From: Jochen Sprickerhof Date: Wed, 6 Jul 2022 08:54:17 +0000 (+0200) Subject: New upstream version 0.9.13 X-Git-Tag: upstream/0.9.13^0 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1f2c3ccb0c9e7fd1fc9eb2004ef4807fc0982934;p=location%2Fdebian%2Flibloc.git New upstream version 0.9.13 --- 1f2c3ccb0c9e7fd1fc9eb2004ef4807fc0982934 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f04b70e --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +*.log +*.mo +*.o +*.tar.xz +.deps/ +.libs/ +Makefile +Makefile.in +/aclocal.m4 +/autom4te.cache +/build-aux +/config.* +/configure +/*.db +/*.db.xz +/libtool +/stamp-h1 +/src/python/location +/src/python/location-importer +/src/python/__init__.py +/src/systemd/location-update.service +/src/systemd/location-update.timer +/test.db +/testdata.db diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..84bba36 --- /dev/null +++ b/COPYING @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..983cb4a --- /dev/null +++ b/Makefile.am @@ -0,0 +1,536 @@ +EXTRA_DIST = +CLEANFILES = +INSTALL_DIRS = +ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS} +AM_MAKEFLAGS = --no-print-directory + +SUBDIRS = . po +BINDINGS = + +OS = $(shell uname -s) + +if ENABLE_PERL +BINDINGS += perl +endif + +AM_CPPFLAGS = \ + -include $(top_builddir)/config.h \ + -DSYSCONFDIR=\""$(sysconfdir)"\" \ + -I${top_srcdir}/src + +AM_CFLAGS = ${my_CFLAGS} \ + -ffunction-sections \ + -fdata-sections + +AM_LDFLAGS = ${my_LDFLAGS} + +# leaving a space here to work around automake's conditionals + ifeq ($(OS),Darwin) + AM_LDFLAGS += -Wl,-dead_strip + else + AM_LDFLAGS += \ + -Wl,--as-needed \ + -Wl,--gc-sections + endif + +LIBLOC_CURRENT=1 +LIBLOC_REVISION=2 +LIBLOC_AGE=0 + +DISTCHECK_CONFIGURE_FLAGS = \ + --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) + +SED_PROCESS = \ + $(AM_V_GEN)$(MKDIR_P) $(dir $@) && $(SED) \ + -e 's,@VERSION\@,$(VERSION),g' \ + -e 's,@prefix\@,$(prefix),g' \ + -e 's,@exec_prefix\@,$(exec_prefix),g' \ + -e 's,@bindir\@,$(bindir),g' \ + -e 's,@libdir\@,$(libdir),g' \ + -e 's,@includedir\@,$(includedir),g' \ + -e 's,@databasedir\@,$(databasedir),g' \ + < $< > $@ || rm $@ + +databasedir = $(localstatedir)/lib/location +pkgconfigdir = $(libdir)/pkgconfig + +# Overwrite Python path +pkgpythondir = $(pythondir)/location + +%: %.in Makefile + $(SED_PROCESS) + +@INTLTOOL_POLICY_RULE@ + +# ------------------------------------------------------------------------------ + +AM_V_ASCIIDOC = $(AM_V_ASCIIDOC_$(V)) +AM_V_ASCIIDOC_ = $(AM_V_ASCIIDOC_$(AM_DEFAULT_VERBOSITY)) +AM_V_ASCIIDOC_0 = @echo " ASCIIDOC" $@; + +AM_V_XSLT = $(AM_V_XSLT_$(V)) +AM_V_XSLT_ = $(AM_V_XSLT_$(AM_DEFAULT_VERBOSITY)) +AM_V_XSLT_0 = @echo " XSLT " $@; + +# ------------------------------------------------------------------------------ + +.PHONY: update-po +update-po: po/POTFILES.in + $(MAKE) -C po update-po + +po/POTFILES.in: Makefile + find $(abs_srcdir)/src -type f \( -name '*.in' -o -name '*.py' \) \ + \! -exec git check-ignore -q {} \; -print | \ + sed -e "s@$(abs_srcdir)/@@g" | LC_ALL=C sort > $@ + +EXTRA_DIST += \ + examples/private-key.pem \ + examples/public-key.pem \ + examples/python/create-database.py \ + examples/python/read-database.py + +pkginclude_HEADERS = \ + src/libloc/libloc.h \ + src/libloc/address.h \ + src/libloc/as.h \ + src/libloc/as-list.h \ + src/libloc/compat.h \ + src/libloc/country.h \ + src/libloc/country-list.h \ + src/libloc/database.h \ + src/libloc/format.h \ + src/libloc/network.h \ + src/libloc/network-list.h \ + src/libloc/private.h \ + src/libloc/stringpool.h \ + src/libloc/resolv.h \ + src/libloc/writer.h + +lib_LTLIBRARIES = \ + src/libloc.la + +src_libloc_la_SOURCES = \ + src/libloc.c \ + src/address.c \ + src/as.c \ + src/as-list.c \ + src/country.c \ + src/country-list.c \ + src/database.c \ + src/network.c \ + src/network-list.c \ + src/resolv.c \ + src/stringpool.c \ + src/writer.c + +EXTRA_DIST += src/libloc.sym + +src_libloc_la_CFLAGS = \ + $(AM_CFLAGS) \ + -DLIBLOC_PRIVATE \ + -fvisibility=hidden + +src_libloc_la_LDFLAGS = \ + $(AM_LDFLAGS) \ + -version-info $(LIBLOC_CURRENT):$(LIBLOC_REVISION):$(LIBLOC_AGE) + +if HAVE_LD_VERSION_SCRIPT +src_libloc_la_LDFLAGS += -Wl,--version-script=$(top_srcdir)/src/libloc.sym +else +src_libloc_la_LDFLAGS += -export-symbols $(top_srcdir)/src/libloc.sym +endif + +src_libloc_la_LIBADD = \ + $(OPENSSL_LIBS) \ + $(RESOLV_LIBS) + +src_libloc_la_DEPENDENCIES = \ + ${top_srcdir}/src/libloc.sym + +noinst_LTLIBRARIES = \ + src/libloc-internal.la + +src_libloc_internal_la_SOURCES = \ + $(src_libloc_la_SOURCES) + +src_libloc_internal_la_CFLAGS = \ + $(src_libloc_la_CFLAGS) + +src_libloc_internal_la_LDFLAGS = \ + $(filter-out -version-info %,$(src_libloc_la_LDFLAGS)) + +src_libloc_internal_la_LIBADD = \ + $(src_libloc_la_LIBADD) + +src_libloc_internal_la_DEPENDENCIES = \ + $(src_libloc_la_DEPENDENCIES) + +pkgconfig_DATA = \ + src/libloc.pc + +EXTRA_DIST += \ + src/libloc.pc.in + +CLEANFILES += \ + src/libloc.pc + +dist_pkgpython_PYTHON = \ + src/python/database.py \ + src/python/downloader.py \ + src/python/export.py \ + src/python/i18n.py \ + src/python/importer.py \ + src/python/logger.py + +pkgpython_PYTHON = \ + src/python/__init__.py + +EXTRA_DIST += \ + src/python/__init__.py.in + +CLEANFILES += \ + src/python/__init__.py + +pyexec_LTLIBRARIES = \ + src/python/_location.la + +src_python__location_la_SOURCES = \ + src/python/locationmodule.c \ + src/python/locationmodule.h \ + src/python/as.c \ + src/python/as.h \ + src/python/country.c \ + src/python/country.h \ + src/python/database.c \ + src/python/database.h \ + src/python/network.c \ + src/python/network.h \ + src/python/writer.c \ + src/python/writer.h + +src_python__location_la_CFLAGS = \ + $(AM_CFLAGS) \ + $(PYTHON_CFLAGS) + +src_python__location_la_LDFLAGS = \ + $(AM_LDFLAGS) \ + -shared \ + -module \ + -avoid-version + +src_python__location_la_LIBADD = \ + src/libloc.la \ + $(PYTHON_LIBS) + +# Compile & install bindings +all-local: $(foreach binding,$(BINDINGS),build-$(binding)) +check-local: $(foreach binding,$(BINDINGS),check-$(binding)) +install-exec-local: $(foreach binding,$(BINDINGS),install-$(binding)) +clean-local: $(foreach binding,$(BINDINGS),clean-$(binding)) +uninstall-local: $(foreach binding,$(BINDINGS),uninstall-$(binding)) + +# Perl Bindings +EXTRA_DIST += \ + src/perl/.gitignore \ + src/perl/Location.xs \ + src/perl/MANIFEST \ + src/perl/Makefile.PL \ + src/perl/lib/Location.pm \ + src/perl/t/Location.t \ + src/perl/typemap + +.PHONY: build-perl +build-perl: + @mkdir -p $(builddir)/src/perl/{lib,t} + @test -e $(builddir)/src/perl/Location.xs || ln -s --relative $(srcdir)/src/perl/Location.xs $(builddir)/src/perl/ + @test -e $(builddir)/src/perl/MANIFEST || ln -s --relative $(srcdir)/src/perl/MANIFEST $(builddir)/src/perl/ + @test -e $(builddir)/src/perl/Makefile.PL || ln -s --relative $(srcdir)/src/perl/Makefile.PL $(builddir)/src/perl/ + @test -e $(builddir)/src/perl/lib/Location.pm || ln -s --relative $(srcdir)/src/perl/lib/Location.pm $(builddir)/src/perl/lib/ + @test -e $(builddir)/src/perl/t/Location.t || ln -s --relative $(srcdir)/src/perl/t/Location.t $(builddir)/src/perl/t/ + @test -e $(builddir)/src/perl/typemap || ln -s --relative $(srcdir)/src/perl/typemap $(builddir)/src/perl/ + + cd $(builddir)/src/perl && $(PERL) Makefile.PL PREFIX="$(prefix)" \ + INC="-I$(abs_srcdir)/src" LIBS="-L$(abs_builddir)/src/.libs -lloc" + cd $(builddir)/src/perl && $(MAKE) LD_RUN_PATH= + +.PHONY: check-perl +check-perl: testdata.db + cd $(builddir)/src/perl && $(MAKE) LD_LIBRARY_PATH="$(abs_builddir)/src/.libs" test \ + database="../../$<" keyfile="$(abs_srcdir)/examples/public-key.pem" + +.PHONY: install-perl +install-perl: + cd $(builddir)/src/perl && $(MAKE) install DESTIDR=$(DESTDIR) + +.PHONY: clean-perl +clean-perl: + cd $(builddir)/src/perl && $(MAKE) distclean + +.PHONY: uninstall-perl +uninstall-perl: + rm -rvf \ + $(DESTDIR)/$(prefix)/lib/*/perl/*/Location.pm \ + $(DESTDIR)/$(prefix)/lib/*/perl/*/auto/Location \ + $(DESTDIR)/$(prefix)/lib/*/perl/*/perllocal.pod \ + $(DESTDIR)/$(prefix)/man/man3/Location.3pm + +bin_SCRIPTS = \ + src/python/location \ + src/python/location-importer + +EXTRA_DIST += \ + src/python/location.in \ + src/python/location-importer.in + +CLEANFILES += \ + src/python/location \ + src/python/location-importer + +# ------------------------------------------------------------------------------ + +if HAVE_SYSTEMD +systemdsystemunit_DATA = \ + src/systemd/location-update.service \ + src/systemd/location-update.timer + +CLEANFILES += \ + $(systemdsystemunit_DATA) + +INSTALL_DIRS += \ + $(systemdsystemunitdir) +endif + +EXTRA_DIST += \ + src/systemd/location-update.service.in \ + src/systemd/location-update.timer.in + +# ------------------------------------------------------------------------------ + +dist_database_DATA = \ + src/signing-key.pem + +# ------------------------------------------------------------------------------ + +TESTS_CFLAGS = \ + $(AM_CFLAGS) \ + -DLIBLOC_PRIVATE \ + -DABS_SRCDIR=\"$(abs_srcdir)\" + +TESTS_LDADD = \ + src/libloc.la \ + src/libloc-internal.la + +TESTS_ENVIRONMENT = \ + TEST_DATA_DIR="$(abs_top_srcdir)/tests/data" + +TESTS = \ + $(check_PROGRAMS) \ + $(dist_check_SCRIPTS) + +EXTRA_DIST += \ + tests/data/location-2022-03-30.db + +CLEANFILES += \ + testdata.db + +testdata.db: examples/python/create-database.py + PYTHONPATH=$(abs_builddir)/src/python/.libs \ + ABS_SRCDIR="$(abs_srcdir)" \ + $(PYTHON) $< $@ + +dist_check_SCRIPTS = \ + tests/python/test-export.py + +check_PROGRAMS = \ + src/test-libloc \ + src/test-stringpool \ + src/test-database \ + src/test-as \ + src/test-network \ + src/test-network-list \ + src/test-country \ + src/test-signature \ + src/test-address + +src_test_libloc_SOURCES = \ + src/test-libloc.c + +src_test_libloc_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_libloc_LDADD = \ + $(TESTS_LDADD) + +src_test_as_SOURCES = \ + src/test-as.c + +src_test_as_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_as_LDADD = \ + $(TESTS_LDADD) + +src_test_country_SOURCES = \ + src/test-country.c + +src_test_country_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_country_LDADD = \ + $(TESTS_LDADD) + +src_test_network_SOURCES = \ + src/test-network.c + +src_test_network_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_network_LDADD = \ + $(TESTS_LDADD) + +src_test_network_list_SOURCES = \ + src/test-network-list.c + +src_test_network_list_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_network_list_LDADD = \ + $(TESTS_LDADD) + +src_test_stringpool_SOURCES = \ + src/test-stringpool.c + +src_test_stringpool_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_stringpool_LDADD = \ + $(TESTS_LDADD) + +src_test_database_SOURCES = \ + src/test-database.c + +src_test_database_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_database_LDADD = \ + $(TESTS_LDADD) + +src_test_signature_SOURCES = \ + src/test-signature.c + +src_test_signature_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_signature_LDADD = \ + $(TESTS_LDADD) + +src_test_address_SOURCES = \ + src/test-address.c + +src_test_address_CFLAGS = \ + $(TESTS_CFLAGS) + +src_test_address_LDADD = \ + $(TESTS_LDADD) + +# ------------------------------------------------------------------------------ + +MANPAGES = \ + $(MANPAGES_3) \ + $(MANPAGES_8) + +MANPAGES_3 = \ + man/libloc.3 \ + man/loc_database_count_as.3 \ + man/loc_database_get_as.3 \ + man/loc_database_get_country.3 \ + man/loc_database_lookup.3 \ + man/loc_database_new.3 \ + man/loc_get_log_priority.3 \ + man/loc_new.3 \ + man/loc_set_log_fn.3 \ + man/loc_set_log_priority.3 + +MANPAGES_8 = \ + man/location.8 + +MANPAGES_TXT = $(MANPAGES_TXT_3) $(MANPAGES_TXT_8) +MANPAGES_TXT_3 = $(patsubst %.3,%.txt,$(MANPAGES_3)) +MANPAGES_TXT_8 = $(patsubst %.8,%.txt,$(MANPAGES_8)) +MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT)) +MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT)) + +.PHONY: man +man: $(MANPAGES) $(MANPAGES_HTML) + +if ENABLE_MAN_PAGES +man_MANS = \ + $(MANPAGES) +endif + +CLEANFILES += \ + $(MANPAGES) \ + $(MANPAGES_HTML) \ + $(MANPAGES_XML) + +EXTRA_DIST += \ + man/asciidoc.conf \ + $(MANPAGES_TXT) + +XSLTPROC_FLAGS = \ + --nonet \ + --stringparam man.output.quietly 1 \ + --stringparam funcsynopsis.style ansi \ + --stringparam man.th.extra1.suppress 1 \ + --stringparam man.authors.section.enabled 1 \ + --stringparam man.copyright.section.enabled 1 + +XSLTPROC_COMMAND_MAN = \ + $(AM_V_XSLT)$(MKDIR_P) $(dir $@) && \ + $(XSLTPROC) -o $@ $(XSLTPROC_FLAGS) \ + http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $< + +# Let XSLT find its source on Mac OS X + ifeq ($(OS),Darwin) +export XML_CATALOG_FILES = /usr/local/etc/xml/catalog + endif + +man/%.xml: man/%.txt man/asciidoc.conf + $(AM_V_ASCIIDOC)$(MKDIR_P) $(dir $@) && \ + $(ASCIIDOC) \ + -f $(abs_srcdir)/man/asciidoc.conf \ + -d manpage -b docbook -o $@ $< + +man/%.3: man/%.xml + $(XSLTPROC_COMMAND_MAN) + +man/%.8: man/%.xml + $(XSLTPROC_COMMAND_MAN) + +man/%.html: man/%.txt man/asciidoc.conf + $(AM_V_ASCIIDOC)$(MKDIR_P) $(dir $@) && \ + $(ASCIIDOC) \ + -f $(abs_srcdir)/man/asciidoc.conf \ + -b html5 -a icons -a theme=flask -o $@ $< + +.PHONY: upload-man +upload-man: $(MANPAGES_HTML) + rsync -avHz --delete --progress $(MANPAGES_HTML) ms@fs01.haj.ipfire.org:/pub/man-pages/$(PACKAGE_NAME)/ + +EXTRA_DIST += \ + debian/build.sh \ + debian/changelog \ + debian/compat \ + debian/control \ + debian/copyright \ + debian/location.install \ + debian/location.manpages \ + debian/location-python.install \ + debian/libloc1.install \ + debian/libloc-dev.install \ + debian/rules \ + debian/source/format + +.PHONY: debian +debian: dist + $(SHELL) debian/build.sh $(PACKAGE_NAME)-$(PACKAGE_VERSION) $(distdir).tar.xz diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..ba2845a --- /dev/null +++ b/autogen.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -e + +if [ -f .git/hooks/pre-commit.sample -a ! -f .git/hooks/pre-commit ] ; then + cp -p .git/hooks/pre-commit.sample .git/hooks/pre-commit && \ + chmod +x .git/hooks/pre-commit && \ + echo "Activated pre-commit hook." +fi + +intltoolize --force --automake +autoreconf --install --symlink + +libdir() { + echo $(cd $1/$(gcc -print-multi-os-directory); pwd) +} + +args="--prefix=/usr \ +--sysconfdir=/etc \ +--libdir=$(libdir /usr/lib)" + +echo +echo "----------------------------------------------------------------" +echo "Initialized build system. For a common configuration please run:" +echo "----------------------------------------------------------------" +echo +echo "./configure CFLAGS='-g -O0' $args" +echo diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..e96e9ce --- /dev/null +++ b/configure.ac @@ -0,0 +1,212 @@ +AC_PREREQ(2.60) +AC_INIT([libloc], + [0.9.13], + [location@lists.ipfire.org], + [libloc], + [https://location.ipfire.org/]) + +AC_CONFIG_SRCDIR([src/libloc.c]) +AC_CONFIG_AUX_DIR([build-aux]) +AM_INIT_AUTOMAKE([ + foreign + 1.11 + -Wall + -Wno-portability + silent-rules + tar-pax + no-dist-gzip + dist-xz + subdir-objects +]) +AC_PROG_CC_STDC +AC_USE_SYSTEM_EXTENSIONS +AC_SYS_LARGEFILE +AC_CONFIG_MACRO_DIR([m4]) +AM_SILENT_RULES([yes]) +LT_INIT([ + disable-static + pic-only +]) +AC_PREFIX_DEFAULT([/usr]) +gl_LD_VERSION_SCRIPT + +IT_PROG_INTLTOOL([0.40.0]) + +# Interpret embedded Python in HTML files +XGETTEXT="${XGETTEXT} -L Python --keyword=_:1,2 --keyword=N_:1,2 --no-location" + +GETTEXT_PACKAGE=${PACKAGE_TARNAME} +AC_SUBST(GETTEXT_PACKAGE) + +AC_PROG_SED +AC_PROG_MKDIR_P + +# - man ------------------------------------------------------------------------ + +have_man_pages=no +AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages], + [do not install man pages])) +AS_IF([test "x$enable_man_pages" != xno], [have_man_pages=yes]) +AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" = "xyes"]) + +AC_PATH_PROG([XSLTPROC], [xsltproc]) + +AC_CHECK_PROGS(ASCIIDOC, [asciidoc]) +if test "${have_man_pages}" = "yes" && test -z "${ASCIIDOC}"; then + AC_MSG_ERROR([Required program 'asciidoc' not found]) +fi +# - debug ---------------------------------------------------------------------- + +AC_ARG_ENABLE([debug], + AS_HELP_STRING([--enable-debug], [enable debug messages @<:@default=disabled@:>@]), + [], [enable_debug=no]) +AS_IF([test "x$enable_debug" = "xyes"], [ + AC_DEFINE(ENABLE_DEBUG, [1], [Debug messages.]) +]) + +AC_CHECK_HEADERS_ONCE([ + arpa/inet.h \ + arpa/nameser.h \ + arpa/nameser_compat.h \ + endian.h \ + netinet/in.h \ + resolv.h \ + string.h \ +]) + +AC_CHECK_FUNCS([ \ + be16toh \ + be32toh \ + be64toh \ + htobe16 \ + htobe32 \ + htobe64 \ + mmap \ + munmap \ + res_query \ + __secure_getenv \ + secure_getenv \ + qsort \ +]) + +my_CFLAGS="\ +-Wall \ +-Wchar-subscripts \ +-Wformat-security \ +-Wmissing-declarations \ +-Wmissing-prototypes \ +-Wnested-externs \ +-Wpointer-arith \ +-Wshadow \ +-Wsign-compare \ +-Wstrict-prototypes \ +-Wtype-limits \ +" +AC_SUBST([my_CFLAGS]) +AC_SUBST([my_LDFLAGS]) + +# Enable -fanalyzer if requested +AC_ARG_ENABLE([analyzer], + AS_HELP_STRING([--enable-analyzer], [enable static analyzer (-fanalyzer) @<:@default=disabled@:>@]), + [], [enable_analyzer=no]) +AS_IF([test "x$enable_analyzer" = "xyes"], + CC_CHECK_FLAGS_APPEND([my_CFLAGS], [CFLAGS], [-fanalyzer]) +) + +# Enable -fno-semantic-interposition (if available) +CC_CHECK_FLAGS_APPEND([my_CFLAGS], [CFLAGS], [-fno-semantic-interposition]) +CC_CHECK_FLAGS_APPEND([my_LDFLAGS], [LDFLAGS], [-fno-semantic-interposition]) + +# ------------------------------------------------------------------------------ + +AC_ARG_WITH([systemd], + AS_HELP_STRING([--with-systemd], [Enable systemd support.]) +) + +AS_IF([test "x$with_systemd" != "xno"], + [PKG_CHECK_MODULES(systemd, [libsystemd], + [have_systemd=yes], [have_systemd=no])], + [have_systemd=no] +) + +AS_IF([test "x$have_systemd" = "xyes"], + [AC_MSG_CHECKING([for systemd system unit directory]) + AC_ARG_WITH([systemdsystemunitdir], + AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files]), + [], [with_systemdsystemunitdir=$($PKG_CONFIG --variable=systemdsystemunitdir systemd)] + ) + + AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir]) + + if test -n "$systemdsystemunitdir" -a "x$systemdsystemunitdir" != xno; then + AC_MSG_RESULT([$systemdsystemunitdir]) + else + AC_MSG_ERROR([not found (try --with-systemdsystemunitdir)]) + fi + ], + [AS_IF([test "x$with_systemd" = "xyes"], + [AC_MSG_ERROR([Systemd support is enabled but no systemd has been found.]) + ]) +]) + +AM_CONDITIONAL(HAVE_SYSTEMD, [test "x$have_systemd" = "xyes"]) + +# ------------------------------------------------------------------------------ + +# Python +AM_PATH_PYTHON([3.9]) +PKG_CHECK_MODULES([PYTHON], [python-${PYTHON_VERSION}]) + +# Perl +AC_PATH_PROG(PERL, perl, no) +AC_SUBST(PERL) + +AX_PROG_PERL_MODULES(ExtUtils::MakeMaker,, AC_MSG_WARN(Need some Perl modules)) + +AC_ARG_ENABLE(perl, AS_HELP_STRING([--disable-perl], [do not build the perl modules]), [],[enable_perl=yes]) +AM_CONDITIONAL(ENABLE_PERL, test "$enable_perl" = "yes") + +dnl Checking for libresolv +case "${host}" in + *-gnu*) + AC_CHECK_LIB(resolv, ns_msg_getflag, [LIBS="-lresolv $LIBS"], AC_MSG_ERROR([libresolv has not been found]), -lresolv) + ;; + *) + AC_CHECK_LIB(resolv, res_init, [LIBS="-lresolv $LIBS"], AC_MSG_ERROR([libresolv has not been found]), -lresolv) + ;; +esac +RESOLV_LIBS="${LIBS}" +AC_SUBST(RESOLV_LIBS) + +dnl Checking for OpenSSL +LIBS= +AC_CHECK_LIB(crypto, EVP_EncryptInit,, AC_MSG_ERROR([libcrypto has not been found])) +OPENSSL_LIBS="${LIBS}" +AC_SUBST(OPENSSL_LIBS) + +AC_CONFIG_HEADERS(config.h) +AC_CONFIG_FILES([ + Makefile + po/Makefile.in +]) + +AC_OUTPUT +AC_MSG_RESULT([ + $PACKAGE $VERSION + ===== + + prefix: ${prefix} + sysconfdir: ${sysconfdir} + libdir: ${libdir} + includedir: ${includedir} + + compiler: ${CC} + cflags: ${CFLAGS} + ldflags: ${LDFLAGS} + + debug: ${enable_debug} + systemd support: ${have_systemd} + + Bindings: + perl: ${enable_perl} +]) diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000..8190b92 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,11 @@ +/.debhelper +/autoreconf.* +/debhelper-build-stamp +/files +/*/ +*.debhelper +*.log +*.substvars +!/patches/ +!/source/ +!/tests/ diff --git a/debian/build.sh b/debian/build.sh new file mode 100644 index 0000000..a11f3a3 --- /dev/null +++ b/debian/build.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +ARCHITECTURES=( amd64 arm64 i386 armhf ) +RELEASES=( buster bullseye bookworm sid ) + +CHROOT_PATH="/var/tmp" + +main() { + if [ $# -lt 2 ]; then + echo "Not enough arguments" >&2 + return 2 + fi + + # Get host architecture + local host_arch="$(dpkg --print-architecture)" + if [ -z "${host_arch}" ]; then + echo "Could not discover host architecture" >&2 + return 1 + fi + + local package="${1}" + local sources="${2}" + + # Create some temporary directory + local tmp="$(mktemp -d)" + + # Extract the sources into it + mkdir -p "${tmp}/sources" + tar xvfa "${sources}" -C "${tmp}/sources" + + # Copy the tarball under the correct Debian name + cp -vf "${sources}" "${tmp}/sources/${package//-/_}.orig.tar.xz" + + # Change into temporary directory + pushd "${tmp}" + + # Build the package for each release + local release + for release in ${RELEASES[@]}; do + local chroot="${release}-${host_arch}-sbuild" + + mkdir -p "${release}" + pushd "${release}" + + # Create a chroot environment + if [ ! -d "/etc/sbuild/chroot/${chroot}" ]; then + if ! sbuild-createchroot --arch="${host_arch}" "${release}" \ + "${CHROOT_PATH}/${chroot}"; then + echo "Could not create chroot for ${release} on ${host_arch}" >&2 + rm -rf "${tmp}" + return 1 + fi + fi + + # And for each architecture we want to support + local arch + for arch in ${ARCHITECTURES[@]}; do + mkdir -p "${arch}" + pushd "${arch}" + + # Copy sources + cp -r "${tmp}/sources" . + + # Run the build process + if ! sbuild --dist="${release}" --host="${arch}" --source "sources/${package}"; then + echo "Could not build package for ${release} on ${arch}" >&2 + rm -rf "${tmp}" + return 1 + fi + + # Remove the sources + rm -rf "sources/${package}" + popd + done + popd + done + + # Remove sources + rm -rf "${tmp}/sources" + popd + + # Done! + echo "SUCCESS!" + echo " You can find your Debian packages in ${tmp}" + return 0 +} + +main "$@" || exit $? diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..2085fe4 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,216 @@ +libloc (0.9.13-1) unstable; urgency=medium + + [ Michael Tremer ] + * tests: Add a simple test that lists all networks + * database: Allocate subnets list only once + * network: Log a more useful message on invalid prefix + * network: Add more debugging output when running exclude + * network: loc_network_subnets: Use correct prefix + * tests: Break after exporting 1000 networks + * configure: Require Python >= 3.9 + * export: Enable flattening for everything + * .gitignore: Ignore *.db files only in main directory + * tests: Import test database + * configure: Bump version to 0.9.13 + + -- Michael Tremer Tue, 12 Apr 2022 12:15:34 +0000 + +libloc (0.9.12-1) unstable; urgency=medium + + [ Michael Tremer ] + * importer: Parse aggregated networks + * database: Return something when no filter criteria is configured + * importer: Correctly hande response codes from Bird + * importer: Silently ignore any table headers + * importer: Skip empty lines + * location: Fix output of list-* commands + * network: Move a couple of helper functions into headers + * network: Add function that counts the bit length of an addres + * network: Drop functions moved in an earlier commit + * network-list: Rewrite summarize algorithm + * network: Allow creating any valid networks + * network: Implement bit length function for IPv4 + * addresses: Implement subtraction for IPv4 + * bogons: Refactor algorithms + * network-list: Cap prefix length based on family + * address: Correctly subtract IPv4 addresses + * bogons: Reset after we have reached the end + * bogons: Don't consider a network legitimate without a country code + * Move all address convenience functions into their own header + * address: Rename in6_addr_cmp into loc_address_cmp + * address: Rename in6_addr_get_bit/in6_addr_set_bit to loc_address_* + * addresses: Use loc_address_family which is now available + * address: Rename increment/decrement functions and modify address in + place + * network: Pass prefix in native length + * strings: Statically allocate all address/network strings + * address: Initialize all bits of IP addresses + * address: Prevent under/overflow when incrementing/decrementing + * network-list: Simplify debugging output on summarize + * network-list: summarize: Break when we exhausted the network range + * network-list: Remove debugging line + * address: Simplify functions + * address: Fix decrementing IP addresses + * address: Fix buffer overwrite + * address: Add some simple tests + * bogons: Add gaps that are only one address wide + * bogons: Skip any subnets of former networks + * as-list: Grow faster to avoid too many re-allocations + * writer: Use AS list internally + * network-list: Grow just like the AS list + * country-list: Grow like AS list + * writer: Use country list internally + * importer: Improve performance of network export query + * writer: I forgot to initalize the country list + * Refactor parsing IP addresses + * address: Set default prefix if none is given + + -- Michael Tremer Wed, 23 Mar 2022 20:11:29 +0000 + +libloc (0.9.11-1) unstable; urgency=medium + + [ Stefan Schantl ] + * export: Remove prefix when exporting countries. + + [ Michael Tremer ] + * ipset: Optimise hash table size + * ipset: Fix hash type for IPv6 + * ipset: Set maxelem to a fixed size + * export: Conditionally enable flattening + * location: Print proper error message for any uncaught exceptions + * export: Allow exporting to stdout + * ipset: The minimum hashsize is 64 + * export: Fix filtering logic + * export: Sightly refactor export logic + * Bump release to 0.9.11 + + [ Peter Müller ] + * location-importer: Fix parsing LACNIC-flavoured inetnums + + -- Michael Tremer Thu, 03 Mar 2022 10:44:44 +0000 + +libloc (0.9.10-1) unstable; urgency=medium + + [ Peter Müller ] + * Non-maintainer upload. + * location-importer: Set "is_drop" to "True" even in case of conflicts + * Process LACNIC geofeed as well + * location-importer: Improve regex for catching historic/orphaned data + * location-importer: Replace "UK" with "GB" + * location-importer.in: Add country code for AWS's "il-central-1" zone + * location-importer.in: Do not make things more complicated than they + are + + [ Michael Tremer ] + * man: Add pages for top level functions + * man: Add man page for loc_database_new + * man: Add man pages for all loc_database_* functions + * export: Make ipset files easily reloadable + + -- Michael Tremer Wed, 16 Feb 2022 08:53:48 +0000 + +libloc (0.9.9-2) unstable; urgency=medium + + * Fix broken Debian build + + -- Michael Tremer Tue, 23 Nov 2021 11:07:22 +0000 + +libloc (0.9.9-1) unstable; urgency=medium + + [ Michael Tremer ] + * database: Make IP address const for lookup + * configure: Enable -fno-semantic-interposition by default + * network: Drop redundant loc_network_match_flag + * network: Drop useless loc_network_match_asn function + * stringpool: Make functions properly private + * Make loc_network_tree_* functions propertly private + * Remove LOC_EXPORT from + loc_network_to_database_v1/loc_network_new_from_database_v1 + * country: Add function that returns flags for special country + * country: Make country codes beginning with X invalid + * network: Make loc_network_match_country_code match special countries + * network: Rename "match" functions to "matches" + + [ Peter Müller ] + * location.txt: Improve manpage + * importer.py: Import JPNIC feed as well + * location-importer: Introduce auxiliary function to sanitise ASNs + * location-importer.in: Add Spamhaus DROP lists + + -- Michael Tremer Sat, 20 Nov 2021 15:12:28 +0000 + +libloc (0.9.8-1) unstable; urgency=medium + + [ Michael Tremer ] + * importer: Do not try to initialise a column that cannot be NULL with + NULL + * configure: Add option to enable GCC's -fanalyzer + * writer: Break when a network cound not be allocated + * stringpool: Allow adding empty strings + * stringpool: Do not call strlen() on potential NULL pointer + * stringpool: Slightly refactor initialization to help the compiler + understand + * stringpool: Avoid memory leak if mmap() fails + * network: Move some helper functions into network.h + * python: Permit passing family to database enumerator + * location: Implement listing bogons + * Move include files to /usr/include/libloc + + [ Peter Müller ] + * location-importer.in: Attempt to provide meaningful AS names if + organisation handles are missing + * location-importer.in: Braindead me accidentally forgot a "break" + statement + + -- Michael Tremer Tue, 21 Sep 2021 10:29:11 +0000 + +libloc (0.9.7-1) unstable; urgency=medium + + [ Valters Jansons ] + * po: Update translations + * systemd: Add Documentation= to location-update + + [ Peter Müller ] + * location-importer.in: emit warnings due to unknown country code for + valid networks only + * location.in: fix search_networks() function call + * location-importer.in: keep track of sources for networks, ASNs, and + organisations + * importer.py: add source information for RIR data feeds + * location-importer.in: track original countries as well + * location-importer.in: track original countries more pythonic + * Implement an additional flag for hostile networks safe to drop + * location-importer.in: Import (technical) AS names from ARIN + * location-importer.in: add source column for overrides as well + * location-importer.in: import additional IP information for Amazon + AWS IP networks + * location-import.in: optimise regular expression for filtering ASN + allocations to other RIRs when parsing ARIN AS names file + + [ Michael Tremer ] + * countries: Fix matching invalid country codes + * Bump version to 0.9.7 + + -- Michael Tremer Fri, 09 Jul 2021 17:16:59 +0000 + +libloc (0.9.6-1) unstable; urgency=medium + + [ Michael Tremer ] + * location: Fix list-networks-by-as + * database: Free mmapped countries section + + [ Peter Müller ] + * location-importer.in: fix typo + * location-importer.in: delete 6to4 IPv6 space as well + * location-importer.in: reduce log noise for unusable networks + * location-importer.in: process unaligned IP ranges in RIR data files + correctly + * location-importer.in: skip networks with unknown country codes + + -- Michael Tremer Wed, 31 Mar 2021 14:06:00 +0100 + +libloc (0.9.5-1) unstable; urgency=medium + + * Initial release. + + -- Stefan Schantl Sun, 27 Oct 2019 18:55:44 +0100 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +10 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..4b1407a --- /dev/null +++ b/debian/control @@ -0,0 +1,87 @@ +Source: libloc +Maintainer: Stefan Schantl +Section: misc +Priority: optional +Standards-Version: 4.3.0 +Build-Depends: + debhelper (>= 11), + dh-python , + asciidoc , + intltool (>=0.40.0), + libpython3-dev , + libssl-dev, + libsystemd-dev, + python3-dev:any , + pkg-config, + systemd, + xsltproc , + docbook-xsl , + git, +Rules-Requires-Root: no +Homepage: https://location.ipfire.org/ +Vcs-Git: https://git.ipfire.org/pub/git/location/libloc.git +Vcs-Browser: https://git.ipfire.org/pub/git/location/libloc.git + +Package: libloc1 +Architecture: any +Section: libs +Pre-Depends: + ${misc:Pre-Depends} +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Recommends: + location (= ${binary:Version}) +Multi-Arch: same +Description: Location library + A library to determine the location of someone on the Internet + +Package: libloc-dev +Architecture: any +Section: libdevel +Depends: + libloc1 (= ${binary:Version}), + ${misc:Depends}, +Suggests: + pkg-config +Multi-Arch: same +Description: Development files for libloc + Install this package if you wish to develop your own programs using + libloc. + +Package: location +Architecture: any +Pre-Depends: + ${misc:Pre-Depends} +Depends: + location-python (= ${binary:Version}), + ${misc:Depends}, + ${python3:Depends} +Multi-Arch: same +Description: CLI utilities for libloc + Commands to determine someone's location on the Internet + +Package: location-importer +Architecture: any +Pre-Depends: + ${misc:Pre-Depends} +Depends: + location-python (= ${binary:Version}), + ${misc:Depends}, + ${python3:Depends} +Multi-Arch: foreign +Description: Tools to author location databases + This package contains tools that are required to build location databases + +Package: location-python +Architecture: any +Section: python +Pre-Depends: + ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${python3:Depends}, + ${shlibs:Depends} +Multi-Arch: foreign +Description: Python modules for libloc + This package contains Python bindings for libloc diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..3bd7654 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,26 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: libloc +Upstream-Contact: Michael Tremer +Source: https://location.ipfire.org/download + +Files: * +Copyright: 2017-2019 IPFire Development team +License: LGPL-2.1 + +Files: debian/* +Copyright: 2019 Stefan Schantl +License: LGPL-2.1 + +License: LGPL-2.1 + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by the + Free Software Foundation; version 2.1 of the License, or (at + your option) any later version. + . + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + . + The complete text of the GNU General Public License + can be found in /usr/share/common-licenses/LGPL-2.1 file. + . diff --git a/debian/genchangelog.sh b/debian/genchangelog.sh new file mode 100755 index 0000000..ab1c198 --- /dev/null +++ b/debian/genchangelog.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e +gitshow () { + local format=$1 + local commit=$2 + + git show --no-patch --format=format:"$format" "$commit" +} + +main () { + if [ $# -lt 1 ]; then + local bn="$(basename $0)" + echo "Usage: $bn " >&2 + echo "Example: $bn 0.9.7..HEAD" >&2 + echo "Example: $bn 0.9.5..0.9.6^" >&2 + return 1 + fi + + local commitrange=$1 + + local commit + for commit in $(git rev-list --reverse "$commitrange"); do + # Skip commits with diffs that only have Makefile.am or d/ changes. + if [ "$(git diff --name-only "${commit}^..${commit}" -- . ':^Makefile.am' ':^debian/' | wc -l)" == 0 ]; then + continue + fi + + local author_name="$(gitshow %an "$commit")" + local author_email="$(gitshow %ae "$commit")" + local subject="$(gitshow %s "$commit")" + + echo "$author_name <$author_email> $subject" + DEBFULLNAME="$author_name" DEBEMAIL="$author_email" debchange --upstream --multimaint-merge "$subject" + done + + debchange --release '' +} + +main "$@" || exit $? diff --git a/debian/gensymbols.sh b/debian/gensymbols.sh new file mode 100755 index 0000000..8523556 --- /dev/null +++ b/debian/gensymbols.sh @@ -0,0 +1,50 @@ +#!/bin/bash +SYMBOLS_PKG=libloc1 +LOCAL_FILE=debian/libloc1.symbols +TEMP_FILE="$(mktemp --tmpdir libloc1.XXXXXX.symbols)" +trap "rm -f ${TEMP_FILE}" EXIT + +generate () { + intltoolize --force --automake + autoreconf --install --symlink + ./configure CFLAGS='-g -O0' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib + + make + + dpkg-gensymbols -p"$SYMBOLS_PKG" -O"$TEMP_FILE" -esrc/.libs/libloc.so.* + sed -i -E -e 's/( [0-9\.]+)-.+$/\1/' "$TEMP_FILE" + + make clean +} + +main () { + local maxver='0.0.0' + if [ -f "$LOCAL_FILE" ]; then + cp "$LOCAL_FILE" "$TEMP_FILE" + maxver="$(grep -E '^ ' "$LOCAL_FILE" | cut -d' ' -f3 | sort -Vru | head -n1)" + echo "Latest version checked: $maxver" + fi + + + local tag + for tag in $(git tag -l --sort=version:refname) + do + if [ "$(echo -e "${maxver}\n${tag}" | sort -Vr | head -n1)" == "$maxver" ]; then + echo "Tag $tag -- skip" + continue + fi + + echo "Tag $tag -- checking" + git switch --quiet --detach "$tag" || return 1 + generate || return 1 + git switch --quiet - || return 1 + done + + echo "Current -- checking" + generate || return 1 + + mv "$TEMP_FILE" "$LOCAL_FILE" + chmod 644 "$LOCAL_FILE" +} + +main "$@" || exit $? diff --git a/debian/libloc-dev.install b/debian/libloc-dev.install new file mode 100644 index 0000000..d93d217 --- /dev/null +++ b/debian/libloc-dev.install @@ -0,0 +1,3 @@ +usr/include/libloc +usr/lib/*/libloc.so +usr/lib/*/pkgconfig diff --git a/debian/libloc1.install b/debian/libloc1.install new file mode 100644 index 0000000..0f8eec4 --- /dev/null +++ b/debian/libloc1.install @@ -0,0 +1 @@ +usr/lib/*/libloc.so.* diff --git a/debian/libloc1.symbols b/debian/libloc1.symbols new file mode 100644 index 0000000..abdc32a --- /dev/null +++ b/debian/libloc1.symbols @@ -0,0 +1,125 @@ +libloc.so.1 libloc1 #MINVER# +* Build-Depends-Package: libloc-dev + LIBLOC_1@LIBLOC_1 0.9.4 + loc_as_cmp@LIBLOC_1 0.9.4 + loc_as_get_name@LIBLOC_1 0.9.4 + loc_as_get_number@LIBLOC_1 0.9.4 + loc_as_list_append@LIBLOC_1 0.9.5 + loc_as_list_clear@LIBLOC_1 0.9.5 + loc_as_list_contains@LIBLOC_1 0.9.5 + loc_as_list_contains_number@LIBLOC_1 0.9.5 + loc_as_list_empty@LIBLOC_1 0.9.5 + loc_as_list_get@LIBLOC_1 0.9.5 + loc_as_list_new@LIBLOC_1 0.9.5 + loc_as_list_ref@LIBLOC_1 0.9.5 + loc_as_list_size@LIBLOC_1 0.9.5 + loc_as_list_unref@LIBLOC_1 0.9.5 + loc_as_new@LIBLOC_1 0.9.4 + loc_as_ref@LIBLOC_1 0.9.4 + loc_as_set_name@LIBLOC_1 0.9.4 + loc_as_unref@LIBLOC_1 0.9.4 + loc_country_cmp@LIBLOC_1 0.9.4 + loc_country_code_is_valid@LIBLOC_1 0.9.4 + loc_country_get_code@LIBLOC_1 0.9.4 + loc_country_get_continent_code@LIBLOC_1 0.9.4 + loc_country_get_name@LIBLOC_1 0.9.4 + loc_country_list_append@LIBLOC_1 0.9.5 + loc_country_list_clear@LIBLOC_1 0.9.5 + loc_country_list_contains@LIBLOC_1 0.9.5 + loc_country_list_contains_code@LIBLOC_1 0.9.5 + loc_country_list_empty@LIBLOC_1 0.9.5 + loc_country_list_get@LIBLOC_1 0.9.5 + loc_country_list_new@LIBLOC_1 0.9.5 + loc_country_list_ref@LIBLOC_1 0.9.5 + loc_country_list_size@LIBLOC_1 0.9.5 + loc_country_list_unref@LIBLOC_1 0.9.5 + loc_country_new@LIBLOC_1 0.9.4 + loc_country_ref@LIBLOC_1 0.9.4 + loc_country_set_continent_code@LIBLOC_1 0.9.4 + loc_country_set_name@LIBLOC_1 0.9.4 + loc_country_special_code_to_flag@LIBLOC_1 0.9.8 + loc_country_unref@LIBLOC_1 0.9.4 + loc_database_count_as@LIBLOC_1 0.9.4 + loc_database_created_at@LIBLOC_1 0.9.4 + loc_database_enumerator_get_asns@LIBLOC_1 0.9.5 + loc_database_enumerator_get_countries@LIBLOC_1 0.9.5 + loc_database_enumerator_new@LIBLOC_1 0.9.4 + loc_database_enumerator_next_as@LIBLOC_1 0.9.4 + loc_database_enumerator_next_country@LIBLOC_1 0.9.4 + loc_database_enumerator_next_network@LIBLOC_1 0.9.4 + loc_database_enumerator_ref@LIBLOC_1 0.9.4 + loc_database_enumerator_set_asns@LIBLOC_1 0.9.5 + loc_database_enumerator_set_countries@LIBLOC_1 0.9.5 + loc_database_enumerator_set_family@LIBLOC_1 0.9.4 + loc_database_enumerator_set_flag@LIBLOC_1 0.9.4 + loc_database_enumerator_set_string@LIBLOC_1 0.9.4 + loc_database_enumerator_unref@LIBLOC_1 0.9.4 + loc_database_get_as@LIBLOC_1 0.9.4 + loc_database_get_country@LIBLOC_1 0.9.4 + loc_database_get_description@LIBLOC_1 0.9.4 + loc_database_get_license@LIBLOC_1 0.9.4 + loc_database_get_vendor@LIBLOC_1 0.9.4 + loc_database_lookup@LIBLOC_1 0.9.4 + loc_database_lookup_from_string@LIBLOC_1 0.9.4 + loc_database_new@LIBLOC_1 0.9.4 + loc_database_ref@LIBLOC_1 0.9.4 + loc_database_unref@LIBLOC_1 0.9.4 + loc_database_verify@LIBLOC_1 0.9.4 + loc_discover_latest_version@LIBLOC_1 0.9.4 + loc_get_log_priority@LIBLOC_1 0.9.4 + loc_network_address_family@LIBLOC_1 0.9.4 + loc_network_cmp@LIBLOC_1 0.9.5 + loc_network_exclude@LIBLOC_1 0.9.5 + loc_network_exclude_list@LIBLOC_1 0.9.5 + loc_network_format_first_address@LIBLOC_1 0.9.4 + loc_network_format_last_address@LIBLOC_1 0.9.4 + loc_network_get_asn@LIBLOC_1 0.9.4 + loc_network_get_country_code@LIBLOC_1 0.9.4 + loc_network_get_first_address@LIBLOC_1 0.9.5 + loc_network_get_last_address@LIBLOC_1 0.9.5 + loc_network_has_flag@LIBLOC_1 0.9.4 + loc_network_is_subnet@LIBLOC_1 0.9.5 + loc_network_list_clear@LIBLOC_1 0.9.5 + loc_network_list_contains@LIBLOC_1 0.9.5 + loc_network_list_dump@LIBLOC_1 0.9.5 + loc_network_list_empty@LIBLOC_1 0.9.5 + loc_network_list_get@LIBLOC_1 0.9.5 + loc_network_list_merge@LIBLOC_1 0.9.5 + loc_network_list_new@LIBLOC_1 0.9.5 + loc_network_list_pop@LIBLOC_1 0.9.5 + loc_network_list_pop_first@LIBLOC_1 0.9.5 + loc_network_list_push@LIBLOC_1 0.9.5 + loc_network_list_ref@LIBLOC_1 0.9.5 + loc_network_list_size@LIBLOC_1 0.9.5 + loc_network_list_unref@LIBLOC_1 0.9.5 + loc_network_matches_address@LIBLOC_1 0.9.8 + loc_network_matches_country_code@LIBLOC_1 0.9.8 + loc_network_new@LIBLOC_1 0.9.4 + loc_network_new_from_string@LIBLOC_1 0.9.4 + loc_network_overlaps@LIBLOC_1 0.9.5 + loc_network_prefix@LIBLOC_1 0.9.5 + loc_network_ref@LIBLOC_1 0.9.4 + loc_network_set_asn@LIBLOC_1 0.9.4 + loc_network_set_country_code@LIBLOC_1 0.9.4 + loc_network_set_flag@LIBLOC_1 0.9.4 + loc_network_str@LIBLOC_1 0.9.4 + loc_network_subnets@LIBLOC_1 0.9.5 + loc_network_unref@LIBLOC_1 0.9.4 + loc_new@LIBLOC_1 0.9.4 + loc_ref@LIBLOC_1 0.9.4 + loc_set_log_fn@LIBLOC_1 0.9.4 + loc_set_log_priority@LIBLOC_1 0.9.4 + loc_unref@LIBLOC_1 0.9.4 + loc_writer_add_as@LIBLOC_1 0.9.4 + loc_writer_add_country@LIBLOC_1 0.9.4 + loc_writer_add_network@LIBLOC_1 0.9.4 + loc_writer_get_description@LIBLOC_1 0.9.4 + loc_writer_get_license@LIBLOC_1 0.9.4 + loc_writer_get_vendor@LIBLOC_1 0.9.4 + loc_writer_new@LIBLOC_1 0.9.4 + loc_writer_ref@LIBLOC_1 0.9.4 + loc_writer_set_description@LIBLOC_1 0.9.4 + loc_writer_set_license@LIBLOC_1 0.9.4 + loc_writer_set_vendor@LIBLOC_1 0.9.4 + loc_writer_unref@LIBLOC_1 0.9.4 + loc_writer_write@LIBLOC_1 0.9.4 diff --git a/debian/location-importer.install b/debian/location-importer.install new file mode 100644 index 0000000..eaae79d --- /dev/null +++ b/debian/location-importer.install @@ -0,0 +1,3 @@ +usr/bin/location-importer +usr/lib/python3*/site-packages/location/database.py +usr/lib/python3*/site-packages/location/importer.py diff --git a/debian/location-perl.install b/debian/location-perl.install new file mode 100644 index 0000000..08e8cc4 --- /dev/null +++ b/debian/location-perl.install @@ -0,0 +1 @@ +usr/lib/*/perl/ diff --git a/debian/location-python.examples b/debian/location-python.examples new file mode 100644 index 0000000..cf2a6ee --- /dev/null +++ b/debian/location-python.examples @@ -0,0 +1 @@ +examples/python/ diff --git a/debian/location-python.install b/debian/location-python.install new file mode 100644 index 0000000..a6004ca --- /dev/null +++ b/debian/location-python.install @@ -0,0 +1 @@ +usr/lib/python3*/site-packages diff --git a/debian/location.install b/debian/location.install new file mode 100644 index 0000000..25d9b6f --- /dev/null +++ b/debian/location.install @@ -0,0 +1,4 @@ +usr/bin/location +var/lib/location/signing-key.pem +src/systemd/*.service /lib/systemd/system/ +src/systemd/*.timer /lib/systemd/system/ diff --git a/debian/location.manpages b/debian/location.manpages new file mode 100644 index 0000000..3e662bb --- /dev/null +++ b/debian/location.manpages @@ -0,0 +1 @@ +man/location.8 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..05b88fd --- /dev/null +++ b/debian/rules @@ -0,0 +1,28 @@ +#!/usr/bin/make -f + +# enable verbose mode +#export DH_VERBOSE=1 + +# enable all hardening build flags +export DEB_BUILD_MAINT_OPTIONS=hardening=+all + +%: + dh $@ --with python3 --with-systemd + +override_dh_auto_configure: + intltoolize --force --automake + dh_auto_configure -- --disable-perl + +override_dh_perl: + dh_perl -d + +override_dh_systemd_enable: + dh_systemd_enable location-update.timer + +override_dh_install: + dh_install + # lintian: unknown-file-in-python-module-directory + rm debian/location-python/usr/lib/python3*/site-packages/_location.la + # linitan: binaries-have-file-conflict (d/location-importer.install) + rm debian/location-python/usr/lib/python3*/site-packages/location/database.py + rm debian/location-python/usr/lib/python3*/site-packages/location/importer.py diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000..19ace6d --- /dev/null +++ b/debian/watch @@ -0,0 +1,3 @@ +version=4 +https://source.ipfire.org/releases/libloc/ \ + @PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ debian uupdate diff --git a/examples/private-key.pem b/examples/private-key.pem new file mode 100644 index 0000000..febe142 --- /dev/null +++ b/examples/private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEILuXwOyh5xQM3KnGb50wk00Lc4I7Hl0wnTCLAbFKpFJQoAoGCCqGSM49 +AwEHoUQDQgAE9RG8+mTIvluN0b/gEyVtcffqHxaOlYDyxzPKaWZPS+3UcNN4lwm2 +F4Tt2oNCqcuGl5hsZFNV7GJbf17h3F8/vA== +-----END EC PRIVATE KEY----- diff --git a/examples/public-key.pem b/examples/public-key.pem new file mode 100644 index 0000000..709563b --- /dev/null +++ b/examples/public-key.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9RG8+mTIvluN0b/gEyVtcffqHxaO +lYDyxzPKaWZPS+3UcNN4lwm2F4Tt2oNCqcuGl5hsZFNV7GJbf17h3F8/vA== +-----END PUBLIC KEY----- diff --git a/examples/python/create-database.py b/examples/python/create-database.py new file mode 100644 index 0000000..04b2dc3 --- /dev/null +++ b/examples/python/create-database.py @@ -0,0 +1,44 @@ +#!/usr/bin/python3 + +import _location as location +import os +import sys + +ABS_SRCDIR = os.environ.get("ABS_SRCDIR", ".") + +private_key_path = os.path.join(ABS_SRCDIR, "examples/private-key.pem") + +with open(private_key_path, "r") as pkey: + w = location.Writer(pkey) + + # Set the vendor + w.vendor = "IPFire Project" + + # Set a description + w.description = "This is a geo location database" + + # Set a license + w.license = "CC" + + # Add a country + c = w.add_country("DE") + c.continent_code = "EU" + c.name = "Germany" + + # Add an AS + a = w.add_as(204867) + a.name = "Lightning Wire Labs GmbH" + + print(a) + + # Add a network + n = w.add_network("2a07:1c44:5800::/40") + n.country_code = "DE" + n.asn = a.number + n.set_flag(location.NETWORK_FLAG_ANYCAST) + + print(n) + + # Write the database to disk + for f in sys.argv[1:]: + w.write(f) diff --git a/examples/python/read-database.py b/examples/python/read-database.py new file mode 100644 index 0000000..28a2c6d --- /dev/null +++ b/examples/python/read-database.py @@ -0,0 +1,22 @@ +#!/usr/bin/python3 + +import location + +# Open the database +d = location.Database("test.db") +print(d) + +# Try to get information about AS123 +a = d.get_as(123) +print(a) + +# Try to get information about AS204867 +a = d.get_as(204867) +print(a) + +# Search for an IP address in the database +n = d.lookup("8.8.8.8") +print(n) + +n = d.lookup("2a07:1c44:5800:1234:1234:1234:1234:1234") +print(n) diff --git a/m4/.gitignore b/m4/.gitignore new file mode 100644 index 0000000..55eaa80 --- /dev/null +++ b/m4/.gitignore @@ -0,0 +1,6 @@ +intltool.m4 +libtool.m4 +ltoptions.m4 +ltsugar.m4 +ltversion.m4 +lt~obsolete.m4 diff --git a/m4/attributes.m4 b/m4/attributes.m4 new file mode 100644 index 0000000..7e080da --- /dev/null +++ b/m4/attributes.m4 @@ -0,0 +1,288 @@ +dnl Macros to check the presence of generic (non-typed) symbols. +dnl Copyright (c) 2006-2008 Diego Pettenò +dnl Copyright (c) 2006-2008 xine project +dnl Copyright (c) 2012 Lucas De Marchi +dnl +dnl This program is free software; you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation; either version 2, or (at your option) +dnl any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU General Public License for more details. +dnl +dnl You should have received a copy of the GNU General Public License +dnl along with this program; if not, write to the Free Software +dnl Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +dnl 02110-1301, USA. +dnl +dnl As a special exception, the copyright owners of the +dnl macro gives unlimited permission to copy, distribute and modify the +dnl configure scripts that are the output of Autoconf when processing the +dnl Macro. You need not follow the terms of the GNU General Public +dnl License when using or distributing such scripts, even though portions +dnl of the text of the Macro appear in them. The GNU General Public +dnl License (GPL) does govern all other use of the material that +dnl constitutes the Autoconf Macro. +dnl +dnl This special exception to the GPL applies to versions of the +dnl Autoconf Macro released by this project. When you make and +dnl distribute a modified version of the Autoconf Macro, you may extend +dnl this special exception to the GPL to apply to your modified version as +dnl well. + +dnl Check if FLAG in ENV-VAR is supported by compiler and append it +dnl to WHERE-TO-APPEND variable +dnl CC_CHECK_FLAG_APPEND([WHERE-TO-APPEND], [ENV-VAR], [FLAG]) + +AC_DEFUN([CC_CHECK_FLAG_APPEND], [ + AC_CACHE_CHECK([if $CC supports flag $3 in envvar $2], + AS_TR_SH([cc_cv_$2_$3]), + [eval "AS_TR_SH([cc_save_$2])='${$2}'" + eval "AS_TR_SH([$2])='-Werror $3'" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([int a = 0; int main(void) { return a; } ])], + [eval "AS_TR_SH([cc_cv_$2_$3])='yes'"], + [eval "AS_TR_SH([cc_cv_$2_$3])='no'"]) + eval "AS_TR_SH([$2])='$cc_save_$2'"]) + + AS_IF([eval test x$]AS_TR_SH([cc_cv_$2_$3])[ = xyes], + [eval "$1='${$1} $3'"]) +]) + +dnl CC_CHECK_FLAGS_APPEND([WHERE-TO-APPEND], [ENV-VAR], [FLAG1 FLAG2]) +AC_DEFUN([CC_CHECK_FLAGS_APPEND], [ + for flag in $3; do + CC_CHECK_FLAG_APPEND($1, $2, $flag) + done +]) + +dnl Check if the flag is supported by linker (cacheable) +dnl CC_CHECK_LDFLAGS([FLAG], [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND]) + +AC_DEFUN([CC_CHECK_LDFLAGS], [ + AC_CACHE_CHECK([if $CC supports $1 flag], + AS_TR_SH([cc_cv_ldflags_$1]), + [ac_save_LDFLAGS="$LDFLAGS" + LDFLAGS="$LDFLAGS $1" + AC_LINK_IFELSE([int main() { return 1; }], + [eval "AS_TR_SH([cc_cv_ldflags_$1])='yes'"], + [eval "AS_TR_SH([cc_cv_ldflags_$1])="]) + LDFLAGS="$ac_save_LDFLAGS" + ]) + + AS_IF([eval test x$]AS_TR_SH([cc_cv_ldflags_$1])[ = xyes], + [$2], [$3]) +]) + +dnl define the LDFLAGS_NOUNDEFINED variable with the correct value for +dnl the current linker to avoid undefined references in a shared object. +AC_DEFUN([CC_NOUNDEFINED], [ + dnl We check $host for which systems to enable this for. + AC_REQUIRE([AC_CANONICAL_HOST]) + + case $host in + dnl FreeBSD (et al.) does not complete linking for shared objects when pthreads + dnl are requested, as different implementations are present; to avoid problems + dnl use -Wl,-z,defs only for those platform not behaving this way. + *-freebsd* | *-openbsd*) ;; + *) + dnl First of all check for the --no-undefined variant of GNU ld. This allows + dnl for a much more readable commandline, so that people can understand what + dnl it does without going to look for what the heck -z defs does. + for possible_flags in "-Wl,--no-undefined" "-Wl,-z,defs"; do + CC_CHECK_LDFLAGS([$possible_flags], [LDFLAGS_NOUNDEFINED="$possible_flags"]) + break + done + ;; + esac + + AC_SUBST([LDFLAGS_NOUNDEFINED]) +]) + +dnl Check for a -Werror flag or equivalent. -Werror is the GCC +dnl and ICC flag that tells the compiler to treat all the warnings +dnl as fatal. We usually need this option to make sure that some +dnl constructs (like attributes) are not simply ignored. +dnl +dnl Other compilers don't support -Werror per se, but they support +dnl an equivalent flag: +dnl - Sun Studio compiler supports -errwarn=%all +AC_DEFUN([CC_CHECK_WERROR], [ + AC_CACHE_CHECK( + [for $CC way to treat warnings as errors], + [cc_cv_werror], + [CC_CHECK_CFLAGS_SILENT([-Werror], [cc_cv_werror=-Werror], + [CC_CHECK_CFLAGS_SILENT([-errwarn=%all], [cc_cv_werror=-errwarn=%all])]) + ]) +]) + +AC_DEFUN([CC_CHECK_ATTRIBUTE], [ + AC_REQUIRE([CC_CHECK_WERROR]) + AC_CACHE_CHECK([if $CC supports __attribute__(( ifelse([$2], , [$1], [$2]) ))], + AS_TR_SH([cc_cv_attribute_$1]), + [ac_save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $cc_cv_werror" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([$3])], + [eval "AS_TR_SH([cc_cv_attribute_$1])='yes'"], + [eval "AS_TR_SH([cc_cv_attribute_$1])='no'"]) + CFLAGS="$ac_save_CFLAGS" + ]) + + AS_IF([eval test x$]AS_TR_SH([cc_cv_attribute_$1])[ = xyes], + [AC_DEFINE( + AS_TR_CPP([SUPPORT_ATTRIBUTE_$1]), 1, + [Define this if the compiler supports __attribute__(( ifelse([$2], , [$1], [$2]) ))] + ) + $4], + [$5]) +]) + +AC_DEFUN([CC_ATTRIBUTE_CONSTRUCTOR], [ + CC_CHECK_ATTRIBUTE( + [constructor],, + [void __attribute__((constructor)) ctor() { int a; }], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_FORMAT], [ + CC_CHECK_ATTRIBUTE( + [format], [format(printf, n, n)], + [void __attribute__((format(printf, 1, 2))) printflike(const char *fmt, ...) { fmt = (void *)0; }], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_FORMAT_ARG], [ + CC_CHECK_ATTRIBUTE( + [format_arg], [format_arg(printf)], + [char *__attribute__((format_arg(1))) gettextlike(const char *fmt) { fmt = (void *)0; }], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_VISIBILITY], [ + CC_CHECK_ATTRIBUTE( + [visibility_$1], [visibility("$1")], + [void __attribute__((visibility("$1"))) $1_function() { }], + [$2], [$3]) +]) + +AC_DEFUN([CC_ATTRIBUTE_NONNULL], [ + CC_CHECK_ATTRIBUTE( + [nonnull], [nonnull()], + [void __attribute__((nonnull())) some_function(void *foo, void *bar) { foo = (void*)0; bar = (void*)0; }], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_UNUSED], [ + CC_CHECK_ATTRIBUTE( + [unused], , + [void some_function(void *foo, __attribute__((unused)) void *bar);], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_SENTINEL], [ + CC_CHECK_ATTRIBUTE( + [sentinel], , + [void some_function(void *foo, ...) __attribute__((sentinel));], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_DEPRECATED], [ + CC_CHECK_ATTRIBUTE( + [deprecated], , + [void some_function(void *foo, ...) __attribute__((deprecated));], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_ALIAS], [ + CC_CHECK_ATTRIBUTE( + [alias], [weak, alias], + [void other_function(void *foo) { } + void some_function(void *foo) __attribute__((weak, alias("other_function")));], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_MALLOC], [ + CC_CHECK_ATTRIBUTE( + [malloc], , + [void * __attribute__((malloc)) my_alloc(int n);], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_PACKED], [ + CC_CHECK_ATTRIBUTE( + [packed], , + [struct astructure { char a; int b; long c; void *d; } __attribute__((packed));], + [$1], [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_CONST], [ + CC_CHECK_ATTRIBUTE( + [const], , + [int __attribute__((const)) twopow(int n) { return 1 << n; } ], + [$1], [$2]) +]) + +AC_DEFUN([CC_FLAG_VISIBILITY], [ + AC_REQUIRE([CC_CHECK_WERROR]) + AC_CACHE_CHECK([if $CC supports -fvisibility=hidden], + [cc_cv_flag_visibility], + [cc_flag_visibility_save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $cc_cv_werror" + CC_CHECK_CFLAGS_SILENT([-fvisibility=hidden], + cc_cv_flag_visibility='yes', + cc_cv_flag_visibility='no') + CFLAGS="$cc_flag_visibility_save_CFLAGS"]) + + AS_IF([test "x$cc_cv_flag_visibility" = "xyes"], + [AC_DEFINE([SUPPORT_FLAG_VISIBILITY], 1, + [Define this if the compiler supports the -fvisibility flag]) + $1], + [$2]) +]) + +AC_DEFUN([CC_FUNC_EXPECT], [ + AC_REQUIRE([CC_CHECK_WERROR]) + AC_CACHE_CHECK([if compiler has __builtin_expect function], + [cc_cv_func_expect], + [ac_save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $cc_cv_werror" + AC_COMPILE_IFELSE([AC_LANG_SOURCE( + [int some_function() { + int a = 3; + return (int)__builtin_expect(a, 3); + }])], + [cc_cv_func_expect=yes], + [cc_cv_func_expect=no]) + CFLAGS="$ac_save_CFLAGS" + ]) + + AS_IF([test "x$cc_cv_func_expect" = "xyes"], + [AC_DEFINE([SUPPORT__BUILTIN_EXPECT], 1, + [Define this if the compiler supports __builtin_expect() function]) + $1], + [$2]) +]) + +AC_DEFUN([CC_ATTRIBUTE_ALIGNED], [ + AC_REQUIRE([CC_CHECK_WERROR]) + AC_CACHE_CHECK([highest __attribute__ ((aligned ())) supported], + [cc_cv_attribute_aligned], + [ac_save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $cc_cv_werror" + for cc_attribute_align_try in 64 32 16 8 4 2; do + AC_COMPILE_IFELSE([AC_LANG_SOURCE([ + int main() { + static char c __attribute__ ((aligned($cc_attribute_align_try))) = 0; + return c; + }])], [cc_cv_attribute_aligned=$cc_attribute_align_try; break]) + done + CFLAGS="$ac_save_CFLAGS" + ]) + + if test "x$cc_cv_attribute_aligned" != "x"; then + AC_DEFINE_UNQUOTED([ATTRIBUTE_ALIGNED_MAX], [$cc_cv_attribute_aligned], + [Define the highest alignment supported]) + fi +]) diff --git a/m4/ax_prog_perl_modules.m4 b/m4/ax_prog_perl_modules.m4 new file mode 100644 index 0000000..70b3230 --- /dev/null +++ b/m4/ax_prog_perl_modules.m4 @@ -0,0 +1,77 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_prog_perl_modules.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_PROG_PERL_MODULES([MODULES], [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +# +# DESCRIPTION +# +# Checks to see if the given perl modules are available. If true the shell +# commands in ACTION-IF-TRUE are executed. If not the shell commands in +# ACTION-IF-FALSE are run. Note if $PERL is not set (for example by +# calling AC_CHECK_PROG, or AC_PATH_PROG), AC_CHECK_PROG(PERL, perl, perl) +# will be run. +# +# MODULES is a space separated list of module names. To check for a +# minimum version of a module, append the version number to the module +# name, separated by an equals sign. +# +# Example: +# +# AX_PROG_PERL_MODULES( Text::Wrap Net::LDAP=1.0.3, , +# AC_MSG_WARN(Need some Perl modules) +# +# LICENSE +# +# Copyright (c) 2009 Dean Povey +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 8 + +AU_ALIAS([AC_PROG_PERL_MODULES], [AX_PROG_PERL_MODULES]) +AC_DEFUN([AX_PROG_PERL_MODULES],[dnl + +m4_define([ax_perl_modules]) +m4_foreach([ax_perl_module], m4_split(m4_normalize([$1])), + [ + m4_append([ax_perl_modules], + [']m4_bpatsubst(ax_perl_module,=,[ ])[' ]) + ]) + +# Make sure we have perl +if test -z "$PERL"; then +AC_CHECK_PROG(PERL,perl,perl) +fi + +if test "x$PERL" != x; then + ax_perl_modules_failed=0 + for ax_perl_module in ax_perl_modules; do + AC_MSG_CHECKING(for perl module $ax_perl_module) + + # Would be nice to log result here, but can't rely on autoconf internals + $PERL -e "use $ax_perl_module; exit" > /dev/null 2>&1 + if test $? -ne 0; then + AC_MSG_RESULT(no); + ax_perl_modules_failed=1 + else + AC_MSG_RESULT(ok); + fi + done + + # Run optional shell commands + if test "$ax_perl_modules_failed" = 0; then + : + $2 + else + : + $3 + fi +else + AC_MSG_WARN(could not find perl) +fi])dnl diff --git a/m4/ld-version-script.m4 b/m4/ld-version-script.m4 new file mode 100644 index 0000000..211d67b --- /dev/null +++ b/m4/ld-version-script.m4 @@ -0,0 +1,48 @@ +# ld-version-script.m4 serial 4 +dnl Copyright (C) 2008-2015 Free Software Foundation, Inc. +dnl This file is free software; the Free Software Foundation +dnl gives unlimited permission to copy and/or distribute it, +dnl with or without modifications, as long as this notice is preserved. + +dnl From Simon Josefsson + +# FIXME: The test below returns a false positive for mingw +# cross-compiles, 'local:' statements does not reduce number of +# exported symbols in a DLL. Use --disable-ld-version-script to work +# around the problem. + +# gl_LD_VERSION_SCRIPT +# -------------------- +# Check if LD supports linker scripts, and define automake conditional +# HAVE_LD_VERSION_SCRIPT if so. +AC_DEFUN([gl_LD_VERSION_SCRIPT], +[ + AC_ARG_ENABLE([ld-version-script], + [AS_HELP_STRING([--enable-ld-version-script], + [enable linker version script (default is enabled when possible)])], + [have_ld_version_script=$enableval], + [AC_CACHE_CHECK([if LD -Wl,--version-script works], + [gl_cv_sys_ld_version_script], + [gl_cv_sys_ld_version_script=no + save_LDFLAGS=$LDFLAGS + LDFLAGS="$LDFLAGS -Wl,--version-script=conftest.map" + echo foo >conftest.map + AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], + [], + [cat > conftest.map <} +{0#{target}{0}} +{0#} +endif::backend-docbook[] + +ifdef::backend-html5[] +[link-inlinemacro] +{target}{0?({0})} +endif::backend-html5[] diff --git a/man/libloc.txt b/man/libloc.txt new file mode 100644 index 0000000..baf98c1 --- /dev/null +++ b/man/libloc.txt @@ -0,0 +1,51 @@ += libloc(3) + +== Name + +libloc - A tool to query the IPFire Location database + +== Synopsis +[verse] + +#include + +`pkg-config --cflags --libs libloc` + +== Description + +`libloc` is a lightweight library which can be used to query the IPFire +Location database. + +See + + * link:loc_new[3] + * link:loc_get_log_priority[3] + * link:loc_set_log_priority[3] + * link:loc_get_log_fn[3] + * link:loc_database_count_as[3] + * link:loc_database_get_as[3] + * link:loc_database_get_country[3] + * link:loc_database_lookup[3] + * link:loc_database_new[3] + +for more information about the functions available. + +== Copying + +Copyright (C) 2022 {author}. + +This library is free software; you can redistribute it and/or modify it under the terms +of the GNU Lesser General Public License as published by the Free Software Foundation; +either version 2.1 of the License, or (at your option) any later version. + +== See Also + +link:location[8] + +== Bug Reports + +Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/; +refer to https://wiki.ipfire.org/devel/bugzilla for details. + +== Authors + +Michael Tremer diff --git a/man/loc_database_count_as.txt b/man/loc_database_count_as.txt new file mode 100644 index 0000000..4cbe151 --- /dev/null +++ b/man/loc_database_count_as.txt @@ -0,0 +1,24 @@ += loc_database_count_as(3) + +== Name + +loc_database_count_as - Return the number of ASes in the database + +== Synopsis +[verse] + +#include + +size_t loc_database_count_as(struct loc_database{empty}* db); + +== Description + +Returns the total number of ASes in the database. + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_database_get_as.txt b/man/loc_database_get_as.txt new file mode 100644 index 0000000..e4b962f --- /dev/null +++ b/man/loc_database_get_as.txt @@ -0,0 +1,31 @@ += loc_database_get_as(3) + +== Name + +loc_database_get_as - Fetch an AS from the database + +== Synopsis +[verse] + +#include + +int loc_database_get_as(struct loc_database{empty}* db, struct loc_as{empty}*{empty}* as, + uint32_t number); + +== Description + +This function retrieves an Autonomous System with the matching _number_ from the database +and stores it in _as_. + +== Return Value + +On success, zero is returned. Otherwise non-zero is being returned and _errno_ is set +accordingly. + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_database_get_country.txt b/man/loc_database_get_country.txt new file mode 100644 index 0000000..b5ab8ec --- /dev/null +++ b/man/loc_database_get_country.txt @@ -0,0 +1,29 @@ += loc_database_get_country(3) + +== Name + +loc_database_get_country - Fetch country information from the database + +== Synopsis + +#include + +int loc_database_get_country(struct loc_database{empty}* db, + struct loc_country{empty}*{empty}* country, const char{empty}* code); + +== Description + +This function fetches information about the country with the matching _code_. + +== Return Value + +On success, zero is returned. Otherwise non-zero is being returned and _errno_ is set +accordingly. + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_database_lookup.txt b/man/loc_database_lookup.txt new file mode 100644 index 0000000..b6f780a --- /dev/null +++ b/man/loc_database_lookup.txt @@ -0,0 +1,37 @@ += loc_database_lookup(3) + +== Name + +loc_database_lookup - Lookup a network from the database + +== Synopsis + +#include + +int loc_database_lookup(struct loc_database{empty}* db, + const struct in6_addr{empty}* address, struct loc_network{empty}*{empty}* network); + +int loc_database_lookup_from_string(struct loc_database{empty}* db, + const char{empty}* string, struct loc_network{empty}*{empty}* network); + +== Description + +The lookup functions try finding a network in the database. + +_loc_database_lookup_ takes the IP address as _struct in6_addr_ format which can either +be a regular IPv6 address or a mapped IPv4 address. + +_loc_database_lookup_string_ takes the IP address as string and will parse it automatically. + +== Return Value + +On success, zero is returned. Otherwise non-zero is being returned and _errno_ is set +accordingly. + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_database_new.txt b/man/loc_database_new.txt new file mode 100644 index 0000000..86a021b --- /dev/null +++ b/man/loc_database_new.txt @@ -0,0 +1,53 @@ += loc_database_new(3) + +== Name + +loc_database_new - Create a new libloc context + +== Synopsis +[verse] + +#include +#include + +struct loc_database; + +int loc_database_new(struct loc_ctx{empty}* ctx, + struct loc_database{empty}*{empty}* database, FILE{empty}* f); + +Reference Counting: + +struct loc_database{empty}* loc_database_ref(struct loc_database{empty}* db); + +struct loc_database{empty}* loc_database_unref(struct loc_database{empty}* db); + +Access some data: + +time_t loc_database_created_at(struct loc_database{empty}* db); + +const char{empty}* loc_database_get_vendor(struct loc_database{empty}* db); + +const char{empty}* loc_database_get_description(struct loc_database{empty}* db); + +const char{empty}* loc_database_get_license(struct loc_database{empty}* db); + +== Description + +loc_database_new() opens a new database from the given file descriptor. +The file descriptor can be closed after this operation because the function is creating +its own copy. + +If the database could be opened successfully, zero is returned. Otherwise a non-zero +return code will indicate an error and errno will be set appropriately. + +Various meta-data about the database can be retrieved with +loc_database_created_at(), loc_database_get_vendor(), loc_database_get_description(), +and loc_database_get_license(). + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_get_log_priority.txt b/man/loc_get_log_priority.txt new file mode 100644 index 0000000..447a8c8 --- /dev/null +++ b/man/loc_get_log_priority.txt @@ -0,0 +1,28 @@ += loc_get_log_priority(3) + +== Name + +loc_get_log_priority - Fetches the log level of a libloc context + +== Synopsis +[verse] + +#include + +int loc_get_log_priority(struct loc_ctx{empty}* ctx); + +== Description + +Returns the log priority of the given context. + +The returned integer is a valid syslog log level as defined in syslog(3). + +The default value is LOG_ERR. + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_new.txt b/man/loc_new.txt new file mode 100644 index 0000000..c1000be --- /dev/null +++ b/man/loc_new.txt @@ -0,0 +1,35 @@ += loc_new(3) + +== Name + +loc_new - Create a new libloc context + +== Synopsis +[verse] + +#include + +struct loc_ctx; + +int loc_new(struct loc_ctx{empty}*{empty}* ctx); + +struct loc_ctx{empty}* loc_ref(struct loc_ctx{empty}* ctx); + +struct loc_ctx{empty}* loc_unref(struct loc_ctx{empty}* ctx); + +== Description + +Every operation in libloc requires to set up a context first. +This is done by calling loc_new(3). + +Every time another part of your code is holding a reference to the context, +you will need to call loc_ref() to increase the reference counter. +If you no longer need the context, you will need to call loc_unref(). + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_set_log_fn.txt b/man/loc_set_log_fn.txt new file mode 100644 index 0000000..00c1854 --- /dev/null +++ b/man/loc_set_log_fn.txt @@ -0,0 +1,29 @@ += loc_set_log_fn(3) + +== Name + +loc_set_log_fn - Sets the log callback function + +== Synopsis +[verse] + +#include + +void loc_set_log_fn(struct loc_ctx{empty}* ctx, + void ({empty}*log_fn)(struct loc_ctx{empty}* ctx, int priority, + const char{empty}* file, int line, const char{empty}* fn, const char{empty}* format, + va_list args) + +== Description + +libloc can use the calling application's logging system by setting this callback. + +It will be called once for each log message according to the configured log level. + +== See Also + +link:libloc[3] + +== Authors + +Michael Tremer diff --git a/man/loc_set_log_priority.txt b/man/loc_set_log_priority.txt new file mode 100644 index 0000000..76556bb --- /dev/null +++ b/man/loc_set_log_priority.txt @@ -0,0 +1,25 @@ += loc_set_log_priority(3) + +== Name + +loc_set_log_priority - Sets the log level of a libloc context + +== Synopsis +[verse] + +#include + +void loc_set_log_priority(struct loc_ctx{empty}* ctx, int priority) + +== Description + +Sets the log priority of the given context. See loc_get_log_priority(3) for more details. + +== See Also + +link:libloc[3] +link:loc_set_log_fn(3) + +== Authors + +Michael Tremer diff --git a/man/location.txt b/man/location.txt new file mode 100644 index 0000000..3dfddf5 --- /dev/null +++ b/man/location.txt @@ -0,0 +1,149 @@ += location(8) + +== NAME +location - Query the location database + +== SYNOPSIS +[verse] +`location export --directory=DIR [--format=FORMAT] [--family=ipv6|ipv4] [ASN|CC ...]` +`location get-as ASN [ASN...]` +`location list-countries [--show-name] [--show-continent]` +`location list-networks-by-as ASN` +`location list-networks-by-cc COUNTRY_CODE` +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]` +`location lookup ADDRESS [ADDRESS...]` +`location search-as STRING` +`location update [--cron=daily|weekly|monthly]` +`location verify` +`location version` + +== DESCRIPTION +`location` retrieves information from the location database. +This data can be used to determine someone's location on the Internet +and for building firewall rulesets to block access from certain ASes +or countries. + +== OPTIONS + +--database FILE:: +-d FILE:: + The path of the database which is being opened. + + + If this option is omitted, the system's database will be opened. + +--quiet:: + Enable quiet mode + +--debug:: + Enable debugging mode + +== COMMANDS + +'export [--directory=DIR] [--format=FORMAT] [--family=ipv6|ipv4] [ASN|CC ...]':: + This command exports the whole database into the given directory. + + + The output can be filtered by only exporting a certain address family, or by passing + a list of country codes and/or ASNs. The default is to export all known countries. + + + The output format can be chosen with the '--format' parameter. For possible formats, + please see below. + + + If the '--directory' option is omitted, the output will be written to stdout which + is useful when you want to load any custom exports straight into nftables or ipset. + +'get-as ASN [ASN...]':: + This command returns the name of the owning organisation of the Autonomous + System. + +'list-countries [--show-name] [--show-continent]':: + Lists all countries known to the database. + + + With the optional parameters '--show-name' and '--show-continent', the name and + continent code will be printed, too. + +'list-networks-by-as [--family=[ipv6|ipv4]] [--format=FORMAT] ASN':: + Lists all networks which belong to this Autonomous System. + + + The '--family' parameter can be used to filter output to only IPv6 or + IPv4 addresses. + + + The '--format' parameter can change the output so that it can be + directly loaded into other software. For details see below. + +'list-networks-by-cc [--family=[ipv6|ipv4]] [--format=FORMAT] COUNTRY_CODE':: + Lists all networks that belong to a country. + + + The country has to be encoded in ISO3166 Alpha-2 notation. + + + See above for usage of the '--family' and '--format' parameters. + +'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]':: + Lists all networks that have a certain flag. + + + See above for usage of the '--family' and '--format' parameters. + +'list-bogons [--family=[ipv6|ipv4]] [--format=FORMAT]':: + Lists all bogons (i.e. networks that are unknown to the database). + + + See above for usage of the '--family' and '--format' parameters. + +'lookup ADDRESS [ADDRESS...]':: + This command returns the network the given IP address has been found in + as well as its Autonomous System if that information is available. + +'search-as STRING':: + Lists all Autonomous Systems which match the given string. + + + The search will be performed case-insensitively. + +'update':: + This command will try to update the local database. + + + It will terminate with a return code of zero if the database has been + successfully updated. 1 on error, 2 on invalid call and 3 if the + database was already the latest version. + + + The '--cron' option allows limiting updates to once a day ('daily'), once a week + ('weekly'), or once a month ('monthly'). If the task is being called, but the + database has been updated recently, an update will be skipped. + +'verify':: + Verifies the downloaded database. + +'version':: + Shows the version information of the downloaded database. + +'--help':: + Shows a short help text on using this program. + +'--version':: + Shows the program's version and exists. + +== EXIT CODES +The 'location' command will normally exit with code zero. +If there has been a problem and the requested action could not be performed, +the exit code is unequal to zero. + +== FORMATS +Some commands allow specifying the output format. This is helpful if the exported +data should be imported into a packet filter for example. +The following formats are understood: + + * 'list' (default): Just lists all networks, one per line + * 'ipset': For ipset + * 'nftables': For nftables + * 'xt_geoip': Returns a list of networks to be loaded into the + xt_geoip kernel module + +== HOW IT WORKS +The downloader checks a DNS record for the latest version of the database. +It will then try to download a file with that version from a mirror server. +If the downloaded file is outdated, the next mirror will be tried until we +have found a file that is recent enough. + +== BUG REPORTS +Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/; +refer to https://wiki.ipfire.org/devel/bugzilla for details. + +== AUTHORS +Michael Tremer diff --git a/po/.gitignore b/po/.gitignore new file mode 100644 index 0000000..a60406b --- /dev/null +++ b/po/.gitignore @@ -0,0 +1,7 @@ +/POTFILES +/Makefile.in.in +/.intltool-merge-cache +/Makefile +/stamp-it +*.gmo +*.pot diff --git a/po/LINGUAS b/po/LINGUAS new file mode 100644 index 0000000..7673daa --- /dev/null +++ b/po/LINGUAS @@ -0,0 +1 @@ +de diff --git a/po/POTFILES.in b/po/POTFILES.in new file mode 100644 index 0000000..5d2cc46 --- /dev/null +++ b/po/POTFILES.in @@ -0,0 +1,12 @@ +src/libloc.pc.in +src/python/__init__.py.in +src/python/database.py +src/python/downloader.py +src/python/export.py +src/python/i18n.py +src/python/importer.py +src/python/location-importer.in +src/python/location.in +src/python/logger.py +src/systemd/location-update.service.in +src/systemd/location-update.timer.in diff --git a/po/de.po b/po/de.po new file mode 100644 index 0000000..3cbcdd7 --- /dev/null +++ b/po/de.po @@ -0,0 +1,258 @@ +# German translations for libloc package. +# Copyright (C) 2018 THE libloc'S COPYRIGHT HOLDER +# This file is distributed under the same license as the libloc package. +# root , 2018. +# +msgid "" +msgstr "" +"Project-Id-Version: libloc 0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-04-15 11:29+0000\n" +"PO-Revision-Date: 2018-02-01 14:05+0000\n" +"Last-Translator: Michael Tremer \n" +"Language-Team: German\n" +"Language: de\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=ASCII\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +msgid "Location Importer Command Line Interface" +msgstr "" + +msgid "Enable debug output" +msgstr "" + +msgid "Enable quiet mode" +msgstr "" + +msgid "Database Hostname" +msgstr "" + +msgid "HOST" +msgstr "" + +msgid "Database Name" +msgstr "" + +msgid "NAME" +msgstr "" + +msgid "Database Username" +msgstr "" + +msgid "USERNAME" +msgstr "" + +msgid "Database Password" +msgstr "" + +msgid "PASSWORD" +msgstr "" + +#. Write Database +msgid "Write database to file" +msgstr "" + +msgid "Database File" +msgstr "" + +msgid "Signing Key" +msgstr "" + +msgid "Backup Signing Key" +msgstr "" + +msgid "Sets the vendor" +msgstr "" + +msgid "Sets a description" +msgstr "" + +msgid "Sets the license" +msgstr "" + +msgid "Database Format Version" +msgstr "" + +#. Update WHOIS +msgid "Update WHOIS Information" +msgstr "" + +msgid "Update BGP Annoucements" +msgstr "" + +msgid "Route Server to connect to" +msgstr "" + +msgid "SERVER" +msgstr "" + +msgid "Update overrides" +msgstr "" + +msgid "Files to import" +msgstr "" + +msgid "Import countries" +msgstr "" + +msgid "File to import" +msgstr "" + +msgid "Location Database Command Line Interface" +msgstr "" + +msgid "Path to database" +msgstr "" + +msgid "Public Signing Key" +msgstr "" + +msgid "Show database version" +msgstr "" + +msgid "Lookup one or multiple IP addresses" +msgstr "" + +msgid "Dump the entire database" +msgstr "" + +#. Update +msgid "Update database" +msgstr "" + +msgid "Update the library only once per interval" +msgstr "" + +msgid "Verify the downloaded database" +msgstr "" + +msgid "Get information about one or multiple Autonomous Systems" +msgstr "" + +msgid "Search for Autonomous Systems that match the string" +msgstr "" + +msgid "Lists all networks in an AS" +msgstr "" + +msgid "Lists all networks in a country" +msgstr "" + +msgid "Lists all networks with flags" +msgstr "" + +msgid "Anonymous Proxies" +msgstr "" + +msgid "Satellite Providers" +msgstr "" + +msgid "Anycasts" +msgstr "" + +msgid "Hostile Networks safe to drop" +msgstr "" + +msgid "Lists all countries" +msgstr "" + +msgid "Show the name of the country" +msgstr "" + +msgid "Show the continent" +msgstr "" + +msgid "Exports data in many formats to load it into packet filters" +msgstr "" + +msgid "Output format" +msgstr "" + +msgid "Output directory" +msgstr "" + +msgid "Specify address family" +msgstr "" + +msgid "List country codes or ASNs to export" +msgstr "" + +#, python-format +msgid "Invalid IP address: %s" +msgstr "" + +#, python-format +msgid "Nothing found for %(address)s" +msgstr "" + +msgid "Network" +msgstr "" + +msgid "Country" +msgstr "" + +msgid "Autonomous System" +msgstr "" + +msgid "Anonymous Proxy" +msgstr "" + +msgid "yes" +msgstr "" + +msgid "Satellite Provider" +msgstr "" + +msgid "Anycast" +msgstr "" + +#, python-format +msgid "Invalid ASN: %s" +msgstr "" + +#, python-format +msgid "Could not find AS%s" +msgstr "" + +#, python-format +msgid "AS%(asn)s belongs to %(name)s" +msgstr "" + +msgid "The database has been updated recently" +msgstr "" + +msgid "You must at least pass one flag" +msgstr "" + +#, python-format +msgid "One Day" +msgid_plural "%(days)s Days" +msgstr[0] "" +msgstr[1] "" + +#, python-format +msgid "One Hour" +msgid_plural "%(hours)s Hours" +msgstr[0] "" +msgstr[1] "" + +#, python-format +msgid "One Minute" +msgid_plural "%(minutes)s Minutes" +msgstr[0] "" +msgstr[1] "" + +#, python-format +msgid "One Second" +msgid_plural "%(seconds)s Seconds" +msgstr[0] "" +msgstr[1] "" + +msgid "Now" +msgstr "" + +#, python-format +msgid "%s ago" +msgstr "" diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..a0ca3cf --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,16 @@ +.dirstamp +.deps/ +.libs/ +*.la +*.lo +*.trs +libloc.pc +test-address +test-as +test-libloc +test-database +test-country +test-network +test-network-list +test-signature +test-stringpool diff --git a/src/address.c b/src/address.c new file mode 100644 index 0000000..dd32dbc --- /dev/null +++ b/src/address.c @@ -0,0 +1,160 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2022 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include + +#include + +#define LOC_ADDRESS_BUFFERS 6 +#define LOC_ADDRESS_BUFFER_LENGTH INET6_ADDRSTRLEN + +static char __loc_address_buffers[LOC_ADDRESS_BUFFERS][LOC_ADDRESS_BUFFER_LENGTH + 1]; +static int __loc_address_buffer_idx = 0; + +static const char* __loc_address6_str(const struct in6_addr* address, char* buffer, size_t length) { + return inet_ntop(AF_INET6, address, buffer, length); +} + +static const char* __loc_address4_str(const struct in6_addr* address, char* buffer, size_t length) { + struct in_addr address4 = { + .s_addr = address->s6_addr32[3], + }; + + return inet_ntop(AF_INET, &address4, buffer, length); +} + +const char* loc_address_str(const struct in6_addr* address) { + if (!address) + return NULL; + + // Select buffer + char* buffer = __loc_address_buffers[__loc_address_buffer_idx++]; + + // Prevent index from overflow + __loc_address_buffer_idx %= LOC_ADDRESS_BUFFERS; + + if (IN6_IS_ADDR_V4MAPPED(address)) + return __loc_address4_str(address, buffer, LOC_ADDRESS_BUFFER_LENGTH); + else + return __loc_address6_str(address, buffer, LOC_ADDRESS_BUFFER_LENGTH); +} + +static void loc_address_from_address4(struct in6_addr* address, + const struct in_addr* address4) { + address->s6_addr32[0] = 0; + address->s6_addr32[1] = 0; + address->s6_addr32[2] = htonl(0xffff); + address->s6_addr32[3] = address4->s_addr; +} + +int loc_address_parse(struct in6_addr* address, unsigned int* prefix, const char* string) { + char buffer[INET6_ADDRSTRLEN + 4]; + int r; + + if (!address || !string) { + errno = EINVAL; + return 1; + } + + // Copy the string into the buffer + r = snprintf(buffer, sizeof(buffer) - 1, "%s", string); + if (r < 0) + return 1; + + // Find / + char* p = strchr(buffer, '/'); + if (p) { + // Terminate the IP address + *p++ = '\0'; + } + + int family = AF_UNSPEC; + + // Try parsing as an IPv6 address + r = inet_pton(AF_INET6, buffer, address); + switch (r) { + // This is not a valid IPv6 address + case 0: + break; + + // This is a valid IPv6 address + case 1: + family = AF_INET6; + break; + + // Unexpected error + default: + return 1; + } + + // Try parsing as an IPv4 address + if (!family) { + struct in_addr address4; + + r = inet_pton(AF_INET, buffer, &address4); + switch (r) { + // This was not a valid IPv4 address + case 0: + break; + + // This was a valid IPv4 address + case 1: + family = AF_INET; + + // Copy the result + loc_address_from_address4(address, &address4); + break; + + // Unexpected error + default: + return 1; + } + } + + // Invalid input + if (family == AF_UNSPEC) { + errno = EINVAL; + return 1; + } + + // Did the user request a prefix? + if (prefix) { + // Set the prefix to the default value + const unsigned int max_prefix = loc_address_family_bit_length(family); + + // Parse the actual string + if (p) { + *prefix = strtol(p, NULL, 10); + + // Check if prefix is within bounds + if (*prefix > max_prefix) { + errno = EINVAL; + return 1; + } + + // If the string didn't contain a prefix, we set the maximum + } else { + *prefix = max_prefix; + } + } + + return 0; +} diff --git a/src/as-list.c b/src/as-list.c new file mode 100644 index 0000000..7154fd2 --- /dev/null +++ b/src/as-list.c @@ -0,0 +1,174 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2020 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include + +#include +#include +#include + +struct loc_as_list { + struct loc_ctx* ctx; + int refcount; + + struct loc_as** elements; + size_t elements_size; + + size_t size; +}; + +static int loc_as_list_grow(struct loc_as_list* list) { + size_t size = list->elements_size * 2; + if (size < 1024) + size = 1024; + + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", + list, size, list->elements_size + size); + + struct loc_as** elements = reallocarray(list->elements, + list->elements_size + size, sizeof(*list->elements)); + if (!elements) + return -errno; + + list->elements = elements; + list->elements_size += size; + + return 0; +} + +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, + struct loc_as_list** list) { + struct loc_as_list* l = calloc(1, sizeof(*l)); + if (!l) + return -ENOMEM; + + l->ctx = loc_ref(ctx); + l->refcount = 1; + + DEBUG(l->ctx, "AS list allocated at %p\n", l); + *list = l; + + return 0; +} + +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { + list->refcount++; + + return list; +} + +static void loc_as_list_free(struct loc_as_list* list) { + DEBUG(list->ctx, "Releasing AS list at %p\n", list); + + loc_as_list_clear(list); + + loc_unref(list->ctx); + free(list); +} + +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { + if (!list) + return NULL; + + if (--list->refcount > 0) + return list; + + loc_as_list_free(list); + return NULL; +} + +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { + return list->size; +} + +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { + return list->size == 0; +} + +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { + if (!list->elements) + return; + + for (unsigned int i = 0; i < list->size; i++) + loc_as_unref(list->elements[i]); + + free(list->elements); + list->elements = NULL; + list->elements_size = 0; + + list->size = 0; +} + +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { + // Check index + if (index >= list->size) + return NULL; + + return loc_as_ref(list->elements[index]); +} + +LOC_EXPORT int loc_as_list_append( + struct loc_as_list* list, struct loc_as* as) { + if (loc_as_list_contains(list, as)) + return 0; + + // Check if we have space left + if (list->size >= list->elements_size) { + int r = loc_as_list_grow(list); + if (r) + return r; + } + + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); + + list->elements[list->size++] = loc_as_ref(as); + + return 0; +} + +LOC_EXPORT int loc_as_list_contains( + struct loc_as_list* list, struct loc_as* as) { + for (unsigned int i = 0; i < list->size; i++) { + if (loc_as_cmp(as, list->elements[i]) == 0) + return 1; + } + + return 0; +} + +LOC_EXPORT int loc_as_list_contains_number( + struct loc_as_list* list, uint32_t number) { + struct loc_as* as; + + int r = loc_as_new(list->ctx, &as, number); + if (r) + return -1; + + r = loc_as_list_contains(list, as); + loc_as_unref(as); + + return r; +} + +static int __loc_as_cmp(const void* as1, const void* as2) { + return loc_as_cmp(*(struct loc_as**)as1, *(struct loc_as**)as2); +} + +LOC_EXPORT void loc_as_list_sort(struct loc_as_list* list) { + // Sort everything + qsort(list->elements, list->size, sizeof(*list->elements), __loc_as_cmp); +} diff --git a/src/as.c b/src/as.c new file mode 100644 index 0000000..2cbd5c8 --- /dev/null +++ b/src/as.c @@ -0,0 +1,157 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include + +#ifdef HAVE_ENDIAN_H +# include +#endif + +#include +#include +#include +#include +#include +#include + +struct loc_as { + struct loc_ctx* ctx; + int refcount; + + uint32_t number; + char* name; +}; + +LOC_EXPORT int loc_as_new(struct loc_ctx* ctx, struct loc_as** as, uint32_t number) { + struct loc_as* a = calloc(1, sizeof(*a)); + if (!a) + return -ENOMEM; + + a->ctx = loc_ref(ctx); + a->refcount = 1; + + a->number = number; + a->name = NULL; + + DEBUG(a->ctx, "AS%u allocated at %p\n", a->number, a); + *as = a; + + return 0; +} + +LOC_EXPORT struct loc_as* loc_as_ref(struct loc_as* as) { + as->refcount++; + + return as; +} + +static void loc_as_free(struct loc_as* as) { + DEBUG(as->ctx, "Releasing AS%u %p\n", as->number, as); + + if (as->name) + free(as->name); + + loc_unref(as->ctx); + free(as); +} + +LOC_EXPORT struct loc_as* loc_as_unref(struct loc_as* as) { + if (--as->refcount > 0) + return NULL; + + loc_as_free(as); + + return NULL; +} + +LOC_EXPORT uint32_t loc_as_get_number(struct loc_as* as) { + return as->number; +} + +LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) { + return as->name; +} + +LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) { + if (as->name) + free(as->name); + + if (name) + as->name = strdup(name); + else + as->name = NULL; + + return 0; +} + +LOC_EXPORT int loc_as_cmp(struct loc_as* as1, struct loc_as* as2) { + if (as1->number > as2->number) + return 1; + + if (as1->number < as2->number) + return -1; + + return 0; +} + +int loc_as_new_from_database_v1(struct loc_ctx* ctx, struct loc_stringpool* pool, + struct loc_as** as, const struct loc_database_as_v1* dbobj) { + uint32_t number = be32toh(dbobj->number); + + int r = loc_as_new(ctx, as, number); + if (r) + return r; + + const char* name = loc_stringpool_get(pool, be32toh(dbobj->name)); + r = loc_as_set_name(*as, name); + if (r) { + loc_as_unref(*as); + return r; + } + + return 0; +} + +int loc_as_to_database_v1(struct loc_as* as, struct loc_stringpool* pool, + struct loc_database_as_v1* dbobj) { + dbobj->number = htobe32(as->number); + + // Save the name string in the string pool + off_t name = loc_stringpool_add(pool, as->name ? as->name : ""); + dbobj->name = htobe32(name); + + return 0; +} + +int loc_as_match_string(struct loc_as* as, const char* string) { + // Match all AS when no search string is set + if (!string) + return 1; + + // Cannot match anything when name is not set + if (!as->name) + return 1; + + // Search if string is in name + if (strcasestr(as->name, string) != NULL) + return 1; + + return 0; +} diff --git a/src/country-list.c b/src/country-list.c new file mode 100644 index 0000000..b2aea8b --- /dev/null +++ b/src/country-list.c @@ -0,0 +1,179 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2020 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include + +#include +#include +#include + +struct loc_country_list { + struct loc_ctx* ctx; + int refcount; + + struct loc_country** elements; + size_t elements_size; + + size_t size; +}; + +static int loc_country_list_grow(struct loc_country_list* list) { + size_t size = list->elements_size * 2; + if (size < 1024) + size = 1024; + + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", + list, size, list->elements_size + size); + + struct loc_country** elements = reallocarray(list->elements, + list->elements_size + size, sizeof(*list->elements)); + if (!elements) + return -errno; + + list->elements = elements; + list->elements_size += size; + + return 0; +} + +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, + struct loc_country_list** list) { + struct loc_country_list* l = calloc(1, sizeof(*l)); + if (!l) + return -ENOMEM; + + l->ctx = loc_ref(ctx); + l->refcount = 1; + + DEBUG(l->ctx, "Country list allocated at %p\n", l); + *list = l; + + return 0; +} + +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { + list->refcount++; + + return list; +} + +static void loc_country_list_free(struct loc_country_list* list) { + DEBUG(list->ctx, "Releasing country list at %p\n", list); + + loc_country_list_clear(list); + + loc_unref(list->ctx); + free(list); +} + +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { + if (!list) + return NULL; + + if (--list->refcount > 0) + return list; + + loc_country_list_free(list); + return NULL; +} + +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { + return list->size; +} + +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { + return list->size == 0; +} + +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { + if (!list->elements) + return; + + for (unsigned int i = 0; i < list->size; i++) + loc_country_unref(list->elements[i]); + + free(list->elements); + list->elements = NULL; + list->elements_size = 0; + + list->size = 0; +} + +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { + // Check index + if (index >= list->size) + return NULL; + + return loc_country_ref(list->elements[index]); +} + +LOC_EXPORT int loc_country_list_append( + struct loc_country_list* list, struct loc_country* country) { + if (loc_country_list_contains(list, country)) + return 0; + + // Check if we have space left + if (list->size >= list->elements_size) { + int r = loc_country_list_grow(list); + if (r) + return r; + } + + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); + + list->elements[list->size++] = loc_country_ref(country); + + return 0; +} + +LOC_EXPORT int loc_country_list_contains( + struct loc_country_list* list, struct loc_country* country) { + for (unsigned int i = 0; i < list->size; i++) { + if (loc_country_cmp(country, list->elements[i]) == 0) + return 1; + } + + return 0; +} + +LOC_EXPORT int loc_country_list_contains_code( + struct loc_country_list* list, const char* code) { + struct loc_country* country; + + int r = loc_country_new(list->ctx, &country, code); + if (r) { + // Ignore invalid country codes which would never match + if (errno == EINVAL) + return 0; + else + return r; + } + + r = loc_country_list_contains(list, country); + loc_country_unref(country); + + return r; +} + +static int __loc_country_cmp(const void* country1, const void* country2) { + return loc_country_cmp(*(struct loc_country**)country1, *(struct loc_country**)country2); +} + +LOC_EXPORT void loc_country_list_sort(struct loc_country_list* list) { + // Sort everything + qsort(list->elements, list->size, sizeof(*list->elements), __loc_country_cmp); +} diff --git a/src/country.c b/src/country.c new file mode 100644 index 0000000..0f13319 --- /dev/null +++ b/src/country.c @@ -0,0 +1,235 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include + +#include +#include +#include +#include +#include + +static const struct loc_special_country { + const char code[3]; + enum loc_network_flags flags; +} loc_special_countries[] = { + { "A1", LOC_NETWORK_FLAG_ANONYMOUS_PROXY }, + { "A2", LOC_NETWORK_FLAG_SATELLITE_PROVIDER }, + { "A3", LOC_NETWORK_FLAG_ANYCAST }, + { "XD", LOC_NETWORK_FLAG_DROP }, + { "", 0 }, +}; + +struct loc_country { + struct loc_ctx* ctx; + int refcount; + + char* code; + char* continent_code; + + char* name; +}; + +LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { + // Check of the country code is valid + if (!loc_country_code_is_valid(country_code)) { + errno = EINVAL; + return 1; + } + + struct loc_country* c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; + + c->ctx = loc_ref(ctx); + c->refcount = 1; + + c->code = strdup(country_code); + + DEBUG(c->ctx, "Country %s allocated at %p\n", c->code, c); + *country = c; + + return 0; +} + +LOC_EXPORT struct loc_country* loc_country_ref(struct loc_country* country) { + country->refcount++; + + return country; +} + +static void loc_country_free(struct loc_country* country) { + DEBUG(country->ctx, "Releasing country %s %p\n", country->code, country); + + if (country->code) + free(country->code); + + if (country->continent_code) + free(country->continent_code); + + if (country->name) + free(country->name); + + loc_unref(country->ctx); + free(country); +} + +LOC_EXPORT struct loc_country* loc_country_unref(struct loc_country* country) { + if (--country->refcount > 0) + return NULL; + + loc_country_free(country); + + return NULL; +} + +LOC_EXPORT const char* loc_country_get_code(struct loc_country* country) { + return country->code; +} + +LOC_EXPORT const char* loc_country_get_continent_code(struct loc_country* country) { + return country->continent_code; +} + +LOC_EXPORT int loc_country_set_continent_code(struct loc_country* country, const char* continent_code) { + // XXX validate input + + // Free previous value + if (country->continent_code) + free(country->continent_code); + + country->continent_code = strdup(continent_code); + + return 0; +} + +LOC_EXPORT const char* loc_country_get_name(struct loc_country* country) { + return country->name; +} + +LOC_EXPORT int loc_country_set_name(struct loc_country* country, const char* name) { + if (country->name) + free(country->name); + + if (name) + country->name = strdup(name); + + return 0; +} + +LOC_EXPORT int loc_country_cmp(struct loc_country* country1, struct loc_country* country2) { + return strcmp(country1->code, country2->code); +} + +int loc_country_new_from_database_v1(struct loc_ctx* ctx, struct loc_stringpool* pool, + struct loc_country** country, const struct loc_database_country_v1* dbobj) { + char buffer[3]; + + // Read country code + loc_country_code_copy(buffer, dbobj->code); + + // Terminate buffer + buffer[2] = '\0'; + + // Create a new country object + int r = loc_country_new(ctx, country, buffer); + if (r) + return r; + + // Continent Code + loc_country_code_copy(buffer, dbobj->continent_code); + + r = loc_country_set_continent_code(*country, buffer); + if (r) + goto FAIL; + + // Set name + const char* name = loc_stringpool_get(pool, be32toh(dbobj->name)); + if (name) { + r = loc_country_set_name(*country, name); + if (r) + goto FAIL; + } + + return 0; + +FAIL: + loc_country_unref(*country); + return r; +} + +int loc_country_to_database_v1(struct loc_country* country, + struct loc_stringpool* pool, struct loc_database_country_v1* dbobj) { + // Add country code + for (unsigned int i = 0; i < 2; i++) { + dbobj->code[i] = country->code[i] ? country->code[i] : '\0'; + } + + // Add continent code + if (country->continent_code) { + for (unsigned int i = 0; i < 2; i++) { + dbobj->continent_code[i] = country->continent_code[i] ? country->continent_code[i] : '\0'; + } + } + + // Save the name string in the string pool + off_t name = loc_stringpool_add(pool, country->name ? country->name : ""); + dbobj->name = htobe32(name); + + return 0; +} + +LOC_EXPORT int loc_country_code_is_valid(const char* cc) { + // It cannot be NULL + if (!cc || !*cc) + return 0; + + // It must be 2 characters long + if (strlen(cc) != 2) + return 0; + + // It must only contain A-Z + for (unsigned int i = 0; i < 2; i++) { + if (cc[i] < 'A' || cc[i] > 'Z') + return 0; + } + + // The code cannot begin with an X (those are reserved for private use) + if (*cc == 'X') + return 0; + + // Looks valid + return 1; +} + +LOC_EXPORT int loc_country_special_code_to_flag(const char* cc) { + // Check if we got some input + if (!cc || !*cc) { + errno = EINVAL; + return -1; + } + + // Return flags for any known special country + for (const struct loc_special_country* country = loc_special_countries; + country->flags; country++) { + if (strcmp(country->code, cc) == 0) + return country->flags; + } + + return 0; +} diff --git a/src/database.c b/src/database.c new file mode 100644 index 0000000..b57407e --- /dev/null +++ b/src/database.c @@ -0,0 +1,1586 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ENDIAN_H +# include +#endif + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct loc_database { + struct loc_ctx* ctx; + int refcount; + + FILE* f; + + enum loc_database_version version; + time_t created_at; + off_t vendor; + off_t description; + off_t license; + + // Signatures + char* signature1; + size_t signature1_length; + char* signature2; + size_t signature2_length; + + // ASes in the database + struct loc_database_as_v1* as_v1; + size_t as_count; + + // Network tree + struct loc_database_network_node_v1* network_nodes_v1; + size_t network_nodes_count; + + // Networks + struct loc_database_network_v1* networks_v1; + size_t networks_count; + + // Countries + struct loc_database_country_v1* countries_v1; + size_t countries_count; + + struct loc_stringpool* pool; +}; + +#define MAX_STACK_DEPTH 256 + +struct loc_node_stack { + off_t offset; + int i; // Is this node 0 or 1? + int depth; +}; + +struct loc_database_enumerator { + struct loc_ctx* ctx; + struct loc_database* db; + enum loc_database_enumerator_mode mode; + int refcount; + + // Search string + char* string; + struct loc_country_list* countries; + struct loc_as_list* asns; + enum loc_network_flags flags; + int family; + + // Flatten output? + int flatten; + + // Index of the AS we are looking at + unsigned int as_index; + + // Index of the country we are looking at + unsigned int country_index; + + // Network state + struct in6_addr network_address; + struct loc_node_stack network_stack[MAX_STACK_DEPTH]; + int network_stack_depth; + unsigned int* networks_visited; + + // For subnet search and bogons + struct loc_network_list* stack; + struct loc_network_list* subnets; + + // For bogons + struct in6_addr gap6_start; + struct in6_addr gap4_start; +}; + +static int loc_database_read_magic(struct loc_database* db) { + struct loc_database_magic magic; + + // Read from file + size_t bytes_read = fread(&magic, 1, sizeof(magic), db->f); + + // Check if we have been able to read enough data + if (bytes_read < sizeof(magic)) { + ERROR(db->ctx, "Could not read enough data to validate magic bytes\n"); + DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic)); + return -ENOMSG; + } + + // Compare magic bytes + if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) { + DEBUG(db->ctx, "Magic value matches\n"); + + // Parse version + db->version = magic.version; + + return 0; + } + + ERROR(db->ctx, "Unrecognized file type\n"); + + // Return an error + return 1; +} + +static int loc_database_read_as_section_v1(struct loc_database* db, + const struct loc_database_header_v1* header) { + off_t as_offset = be32toh(header->as_offset); + size_t as_length = be32toh(header->as_length); + + DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", (intmax_t)as_offset, as_length); + + if (as_length > 0) { + db->as_v1 = mmap(NULL, as_length, PROT_READ, + MAP_SHARED, fileno(db->f), as_offset); + + if (db->as_v1 == MAP_FAILED) + return -errno; + } + + db->as_count = as_length / sizeof(*db->as_v1); + + INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count); + + return 0; +} + +static int loc_database_read_network_nodes_section_v1(struct loc_database* db, + const struct loc_database_header_v1* header) { + off_t network_nodes_offset = be32toh(header->network_tree_offset); + size_t network_nodes_length = be32toh(header->network_tree_length); + + DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n", + (intmax_t)network_nodes_offset, network_nodes_length); + + if (network_nodes_length > 0) { + db->network_nodes_v1 = mmap(NULL, network_nodes_length, PROT_READ, + MAP_SHARED, fileno(db->f), network_nodes_offset); + + if (db->network_nodes_v1 == MAP_FAILED) + return -errno; + } + + db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v1); + + INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count); + + return 0; +} + +static int loc_database_read_networks_section_v1(struct loc_database* db, + const struct loc_database_header_v1* header) { + off_t networks_offset = be32toh(header->network_data_offset); + size_t networks_length = be32toh(header->network_data_length); + + DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n", + (intmax_t)networks_offset, networks_length); + + if (networks_length > 0) { + db->networks_v1 = mmap(NULL, networks_length, PROT_READ, + MAP_SHARED, fileno(db->f), networks_offset); + + if (db->networks_v1 == MAP_FAILED) + return -errno; + } + + db->networks_count = networks_length / sizeof(*db->networks_v1); + + INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count); + + return 0; +} + +static int loc_database_read_countries_section_v1(struct loc_database* db, + const struct loc_database_header_v1* header) { + off_t countries_offset = be32toh(header->countries_offset); + size_t countries_length = be32toh(header->countries_length); + + DEBUG(db->ctx, "Reading countries section from %jd (%zu bytes)\n", + (intmax_t)countries_offset, countries_length); + + if (countries_length > 0) { + db->countries_v1 = mmap(NULL, countries_length, PROT_READ, + MAP_SHARED, fileno(db->f), countries_offset); + + if (db->countries_v1 == MAP_FAILED) + return -errno; + } + + db->countries_count = countries_length / sizeof(*db->countries_v1); + + INFO(db->ctx, "Read %zu countries from the database\n", + db->countries_count); + + return 0; +} + +static int loc_database_read_signature(struct loc_database* db, + char** dst, char* src, size_t length) { + // Check for a plausible signature length + if (length > LOC_SIGNATURE_MAX_LENGTH) { + ERROR(db->ctx, "Signature too long: %zu\n", length); + return -EINVAL; + } + + DEBUG(db->ctx, "Reading signature of %zu bytes\n", length); + + // Allocate space + *dst = malloc(length); + if (!*dst) + return -ENOMEM; + + // Copy payload + memcpy(*dst, src, length); + + return 0; +} + +static int loc_database_read_header_v1(struct loc_database* db) { + struct loc_database_header_v1 header; + int r; + + // Read from file + size_t size = fread(&header, 1, sizeof(header), db->f); + + if (size < sizeof(header)) { + ERROR(db->ctx, "Could not read enough data for header\n"); + return -ENOMSG; + } + + // Copy over data + db->created_at = be64toh(header.created_at); + db->vendor = be32toh(header.vendor); + db->description = be32toh(header.description); + db->license = be32toh(header.license); + + db->signature1_length = be16toh(header.signature1_length); + db->signature2_length = be16toh(header.signature2_length); + + // Read signatures + if (db->signature1_length) { + r = loc_database_read_signature(db, &db->signature1, + header.signature1, db->signature1_length); + if (r) + return r; + } + + if (db->signature2_length) { + r = loc_database_read_signature(db, &db->signature2, + header.signature2, db->signature2_length); + if (r) + return r; + } + + // Open pool + off_t pool_offset = be32toh(header.pool_offset); + size_t pool_length = be32toh(header.pool_length); + + r = loc_stringpool_open(db->ctx, &db->pool, + db->f, pool_length, pool_offset); + if (r) + return r; + + // AS section + r = loc_database_read_as_section_v1(db, &header); + if (r) + return r; + + // Network Nodes + r = loc_database_read_network_nodes_section_v1(db, &header); + if (r) + return r; + + // Networks + r = loc_database_read_networks_section_v1(db, &header); + if (r) + return r; + + // countries + r = loc_database_read_countries_section_v1(db, &header); + if (r) + return r; + + return 0; +} + +static int loc_database_read_header(struct loc_database* db) { + DEBUG(db->ctx, "Database version is %u\n", db->version); + + switch (db->version) { + case LOC_DATABASE_VERSION_1: + return loc_database_read_header_v1(db); + + default: + ERROR(db->ctx, "Incompatible database version: %u\n", db->version); + return 1; + } +} + +static int loc_database_read(struct loc_database* db, FILE* f) { + clock_t start = clock(); + + int fd = fileno(f); + + // Clone file descriptor + fd = dup(fd); + if (!fd) { + ERROR(db->ctx, "Could not duplicate file descriptor\n"); + return -1; + } + + // Reopen the file so that we can keep our own file handle + db->f = fdopen(fd, "r"); + if (!db->f) { + ERROR(db->ctx, "Could not re-open database file\n"); + return -1; + } + + // Rewind to the start of the file + rewind(db->f); + + // Read magic bytes + int r = loc_database_read_magic(db); + if (r) + return r; + + // Read the header + r = loc_database_read_header(db); + if (r) + return r; + + clock_t end = clock(); + + INFO(db->ctx, "Opened database in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000); + + return 0; +} + +LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) { + // Fail on invalid file handle + if (!f) + return -EINVAL; + + struct loc_database* db = calloc(1, sizeof(*db)); + if (!db) + return -ENOMEM; + + // Reference context + db->ctx = loc_ref(ctx); + db->refcount = 1; + + DEBUG(db->ctx, "Database object allocated at %p\n", db); + + int r = loc_database_read(db, f); + if (r) { + loc_database_unref(db); + return r; + } + + *database = db; + + return 0; +} + +LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) { + db->refcount++; + + return db; +} + +static void loc_database_free(struct loc_database* db) { + int r; + + DEBUG(db->ctx, "Releasing database %p\n", db); + + // Removing all ASes + if (db->as_v1) { + r = munmap(db->as_v1, db->as_count * sizeof(*db->as_v1)); + if (r) + ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno)); + } + + // Remove mapped network sections + if (db->networks_v1) { + r = munmap(db->networks_v1, db->networks_count * sizeof(*db->networks_v1)); + if (r) + ERROR(db->ctx, "Could not unmap networks section: %s\n", strerror(errno)); + } + + // Remove mapped network nodes section + if (db->network_nodes_v1) { + r = munmap(db->network_nodes_v1, db->network_nodes_count * sizeof(*db->network_nodes_v1)); + if (r) + ERROR(db->ctx, "Could not unmap network nodes section: %s\n", strerror(errno)); + } + + // Remove mapped countries section + if (db->countries_v1) { + r = munmap(db->countries_v1, db->countries_count * sizeof(*db->countries_v1)); + if (r) + ERROR(db->ctx, "Could not unmap countries section: %s\n", strerror(errno)); + } + + if (db->pool) + loc_stringpool_unref(db->pool); + + // Free signature + if (db->signature1) + free(db->signature1); + if (db->signature2) + free(db->signature2); + + // Close database file + if (db->f) + fclose(db->f); + + loc_unref(db->ctx); + free(db); +} + +LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) { + if (--db->refcount > 0) + return NULL; + + loc_database_free(db); + return NULL; +} + +LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { + // Cannot do this when no signature is available + if (!db->signature1 && !db->signature2) { + DEBUG(db->ctx, "No signature available to verify\n"); + return 1; + } + + // Start the stopwatch + clock_t start = clock(); + + // Load public key + EVP_PKEY* pkey = PEM_read_PUBKEY(f, NULL, NULL, NULL); + if (!pkey) { + char* error = ERR_error_string(ERR_get_error(), NULL); + ERROR(db->ctx, "Could not parse public key: %s\n", error); + + return -1; + } + + int r = 0; + + EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); + + // Initialise hash function + r = EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey); + if (r != 1) { + ERROR(db->ctx, "Error initializing signature validation: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + r = 1; + + goto CLEANUP; + } + + // Reset file to start + rewind(db->f); + + // Read magic + struct loc_database_magic magic; + fread(&magic, 1, sizeof(magic), db->f); + + hexdump(db->ctx, &magic, sizeof(magic)); + + // Feed magic into the hash + r = EVP_DigestVerifyUpdate(mdctx, &magic, sizeof(magic)); + if (r != 1) { + ERROR(db->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + r = 1; + + goto CLEANUP; + } + + // Read the header + struct loc_database_header_v1 header_v1; + size_t bytes_read; + + switch (db->version) { + case LOC_DATABASE_VERSION_1: + bytes_read = fread(&header_v1, 1, sizeof(header_v1), db->f); + if (bytes_read < sizeof(header_v1)) { + ERROR(db->ctx, "Could not read header\n"); + r = 1; + + goto CLEANUP; + } + + // Clear signatures + memset(header_v1.signature1, '\0', sizeof(header_v1.signature1)); + header_v1.signature1_length = 0; + memset(header_v1.signature2, '\0', sizeof(header_v1.signature2)); + header_v1.signature2_length = 0; + + hexdump(db->ctx, &header_v1, sizeof(header_v1)); + + // Feed header into the hash + r = EVP_DigestVerifyUpdate(mdctx, &header_v1, sizeof(header_v1)); + if (r != 1) { + ERROR(db->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + r = 1; + + goto CLEANUP; + } + break; + + default: + ERROR(db->ctx, "Cannot compute hash for database with format %d\n", + db->version); + r = -EINVAL; + goto CLEANUP; + } + + // Walk through the file in chunks of 64kB + char buffer[64 * 1024]; + + while (!feof(db->f)) { + bytes_read = fread(buffer, 1, sizeof(buffer), db->f); + + hexdump(db->ctx, buffer, bytes_read); + + r = EVP_DigestVerifyUpdate(mdctx, buffer, bytes_read); + if (r != 1) { + ERROR(db->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + r = 1; + + goto CLEANUP; + } + } + + // Check first signature + if (db->signature1) { + hexdump(db->ctx, db->signature1, db->signature1_length); + + r = EVP_DigestVerifyFinal(mdctx, + (unsigned char*)db->signature1, db->signature1_length); + + if (r == 0) { + DEBUG(db->ctx, "The first signature is invalid\n"); + r = 1; + } else if (r == 1) { + DEBUG(db->ctx, "The first signature is valid\n"); + r = 0; + } else { + ERROR(db->ctx, "Error verifying the first signature: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + r = -1; + } + } + + // Check second signature only when the first one was invalid + if (r && db->signature2) { + hexdump(db->ctx, db->signature2, db->signature2_length); + + r = EVP_DigestVerifyFinal(mdctx, + (unsigned char*)db->signature2, db->signature2_length); + + if (r == 0) { + DEBUG(db->ctx, "The second signature is invalid\n"); + r = 1; + } else if (r == 1) { + DEBUG(db->ctx, "The second signature is valid\n"); + r = 0; + } else { + ERROR(db->ctx, "Error verifying the second signature: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + r = -1; + } + } + + clock_t end = clock(); + INFO(db->ctx, "Signature checked in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000); + +CLEANUP: + // Cleanup + EVP_MD_CTX_free(mdctx); + EVP_PKEY_free(pkey); + + return r; +} + +LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) { + return db->created_at; +} + +LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) { + return loc_stringpool_get(db->pool, db->vendor); +} + +LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) { + return loc_stringpool_get(db->pool, db->description); +} + +LOC_EXPORT const char* loc_database_get_license(struct loc_database* db) { + return loc_stringpool_get(db->pool, db->license); +} + +LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) { + return db->as_count; +} + +// Returns the AS at position pos +static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) { + if ((size_t)pos >= db->as_count) + return -EINVAL; + + DEBUG(db->ctx, "Fetching AS at position %jd\n", (intmax_t)pos); + + int r; + switch (db->version) { + case LOC_DATABASE_VERSION_1: + r = loc_as_new_from_database_v1(db->ctx, db->pool, as, db->as_v1 + pos); + break; + + default: + return -1; + } + + if (r == 0) { + DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as)); + } + + return r; +} + +// Performs a binary search to find the AS in the list +LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) { + off_t lo = 0; + off_t hi = db->as_count - 1; + +#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); +#endif + + while (lo <= hi) { + off_t i = (lo + hi) / 2; + + // Fetch AS in the middle between lo and hi + int r = loc_database_fetch_as(db, as, i); + if (r) + return r; + + // Check if this is a match + uint32_t as_number = loc_as_get_number(*as); + if (as_number == number) { +#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, + (double)(end - start) / CLOCKS_PER_SEC * 1000); +#endif + + return 0; + } + + // If it wasn't, we release the AS and + // adjust our search pointers + loc_as_unref(*as); + + if (as_number < number) { + lo = i + 1; + } else + hi = i - 1; + } + + // Nothing found + *as = NULL; + + return 1; +} + +// Returns the network at position pos +static int loc_database_fetch_network(struct loc_database* db, struct loc_network** network, + struct in6_addr* address, unsigned int prefix, off_t pos) { + if ((size_t)pos >= db->networks_count) { + DEBUG(db->ctx, "Network ID out of range: %jd/%jd\n", + (intmax_t)pos, (intmax_t)db->networks_count); + return -EINVAL; + } + + + DEBUG(db->ctx, "Fetching network at position %jd\n", (intmax_t)pos); + + int r; + switch (db->version) { + case LOC_DATABASE_VERSION_1: + r = loc_network_new_from_database_v1(db->ctx, network, + address, prefix, db->networks_v1 + pos); + break; + + default: + return -1; + } + + if (r == 0) + DEBUG(db->ctx, "Got network %s\n", loc_network_str(*network)); + + return r; +} + +static int __loc_database_node_is_leaf(const struct loc_database_network_node_v1* node) { + return (node->network != htobe32(0xffffffff)); +} + +static int __loc_database_lookup_handle_leaf(struct loc_database* db, const struct in6_addr* address, + struct loc_network** network, struct in6_addr* network_address, unsigned int prefix, + const struct loc_database_network_node_v1* node) { + off_t network_index = be32toh(node->network); + + DEBUG(db->ctx, "Handling leaf node at %jd (%jd)\n", (intmax_t)(node - db->network_nodes_v1), (intmax_t)network_index); + + // Fetch the network + int r = loc_database_fetch_network(db, network, + network_address, prefix, network_index); + if (r) { + ERROR(db->ctx, "Could not fetch network %jd from database\n", (intmax_t)network_index); + return r; + } + + // Check if the given IP address is inside the network + if (!loc_network_matches_address(*network, address)) { + DEBUG(db->ctx, "Searched address is not part of the network\n"); + + loc_network_unref(*network); + *network = NULL; + return 1; + } + + // A network was found and the IP address matches + return 0; +} + +// Searches for an exact match along the path +static int __loc_database_lookup(struct loc_database* db, const struct in6_addr* address, + struct loc_network** network, struct in6_addr* network_address, + const struct loc_database_network_node_v1* node, unsigned int level) { + int r; + off_t node_index; + + // Follow the path + int bit = loc_address_get_bit(address, level); + loc_address_set_bit(network_address, level, bit); + + if (bit == 0) + node_index = be32toh(node->zero); + else + node_index = be32toh(node->one); + + // If the node index is zero, the tree ends here + // and we cannot descend any further + if (node_index > 0) { + // Check boundaries + if ((size_t)node_index >= db->network_nodes_count) + return -EINVAL; + + // Move on to the next node + r = __loc_database_lookup(db, address, network, network_address, + db->network_nodes_v1 + node_index, level + 1); + + // End here if a result was found + if (r == 0) + return r; + + // Raise any errors + else if (r < 0) + return r; + + DEBUG(db->ctx, "No match found below level %u\n", level); + } else { + DEBUG(db->ctx, "Tree ended at level %u\n", level); + } + + // If this node has a leaf, we will check if it matches + if (__loc_database_node_is_leaf(node)) { + r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node); + if (r <= 0) + return r; + } + + return 1; +} + +LOC_EXPORT int loc_database_lookup(struct loc_database* db, + const struct in6_addr* address, struct loc_network** network) { + struct in6_addr network_address; + memset(&network_address, 0, sizeof(network_address)); + + *network = NULL; + +#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); +#endif + + int r = __loc_database_lookup(db, address, network, &network_address, + db->network_nodes_v1, 0); + +#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(db->ctx, "Executed network search in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000); +#endif + + return r; +} + +LOC_EXPORT int loc_database_lookup_from_string(struct loc_database* db, + const char* string, struct loc_network** network) { + struct in6_addr address; + + int r = loc_address_parse(&address, NULL, string); + if (r) + return r; + + return loc_database_lookup(db, &address, network); +} + +// Returns the country at position pos +static int loc_database_fetch_country(struct loc_database* db, + struct loc_country** country, off_t pos) { + if ((size_t)pos >= db->countries_count) + return -EINVAL; + + DEBUG(db->ctx, "Fetching country at position %jd\n", (intmax_t)pos); + + int r; + switch (db->version) { + case LOC_DATABASE_VERSION_1: + r = loc_country_new_from_database_v1(db->ctx, db->pool, country, db->countries_v1 + pos); + break; + + default: + return -1; + } + + if (r == 0) { + DEBUG(db->ctx, "Got country %s\n", loc_country_get_code(*country)); + } + + return r; +} + +// Performs a binary search to find the country in the list +LOC_EXPORT int loc_database_get_country(struct loc_database* db, + struct loc_country** country, const char* code) { + off_t lo = 0; + off_t hi = db->countries_count - 1; + +#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); +#endif + + while (lo <= hi) { + off_t i = (lo + hi) / 2; + + // Fetch country in the middle between lo and hi + int r = loc_database_fetch_country(db, country, i); + if (r) + return r; + + // Check if this is a match + const char* cc = loc_country_get_code(*country); + int result = strcmp(code, cc); + + if (result == 0) { +#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, + (double)(end - start) / CLOCKS_PER_SEC * 1000); +#endif + + return 0; + } + + // If it wasn't, we release the country and + // adjust our search pointers + loc_country_unref(*country); + + if (result > 0) { + lo = i + 1; + } else + hi = i - 1; + } + + // Nothing found + *country = NULL; + + return 1; +} + +// Enumerator + +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); + + // Release all references + loc_database_unref(enumerator->db); + loc_unref(enumerator->ctx); + + if (enumerator->string) + free(enumerator->string); + + if (enumerator->countries) + loc_country_list_unref(enumerator->countries); + + if (enumerator->asns) + loc_as_list_unref(enumerator->asns); + + // Free network search + free(enumerator->networks_visited); + + // Free subnet/bogons stack + if (enumerator->stack) + loc_network_list_unref(enumerator->stack); + + if (enumerator->subnets) + loc_network_list_unref(enumerator->subnets); + + free(enumerator); +} + +LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); + if (!e) + return -ENOMEM; + + // Reference context + e->ctx = loc_ref(db->ctx); + e->db = loc_database_ref(db); + e->mode = mode; + e->refcount = 1; + + // Flatten output? + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + + // Initialise graph search + e->network_stack_depth = 1; + e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); + + // Allocate stack + int r = loc_network_list_new(e->ctx, &e->stack); + if (r) { + loc_database_enumerator_free(e); + return r; + } + + // Initialize bogon search + loc_address_reset(&e->gap6_start, AF_INET6); + loc_address_reset(&e->gap4_start, AF_INET); + + DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); + + *enumerator = e; + return 0; +} + +LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator) { + enumerator->refcount++; + + return enumerator; +} + +LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { + if (!enumerator) + return NULL; + + if (--enumerator->refcount > 0) + return enumerator; + + loc_database_enumerator_free(enumerator); + return NULL; +} + +LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string) { + enumerator->string = strdup(string); + + // Make the string lowercase + for (char *p = enumerator->string; *p; p++) + *p = tolower(*p); + + return 0; +} + +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( + struct loc_database_enumerator* enumerator) { + if (!enumerator->countries) + return NULL; + + return loc_country_list_ref(enumerator->countries); +} + +LOC_EXPORT int loc_database_enumerator_set_countries( + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { + if (enumerator->countries) + loc_country_list_unref(enumerator->countries); + + enumerator->countries = loc_country_list_ref(countries); + + return 0; +} + +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( + struct loc_database_enumerator* enumerator) { + if (!enumerator->asns) + return NULL; + + return loc_as_list_ref(enumerator->asns); +} + +LOC_EXPORT int loc_database_enumerator_set_asns( + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { + if (enumerator->asns) + loc_as_list_unref(enumerator->asns); + + enumerator->asns = loc_as_list_ref(asns); + + return 0; +} + +LOC_EXPORT int loc_database_enumerator_set_flag( + struct loc_database_enumerator* enumerator, enum loc_network_flags flag) { + enumerator->flags |= flag; + + return 0; +} + +LOC_EXPORT int loc_database_enumerator_set_family( + struct loc_database_enumerator* enumerator, int family) { + enumerator->family = family; + + return 0; +} + +LOC_EXPORT int loc_database_enumerator_next_as( + struct loc_database_enumerator* enumerator, struct loc_as** as) { + *as = NULL; + + // Do not do anything if not in AS mode + if (enumerator->mode != LOC_DB_ENUMERATE_ASES) + return 0; + + struct loc_database* db = enumerator->db; + + while (enumerator->as_index < db->as_count) { + // Fetch the next AS + int r = loc_database_fetch_as(db, as, enumerator->as_index++); + if (r) + return r; + + r = loc_as_match_string(*as, enumerator->string); + if (r == 1) { + DEBUG(enumerator->ctx, "AS%d (%s) matches %s\n", + loc_as_get_number(*as), loc_as_get_name(*as), enumerator->string); + + return 0; + } + + // No match + loc_as_unref(*as); + *as = NULL; + } + + // Reset the index + enumerator->as_index = 0; + + // We have searched through all of them + return 0; +} + +static int loc_database_enumerator_stack_push_node( + struct loc_database_enumerator* e, off_t offset, int i, int depth) { + // Do not add empty nodes + if (!offset) + return 0; + + // Check if there is any space left on the stack + if (e->network_stack_depth >= MAX_STACK_DEPTH) { + ERROR(e->ctx, "Maximum stack size reached: %d\n", e->network_stack_depth); + return -1; + } + + // Increase stack size + int s = ++e->network_stack_depth; + + DEBUG(e->ctx, "Added node %jd to stack (%d)\n", (intmax_t)offset, depth); + + e->network_stack[s].offset = offset; + e->network_stack[s].i = i; + e->network_stack[s].depth = depth; + + return 0; +} + +static int loc_database_enumerator_match_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // If family is set, it must match + if (enumerator->family && loc_network_address_family(network) != enumerator->family) { + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); + return 0; + } + + // Match if no filter criteria is configured + if (!enumerator->countries && !enumerator->asns && !enumerator->flags) + return 1; + + // Check if the country code matches + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { + const char* country_code = loc_network_get_country_code(network); + + if (loc_country_list_contains_code(enumerator->countries, country_code)) { + DEBUG(enumerator->ctx, "Matched network %p because of its country code\n", network); + return 1; + } + } + + // Check if the ASN matches + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { + uint32_t asn = loc_network_get_asn(network); + + if (loc_as_list_contains_number(enumerator->asns, asn)) { + DEBUG(enumerator->ctx, "Matched network %p because of its ASN\n", network); + return 1; + } + } + + // Check if flags match + if (enumerator->flags && loc_network_has_flag(network, enumerator->flags)) { + DEBUG(enumerator->ctx, "Matched network %p because of its flags\n", network); + return 1; + } + + // Not a match + return 0; +} + +static int __loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { + // Return top element from the stack + while (1) { + *network = loc_network_list_pop_first(enumerator->stack); + + // Stack is empty + if (!*network) + break; + + // Return everything if filter isn't enabled, or only return matches + if (!filter || loc_database_enumerator_match_network(enumerator, *network)) + return 0; + + // Throw away anything that doesn't match + loc_network_unref(*network); + *network = NULL; + } + + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); + + // Perform DFS + while (enumerator->network_stack_depth > 0) { + DEBUG(enumerator->ctx, "Stack depth: %u\n", enumerator->network_stack_depth); + + // Get object from top of the stack + struct loc_node_stack* node = &enumerator->network_stack[enumerator->network_stack_depth]; + + // Remove the node from the stack if we have already visited it + if (enumerator->networks_visited[node->offset]) { + enumerator->network_stack_depth--; + continue; + } + + // Mark the bits on the path correctly + loc_address_set_bit(&enumerator->network_address, + (node->depth > 0) ? node->depth - 1 : 0, node->i); + + DEBUG(enumerator->ctx, "Looking at node %jd\n", (intmax_t)node->offset); + enumerator->networks_visited[node->offset]++; + + // Pop node from top of the stack + struct loc_database_network_node_v1* n = + enumerator->db->network_nodes_v1 + node->offset; + + // Add edges to stack + int r = loc_database_enumerator_stack_push_node(enumerator, + be32toh(n->one), 1, node->depth + 1); + + if (r) + return r; + + r = loc_database_enumerator_stack_push_node(enumerator, + be32toh(n->zero), 0, node->depth + 1); + + if (r) + return r; + + // Check if this node is a leaf and has a network object + if (__loc_database_node_is_leaf(n)) { + off_t network_index = be32toh(n->network); + + DEBUG(enumerator->ctx, "Node has a network at %jd\n", (intmax_t)network_index); + + // Fetch the network object + r = loc_database_fetch_network(enumerator->db, network, + &enumerator->network_address, node->depth, network_index); + + // Break on any errors + if (r) + return r; + + // Return all networks when the filter is disabled, or check for match + if (!filter || loc_database_enumerator_match_network(enumerator, *network)) + return 0; + + // Does not seem to be a match, so we cleanup and move on + loc_network_unref(*network); + *network = NULL; + } + } + + // Reached the end of the search + return 0; +} + +static int __loc_database_enumerator_next_network_flattened( + struct loc_database_enumerator* enumerator, struct loc_network** network) { + // Fetch the next network + int r = __loc_database_enumerator_next_network(enumerator, network, 1); + if (r) + return r; + + // End if we could not read another network + if (!*network) + return 0; + + struct loc_network* subnet = NULL; + + // Create a list with all subnets + if (!enumerator->subnets) { + r = loc_network_list_new(enumerator->ctx, &enumerator->subnets); + if (r) + return r; + } + + // Search all subnets from the database + while (1) { + // Fetch the next network in line + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); + if (r) { + loc_network_unref(subnet); + loc_network_list_clear(enumerator->subnets); + + return r; + } + + // End if we did not receive another subnet + if (!subnet) + break; + + // Collect all subnets in a list + if (loc_network_is_subnet(*network, subnet)) { + r = loc_network_list_push(enumerator->subnets, subnet); + if (r) { + loc_network_unref(subnet); + loc_network_list_clear(enumerator->subnets); + + return r; + } + + loc_network_unref(subnet); + continue; + } + + // If this is not a subnet, we push it back onto the stack and break + r = loc_network_list_push(enumerator->stack, subnet); + if (r) { + loc_network_unref(subnet); + loc_network_list_clear(enumerator->subnets); + + return r; + } + + loc_network_unref(subnet); + break; + } + + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", + loc_network_list_size(enumerator->subnets)); + + // We can abort here if the network has no subnets + if (loc_network_list_empty(enumerator->subnets)) { + loc_network_list_clear(enumerator->subnets); + + return 0; + } + + // If the network has any subnets, we will break it into smaller parts + // without the subnets. + struct loc_network_list* excluded = loc_network_exclude_list(*network, enumerator->subnets); + if (!excluded) { + loc_network_list_clear(enumerator->subnets); + return 1; + } + + // Merge subnets onto the stack + r = loc_network_list_merge(enumerator->stack, enumerator->subnets); + if (r) { + loc_network_list_clear(enumerator->subnets); + loc_network_list_unref(excluded); + + return r; + } + + // Push excluded list onto the stack + r = loc_network_list_merge(enumerator->stack, excluded); + if (r) { + loc_network_list_clear(enumerator->subnets); + loc_network_list_unref(excluded); + + return r; + } + + loc_network_list_clear(enumerator->subnets); + loc_network_list_unref(excluded); + + // Drop the network and restart the whole process again to pick the next network + loc_network_unref(*network); + + return __loc_database_enumerator_next_network_flattened(enumerator, network); +} + +/* + This function finds all bogons (i.e. gaps) between the input networks +*/ +static int __loc_database_enumerator_next_bogon( + struct loc_database_enumerator* enumerator, struct loc_network** bogon) { + int r; + + // Return top element from the stack + while (1) { + *bogon = loc_network_list_pop_first(enumerator->stack); + + // Stack is empty + if (!*bogon) + break; + + // Return result + return 0; + } + + struct loc_network* network = NULL; + struct in6_addr* gap_start = NULL; + struct in6_addr gap_end = IN6ADDR_ANY_INIT; + + while (1) { + r = __loc_database_enumerator_next_network(enumerator, &network, 1); + if (r) + return r; + + // We have read the last network + if (!network) + goto FINISH; + + const char* country_code = loc_network_get_country_code(network); + + /* + Skip anything that does not have a country code + + Even if a network is part of the routing table, and the database provides + an ASN, this does not mean that this is a legitimate announcement. + */ + if (country_code && !*country_code) { + loc_network_unref(network); + continue; + } + + // Determine the network family + int family = loc_network_address_family(network); + + switch (family) { + case AF_INET6: + gap_start = &enumerator->gap6_start; + break; + + case AF_INET: + gap_start = &enumerator->gap4_start; + break; + + default: + ERROR(enumerator->ctx, "Unsupported network family %d\n", family); + errno = ENOTSUP; + return 1; + } + + const struct in6_addr* first_address = loc_network_get_first_address(network); + const struct in6_addr* last_address = loc_network_get_last_address(network); + + // Skip if this network is a subnet of a former one + if (loc_address_cmp(gap_start, last_address) >= 0) { + loc_network_unref(network); + continue; + } + + // Search where the gap could end + gap_end = *first_address; + loc_address_decrement(&gap_end); + + // There is a gap + if (loc_address_cmp(gap_start, &gap_end) <= 0) { + r = loc_network_list_summarize(enumerator->ctx, + gap_start, &gap_end, &enumerator->stack); + if (r) { + loc_network_unref(network); + return r; + } + } + + // The gap now starts after this network + *gap_start = *last_address; + loc_address_increment(gap_start); + + loc_network_unref(network); + + // Try to return something + *bogon = loc_network_list_pop_first(enumerator->stack); + if (*bogon) + break; + } + + return 0; + +FINISH: + + if (!loc_address_all_zeroes(&enumerator->gap6_start)) { + r = loc_address_reset_last(&gap_end, AF_INET6); + if (r) + return r; + + if (loc_address_cmp(&enumerator->gap6_start, &gap_end) <= 0) { + r = loc_network_list_summarize(enumerator->ctx, + &enumerator->gap6_start, &gap_end, &enumerator->stack); + if (r) + return r; + } + + // Reset start + loc_address_reset(&enumerator->gap6_start, AF_INET6); + } + + if (!loc_address_all_zeroes(&enumerator->gap4_start)) { + r = loc_address_reset_last(&gap_end, AF_INET); + if (r) + return r; + + if (loc_address_cmp(&enumerator->gap4_start, &gap_end) <= 0) { + r = loc_network_list_summarize(enumerator->ctx, + &enumerator->gap4_start, &gap_end, &enumerator->stack); + if (r) + return r; + } + + // Reset start + loc_address_reset(&enumerator->gap4_start, AF_INET); + } + + // Try to return something + *bogon = loc_network_list_pop_first(enumerator->stack); + + return 0; +} + +LOC_EXPORT int loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network) { + switch (enumerator->mode) { + case LOC_DB_ENUMERATE_NETWORKS: + // Flatten output? + if (enumerator->flatten) + return __loc_database_enumerator_next_network_flattened(enumerator, network); + + return __loc_database_enumerator_next_network(enumerator, network, 1); + + case LOC_DB_ENUMERATE_BOGONS: + return __loc_database_enumerator_next_bogon(enumerator, network); + + default: + return 0; + } +} + +LOC_EXPORT int loc_database_enumerator_next_country( + struct loc_database_enumerator* enumerator, struct loc_country** country) { + *country = NULL; + + // Do not do anything if not in country mode + if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES) + return 0; + + struct loc_database* db = enumerator->db; + + while (enumerator->country_index < db->countries_count) { + // Fetch the next country + int r = loc_database_fetch_country(db, country, enumerator->country_index++); + if (r) + return r; + + // We do not filter here, so it always is a match + return 0; + } + + // Reset the index + enumerator->country_index = 0; + + // We have searched through all of them + return 0; +} diff --git a/src/libloc.c b/src/libloc.c new file mode 100644 index 0000000..cf2d740 --- /dev/null +++ b/src/libloc.c @@ -0,0 +1,132 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct loc_ctx { + int refcount; + void (*log_fn)(struct loc_ctx* ctx, + int priority, const char *file, int line, const char *fn, + const char *format, va_list args); + int log_priority; +}; + +void loc_log(struct loc_ctx* ctx, + int priority, const char* file, int line, const char* fn, + const char* format, ...) { + va_list args; + + va_start(args, format); + ctx->log_fn(ctx, priority, file, line, fn, format, args); + va_end(args); +} + +static void log_stderr(struct loc_ctx* ctx, + int priority, const char* file, int line, const char* fn, + const char* format, va_list args) { + fprintf(stderr, "libloc: %s: ", fn); + vfprintf(stderr, format, args); +} + +static int log_priority(const char* priority) { + char *endptr; + + int prio = strtol(priority, &endptr, 10); + + if (endptr[0] == '\0' || isspace(endptr[0])) + return prio; + + if (strncmp(priority, "err", 3) == 0) + return LOG_ERR; + + if (strncmp(priority, "info", 4) == 0) + return LOG_INFO; + + if (strncmp(priority, "debug", 5) == 0) + return LOG_DEBUG; + + return 0; +} + +LOC_EXPORT int loc_new(struct loc_ctx** ctx) { + struct loc_ctx* c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; + + c->refcount = 1; + c->log_fn = log_stderr; + c->log_priority = LOG_ERR; + + const char* env = secure_getenv("LOC_LOG"); + if (env) + loc_set_log_priority(c, log_priority(env)); + + INFO(c, "ctx %p created\n", c); + DEBUG(c, "log_priority=%d\n", c->log_priority); + *ctx = c; + + return 0; +} + +LOC_EXPORT struct loc_ctx* loc_ref(struct loc_ctx* ctx) { + if (!ctx) + return NULL; + + ctx->refcount++; + + return ctx; +} + +LOC_EXPORT struct loc_ctx* loc_unref(struct loc_ctx* ctx) { + if (!ctx) + return NULL; + + if (--ctx->refcount > 0) + return NULL; + + INFO(ctx, "context %p released\n", ctx); + free(ctx); + + return NULL; +} + +LOC_EXPORT void loc_set_log_fn(struct loc_ctx* ctx, + void (*log_fn)(struct loc_ctx* ctx, int priority, const char* file, + int line, const char* fn, const char* format, va_list args)) { + ctx->log_fn = log_fn; + INFO(ctx, "custom logging function %p registered\n", log_fn); +} + +LOC_EXPORT int loc_get_log_priority(struct loc_ctx* ctx) { + return ctx->log_priority; +} + +LOC_EXPORT void loc_set_log_priority(struct loc_ctx* ctx, int priority) { + ctx->log_priority = priority; +} diff --git a/src/libloc.pc.in b/src/libloc.pc.in new file mode 100644 index 0000000..52474c2 --- /dev/null +++ b/src/libloc.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libloc +Description: A library to determine the location of someone on the Internet +Version: @VERSION@ +Libs: -L${libdir} -lloc +Libs.private: +Cflags: -I${includedir} diff --git a/src/libloc.sym b/src/libloc.sym new file mode 100644 index 0000000..29e17f0 --- /dev/null +++ b/src/libloc.sym @@ -0,0 +1,148 @@ +LIBLOC_1 { +global: + loc_ref; + loc_get_log_priority; + loc_set_log_fn; + loc_unref; + loc_set_log_priority; + loc_new; + loc_discover_latest_version; + + # AS + loc_as_cmp; + loc_as_get_name; + loc_as_get_number; + loc_as_new; + loc_as_ref; + loc_as_set_name; + loc_as_unref; + + # AS List + loc_as_list_append; + loc_as_list_clear; + loc_as_list_contains; + loc_as_list_contains_number; + loc_as_list_empty; + loc_as_list_get; + loc_as_list_new; + loc_as_list_ref; + loc_as_list_size; + loc_as_list_sort; + loc_as_list_unref; + + # Country + loc_country_cmp; + loc_country_code_is_valid; + loc_country_get_code; + loc_country_get_continent_code; + loc_country_get_name; + loc_country_new; + loc_country_ref; + loc_country_set_continent_code; + loc_country_set_name; + loc_country_special_code_to_flag; + loc_country_unref; + + # Country List + loc_country_list_append; + loc_country_list_clear; + loc_country_list_contains; + loc_country_list_contains_code; + loc_country_list_empty; + loc_country_list_get; + loc_country_list_new; + loc_country_list_ref; + loc_country_list_size; + loc_country_list_sort; + loc_country_list_unref; + + # Database + loc_database_add_as; + loc_database_count_as; + loc_database_created_at; + loc_database_get_as; + loc_database_get_country; + loc_database_get_description; + loc_database_get_license; + loc_database_get_vendor; + loc_database_lookup; + loc_database_lookup_from_string; + loc_database_new; + loc_database_ref; + loc_database_unref; + loc_database_verify; + + # Database Enumerator + loc_database_enumerator_get_asns; + loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; + loc_database_enumerator_set_asns; + loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; + loc_database_enumerator_set_string; + loc_database_enumerator_unref; + + # Network + loc_network_address_family; + loc_network_cmp; + loc_network_exclude; + loc_network_exclude_list; + loc_network_format_first_address; + loc_network_format_last_address; + loc_network_get_asn; + loc_network_get_country_code; + loc_network_get_first_address; + loc_network_get_last_address; + loc_network_has_flag; + loc_network_is_subnet; + loc_network_matches_address; + loc_network_matches_country_code; + loc_network_new; + loc_network_new_from_string; + loc_network_overlaps; + loc_network_prefix; + loc_network_ref; + loc_network_set_asn; + loc_network_set_country_code; + loc_network_set_flag; + loc_network_str; + loc_network_subnets; + loc_network_unref; + + # Network List + loc_network_list_clear; + loc_network_list_contains; + loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; + loc_network_list_merge; + loc_network_list_new; + loc_network_list_pop; + loc_network_list_pop_first; + loc_network_list_push; + loc_network_list_ref; + loc_network_list_size; + loc_network_list_unref; + + # Writer + loc_writer_add_as; + loc_writer_add_country; + loc_writer_add_network; + loc_writer_get_description; + loc_writer_get_license; + loc_writer_get_vendor; + loc_writer_new; + loc_writer_ref; + loc_writer_set_description; + loc_writer_set_license; + loc_writer_set_vendor; + loc_writer_unref; + loc_writer_write; +local: + *; +}; diff --git a/src/libloc/address.h b/src/libloc/address.h new file mode 100644 index 0000000..f7fe333 --- /dev/null +++ b/src/libloc/address.h @@ -0,0 +1,283 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2022 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_ADDRESS_H +#define LIBLOC_ADDRESS_H + +#ifdef LIBLOC_PRIVATE + +#include +#include + +/* + All of these functions are private and for internal use only +*/ + +const char* loc_address_str(const struct in6_addr* address); +int loc_address_parse(struct in6_addr* address, unsigned int* prefix, const char* string); + +static inline int loc_address_family(const struct in6_addr* address) { + if (IN6_IS_ADDR_V4MAPPED(address)) + return AF_INET; + else + return AF_INET6; +} + +static inline unsigned int loc_address_family_bit_length(const int family) { + switch (family) { + case AF_INET6: + return 128; + + case AF_INET: + return 32; + + default: + return 0; + } +} + +/* + Checks whether prefix is valid for the given address +*/ +static inline int loc_address_valid_prefix(const struct in6_addr* address, unsigned int prefix) { + const int family = loc_address_family(address); + + // What is the largest possible prefix? + const unsigned int bit_length = loc_address_family_bit_length(family); + + if (prefix <= bit_length) + return 1; + + return 0; +} + +static inline int loc_address_cmp(const struct in6_addr* a1, const struct in6_addr* a2) { + for (unsigned int i = 0; i < 16; i++) { + if (a1->s6_addr[i] > a2->s6_addr[i]) + return 1; + + else if (a1->s6_addr[i] < a2->s6_addr[i]) + return -1; + } + + return 0; +} + +#define foreach_octet_in_address(octet, address) \ + for (octet = (IN6_IS_ADDR_V4MAPPED(address) ? 12 : 0); octet <= 15; octet++) + +#define foreach_octet_in_address_reverse(octet, address) \ + for (octet = 15; octet >= (IN6_IS_ADDR_V4MAPPED(address) ? 12 : 0); octet--) + +static inline int loc_address_all_zeroes(const struct in6_addr* address) { + int octet = 0; + + foreach_octet_in_address(octet, address) { + if (address->s6_addr[octet]) + return 0; + } + + return 1; +} + +static inline int loc_address_all_ones(const struct in6_addr* address) { + int octet = 0; + + foreach_octet_in_address(octet, address) { + if (address->s6_addr[octet] < 255) + return 0; + } + + return 1; +} + +static inline int loc_address_get_bit(const struct in6_addr* address, unsigned int i) { + return ((address->s6_addr[i / 8] >> (7 - (i % 8))) & 1); +} + +static inline void loc_address_set_bit(struct in6_addr* address, unsigned int i, unsigned int val) { + address->s6_addr[i / 8] ^= (-val ^ address->s6_addr[i / 8]) & (1 << (7 - (i % 8))); +} + +static inline struct in6_addr loc_prefix_to_bitmask(const unsigned int prefix) { + struct in6_addr bitmask; + + for (unsigned int i = 0; i < 16; i++) + bitmask.s6_addr[i] = 0; + + for (int i = prefix, j = 0; i > 0; i -= 8, j++) { + if (i >= 8) + bitmask.s6_addr[j] = 0xff; + else + bitmask.s6_addr[j] = 0xff << (8 - i); + } + + return bitmask; +} + +static inline unsigned int loc_address_bit_length(const struct in6_addr* address) { + int octet = 0; + foreach_octet_in_address(octet, address) { + if (address->s6_addr[octet]) + return (15 - octet) * 8 + 32 - __builtin_clz(address->s6_addr[octet]); + } + + return 0; +} + +static inline int loc_address_reset(struct in6_addr* address, int family) { + switch (family) { + case AF_INET6: + address->s6_addr32[0] = 0x00000000; + address->s6_addr32[1] = 0x00000000; + address->s6_addr32[2] = 0x00000000; + address->s6_addr32[3] = 0x00000000; + return 0; + + case AF_INET: + address->s6_addr32[0] = 0x00000000; + address->s6_addr32[1] = 0x00000000; + address->s6_addr32[2] = htonl(0xffff); + address->s6_addr32[3] = 0x00000000; + return 0; + } + + return -1; +} + +static inline int loc_address_reset_last(struct in6_addr* address, int family) { + switch (family) { + case AF_INET6: + address->s6_addr32[0] = 0xffffffff; + address->s6_addr32[1] = 0xffffffff; + address->s6_addr32[2] = 0xffffffff; + address->s6_addr32[3] = 0xffffffff; + return 0; + + case AF_INET: + address->s6_addr32[0] = 0x00000000; + address->s6_addr32[1] = 0x00000000; + address->s6_addr32[2] = htonl(0xffff); + address->s6_addr32[3] = 0xffffffff; + return 0; + } + + return -1; +} + +static inline struct in6_addr loc_address_and( + const struct in6_addr* address, const struct in6_addr* bitmask) { + struct in6_addr a; + + // Perform bitwise AND + for (unsigned int i = 0; i < 4; i++) + a.s6_addr32[i] = address->s6_addr32[i] & bitmask->s6_addr32[i]; + + return a; +} + +static inline struct in6_addr loc_address_or( + const struct in6_addr* address, const struct in6_addr* bitmask) { + struct in6_addr a; + + // Perform bitwise OR + for (unsigned int i = 0; i < 4; i++) + a.s6_addr32[i] = address->s6_addr32[i] | ~bitmask->s6_addr32[i]; + + return a; +} + +static inline int loc_address_sub(struct in6_addr* result, + const struct in6_addr* address1, const struct in6_addr* address2) { + int family1 = loc_address_family(address1); + int family2 = loc_address_family(address2); + + // Address family must match + if (family1 != family2) { + errno = EINVAL; + return 1; + } + + // Clear result + int r = loc_address_reset(result, family1); + if (r) + return r; + + int octet = 0; + int remainder = 0; + + foreach_octet_in_address_reverse(octet, address1) { + int x = address1->s6_addr[octet] - address2->s6_addr[octet] + remainder; + + // Store remainder for the next iteration + remainder = (x >> 8); + + result->s6_addr[octet] = x & 0xff; + } + + return 0; +} + +static inline void loc_address_increment(struct in6_addr* address) { + // Prevent overflow when everything is ones + if (loc_address_all_ones(address)) + return; + + int octet = 0; + foreach_octet_in_address_reverse(octet, address) { + if (address->s6_addr[octet] < 255) { + address->s6_addr[octet]++; + break; + } else { + address->s6_addr[octet] = 0; + } + } +} + +static inline void loc_address_decrement(struct in6_addr* address) { + // Prevent underflow when everything is ones + if (loc_address_all_zeroes(address)) + return; + + int octet = 0; + foreach_octet_in_address_reverse(octet, address) { + if (address->s6_addr[octet] > 0) { + address->s6_addr[octet]--; + break; + } else { + address->s6_addr[octet] = 255; + } + } +} + +static inline int loc_address_count_trailing_zero_bits(const struct in6_addr* address) { + int zeroes = 0; + + int octet = 0; + foreach_octet_in_address_reverse(octet, address) { + if (address->s6_addr[octet]) { + zeroes += __builtin_ctz(address->s6_addr[octet]); + break; + } else + zeroes += 8; + } + + return zeroes; +} + +#endif /* LIBLOC_PRIVATE */ + +#endif /* LIBLOC_ADDRESS_H */ diff --git a/src/libloc/as-list.h b/src/libloc/as-list.h new file mode 100644 index 0000000..bd1d4e6 --- /dev/null +++ b/src/libloc/as-list.h @@ -0,0 +1,43 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_AS_LIST_H +#define LIBLOC_AS_LIST_H + +#include +#include + +struct loc_as_list; + +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); + +size_t loc_as_list_size(struct loc_as_list* list); +int loc_as_list_empty(struct loc_as_list* list); +void loc_as_list_clear(struct loc_as_list* list); + +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); + +int loc_as_list_contains( + struct loc_as_list* list, struct loc_as* as); +int loc_as_list_contains_number( + struct loc_as_list* list, uint32_t number); + +void loc_as_list_sort(struct loc_as_list* list); + +#endif diff --git a/src/libloc/as.h b/src/libloc/as.h new file mode 100644 index 0000000..05e0188 --- /dev/null +++ b/src/libloc/as.h @@ -0,0 +1,49 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_AS_H +#define LIBLOC_AS_H + +#include + +#include +#include +#include + +struct loc_as; +int loc_as_new(struct loc_ctx* ctx, struct loc_as** as, uint32_t number); +struct loc_as* loc_as_ref(struct loc_as* as); +struct loc_as* loc_as_unref(struct loc_as* as); + +uint32_t loc_as_get_number(struct loc_as* as); + +const char* loc_as_get_name(struct loc_as* as); +int loc_as_set_name(struct loc_as* as, const char* name); + +int loc_as_cmp(struct loc_as* as1, struct loc_as* as2); + +#ifdef LIBLOC_PRIVATE + +int loc_as_new_from_database_v1(struct loc_ctx* ctx, struct loc_stringpool* pool, + struct loc_as** as, const struct loc_database_as_v1* dbobj); +int loc_as_to_database_v1(struct loc_as* as, struct loc_stringpool* pool, + struct loc_database_as_v1* dbobj); + +int loc_as_match_string(struct loc_as* as, const char* string); + +#endif + +#endif diff --git a/src/libloc/compat.h b/src/libloc/compat.h new file mode 100644 index 0000000..8750976 --- /dev/null +++ b/src/libloc/compat.h @@ -0,0 +1,40 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_COMPAT_H +#define LIBLOC_COMPAT_H + +#ifdef __APPLE__ +/* Hacks to make this library compile on Mac OS X */ + +#include +#define be16toh(x) OSSwapBigToHostInt16(x) +#define htobe16(x) OSSwapHostToBigInt16(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define htobe32(x) OSSwapHostToBigInt32(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define htobe64(x) OSSwapHostToBigInt64(x) + +#ifndef s6_addr16 +# define s6_addr16 __u6_addr.__u6_addr16 +#endif +#ifndef s6_addr32 +# define s6_addr32 __u6_addr.__u6_addr32 +#endif + +#endif + +#endif diff --git a/src/libloc/country-list.h b/src/libloc/country-list.h new file mode 100644 index 0000000..a479aed --- /dev/null +++ b/src/libloc/country-list.h @@ -0,0 +1,45 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_COUNTRY_LIST_H +#define LIBLOC_COUNTRY_LIST_H + +#include + +#include +#include + +struct loc_country_list; + +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); + +size_t loc_country_list_size(struct loc_country_list* list); +int loc_country_list_empty(struct loc_country_list* list); +void loc_country_list_clear(struct loc_country_list* list); + +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); + +int loc_country_list_contains( + struct loc_country_list* list, struct loc_country* country); +int loc_country_list_contains_code( + struct loc_country_list* list, const char* code); + +void loc_country_list_sort(struct loc_country_list* list); + +#endif diff --git a/src/libloc/country.h b/src/libloc/country.h new file mode 100644 index 0000000..98ee8cb --- /dev/null +++ b/src/libloc/country.h @@ -0,0 +1,59 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_COUNTRY_H +#define LIBLOC_COUNTRY_H + +#include +#include +#include + +struct loc_country; +int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code); +struct loc_country* loc_country_ref(struct loc_country* country); +struct loc_country* loc_country_unref(struct loc_country* country); + +const char* loc_country_get_code(struct loc_country* country); + +const char* loc_country_get_continent_code(struct loc_country* country); +int loc_country_set_continent_code(struct loc_country* country, const char* continent_code); + +const char* loc_country_get_name(struct loc_country* country); +int loc_country_set_name(struct loc_country* country, const char* name); + +int loc_country_cmp(struct loc_country* country1, struct loc_country* country2); + +int loc_country_code_is_valid(const char* cc); +int loc_country_special_code_to_flag(const char* cc); + +#ifdef LIBLOC_PRIVATE + +#include + +int loc_country_new_from_database_v1(struct loc_ctx* ctx, struct loc_stringpool* pool, + struct loc_country** country, const struct loc_database_country_v1* dbobj); +int loc_country_to_database_v1(struct loc_country* country, + struct loc_stringpool* pool, struct loc_database_country_v1* dbobj); + +static inline void loc_country_code_copy(char* dst, const char* src) { + for (unsigned int i = 0; i < 2; i++) { + dst[i] = src[i]; + } +} + +#endif + +#endif diff --git a/src/libloc/database.h b/src/libloc/database.h new file mode 100644 index 0000000..220f0fb --- /dev/null +++ b/src/libloc/database.h @@ -0,0 +1,87 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_DATABASE_H +#define LIBLOC_DATABASE_H + +#include +#include +#include + +#include +#include +#include +#include +#include + +struct loc_database; +int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); +struct loc_database* loc_database_ref(struct loc_database* db); +struct loc_database* loc_database_unref(struct loc_database* db); + +int loc_database_verify(struct loc_database* db, FILE* f); + +time_t loc_database_created_at(struct loc_database* db); +const char* loc_database_get_vendor(struct loc_database* db); +const char* loc_database_get_description(struct loc_database* db); +const char* loc_database_get_license(struct loc_database* db); + +int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number); +size_t loc_database_count_as(struct loc_database* db); + +int loc_database_lookup(struct loc_database* db, + const struct in6_addr* address, struct loc_network** network); +int loc_database_lookup_from_string(struct loc_database* db, + const char* string, struct loc_network** network); + +int loc_database_get_country(struct loc_database* db, + struct loc_country** country, const char* code); + +enum loc_database_enumerator_mode { + LOC_DB_ENUMERATE_NETWORKS = 1, + LOC_DB_ENUMERATE_ASES = 2, + LOC_DB_ENUMERATE_COUNTRIES = 3, + LOC_DB_ENUMERATE_BOGONS = 4, +}; + +enum loc_database_enumerator_flags { + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), +}; + +struct loc_database_enumerator; +int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); +struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); +struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + +int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); +int loc_database_enumerator_set_countries( + struct loc_database_enumerator* enumerator, struct loc_country_list* countries); +struct loc_as_list* loc_database_enumerator_get_asns( + struct loc_database_enumerator* enumerator); +int loc_database_enumerator_set_asns( + struct loc_database_enumerator* enumerator, struct loc_as_list* asns); +int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); +int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); +int loc_database_enumerator_next_as( + struct loc_database_enumerator* enumerator, struct loc_as** as); +int loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network); +int loc_database_enumerator_next_country( + struct loc_database_enumerator* enumerator, struct loc_country** country); + +#endif diff --git a/src/libloc/format.h b/src/libloc/format.h new file mode 100644 index 0000000..a04c089 --- /dev/null +++ b/src/libloc/format.h @@ -0,0 +1,128 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_FORMAT_H +#define LIBLOC_FORMAT_H + +#include + +#define LOC_DATABASE_MAGIC "LOCDBXX" + +enum loc_database_version { + LOC_DATABASE_VERSION_UNSET = 0, + LOC_DATABASE_VERSION_1 = 1, +}; + +#define LOC_DATABASE_VERSION_LATEST LOC_DATABASE_VERSION_1 + +#ifdef LIBLOC_PRIVATE + +#define LOC_DATABASE_DOMAIN "_v%u._db.location.ipfire.org" + +#define LOC_DATABASE_PAGE_SIZE 4096 +#define LOC_SIGNATURE_MAX_LENGTH (LOC_DATABASE_PAGE_SIZE / 2) + +struct loc_database_magic { + char magic[7]; + + // Database version information + uint8_t version; +}; + +struct loc_database_header_v1 { + // UNIX timestamp when the database was created + uint64_t created_at; + + // Vendor who created the database + uint32_t vendor; + + // Description of the database + uint32_t description; + + // License of the database + uint32_t license; + + // Tells us where the ASes start + uint32_t as_offset; + uint32_t as_length; + + // Tells us where the networks start + uint32_t network_data_offset; + uint32_t network_data_length; + + // Tells us where the network nodes start + uint32_t network_tree_offset; + uint32_t network_tree_length; + + // Tells us where the countries start + uint32_t countries_offset; + uint32_t countries_length; + + // Tells us where the pool starts + uint32_t pool_offset; + uint32_t pool_length; + + // Signatures + uint16_t signature1_length; + uint16_t signature2_length; + char signature1[LOC_SIGNATURE_MAX_LENGTH]; + char signature2[LOC_SIGNATURE_MAX_LENGTH]; + + // Add some padding for future extensions + char padding[32]; +}; + +struct loc_database_network_node_v1 { + uint32_t zero; + uint32_t one; + + uint32_t network; +}; + +struct loc_database_network_v1 { + // The start address and prefix will be encoded in the tree + + // The country this network is located in + char country_code[2]; + + // ASN + uint32_t asn; + + // Flags + uint16_t flags; + + // Reserved + char padding[2]; +}; + +struct loc_database_as_v1 { + // The AS number + uint32_t number; + + // Name + uint32_t name; +}; + +struct loc_database_country_v1 { + char code[2]; + char continent_code[2]; + + // Name in the string pool + uint32_t name; +}; + +#endif +#endif diff --git a/src/libloc/libloc.h b/src/libloc/libloc.h new file mode 100644 index 0000000..938ed75 --- /dev/null +++ b/src/libloc/libloc.h @@ -0,0 +1,43 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_H +#define LIBLOC_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct loc_ctx; +struct loc_ctx *loc_ref(struct loc_ctx* ctx); +struct loc_ctx *loc_unref(struct loc_ctx* ctx); + +int loc_new(struct loc_ctx** ctx); +void loc_set_log_fn(struct loc_ctx* ctx, + void (*log_fn)(struct loc_ctx* ctx, + int priority, const char* file, int line, const char* fn, + const char* format, va_list args)); +int loc_get_log_priority(struct loc_ctx* ctx); +void loc_set_log_priority(struct loc_ctx* ctx, int priority); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/src/libloc/network-list.h b/src/libloc/network-list.h new file mode 100644 index 0000000..988a8a0 --- /dev/null +++ b/src/libloc/network-list.h @@ -0,0 +1,46 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2020 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_NETWORK_LIST_H +#define LIBLOC_NETWORK_LIST_H + +#include + +struct loc_network_list; +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); +size_t loc_network_list_size(struct loc_network_list* list); +int loc_network_list_empty(struct loc_network_list* list); +void loc_network_list_clear(struct loc_network_list* list); +void loc_network_list_dump(struct loc_network_list* list); +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); +struct loc_network* loc_network_list_pop(struct loc_network_list* list); +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); + +#ifdef LIBLOC_PRIVATE + +#include + +int loc_network_list_summarize(struct loc_ctx* ctx, + const struct in6_addr* first, const struct in6_addr* last, struct loc_network_list** list); + +#endif /* LOC_PRIVATE */ + +#endif diff --git a/src/libloc/network.h b/src/libloc/network.h new file mode 100644 index 0000000..024a0f1 --- /dev/null +++ b/src/libloc/network.h @@ -0,0 +1,97 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017-2021 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_NETWORK_H +#define LIBLOC_NETWORK_H + +#include + +#include +#include +#include + +enum loc_network_flags { + LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 + LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2 + LOC_NETWORK_FLAG_ANYCAST = (1 << 2), // A3 + LOC_NETWORK_FLAG_DROP = (1 << 3), // XD +}; + +struct loc_network; +int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* first_address, unsigned int prefix); +int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, + const char* address_string); +struct loc_network* loc_network_ref(struct loc_network* network); +struct loc_network* loc_network_unref(struct loc_network* network); +const char* loc_network_str(struct loc_network* network); +int loc_network_address_family(struct loc_network* network); +unsigned int loc_network_prefix(struct loc_network* network); + +const struct in6_addr* loc_network_get_first_address(struct loc_network* network); +const char* loc_network_format_first_address(struct loc_network* network); +const struct in6_addr* loc_network_get_last_address(struct loc_network* network); +const char* loc_network_format_last_address(struct loc_network* network); +int loc_network_matches_address(struct loc_network* network, const struct in6_addr* address); + +const char* loc_network_get_country_code(struct loc_network* network); +int loc_network_set_country_code(struct loc_network* network, const char* country_code); +int loc_network_matches_country_code(struct loc_network* network, const char* country_code); + +uint32_t loc_network_get_asn(struct loc_network* network); +int loc_network_set_asn(struct loc_network* network, uint32_t asn); + +int loc_network_has_flag(struct loc_network* network, uint32_t flag); +int loc_network_set_flag(struct loc_network* network, uint32_t flag); + +int loc_network_cmp(struct loc_network* self, struct loc_network* other); +int loc_network_overlaps(struct loc_network* self, struct loc_network* other); +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); +int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); +struct loc_network_list* loc_network_exclude( + struct loc_network* self, struct loc_network* other); +struct loc_network_list* loc_network_exclude_list( + struct loc_network* network, struct loc_network_list* list); + +#ifdef LIBLOC_PRIVATE + +int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); +int loc_network_new_from_database_v1(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix, const struct loc_database_network_v1* dbobj); + +struct loc_network_tree; +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree); +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree); +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree); +int loc_network_tree_walk(struct loc_network_tree* tree, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data); +int loc_network_tree_dump(struct loc_network_tree* tree); +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network); +size_t loc_network_tree_count_networks(struct loc_network_tree* tree); +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree); + +struct loc_network_tree_node; +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node); +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node); +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node); +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index); + +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node); +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node); + +#endif +#endif diff --git a/src/libloc/private.h b/src/libloc/private.h new file mode 100644 index 0000000..2ee97d1 --- /dev/null +++ b/src/libloc/private.h @@ -0,0 +1,105 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_PRIVATE_H +#define LIBLOC_PRIVATE_H + +#ifdef LIBLOC_PRIVATE + +#include +#include + +#include + +static inline void __attribute__((always_inline, format(printf, 2, 3))) +loc_log_null(struct loc_ctx *ctx, const char *format, ...) {} + +#define loc_log_cond(ctx, prio, arg...) \ + do { \ + if (loc_get_log_priority(ctx) >= prio) \ + loc_log(ctx, prio, __FILE__, __LINE__, __FUNCTION__, ## arg); \ + } while (0) + +#ifdef ENABLE_DEBUG +# define DEBUG(ctx, arg...) loc_log_cond(ctx, LOG_DEBUG, ## arg) +#else +# define DEBUG(ctx, arg...) loc_log_null(ctx, ## arg) +#endif + +#define INFO(ctx, arg...) loc_log_cond(ctx, LOG_INFO, ## arg) +#define ERROR(ctx, arg...) loc_log_cond(ctx, LOG_ERR, ## arg) + +#ifndef HAVE_SECURE_GETENV +# ifdef HAVE___SECURE_GETENV +# define secure_getenv __secure_getenv +# else +# define secure_getenv getenv +# endif +#endif + +#define LOC_EXPORT __attribute__ ((visibility("default"))) + +void loc_log(struct loc_ctx *ctx, + int priority, const char *file, int line, const char *fn, + const char *format, ...) __attribute__((format(printf, 6, 7))); + + +static inline void hexdump(struct loc_ctx* ctx, const void* addr, size_t len) { + char buffer_hex[16 * 3 + 6]; + char buffer_ascii[17]; + + unsigned int i = 0; + unsigned char* p = (unsigned char*)addr; + + DEBUG(ctx, "Dumping %zu byte(s)\n", len); + + // Process every byte in the data + for (i = 0; i < len; i++) { + // Multiple of 16 means new line (with line offset) + if ((i % 16) == 0) { + // Just don't print ASCII for the zeroth line + if (i != 0) + DEBUG(ctx, " %s %s\n", buffer_hex, buffer_ascii); + + // Output the offset. + sprintf(buffer_hex, "%04x ", i); + } + + // Now the hex code for the specific character + sprintf(buffer_hex + 5 + ((i % 16) * 3), " %02x", p[i]); + + // And store a printable ASCII character for later + if ((p[i] < 0x20) || (p[i] > 0x7e)) + buffer_ascii[i % 16] = '.'; + else + buffer_ascii[i % 16] = p[i]; + + // Terminate string + buffer_ascii[(i % 16) + 1] = '\0'; + } + + // Pad out last line if not exactly 16 characters + while ((i % 16) != 0) { + sprintf(buffer_hex + 5 + ((i % 16) * 3), " "); + i++; + } + + // And print the final bit + DEBUG(ctx, " %s %s\n", buffer_hex, buffer_ascii); +} + +#endif +#endif diff --git a/src/libloc/resolv.h b/src/libloc/resolv.h new file mode 100644 index 0000000..dd13d60 --- /dev/null +++ b/src/libloc/resolv.h @@ -0,0 +1,26 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_RESOLV_H +#define LIBLOC_RESOLV_H + +#include + +#include + +int loc_discover_latest_version(struct loc_ctx* ctx, unsigned int version, time_t* t); + +#endif diff --git a/src/libloc/stringpool.h b/src/libloc/stringpool.h new file mode 100644 index 0000000..932aad7 --- /dev/null +++ b/src/libloc/stringpool.h @@ -0,0 +1,44 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_STRINGPOOL_H +#define LIBLOC_STRINGPOOL_H + +#ifdef LIBLOC_PRIVATE + +#include +#include + +#include + +struct loc_stringpool; +int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool); +int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool, + FILE* f, size_t length, off_t offset); + +struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool); +struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool); + +const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset); +size_t loc_stringpool_get_size(struct loc_stringpool* pool); + +off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string); +void loc_stringpool_dump(struct loc_stringpool* pool); + +size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f); + +#endif +#endif diff --git a/src/libloc/writer.h b/src/libloc/writer.h new file mode 100644 index 0000000..eae9548 --- /dev/null +++ b/src/libloc/writer.h @@ -0,0 +1,49 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_WRITER_H +#define LIBLOC_WRITER_H + +#include + +#include +#include +#include +#include +#include + +struct loc_writer; + +int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer, + FILE* fkey1, FILE* fkey2); + +struct loc_writer* loc_writer_ref(struct loc_writer* writer); +struct loc_writer* loc_writer_unref(struct loc_writer* writer); + +const char* loc_writer_get_vendor(struct loc_writer* writer); +int loc_writer_set_vendor(struct loc_writer* writer, const char* vendor); +const char* loc_writer_get_description(struct loc_writer* writer); +int loc_writer_set_description(struct loc_writer* writer, const char* description); +const char* loc_writer_get_license(struct loc_writer* writer); +int loc_writer_set_license(struct loc_writer* writer, const char* license); + +int loc_writer_add_as(struct loc_writer* writer, struct loc_as** as, uint32_t number); +int loc_writer_add_network(struct loc_writer* writer, struct loc_network** network, const char* string); +int loc_writer_add_country(struct loc_writer* writer, struct loc_country** country, const char* country_code); + +int loc_writer_write(struct loc_writer* writer, FILE* f, enum loc_database_version); + +#endif diff --git a/src/network-list.c b/src/network-list.c new file mode 100644 index 0000000..e2bbf05 --- /dev/null +++ b/src/network-list.c @@ -0,0 +1,388 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2020 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include + +#include +#include +#include +#include + +struct loc_network_list { + struct loc_ctx* ctx; + int refcount; + + struct loc_network** elements; + size_t elements_size; + + size_t size; +}; + +static int loc_network_list_grow(struct loc_network_list* list) { + size_t size = list->elements_size * 2; + if (size < 1024) + size = 1024; + + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", + list, size, list->elements_size + size); + + struct loc_network** elements = reallocarray(list->elements, + list->elements_size + size, sizeof(*list->elements)); + if (!elements) + return -errno; + + list->elements = elements; + list->elements_size += size; + + return 0; +} + +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, + struct loc_network_list** list) { + struct loc_network_list* l = calloc(1, sizeof(*l)); + if (!l) + return -ENOMEM; + + l->ctx = loc_ref(ctx); + l->refcount = 1; + + DEBUG(l->ctx, "Network list allocated at %p\n", l); + *list = l; + return 0; +} + +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { + list->refcount++; + + return list; +} + +static void loc_network_list_free(struct loc_network_list* list) { + DEBUG(list->ctx, "Releasing network list at %p\n", list); + + // Remove all content + loc_network_list_clear(list); + + loc_unref(list->ctx); + free(list); +} + +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { + if (!list) + return NULL; + + if (--list->refcount > 0) + return list; + + loc_network_list_free(list); + return NULL; +} + +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { + return list->size; +} + +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + return list->size == 0; +} + +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { + if (!list->elements) + return; + + for (unsigned int i = 0; i < list->size; i++) + loc_network_unref(list->elements[i]); + + free(list->elements); + list->elements = NULL; + list->elements_size = 0; + + list->size = 0; +} + +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { + struct loc_network* network; + + for (unsigned int i = 0; i < list->size; i++) { + network = list->elements[i]; + + INFO(list->ctx, "%4d: %s\n", + i, loc_network_str(network)); + } +} + +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { + // Check index + if (index >= list->size) + return NULL; + + return loc_network_ref(list->elements[index]); +} + +static off_t loc_network_list_find(struct loc_network_list* list, + struct loc_network* network, int* found) { + // Insert at the beginning for an empty list + if (loc_network_list_empty(list)) + return 0; + + off_t lo = 0; + off_t hi = list->size - 1; + int result; + + // Since we are working on an ordered list, there is often a good chance that + // the network we are looking for is at the end or has to go to the end. + if (hi >= 0) { + result = loc_network_cmp(network, list->elements[hi]); + + // Match, so we are done + if (result == 0) { + *found = 1; + + return hi; + + // This needs to be added after the last one + } else if (result > 0) { + *found = 0; + + return hi + 1; + } + } + +#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); +#endif + + off_t i = 0; + + while (lo <= hi) { + i = (lo + hi) / 2; + + // Check if this is a match + result = loc_network_cmp(network, list->elements[i]); + + if (result == 0) { + *found = 1; + +#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(list->ctx, "Found network in %.4fms at %jd\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); +#endif + + return i; + } + + if (result > 0) { + lo = i + 1; + i++; + } else { + hi = i - 1; + } + } + + *found = 0; + +#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); +#endif + + return i; +} + +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { + int found = 0; + + off_t index = loc_network_list_find(list, network, &found); + + // The network has been found on the list. Nothing to do. + if (found) + return 0; + + DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", + list, network, (intmax_t)index); + + // Check if we have space left + if (list->size >= list->elements_size) { + int r = loc_network_list_grow(list); + if (r) + return r; + } + + // The list is now larger + list->size++; + + // Move all elements out of the way + for (unsigned int i = list->size - 1; i > index; i--) + list->elements[i] = list->elements[i - 1]; + + // Add the new element at the right place + list->elements[index] = loc_network_ref(network); + + return 0; +} + +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { + // Return nothing when empty + if (loc_network_list_empty(list)) { + DEBUG(list->ctx, "%p: Popped empty stack\n", list); + return NULL; + } + + struct loc_network* network = list->elements[--list->size]; + + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); + + return network; +} + +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { + // Return nothing when empty + if (loc_network_list_empty(list)) { + DEBUG(list->ctx, "%p: Popped empty stack\n", list); + return NULL; + } + + struct loc_network* network = list->elements[0]; + + // Move all elements to the top of the stack + for (unsigned int i = 0; i < list->size - 1; i++) { + list->elements[i] = list->elements[i+1]; + } + + // The list is shorter now + --list->size; + + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); + + return network; +} + +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { + int found = 0; + + loc_network_list_find(list, network, &found); + + return found; +} + +LOC_EXPORT int loc_network_list_merge( + struct loc_network_list* self, struct loc_network_list* other) { + int r; + + for (unsigned int i = 0; i < other->size; i++) { + r = loc_network_list_push(self, other->elements[i]); + if (r) + return r; + } + + return 0; +} + +int loc_network_list_summarize(struct loc_ctx* ctx, + const struct in6_addr* first, const struct in6_addr* last, struct loc_network_list** list) { + int r; + + if (!list) { + errno = EINVAL; + return 1; + } + + DEBUG(ctx, "Summarizing %s - %s\n", loc_address_str(first), loc_address_str(last)); + + const int family1 = loc_address_family(first); + const int family2 = loc_address_family(last); + + // Check if address families match + if (family1 != family2) { + ERROR(ctx, "Address families do not match\n"); + errno = EINVAL; + return 1; + } + + // Check if the last address is larger than the first address + if (loc_address_cmp(first, last) >= 0) { + ERROR(ctx, "The first address must be smaller than the last address\n"); + errno = EINVAL; + return 1; + } + + struct loc_network* network = NULL; + struct in6_addr start = *first; + + const int family_bit_length = loc_address_family_bit_length(family1); + + while (loc_address_cmp(&start, last) <= 0) { + struct in6_addr num; + int bits1; + + // Find the number of trailing zeroes of the start address + if (loc_address_all_zeroes(&start)) + bits1 = family_bit_length; + else { + bits1 = loc_address_count_trailing_zero_bits(&start); + if (bits1 > family_bit_length) + bits1 = family_bit_length; + } + + // Subtract the start address from the last address and add one + // (i.e. how many addresses are in this network?) + r = loc_address_sub(&num, last, &start); + if (r) + return r; + + loc_address_increment(&num); + + // How many bits do we need to represent this address? + int bits2 = loc_address_bit_length(&num) - 1; + + // Select the smaller one + int bits = (bits1 > bits2) ? bits2 : bits1; + + // Create a network + r = loc_network_new(ctx, &network, &start, family_bit_length - bits); + if (r) + return r; + + DEBUG(ctx, "Found network %s\n", loc_network_str(network)); + + // Push network on the list + r = loc_network_list_push(*list, network); + if (r) { + loc_network_unref(network); + return r; + } + + // The next network starts right after this one + start = *loc_network_get_last_address(network); + + // If we have reached the end of possible IP addresses, we stop + if (loc_address_all_ones(&start)) + break; + + loc_address_increment(&start); + } + + return 0; +} diff --git a/src/network.c b/src/network.c new file mode 100644 index 0000000..37a483e --- /dev/null +++ b/src/network.c @@ -0,0 +1,889 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ENDIAN_H +# include +#endif + +#include +#include +#include +#include +#include +#include +#include + +struct loc_network { + struct loc_ctx* ctx; + int refcount; + + int family; + struct in6_addr first_address; + struct in6_addr last_address; + unsigned int prefix; + + char country_code[3]; + uint32_t asn; + enum loc_network_flags flags; + + char string[INET6_ADDRSTRLEN + 4]; +}; + +LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix) { + // Validate the prefix + if (!loc_address_valid_prefix(address, prefix)) { + ERROR(ctx, "Invalid prefix in %s: %u\n", loc_address_str(address), prefix); + errno = EINVAL; + return 1; + } + + struct loc_network* n = calloc(1, sizeof(*n)); + if (!n) { + errno = ENOMEM; + return 1; + } + + n->ctx = loc_ref(ctx); + n->refcount = 1; + + // Store the prefix + if (IN6_IS_ADDR_V4MAPPED(address)) + n->prefix = prefix + 96; + else + n->prefix = prefix; + + // Convert the prefix into a bitmask + struct in6_addr bitmask = loc_prefix_to_bitmask(n->prefix); + + // Store the first and last address in the network + n->first_address = loc_address_and(address, &bitmask); + n->last_address = loc_address_or(&n->first_address, &bitmask); + + // Set family + n->family = loc_address_family(&n->first_address); + + DEBUG(n->ctx, "Network allocated at %p\n", n); + *network = n; + return 0; +} + +LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, + struct loc_network** network, const char* string) { + struct in6_addr address; + unsigned int prefix; + + // Parse the input + int r = loc_address_parse(&address, &prefix, string); + if (r) { + ERROR(ctx, "Could not parse network %s: %m\n", string); + return r; + } + + // Create a new network + return loc_network_new(ctx, network, &address, prefix); +} + +LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { + network->refcount++; + + return network; +} + +static void loc_network_free(struct loc_network* network) { + DEBUG(network->ctx, "Releasing network at %p\n", network); + + loc_unref(network->ctx); + free(network); +} + +LOC_EXPORT struct loc_network* loc_network_unref(struct loc_network* network) { + if (!network) + return NULL; + + if (--network->refcount > 0) + return network; + + loc_network_free(network); + return NULL; +} + +LOC_EXPORT const char* loc_network_str(struct loc_network* network) { + if (!*network->string) { + // Format the address + const char* address = loc_address_str(&network->first_address); + if (!address) + return NULL; + + // Fetch the prefix + unsigned int prefix = loc_network_prefix(network); + + // Format the string + int r = snprintf(network->string, sizeof(network->string) - 1, + "%s/%u", address, prefix); + if (r < 0) { + ERROR(network->ctx, "Could not format network string: %m\n"); + *network->string = '\0'; + return NULL; + } + } + + return network->string; +} + +LOC_EXPORT int loc_network_address_family(struct loc_network* network) { + return network->family; +} + +LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { + switch (network->family) { + case AF_INET6: + return network->prefix; + + case AF_INET: + return network->prefix - 96; + } + + return 0; +} + +LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) { + return &network->first_address; +} + +LOC_EXPORT const char* loc_network_format_first_address(struct loc_network* network) { + return loc_address_str(&network->first_address); +} + +LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) { + return &network->last_address; +} + +LOC_EXPORT const char* loc_network_format_last_address(struct loc_network* network) { + return loc_address_str(&network->last_address); +} + +LOC_EXPORT int loc_network_matches_address(struct loc_network* network, const struct in6_addr* address) { + // Address must be larger than the start address + if (loc_address_cmp(&network->first_address, address) > 0) + return 0; + + // Address must be smaller than the last address + if (loc_address_cmp(&network->last_address, address) < 0) + return 0; + + // The address is inside this network + return 1; +} + +LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) { + return network->country_code; +} + +LOC_EXPORT int loc_network_set_country_code(struct loc_network* network, const char* country_code) { + // Set empty country code + if (!country_code || !*country_code) { + *network->country_code = '\0'; + return 0; + } + + // Check country code + if (!loc_country_code_is_valid(country_code)) + return -EINVAL; + + loc_country_code_copy(network->country_code, country_code); + + return 0; +} + +LOC_EXPORT int loc_network_matches_country_code(struct loc_network* network, const char* country_code) { + // Search for any special flags + const int flag = loc_country_special_code_to_flag(country_code); + + // If we found a flag, we will return whether it is set or not + if (flag) + return loc_network_has_flag(network, flag); + + // Check country code + if (!loc_country_code_is_valid(country_code)) + return -EINVAL; + + // Check for an exact match + return (network->country_code[0] == country_code[0]) + && (network->country_code[1] == country_code[1]); +} + +LOC_EXPORT uint32_t loc_network_get_asn(struct loc_network* network) { + return network->asn; +} + +LOC_EXPORT int loc_network_set_asn(struct loc_network* network, uint32_t asn) { + network->asn = asn; + + return 0; +} + +LOC_EXPORT int loc_network_has_flag(struct loc_network* network, uint32_t flag) { + return network->flags & flag; +} + +LOC_EXPORT int loc_network_set_flag(struct loc_network* network, uint32_t flag) { + network->flags |= flag; + + return 0; +} + +LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { + // Compare address + int r = loc_address_cmp(&self->first_address, &other->first_address); + if (r) + return r; + + // Compare prefix + if (self->prefix > other->prefix) + return 1; + else if (self->prefix < other->prefix) + return -1; + + // Both networks are equal + return 0; +} + +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { + // Either of the start addresses must be in the other subnet + if (loc_network_matches_address(self, &other->first_address)) + return 1; + + if (loc_network_matches_address(other, &self->first_address)) + return 1; + + // Or either of the end addresses is in the other subnet + if (loc_network_matches_address(self, &other->last_address)) + return 1; + + if (loc_network_matches_address(other, &self->last_address)) + return 1; + + return 0; +} + +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { + // The prefix must be smaller (this avoids the more complex comparisons later) + if (self->prefix > other->prefix) + return 0; + + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. + if (loc_address_cmp(&self->first_address, &other->first_address) > 0) + return 0; + + // If the end address of the other network is greater than this network, + // it cannot be a subnet. + if (loc_address_cmp(&self->last_address, &other->last_address) < 0) + return 0; + + return 1; +} + +LOC_EXPORT int loc_network_subnets(struct loc_network* network, + struct loc_network** subnet1, struct loc_network** subnet2) { + int r; + *subnet1 = NULL; + *subnet2 = NULL; + + // New prefix length + unsigned int prefix = loc_network_prefix(network) + 1; + + // Check if the new prefix is valid + if (!loc_address_valid_prefix(&network->first_address, prefix)) { + ERROR(network->ctx, "Invalid prefix: %d\n", prefix); + errno = EINVAL; + return 1; + } + + // Create the first half of the network + r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix); + if (r) + return r; + + // The next subnet starts after the first one + struct in6_addr first_address = (*subnet1)->last_address; + loc_address_increment(&first_address); + + // Create the second half of the network + r = loc_network_new(network->ctx, subnet2, &first_address, prefix); + if (r) + return r; + + // Copy country code + const char* country_code = loc_network_get_country_code(network); + if (country_code) { + loc_network_set_country_code(*subnet1, country_code); + loc_network_set_country_code(*subnet2, country_code); + } + + // Copy ASN + uint32_t asn = loc_network_get_asn(network); + if (asn) { + loc_network_set_asn(*subnet1, asn); + loc_network_set_asn(*subnet2, asn); + } + + // Copy flags + loc_network_set_flag(*subnet1, network->flags); + loc_network_set_flag(*subnet2, network->flags); + + return 0; +} + +static int __loc_network_exclude(struct loc_network* network, + struct loc_network* other, struct loc_network_list* list) { + struct loc_network* subnet1 = NULL; + struct loc_network* subnet2 = NULL; + + int r = loc_network_subnets(network, &subnet1, &subnet2); + if (r) + goto ERROR; + + if (loc_network_cmp(other, subnet1) == 0) { + r = loc_network_list_push(list, subnet2); + if (r) + goto ERROR; + + } else if (loc_network_cmp(other, subnet2) == 0) { + r = loc_network_list_push(list, subnet1); + if (r) + goto ERROR; + + } else if (loc_network_is_subnet(subnet1, other)) { + r = loc_network_list_push(list, subnet2); + if (r) + goto ERROR; + + r = __loc_network_exclude(subnet1, other, list); + if (r) + goto ERROR; + + } else if (loc_network_is_subnet(subnet2, other)) { + r = loc_network_list_push(list, subnet1); + if (r) + goto ERROR; + + r = __loc_network_exclude(subnet2, other, list); + if (r) + goto ERROR; + + } else { + ERROR(network->ctx, "We should never get here\n"); + r = 1; + goto ERROR; + } + +ERROR: + if (subnet1) + loc_network_unref(subnet1); + + if (subnet2) + loc_network_unref(subnet2); + + if (r) + DEBUG(network->ctx, "%s has failed with %d\n", __FUNCTION__, r); + + return r; +} + +static int __loc_network_exclude_to_list(struct loc_network* self, + struct loc_network* other, struct loc_network_list* list) { + // Other must be a subnet of self + if (!loc_network_is_subnet(self, other)) { + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); + + // Exit silently + return 0; + } + + // We cannot perform this operation if both networks equal + if (loc_network_cmp(self, other) == 0) { + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); + + // Exit silently + return 0; + } + + return __loc_network_exclude(self, other, list); +} + +LOC_EXPORT struct loc_network_list* loc_network_exclude( + struct loc_network* self, struct loc_network* other) { + struct loc_network_list* list; + + DEBUG(self->ctx, "Returning %s excluding %s...\n", + loc_network_str(self), loc_network_str(other)); + + // Create a new list with the result + int r = loc_network_list_new(self->ctx, &list); + if (r) { + ERROR(self->ctx, "Could not create network list: %d\n", r); + + return NULL; + } + + r = __loc_network_exclude_to_list(self, other, list); + if (r) { + loc_network_list_unref(list); + + return NULL; + } + + // Return the result + return list; +} + +LOC_EXPORT struct loc_network_list* loc_network_exclude_list( + struct loc_network* network, struct loc_network_list* list) { + struct loc_network_list* to_check; + + // Create a new list with all networks to look at + int r = loc_network_list_new(network->ctx, &to_check); + if (r) + return NULL; + + struct loc_network* subnet = NULL; + struct loc_network_list* subnets = NULL; + + for (unsigned int i = 0; i < loc_network_list_size(list); i++) { + subnet = loc_network_list_get(list, i); + + // Find all excluded networks + if (!loc_network_list_contains(to_check, subnet)) { + r = __loc_network_exclude_to_list(network, subnet, to_check); + if (r) { + loc_network_list_unref(to_check); + loc_network_unref(subnet); + + return NULL; + } + } + + // Cleanup + loc_network_unref(subnet); + } + + r = loc_network_list_new(network->ctx, &subnets); + if (r) { + loc_network_list_unref(to_check); + return NULL; + } + + off_t smallest_subnet = 0; + + while (!loc_network_list_empty(to_check)) { + struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check); + + // Check whether the subnet to check is part of the input list + if (loc_network_list_contains(list, subnet_to_check)) { + loc_network_unref(subnet_to_check); + continue; + } + + // Marks whether this subnet passed all checks + int passed = 1; + + for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) { + subnet = loc_network_list_get(list, i); + + // Drop this subnet if is a subnet of another subnet + if (loc_network_is_subnet(subnet, subnet_to_check)) { + passed = 0; + loc_network_unref(subnet); + break; + } + + // Break it down if it overlaps + if (loc_network_overlaps(subnet, subnet_to_check)) { + passed = 0; + + __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); + + loc_network_unref(subnet); + break; + } + + // If the subnet is strictly greater, we do not need to continue the search + r = loc_network_cmp(subnet, subnet_to_check); + if (r > 0) { + loc_network_unref(subnet); + break; + + // If it is strictly smaller, we can continue the search from here next + // time because all networks that are to be checked can only be larger + // than this one. + } else if (r < 0) { + smallest_subnet = i; + } + + loc_network_unref(subnet); + } + + if (passed) { + r = loc_network_list_push(subnets, subnet_to_check); + } + + loc_network_unref(subnet_to_check); + } + + loc_network_list_unref(to_check); + + return subnets; +} + +int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); + + // Add ASN + dbobj->asn = htobe32(network->asn); + + // Flags + dbobj->flags = htobe16(network->flags); + + return 0; +} + +int loc_network_new_from_database_v1(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix, const struct loc_database_network_v1* dbobj) { + char country_code[3] = "\0\0"; + + // Adjust prefix for IPv4 + if (IN6_IS_ADDR_V4MAPPED(address)) + prefix -= 96; + + int r = loc_network_new(ctx, network, address, prefix); + if (r) { + ERROR(ctx, "Could not allocate a new network: %s", strerror(-r)); + return r; + } + + // Import country code + loc_country_code_copy(country_code, dbobj->country_code); + + r = loc_network_set_country_code(*network, country_code); + if (r) { + ERROR(ctx, "Could not set country code: %s\n", country_code); + return r; + } + + // Import ASN + uint32_t asn = be32toh(dbobj->asn); + r = loc_network_set_asn(*network, asn); + if (r) { + ERROR(ctx, "Could not set ASN: %d\n", asn); + return r; + } + + // Import flags + int flags = be16toh(dbobj->flags); + r = loc_network_set_flag(*network, flags); + if (r) { + ERROR(ctx, "Could not set flags: %d\n", flags); + return r; + } + + return 0; +} + +struct loc_network_tree { + struct loc_ctx* ctx; + int refcount; + + struct loc_network_tree_node* root; +}; + +struct loc_network_tree_node { + struct loc_ctx* ctx; + int refcount; + + struct loc_network_tree_node* zero; + struct loc_network_tree_node* one; + + struct loc_network* network; +}; + +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { + struct loc_network_tree* t = calloc(1, sizeof(*t)); + if (!t) + return -ENOMEM; + + t->ctx = loc_ref(ctx); + t->refcount = 1; + + // Create the root node + int r = loc_network_tree_node_new(ctx, &t->root); + if (r) { + loc_network_tree_unref(t); + return r; + } + + DEBUG(t->ctx, "Network tree allocated at %p\n", t); + *tree = t; + return 0; +} + +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { + return loc_network_tree_node_ref(tree->root); +} + +static struct loc_network_tree_node* loc_network_tree_get_node(struct loc_network_tree_node* node, int path) { + struct loc_network_tree_node** n; + + if (path == 0) + n = &node->zero; + else + n = &node->one; + + // If the desired node doesn't exist, yet, we will create it + if (*n == NULL) { + int r = loc_network_tree_node_new(node->ctx, n); + if (r) + return NULL; + } + + return *n; +} + +static struct loc_network_tree_node* loc_network_tree_get_path(struct loc_network_tree* tree, const struct in6_addr* address, unsigned int prefix) { + struct loc_network_tree_node* node = tree->root; + + for (unsigned int i = 0; i < prefix; i++) { + // Check if the ith bit is one or zero + node = loc_network_tree_get_node(node, loc_address_get_bit(address, i)); + } + + return node; +} + +static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_node* node, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data) { + int r; + + // Finding a network ends the walk here + if (node->network) { + if (filter_callback) { + int f = filter_callback(node->network, data); + if (f < 0) + return f; + + // Skip network if filter function returns value greater than zero + if (f > 0) + return 0; + } + + r = callback(node->network, data); + if (r) + return r; + } + + // Walk down on the left side of the tree first + if (node->zero) { + r = __loc_network_tree_walk(ctx, node->zero, filter_callback, callback, data); + if (r) + return r; + } + + // Then walk on the other side + if (node->one) { + r = __loc_network_tree_walk(ctx, node->one, filter_callback, callback, data); + if (r) + return r; + } + + return 0; +} + +int loc_network_tree_walk(struct loc_network_tree* tree, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data) { + return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); +} + +static void loc_network_tree_free(struct loc_network_tree* tree) { + DEBUG(tree->ctx, "Releasing network tree at %p\n", tree); + + loc_network_tree_node_unref(tree->root); + + loc_unref(tree->ctx); + free(tree); +} + +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { + if (--tree->refcount > 0) + return tree; + + loc_network_tree_free(tree); + return NULL; +} + +static int __loc_network_tree_dump(struct loc_network* network, void* data) { + DEBUG(network->ctx, "Dumping network at %p\n", network); + + const char* s = loc_network_str(network); + if (!s) + return 1; + + INFO(network->ctx, "%s\n", s); + + return 0; +} + +int loc_network_tree_dump(struct loc_network_tree* tree) { + DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); + + return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); +} + +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { + DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); + + struct loc_network_tree_node* node = loc_network_tree_get_path(tree, + &network->first_address, network->prefix); + if (!node) { + ERROR(tree->ctx, "Could not find a node\n"); + return -ENOMEM; + } + + // Check if node has not been set before + if (node->network) { + DEBUG(tree->ctx, "There is already a network at this path\n"); + return -EBUSY; + } + + // Point node to the network + node->network = loc_network_ref(network); + + return 0; +} + +static int __loc_network_tree_count(struct loc_network* network, void* data) { + size_t* counter = (size_t*)data; + + // Increase the counter for each network + counter++; + + return 0; +} + +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { + size_t counter = 0; + + int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); + if (r) + return r; + + return counter; +} + +static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) { + size_t counter = 1; + + if (node->zero) + counter += __loc_network_tree_count_nodes(node->zero); + + if (node->one) + counter += __loc_network_tree_count_nodes(node->one); + + return counter; +} + +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { + return __loc_network_tree_count_nodes(tree->root); +} + +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { + struct loc_network_tree_node* n = calloc(1, sizeof(*n)); + if (!n) + return -ENOMEM; + + n->ctx = loc_ref(ctx); + n->refcount = 1; + + n->zero = n->one = NULL; + + DEBUG(n->ctx, "Network node allocated at %p\n", n); + *node = n; + return 0; +} + +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { + if (node) + node->refcount++; + + return node; +} + +static void loc_network_tree_node_free(struct loc_network_tree_node* node) { + DEBUG(node->ctx, "Releasing network node at %p\n", node); + + if (node->network) + loc_network_unref(node->network); + + if (node->zero) + loc_network_tree_node_unref(node->zero); + + if (node->one) + loc_network_tree_node_unref(node->one); + + loc_unref(node->ctx); + free(node); +} + +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { + if (!node) + return NULL; + + if (--node->refcount > 0) + return node; + + loc_network_tree_node_free(node); + return NULL; +} + +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { + if (index == 0) + node = node->zero; + else + node = node->one; + + if (!node) + return NULL; + + return loc_network_tree_node_ref(node); +} + +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + return (!!node->network); +} + +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); +} diff --git a/src/perl/.gitignore b/src/perl/.gitignore new file mode 100644 index 0000000..b75ad14 --- /dev/null +++ b/src/perl/.gitignore @@ -0,0 +1,6 @@ +/Location.bs +/Location.c +/MYMETA.json +/MYMETA.yml +/blib +/pm_to_blib diff --git a/src/perl/Location.xs b/src/perl/Location.xs new file mode 100644 index 0000000..6f21f2b --- /dev/null +++ b/src/perl/Location.xs @@ -0,0 +1,321 @@ +#define PERL_NO_GET_CONTEXT +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" + +#include +#include + +#include +#include +#include +#include + +MODULE = Location PACKAGE = Location + +struct loc_database * +init(file) + char* file; + + CODE: + struct loc_ctx* ctx = NULL; + + // Initialise location context + int err = loc_new(&ctx); + if (err < 0) + croak("Could not initialize libloc context: %d\n", err); + + // Open the database file for reading + FILE* f = fopen(file, "r"); + if (!f) { + loc_unref(ctx); + + croak("Could not open file for reading: %s: %s\n", + file, strerror(errno)); + } + + // Parse the database + struct loc_database* db = NULL; + err = loc_database_new(ctx, &db, f); + + // We can close the database file straight away + // because loc_database_new creates a copy of the file descriptor + fclose(f); + + if (err) { + loc_unref(ctx); + + croak("Could not read database: %s\n", file); + } + + // Cleanup + loc_unref(ctx); + + RETVAL = db; + OUTPUT: + RETVAL + +# +# Database functions +# +bool +verify(db, keyfile) + struct loc_database* db; + char* keyfile; + + CODE: + // Try to open the keyfile + FILE* f = fopen(keyfile, "r"); + if (!f) { + croak("Could not open keyfile %s: %s\n", + keyfile, strerror(errno)); + } + + // Verify the database + int status = loc_database_verify(db, f); + if (status) { + RETVAL = false; + fclose(f); + + croak("Could not verify the database signature\n"); + } + + // Database was validated successfully + RETVAL = true; + + // Close the keyfile + fclose(f); + OUTPUT: + RETVAL + +const char* +get_vendor(db) + struct loc_database* db; + + CODE: + // Get vendor + RETVAL = loc_database_get_vendor(db); + OUTPUT: + RETVAL + +const char* +get_description(db) + struct loc_database* db; + + CODE: + // Get database description + RETVAL = loc_database_get_description(db); + OUTPUT: + RETVAL + +const char* +get_license(db) + struct loc_database* db; + + CODE: + // Get database license + RETVAL = loc_database_get_license(db); + OUTPUT: + RETVAL + +void +database_countries(db) + struct loc_database* db; + + PPCODE: + // Create Database enumerator + struct loc_database_enumerator* enumerator; + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); + + if (err) { + croak("Could not create a database enumerator\n"); + } + + // Init and enumerate first country. + struct loc_country* country; + err = loc_database_enumerator_next_country(enumerator, &country); + if (err) { + croak("Could not enumerate next country\n"); + } + + while (country) { + // Extract the country code. + const char* ccode = loc_country_get_code(country); + + // Push country code. + XPUSHs(sv_2mortal(newSVpv(ccode, 2))); + + // Unref country pointer. + loc_country_unref(country); + + // Enumerate next item. + err = loc_database_enumerator_next_country(enumerator, &country); + if (err) { + croak("Could not enumerate next country\n"); + } + } + + loc_database_enumerator_unref(enumerator); + +# +# Lookup functions +# +SV* +lookup_country_code(db, address) + struct loc_database* db; + char* address; + + CODE: + RETVAL = &PL_sv_undef; + + // Lookup network + struct loc_network *network; + int err = loc_database_lookup_from_string(db, address, &network); + if (!err) { + // Extract the country code + const char* country_code = loc_network_get_country_code(network); + RETVAL = newSVpv(country_code, strlen(country_code)); + + loc_network_unref(network); + } + OUTPUT: + RETVAL + +bool +lookup_network_has_flag(db, address, flag) + struct loc_database* db; + char* address; + char* flag; + + CODE: + RETVAL = false; + + enum loc_network_flags iv = 0; + + if (strcmp("LOC_NETWORK_FLAG_ANONYMOUS_PROXY", flag) == 0) + iv |= LOC_NETWORK_FLAG_ANONYMOUS_PROXY; + else if (strcmp("LOC_NETWORK_FLAG_SATELLITE_PROVIDER", flag) == 0) + iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER; + else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0) + iv |= LOC_NETWORK_FLAG_ANYCAST; + else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0) + iv |= LOC_NETWORK_FLAG_DROP; + else + croak("Invalid flag"); + + // Lookup network + struct loc_network *network; + int err = loc_database_lookup_from_string(db, address, &network); + + if (!err) { + // Check if the network has the given flag. + if (loc_network_has_flag(network, iv)) { + RETVAL = true; + } + + loc_network_unref(network); + } + + OUTPUT: + RETVAL + +SV* +lookup_asn(db, address) + struct loc_database* db; + char* address; + + CODE: + RETVAL = &PL_sv_undef; + + // Lookup network + struct loc_network *network; + int err = loc_database_lookup_from_string(db, address, &network); + if (!err) { + // Extract the ASN + unsigned int as_number = loc_network_get_asn(network); + if (as_number > 0) { + RETVAL = newSViv(as_number); + } + + loc_network_unref(network); + } + OUTPUT: + RETVAL + +# +# Get functions +# +SV* +get_country_name(db, ccode) + struct loc_database* db; + char* ccode; + + CODE: + RETVAL = &PL_sv_undef; + + // Lookup country code + struct loc_country *country; + int err = loc_database_get_country(db, &country, ccode); + if(!err) { + // Extract the name for the given country code. + const char* country_name = loc_country_get_name(country); + RETVAL = newSVpv(country_name, strlen(country_name)); + + loc_country_unref(country); + } + + OUTPUT: + RETVAL + +SV* +get_continent_code(db, ccode) + struct loc_database* db; + char* ccode; + + CODE: + RETVAL = &PL_sv_undef; + + // Lookup country code + struct loc_country *country; + int err = loc_database_get_country(db, &country, ccode); + if(!err) { + //Extract the continent code for the given country code. + const char* continent_code = loc_country_get_continent_code(country); + RETVAL = newSVpv(continent_code, strlen(continent_code)); + + loc_country_unref(country); + } + + OUTPUT: + RETVAL + +SV* +get_as_name(db, as_number) + struct loc_database* db; + unsigned int as_number; + + CODE: + RETVAL = &PL_sv_undef; + + // Lookup AS. + struct loc_as *as; + int err = loc_database_get_as(db, &as, as_number); + if(!err) { + // Get the name of the given AS number. + const char* as_name = loc_as_get_name(as); + + RETVAL = newSVpv(as_name, strlen(as_name)); + + loc_as_unref(as); + } + + OUTPUT: + RETVAL + +void +DESTROY(db) + struct loc_database* db; + + CODE: + // Close database + loc_database_unref(db); diff --git a/src/perl/MANIFEST b/src/perl/MANIFEST new file mode 100644 index 0000000..931285c --- /dev/null +++ b/src/perl/MANIFEST @@ -0,0 +1,6 @@ +Location.xs +Makefile.PL +MANIFEST +typemap +t/Location.t +lib/Location.pm diff --git a/src/perl/Makefile.PL b/src/perl/Makefile.PL new file mode 100644 index 0000000..38d5390 --- /dev/null +++ b/src/perl/Makefile.PL @@ -0,0 +1,16 @@ +use ExtUtils::MakeMaker; +# See lib/ExtUtils/MakeMaker.pm for details of how to influence +# the contents of the Makefile that is written. +WriteMakefile( + NAME => 'Location', + VERSION_FROM => 'lib/Location.pm', + PREREQ_PM => {}, + ABSTRACT_FROM => 'lib/Location.pm', + AUTHOR => 'Stefan Schantl ', + LICENSE => 'lgpl', + LIBS => ['-lloc'], + DEFINE => '', # e.g., '-DHAVE_SOMETHING' + INC => '-I. -I../../', + # Un-comment this if you add C files to link with later: + # OBJECT => '$(O_FILES)', # link all the C files too +); diff --git a/src/perl/lib/Location.pm b/src/perl/lib/Location.pm new file mode 100644 index 0000000..2782d53 --- /dev/null +++ b/src/perl/lib/Location.pm @@ -0,0 +1,69 @@ +package Location; + +use 5.028001; +use strict; +use warnings; + +require Exporter; + +our @ISA = qw(Exporter); + +# Items to export into callers namespace by default. Note: do not export +# names by default without a very good reason. Use EXPORT_OK instead. +# Do not simply export all your public functions/methods/constants. + +# This allows declaration use Location ':all'; +# If you do not need this, moving things directly into @EXPORT or @EXPORT_OK +# will save memory. +our %EXPORT_TAGS = ( 'all' => [ qw() ] ); + +our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); + +our @EXPORT = qw(); + +our $VERSION = '0.01'; + +require XSLoader; +XSLoader::load('Location', $VERSION); + +# Preloaded methods go here. + +1; +__END__ +# Below is stub documentation for your module. You'd better edit it! + +=head1 NAME + +Location - Provides a simple interface to libloc. + +=head1 SYNOPSIS + + use Location; + +=head1 DESCRIPTION + +Location is a simple interface to libloc - A library to determine someones +location on the Internet. (https://git.ipfire.org/?p=location/libloc.git;a=summary) + +=head2 EXPORT + +None by default. + +=head1 SEE ALSO + +https://git.ipfire.org/?p=location/libloc.git;a=summary + +=head1 AUTHOR + +Stefan Schantl, stefan.schantl@ipfire.org + +=head1 COPYRIGHT AND LICENSE + +Copyright (C) 2019 by Stefan Schantl + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself, either Perl version 5.28.1 or, +at your option, any later version of Perl 5 you may have available. + + +=cut diff --git a/src/perl/t/Location.t b/src/perl/t/Location.t new file mode 100644 index 0000000..e256aec --- /dev/null +++ b/src/perl/t/Location.t @@ -0,0 +1,73 @@ +# Before 'make install' is performed this script should be runnable with +# 'make test'. After 'make install' it should work as 'perl Location.t' + +######################### + +# change 'tests => 1' to 'tests => last_test_to_print'; + +use strict; +use warnings; + +# Where to find the test database. +my $testdb = $ENV{'database'}; +my $keyfile = $ENV{'keyfile'}; + +use Test::More tests => 12; +BEGIN { use_ok('Location') }; + +######################### + +# Insert your test code below, the Test::More module is use()ed here so read +# its man page ( perldoc Test::More ) for help writing this test script. + +# Address which should be used for database lookup. +my $address = "2a07:1c44:5800::1"; + +# Connect to the database. +my $db = &Location::init("$testdb"); + +# Verify +my $status = &Location::verify($db, $keyfile); +ok($status, "This database is valid"); + +my $vendor = &Location::get_vendor($db); +ok($vendor eq "IPFire Project", "Test 1 - Get Database Vendor"); + +my $license = &Location::get_license($db); +ok($license eq "CC", "Test 2 - Get Database license"); + +my $description = &Location::get_description($db); +ok($description eq "This is a geo location database", "Test 3 - Get Database Description"); + +my $country_code = &Location::lookup_country_code($db, $address); +ok($country_code eq "DE", "Test 4 - Lookup country code for $address"); + +$country_code = &Location::lookup_country_code($db, "1.1.1.1"); +if(defined($country_code)) { fail("Test 5 - Lookup country code for address not in Database."); } + +$country_code = &Location::lookup_country_code($db, "a.b.c.d"); +if(defined($country_code)) { fail("Test 6 - Lookup country code for invalid address.") } + +my $as_number = &Location::lookup_asn($db, $address); +ok($as_number eq "204867", "Test 7 - Lookup Autonomous System Number for $address."); + +$as_number = &Location::lookup_asn($db, "1.1.1.1"); +if(defined($as_number)) { fail("Test 8 - Lookup Autonomous System Number for address not in Database.") } + +$as_number = &Location::lookup_asn($db, "a.b.c.d"); +if(defined($as_number)) { fail("Test 9 - Lookup Autonomous System Number for invalid address.") } + +my $as_name = &Location::get_as_name($db, "204867"); +ok($as_name eq "Lightning Wire Labs GmbH", "Test 10 - Get name for AS204867."); + +my @locations = &Location::database_countries($db); +ok(@locations != 0, "Test 11 - Get database countries."); + +my $network_flag_anycast = &Location::lookup_network_has_flag($db, $address, "LOC_NETWORK_FLAG_ANYCAST"); +ok($network_flag_anycast, "Network has Anycast flag."); + +my $country_name = &Location::get_country_name($db, "DE"); +ok($country_name eq "Germany", "Test 13 - Got country name: $country_name"); + +my $continent_code = &Location::get_continent_code($db, "DE"); +ok($continent_code eq "EU", "Test 14 - Got continent code $continent_code for country code 'DE'"); diff --git a/src/perl/typemap b/src/perl/typemap new file mode 100644 index 0000000..c53fb53 --- /dev/null +++ b/src/perl/typemap @@ -0,0 +1,2 @@ +TYPEMAP +struct loc_database * T_PTRREF diff --git a/src/python/__init__.py.in b/src/python/__init__.py.in new file mode 100644 index 0000000..bd94d35 --- /dev/null +++ b/src/python/__init__.py.in @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +__version__ = "@VERSION@" + +# Import everything from the C module +from _location import * + +# Initialise logging +from . import logger diff --git a/src/python/as.c b/src/python/as.c new file mode 100644 index 0000000..4cf9987 --- /dev/null +++ b/src/python/as.c @@ -0,0 +1,159 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include + +#include +#include + +#include "locationmodule.h" +#include "as.h" + +PyObject* new_as(PyTypeObject* type, struct loc_as* as) { + ASObject* self = (ASObject*)type->tp_alloc(type, 0); + if (self) { + self->as = loc_as_ref(as); + } + + return (PyObject*)self; +} + +static PyObject* AS_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + ASObject* self = (ASObject*)type->tp_alloc(type, 0); + + return (PyObject*)self; +} + +static void AS_dealloc(ASObject* self) { + if (self->as) + loc_as_unref(self->as); + + Py_TYPE(self)->tp_free((PyObject* )self); +} + +static int AS_init(ASObject* self, PyObject* args, PyObject* kwargs) { + uint32_t number = 0; + + if (!PyArg_ParseTuple(args, "i", &number)) + return -1; + + // Create the AS object + int r = loc_as_new(loc_ctx, &self->as, number); + if (r) + return -1; + + return 0; +} + +static PyObject* AS_repr(ASObject* self) { + uint32_t number = loc_as_get_number(self->as); + const char* name = loc_as_get_name(self->as); + + if (name) + return PyUnicode_FromFormat("", number, name); + + return PyUnicode_FromFormat("", number); +} + +static PyObject* AS_str(ASObject* self) { + uint32_t number = loc_as_get_number(self->as); + const char* name = loc_as_get_name(self->as); + + if (name) + return PyUnicode_FromFormat("AS%d - %s", number, name); + + return PyUnicode_FromFormat("AS%d", number); +} + +static PyObject* AS_get_number(ASObject* self) { + uint32_t number = loc_as_get_number(self->as); + + return PyLong_FromLong(number); +} + +static PyObject* AS_get_name(ASObject* self) { + const char* name = loc_as_get_name(self->as); + + return PyUnicode_FromString(name); +} + +static int AS_set_name(ASObject* self, PyObject* value) { + const char* name = PyUnicode_AsUTF8(value); + + int r = loc_as_set_name(self->as, name); + if (r) { + PyErr_Format(PyExc_ValueError, "Could not set name: %s", name); + return r; + } + + return 0; +} + +static PyObject* AS_richcompare(ASObject* self, ASObject* other, int op) { + int r = loc_as_cmp(self->as, other->as); + + switch (op) { + case Py_EQ: + if (r == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; + + case Py_LT: + if (r < 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; + + default: + break; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static struct PyGetSetDef AS_getsetters[] = { + { + "name", + (getter)AS_get_name, + (setter)AS_set_name, + NULL, + NULL, + }, + { + "number", + (getter)AS_get_number, + NULL, + NULL, + NULL, + }, + { NULL }, +}; + +PyTypeObject ASType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "location.AS", + .tp_basicsize = sizeof(ASObject), + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_new = AS_new, + .tp_dealloc = (destructor)AS_dealloc, + .tp_init = (initproc)AS_init, + .tp_doc = "AS object", + .tp_getset = AS_getsetters, + .tp_repr = (reprfunc)AS_repr, + .tp_str = (reprfunc)AS_str, + .tp_richcompare = (richcmpfunc)AS_richcompare, +}; diff --git a/src/python/as.h b/src/python/as.h new file mode 100644 index 0000000..d7fe36a --- /dev/null +++ b/src/python/as.h @@ -0,0 +1,34 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef PYTHON_LOCATION_AS_H +#define PYTHON_LOCATION_AS_H + +#include + +#include +#include + +typedef struct { + PyObject_HEAD + struct loc_as* as; +} ASObject; + +extern PyTypeObject ASType; + +PyObject* new_as(PyTypeObject* type, struct loc_as* as); + +#endif /* PYTHON_LOCATION_AS_H */ diff --git a/src/python/country.c b/src/python/country.c new file mode 100644 index 0000000..4bb6a31 --- /dev/null +++ b/src/python/country.c @@ -0,0 +1,178 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include + +#include +#include + +#include "locationmodule.h" +#include "country.h" + +PyObject* new_country(PyTypeObject* type, struct loc_country* country) { + CountryObject* self = (CountryObject*)type->tp_alloc(type, 0); + if (self) { + self->country = loc_country_ref(country); + } + + return (PyObject*)self; +} + +static PyObject* Country_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + CountryObject* self = (CountryObject*)type->tp_alloc(type, 0); + + return (PyObject*)self; +} + +static void Country_dealloc(CountryObject* self) { + if (self->country) + loc_country_unref(self->country); + + Py_TYPE(self)->tp_free((PyObject* )self); +} + +static int Country_init(CountryObject* self, PyObject* args, PyObject* kwargs) { + const char* country_code = NULL; + + if (!PyArg_ParseTuple(args, "s", &country_code)) + return -1; + + // Create the country object + int r = loc_country_new(loc_ctx, &self->country, country_code); + if (r) + return -1; + + return 0; +} + +static PyObject* Country_repr(CountryObject* self) { + const char* code = loc_country_get_code(self->country); + const char* name = loc_country_get_name(self->country); + + if (name) + return PyUnicode_FromFormat("", code, name); + + return PyUnicode_FromFormat("", code); +} + +static PyObject* Country_get_code(CountryObject* self) { + const char* code = loc_country_get_code(self->country); + + return PyUnicode_FromString(code); +} + +static PyObject* Country_str(CountryObject* self) { + return Country_get_code(self); +} + +static PyObject* Country_get_name(CountryObject* self) { + const char* name = loc_country_get_name(self->country); + + return PyUnicode_FromString(name); +} + +static int Country_set_name(CountryObject* self, PyObject* value) { + const char* name = PyUnicode_AsUTF8(value); + + int r = loc_country_set_name(self->country, name); + if (r) { + PyErr_Format(PyExc_ValueError, "Could not set name: %s", name); + return r; + } + + return 0; +} + +static PyObject* Country_get_continent_code(CountryObject* self) { + const char* code = loc_country_get_continent_code(self->country); + + return PyUnicode_FromString(code); +} + +static int Country_set_continent_code(CountryObject* self, PyObject* value) { + const char* code = PyUnicode_AsUTF8(value); + + int r = loc_country_set_continent_code(self->country, code); + if (r) { + PyErr_Format(PyExc_ValueError, "Could not set continent code: %s", code); + return r; + } + + return 0; +} + +static PyObject* Country_richcompare(CountryObject* self, CountryObject* other, int op) { + int r = loc_country_cmp(self->country, other->country); + + switch (op) { + case Py_EQ: + if (r == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; + + case Py_LT: + if (r < 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; + + default: + break; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static struct PyGetSetDef Country_getsetters[] = { + { + "code", + (getter)Country_get_code, + NULL, + NULL, + NULL, + }, + { + "name", + (getter)Country_get_name, + (setter)Country_set_name, + NULL, + NULL, + }, + { + "continent_code", + (getter)Country_get_continent_code, + (setter)Country_set_continent_code, + NULL, + NULL, + }, + { NULL }, +}; + +PyTypeObject CountryType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "location.Country", + .tp_basicsize = sizeof(CountryObject), + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_new = Country_new, + .tp_dealloc = (destructor)Country_dealloc, + .tp_init = (initproc)Country_init, + .tp_doc = "Country object", + .tp_getset = Country_getsetters, + .tp_repr = (reprfunc)Country_repr, + .tp_str = (reprfunc)Country_str, + .tp_richcompare = (richcmpfunc)Country_richcompare, +}; diff --git a/src/python/country.h b/src/python/country.h new file mode 100644 index 0000000..346163c --- /dev/null +++ b/src/python/country.h @@ -0,0 +1,33 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef PYTHON_LOCATION_COUNTRY_H +#define PYTHON_LOCATION_COUNTRY_H + +#include + +#include + +typedef struct { + PyObject_HEAD + struct loc_country* country; +} CountryObject; + +extern PyTypeObject CountryType; + +PyObject* new_country(PyTypeObject* type, struct loc_country* country); + +#endif /* PYTHON_LOCATION_COUNTRY_H */ diff --git a/src/python/database.c b/src/python/database.c new file mode 100644 index 0000000..ca56748 --- /dev/null +++ b/src/python/database.c @@ -0,0 +1,637 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include + +#include +#include +#include +#include + +#include "locationmodule.h" +#include "as.h" +#include "country.h" +#include "database.h" +#include "network.h" + +static PyObject* Database_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + DatabaseObject* self = (DatabaseObject*)type->tp_alloc(type, 0); + + return (PyObject*)self; +} + +static void Database_dealloc(DatabaseObject* self) { + if (self->db) + loc_database_unref(self->db); + + if (self->path) + free(self->path); + + Py_TYPE(self)->tp_free((PyObject* )self); +} + +static int Database_init(DatabaseObject* self, PyObject* args, PyObject* kwargs) { + const char* path = NULL; + + if (!PyArg_ParseTuple(args, "s", &path)) + return -1; + + self->path = strdup(path); + + // Open the file for reading + FILE* f = fopen(self->path, "r"); + if (!f) { + PyErr_SetFromErrno(PyExc_IOError); + return -1; + } + + // Load the database + int r = loc_database_new(loc_ctx, &self->db, f); + fclose(f); + + // Return on any errors + if (r) + return -1; + + return 0; +} + +static PyObject* Database_repr(DatabaseObject* self) { + return PyUnicode_FromFormat("", self->path); +} + +static PyObject* Database_verify(DatabaseObject* self, PyObject* args) { + PyObject* public_key = NULL; + FILE* f = NULL; + + // Parse arguments + if (!PyArg_ParseTuple(args, "O", &public_key)) + return NULL; + + // Convert into FILE* + int fd = PyObject_AsFileDescriptor(public_key); + if (fd < 0) + return NULL; + + // Re-open file descriptor + f = fdopen(fd, "r"); + if (!f) { + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + + int r = loc_database_verify(self->db, f); + + if (r == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* Database_get_description(DatabaseObject* self) { + const char* description = loc_database_get_description(self->db); + + return PyUnicode_FromString(description); +} + +static PyObject* Database_get_vendor(DatabaseObject* self) { + const char* vendor = loc_database_get_vendor(self->db); + + return PyUnicode_FromString(vendor); +} + +static PyObject* Database_get_license(DatabaseObject* self) { + const char* license = loc_database_get_license(self->db); + + return PyUnicode_FromString(license); +} + +static PyObject* Database_get_created_at(DatabaseObject* self) { + time_t created_at = loc_database_created_at(self->db); + + return PyLong_FromLong(created_at); +} + +static PyObject* Database_get_as(DatabaseObject* self, PyObject* args) { + struct loc_as* as = NULL; + uint32_t number = 0; + + if (!PyArg_ParseTuple(args, "i", &number)) + return NULL; + + // Try to retrieve the AS + int r = loc_database_get_as(self->db, &as, number); + + // We got an AS + if (r == 0) { + PyObject* obj = new_as(&ASType, as); + loc_as_unref(as); + + return obj; + + // Nothing found + } else if (r == 1) { + Py_RETURN_NONE; + } + + // Unexpected error + return NULL; +} + +static PyObject* Database_get_country(DatabaseObject* self, PyObject* args) { + const char* country_code = NULL; + + if (!PyArg_ParseTuple(args, "s", &country_code)) + return NULL; + + struct loc_country* country; + int r = loc_database_get_country(self->db, &country, country_code); + if (r) { + Py_RETURN_NONE; + } + + PyObject* obj = new_country(&CountryType, country); + loc_country_unref(country); + + return obj; +} + +static PyObject* Database_lookup(DatabaseObject* self, PyObject* args) { + struct loc_network* network = NULL; + const char* address = NULL; + + if (!PyArg_ParseTuple(args, "s", &address)) + return NULL; + + // Try to retrieve a matching network + int r = loc_database_lookup_from_string(self->db, address, &network); + + // We got a network + if (r == 0) { + PyObject* obj = new_network(&NetworkType, network); + loc_network_unref(network); + + return obj; + + // Nothing found + } else if (r == 1) { + Py_RETURN_NONE; + + // Invalid input + } else if (r == -EINVAL) { + PyErr_Format(PyExc_ValueError, "Invalid IP address: %s", address); + return NULL; + } + + // Unexpected error + return NULL; +} + +static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database_enumerator* enumerator) { + DatabaseEnumeratorObject* self = (DatabaseEnumeratorObject*)type->tp_alloc(type, 0); + if (self) { + self->enumerator = loc_database_enumerator_ref(enumerator); + } + + return (PyObject*)self; +} + +static PyObject* Database_iterate_all(DatabaseObject* self, + enum loc_database_enumerator_mode what, int family, int flags) { + struct loc_database_enumerator* enumerator; + + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + + // Set family + if (family) + loc_database_enumerator_set_family(enumerator, family); + + PyObject* obj = new_database_enumerator(&DatabaseEnumeratorType, enumerator); + loc_database_enumerator_unref(enumerator); + + return obj; +} + +static PyObject* Database_ases(DatabaseObject* self) { + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, AF_UNSPEC, 0); +} + +static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + const char* string = NULL; + + if (!PyArg_ParseTuple(args, "s", &string)) + return NULL; + + struct loc_database_enumerator* enumerator; + + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + + // Search string we are searching for + loc_database_enumerator_set_string(enumerator, string); + + PyObject* obj = new_database_enumerator(&DatabaseEnumeratorType, enumerator); + loc_database_enumerator_unref(enumerator); + + return obj; +} + +static PyObject* Database_networks(DatabaseObject* self) { + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, AF_UNSPEC, 0); +} + +static PyObject* Database_networks_flattened(DatabaseObject *self) { + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, AF_UNSPEC, + LOC_DB_ENUMERATOR_FLAGS_FLATTEN); +} + +static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; + PyObject* country_codes = NULL; + PyObject* asn_list = NULL; + int flags = 0; + int family = 0; + int flatten = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + + // Set country code we are searching for + if (country_codes) { + struct loc_country_list* countries; + r = loc_country_list_new(loc_ctx, &countries); + if (r) { + PyErr_SetString(PyExc_SystemError, "Could not create country list"); + return NULL; + } + + for (int i = 0; i < PyList_Size(country_codes); i++) { + PyObject* item = PyList_GetItem(country_codes, i); + + if (!PyUnicode_Check(item)) { + PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); + loc_country_list_unref(countries); + return NULL; + } + + const char* country_code = PyUnicode_AsUTF8(item); + + struct loc_country* country; + r = loc_country_new(loc_ctx, &country, country_code); + if (r) { + if (r == -EINVAL) { + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); + } else { + PyErr_SetString(PyExc_SystemError, "Could not create country"); + } + + loc_country_list_unref(countries); + return NULL; + } + + // Append it to the list + r = loc_country_list_append(countries, country); + if (r) { + PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); + + loc_country_list_unref(countries); + loc_country_unref(country); + return NULL; + } + + loc_country_unref(country); + } + + r = loc_database_enumerator_set_countries(enumerator, countries); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + + loc_country_list_unref(countries); + return NULL; + } + + loc_country_list_unref(countries); + } + + // Set the ASN we are searching for + if (asn_list) { + struct loc_as_list* asns; + r = loc_as_list_new(loc_ctx, &asns); + if (r) { + PyErr_SetString(PyExc_SystemError, "Could not create AS list"); + return NULL; + } + + for (int i = 0; i < PyList_Size(asn_list); i++) { + PyObject* item = PyList_GetItem(asn_list, i); + + if (!PyLong_Check(item)) { + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); + + loc_as_list_unref(asns); + return NULL; + } + + unsigned long number = PyLong_AsLong(item); + + struct loc_as* as; + r = loc_as_new(loc_ctx, &as, number); + if (r) { + PyErr_SetString(PyExc_SystemError, "Could not create AS"); + + loc_as_list_unref(asns); + loc_as_unref(as); + return NULL; + } + + r = loc_as_list_append(asns, as); + if (r) { + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); + + loc_as_list_unref(asns); + loc_as_unref(as); + return NULL; + } + + loc_as_unref(as); + } + + r = loc_database_enumerator_set_asns(enumerator, asns); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + + loc_as_list_unref(asns); + return NULL; + } + + loc_as_list_unref(asns); + } + + // Set the flags we are searching for + if (flags) { + r = loc_database_enumerator_set_flag(enumerator, flags); + + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + } + + // Set the family we are searching for + if (family) { + r = loc_database_enumerator_set_family(enumerator, family); + + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + } + + PyObject* obj = new_database_enumerator(&DatabaseEnumeratorType, enumerator); + loc_database_enumerator_unref(enumerator); + + return obj; +} + +static PyObject* Database_countries(DatabaseObject* self) { + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, AF_UNSPEC, 0); +} + +static PyObject* Database_list_bogons(DatabaseObject* self, PyObject* args, PyObject* kwargs) { + char* kwlist[] = { "family", NULL }; + int family = AF_UNSPEC; + + // Parse arguments + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &family)) + return NULL; + + return Database_iterate_all(self, LOC_DB_ENUMERATE_BOGONS, family, 0); +} + +static struct PyMethodDef Database_methods[] = { + { + "get_as", + (PyCFunction)Database_get_as, + METH_VARARGS, + NULL, + }, + { + "get_country", + (PyCFunction)Database_get_country, + METH_VARARGS, + NULL, + }, + { + "list_bogons", + (PyCFunction)Database_list_bogons, + METH_VARARGS|METH_KEYWORDS, + NULL, + }, + { + "lookup", + (PyCFunction)Database_lookup, + METH_VARARGS, + NULL, + }, + { + "search_as", + (PyCFunction)Database_search_as, + METH_VARARGS, + NULL, + }, + { + "search_networks", + (PyCFunction)Database_search_networks, + METH_VARARGS|METH_KEYWORDS, + NULL, + }, + { + "verify", + (PyCFunction)Database_verify, + METH_VARARGS, + NULL, + }, + { NULL }, +}; + +static struct PyGetSetDef Database_getsetters[] = { + { + "ases", + (getter)Database_ases, + NULL, + NULL, + NULL, + }, + { + "countries", + (getter)Database_countries, + NULL, + NULL, + NULL, + }, + { + "created_at", + (getter)Database_get_created_at, + NULL, + NULL, + NULL, + }, + { + "description", + (getter)Database_get_description, + NULL, + NULL, + NULL, + }, + { + "license", + (getter)Database_get_license, + NULL, + NULL, + NULL, + }, + { + "networks", + (getter)Database_networks, + NULL, + NULL, + NULL, + }, + { + "networks_flattened", + (getter)Database_networks_flattened, + NULL, + NULL, + NULL, + }, + { + "vendor", + (getter)Database_get_vendor, + NULL, + NULL, + NULL, + }, + { NULL }, +}; + +PyTypeObject DatabaseType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "location.Database", + .tp_basicsize = sizeof(DatabaseObject), + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_new = Database_new, + .tp_dealloc = (destructor)Database_dealloc, + .tp_init = (initproc)Database_init, + .tp_doc = "Database object", + .tp_methods = Database_methods, + .tp_getset = Database_getsetters, + .tp_repr = (reprfunc)Database_repr, +}; + +static PyObject* DatabaseEnumerator_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + DatabaseEnumeratorObject* self = (DatabaseEnumeratorObject*)type->tp_alloc(type, 0); + + return (PyObject*)self; +} + +static void DatabaseEnumerator_dealloc(DatabaseEnumeratorObject* self) { + loc_database_enumerator_unref(self->enumerator); + + Py_TYPE(self)->tp_free((PyObject* )self); +} + +static PyObject* DatabaseEnumerator_next(DatabaseEnumeratorObject* self) { + struct loc_network* network = NULL; + + // Enumerate all networks + int r = loc_database_enumerator_next_network(self->enumerator, &network); + if (r) { + PyErr_SetFromErrno(PyExc_ValueError); + return NULL; + } + + // A network was found + if (network) { + PyObject* obj = new_network(&NetworkType, network); + loc_network_unref(network); + + return obj; + } + + // Enumerate all ASes + struct loc_as* as = NULL; + + r = loc_database_enumerator_next_as(self->enumerator, &as); + if (r) { + PyErr_SetFromErrno(PyExc_ValueError); + return NULL; + } + + if (as) { + PyObject* obj = new_as(&ASType, as); + loc_as_unref(as); + + return obj; + } + + // Enumerate all countries + struct loc_country* country = NULL; + + r = loc_database_enumerator_next_country(self->enumerator, &country); + if (r) { + PyErr_SetFromErrno(PyExc_ValueError); + return NULL; + } + + if (country) { + PyObject* obj = new_country(&CountryType, country); + loc_country_unref(country); + + return obj; + } + + // Nothing found, that means the end + PyErr_SetNone(PyExc_StopIteration); + return NULL; +} + +PyTypeObject DatabaseEnumeratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "location.DatabaseEnumerator", + .tp_basicsize = sizeof(DatabaseEnumeratorObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_alloc = PyType_GenericAlloc, + .tp_new = DatabaseEnumerator_new, + .tp_dealloc = (destructor)DatabaseEnumerator_dealloc, + .tp_iter = PyObject_SelfIter, + .tp_iternext = (iternextfunc)DatabaseEnumerator_next, +}; diff --git a/src/python/database.h b/src/python/database.h new file mode 100644 index 0000000..88b839b --- /dev/null +++ b/src/python/database.h @@ -0,0 +1,39 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef PYTHON_LOCATION_DATABASE_H +#define PYTHON_LOCATION_DATABASE_H + +#include + +#include + +typedef struct { + PyObject_HEAD + struct loc_database* db; + char* path; +} DatabaseObject; + +extern PyTypeObject DatabaseType; + +typedef struct { + PyObject_HEAD + struct loc_database_enumerator* enumerator; +} DatabaseEnumeratorObject; + +extern PyTypeObject DatabaseEnumeratorType; + +#endif /* PYTHON_LOCATION_DATABASE_H */ diff --git a/src/python/database.py b/src/python/database.py new file mode 100644 index 0000000..5d79941 --- /dev/null +++ b/src/python/database.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python + +""" + A lightweight wrapper around psycopg2. + + Originally part of the Tornado framework. The tornado.database module + is slated for removal in Tornado 3.0, and it is now available separately + as torndb. +""" + +import logging +import psycopg2 + +log = logging.getLogger("location.database") +log.propagate = 1 + +class Connection(object): + """ + A lightweight wrapper around MySQLdb DB-API connections. + + The main value we provide is wrapping rows in a dict/object so that + columns can be accessed by name. Typical usage:: + + db = torndb.Connection("localhost", "mydatabase") + for article in db.query("SELECT * FROM articles"): + print article.title + + Cursors are hidden by the implementation, but other than that, the methods + are very similar to the DB-API. + + We explicitly set the timezone to UTC and the character encoding to + UTF-8 on all connections to avoid time zone and encoding errors. + """ + def __init__(self, host, database, user=None, password=None): + self.host = host + self.database = database + + self._db = None + self._db_args = { + "host" : host, + "database" : database, + "user" : user, + "password" : password, + "sslmode" : "require", + } + + try: + self.reconnect() + except Exception: + log.error("Cannot connect to database on %s", self.host, exc_info=True) + + def __del__(self): + self.close() + + def close(self): + """ + Closes this database connection. + """ + if getattr(self, "_db", None) is not None: + self._db.close() + self._db = None + + def reconnect(self): + """ + Closes the existing database connection and re-opens it. + """ + self.close() + + self._db = psycopg2.connect(**self._db_args) + self._db.autocommit = True + + # Initialize the timezone setting. + self.execute("SET TIMEZONE TO 'UTC'") + + def query(self, query, *parameters, **kwparameters): + """ + Returns a row list for the given query and parameters. + """ + cursor = self._cursor() + try: + self._execute(cursor, query, parameters, kwparameters) + column_names = [d[0] for d in cursor.description] + return [Row(zip(column_names, row)) for row in cursor] + finally: + cursor.close() + + def get(self, query, *parameters, **kwparameters): + """ + Returns the first row returned for the given query. + """ + rows = self.query(query, *parameters, **kwparameters) + if not rows: + return None + elif len(rows) > 1: + raise Exception("Multiple rows returned for Database.get() query") + else: + return rows[0] + + def execute(self, query, *parameters, **kwparameters): + """ + Executes the given query, returning the lastrowid from the query. + """ + return self.execute_lastrowid(query, *parameters, **kwparameters) + + def execute_lastrowid(self, query, *parameters, **kwparameters): + """ + Executes the given query, returning the lastrowid from the query. + """ + cursor = self._cursor() + try: + self._execute(cursor, query, parameters, kwparameters) + return cursor.lastrowid + finally: + cursor.close() + + def execute_rowcount(self, query, *parameters, **kwparameters): + """ + Executes the given query, returning the rowcount from the query. + """ + cursor = self._cursor() + try: + self._execute(cursor, query, parameters, kwparameters) + return cursor.rowcount + finally: + cursor.close() + + def executemany(self, query, parameters): + """ + Executes the given query against all the given param sequences. + + We return the lastrowid from the query. + """ + return self.executemany_lastrowid(query, parameters) + + def executemany_lastrowid(self, query, parameters): + """ + Executes the given query against all the given param sequences. + + We return the lastrowid from the query. + """ + cursor = self._cursor() + try: + cursor.executemany(query, parameters) + return cursor.lastrowid + finally: + cursor.close() + + def executemany_rowcount(self, query, parameters): + """ + Executes the given query against all the given param sequences. + + We return the rowcount from the query. + """ + cursor = self._cursor() + + try: + cursor.executemany(query, parameters) + return cursor.rowcount + finally: + cursor.close() + + def _ensure_connected(self): + if self._db is None: + log.warning("Database connection was lost...") + + self.reconnect() + + def _cursor(self): + self._ensure_connected() + return self._db.cursor() + + def _execute(self, cursor, query, parameters, kwparameters): + log.debug("SQL Query: %s" % (query % (kwparameters or parameters))) + + try: + return cursor.execute(query, kwparameters or parameters) + except (OperationalError, psycopg2.ProgrammingError): + log.error("Error connecting to database on %s", self.host) + self.close() + raise + + def transaction(self): + return Transaction(self) + + +class Row(dict): + """A dict that allows for object-like property access syntax.""" + def __getattr__(self, name): + try: + return self[name] + except KeyError: + raise AttributeError(name) + + +class Transaction(object): + def __init__(self, db): + self.db = db + + self.db.execute("START TRANSACTION") + + def __enter__(self): + return self + + def __exit__(self, exctype, excvalue, traceback): + if exctype is not None: + self.db.execute("ROLLBACK") + else: + self.db.execute("COMMIT") + + +# Alias some common exceptions +IntegrityError = psycopg2.IntegrityError +OperationalError = psycopg2.OperationalError diff --git a/src/python/downloader.py b/src/python/downloader.py new file mode 100644 index 0000000..05f7872 --- /dev/null +++ b/src/python/downloader.py @@ -0,0 +1,211 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import logging +import lzma +import os +import random +import stat +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +from . import __version__ +from _location import Database, DATABASE_VERSION_LATEST + +DATABASE_FILENAME = "location.db.xz" +MIRRORS = ( + "https://location.ipfire.org/databases/", +) + +# Initialise logging +log = logging.getLogger("location.downloader") +log.propagate = 1 + +class Downloader(object): + def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None): + self.version = version + + # Set mirrors or use defaults + self.mirrors = list(mirrors or MIRRORS) + + # Randomize mirrors + random.shuffle(self.mirrors) + + # Get proxies from environment + self.proxies = self._get_proxies() + + def _get_proxies(self): + proxies = {} + + for protocol in ("https", "http"): + proxy = os.environ.get("%s_proxy" % protocol, None) + + if proxy: + proxies[protocol] = proxy + + return proxies + + def _make_request(self, url, baseurl=None, headers={}): + if baseurl: + url = urllib.parse.urljoin(baseurl, url) + + req = urllib.request.Request(url, method="GET") + + # Update headers + headers.update({ + "User-Agent" : "location/%s" % __version__, + }) + + # Set headers + for header in headers: + req.add_header(header, headers[header]) + + # Set proxies + for protocol in self.proxies: + req.set_proxy(self.proxies[protocol], protocol) + + return req + + def _send_request(self, req, **kwargs): + # Log request headers + log.debug("HTTP %s Request to %s" % (req.method, req.host)) + log.debug(" URL: %s" % req.full_url) + log.debug(" Headers:") + for k, v in req.header_items(): + log.debug(" %s: %s" % (k, v)) + + try: + res = urllib.request.urlopen(req, **kwargs) + + except urllib.error.HTTPError as e: + # Log response headers + log.debug("HTTP Response: %s" % e.code) + log.debug(" Headers:") + for header in e.headers: + log.debug(" %s: %s" % (header, e.headers[header])) + + # Raise all other errors + raise e + + # Log response headers + log.debug("HTTP Response: %s" % res.code) + log.debug(" Headers:") + for k, v in res.getheaders(): + log.debug(" %s: %s" % (k, v)) + + return res + + def download(self, public_key, timestamp=None, tmpdir=None, **kwargs): + url = "%s/%s" % (self.version, DATABASE_FILENAME) + + headers = {} + if timestamp: + headers["If-Modified-Since"] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), + ) + + t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) + with t: + # Try all mirrors + for mirror in self.mirrors: + # Prepare HTTP request + req = self._make_request(url, baseurl=mirror, headers=headers) + + try: + with self._send_request(req) as res: + decompressor = lzma.LZMADecompressor() + + # Read all data + while True: + buf = res.read(1024) + if not buf: + break + + # Decompress data + buf = decompressor.decompress(buf) + if buf: + t.write(buf) + + # Write all data to disk + t.flush() + + # Catch decompression errors + except lzma.LZMAError as e: + log.warning("Could not decompress downloaded file: %s" % e) + continue + + except urllib.error.HTTPError as e: + # The file on the server was too old + if e.code == 304: + log.warning("%s is serving an outdated database. Trying next mirror..." % mirror) + + # Log any other HTTP errors + else: + log.warning("%s reported: %s" % (mirror, e)) + + # Throw away any downloaded content and try again + t.truncate() + + else: + # Check if the downloaded database is recent + if not self._check_database(t, public_key, timestamp): + log.warning("Downloaded database is outdated. Trying next mirror...") + + # Throw away the data and try again + t.truncate() + continue + + # Make the file readable for everyone + os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH) + + # Return temporary file + return t + + # Delete the temporary file after unsuccessful downloads + os.unlink(t.name) + + raise FileNotFoundError(url) + + def _check_database(self, f, public_key, timestamp=None): + """ + Checks the downloaded database if it can be opened, + verified and if it is recent enough + """ + log.debug("Opening downloaded database at %s" % f.name) + + db = Database(f.name) + + # Database is not recent + if timestamp and db.created_at < timestamp: + return False + + log.info("Downloaded new database from %s" % (time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at), + ))) + + # Verify the database + with open(public_key, "r") as f: + if not db.verify(f): + log.error("Could not verify database") + return False + + return True diff --git a/src/python/export.py b/src/python/export.py new file mode 100644 index 0000000..3cdece4 --- /dev/null +++ b/src/python/export.py @@ -0,0 +1,291 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020-2021 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import functools +import io +import ipaddress +import logging +import math +import os +import socket +import sys + +from .i18n import _ +import _location + +# Initialise logging +log = logging.getLogger("location.export") +log.propagate = 1 + +FLAGS = { + _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", + _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", + _location.NETWORK_FLAG_ANYCAST : "A3", + _location.NETWORK_FLAG_DROP : "XD", +} + +class OutputWriter(object): + suffix = "networks" + mode = "w" + + def __init__(self, name, family=None, directory=None, f=None): + self.name = name + self.family = family + self.directory = directory + + # Open output file + if f: + self.f = f + elif self.directory: + self.f = open(self.filename, self.mode) + elif "b" in self.mode: + self.f = io.BytesIO() + else: + self.f = io.StringIO() + + # Call any custom initialization + self.init() + + # Immediately write the header + self._write_header() + + def init(self): + """ + To be overwritten by anything that inherits from this + """ + pass + + def __repr__(self): + return "<%s %s f=%s>" % (self.__class__.__name__, self, self.f) + + @functools.cached_property + def tag(self): + families = { + socket.AF_INET6 : "6", + socket.AF_INET : "4", + } + + return "%sv%s" % (self.name, families.get(self.family, "?")) + + @functools.cached_property + def filename(self): + if self.directory: + return os.path.join(self.directory, "%s.%s" % (self.tag, self.suffix)) + + def _write_header(self): + """ + The header of the file + """ + pass + + def _write_footer(self): + """ + The footer of the file + """ + pass + + def write(self, network): + self.f.write("%s\n" % network) + + def finish(self): + """ + Called when all data has been written + """ + self._write_footer() + + # Flush all output + self.f.flush() + + def print(self): + """ + Prints the entire output line by line + """ + if isinstance(self.f, io.BytesIO): + raise TypeError(_("Won't write binary output to stdout")) + + # Go back to the beginning + self.f.seek(0) + + # Iterate over everything line by line + for line in self.f: + sys.stdout.write(line) + + +class IpsetOutputWriter(OutputWriter): + """ + For ipset + """ + suffix = "ipset" + + # The value is being used if we don't know any better + DEFAULT_HASHSIZE = 64 + + # We aim for this many networks in a bucket on average. This allows us to choose + # how much memory we want to sacrifice to gain better performance. The lower the + # factor, the faster a lookup will be, but it will use more memory. + # We will aim for only using three quarters of all buckets to avoid any searches + # through the linked lists. + HASHSIZE_FACTOR = 0.75 + + def init(self): + # Count all networks + self.networks = 0 + + @property + def hashsize(self): + """ + Calculates an optimized hashsize + """ + # Return the default value if we don't know the size of the set + if not self.networks: + return self.DEFAULT_HASHSIZE + + # Find the nearest power of two that is larger than the number of networks + # divided by the hashsize factor. + exponent = math.log(self.networks / self.HASHSIZE_FACTOR, 2) + + # Return the size of the hash (the minimum is 64) + return max(2 ** math.ceil(exponent), 64) + + def _write_header(self): + # This must have a fixed size, because we will write the header again in the end + self.f.write("create %s hash:net family inet%s" % ( + self.tag, + "6" if self.family == socket.AF_INET6 else "" + )) + self.f.write(" hashsize %8d maxelem 1048576 -exist\n" % self.hashsize) + self.f.write("flush %s\n" % self.tag) + + def write(self, network): + self.f.write("add %s %s\n" % (self.tag, network)) + + # Increment network counter + self.networks += 1 + + def _write_footer(self): + # Jump back to the beginning of the file + self.f.seek(0) + + # Rewrite the header with better configuration + self._write_header() + + +class NftablesOutputWriter(OutputWriter): + """ + For nftables + """ + suffix = "set" + + def _write_header(self): + self.f.write("define %s = {\n" % self.tag) + + def _write_footer(self): + self.f.write("}\n") + + def write(self, network): + self.f.write(" %s,\n" % network) + + +class XTGeoIPOutputWriter(OutputWriter): + """ + Formats the output in that way, that it can be loaded by + the xt_geoip kernel module from xtables-addons. + """ + mode = "wb" + + @property + def tag(self): + return self.name + + @property + def suffix(self): + return "iv%s" % ("6" if self.family == socket.AF_INET6 else "4") + + def write(self, network): + self.f.write(network._first_address) + self.f.write(network._last_address) + + +formats = { + "ipset" : IpsetOutputWriter, + "list" : OutputWriter, + "nftables" : NftablesOutputWriter, + "xt_geoip" : XTGeoIPOutputWriter, +} + +class Exporter(object): + def __init__(self, db, writer): + self.db, self.writer = db, writer + + def export(self, directory, families, countries, asns): + for family in families: + log.debug("Exporting family %s" % family) + + writers = {} + + # Create writers for countries + for country_code in countries: + writers[country_code] = self.writer(country_code, family=family, directory=directory) + + # Create writers for ASNs + for asn in asns: + writers[asn] = self.writer("AS%s" % asn, family=family, directory=directory) + + # Filter countries from special country codes + country_codes = [ + country_code for country_code in countries if not country_code in FLAGS.values() + ] + + # Get all networks that match the family + networks = self.db.search_networks(family=family, + country_codes=country_codes, asns=asns, flatten=True) + + # Walk through all networks + for network in networks: + # Write matching countries + try: + writers[network.country_code].write(network) + except KeyError: + pass + + # Write matching ASNs + try: + writers[network.asn].write(network) + except KeyError: + pass + + # Handle flags + for flag in FLAGS: + if network.has_flag(flag): + # Fetch the "fake" country code + country = FLAGS[flag] + + try: + writers[country].write(network) + except KeyError: + pass + + # Write everything to the filesystem + for writer in writers.values(): + writer.finish() + + # Print to stdout + if not directory: + for writer in writers.values(): + writer.print() diff --git a/src/python/i18n.py b/src/python/i18n.py new file mode 100644 index 0000000..2161aa6 --- /dev/null +++ b/src/python/i18n.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import gettext + +def _(singular, plural=None, n=None): + if plural: + return gettext.dngettext("libloc", singular, plural, n) + + return gettext.dgettext("libloc", singular) diff --git a/src/python/importer.py b/src/python/importer.py new file mode 100644 index 0000000..dee36ed --- /dev/null +++ b/src/python/importer.py @@ -0,0 +1,250 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import gzip +import logging +import urllib.request + +# Initialise logging +log = logging.getLogger("location.importer") +log.propagate = 1 + +WHOIS_SOURCES = { + # African Network Information Centre + "AFRINIC": [ + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz" + ], + + # Asia Pacific Network Information Centre + "APNIC": [ + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz" + ], + + # American Registry for Internet Numbers + # XXX there is nothing useful for us in here + # ARIN: [ + # "https://ftp.arin.net/pub/rr/arin.db" + # ], + + # Japan Network Information Center + "JPNIC": [ + "https://ftp.nic.ad.jp/jpirr/jpirr.db.gz" + ], + + # Latin America and Caribbean Network Information Centre + "LACNIC": [ + "https://ftp.lacnic.net/lacnic/dbase/lacnic.db.gz" + ], + + # Réseaux IP Européens + "RIPE": [ + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz" + ], +} + +EXTENDED_SOURCES = { + # African Network Information Centre + # "ARIN": [ + # "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest" + # ], + + # Asia Pacific Network Information Centre + # "APNIC": [ + # "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest" + # ], + + # American Registry for Internet Numbers + "ARIN": [ + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest" + ], + + # Latin America and Caribbean Network Information Centre + "LACNIC": [ + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest" + ], + + # Réseaux IP Européens + # "RIPE": [ + # "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest" + # ], +} + +class Downloader(object): + def __init__(self): + self.proxy = None + + def set_proxy(self, url): + """ + Sets a HTTP proxy that is used to perform all requests + """ + log.info("Using proxy %s" % url) + self.proxy = url + + def request(self, url, data=None, return_blocks=False): + req = urllib.request.Request(url, data=data) + + # Configure proxy + if self.proxy: + req.set_proxy(self.proxy, "http") + + return DownloaderContext(self, req, return_blocks=return_blocks) + + +class DownloaderContext(object): + def __init__(self, downloader, request, return_blocks=False): + self.downloader = downloader + self.request = request + + # Should we return one block or a single line? + self.return_blocks = return_blocks + + # Save the response object + self.response = None + + def __enter__(self): + log.info("Retrieving %s..." % self.request.full_url) + + # Send request + self.response = urllib.request.urlopen(self.request) + + # Log the response headers + log.debug("Response Headers:") + for header in self.headers: + log.debug(" %s: %s" % (header, self.get_header(header))) + + return self + + def __exit__(self, type, value, traceback): + pass + + def __iter__(self): + """ + Makes the object iterable by going through each block + """ + if self.return_blocks: + return iterate_over_blocks(self.body) + + return iterate_over_lines(self.body) + + @property + def headers(self): + if self.response: + return self.response.headers + + def get_header(self, name): + if self.headers: + return self.headers.get(name) + + @property + def body(self): + """ + Returns a file-like object with the decoded content + of the response. + """ + content_type = self.get_header("Content-Type") + + # Decompress any gzipped response on the fly + if content_type in ("application/x-gzip", "application/gzip"): + return gzip.GzipFile(fileobj=self.response, mode="rb") + + # Return the response by default + return self.response + + +def read_blocks(f): + for block in iterate_over_blocks(f): + type = None + data = {} + + for i, line in enumerate(block): + key, value = line.split(":", 1) + + # The key of the first line defines the type + if i == 0: + type = key + + # Store value + data[key] = value.strip() + + yield type, data + +def iterate_over_blocks(f, charsets=("utf-8", "latin1")): + block = [] + + for line in f: + # Convert to string + for charset in charsets: + try: + line = line.decode(charset) + except UnicodeDecodeError: + continue + else: + break + + # Skip commented lines + if line.startswith("#") or line.startswith("%"): + continue + + # Strip line-endings + line = line.rstrip() + + # Remove any comments at the end of line + line, hash, comment = line.partition("#") + + if comment: + # Strip any whitespace before the comment + line = line.rstrip() + + # If the line is now empty, we move on + if not line: + continue + + if line: + block.append(line) + continue + + # End the block on an empty line + if block: + yield block + + # Reset the block + block = [] + + # Return the last block + if block: + yield block + + +def iterate_over_lines(f): + for line in f: + # Decode the line + line = line.decode() + + # Strip the ending + yield line.rstrip() diff --git a/src/python/location-importer.in b/src/python/location-importer.in new file mode 100644 index 0000000..bee9186 --- /dev/null +++ b/src/python/location-importer.in @@ -0,0 +1,1535 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020-2022 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import argparse +import ipaddress +import json +import logging +import math +import re +import socket +import sys +import telnetlib + +# Load our location module +import location +import location.database +import location.importer +from location.i18n import _ + +# Initialise logging +log = logging.getLogger("location.importer") +log.propagate = 1 + +# Define constants +VALID_ASN_RANGES = ( + (1, 23455), + (23457, 64495), + (131072, 4199999999), +) + + +class CLI(object): + def parse_cli(self): + parser = argparse.ArgumentParser( + description=_("Location Importer Command Line Interface"), + ) + subparsers = parser.add_subparsers() + + # Global configuration flags + parser.add_argument("--debug", action="store_true", + help=_("Enable debug output")) + parser.add_argument("--quiet", action="store_true", + help=_("Enable quiet mode")) + + # version + parser.add_argument("--version", action="version", + version="%(prog)s @VERSION@") + + # Database + parser.add_argument("--database-host", required=True, + help=_("Database Hostname"), metavar=_("HOST")) + parser.add_argument("--database-name", required=True, + help=_("Database Name"), metavar=_("NAME")) + parser.add_argument("--database-username", required=True, + help=_("Database Username"), metavar=_("USERNAME")) + parser.add_argument("--database-password", required=True, + help=_("Database Password"), metavar=_("PASSWORD")) + + # Write Database + write = subparsers.add_parser("write", help=_("Write database to file")) + write.set_defaults(func=self.handle_write) + write.add_argument("file", nargs=1, help=_("Database File")) + write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key")) + write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key")) + write.add_argument("--vendor", nargs="?", help=_("Sets the vendor")) + write.add_argument("--description", nargs="?", help=_("Sets a description")) + write.add_argument("--license", nargs="?", help=_("Sets the license")) + write.add_argument("--version", type=int, help=_("Database Format Version")) + + # Update WHOIS + update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information")) + update_whois.set_defaults(func=self.handle_update_whois) + + # Update announcements + update_announcements = subparsers.add_parser("update-announcements", + help=_("Update BGP Annoucements")) + update_announcements.set_defaults(func=self.handle_update_announcements) + update_announcements.add_argument("server", nargs=1, + help=_("Route Server to connect to"), metavar=_("SERVER")) + + # Update overrides + update_overrides = subparsers.add_parser("update-overrides", + help=_("Update overrides"), + ) + update_overrides.add_argument( + "files", nargs="+", help=_("Files to import"), + ) + update_overrides.set_defaults(func=self.handle_update_overrides) + + # Import countries + import_countries = subparsers.add_parser("import-countries", + help=_("Import countries"), + ) + import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"), + help=_("File to import")) + import_countries.set_defaults(func=self.handle_import_countries) + + args = parser.parse_args() + + # Configure logging + if args.debug: + location.logger.set_level(logging.DEBUG) + elif args.quiet: + location.logger.set_level(logging.WARNING) + + # Print usage if no action was given + if not "func" in args: + parser.print_usage() + sys.exit(2) + + return args + + def run(self): + # Parse command line arguments + args = self.parse_cli() + + # Initialise database + self.db = self._setup_database(args) + + # Call function + ret = args.func(args) + + # Return with exit code + if ret: + sys.exit(ret) + + # Otherwise just exit + sys.exit(0) + + def _setup_database(self, ns): + """ + Initialise the database + """ + # Connect to database + db = location.database.Connection( + host=ns.database_host, database=ns.database_name, + user=ns.database_username, password=ns.database_password, + ) + + with db.transaction(): + db.execute(""" + -- announcements + CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint, + first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); + CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); + CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); + + -- autnums + CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); + ALTER TABLE autnums ADD COLUMN IF NOT EXISTS source text; + CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number); + + -- countries + CREATE TABLE IF NOT EXISTS countries( + country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL); + CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code); + + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + ALTER TABLE networks ADD COLUMN IF NOT EXISTS original_countries text[]; + ALTER TABLE networks ADD COLUMN IF NOT EXISTS source text; + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides + CREATE TABLE IF NOT EXISTS autnum_overrides( + number bigint NOT NULL, + name text, + country text, + is_anonymous_proxy boolean, + is_satellite_provider boolean, + is_anycast boolean + ); + CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number + ON autnum_overrides(number); + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS source text; + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; + + CREATE TABLE IF NOT EXISTS network_overrides( + network inet NOT NULL, + country text, + is_anonymous_proxy boolean, + is_satellite_provider boolean, + is_anycast boolean + ); + CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network + ON network_overrides(network); + CREATE INDEX IF NOT EXISTS network_overrides_search + ON network_overrides USING GIST(network inet_ops); + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS source text; + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; + """) + + return db + + def handle_write(self, ns): + """ + Compiles a database in libloc format out of what is in the database + """ + # Allocate a writer + writer = location.Writer(ns.signing_key, ns.backup_signing_key) + + # Set all metadata + if ns.vendor: + writer.vendor = ns.vendor + + if ns.description: + writer.description = ns.description + + if ns.license: + writer.license = ns.license + + # Add all Autonomous Systems + log.info("Writing Autonomous Systems...") + + # Select all ASes with a name + rows = self.db.query(""" + SELECT + autnums.number AS number, + COALESCE( + (SELECT overrides.name FROM autnum_overrides overrides + WHERE overrides.number = autnums.number), + autnums.name + ) AS name + FROM autnums + WHERE name <> %s ORDER BY number + """, "") + + for row in rows: + a = writer.add_as(row.number) + a.name = row.name + + # Add all networks + log.info("Writing networks...") + + # Select all known networks + rows = self.db.query(""" + WITH known_networks AS ( + SELECT network FROM announcements + UNION + SELECT network FROM networks + UNION + SELECT network FROM network_overrides + ), + + ordered_networks AS ( + SELECT + known_networks.network AS network, + announcements.autnum AS autnum, + networks.country AS country, + + -- Must be part of returned values for ORDER BY clause + masklen(announcements.network) AS sort_a, + masklen(networks.network) AS sort_b + FROM + known_networks + LEFT JOIN + announcements ON known_networks.network <<= announcements.network + LEFT JOIN + networks ON known_networks.network <<= networks.network + ORDER BY + known_networks.network, + sort_a DESC, + sort_b DESC + ) + + -- Return a list of those networks enriched with all + -- other information that we store in the database + SELECT + DISTINCT ON (network) + network, + autnum, + + -- Country + COALESCE( + ( + SELECT country FROM network_overrides overrides + WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT country FROM autnum_overrides overrides + WHERE networks.autnum = overrides.number + ), + networks.country + ) AS country, + + -- Flags + COALESCE( + ( + SELECT is_anonymous_proxy FROM network_overrides overrides + WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anonymous_proxy FROM autnum_overrides overrides + WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_anonymous_proxy, + COALESCE( + ( + SELECT is_satellite_provider FROM network_overrides overrides + WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_satellite_provider FROM autnum_overrides overrides + WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_satellite_provider, + COALESCE( + ( + SELECT is_anycast FROM network_overrides overrides + WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anycast FROM autnum_overrides overrides + WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_anycast, + COALESCE( + ( + SELECT is_drop FROM network_overrides overrides + WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_drop FROM autnum_overrides overrides + WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_drop + FROM + ordered_networks networks + """) + + for row in rows: + network = writer.add_network(row.network) + + # Save country + if row.country: + network.country_code = row.country + + # Save ASN + if row.autnum: + network.asn = row.autnum + + # Set flags + if row.is_anonymous_proxy: + network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY) + + if row.is_satellite_provider: + network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER) + + if row.is_anycast: + network.set_flag(location.NETWORK_FLAG_ANYCAST) + + if row.is_drop: + network.set_flag(location.NETWORK_FLAG_DROP) + + # Add all countries + log.info("Writing countries...") + rows = self.db.query("SELECT * FROM countries ORDER BY country_code") + + for row in rows: + c = writer.add_country(row.country_code) + c.continent_code = row.continent_code + c.name = row.name + + # Write everything to file + log.info("Writing database to file...") + for file in ns.file: + writer.write(file) + + def handle_update_whois(self, ns): + downloader = location.importer.Downloader() + + # Download all sources + with self.db.transaction(): + # Create some temporary tables to store parsed data + self.db.execute(""" + CREATE TEMPORARY TABLE _autnums(number integer NOT NULL, organization text NOT NULL, source text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _autnums_number ON _autnums(number); + + CREATE TEMPORARY TABLE _organizations(handle text NOT NULL, name text NOT NULL, source text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); + + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, original_countries text[] NOT NULL, source text NOT NULL) + ON COMMIT DROP; + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); + """) + + # Remove all previously imported content + self.db.execute(""" + TRUNCATE TABLE networks; + """) + + # Fetch all valid country codes to check parsed networks aganist... + rows = self.db.query("SELECT * FROM countries ORDER BY country_code") + validcountries = [] + + for row in rows: + validcountries.append(row.country_code) + + for source_key in location.importer.WHOIS_SOURCES: + for single_url in location.importer.WHOIS_SOURCES[source_key]: + with downloader.request(single_url, return_blocks=True) as f: + for block in f: + self._parse_block(block, source_key, validcountries) + + # Process all parsed networks from every RIR we happen to have access to, + # insert the largest network chunks into the networks table immediately... + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") + + for family in (row.family for row in families): + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) + + self.db.execute("INSERT INTO networks(network, country, original_countries, source) \ + SELECT network, country, original_countries, source FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) + + # ... determine any other prefixes for this network family, ... + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) + + # ... and insert networks with this prefix in case they provide additional + # information (i. e. subnet of a larger chunk with a different country) + for prefix in (row.prefix for row in prefixes): + self.db.execute(""" + WITH candidates AS ( + SELECT + _rirdata.network, + _rirdata.country, + _rirdata.original_countries, + _rirdata.source + FROM + _rirdata + WHERE + family(_rirdata.network) = %s + AND + masklen(_rirdata.network) = %s + ), + filtered AS ( + SELECT + DISTINCT ON (c.network) + c.network, + c.country, + c.original_countries, + c.source, + masklen(networks.network), + networks.country AS parent_country + FROM + candidates c + LEFT JOIN + networks + ON + c.network << networks.network + ORDER BY + c.network, + masklen(networks.network) DESC NULLS LAST + ) + INSERT INTO + networks(network, country, original_countries, source) + SELECT + network, + country, + original_countries, + source + FROM + filtered + WHERE + parent_country IS NULL + OR + country <> parent_country + ON CONFLICT DO NOTHING""", + family, prefix, + ) + + self.db.execute(""" + INSERT INTO autnums(number, name, source) + SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums + JOIN _organizations ON _autnums.organization = _organizations.handle + ON CONFLICT (number) DO UPDATE SET name = excluded.name; + """) + + # Download all extended sources + for source_key in location.importer.EXTENDED_SOURCES: + for single_url in location.importer.EXTENDED_SOURCES[source_key]: + with self.db.transaction(): + # Download data + with downloader.request(single_url) as f: + for line in f: + self._parse_line(line, source_key, validcountries) + + # Download and import (technical) AS names from ARIN + self._import_as_names_from_arin() + + def _check_parsed_network(self, network): + """ + Assistive function to detect and subsequently sort out parsed + networks from RIR data (both Whois and so-called "extended sources"), + which are or have... + + (a) not globally routable (RFC 1918 space, et al.) + (b) covering a too large chunk of the IP address space (prefix length + is < 7 for IPv4 networks, and < 10 for IPv6) + (c) "0.0.0.0" or "::" as a network address + (d) are too small for being publicly announced (we have decided not to + process them at the moment, as they significantly enlarge our + database without providing very helpful additional information) + + This unfortunately is necessary due to brain-dead clutter across + various RIR databases, causing mismatches and eventually disruptions. + + We will return False in case a network is not suitable for adding + it to our database, and True otherwise. + """ + + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): + return False + + if not network.is_global: + log.debug("Skipping non-globally routable network: %s" % network) + return False + + if network.version == 4: + if network.prefixlen < 7: + log.debug("Skipping too big IP chunk: %s" % network) + return False + + if network.prefixlen > 24: + log.debug("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "0.0.0.0": + log.debug("Skipping network based on 0.0.0.0: %s" % network) + return False + + elif network.version == 6: + if network.prefixlen < 10: + log.debug("Skipping too big IP chunk: %s" % network) + return False + + if network.prefixlen > 48: + log.debug("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "::": + log.debug("Skipping network based on '::': %s" % network) + return False + + else: + # This should not happen... + log.warning("Skipping network of unknown family, this should not happen: %s" % network) + return False + + # In case we have made it here, the network is considered to + # be suitable for libloc consumption... + return True + + def _check_parsed_asn(self, asn): + """ + Assistive function to filter Autonomous System Numbers not being suitable + for adding to our database. Returns False in such cases, and True otherwise. + """ + + for start, end in VALID_ASN_RANGES: + if start <= asn and end >= asn: + return True + + log.info("Supplied ASN %s out of publicly routable ASN ranges" % asn) + return False + + def _parse_block(self, block, source_key, validcountries = None): + # Get first line to find out what type of block this is + line = block[0] + + # aut-num + if line.startswith("aut-num:"): + return self._parse_autnum_block(block, source_key) + + # inetnum + if line.startswith("inet6num:") or line.startswith("inetnum:"): + return self._parse_inetnum_block(block, source_key, validcountries) + + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block, source_key) + + def _parse_autnum_block(self, block, source_key): + autnum = {} + for line in block: + # Split line + key, val = split_line(line) + + if key == "aut-num": + m = re.match(r"^(AS|as)(\d+)", val) + if m: + autnum["asn"] = m.group(2) + + elif key == "org": + autnum[key] = val.upper() + + elif key == "descr": + # Save the first description line as well... + if not key in autnum: + autnum[key] = val + + # Skip empty objects + if not autnum or not "asn" in autnum: + return + + # Insert a dummy organisation handle into our temporary organisations + # table in case the AS does not have an organisation handle set, but + # has a description (a quirk often observed in APNIC area), so we can + # later display at least some string for this AS. + if not "org" in autnum: + if "descr" in autnum: + autnum["org"] = "LIBLOC-%s-ORGHANDLE" % autnum.get("asn") + + self.db.execute("INSERT INTO _organizations(handle, name, source) \ + VALUES(%s, %s, %s) ON CONFLICT (handle) DO NOTHING", + autnum.get("org"), autnum.get("descr"), source_key, + ) + else: + log.warning("ASN %s neither has an organisation handle nor a description line set, omitting" % \ + autnum.get("asn")) + return + + # Insert into database + self.db.execute("INSERT INTO _autnums(number, organization, source) \ + VALUES(%s, %s, %s) ON CONFLICT (number) DO UPDATE SET \ + organization = excluded.organization", + autnum.get("asn"), autnum.get("org"), source_key, + ) + + def _parse_inetnum_block(self, block, source_key, validcountries = None): + log.debug("Parsing inetnum block:") + + inetnum = {} + for line in block: + log.debug(line) + + # Split line + key, val = split_line(line) + + # Filter any inetnum records which are only referring to IP space + # not managed by that specific RIR... + if key == "netname": + if re.match(r"^(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK|STUB-[\d-]{3,}SLASH\d{1,2})", val.strip()): + log.debug("Skipping record indicating historic/orphaned data: %s" % val.strip()) + return + + if key == "inetnum": + start_address, delim, end_address = val.partition("-") + + # Strip any excess space + start_address, end_address = start_address.rstrip(), end_address.strip() + + # Handle "inetnum" formatting in LACNIC DB (e.g. "24.152.8/22" instead of "24.152.8.0/22") + if start_address and not (delim or end_address): + try: + start_address = ipaddress.ip_network(start_address, strict=False) + except ValueError: + start_address = start_address.split("/") + ldigits = start_address[0].count(".") + + # How many octets do we need to add? + # (LACNIC does not seem to have a /8 or greater assigned, so the following should suffice.) + if ldigits == 1: + start_address = start_address[0] + ".0.0/" + start_address[1] + elif ldigits == 2: + start_address = start_address[0] + ".0/" + start_address[1] + else: + log.warning("Could not recover IPv4 address from line in LACNIC DB format: %s" % line) + return + + try: + start_address = ipaddress.ip_network(start_address, strict=False) + except ValueError: + log.warning("Could not parse line in LACNIC DB format: %s" % line) + return + + # Enumerate first and last IP address of this network + end_address = start_address[-1] + start_address = start_address[0] + + else: + # Convert to IP address + try: + start_address = ipaddress.ip_address(start_address) + end_address = ipaddress.ip_address(end_address) + except ValueError: + log.warning("Could not parse line: %s" % line) + return + + inetnum["inetnum"] = list(ipaddress.summarize_address_range(start_address, end_address)) + + elif key == "inet6num": + inetnum[key] = [ipaddress.ip_network(val, strict=False)] + + elif key == "country": + val = val.upper() + + # Catch RIR data objects with more than one country code... + if not key in inetnum: + inetnum[key] = [] + else: + if val in inetnum.get("country"): + # ... but keep this list distinct... + continue + + # When people set country codes to "UK", they actually mean "GB" + if val == "UK": + val = "GB" + + inetnum[key].append(val) + + # Skip empty objects + if not inetnum or not "country" in inetnum: + return + + # Prepare skipping objects with unknown country codes... + invalidcountries = [singlecountry for singlecountry in inetnum.get("country") if singlecountry not in validcountries] + + # Iterate through all networks enumerated from above, check them for plausibility and insert + # them into the database, if _check_parsed_network() succeeded + for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"): + if self._check_parsed_network(single_network): + + # Skip objects with unknown country codes if they are valid to avoid log spam... + if validcountries and invalidcountries: + log.warning("Skipping network with bogus countr(y|ies) %s (original countries: %s): %s" % \ + (invalidcountries, inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum"))) + break + + # Everything is fine here, run INSERT statement... + self.db.execute("INSERT INTO _rirdata(network, country, original_countries, source) \ + VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % single_network, inetnum.get("country")[0], inetnum.get("country"), source_key, + ) + + def _parse_org_block(self, block, source_key): + org = {} + for line in block: + # Split line + key, val = split_line(line) + + if key == "organisation": + org[key] = val.upper() + elif key == "org-name": + org[key] = val + + # Skip empty objects + if not org: + return + + self.db.execute("INSERT INTO _organizations(handle, name, source) \ + VALUES(%s, %s, %s) ON CONFLICT (handle) DO \ + UPDATE SET name = excluded.name", + org.get("organisation"), org.get("org-name"), source_key, + ) + + def _parse_line(self, line, source_key, validcountries = None): + # Skip version line + if line.startswith("2"): + return + + # Skip comments + if line.startswith("#"): + return + + try: + registry, country_code, type, line = line.split("|", 3) + except: + log.warning("Could not parse line: %s" % line) + return + + # Skip any lines that are for stats only or do not have a country + # code at all (avoids log spam below) + if not country_code or country_code == '*': + return + + # Skip objects with unknown country codes + if validcountries and country_code not in validcountries: + log.warning("Skipping line with bogus country '%s': %s" % \ + (country_code, line)) + return + + if type in ("ipv6", "ipv4"): + return self._parse_ip_line(country_code, type, line, source_key) + + def _parse_ip_line(self, country, type, line, source_key): + try: + address, prefix, date, status, organization = line.split("|") + except ValueError: + organization = None + + # Try parsing the line without organization + try: + address, prefix, date, status = line.split("|") + except ValueError: + log.warning("Unhandled line format: %s" % line) + return + + # Skip anything that isn't properly assigned + if not status in ("assigned", "allocated"): + return + + # Cast prefix into an integer + try: + prefix = int(prefix) + except: + log.warning("Invalid prefix: %s" % prefix) + return + + # Fix prefix length for IPv4 + if type == "ipv4": + prefix = 32 - int(math.log(prefix, 2)) + + # Try to parse the address + try: + network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False) + except ValueError: + log.warning("Invalid IP address: %s" % address) + return + + if not self._check_parsed_network(network): + return + + self.db.execute("INSERT INTO networks(network, country, original_countries, source) \ + VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO \ + UPDATE SET country = excluded.country", + "%s" % network, country, [country], source_key, + ) + + def _import_as_names_from_arin(self): + downloader = location.importer.Downloader() + + # XXX: Download AS names file from ARIN (note that these names appear to be quite + # technical, not intended for human consumption, as description fields in + # organisation handles for other RIRs are - however, this is what we have got, + # and in some cases, it might be still better than nothing) + with downloader.request("https://ftp.arin.net/info/asn.txt", return_blocks=False) as f: + for line in f: + # Convert binary line to string... + line = str(line) + + # ... valid lines start with a space, followed by the number of the Autonomous System ... + if not line.startswith(" "): + continue + + # Split line and check if there is a valid ASN in it... + asn, name = line.split()[0:2] + + try: + asn = int(asn) + except ValueError: + log.debug("Skipping ARIN AS names line not containing an integer for ASN") + continue + + # Filter invalid ASNs... + if not self._check_parsed_asn(asn): + continue + + # Skip any AS name that appears to be a placeholder for a different RIR or entity... + if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|DNIC|LACNIC|RIPE|IANA)(?:\d?$|\-)", name): + continue + + # Bail out in case the AS name contains anything we do not expect here... + if re.search(r"[^a-zA-Z0-9-_]", name): + log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \ + (asn, name)) + + # Things look good here, run INSERT statement and skip this one if we already have + # a (better?) name for this Autonomous System... + self.db.execute(""" + INSERT INTO autnums( + number, + name, + source + ) VALUES (%s, %s, %s) + ON CONFLICT (number) DO NOTHING""", + asn, + name, + "ARIN", + ) + + def handle_update_announcements(self, ns): + server = ns.server[0] + + with self.db.transaction(): + if server.startswith("/"): + self._handle_update_announcements_from_bird(server) + else: + self._handle_update_announcements_from_telnet(server) + + # Purge anything we never want here + self.db.execute(""" + -- Delete default routes + DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0'; + + -- Delete anything that is not global unicast address space + DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3'; + + -- DELETE "current network" address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8'; + + -- DELETE local loopback address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8'; + + -- DELETE RFC 1918 address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16'; + + -- DELETE test, benchmark and documentation address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24'; + + -- DELETE CGNAT address space (RFC 6598) + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10'; + + -- DELETE link local address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16'; + + -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068) + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24'; + DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16'; + + -- DELETE multicast and reserved address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4'; + + -- Delete networks that are too small to be in the global routing table + DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48; + DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24; + + -- Delete any non-public or reserved ASNs + DELETE FROM announcements WHERE NOT ( + (autnum >= 1 AND autnum <= 23455) + OR + (autnum >= 23457 AND autnum <= 64495) + OR + (autnum >= 131072 AND autnum <= 4199999999) + ); + + -- Delete everything that we have not seen for 14 days + DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days'; + """) + + def _handle_update_announcements_from_bird(self, server): + # Pre-compile the regular expression for faster searching + route = re.compile(b"^\s(.+?)\s+.+?\[(?:AS(.*?))?.\]$") + + log.info("Requesting routing table from Bird (%s)" % server) + + aggregated_networks = [] + + # Send command to list all routes + for line in self._bird_cmd(server, "show route"): + m = route.match(line) + if not m: + # Skip empty lines + if not line: + pass + + # Ignore any header lines with the name of the routing table + elif line.startswith(b"Table"): + pass + + # Log anything else + else: + log.debug("Could not parse line: %s" % line.decode()) + + continue + + # Fetch the extracted network and ASN + network, autnum = m.groups() + + # Decode into strings + if network: + network = network.decode() + if autnum: + autnum = autnum.decode() + + # Collect all aggregated networks + if not autnum: + log.debug("%s is an aggregated network" % network) + aggregated_networks.append(network) + continue + + # Insert it into the database + self.db.execute("INSERT INTO announcements(network, autnum) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP", + network, autnum, + ) + + # Process any aggregated networks + for network in aggregated_networks: + log.debug("Processing aggregated network %s" % network) + + # Run "show route all" for each network + for line in self._bird_cmd(server, "show route %s all" % network): + # Try finding the path + m = re.match(b"\s+BGP\.as_path:.* (\d+) {\d+}$", line) + if m: + # Select the last AS number in the path + autnum = m.group(1).decode() + + # Insert it into the database + self.db.execute("INSERT INTO announcements(network, autnum) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP", + network, autnum, + ) + + # We don't need to process any more + break + + def _handle_update_announcements_from_telnet(self, server): + # Pre-compile regular expression for routes + route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL) + + with telnetlib.Telnet(server) as t: + # Enable debug mode + #if ns.debug: + # t.set_debuglevel(10) + + # Wait for console greeting + greeting = t.read_until(b"> ", timeout=30) + if not greeting: + log.error("Could not get a console prompt") + return 1 + + # Disable pagination + t.write(b"terminal length 0\n") + + # Wait for the prompt to return + t.read_until(b"> ") + + # Fetch the routing tables + for protocol in ("ipv6", "ipv4"): + log.info("Requesting %s routing table" % protocol) + + # Request the full unicast routing table + t.write(b"show bgp %s unicast\n" % protocol.encode()) + + # Read entire header which ends with "Path" + t.read_until(b"Path\r\n") + + while True: + # Try reading a full entry + # Those might be broken across multiple lines but ends with i + line = t.read_until(b"i\r\n", timeout=5) + if not line: + break + + # Show line for debugging + #log.debug(repr(line)) + + # Try finding a route in here + m = route.match(line) + if m: + network, autnum = m.groups() + + # Convert network to string + network = network.decode() + + # Append /24 for IPv4 addresses + if not "/" in network and not ":" in network: + network = "%s/24" % network + + # Convert AS number to integer + autnum = int(autnum) + + log.info("Found announcement for %s by %s" % (network, autnum)) + + self.db.execute("INSERT INTO announcements(network, autnum) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP", + network, autnum, + ) + + log.info("Finished reading the %s routing table" % protocol) + + def _bird_cmd(self, socket_path, command): + # Connect to the socket + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.connect(socket_path) + + # Allocate some buffer + buffer = b"" + + log.debug("Sending Bird command: %s" % command) + + # Send the command + s.send(b"%s\n" % command.encode()) + + while True: + # Fill up the buffer + buffer += s.recv(4096) + + while True: + # Search for the next newline + pos = buffer.find(b"\n") + + # If we cannot find one, we go back and read more data + if pos <= 0: + break + + # Cut after the newline character + pos += 1 + + # Split the line we want and keep the rest in buffer + line, buffer = buffer[:pos], buffer[pos:] + + # Try parsing any status lines + if len(line) > 4 and line[:4].isdigit() and line[4] in (32, 45): + code, delim, line = int(line[:4]), line[4], line[5:] + + log.debug("Received response code %s from bird" % code) + + # End of output + if code == 0: + return + + # Ignore hello line + elif code == 1: + continue + + # Otherwise return the line + yield line + + def handle_update_overrides(self, ns): + with self.db.transaction(): + # Drop all data that we have + self.db.execute(""" + TRUNCATE TABLE autnum_overrides; + TRUNCATE TABLE network_overrides; + """) + + # Update overrides for various cloud providers big enough to publish their own IP + # network allocation lists in a machine-readable format... + self._update_overrides_for_aws() + + # Update overrides for Spamhaus DROP feeds... + self._update_overrides_for_spamhaus_drop() + + for file in ns.files: + log.info("Reading %s..." % file) + + with open(file, "rb") as f: + for type, block in location.importer.read_blocks(f): + if type == "net": + network = block.get("net") + # Try to parse and normalise the network + try: + network = ipaddress.ip_network(network, strict=False) + except ValueError as e: + log.warning("Invalid IP network: %s: %s" % (network, e)) + continue + + # Prevent that we overwrite all networks + if network.prefixlen == 0: + log.warning("Skipping %s: You cannot overwrite default" % network) + continue + + self.db.execute(""" + INSERT INTO network_overrides( + network, + country, + source, + is_anonymous_proxy, + is_satellite_provider, + is_anycast, + is_drop + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (network) DO NOTHING""", + "%s" % network, + block.get("country"), + "manual", + self._parse_bool(block, "is-anonymous-proxy"), + self._parse_bool(block, "is-satellite-provider"), + self._parse_bool(block, "is-anycast"), + self._parse_bool(block, "drop"), + ) + + elif type == "aut-num": + autnum = block.get("aut-num") + + # Check if AS number begins with "AS" + if not autnum.startswith("AS"): + log.warning("Invalid AS number: %s" % autnum) + continue + + # Strip "AS" + autnum = autnum[2:] + + self.db.execute(""" + INSERT INTO autnum_overrides( + number, + name, + country, + source, + is_anonymous_proxy, + is_satellite_provider, + is_anycast, + is_drop + ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT DO NOTHING""", + autnum, + block.get("name"), + block.get("country"), + "manual", + self._parse_bool(block, "is-anonymous-proxy"), + self._parse_bool(block, "is-satellite-provider"), + self._parse_bool(block, "is-anycast"), + self._parse_bool(block, "drop"), + ) + + else: + log.warning("Unsupported type: %s" % type) + + def _update_overrides_for_aws(self): + # Download Amazon AWS IP allocation file to create overrides... + downloader = location.importer.Downloader() + + try: + with downloader.request("https://ip-ranges.amazonaws.com/ip-ranges.json", return_blocks=False) as f: + aws_ip_dump = json.load(f.body) + except Exception as e: + log.error("unable to preprocess Amazon AWS IP ranges: %s" % e) + return + + # XXX: Set up a dictionary for mapping a region name to a country. Unfortunately, + # there seems to be no machine-readable version available of this other than + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html + # (worse, it seems to be incomplete :-/ ); https://www.cloudping.cloud/endpoints + # was helpful here as well. + aws_region_country_map = { + "af-south-1": "ZA", + "ap-east-1": "HK", + "ap-south-1": "IN", + "ap-south-2": "IN", + "ap-northeast-3": "JP", + "ap-northeast-2": "KR", + "ap-southeast-1": "SG", + "ap-southeast-2": "AU", + "ap-southeast-3": "MY", + "ap-southeast-4": "AU", + "ap-northeast-1": "JP", + "ca-central-1": "CA", + "eu-central-1": "DE", + "eu-central-2": "CH", + "eu-west-1": "IE", + "eu-west-2": "GB", + "eu-south-1": "IT", + "eu-south-2": "ES", + "eu-west-3": "FR", + "eu-north-1": "SE", + "il-central-1": "IL", # XXX: This one is not documented anywhere except for ip-ranges.json itself + "me-central-1": "AE", + "me-south-1": "BH", + "sa-east-1": "BR" + } + + # Fetch all valid country codes to check parsed networks aganist... + rows = self.db.query("SELECT * FROM countries ORDER BY country_code") + validcountries = [] + + for row in rows: + validcountries.append(row.country_code) + + with self.db.transaction(): + for snetwork in aws_ip_dump["prefixes"] + aws_ip_dump["ipv6_prefixes"]: + try: + network = ipaddress.ip_network(snetwork.get("ip_prefix") or snetwork.get("ipv6_prefix"), strict=False) + except ValueError: + log.warning("Unable to parse line: %s" % snetwork) + continue + + # Sanitize parsed networks... + if not self._check_parsed_network(network): + continue + + # Determine region of this network... + region = snetwork["region"] + cc = None + is_anycast = False + + # Any region name starting with "us-" will get "US" country code assigned straight away... + if region.startswith("us-"): + cc = "US" + elif region.startswith("cn-"): + # ... same goes for China ... + cc = "CN" + elif region == "GLOBAL": + # ... funny region name for anycast-like networks ... + is_anycast = True + elif region in aws_region_country_map: + # ... assign looked up country code otherwise ... + cc = aws_region_country_map[region] + else: + # ... and bail out if we are missing something here + log.warning("Unable to determine country code for line: %s" % snetwork) + continue + + # Skip networks with unknown country codes + if not is_anycast and validcountries and cc not in validcountries: + log.warning("Skipping Amazon AWS network with bogus country '%s': %s" % \ + (cc, network)) + return + + # Conduct SQL statement... + self.db.execute(""" + INSERT INTO network_overrides( + network, + country, + source, + is_anonymous_proxy, + is_satellite_provider, + is_anycast + ) VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (network) DO NOTHING""", + "%s" % network, + cc, + "Amazon AWS IP feed", + None, + None, + is_anycast, + ) + + + def _update_overrides_for_spamhaus_drop(self): + downloader = location.importer.Downloader() + + ip_urls = [ + "https://www.spamhaus.org/drop/drop.txt", + "https://www.spamhaus.org/drop/edrop.txt", + "https://www.spamhaus.org/drop/dropv6.txt" + ] + + asn_urls = [ + "https://www.spamhaus.org/drop/asndrop.txt" + ] + + for url in ip_urls: + try: + with downloader.request(url, return_blocks=False) as f: + fcontent = f.body.readlines() + except Exception as e: + log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e)) + return + + # Iterate through every line, filter comments and add remaining networks to + # the override table in case they are valid... + with self.db.transaction(): + for sline in fcontent: + + # The response is assumed to be encoded in UTF-8... + sline = sline.decode("utf-8") + + # Comments start with a semicolon... + if sline.startswith(";"): + continue + + # Extract network and ignore anything afterwards... + try: + network = ipaddress.ip_network(sline.split()[0], strict=False) + except ValueError: + log.error("Unable to parse line: %s" % sline) + continue + + # Sanitize parsed networks... + if not self._check_parsed_network(network): + log.warning("Skipping bogus network found in Spamhaus DROP URL %s: %s" % \ + (url, network)) + continue + + # Conduct SQL statement... + self.db.execute(""" + INSERT INTO network_overrides( + network, + source, + is_drop + ) VALUES (%s, %s, %s) + ON CONFLICT (network) DO UPDATE SET is_drop = True""", + "%s" % network, + "Spamhaus DROP lists", + True + ) + + for url in asn_urls: + try: + with downloader.request(url, return_blocks=False) as f: + fcontent = f.body.readlines() + except Exception as e: + log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e)) + return + + # Iterate through every line, filter comments and add remaining ASNs to + # the override table in case they are valid... + with self.db.transaction(): + for sline in fcontent: + + # The response is assumed to be encoded in UTF-8... + sline = sline.decode("utf-8") + + # Comments start with a semicolon... + if sline.startswith(";"): + continue + + # Throw away anything after the first space... + sline = sline.split()[0] + + # ... strip the "AS" prefix from it ... + sline = sline.strip("AS") + + # ... and convert it into an integer. Voila. + asn = int(sline) + + # Filter invalid ASNs... + if not self._check_parsed_asn(asn): + log.warning("Skipping bogus ASN found in Spamhaus DROP URL %s: %s" % \ + (url, asn)) + continue + + # Conduct SQL statement... + self.db.execute(""" + INSERT INTO autnum_overrides( + number, + source, + is_drop + ) VALUES (%s, %s, %s) + ON CONFLICT (number) DO UPDATE SET is_drop = True""", + "%s" % asn, + "Spamhaus ASN-DROP list", + True + ) + + @staticmethod + def _parse_bool(block, key): + val = block.get(key) + + # There is no point to proceed when we got None + if val is None: + return + + # Convert to lowercase + val = val.lower() + + # True + if val in ("yes", "1"): + return True + + # False + if val in ("no", "0"): + return False + + # Default to None + return None + + def handle_import_countries(self, ns): + with self.db.transaction(): + # Drop all data that we have + self.db.execute("TRUNCATE TABLE countries") + + for file in ns.file: + for line in file: + line = line.rstrip() + + # Ignore any comments + if line.startswith("#"): + continue + + try: + country_code, continent_code, name = line.split(maxsplit=2) + except: + log.warning("Could not parse line: %s" % line) + continue + + self.db.execute("INSERT INTO countries(country_code, name, continent_code) \ + VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code) + + +def split_line(line): + key, colon, val = line.partition(":") + + # Strip any excess space + key = key.strip() + val = val.strip() + + return key, val + +def main(): + # Run the command line interface + c = CLI() + c.run() + +main() diff --git a/src/python/location.in b/src/python/location.in new file mode 100644 index 0000000..233cea0 --- /dev/null +++ b/src/python/location.in @@ -0,0 +1,644 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2017-2021 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import argparse +import datetime +import ipaddress +import logging +import os +import re +import shutil +import socket +import sys +import time + +# Load our location module +import location +import location.downloader +import location.export + +from location.i18n import _ + +# Setup logging +log = logging.getLogger("location") + +# Output formatters + +class CLI(object): + def parse_cli(self): + parser = argparse.ArgumentParser( + description=_("Location Database Command Line Interface"), + ) + subparsers = parser.add_subparsers() + + # Global configuration flags + parser.add_argument("--debug", action="store_true", + help=_("Enable debug output")) + parser.add_argument("--quiet", action="store_true", + help=_("Enable quiet mode")) + + # version + parser.add_argument("--version", action="version", + version="%(prog)s @VERSION@") + + # database + parser.add_argument("--database", "-d", + default="@databasedir@/database.db", help=_("Path to database"), + ) + + # public key + parser.add_argument("--public-key", "-k", + default="@databasedir@/signing-key.pem", help=_("Public Signing Key"), + ) + + # Show the database version + version = subparsers.add_parser("version", + help=_("Show database version")) + version.set_defaults(func=self.handle_version) + + # lookup an IP address + lookup = subparsers.add_parser("lookup", + help=_("Lookup one or multiple IP addresses"), + ) + lookup.add_argument("address", nargs="+") + lookup.set_defaults(func=self.handle_lookup) + + # Dump the whole database + dump = subparsers.add_parser("dump", + help=_("Dump the entire database"), + ) + dump.add_argument("output", nargs="?", type=argparse.FileType("w")) + dump.set_defaults(func=self.handle_dump) + + # Update + update = subparsers.add_parser("update", help=_("Update database")) + update.add_argument("--cron", + help=_("Update the library only once per interval"), + choices=("daily", "weekly", "monthly"), + ) + update.set_defaults(func=self.handle_update) + + # Verify + verify = subparsers.add_parser("verify", + help=_("Verify the downloaded database")) + verify.set_defaults(func=self.handle_verify) + + # Get AS + get_as = subparsers.add_parser("get-as", + help=_("Get information about one or multiple Autonomous Systems"), + ) + get_as.add_argument("asn", nargs="+") + get_as.set_defaults(func=self.handle_get_as) + + # Search for AS + search_as = subparsers.add_parser("search-as", + help=_("Search for Autonomous Systems that match the string"), + ) + search_as.add_argument("query", nargs=1) + search_as.set_defaults(func=self.handle_search_as) + + # List all networks in an AS + list_networks_by_as = subparsers.add_parser("list-networks-by-as", + help=_("Lists all networks in an AS"), + ) + list_networks_by_as.add_argument("asn", nargs=1, type=int) + list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4")) + list_networks_by_as.add_argument("--format", + choices=location.export.formats.keys(), default="list") + list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as) + + # List all networks in a country + list_networks_by_cc = subparsers.add_parser("list-networks-by-cc", + help=_("Lists all networks in a country"), + ) + list_networks_by_cc.add_argument("country_code", nargs=1) + list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4")) + list_networks_by_cc.add_argument("--format", + choices=location.export.formats.keys(), default="list") + list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc) + + # List all networks with flags + list_networks_by_flags = subparsers.add_parser("list-networks-by-flags", + help=_("Lists all networks with flags"), + ) + list_networks_by_flags.add_argument("--anonymous-proxy", + action="store_true", help=_("Anonymous Proxies"), + ) + list_networks_by_flags.add_argument("--satellite-provider", + action="store_true", help=_("Satellite Providers"), + ) + list_networks_by_flags.add_argument("--anycast", + action="store_true", help=_("Anycasts"), + ) + list_networks_by_flags.add_argument("--drop", + action="store_true", help=_("Hostile Networks safe to drop"), + ) + list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) + list_networks_by_flags.add_argument("--format", + choices=location.export.formats.keys(), default="list") + list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags) + + # List bogons + list_bogons = subparsers.add_parser("list-bogons", + help=_("Lists all bogons"), + ) + list_bogons.add_argument("--family", choices=("ipv6", "ipv4")) + list_bogons.add_argument("--format", + choices=location.export.formats.keys(), default="list") + list_bogons.set_defaults(func=self.handle_list_bogons) + + # List countries + list_countries = subparsers.add_parser("list-countries", + help=_("Lists all countries"), + ) + list_countries.add_argument("--show-name", + action="store_true", help=_("Show the name of the country"), + ) + list_countries.add_argument("--show-continent", + action="store_true", help=_("Show the continent"), + ) + list_countries.set_defaults(func=self.handle_list_countries) + + # Export + export = subparsers.add_parser("export", + help=_("Exports data in many formats to load it into packet filters"), + ) + export.add_argument("--format", help=_("Output format"), + choices=location.export.formats.keys(), default="list") + export.add_argument("--directory", help=_("Output directory")) + export.add_argument("--family", + help=_("Specify address family"), choices=("ipv6", "ipv4"), + ) + export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export")) + export.set_defaults(func=self.handle_export) + + args = parser.parse_args() + + # Configure logging + if args.debug: + location.logger.set_level(logging.DEBUG) + elif args.quiet: + location.logger.set_level(logging.WARNING) + + # Print usage if no action was given + if not "func" in args: + parser.print_usage() + sys.exit(2) + + return args + + def run(self): + # Parse command line arguments + args = self.parse_cli() + + # Open database + try: + db = location.Database(args.database) + except FileNotFoundError as e: + # Allow continuing without a database + if args.func == self.handle_update: + db = None + + else: + sys.stderr.write("location: Could not open database %s: %s\n" \ + % (args.database, e)) + sys.exit(1) + + # Translate family (if present) + if "family" in args: + if args.family == "ipv6": + args.family = socket.AF_INET6 + elif args.family == "ipv4": + args.family = socket.AF_INET + else: + args.family = 0 + + # Call function + try: + ret = args.func(db, args) + + # Catch invalid inputs + except ValueError as e: + sys.stderr.write("%s\n" % e) + ret = 2 + + # Catch any other exceptions + except Exception as e: + sys.stderr.write("%s\n" % e) + ret = 1 + + # Return with exit code + if ret: + sys.exit(ret) + + # Otherwise just exit + sys.exit(0) + + def handle_version(self, db, ns): + """ + Print the version of the database + """ + t = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at), + ) + + print(t) + + def handle_lookup(self, db, ns): + ret = 0 + + format = " %-24s: %s" + + for address in ns.address: + try: + network = db.lookup(address) + except ValueError: + print(_("Invalid IP address: %s") % address, file=sys.stderr) + return 2 + + args = { + "address" : address, + "network" : network, + } + + # Nothing found? + if not network: + print(_("Nothing found for %(address)s") % args, file=sys.stderr) + ret = 1 + continue + + print("%s:" % address) + print(format % (_("Network"), network)) + + # Print country + if network.country_code: + country = db.get_country(network.country_code) + + print(format % ( + _("Country"), + country.name if country else network.country_code), + ) + + # Print AS information + if network.asn: + autonomous_system = db.get_as(network.asn) + + print(format % ( + _("Autonomous System"), + autonomous_system or "AS%s" % network.asn), + ) + + # Anonymous Proxy + if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY): + print(format % ( + _("Anonymous Proxy"), _("yes"), + )) + + # Satellite Provider + if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER): + print(format % ( + _("Satellite Provider"), _("yes"), + )) + + # Anycast + if network.has_flag(location.NETWORK_FLAG_ANYCAST): + print(format % ( + _("Anycast"), _("yes"), + )) + + # Hostile Network + if network.has_flag(location.NETWORK_FLAG_DROP): + print(format % ( + _("Hostile Network safe to drop"), _("yes"), + )) + + return ret + + def handle_dump(self, db, ns): + # Use output file or write to stdout + f = ns.output or sys.stdout + + # Format everything like this + format = "%-24s %s\n" + + # Write metadata + f.write("#\n# Location Database Export\n#\n") + + f.write("# Generated: %s\n" % time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at), + )) + + if db.vendor: + f.write("# Vendor: %s\n" % db.vendor) + + if db.license: + f.write("# License: %s\n" % db.license) + + f.write("#\n") + + if db.description: + for line in db.description.splitlines(): + line = "# %s" % line + f.write("%s\n" % line.rstrip()) + + f.write("#\n") + + # Iterate over all ASes + for a in db.ases: + f.write("\n") + f.write(format % ("aut-num:", "AS%s" % a.number)) + f.write(format % ("name:", a.name)) + + flags = { + location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:", + location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:", + location.NETWORK_FLAG_ANYCAST : "is-anycast:", + location.NETWORK_FLAG_DROP : "drop:", + } + + # Iterate over all networks + for n in db.networks: + f.write("\n") + f.write(format % ("net:", n)) + + if n.country_code: + f.write(format % ("country:", n.country_code)) + + if n.asn: + f.write(format % ("aut-num:", n.asn)) + + # Print all flags + for flag in flags: + if n.has_flag(flag): + f.write(format % (flags[flag], "yes")) + + def handle_get_as(self, db, ns): + """ + Gets information about Autonomous Systems + """ + ret = 0 + + for asn in ns.asn: + try: + asn = int(asn) + except ValueError: + print(_("Invalid ASN: %s") % asn, file=sys.stderr) + ret = 1 + continue + + # Fetch AS from database + a = db.get_as(asn) + + # Nothing found + if not a: + print(_("Could not find AS%s") % asn, file=sys.stderr) + ret = 1 + continue + + print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name }) + + return ret + + def handle_search_as(self, db, ns): + for query in ns.query: + # Print all matches ASes + for a in db.search_as(query): + print(a) + + def handle_update(self, db, ns): + if ns.cron and db: + now = time.time() + + if ns.cron == "daily": + delta = datetime.timedelta(days=1) + elif ns.cron == "weekly": + delta = datetime.timedelta(days=7) + elif ns.cron == "monthly": + delta = datetime.timedelta(days=30) + + delta = delta.total_seconds() + + # Check if the database has recently been updated + if db.created_at >= (now - delta): + log.info( + _("The database has been updated recently"), + ) + return 3 + + # Fetch the timestamp we need from DNS + t = location.discover_latest_version() + + # Check the version of the local database + if db and t and db.created_at >= t: + log.info("Already on the latest version") + return + + # Download the database into the correct directory + tmpdir = os.path.dirname(ns.database) + + # Create a downloader + d = location.downloader.Downloader() + + # Try downloading a new database + try: + t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir) + + # If no file could be downloaded, log a message + except FileNotFoundError as e: + log.error("Could not download a new database") + return 1 + + # If we have not received a new file, there is nothing to do + if not t: + return 3 + + # Move temporary file to destination + shutil.move(t.name, ns.database) + + return 0 + + def handle_verify(self, db, ns): + # Verify the database + with open(ns.public_key, "r") as f: + if not db.verify(f): + log.error("Could not verify database") + return 1 + + # Success + log.debug("Database successfully verified") + return 0 + + def __get_output_formatter(self, ns): + try: + cls = location.export.formats[ns.format] + except KeyError: + cls = location.export.OutputFormatter + + return cls + + def handle_list_countries(self, db, ns): + for country in db.countries: + line = [ + country.code, + ] + + if ns.show_continent: + line.append(country.continent_code) + + if ns.show_name: + line.append(country.name) + + # Format the output + line = " ".join(line) + + # Print the output + print(line) + + def handle_list_networks_by_as(self, db, ns): + writer = self.__get_output_formatter(ns) + + for asn in ns.asn: + f = writer("AS%s" % asn, f=sys.stdout) + + # Print all matching networks + for n in db.search_networks(asns=[asn], family=ns.family): + f.write(n) + + f.finish() + + def handle_list_networks_by_cc(self, db, ns): + writer = self.__get_output_formatter(ns) + + for country_code in ns.country_code: + # Open standard output + f = writer(country_code, f=sys.stdout) + + # Print all matching networks + for n in db.search_networks(country_codes=[country_code], family=ns.family): + f.write(n) + + f.finish() + + def handle_list_networks_by_flags(self, db, ns): + flags = 0 + + if ns.anonymous_proxy: + flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY + + if ns.satellite_provider: + flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER + + if ns.anycast: + flags |= location.NETWORK_FLAG_ANYCAST + + if ns.drop: + flags |= location.NETWORK_FLAG_DROP + + if not flags: + raise ValueError(_("You must at least pass one flag")) + + writer = self.__get_output_formatter(ns) + f = writer("custom", f=sys.stdout) + + for n in db.search_networks(flags=flags, family=ns.family): + f.write(n) + + f.finish() + + def handle_list_bogons(self, db, ns): + writer = self.__get_output_formatter(ns) + f = writer("bogons", f=sys.stdout) + + for n in db.list_bogons(family=ns.family): + f.write(n) + + f.finish() + + def handle_export(self, db, ns): + countries, asns = [], [] + + # Translate family + if ns.family: + families = [ ns.family ] + else: + families = [ socket.AF_INET6, socket.AF_INET ] + + for object in ns.objects: + m = re.match("^AS(\d+)$", object) + if m: + object = int(m.group(1)) + + asns.append(object) + + elif location.country_code_is_valid(object) \ + or object in ("A1", "A2", "A3", "XD"): + countries.append(object) + + else: + log.warning("Invalid argument: %s" % object) + continue + + # Default to exporting all countries + if not countries and not asns: + countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries] + + # Select the output format + writer = self.__get_output_formatter(ns) + + e = location.export.Exporter(db, writer) + e.export(ns.directory, countries=countries, asns=asns, families=families) + + +def format_timedelta(t): + s = [] + + if t.days: + s.append( + _("One Day", "%(days)s Days", t.days) % { "days" : t.days, } + ) + + hours = t.seconds // 3600 + if hours: + s.append( + _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, } + ) + + minutes = (t.seconds % 3600) // 60 + if minutes: + s.append( + _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, } + ) + + seconds = t.seconds % 60 + if t.seconds: + s.append( + _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, } + ) + + if not s: + return _("Now") + + return _("%s ago") % ", ".join(s) + +def main(): + # Run the command line interface + c = CLI() + c.run() + +main() diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c new file mode 100644 index 0000000..15f661b --- /dev/null +++ b/src/python/locationmodule.c @@ -0,0 +1,180 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017-2021 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include + +#include +#include + +#include "locationmodule.h" +#include "as.h" +#include "country.h" +#include "database.h" +#include "network.h" +#include "writer.h" + +/* Declare global context */ +struct loc_ctx* loc_ctx; + +PyMODINIT_FUNC PyInit__location(void); + +static void location_free(void) { + // Release context + if (loc_ctx) + loc_unref(loc_ctx); +} + +static PyObject* set_log_level(PyObject* m, PyObject* args) { + int priority = LOG_INFO; + + if (!PyArg_ParseTuple(args, "i", &priority)) + return NULL; + + loc_set_log_priority(loc_ctx, priority); + + Py_RETURN_NONE; +} + +static PyObject* discover_latest_version(PyObject* m, PyObject* args) { + unsigned int version = LOC_DATABASE_VERSION_LATEST; + + if (!PyArg_ParseTuple(args, "|i", &version)) + return NULL; + + time_t t = 0; + + int r = loc_discover_latest_version(loc_ctx, version, &t); + if (r) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLong(t); +} + +static PyObject* country_code_is_valid(PyObject* m, PyObject* args) { + const char* country_code = NULL; + + if (!PyArg_ParseTuple(args, "s", &country_code)) + return NULL; + + if (loc_country_code_is_valid(country_code)) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyMethodDef location_module_methods[] = { + { + "country_code_is_valid", + (PyCFunction)country_code_is_valid, + METH_VARARGS, + NULL, + }, + { + "discover_latest_version", + (PyCFunction)discover_latest_version, + METH_VARARGS, + NULL, + }, + { + "set_log_level", + (PyCFunction)set_log_level, + METH_VARARGS, + NULL, + }, + { NULL }, +}; + +static struct PyModuleDef location_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_location", + .m_size = -1, + .m_doc = "Python module for libloc", + .m_methods = location_module_methods, + .m_free = (freefunc)location_free, +}; + +PyMODINIT_FUNC PyInit__location(void) { + // Initialise loc context + int r = loc_new(&loc_ctx); + if (r) + return NULL; + + PyObject* m = PyModule_Create(&location_module); + if (!m) + return NULL; + + // AS + if (PyType_Ready(&ASType) < 0) + return NULL; + + Py_INCREF(&ASType); + PyModule_AddObject(m, "AS", (PyObject *)&ASType); + + // Country + if (PyType_Ready(&CountryType) < 0) + return NULL; + + Py_INCREF(&CountryType); + PyModule_AddObject(m, "Country", (PyObject *)&CountryType); + + // Database + if (PyType_Ready(&DatabaseType) < 0) + return NULL; + + Py_INCREF(&DatabaseType); + PyModule_AddObject(m, "Database", (PyObject *)&DatabaseType); + + // Database Enumerator + if (PyType_Ready(&DatabaseEnumeratorType) < 0) + return NULL; + + Py_INCREF(&DatabaseEnumeratorType); + //PyModule_AddObject(m, "DatabaseEnumerator", (PyObject *)&DatabaseEnumeratorType); + + // Network + if (PyType_Ready(&NetworkType) < 0) + return NULL; + + Py_INCREF(&NetworkType); + PyModule_AddObject(m, "Network", (PyObject *)&NetworkType); + + // Writer + if (PyType_Ready(&WriterType) < 0) + return NULL; + + Py_INCREF(&WriterType); + PyModule_AddObject(m, "Writer", (PyObject *)&WriterType); + + // Add flags + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANONYMOUS_PROXY", LOC_NETWORK_FLAG_ANONYMOUS_PROXY)) + return NULL; + + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_SATELLITE_PROVIDER", LOC_NETWORK_FLAG_SATELLITE_PROVIDER)) + return NULL; + + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST)) + return NULL; + + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP)) + return NULL; + + // Add latest database version + if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST)) + return NULL; + + return m; +} diff --git a/src/python/locationmodule.h b/src/python/locationmodule.h new file mode 100644 index 0000000..e267986 --- /dev/null +++ b/src/python/locationmodule.h @@ -0,0 +1,24 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef PYTHON_LOCATION_MODULE_H +#define PYTHON_LOCATION_MODULE_H + +#include + +extern struct loc_ctx* loc_ctx; + +#endif /* PYTHON_LOCATION_MODULE_H */ diff --git a/src/python/logger.py b/src/python/logger.py new file mode 100644 index 0000000..0bdf9ec --- /dev/null +++ b/src/python/logger.py @@ -0,0 +1,46 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2020 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import logging +import logging.handlers + +# Initialise root logger +log = logging.getLogger("location") +log.setLevel(logging.INFO) + +# Log to console +handler = logging.StreamHandler() +handler.setLevel(logging.DEBUG) +log.addHandler(handler) + +# Log to syslog +handler = logging.handlers.SysLogHandler(address="/dev/log", + facility=logging.handlers.SysLogHandler.LOG_DAEMON) +handler.setLevel(logging.INFO) +log.addHandler(handler) + +# Format syslog messages +formatter = logging.Formatter("%(message)s") +handler.setFormatter(formatter) + +def set_level(level): + """ + Sets the log level for the root logger + """ + log.setLevel(level) diff --git a/src/python/network.c b/src/python/network.c new file mode 100644 index 0000000..474b6de --- /dev/null +++ b/src/python/network.c @@ -0,0 +1,344 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include + +#include +#include + +#include +#include +#include + +#include "locationmodule.h" +#include "network.h" + +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { + PyObject* list = PyList_New(0); + if (!networks) + return list; + + while (!loc_network_list_empty(networks)) { + struct loc_network* network = loc_network_list_pop(networks); + + PyObject* n = new_network(&NetworkType, network); + PyList_Append(list, n); + + loc_network_unref(network); + Py_DECREF(n); + } + + return list; +} + +PyObject* new_network(PyTypeObject* type, struct loc_network* network) { + NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); + if (self) { + self->network = loc_network_ref(network); + } + + return (PyObject*)self; +} + +static PyObject* Network_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); + + return (PyObject*)self; +} + +static void Network_dealloc(NetworkObject* self) { + if (self->network) + loc_network_unref(self->network); + + Py_TYPE(self)->tp_free((PyObject* )self); +} + +static int Network_init(NetworkObject* self, PyObject* args, PyObject* kwargs) { + const char* network = NULL; + + if (!PyArg_ParseTuple(args, "s", &network)) + return -1; + + // Load the Network + int r = loc_network_new_from_string(loc_ctx, &self->network, network); + if (r) { + PyErr_Format(PyExc_ValueError, "Invalid network: %s", network); + return -1; + } + + return 0; +} + +static PyObject* Network_repr(NetworkObject* self) { + const char* network = loc_network_str(self->network); + + return PyUnicode_FromFormat("", network); +} + +static PyObject* Network_str(NetworkObject* self) { + const char* network = loc_network_str(self->network); + + return PyUnicode_FromString(network); +} + +static PyObject* Network_get_country_code(NetworkObject* self) { + const char* country_code = loc_network_get_country_code(self->network); + + return PyUnicode_FromString(country_code); +} + +static int Network_set_country_code(NetworkObject* self, PyObject* value) { + const char* country_code = PyUnicode_AsUTF8(value); + + int r = loc_network_set_country_code(self->network, country_code); + if (r) { + if (r == -EINVAL) + PyErr_Format(PyExc_ValueError, + "Invalid country code: %s", country_code); + + return -1; + } + + return 0; +} + +static PyObject* Network_get_asn(NetworkObject* self) { + uint32_t asn = loc_network_get_asn(self->network); + + if (asn) + return PyLong_FromLong(asn); + + Py_RETURN_NONE; +} + +static int Network_set_asn(NetworkObject* self, PyObject* value) { + long int asn = PyLong_AsLong(value); + + // Check if the ASN is within the valid range + if (asn <= 0) { + PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn); + return -1; + } + +#if (__WORDSIZE > 32) + // Check whether the input was longer than 32 bit + if (asn > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn); + return -1; + } +#endif + + int r = loc_network_set_asn(self->network, asn); + if (r) + return -1; + + return 0; +} + +static PyObject* Network_has_flag(NetworkObject* self, PyObject* args) { + enum loc_network_flags flag = 0; + + if (!PyArg_ParseTuple(args, "i", &flag)) + return NULL; + + if (loc_network_has_flag(self->network, flag)) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { + enum loc_network_flags flag = 0; + + if (!PyArg_ParseTuple(args, "i", &flag)) + return NULL; + + int r = loc_network_set_flag(self->network, flag); + + if (r) { + // What exception to throw here? + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL; + + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) + return NULL; + + struct loc_network_list* list = loc_network_exclude(self->network, other->network); + + // Convert to Python objects + PyObject* obj = PyList_FromNetworkList(list); + loc_network_list_unref(list); + + return obj; +} + +static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL; + + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) + return NULL; + + if (loc_network_is_subnet(other->network, self->network)) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* Network_get_family(NetworkObject* self) { + int family = loc_network_address_family(self->network); + + return PyLong_FromLong(family); +} + +static PyObject* Network_get_first_address(NetworkObject* self) { + const char* address = loc_network_format_first_address(self->network); + + return PyUnicode_FromString(address); +} + +static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { + struct in_addr address4; + + // Convert IPv4 addresses to struct in_addr + if (IN6_IS_ADDR_V4MAPPED(address6)) { + address4.s_addr = address6->s6_addr32[3]; + + return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)); + } + + // Return IPv6 addresses as they are + return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); +} + +static PyObject* Network_get__first_address(NetworkObject* self) { + const struct in6_addr* address = loc_network_get_first_address(self->network); + + return PyBytes_FromAddress(address); +} + +static PyObject* Network_get_last_address(NetworkObject* self) { + const char* address = loc_network_format_last_address(self->network); + + return PyUnicode_FromString(address); +} + +static PyObject* Network_get__last_address(NetworkObject* self) { + const struct in6_addr* address = loc_network_get_last_address(self->network); + + return PyBytes_FromAddress(address); +} + +static struct PyMethodDef Network_methods[] = { + { + "exclude", + (PyCFunction)Network_exclude, + METH_VARARGS, + NULL, + }, + { + "has_flag", + (PyCFunction)Network_has_flag, + METH_VARARGS, + NULL, + }, + { + "is_subnet_of", + (PyCFunction)Network_is_subnet_of, + METH_VARARGS, + NULL, + }, + { + "set_flag", + (PyCFunction)Network_set_flag, + METH_VARARGS, + NULL, + }, + { NULL }, +}; + +static struct PyGetSetDef Network_getsetters[] = { + { + "asn", + (getter)Network_get_asn, + (setter)Network_set_asn, + NULL, + NULL, + }, + { + "country_code", + (getter)Network_get_country_code, + (setter)Network_set_country_code, + NULL, + NULL, + }, + { + "family", + (getter)Network_get_family, + NULL, + NULL, + NULL, + }, + { + "first_address", + (getter)Network_get_first_address, + NULL, + NULL, + NULL, + }, + { + "_first_address", + (getter)Network_get__first_address, + NULL, + NULL, + NULL, + }, + { + "last_address", + (getter)Network_get_last_address, + NULL, + NULL, + NULL, + }, + { + "_last_address", + (getter)Network_get__last_address, + NULL, + NULL, + NULL, + }, + { NULL }, +}; + +PyTypeObject NetworkType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "location.Network", + .tp_basicsize = sizeof(NetworkObject), + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_new = Network_new, + .tp_dealloc = (destructor)Network_dealloc, + .tp_init = (initproc)Network_init, + .tp_doc = "Network object", + .tp_methods = Network_methods, + .tp_getset = Network_getsetters, + .tp_repr = (reprfunc)Network_repr, + .tp_str = (reprfunc)Network_str, +}; diff --git a/src/python/network.h b/src/python/network.h new file mode 100644 index 0000000..b137e72 --- /dev/null +++ b/src/python/network.h @@ -0,0 +1,33 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef PYTHON_LOCATION_NETWORK_H +#define PYTHON_LOCATION_NETWORK_H + +#include + +#include + +typedef struct { + PyObject_HEAD + struct loc_network* network; +} NetworkObject; + +extern PyTypeObject NetworkType; + +PyObject* new_network(PyTypeObject* type, struct loc_network* network); + +#endif /* PYTHON_LOCATION_NETWORK_H */ diff --git a/src/python/writer.c b/src/python/writer.c new file mode 100644 index 0000000..c54596a --- /dev/null +++ b/src/python/writer.c @@ -0,0 +1,310 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include + +#include +#include + +#include "locationmodule.h" +#include "as.h" +#include "country.h" +#include "network.h" +#include "writer.h" + +static PyObject* Writer_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + WriterObject* self = (WriterObject*)type->tp_alloc(type, 0); + + return (PyObject*)self; +} + +static void Writer_dealloc(WriterObject* self) { + if (self->writer) + loc_writer_unref(self->writer); + + Py_TYPE(self)->tp_free((PyObject* )self); +} + +static int Writer_init(WriterObject* self, PyObject* args, PyObject* kwargs) { + PyObject* private_key1 = NULL; + PyObject* private_key2 = NULL; + FILE* f1 = NULL; + FILE* f2 = NULL; + int fd; + + // Parse arguments + if (!PyArg_ParseTuple(args, "|OO", &private_key1, &private_key2)) + return -1; + + // Ignore None + if (private_key1 == Py_None) { + Py_DECREF(private_key1); + private_key1 = NULL; + } + + if (private_key2 == Py_None) { + Py_DECREF(private_key2); + private_key2 = NULL; + } + + // Convert into FILE* + if (private_key1) { + fd = PyObject_AsFileDescriptor(private_key1); + if (fd < 0) + return -1; + + // Re-open file descriptor + f2 = fdopen(fd, "r"); + if (!f2) { + PyErr_SetFromErrno(PyExc_IOError); + return -1; + } + } + + if (private_key2) { + fd = PyObject_AsFileDescriptor(private_key2); + if (fd < 0) + return -1; + + // Re-open file descriptor + f2 = fdopen(fd, "r"); + if (!f2) { + PyErr_SetFromErrno(PyExc_IOError); + return -1; + } + } + + // Create the writer object + return loc_writer_new(loc_ctx, &self->writer, f1, f2); +} + +static PyObject* Writer_get_vendor(WriterObject* self) { + const char* vendor = loc_writer_get_vendor(self->writer); + + return PyUnicode_FromString(vendor); +} + +static int Writer_set_vendor(WriterObject* self, PyObject* value) { + const char* vendor = PyUnicode_AsUTF8(value); + + int r = loc_writer_set_vendor(self->writer, vendor); + if (r) { + PyErr_Format(PyExc_ValueError, "Could not set vendor: %s", vendor); + return r; + } + + return 0; +} + +static PyObject* Writer_get_description(WriterObject* self) { + const char* description = loc_writer_get_description(self->writer); + + return PyUnicode_FromString(description); +} + +static int Writer_set_description(WriterObject* self, PyObject* value) { + const char* description = PyUnicode_AsUTF8(value); + + int r = loc_writer_set_description(self->writer, description); + if (r) { + PyErr_Format(PyExc_ValueError, "Could not set description: %s", description); + return r; + } + + return 0; +} + +static PyObject* Writer_get_license(WriterObject* self) { + const char* license = loc_writer_get_license(self->writer); + + return PyUnicode_FromString(license); +} + +static int Writer_set_license(WriterObject* self, PyObject* value) { + const char* license = PyUnicode_AsUTF8(value); + + int r = loc_writer_set_license(self->writer, license); + if (r) { + PyErr_Format(PyExc_ValueError, "Could not set license: %s", license); + return r; + } + + return 0; +} + +static PyObject* Writer_add_as(WriterObject* self, PyObject* args) { + struct loc_as* as; + uint32_t number = 0; + + if (!PyArg_ParseTuple(args, "i", &number)) + return NULL; + + // Create AS object + int r = loc_writer_add_as(self->writer, &as, number); + if (r) + return NULL; + + PyObject* obj = new_as(&ASType, as); + loc_as_unref(as); + + return obj; +} + +static PyObject* Writer_add_country(WriterObject* self, PyObject* args) { + struct loc_country* country; + const char* country_code; + + if (!PyArg_ParseTuple(args, "s", &country_code)) + return NULL; + + // Create country object + int r = loc_writer_add_country(self->writer, &country, country_code); + if (r) { + switch (r) { + case -EINVAL: + PyErr_SetString(PyExc_ValueError, "Invalid network"); + break; + + default: + return NULL; + } + } + + PyObject* obj = new_country(&CountryType, country); + loc_country_unref(country); + + return obj; +} + +static PyObject* Writer_add_network(WriterObject* self, PyObject* args) { + struct loc_network* network; + const char* string = NULL; + + if (!PyArg_ParseTuple(args, "s", &string)) + return NULL; + + // Create network object + int r = loc_writer_add_network(self->writer, &network, string); + if (r) { + switch (r) { + case -EINVAL: + PyErr_SetString(PyExc_ValueError, "Invalid network"); + break; + + case -EBUSY: + PyErr_SetString(PyExc_IndexError, "A network already exists here"); + break; + } + + return NULL; + } + + PyObject* obj = new_network(&NetworkType, network); + loc_network_unref(network); + + return obj; +} + +static PyObject* Writer_write(WriterObject* self, PyObject* args) { + const char* path = NULL; + int version = LOC_DATABASE_VERSION_UNSET; + + if (!PyArg_ParseTuple(args, "s|i", &path, &version)) + return NULL; + + FILE* f = fopen(path, "w+"); + if (!f) { + PyErr_Format(PyExc_IOError, strerror(errno)); + return NULL; + } + + int r = loc_writer_write(self->writer, f, (enum loc_database_version)version); + fclose(f); + + // Raise any errors + if (r) { + PyErr_Format(PyExc_IOError, strerror(errno)); + return NULL; + } + + Py_RETURN_NONE; +} + +static struct PyMethodDef Writer_methods[] = { + { + "add_as", + (PyCFunction)Writer_add_as, + METH_VARARGS, + NULL, + }, + { + "add_country", + (PyCFunction)Writer_add_country, + METH_VARARGS, + NULL, + }, + { + "add_network", + (PyCFunction)Writer_add_network, + METH_VARARGS, + NULL, + }, + { + "write", + (PyCFunction)Writer_write, + METH_VARARGS, + NULL, + }, + { NULL }, +}; + +static struct PyGetSetDef Writer_getsetters[] = { + { + "description", + (getter)Writer_get_description, + (setter)Writer_set_description, + NULL, + NULL, + }, + { + "license", + (getter)Writer_get_license, + (setter)Writer_set_license, + NULL, + NULL, + }, + { + "vendor", + (getter)Writer_get_vendor, + (setter)Writer_set_vendor, + NULL, + NULL, + }, + { NULL }, +}; + +PyTypeObject WriterType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "location.Writer", + .tp_basicsize = sizeof(WriterObject), + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_new = Writer_new, + .tp_dealloc = (destructor)Writer_dealloc, + .tp_init = (initproc)Writer_init, + .tp_doc = "Writer object", + .tp_methods = Writer_methods, + .tp_getset = Writer_getsetters, +}; diff --git a/src/python/writer.h b/src/python/writer.h new file mode 100644 index 0000000..10ca26b --- /dev/null +++ b/src/python/writer.h @@ -0,0 +1,31 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef PYTHON_LOCATION_WRITER_H +#define PYTHON_LOCATION_WRITER_H + +#include + +#include + +typedef struct { + PyObject_HEAD + struct loc_writer* writer; +} WriterObject; + +extern PyTypeObject WriterType; + +#endif /* PYTHON_LOCATION_WRITER_H */ diff --git a/src/resolv.c b/src/resolv.c new file mode 100644 index 0000000..1c4cd75 --- /dev/null +++ b/src/resolv.c @@ -0,0 +1,150 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include + +#include +#include +#include + +static int parse_timestamp(const unsigned char* txt, time_t* t) { + struct tm ts; + + // Parse timestamp + char* p = strptime((const char*)txt, "%a, %d %b %Y %H:%M:%S GMT", &ts); + + // If the whole string has been parsed, we convert the parse value to time_t + if (p && !*p) { + *t = timegm(&ts); + + // Otherwise we reset t + } else { + *t = 0; + return -1; + } + + return 0; +} + +LOC_EXPORT int loc_discover_latest_version(struct loc_ctx* ctx, + unsigned int version, time_t* t) { + // Initialise the resolver + int r = res_init(); + if (r) { + ERROR(ctx, "res_init() failed\n"); + return r; + } + + // Make domain + char domain[64]; + snprintf(domain, 63, LOC_DATABASE_DOMAIN, version); + + unsigned char answer[PACKETSZ]; + int len; + + DEBUG(ctx, "Querying %s\n", domain); + + // Send a query + if ((len = res_query(domain, C_IN, T_TXT, answer, sizeof(answer))) < 0 || len > PACKETSZ) { + ERROR(ctx, "Could not query %s: \n", domain); + + return -1; + } + + unsigned char* end = answer + len; + unsigned char* payload = answer + sizeof(HEADER); + + // Expand domain name + char host[128]; + if ((len = dn_expand(answer, end, payload, host, sizeof(host))) < 0) { + ERROR(ctx, "dn_expand() failed\n"); + return -1; + } + + // Payload starts after hostname + payload += len; + + if (payload > end - 4) { + ERROR(ctx, "DNS reply too short\n"); + return -1; + } + + int type; + GETSHORT(type, payload); + if (type != T_TXT) { + ERROR(ctx, "DNS reply of unexpected type: %d\n", type); + return -1; + } + + // Skip class + payload += INT16SZ; + + // Walk through CNAMEs + unsigned int size = 0; + int ttl __attribute__ ((unused)); + do { + payload += size; + + if ((len = dn_expand(answer, end, payload, host, sizeof(host))) < 0) { + ERROR(ctx, "dn_expand() failed\n"); + return -1; + } + + payload += len; + + if (payload > end - 10) { + ERROR(ctx, "DNS reply too short\n"); + return -1; + } + + // Skip type, class, ttl + GETSHORT(type, payload); + payload += INT16SZ; + GETLONG(ttl, payload); + + // Read size + GETSHORT(size, payload); + if (payload + size < answer || payload + size > end) { + ERROR(ctx, "DNS RR overflow\n"); + return -1; + } + } while (type == T_CNAME); + + if (type != T_TXT) { + ERROR(ctx, "Not a TXT record\n"); + return -1; + } + + if (!size || (len = *payload) >= size || !len) { + ERROR(ctx, "Broken TXT record (len = %d, size = %d)\n", len, size); + return -1; + } + + // Get start of the string + unsigned char* txt = payload + 1; + txt[len] = '\0'; + + DEBUG(ctx, "Resolved to: %s\n", txt); + + // Parse timestamp + r = parse_timestamp(txt, t); + + return r; +} diff --git a/src/signing-key.pem b/src/signing-key.pem new file mode 100644 index 0000000..13931af --- /dev/null +++ b/src/signing-key.pem @@ -0,0 +1,8 @@ +Created: Dec 10, 2019 by Michael Tremer +Algorithm: NIST/SECG curve over a 521 bit prime field (secp521r1) +-----BEGIN PUBLIC KEY----- +MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQB1PzZdV9DE59LVCdXy/9cRgvTy9lx +L5tV10awZDilwQ9GR8x/irJE8ctGSnZ5HgbOk+gilurmC5JlJmTjZrW7tt8Awiu8 +ir3y2n7XXyiVGIzTHrA6Tw7SG+H9LzuIl0wCg6s6svnXVDyho7b0tSZPUGKMI28q +CUXef0jvZ9+ncTiJh1w= +-----END PUBLIC KEY----- diff --git a/src/stringpool.c b/src/stringpool.c new file mode 100644 index 0000000..187c9d3 --- /dev/null +++ b/src/stringpool.c @@ -0,0 +1,271 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +enum loc_stringpool_mode { + STRINGPOOL_DEFAULT, + STRINGPOOL_MMAP, +}; + +struct loc_stringpool { + struct loc_ctx* ctx; + int refcount; + + enum loc_stringpool_mode mode; + + char* data; + ssize_t length; + + char* pos; +}; + +static off_t loc_stringpool_get_offset(struct loc_stringpool* pool, const char* pos) { + if (pos < pool->data) + return -EFAULT; + + if (pos > (pool->data + pool->length)) + return -EFAULT; + + return pos - pool->data; +} + +static char* __loc_stringpool_get(struct loc_stringpool* pool, off_t offset) { + if (offset < 0 || offset >= pool->length) + return NULL; + + return pool->data + offset; +} + +static int loc_stringpool_grow(struct loc_stringpool* pool, size_t length) { + DEBUG(pool->ctx, "Growing string pool to %zu bytes\n", length); + + // Save pos pointer + off_t pos = loc_stringpool_get_offset(pool, pool->pos); + + // Reallocate data section + pool->data = realloc(pool->data, length); + if (!pool->data) + return -ENOMEM; + + pool->length = length; + + // Restore pos + pool->pos = __loc_stringpool_get(pool, pos); + + return 0; +} + +static off_t loc_stringpool_append(struct loc_stringpool* pool, const char* string) { + if (!string) + return -EINVAL; + + DEBUG(pool->ctx, "Appending '%s' to string pool at %p\n", string, pool); + + // Make sure we have enough space + int r = loc_stringpool_grow(pool, pool->length + strlen(string) + 1); + if (r) { + errno = r; + return -1; + } + + off_t offset = loc_stringpool_get_offset(pool, pool->pos); + + // Copy string byte by byte + while (*string) + *pool->pos++ = *string++; + + // Terminate the string + *pool->pos++ = '\0'; + + return offset; +} + +static void loc_stringpool_free(struct loc_stringpool* pool) { + DEBUG(pool->ctx, "Releasing string pool %p\n", pool); + int r; + + switch (pool->mode) { + case STRINGPOOL_DEFAULT: + if (pool->data) + free(pool->data); + break; + + case STRINGPOOL_MMAP: + if (pool->data) { + r = munmap(pool->data, pool->length); + if (r) + ERROR(pool->ctx, "Could not unmap data at %p: %s\n", + pool->data, strerror(errno)); + } + break; + } + + loc_unref(pool->ctx); + free(pool); +} + +int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool) { + struct loc_stringpool* p = calloc(1, sizeof(*p)); + if (!p) + return 1; + + p->ctx = loc_ref(ctx); + p->refcount = 1; + + // Save mode + p->mode = STRINGPOOL_DEFAULT; + + *pool = p; + + return 0; +} + +static int loc_stringpool_mmap(struct loc_stringpool* pool, FILE* f, size_t length, off_t offset) { + if (pool->mode != STRINGPOOL_MMAP) + return -EINVAL; + + DEBUG(pool->ctx, "Reading string pool starting from %jd (%zu bytes)\n", (intmax_t)offset, length); + + // Map file content into memory + pool->data = pool->pos = mmap(NULL, length, PROT_READ, + MAP_PRIVATE, fileno(f), offset); + + // Store size of section + pool->length = length; + + if (pool->data == MAP_FAILED) + return 1; + + return 0; +} + +int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool, + FILE* f, size_t length, off_t offset) { + struct loc_stringpool* p = NULL; + + // Allocate a new stringpool + int r = loc_stringpool_new(ctx, &p); + if (r) + return r; + + // Change mode to mmap + p->mode = STRINGPOOL_MMAP; + + // Map data into memory + if (length > 0) { + r = loc_stringpool_mmap(p, f, length, offset); + if (r) { + loc_stringpool_free(p); + return r; + } + } + + *pool = p; + return 0; +} + +struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool) { + pool->refcount++; + + return pool; +} + +struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool) { + if (--pool->refcount > 0) + return NULL; + + loc_stringpool_free(pool); + + return NULL; +} + +static off_t loc_stringpool_get_next_offset(struct loc_stringpool* pool, off_t offset) { + const char* string = loc_stringpool_get(pool, offset); + if (!string) + return offset; + + return offset + strlen(string) + 1; +} + +const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) { + return __loc_stringpool_get(pool, offset); +} + +size_t loc_stringpool_get_size(struct loc_stringpool* pool) { + return loc_stringpool_get_offset(pool, pool->pos); +} + +static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) { + if (!s || !*s) + return -EINVAL; + + off_t offset = 0; + while (offset < pool->length) { + const char* string = loc_stringpool_get(pool, offset); + if (!string) + break; + + int r = strcmp(s, string); + if (r == 0) + return offset; + + offset = loc_stringpool_get_next_offset(pool, offset); + } + + return -ENOENT; +} + +off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string) { + off_t offset = loc_stringpool_find(pool, string); + if (offset >= 0) { + DEBUG(pool->ctx, "Found '%s' at position %jd\n", string, (intmax_t)offset); + return offset; + } + + return loc_stringpool_append(pool, string); +} + +void loc_stringpool_dump(struct loc_stringpool* pool) { + off_t offset = 0; + + while (offset < pool->length) { + const char* string = loc_stringpool_get(pool, offset); + if (!string) + break; + + printf("%jd (%zu): %s\n", (intmax_t)offset, strlen(string), string); + + offset = loc_stringpool_get_next_offset(pool, offset); + } +} + +size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f) { + size_t size = loc_stringpool_get_size(pool); + + return fwrite(pool->data, sizeof(*pool->data), size, f); +} diff --git a/src/systemd/location-update.service.in b/src/systemd/location-update.service.in new file mode 100644 index 0000000..1c8e116 --- /dev/null +++ b/src/systemd/location-update.service.in @@ -0,0 +1,8 @@ +[Unit] +Description=Automatic Location Database Updater +Documentation=man:location(8) https://man-pages.ipfire.org/libloc/location.html +Requires=network.target + +[Service] +Type=oneshot +ExecStart=@bindir@/location update diff --git a/src/systemd/location-update.timer.in b/src/systemd/location-update.timer.in new file mode 100644 index 0000000..1b56a04 --- /dev/null +++ b/src/systemd/location-update.timer.in @@ -0,0 +1,9 @@ +[Unit] +Description=Update the location database once a day + +[Timer] +OnCalendar=daily +RandomizedDelaySec=24h + +[Install] +WantedBy=timers.target diff --git a/src/test-address.c b/src/test-address.c new file mode 100644 index 0000000..7012e41 --- /dev/null +++ b/src/test-address.c @@ -0,0 +1,138 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2022 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include + +#include +#include +#include + +static int perform_tests(struct loc_ctx* ctx, const int family) { + struct in6_addr address = IN6ADDR_ANY_INIT; + const char* e = NULL; + const char* s = NULL; + + // Reset IP address + loc_address_reset(&address, family); + + if (!loc_address_all_zeroes(&address)) { + fprintf(stderr, "IP address isn't all zeroes\n"); + return 1; + } + + if (loc_address_all_ones(&address)) { + fprintf(stderr, "IP address isn't all ones\n"); + return 1; + } + + switch (family) { + case AF_INET6: + e = "::"; + break; + + case AF_INET: + e = "0.0.0.0"; + break; + } + + // Convert this to a string a few times + for (unsigned int i = 0; i < 100; i++) { + s = loc_address_str(&address); + + printf("Iteration %d: %s\n", i, s); + + if (strcmp(s, e) != 0) { + fprintf(stderr, "IP address was formatted in an invalid format: %s\n", s); + return 1; + } + } + + // Increment the IP address + loc_address_increment(&address); + + switch (family) { + case AF_INET6: + e = "::1"; + break; + + case AF_INET: + e = "0.0.0.1"; + break; + } + + s = loc_address_str(&address); + + printf("Incremented IP address to %s\n", s); + + if (strcmp(s, e) != 0) { + printf("IP address has been incremented incorrectly: %s\n", s); + return 1; + } + + if (loc_address_all_zeroes(&address)) { + printf("IP address shouldn't be all zeroes any more\n"); + return 1; + } + + if (loc_address_all_ones(&address)) { + printf("IP address shouldn't be all ones any more\n"); + return 1; + } + + // Decrement the IP address + loc_address_decrement(&address); + + s = loc_address_str(&address); + + printf("Incremented IP address to %s\n", s); + + if (!loc_address_all_zeroes(&address)) { + printf("IP address hasn't been decremented correctly: %s\n", + loc_address_str(&address)); + return 1; + } + + return 0; +} + +int main(int argc, char** argv) { + struct loc_ctx* ctx = NULL; + int r = EXIT_FAILURE; + + int err = loc_new(&ctx); + if (err < 0) + exit(r); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Perform all tests for IPv6 + r = perform_tests(ctx, AF_INET6); + if (r) + goto ERROR; + + // Perform all tests for IPv4 + r = perform_tests(ctx, AF_INET); + if (r) + goto ERROR; + +ERROR: + loc_unref(ctx); + + return r; +} diff --git a/src/test-as.c b/src/test-as.c new file mode 100644 index 0000000..91b62fd --- /dev/null +++ b/src/test-as.c @@ -0,0 +1,128 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include + +#include +#include +#include + +#define TEST_AS_COUNT 5000 + +int main(int argc, char** argv) { + int err; + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); + if (err < 0) + exit(EXIT_FAILURE); + + char name[256]; + for (unsigned int i = 1; i <= TEST_AS_COUNT; i++) { + struct loc_as* as; + loc_writer_add_as(writer, &as, i); + + sprintf(name, "Test AS%u", i); + loc_as_set_name(as, name); + + loc_as_unref(as); + } + + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); + if (err) { + fprintf(stderr, "Could not write database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + loc_writer_unref(writer); + + // And open it again from disk + struct loc_database* db; + err = loc_database_new(ctx, &db, f); + if (err) { + fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + size_t as_count = loc_database_count_as(db); + if (as_count != TEST_AS_COUNT) { + fprintf(stderr, "Could not read all ASes\n"); + exit(EXIT_FAILURE); + } + + struct loc_as* as; + for (unsigned int i = 1; i <= 10; i++) { + err = loc_database_get_as(db, &as, i); + if (err) { + fprintf(stderr, "Could not find AS%d\n", i); + exit(EXIT_FAILURE); + } + + loc_as_unref(as); + } + + // Enumerator + + struct loc_database_enumerator* enumerator; + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); + if (err) { + fprintf(stderr, "Could not create a database enumerator\n"); + exit(EXIT_FAILURE); + } + + loc_database_enumerator_set_string(enumerator, "10"); + + err = loc_database_enumerator_next_as(enumerator, &as); + if (err) { + fprintf(stderr, "Could not enumerate next AS\n"); + exit(EXIT_FAILURE); + } + + while (as) { + printf("Found AS%d: %s\n", loc_as_get_number(as), loc_as_get_name(as)); + + err = loc_database_enumerator_next_as(enumerator, &as); + if (err) { + fprintf(stderr, "Could not enumerate next AS\n"); + exit(EXIT_FAILURE); + } + } + + loc_database_enumerator_unref(enumerator); + loc_database_unref(db); + loc_unref(ctx); + fclose(f); + + return EXIT_SUCCESS; +} diff --git a/src/test-country.c b/src/test-country.c new file mode 100644 index 0000000..c6aff49 --- /dev/null +++ b/src/test-country.c @@ -0,0 +1,195 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + struct loc_country* country; + int flag; + int err; + + // Check some valid country codes + if (!loc_country_code_is_valid("DE")) { + fprintf(stderr, "Valid country code detected as invalid: %s\n", "DE"); + exit(EXIT_FAILURE); + } + + // Check some invalid country codes + if (loc_country_code_is_valid("X1")) { + fprintf(stderr, "Invalid country code detected as valid: %s\n", "X1"); + exit(EXIT_FAILURE); + } + + // Test special country codes + flag = loc_country_special_code_to_flag("XX"); + if (flag) { + fprintf(stderr, "Unexpectedly received a flag for XX: %d\n", flag); + exit(EXIT_FAILURE); + } + + // A1 + flag = loc_country_special_code_to_flag("A1"); + if (flag != LOC_NETWORK_FLAG_ANONYMOUS_PROXY) { + fprintf(stderr, "Got a wrong flag for A1: %d\n", flag); + exit(EXIT_FAILURE); + } + + // A2 + flag = loc_country_special_code_to_flag("A2"); + if (flag != LOC_NETWORK_FLAG_SATELLITE_PROVIDER) { + fprintf(stderr, "Got a wrong flag for A2: %d\n", flag); + exit(EXIT_FAILURE); + } + + // A3 + flag = loc_country_special_code_to_flag("A3"); + if (flag != LOC_NETWORK_FLAG_ANYCAST) { + fprintf(stderr, "Got a wrong flag for A3: %d\n", flag); + exit(EXIT_FAILURE); + } + + // XD + flag = loc_country_special_code_to_flag("XD"); + if (flag != LOC_NETWORK_FLAG_DROP) { + fprintf(stderr, "Got a wrong flag for XD: %d\n", flag); + exit(EXIT_FAILURE); + } + + // NULL input + flag = loc_country_special_code_to_flag(NULL); + if (flag >= 0) { + fprintf(stderr, "loc_country_special_code_to_flag didn't throw an error for NULL\n"); + exit(EXIT_FAILURE); + } + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); + if (err < 0) + exit(EXIT_FAILURE); + + // Create a country + err = loc_writer_add_country(writer, &country, "DE"); + if (err) { + fprintf(stderr, "Could not create country\n"); + exit(EXIT_FAILURE); + } + + // Set name & continent + loc_country_set_name(country, "Testistan"); + loc_country_set_continent_code(country, "YY"); + + // Free country + loc_country_unref(country); + + // Add another country + err = loc_writer_add_country(writer, &country, "YY"); + if (err) { + fprintf(stderr, "Could not create country: YY\n"); + exit(EXIT_FAILURE); + } + loc_country_unref(country); + + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); + if (err) { + fprintf(stderr, "Could not write database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + loc_writer_unref(writer); + + // And open it again from disk + struct loc_database* db; + err = loc_database_new(ctx, &db, f); + if (err) { + fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + // Lookup an address in the subnet + err = loc_database_get_country(db, &country, "YY"); + if (err) { + fprintf(stderr, "Could not find country: YY\n"); + exit(EXIT_FAILURE); + } + loc_country_unref(country); + + struct loc_network* network = NULL; + + // Create a test network + err = loc_network_new_from_string(ctx, &network, "2001:db8::/64"); + if (err) { + fprintf(stderr, "Could not create network: %m\n"); + exit(EXIT_FAILURE); + } + + // Set country code & flag + loc_network_set_country_code(network, "YY"); + loc_network_set_flag(network, LOC_NETWORK_FLAG_ANONYMOUS_PROXY); + + // Check if this network matches its own country code + err = loc_network_matches_country_code(network, "YY"); + if (!err) { + fprintf(stderr, "Network does not match its own country code\n"); + exit(EXIT_FAILURE); + } + + // Check if this network matches the special country code + err = loc_network_matches_country_code(network, "A1"); + if (!err) { + fprintf(stderr, "Network does not match the special country code A1\n"); + exit(EXIT_FAILURE); + } + + // Check if this network does not match another special country code + err = loc_network_matches_country_code(network, "A2"); + if (err) { + fprintf(stderr, "Network matches another special country code A2\n"); + exit(EXIT_FAILURE); + } + + loc_network_unref(network); + + loc_database_unref(db); + loc_unref(ctx); + fclose(f); + + return EXIT_SUCCESS; +} diff --git a/src/test-database.c b/src/test-database.c new file mode 100644 index 0000000..7037f6a --- /dev/null +++ b/src/test-database.c @@ -0,0 +1,231 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +const char* VENDOR = "Test Vendor"; +const char* DESCRIPTION = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + "Proin ultrices pulvinar dolor, et sollicitudin eros ultricies " + "vitae. Nam in volutpat libero. Nulla facilisi. Pellentesque " + "tempor felis enim. Integer congue nisi in maximus pretium. " + "Pellentesque et turpis elementum, luctus mi at, interdum erat. " + "Maecenas ut venenatis nunc."; +const char* LICENSE = "CC"; + +const char* networks[] = { + "2001:db8::/32", + "2001:db8:1000::/48", + "2001:db8:2000::/48", + "2001:db8:2020::/48", + NULL, +}; + +static int attempt_to_open(struct loc_ctx* ctx, char* path) { + FILE* f = fopen(path, "r"); + if (!f) + return -1; + + struct loc_database* db; + int r = loc_database_new(ctx, &db, f); + + if (r == 0) { + fprintf(stderr, "Opening %s was unexpectedly successful\n", path); + loc_database_unref(db); + + r = 1; + } + + // Close the file again + fclose(f); + + return r; +} + +int main(int argc, char** argv) { + int err; + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Try opening an empty file + err = attempt_to_open(ctx, "/dev/null"); + if (err == 0) + exit(EXIT_FAILURE); + + // Try opening a file with all zeroes + err = attempt_to_open(ctx, "/dev/zero"); + if (err == 0) + exit(EXIT_FAILURE); + + // Try opening a file with random data + err = attempt_to_open(ctx, "/dev/urandom"); + if (err == 0) + exit(EXIT_FAILURE); + + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); + if (err < 0) + exit(EXIT_FAILURE); + + // Set the vendor + err = loc_writer_set_vendor(writer, VENDOR); + if (err) { + fprintf(stderr, "Could not set vendor\n"); + exit(EXIT_FAILURE); + } + + // Retrieve vendor + const char* vendor = loc_writer_get_vendor(writer); + if (vendor) { + printf("Vendor is: %s\n", vendor); + } else { + fprintf(stderr, "Could not retrieve vendor\n"); + exit(EXIT_FAILURE); + } + + // Set a description + err = loc_writer_set_description(writer, DESCRIPTION); + if (err) { + fprintf(stderr, "Could not set description\n"); + exit(EXIT_FAILURE); + } + + // Retrieve description + const char* description = loc_writer_get_description(writer); + if (description) { + printf("Description is: %s\n", description); + } else { + fprintf(stderr, "Could not retrieve description\n"); + exit(EXIT_FAILURE); + } + + // Set a license + err = loc_writer_set_license(writer, LICENSE); + if (err) { + fprintf(stderr, "Could not set license\n"); + exit(EXIT_FAILURE); + } + + // Retrieve license + const char* license = loc_writer_get_license(writer); + if (license) { + printf("License is: %s\n", license); + } else { + fprintf(stderr, "Could not retrieve license\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* network = NULL; + + // Add some networks + const char** n = networks; + while (*n) { + err = loc_writer_add_network(writer, &network, *n); + if (err) { + fprintf(stderr, "Could not add network %s\n", *n); + exit(EXIT_FAILURE); + } + + // Set a country + loc_network_set_country_code(network, "XX"); + + // Next one + n++; + } + + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); + if (err) { + fprintf(stderr, "Could not write database: %s\n", strerror(err)); + exit(EXIT_FAILURE); + } + loc_writer_unref(writer); + + // And open it again from disk + struct loc_database* db; + err = loc_database_new(ctx, &db, f); + if (err) { + fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + // Try reading something from the database + vendor = loc_database_get_vendor(db); + if (!vendor) { + fprintf(stderr, "Could not retrieve vendor\n"); + exit(EXIT_FAILURE); + } else if (strcmp(vendor, VENDOR) != 0) { + fprintf(stderr, "Vendor doesn't match: %s != %s\n", vendor, VENDOR); + exit(EXIT_FAILURE); + } + + // Enumerator + struct loc_database_enumerator* enumerator; + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); + if (err) { + fprintf(stderr, "Could not initialise the enumerator: %d\n", err); + exit(EXIT_FAILURE); + } + + // Walk through all networks + while (1) { + err = loc_database_enumerator_next_network(enumerator, &network); + if (err) { + fprintf(stderr, "Error fetching the next network: %d\n", err); + exit(EXIT_FAILURE); + } + + if (!network) + break; + + const char* s = loc_network_str(network); + printf("Got network: %s\n", s); + } + + // Free the enumerator + loc_database_enumerator_unref(enumerator); + + // Close the database + loc_database_unref(db); + loc_unref(ctx); + fclose(f); + + return EXIT_SUCCESS; +} diff --git a/src/test-libloc.c b/src/test-libloc.c new file mode 100644 index 0000000..41512e1 --- /dev/null +++ b/src/test-libloc.c @@ -0,0 +1,43 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, char** argv) { + struct loc_ctx *ctx; + + int err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + printf("version %s\n", VERSION); + + loc_unref(ctx); + return EXIT_SUCCESS; +} diff --git a/src/test-network-list.c b/src/test-network-list.c new file mode 100644 index 0000000..70a6b89 --- /dev/null +++ b/src/test-network-list.c @@ -0,0 +1,183 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +int main(int argc, char** argv) { + int err; + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Create a network + struct loc_network* network1; + err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); + if (err) { + fprintf(stderr, "Could not create the network1\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* subnet1; + err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); + if (err) { + fprintf(stderr, "Could not create the subnet1\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* subnet2; + err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); + if (err) { + fprintf(stderr, "Could not create the subnet2\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* subnet3; + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48"); + if (err) { + fprintf(stderr, "Could not create the subnet3\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* subnet4; + err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48"); + if (err) { + fprintf(stderr, "Could not create the subnet4\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* subnet5; + err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48"); + if (err) { + fprintf(stderr, "Could not create the subnet5\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* subnet6; + err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48"); + if (err) { + fprintf(stderr, "Could not create the subnet6\n"); + exit(EXIT_FAILURE); + } + + // Make a list with both subnets + struct loc_network_list* subnets; + err = loc_network_list_new(ctx, &subnets); + if (err) { + fprintf(stderr, "Could not create subnets list\n"); + exit(EXIT_FAILURE); + } + + size_t size = loc_network_list_size(subnets); + if (size > 0) { + fprintf(stderr, "The list is not empty: %zu\n", size); + exit(EXIT_FAILURE); + } + + err = loc_network_list_push(subnets, subnet1); + if (err) { + fprintf(stderr, "Could not add subnet1 to subnets list\n"); + exit(EXIT_FAILURE); + } + + if (loc_network_list_empty(subnets)) { + fprintf(stderr, "The subnets list reports that it is empty\n"); + exit(EXIT_FAILURE); + } + + err = loc_network_list_push(subnets, subnet2); + if (err) { + fprintf(stderr, "Could not add subnet2 to subnets list\n"); + exit(EXIT_FAILURE); + } + + // Add the fourth one next + err = loc_network_list_push(subnets, subnet4); + if (err) { + fprintf(stderr, "Could not add subnet4 to subnets list\n"); + exit(EXIT_FAILURE); + } + + // Add the third one + err = loc_network_list_push(subnets, subnet3); + if (err) { + fprintf(stderr, "Could not add subnet3 to subnets list\n"); + exit(EXIT_FAILURE); + } + + // Add more subnets + err = loc_network_list_push(subnets, subnet5); + if (err) { + fprintf(stderr, "Could not add subnet5 to subnets list\n"); + exit(EXIT_FAILURE); + } + + err = loc_network_list_push(subnets, subnet6); + if (err) { + fprintf(stderr, "Could not add subnet6 to subnets list\n"); + exit(EXIT_FAILURE); + } + + loc_network_list_dump(subnets); + + size = loc_network_list_size(subnets); + if (size != 6) { + fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); + exit(EXIT_FAILURE); + } + + // Exclude subnet1 from network1 + struct loc_network_list* excluded = loc_network_exclude(network1, subnet1); + if (!excluded) { + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n"); + exit(EXIT_FAILURE); + } + + loc_network_list_dump(excluded); + + // Exclude all subnets from network1 + excluded = loc_network_exclude_list(network1, subnets); + if (!excluded) { + fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n"); + exit(EXIT_FAILURE); + } + + loc_network_list_dump(excluded); + + if (excluded) + loc_network_list_unref(excluded); + + loc_network_list_unref(subnets); + loc_network_unref(network1); + loc_network_unref(subnet1); + loc_network_unref(subnet2); + loc_unref(ctx); + + return EXIT_SUCCESS; +} diff --git a/src/test-network.c b/src/test-network.c new file mode 100644 index 0000000..1c49d60 --- /dev/null +++ b/src/test-network.c @@ -0,0 +1,344 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + int err; + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + +#if 0 + struct loc_network_tree* tree; + err = loc_network_tree_new(ctx, &tree); + if (err) { + fprintf(stderr, "Could not create the network tree\n"); + exit(EXIT_FAILURE); + } +#endif + + struct in6_addr address; + err = inet_pton(AF_INET6, "2001:db8::1", &address); + if (err != 1) { + fprintf(stderr, "Could not parse IP address\n"); + exit(EXIT_FAILURE); + } + + // Create a network + struct loc_network* network1; + err = loc_network_new_from_string(ctx, &network1, "2001:db8::1/32"); + if (err) { + fprintf(stderr, "Could not create the network\n"); + exit(EXIT_FAILURE); + } + + err = loc_network_set_country_code(network1, "DE"); + if (err) { + fprintf(stderr, "Could not set country code\n"); + exit(EXIT_FAILURE); + } + +#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network1); + if (err) { + fprintf(stderr, "Could not add network to the tree\n"); + exit(EXIT_FAILURE); + } +#endif + + // Check if the first and last addresses are correct + const char* string = loc_network_format_first_address(network1); + if (!string) { + fprintf(stderr, "Did get NULL instead of a string for the first address\n"); + exit(EXIT_FAILURE); + } + + if (strcmp(string, "2001:db8::") != 0) { + fprintf(stderr, "Got an incorrect first address: %s\n", string); + exit(EXIT_FAILURE); + } + + string = loc_network_format_last_address(network1); + if (!string) { + fprintf(stderr, "Did get NULL instead of a string for the last address\n"); + exit(EXIT_FAILURE); + } + + if (strcmp(string, "2001:db8:ffff:ffff:ffff:ffff:ffff:ffff") != 0) { + fprintf(stderr, "Got an incorrect last address: %s\n", string); + exit(EXIT_FAILURE); + } + + err = loc_network_matches_address(network1, &address); + if (!err) { + fprintf(stderr, "Network1 does not match address\n"); + exit(EXIT_FAILURE); + } + + struct loc_network* network2; + err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); + if (err) { + fprintf(stderr, "Could not create the network\n"); + exit(EXIT_FAILURE); + } + + err = loc_network_set_country_code(network2, "DE"); + if (err) { + fprintf(stderr, "Could not set country code\n"); + exit(EXIT_FAILURE); + } + +#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network2); + if (err) { + fprintf(stderr, "Could not add network to the tree\n"); + exit(EXIT_FAILURE); + } + + // Dump the tree + err = loc_network_tree_dump(tree); + if (err) { + fprintf(stderr, "Error dumping tree: %d\n", err); + exit(EXIT_FAILURE); + } + + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); +#endif + + // Check equals function + err = loc_network_cmp(network1, network1); + if (err) { + fprintf(stderr, "Network is not equal with itself\n"); + exit(EXIT_FAILURE); + } + + err = loc_network_cmp(network1, network2); + if (!err) { + fprintf(stderr, "Networks equal unexpectedly\n"); + exit(EXIT_FAILURE); + } + + // Check subnet function + err = loc_network_is_subnet(network1, network2); + if (!err) { + fprintf(stderr, "Subnet check 1 failed: %d\n", err); + exit(EXIT_FAILURE); + } + + err = loc_network_is_subnet(network2, network1); + if (err) { + fprintf(stderr, "Subnet check 2 failed: %d\n", err); + exit(EXIT_FAILURE); + } + + // Make subnets + struct loc_network* subnet1 = NULL; + struct loc_network* subnet2 = NULL; + + err = loc_network_subnets(network1, &subnet1, &subnet2); + if (err || !subnet1 || !subnet2) { + fprintf(stderr, "Could not find subnets of network: %d\n", err); + exit(EXIT_FAILURE); + } + + const char* s = loc_network_str(subnet1); + printf("Received subnet1 = %s\n", s); + + s = loc_network_str(subnet2); + printf("Received subnet2 = %s\n", s); + + if (!loc_network_is_subnet(network1, subnet1)) { + fprintf(stderr, "Subnet1 is not a subnet\n"); + exit(EXIT_FAILURE); + } + + if (!loc_network_is_subnet(network1, subnet2)) { + fprintf(stderr, "Subnet2 is not a subnet\n"); + exit(EXIT_FAILURE); + } + + if (!loc_network_overlaps(network1, subnet1)) { + fprintf(stderr, "Network1 does not seem to contain subnet1\n"); + exit(EXIT_FAILURE); + } + + if (!loc_network_overlaps(network1, subnet2)) { + fprintf(stderr, "Network1 does not seem to contain subnet2\n"); + exit(EXIT_FAILURE); + } + + loc_network_unref(subnet1); + loc_network_unref(subnet2); + + struct loc_network_list* excluded = loc_network_exclude(network1, network2); + if (!excluded) { + fprintf(stderr, "Could not create excluded list\n"); + exit(EXIT_FAILURE); + } + + loc_network_list_dump(excluded); + loc_network_list_unref(excluded); + + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); + if (err < 0) + exit(EXIT_FAILURE); + + struct loc_network* network3; + err = loc_writer_add_network(writer, &network3, "2001:db8::/64"); + if (err) { + fprintf(stderr, "Could not add network\n"); + exit(EXIT_FAILURE); + } + + // Set country code + loc_network_set_country_code(network3, "XX"); + + struct loc_network* network4; + err = loc_writer_add_network(writer, &network4, "2001:db8:ffff::/64"); + if (err) { + fprintf(stderr, "Could not add network\n"); + exit(EXIT_FAILURE); + } + + // Set country code + loc_network_set_country_code(network4, "XY"); + + // Set ASN + loc_network_set_asn(network4, 1024); + + // Try adding an invalid network + struct loc_network* network; + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); + if (!err) { + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); + exit(EXIT_FAILURE); + } + + // Try adding a single address + err = loc_writer_add_network(writer, &network, "2001:db8::"); + if (err) { + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); + exit(EXIT_FAILURE); + } + + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); + if (err) { + fprintf(stderr, "Could not write database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + loc_writer_unref(writer); + + loc_network_unref(network1); + loc_network_unref(network2); + loc_network_unref(network3); + loc_network_unref(network4); + +#if 0 + loc_network_tree_unref(tree); +#endif + + // And open it again from disk + struct loc_database* db; + err = loc_database_new(ctx, &db, f); + if (err) { + fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + // Lookup an address in the subnet + err = loc_database_lookup_from_string(db, "2001:db8::", &network1); + if (err) { + fprintf(stderr, "Could not look up 2001:db8::\n"); + exit(EXIT_FAILURE); + } + loc_network_unref(network1); + + // Lookup an address outside the subnet + err = loc_database_lookup_from_string(db, "2001:db8:fffe:1::", &network1); + if (err == 0) { + fprintf(stderr, "Could look up 2001:db8:fffe:1::, but I shouldn't\n"); + exit(EXIT_FAILURE); + } + loc_network_unref(network1); + + const struct bit_length_test { + const char* network; + unsigned int bit_length; + } bit_length_tests[] = { + { "::/0", 0 }, + { "2001::/128", 126 }, + { "1.0.0.0/32", 25 }, + { "0.0.0.1/32", 1 }, + { "255.255.255.255/32", 32 }, + { NULL, 0, }, + }; + + for (const struct bit_length_test* t = bit_length_tests; t->network; t++) { + err = loc_network_new_from_string(ctx, &network1, t->network); + if (err) { + fprintf(stderr, "Could not create network %s: %m\n", t->network); + exit(EXIT_FAILURE); + } + + const struct in6_addr* addr = loc_network_get_first_address(network1); + + unsigned int bit_length = loc_address_bit_length(addr); + + if (bit_length != t->bit_length) { + printf("Bit length of %s didn't match: %u != %u\n", + t->network, t->bit_length, bit_length); + loc_network_unref(network1); + exit(EXIT_FAILURE); + } + + loc_network_unref(network1); + } + + loc_unref(ctx); + fclose(f); + + return EXIT_SUCCESS; +} diff --git a/src/test-signature.c b/src/test-signature.c new file mode 100644 index 0000000..bc66c96 --- /dev/null +++ b/src/test-signature.c @@ -0,0 +1,120 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2019 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +int main(int argc, char** argv) { + int err; + + // Open public key + FILE* public_key = fopen(ABS_SRCDIR "/examples/public-key.pem", "r"); + if (!public_key) { + fprintf(stderr, "Could not open public key file: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + // Open private key + FILE* private_key1 = fopen(ABS_SRCDIR "/examples/private-key.pem", "r"); + if (!private_key1) { + fprintf(stderr, "Could not open private key file: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + FILE* private_key2 = fopen(ABS_SRCDIR "/examples/private-key.pem", "r"); + if (!private_key2) { + fprintf(stderr, "Could not open private key file: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Create an empty database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, private_key1, private_key2); + if (err < 0) + exit(EXIT_FAILURE); + + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); + if (err) { + fprintf(stderr, "Could not write database: %s\n", strerror(err)); + exit(EXIT_FAILURE); + } + loc_writer_unref(writer); + + // And open it again from disk + struct loc_database* db; + err = loc_database_new(ctx, &db, f); + if (err) { + fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + // Verify the database signature + err = loc_database_verify(db, public_key); + if (err) { + fprintf(stderr, "Could not verify the database: %d\n", err); + exit(EXIT_FAILURE); + } + + // Open another public key + public_key = freopen(ABS_SRCDIR "/src/signing-key.pem", "r", public_key); + if (!public_key) { + fprintf(stderr, "Could not open public key file: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + // Verify with an incorrect key + err = loc_database_verify(db, public_key); + if (err == 0) { + fprintf(stderr, "Database was verified with an incorrect key: %d\n", err); + exit(EXIT_FAILURE); + } + + // Close the database + loc_database_unref(db); + loc_unref(ctx); + fclose(f); + + fclose(private_key1); + fclose(private_key2); + fclose(public_key); + + return EXIT_SUCCESS; +} diff --git a/src/test-stringpool.c b/src/test-stringpool.c new file mode 100644 index 0000000..392aa29 --- /dev/null +++ b/src/test-stringpool.c @@ -0,0 +1,123 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static const char* characters = "012345789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +static char* random_string(size_t size) { + char* string = malloc(size + 1); + + char* p = string; + for (unsigned int i = 0; i < size; i++) { + *p++ = characters[rand() % strlen(characters)]; + } + *p = '\0'; + + return string; +} + +int main(int argc, char** argv) { + // Initialize the RNG + time_t now = time(NULL); + srand(now); + + int err; + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + + // Create the stringpool + struct loc_stringpool* pool; + err = loc_stringpool_new(ctx, &pool); + if (err < 0) + exit(EXIT_FAILURE); + + // Try reading some invalid string + const char* s = loc_stringpool_get(pool, 100); + if (s != NULL) { + fprintf(stderr, "An unexpected string was returned: %s\n", s); + exit(EXIT_FAILURE); + } + + // Append a string + off_t pos = loc_stringpool_add(pool, "ABC"); + if (pos < 0) { + fprintf(stderr, "Could not add string: %s\n", strerror(-pos)); + exit(EXIT_FAILURE); + } + + printf("Added string at %jd\n", (intmax_t)pos); + + // Must start at first byte + if (pos != 0) { + fprintf(stderr, "First string didn't start at the first byte\n"); + exit(EXIT_FAILURE); + } + + // Append the same string again + pos = loc_stringpool_add(pool, "ABC"); + if (pos != 0) { + fprintf(stderr, "Same string was added at a different position again\n"); + exit(EXIT_FAILURE); + } + + // Append another string + pos = loc_stringpool_add(pool, "DEF"); + if (pos == 0) { + fprintf(stderr, "Second string was added at the first address\n"); + exit(EXIT_FAILURE); + } + + // Add 10000 random strings + for (unsigned int i = 0; i < 10000; i++) { + char* string = random_string(3); + + pos = loc_stringpool_add(pool, string); + free(string); + + if (pos < 0) { + fprintf(stderr, "Could not add string %d: %s\n", i, strerror(-pos)); + exit(EXIT_FAILURE); + } + } + + // Dump pool + loc_stringpool_dump(pool); + + loc_stringpool_unref(pool); + loc_unref(ctx); + + return EXIT_SUCCESS; +} diff --git a/src/writer.c b/src/writer.c new file mode 100644 index 0000000..4420972 --- /dev/null +++ b/src/writer.c @@ -0,0 +1,749 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ENDIAN_H +# include +#endif + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct loc_writer { + struct loc_ctx* ctx; + int refcount; + + struct loc_stringpool* pool; + off_t vendor; + off_t description; + off_t license; + + // Private keys to sign any databases + EVP_PKEY* private_key1; + EVP_PKEY* private_key2; + + // Signatures + char signature1[LOC_SIGNATURE_MAX_LENGTH]; + size_t signature1_length; + char signature2[LOC_SIGNATURE_MAX_LENGTH]; + size_t signature2_length; + + struct loc_network_tree* networks; + + struct loc_as_list* as_list; + struct loc_country_list* country_list; +}; + +static int parse_private_key(struct loc_writer* writer, EVP_PKEY** private_key, FILE* f) { + // Free any previously loaded keys + if (*private_key) + EVP_PKEY_free(*private_key); + + // Read the key + *private_key = PEM_read_PrivateKey(f, NULL, NULL, NULL); + + // Log any errors + if (!*private_key) { + char* error = ERR_error_string(ERR_get_error(), NULL); + ERROR(writer->ctx, "Could not parse private key: %s\n", error); + + return -1; + } + + return 0; +} + +LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer, + FILE* fkey1, FILE* fkey2) { + struct loc_writer* w = calloc(1, sizeof(*w)); + if (!w) + return -ENOMEM; + + w->ctx = loc_ref(ctx); + w->refcount = 1; + + int r = loc_stringpool_new(ctx, &w->pool); + if (r) { + loc_writer_unref(w); + return r; + } + + // Initialize the network tree + r = loc_network_tree_new(ctx, &w->networks); + if (r) { + loc_writer_unref(w); + return r; + } + + // Initialize AS list + r = loc_as_list_new(ctx, &w->as_list); + if (r) { + loc_writer_unref(w); + return r; + } + + // Initialize countries list + r = loc_country_list_new(ctx, &w->country_list); + if (r) { + loc_writer_unref(w); + return r; + } + + // Load the private keys to sign databases + if (fkey1) { + r = parse_private_key(w, &w->private_key1, fkey1); + if (r) { + loc_writer_unref(w); + return r; + } + } + + if (fkey2) { + r = parse_private_key(w, &w->private_key2, fkey2); + if (r) { + loc_writer_unref(w); + return r; + } + } + + *writer = w; + return 0; +} + +LOC_EXPORT struct loc_writer* loc_writer_ref(struct loc_writer* writer) { + writer->refcount++; + + return writer; +} + +static void loc_writer_free(struct loc_writer* writer) { + DEBUG(writer->ctx, "Releasing writer at %p\n", writer); + + // Free private keys + if (writer->private_key1) + EVP_PKEY_free(writer->private_key1); + if (writer->private_key2) + EVP_PKEY_free(writer->private_key2); + + // Unref all AS + if (writer->as_list) + loc_as_list_unref(writer->as_list); + + // Unref all countries + if (writer->country_list) + loc_country_list_unref(writer->country_list); + + // Release network tree + if (writer->networks) + loc_network_tree_unref(writer->networks); + + // Unref the string pool + loc_stringpool_unref(writer->pool); + + loc_unref(writer->ctx); + free(writer); +} + +LOC_EXPORT struct loc_writer* loc_writer_unref(struct loc_writer* writer) { + if (--writer->refcount > 0) + return writer; + + loc_writer_free(writer); + + return NULL; +} + +LOC_EXPORT const char* loc_writer_get_vendor(struct loc_writer* writer) { + return loc_stringpool_get(writer->pool, writer->vendor); +} + +LOC_EXPORT int loc_writer_set_vendor(struct loc_writer* writer, const char* vendor) { + // Add the string to the string pool + off_t offset = loc_stringpool_add(writer->pool, vendor); + if (offset < 0) + return offset; + + writer->vendor = offset; + return 0; +} + +LOC_EXPORT const char* loc_writer_get_description(struct loc_writer* writer) { + return loc_stringpool_get(writer->pool, writer->description); +} + +LOC_EXPORT int loc_writer_set_description(struct loc_writer* writer, const char* description) { + // Add the string to the string pool + off_t offset = loc_stringpool_add(writer->pool, description); + if (offset < 0) + return offset; + + writer->description = offset; + return 0; +} + +LOC_EXPORT const char* loc_writer_get_license(struct loc_writer* writer) { + return loc_stringpool_get(writer->pool, writer->license); +} + +LOC_EXPORT int loc_writer_set_license(struct loc_writer* writer, const char* license) { + // Add the string to the string pool + off_t offset = loc_stringpool_add(writer->pool, license); + if (offset < 0) + return offset; + + writer->license = offset; + return 0; +} + +LOC_EXPORT int loc_writer_add_as(struct loc_writer* writer, struct loc_as** as, uint32_t number) { + // Create a new AS object + int r = loc_as_new(writer->ctx, as, number); + if (r) + return r; + + // Append it to the list + return loc_as_list_append(writer->as_list, *as); +} + +LOC_EXPORT int loc_writer_add_network(struct loc_writer* writer, struct loc_network** network, const char* string) { + int r; + + // Create a new network object + r = loc_network_new_from_string(writer->ctx, network, string); + if (r) + return r; + + // Add it to the local tree + return loc_network_tree_add_network(writer->networks, *network); +} + +LOC_EXPORT int loc_writer_add_country(struct loc_writer* writer, struct loc_country** country, const char* country_code) { + // Allocate a new country + int r = loc_country_new(writer->ctx, country, country_code); + if (r) + return r; + + // Append it to the list + return loc_country_list_append(writer->country_list, *country); +} + +static void make_magic(struct loc_writer* writer, struct loc_database_magic* magic, + enum loc_database_version version) { + // Copy magic bytes + for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++) + magic->magic[i] = LOC_DATABASE_MAGIC[i]; + + // Set version + magic->version = version; +} + +static void align_page_boundary(off_t* offset, FILE* f) { + // Move to next page boundary + while (*offset % LOC_DATABASE_PAGE_SIZE > 0) + *offset += fwrite("", 1, 1, f); +} + +static int loc_database_write_pool(struct loc_writer* writer, + struct loc_database_header_v1* header, off_t* offset, FILE* f) { + // Save the offset of the pool section + DEBUG(writer->ctx, "Pool starts at %jd bytes\n", (intmax_t)*offset); + header->pool_offset = htobe32(*offset); + + // Write the pool + size_t pool_length = loc_stringpool_write(writer->pool, f); + *offset += pool_length; + + DEBUG(writer->ctx, "Pool has a length of %zu bytes\n", pool_length); + header->pool_length = htobe32(pool_length); + + return 0; +} + +static int loc_database_write_as_section(struct loc_writer* writer, + struct loc_database_header_v1* header, off_t* offset, FILE* f) { + DEBUG(writer->ctx, "AS section starts at %jd bytes\n", (intmax_t)*offset); + header->as_offset = htobe32(*offset); + + // Sort the AS list first + loc_as_list_sort(writer->as_list); + + const size_t as_count = loc_as_list_size(writer->as_list); + + struct loc_database_as_v1 block; + size_t block_length = 0; + + for (unsigned int i = 0; i < as_count; i++) { + struct loc_as* as = loc_as_list_get(writer->as_list, i); + if (!as) + return 1; + + // Convert AS into database format + loc_as_to_database_v1(as, writer->pool, &block); + + // Write to disk + *offset += fwrite(&block, 1, sizeof(block), f); + block_length += sizeof(block); + + // Unref AS + loc_as_unref(as); + } + + DEBUG(writer->ctx, "AS section has a length of %zu bytes\n", block_length); + header->as_length = htobe32(block_length); + + align_page_boundary(offset, f); + + return 0; +} + +struct node { + TAILQ_ENTRY(node) nodes; + + struct loc_network_tree_node* node; + + // Indices of the child nodes + uint32_t index_zero; + uint32_t index_one; +}; + +static struct node* make_node(struct loc_network_tree_node* node) { + struct node* n = malloc(sizeof(*n)); + if (!n) + return NULL; + + n->node = loc_network_tree_node_ref(node); + n->index_zero = n->index_one = 0; + + return n; +} + +static void free_node(struct node* node) { + loc_network_tree_node_unref(node->node); + + free(node); +} + +struct network { + TAILQ_ENTRY(network) networks; + + struct loc_network* network; +}; + +static struct network* make_network(struct loc_network* network) { + struct network* n = malloc(sizeof(*n)); + if (!n) + return NULL; + + n->network = loc_network_ref(network); + + return n; +} + +static void free_network(struct network* network) { + loc_network_unref(network->network); + + free(network); +} + +static int loc_database_write_networks(struct loc_writer* writer, + struct loc_database_header_v1* header, off_t* offset, FILE* f) { + // Write the network tree + DEBUG(writer->ctx, "Network tree starts at %jd bytes\n", (intmax_t)*offset); + header->network_tree_offset = htobe32(*offset); + + size_t network_tree_length = 0; + size_t network_data_length = 0; + + struct node* node; + struct node* child_node; + + uint32_t index = 0; + uint32_t network_index = 0; + + struct loc_database_network_v1 db_network; + struct loc_database_network_node_v1 db_node; + + // Initialize queue for nodes + TAILQ_HEAD(node_t, node) nodes; + TAILQ_INIT(&nodes); + + // Initialize queue for networks + TAILQ_HEAD(network_t, network) networks; + TAILQ_INIT(&networks); + + // Add root + struct loc_network_tree_node* root = loc_network_tree_get_root(writer->networks); + node = make_node(root); + if (!node) + return 1; + + TAILQ_INSERT_TAIL(&nodes, node, nodes); + + while (!TAILQ_EMPTY(&nodes)) { + // Pop first node in list + node = TAILQ_FIRST(&nodes); + TAILQ_REMOVE(&nodes, node, nodes); + + DEBUG(writer->ctx, "Processing node %p\n", node); + + // Get child nodes + struct loc_network_tree_node* node_zero = loc_network_tree_node_get(node->node, 0); + if (node_zero) { + node->index_zero = ++index; + + child_node = make_node(node_zero); + loc_network_tree_node_unref(node_zero); + + TAILQ_INSERT_TAIL(&nodes, child_node, nodes); + } + + struct loc_network_tree_node* node_one = loc_network_tree_node_get(node->node, 1); + if (node_one) { + node->index_one = ++index; + + child_node = make_node(node_one); + loc_network_tree_node_unref(node_one); + + TAILQ_INSERT_TAIL(&nodes, child_node, nodes); + } + + // Prepare what we are writing to disk + db_node.zero = htobe32(node->index_zero); + db_node.one = htobe32(node->index_one); + + if (loc_network_tree_node_is_leaf(node->node)) { + struct loc_network* network = loc_network_tree_node_get_network(node->node); + + // Append network to be written out later + struct network* nw = make_network(network); + if (!nw) { + free_node(node); + return 1; + } + TAILQ_INSERT_TAIL(&networks, nw, networks); + + db_node.network = htobe32(network_index++); + loc_network_unref(network); + } else { + db_node.network = htobe32(0xffffffff); + } + + // Write the current node + DEBUG(writer->ctx, "Writing node %p (0 = %d, 1 = %d)\n", + node, node->index_zero, node->index_one); + + *offset += fwrite(&db_node, 1, sizeof(db_node), f); + network_tree_length += sizeof(db_node); + + free_node(node); + } + + loc_network_tree_node_unref(root); + + header->network_tree_length = htobe32(network_tree_length); + + align_page_boundary(offset, f); + + DEBUG(writer->ctx, "Networks data section starts at %jd bytes\n", (intmax_t)*offset); + header->network_data_offset = htobe32(*offset); + + // We have now written the entire tree and have all networks + // in a queue in order as they are indexed + while (!TAILQ_EMPTY(&networks)) { + struct network* nw = TAILQ_FIRST(&networks); + TAILQ_REMOVE(&networks, nw, networks); + + // Prepare what we are writing to disk + int r = loc_network_to_database_v1(nw->network, &db_network); + if (r) + return r; + + *offset += fwrite(&db_network, 1, sizeof(db_network), f); + network_data_length += sizeof(db_network); + + free_network(nw); + } + + header->network_data_length = htobe32(network_data_length); + + align_page_boundary(offset, f); + + return 0; +} + +static int loc_database_write_countries(struct loc_writer* writer, + struct loc_database_header_v1* header, off_t* offset, FILE* f) { + DEBUG(writer->ctx, "Countries section starts at %jd bytes\n", (intmax_t)*offset); + header->countries_offset = htobe32(*offset); + + const size_t countries_count = loc_country_list_size(writer->country_list); + + struct loc_database_country_v1 block; + size_t block_length = 0; + + for (unsigned int i = 0; i < countries_count; i++) { + struct loc_country* country = loc_country_list_get(writer->country_list, i); + + // Convert country into database format + loc_country_to_database_v1(country, writer->pool, &block); + + // Write to disk + *offset += fwrite(&block, 1, sizeof(block), f); + block_length += sizeof(block); + } + + DEBUG(writer->ctx, "Countries section has a length of %zu bytes\n", block_length); + header->countries_length = htobe32(block_length); + + align_page_boundary(offset, f); + + return 0; +} + +static int loc_writer_create_signature(struct loc_writer* writer, + struct loc_database_header_v1* header, FILE* f, EVP_PKEY* private_key, + char* signature, size_t* length) { + DEBUG(writer->ctx, "Creating signature...\n"); + + // Read file from the beginning + rewind(f); + + // Create a new context for signing + EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); + + // Initialise the context + int r = EVP_DigestSignInit(mdctx, NULL, NULL, NULL, private_key); + if (r != 1) { + ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + goto END; + } + + // Read magic + struct loc_database_magic magic; + fread(&magic, 1, sizeof(magic), f); + + hexdump(writer->ctx, &magic, sizeof(magic)); + + // Feed magic into the signature + r = EVP_DigestSignUpdate(mdctx, &magic, sizeof(magic)); + if (r != 1) { + ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + goto END; + } + + hexdump(writer->ctx, header, sizeof(*header)); + + // Feed the header into the signature + r = EVP_DigestSignUpdate(mdctx, header, sizeof(*header)); + if (r != 1) { + ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + goto END; + } + + // Skip header + fseek(f, sizeof(*header), SEEK_CUR); + + // Walk through the file in chunks of 64kB + char buffer[64 * 1024]; + while (!feof(f)) { + size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); + + if (ferror(f)) { + ERROR(writer->ctx, "Error reading from file: %s\n", strerror(errno)); + r = errno; + goto END; + } + + hexdump(writer->ctx, buffer, bytes_read); + + r = EVP_DigestSignUpdate(mdctx, buffer, bytes_read); + if (r != 1) { + ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + r = -1; + goto END; + } + } + + // Compute the signature + r = EVP_DigestSignFinal(mdctx, + (unsigned char*)signature, length); + if (r != 1) { + ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL)); + r = -1; + goto END; + } + + DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *length); + r = 0; + + // Dump signature + hexdump(writer->ctx, signature, *length); + +END: + EVP_MD_CTX_free(mdctx); + + return r; +} + +LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f, enum loc_database_version version) { + // Check version + switch (version) { + case LOC_DATABASE_VERSION_UNSET: + version = LOC_DATABASE_VERSION_LATEST; + break; + + case LOC_DATABASE_VERSION_1: + break; + + default: + ERROR(writer->ctx, "Invalid database version: %d\n", version); + return -1; + } + + DEBUG(writer->ctx, "Writing database in version %d\n", version); + + struct loc_database_magic magic; + make_magic(writer, &magic, version); + + // Make the header + struct loc_database_header_v1 header; + header.vendor = htobe32(writer->vendor); + header.description = htobe32(writer->description); + header.license = htobe32(writer->license); + + time_t now = time(NULL); + header.created_at = htobe64(now); + + // Clear the signatures + memset(header.signature1, '\0', sizeof(header.signature1)); + header.signature1_length = 0; + memset(header.signature2, '\0', sizeof(header.signature2)); + header.signature2_length = 0; + + // Clear the padding + memset(header.padding, '\0', sizeof(header.padding)); + + int r; + off_t offset = 0; + + // Start writing at the beginning of the file + r = fseek(f, 0, SEEK_SET); + if (r) + return r; + + // Write the magic + offset += fwrite(&magic, 1, sizeof(magic), f); + + // Skip the space we need to write the header later + r = fseek(f, sizeof(header), SEEK_CUR); + if (r) { + DEBUG(writer->ctx, "Could not seek to position after header\n"); + return r; + } + offset += sizeof(header); + + align_page_boundary(&offset, f); + + // Write all ASes + r = loc_database_write_as_section(writer, &header, &offset, f); + if (r) + return r; + + // Write all networks + r = loc_database_write_networks(writer, &header, &offset, f); + if (r) + return r; + + // Write countries + r = loc_database_write_countries(writer, &header, &offset, f); + if (r) + return r; + + // Write pool + r = loc_database_write_pool(writer, &header, &offset, f); + if (r) + return r; + + // Create the signatures + if (writer->private_key1) { + DEBUG(writer->ctx, "Creating signature with first private key\n"); + + writer->signature1_length = sizeof(writer->signature1); + + r = loc_writer_create_signature(writer, &header, f, + writer->private_key1, writer->signature1, &writer->signature1_length); + if (r) + return r; + } + + if (writer->private_key2) { + DEBUG(writer->ctx, "Creating signature with second private key\n"); + + writer->signature2_length = sizeof(writer->signature2); + + r = loc_writer_create_signature(writer, &header, f, + writer->private_key2, writer->signature2, &writer->signature2_length); + if (r) + return r; + } + + // Copy the signatures into the header + if (writer->signature1_length) { + DEBUG(writer->ctx, "Copying first signature of %zu byte(s)\n", + writer->signature1_length); + + memcpy(header.signature1, writer->signature1, writer->signature1_length); + header.signature1_length = htobe16(writer->signature1_length); + } + + if (writer->signature2_length) { + DEBUG(writer->ctx, "Copying second signature of %zu byte(s)\n", + writer->signature1_length); + + memcpy(header.signature2, writer->signature2, writer->signature2_length); + header.signature2_length = htobe16(writer->signature2_length); + } + + // Write the header + r = fseek(f, sizeof(magic), SEEK_SET); + if (r) + return r; + + fwrite(&header, 1, sizeof(header), f); + + return r; +} diff --git a/tests/data/location-2022-03-30.db b/tests/data/location-2022-03-30.db new file mode 100644 index 0000000..fff8d34 Binary files /dev/null and b/tests/data/location-2022-03-30.db differ diff --git a/tests/python/test-export.py b/tests/python/test-export.py new file mode 100755 index 0000000..419b105 --- /dev/null +++ b/tests/python/test-export.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2022 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import location +import os +import unittest + +TEST_DATA_DIR = os.environ["TEST_DATA_DIR"] + +class Test(unittest.TestCase): + def setUp(self): + path = os.path.join(TEST_DATA_DIR, "location-2022-03-30.db") + + # Load the database + self.db = location.Database(path) + + def test_list_networks(self): + """ + Lists all available networks + """ + for network in self.db.networks: + print(network) + + def test_list_networks_flattened(self): + """ + Lists all networks but flattened + """ + for i, network in enumerate(self.db.networks_flattened): + # Break after the first 1000 iterations + if i >= 1000: + break + + print(network) + + +if __name__ == "__main__": + unittest.main()