src/loc/network.h \
src/loc/private.h \
src/loc/stringpool.h \
+ src/loc/resolv.h \
src/loc/writer.h
lib_LTLIBRARIES = \
src/country.c \
src/database.c \
src/network.c \
+ src/resolv.c \
src/stringpool.c \
src/writer.c
src_libloc_la_LDFLAGS += -export-symbols $(top_srcdir)/src/libloc.sym
endif
+src_libloc_la_LIBADD = \
+ $(RESOLV_LIBS)
+
src_libloc_la_DEPENDENCIES = \
${top_srcdir}/src/libloc.sym
AC_CHECK_HEADERS_ONCE([
arpa/inet.h \
+ arpa/nameser.h \
endian.h \
netinet/in.h \
+ resolv.h \
string.h \
])
AC_CHECK_FUNCS([ \
- be16toh \
- be32toh \
- be64toh \
- htobe16 \
- htobe32 \
- htobe64 \
- mmap \
- munmap \
+ be16toh \
+ be32toh \
+ be64toh \
+ htobe16 \
+ htobe32 \
+ htobe64 \
+ mmap \
+ munmap \
+ res_query \
__secure_getenv \
secure_getenv \
- qsort \
+ qsort \
])
my_CFLAGS="\
AC_ARG_ENABLE(perl, AS_HELP_STRING([--disable-perl], [do not build the perl modules]), [],[enable_perl=yes])
AM_CONDITIONAL(ENABLE_PERL, test "$enable_perl" = "yes")
+dnl Checking for libresolv
+AC_CHECK_LIB(resolv, ns_msg_getflag, [LIBS="-lresolv $LIBS"], AC_MSG_ERROR([libresolv has not been found]), -lresolv)
+RESOLV_LIBS="${LIBS}"
+
AC_CONFIG_HEADERS(config.h)
AC_CONFIG_FILES([
Makefile
If there has been a problem and the requested action could not be performed,
the exit code is unequal to zero.
+== HOW IT WORKS
+The downloader checks a DNS record for the latest version of the database.
+It will then try to download a file with that version from a mirror server.
+If the downloaded file is outdated, the next mirror will be tried until we
+have found a file that is recent enough.
+
== BUGS
Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.
loc_unref;
loc_set_log_priority;
loc_new;
+ loc_discover_latest_version;
# AS
loc_as_cmp;
loc_database_enumerator_ref;
loc_database_enumerator_set_asn;
loc_database_enumerator_set_country_code;
+ loc_database_enumerator_set_flag;
loc_database_enumerator_set_string;
loc_database_enumerator_unref;
#define LOC_DATABASE_VERSION 0
+#define STR(x) #x
+#define LOC_DATABASE_DOMAIN_LATEST(version) "_latest._v" STR(version) ".location.ipfire.org"
+
#define LOC_DATABASE_PAGE_SIZE 4096
struct loc_database_magic {
--- /dev/null
+/*
+ libloc - A library to determine the location of someone on the Internet
+
+ Copyright (C) 2019 IPFire Development Team <info@ipfire.org>
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+*/
+
+#ifndef LIBLOC_RESOLV_H
+#define LIBLOC_RESOLV_H
+
+#include <time.h>
+
+#include <loc/libloc.h>
+
+int loc_discover_latest_version(struct loc_ctx* ctx, const char* domain, time_t* t);
+
+#endif
###############################################################################
import argparse
+import datetime
import gettext
import logging
import logging.handlers
return gettext.dgettext("libloc", singular)
-class NotModifiedError(Exception):
- """
- Raised when the file has not been modified on the server
- """
- pass
-
class Downloader(object):
def __init__(self, mirrors):
for header in e.headers:
log.debug(" %s: %s" % (header, e.headers[header]))
- # Handle 304
- if e.code == 304:
- raise NotModifiedError() from e
-
# Raise all other errors
raise e
return res
- def download(self, url, mtime=None, **kwargs):
+ def download(self, url, timestamp=None, **kwargs):
headers = {}
- if mtime:
- headers["If-Modified-Since"] = time.strftime(
- "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(mtime),
+ if timestamp:
+ headers["If-Modified-Since"] = timestamp.strftime(
+ "%a, %d %b %Y %H:%M:%S GMT",
)
t = tempfile.NamedTemporaryFile(delete=False)
if buf:
t.write(buf)
- # Write all data to disk
- t.flush()
-
- # Nothing to do when the database on the server is up to date
- except NotModifiedError:
- log.info("Local database is up to date")
- return
+ # Write all data to disk
+ t.flush()
# Catch decompression errors
except lzma.LZMAError as e:
log.warning("Could not decompress downloaded file: %s" % e)
continue
- # XXX what do we catch here?
except urllib.error.HTTPError as e:
- if e.code == 404:
- continue
+ # The file on the server was too old
+ if e.code == 304:
+ log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
- # Truncate the target file and drop downloaded content
- try:
- t.truncate()
- except OSError:
- pass
+ # Log any other HTTP errors
+ else:
+ log.warning("%s reported: %s" % (mirror, e))
+
+ # Throw away any downloaded content and try again
+ t.truncate()
- raise e
+ else:
+ # Check if the downloaded database is recent
+ if not self._check_database(t, timestamp):
+ log.warning("Downloaded database is outdated. Trying next mirror...")
- # Return temporary file
- return t
+ # Throw away the data and try again
+ t.truncate()
+ continue
+
+ # Return temporary file
+ return t
raise FileNotFoundError(url)
+ def _check_database(self, f, timestamp=None):
+ """
+ Checks the downloaded database if it can be opened,
+ verified and if it is recent enough
+ """
+ log.debug("Opening downloaded database at %s" % f.name)
+
+ db = location.Database(f.name)
+
+ # Database is not recent
+ if timestamp and db.created_at < timestamp.timestamp():
+ return False
+
+ log.info("Downloaded new database from %s" % (time.strftime(
+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
+ )))
+
+ return True
+
class CLI(object):
def __init__(self):
sys.exit(0)
def handle_update(self, ns):
- mtime = None
+ # Fetch the version we need from DNS
+ t = location.discover_latest_version()
+
+ # Parse timestamp into datetime format
+ try:
+ timestamp = datetime.datetime.fromtimestamp(t)
+ except:
+ raise
# Open database
try:
db = location.Database(ns.database)
- # Get mtime of the old file
- mtime = os.path.getmtime(ns.database)
+ # Check if we are already on the latest version
+ if db.created_at >= timestamp.timestamp():
+ log.info("Already on the latest version")
+ return
+
except FileNotFoundError as e:
db = None
# Try downloading a new database
try:
- t = self.downloader.download(DATABASE_FILENAME, mtime=mtime)
+ t = self.downloader.download(DATABASE_FILENAME, timestamp=timestamp)
# If no file could be downloaded, log a message
except FileNotFoundError as e:
if not t:
return 0
- # Save old database creation time
- created_at = db.created_at if db else 0
-
- # Try opening the downloaded file
- try:
- db = location.Database(t.name)
- except Exception as e:
- raise e
-
- # Check if the downloaded file is newer
- if db.created_at <= created_at:
- log.warning("Downloaded database is older than the current version")
- return 1
-
- log.info("Downloaded new database from %s" % (time.strftime(
- "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
- )))
-
# Write temporary file to destination
shutil.copyfile(t.name, ns.database)
#include <Python.h>
#include <syslog.h>
+#include <loc/resolv.h>
+
#include "locationmodule.h"
#include "as.h"
#include "country.h"
Py_RETURN_NONE;
}
+static PyObject* discover_latest_version(PyObject* m, PyObject* args) {
+ const char* domain = NULL;
+
+ if (!PyArg_ParseTuple(args, "|s", &domain))
+ return NULL;
+
+ time_t t = 0;
+
+ int r = loc_discover_latest_version(loc_ctx, domain, &t);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_OSError);
+ return NULL;
+ }
+
+ return PyLong_FromUnsignedLong(t);
+}
+
static PyMethodDef location_module_methods[] = {
+ {
+ "discover_latest_version",
+ (PyCFunction)discover_latest_version,
+ METH_VARARGS,
+ NULL,
+ },
{
"set_log_level",
(PyCFunction)set_log_level,
--- /dev/null
+/*
+ libloc - A library to determine the location of someone on the Internet
+
+ Copyright (C) 2019 IPFire Development Team <info@ipfire.org>
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+*/
+
+#include <resolv.h>
+#include <string.h>
+#include <time.h>
+
+#include <loc/format.h>
+#include <loc/private.h>
+#include <loc/resolv.h>
+
+static int parse_timestamp(const unsigned char* txt, time_t* t) {
+ struct tm ts;
+
+ // Parse timestamp
+ char* p = strptime((const char*)txt, "%a, %d %b %Y %H:%M:%S GMT", &ts);
+
+ // If the whole string has been parsed, we convert the parse value to time_t
+ if (p && !*p) {
+ *t = mktime(&ts);
+
+ // Otherwise we reset t
+ } else {
+ *t = 0;
+ return -1;
+ }
+
+ return 0;
+}
+
+LOC_EXPORT int loc_discover_latest_version(struct loc_ctx* ctx, const char* domain, time_t* t) {
+ // Initialise the resolver
+ int r = res_init();
+ if (r) {
+ ERROR(ctx, "res_init() failed\n");
+ return r;
+ }
+
+ // Fall back to default domain
+ if (!domain)
+ domain = LOC_DATABASE_DOMAIN_LATEST(LOC_DATABASE_VERSION);
+
+ unsigned char answer[PACKETSZ];
+ int len;
+
+ DEBUG(ctx, "Querying %s\n", domain);
+
+ // Send a query
+ if ((len = res_query(domain, C_IN, T_TXT, answer, sizeof(answer))) < 0 || len > PACKETSZ) {
+ ERROR(ctx, "Could not query %s: \n", domain);
+
+ return -1;
+ }
+
+ unsigned char* end = answer + len;
+ unsigned char* payload = answer + sizeof(HEADER);
+
+ // Expand domain name
+ char host[128];
+ if ((len = dn_expand(answer, end, payload, host, sizeof(host))) < 0) {
+ ERROR(ctx, "dn_expand() failed\n");
+ return -1;
+ }
+
+ // Payload starts after hostname
+ payload += len;
+
+ if (payload > end - 4) {
+ ERROR(ctx, "DNS reply too short\n");
+ return -1;
+ }
+
+ int type;
+ GETSHORT(type, payload);
+ if (type != T_TXT) {
+ ERROR(ctx, "DNS reply of unexpected type: %d\n", type);
+ return -1;
+ }
+
+ // Skip class
+ payload += INT16SZ;
+
+ // Walk through CNAMEs
+ unsigned int size = 0;
+ int ttl;
+ do {
+ payload += size;
+
+ if ((len = dn_expand(answer, end, payload, host, sizeof(host))) < 0) {
+ ERROR(ctx, "dn_expand() failed\n");
+ return -1;
+ }
+
+ payload += len;
+
+ if (payload > end - 10) {
+ ERROR(ctx, "DNS reply too short\n");
+ return -1;
+ }
+
+ // Skip type, class, ttl
+ GETSHORT(type, payload);
+ payload += INT16SZ;
+ GETLONG(ttl, payload);
+
+ // Read size
+ GETSHORT(size, payload);
+ if (payload + size < answer || payload + size > end) {
+ ERROR(ctx, "DNS RR overflow\n");
+ return -1;
+ }
+ } while (type == T_CNAME);
+
+ if (type != T_TXT) {
+ ERROR(ctx, "Not a TXT record\n");
+ return -1;
+ }
+
+ if (!size || (len = *payload) >= size || !len) {
+ ERROR(ctx, "Broken TXT record (len = %d, size = %d)\n", len, size);
+ return -1;
+ }
+
+ // Get start of the string
+ unsigned char* txt = payload + 1;
+ txt[len] = '\0';
+
+ DEBUG(ctx, "Resolved to: %s\n", txt);
+
+ // Parse timestamp
+ r = parse_timestamp(txt, t);
+
+ return r;
+}
projects / location / libloc.git / commitdiff
