+From 627bf1daaae1510cfd4016297ed16b82df209aae Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 18 Nov 2020 13:33:45 +0000
+Subject: [PATCH 71/77] python: Remove unnecessary db object from writers
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 12 ++++++------
+ 1 file changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 4219957..5bc9f30 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -39,8 +39,8 @@ class OutputWriter(object):
+ suffix = "networks"
+ mode = "w"
+
+- def __init__(self, db, f, prefix=None, flatten=True):
+- self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
++ def __init__(self, f, prefix=None, flatten=True):
++ self.f, self.prefix, self.flatten = f, prefix, flatten
+
+ # The previously written network
+ self._last_network = None
+@@ -49,13 +49,13 @@ class OutputWriter(object):
+ self._write_header()
+
+ @classmethod
+- def open(cls, db, filename, **kwargs):
++ def open(cls, filename, **kwargs):
+ """
+ Convenience function to open a file
+ """
+ f = open(filename, cls.mode)
+
+- return cls(db, f, **kwargs)
++ return cls(f, **kwargs)
+
+ def __repr__(self):
+ return "<%s f=%s>" % (self.__class__.__name__, self.f)
+@@ -172,7 +172,7 @@ class Exporter(object):
+ directory, prefix=country_code, suffix=self.writer.suffix, family=family,
+ )
+
+- writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
++ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
+
+ # Create writers for ASNs
+ for asn in asns:
+@@ -180,7 +180,7 @@ class Exporter(object):
+ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
+ )
+
+- writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
++ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
+
+ # Filter countries from special country codes
+ country_codes = [
+--
+2.20.1
+
+From 9cb56ac9adafafa6e452009c2fa2d42e94474e11 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed, 18 Nov 2020 13:34:50 +0000
+Subject: [PATCH 72/77] location: End lookup after an invalid IP address was
+ passed
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/location.in | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/python/location.in b/src/python/location.in
+index 0d09210..6885ea0 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -253,6 +253,7 @@ class CLI(object):
+ network = db.lookup(address)
+ except ValueError:
+ print(_("Invalid IP address: %s") % address, file=sys.stderr)
++ return 2
+
+ args = {
+ "address" : address,
+--
+2.20.1
+
+From 2a550d12208f8bc8002e05ac08613312df26b20e Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 19 Nov 2020 12:03:33 +0000
+Subject: [PATCH 73/77] python: Fix download of database
+
+This was all messed up in 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1
+when the change of type for the timestamp wasn't changed everywhere
+else.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/downloader.py | 6 +++---
+ src/python/location.in | 2 +-
+ 2 files changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/src/python/downloader.py b/src/python/downloader.py
+index 87bbb68..05f7872 100644
+--- a/src/python/downloader.py
++++ b/src/python/downloader.py
+@@ -119,8 +119,8 @@ class Downloader(object):
+
+ headers = {}
+ if timestamp:
+- headers["If-Modified-Since"] = timestamp.strftime(
+- "%a, %d %b %Y %H:%M:%S GMT",
++ headers["If-Modified-Since"] = time.strftime(
++ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
+ )
+
+ t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
+@@ -195,7 +195,7 @@ class Downloader(object):
+ db = Database(f.name)
+
+ # Database is not recent
+- if timestamp and db.created_at < timestamp.timestamp():
++ if timestamp and db.created_at < timestamp:
+ return False
+
+ log.info("Downloaded new database from %s" % (time.strftime(
+diff --git a/src/python/location.in b/src/python/location.in
+index 6885ea0..b30beae 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -433,7 +433,7 @@ class CLI(object):
+
+ # Try downloading a new database
+ try:
+- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
++ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
+
+ # If no file could be downloaded, log a message
+ except FileNotFoundError as e:
+--
+2.20.1
+
+From a1a00053300cff3c0f690d52377c76c83c2a08b2 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 19 Nov 2020 12:34:11 +0000
+Subject: [PATCH 74/77] python: Add property to return IP addresses as bytes
+
+This avoids calling inet_pton to parse IP addresses from string
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/libloc.sym | 2 ++
+ src/loc/network.h | 2 ++
+ src/network.c | 8 ++++++++
+ src/python/database.c | 2 +-
+ src/python/export.py | 7 ++-----
+ src/python/network.c | 40 ++++++++++++++++++++++++++++++++++++++++
+ 6 files changed, 55 insertions(+), 6 deletions(-)
+
+diff --git a/src/libloc.sym b/src/libloc.sym
+index 53273cd..406dd15 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -113,6 +113,8 @@ global:
+ loc_network_format_last_address;
+ loc_network_get_asn;
+ loc_network_get_country_code;
++ loc_network_get_first_address;
++ loc_network_get_last_address;
+ loc_network_gt;
+ loc_network_has_flag;
+ loc_network_is_subnet;
+diff --git a/src/loc/network.h b/src/loc/network.h
+index d86b685..4b7410c 100644
+--- a/src/loc/network.h
++++ b/src/loc/network.h
+@@ -39,7 +39,9 @@ struct loc_network* loc_network_unref(struct loc_network* network);
+ char* loc_network_str(struct loc_network* network);
+ int loc_network_address_family(struct loc_network* network);
+
++const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
+ char* loc_network_format_first_address(struct loc_network* network);
++const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
+ char* loc_network_format_last_address(struct loc_network* network);
+ int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
+
+diff --git a/src/network.c b/src/network.c
+index 28ca2df..4c8787a 100644
+--- a/src/network.c
++++ b/src/network.c
+@@ -343,10 +343,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
+ return string;
+ }
+
++LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
++ return &network->first_address;
++}
++
+ LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
+ return loc_network_format_address(network, &network->first_address);
+ }
+
++LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
++ return &network->last_address;
++}
++
+ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
+ return loc_network_format_address(network, &network->last_address);
+ }
+diff --git a/src/python/database.c b/src/python/database.c
+index ed22275..f385c61 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -329,7 +329,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ if (r) {
+ PyErr_SetFromErrno(PyExc_SystemError);
+
+- loc_as_list_unref(countries);
++ loc_country_list_unref(countries);
+ return NULL;
+ }
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 5bc9f30..6b39878 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -142,11 +142,8 @@ class XTGeoIPOutputWriter(OutputWriter):
+ mode = "wb"
+
+ def _write_network(self, network):
+- for address in (network.first_address, network.last_address):
+- # Convert this into a string of bits
+- bytes = socket.inet_pton(network.family, address)
+-
+- self.f.write(bytes)
++ for address in (network._first_address, network._last_address):
++ self.f.write(address)
+
+
+ formats = {
+diff --git a/src/python/network.c b/src/python/network.c
+index ed91d65..742b472 100644
+--- a/src/python/network.c
++++ b/src/python/network.c
+@@ -215,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
+ return obj;
+ }
+
++static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
++ struct in_addr address4;
++
++ // Convert IPv4 addresses to struct in_addr
++ if (IN6_IS_ADDR_V4MAPPED(address6)) {
++ address4.s_addr = address6->s6_addr32[3];
++
++ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
++ }
++
++ // Return IPv6 addresses as they are
++ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
++}
++
++static PyObject* Network_get__first_address(NetworkObject* self) {
++ const struct in6_addr* address = loc_network_get_first_address(self->network);
++
++ return PyBytes_FromAddress(address);
++}
++
+ static PyObject* Network_get_last_address(NetworkObject* self) {
+ char* address = loc_network_format_last_address(self->network);
+
+@@ -224,6 +244,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
+ return obj;
+ }
+
++static PyObject* Network_get__last_address(NetworkObject* self) {
++ const struct in6_addr* address = loc_network_get_last_address(self->network);
++
++ return PyBytes_FromAddress(address);
++}
++
+ static struct PyMethodDef Network_methods[] = {
+ {
+ "exclude",
+@@ -281,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = {
+ NULL,
+ NULL,
+ },
++ {
++ "_first_address",
++ (getter)Network_get__first_address,
++ NULL,
++ NULL,
++ NULL,
++ },
+ {
+ "last_address",
+ (getter)Network_get_last_address,
+@@ -288,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = {
+ NULL,
+ NULL,
+ },
++ {
++ "_last_address",
++ (getter)Network_get__last_address,
++ NULL,
++ NULL,
++ NULL,
++ },
+ { NULL },
+ };
+
+--
+2.20.1
+
+From 90d2194a876c223f9124ce9e27bdee6a6b49ff6a Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 19 Nov 2020 12:40:01 +0000
+Subject: [PATCH 75/77] export: Remove old flattening feature
+
+The database enumerator now only returns networks that will
+never overlap.
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 34 ++++++----------------------------
+ 1 file changed, 6 insertions(+), 28 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 6b39878..4702bcf 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -39,11 +39,8 @@ class OutputWriter(object):
+ suffix = "networks"
+ mode = "w"
+
+- def __init__(self, f, prefix=None, flatten=True):
+- self.f, self.prefix, self.flatten = f, prefix, flatten
+-
+- # The previously written network
+- self._last_network = None
++ def __init__(self, f, prefix=None):
++ self.f, self.prefix = f, prefix
+
+ # Immediately write the header
+ self._write_header()
+@@ -60,18 +57,6 @@ class OutputWriter(object):
+ def __repr__(self):
+ return "<%s f=%s>" % (self.__class__.__name__, self.f)
+
+- def _flatten(self, network):
+- """
+- Checks if the given network needs to be written to file,
+- or if it is a subnet of the previously written network.
+- """
+- if self._last_network and network.is_subnet_of(self._last_network):
+- return True
+-
+- # Remember this network for the next call
+- self._last_network = network
+- return False
+-
+ def _write_header(self):
+ """
+ The header of the file
+@@ -84,15 +69,8 @@ class OutputWriter(object):
+ """
+ pass
+
+- def _write_network(self, network):
+- self.f.write("%s\n" % network)
+-
+ def write(self, network):
+- if self.flatten and self._flatten(network):
+- log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
+- return
+-
+- return self._write_network(network)
++ self.f.write("%s\n" % network)
+
+ def finish(self):
+ """
+@@ -113,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
+ def _write_header(self):
+ self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
+
+- def _write_network(self, network):
++ def write(self, network):
+ self.f.write("add %s %s\n" % (self.prefix, network))
+
+
+@@ -129,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
+ def _write_footer(self):
+ self.f.write("}\n")
+
+- def _write_network(self, network):
++ def write(self, network):
+ self.f.write(" %s,\n" % network)
+
+
+@@ -141,7 +119,7 @@ class XTGeoIPOutputWriter(OutputWriter):
+ suffix = "iv"
+ mode = "wb"
+
+- def _write_network(self, network):
++ def write(self, network):
+ for address in (network._first_address, network._last_address):
+ self.f.write(address)
+
+--
+2.20.1
+
+From 90188dad86223380b0854b523b63ec024117c5f2 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 19 Nov 2020 12:41:19 +0000
+Subject: [PATCH 76/77] export: Speed-up export in xt_geoip format
+
+Removing the loop avoids creating a tuple and then iterating over it
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/python/export.py | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 4702bcf..f0eae26 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -120,8 +120,8 @@ class XTGeoIPOutputWriter(OutputWriter):
+ mode = "wb"
+
+ def write(self, network):
+- for address in (network._first_address, network._last_address):
+- self.f.write(address)
++ self.f.write(network._first_address)
++ self.f.write(network._last_address)
+
+
+ formats = {
+--
+2.20.1
+
+From 6661692f3bc8f788af8b75ae25561f4cc4a2acb5 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu, 19 Nov 2020 12:48:46 +0000
+Subject: [PATCH 77/77] database: Disable some useless code when not running in
+ debug mode
+
+Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+---
+ src/database.c | 16 +++++++++++++++-
+ 1 file changed, 15 insertions(+), 1 deletion(-)
+
+diff --git a/src/database.c b/src/database.c
+index 83dd752..ef4f505 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -619,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
+ }
+
+ clock_t end = clock();
+- DEBUG(db->ctx, "Signature checked in %.4fms\n",
++ INFO(db->ctx, "Signature checked in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
+
+ CLEANUP:
+@@ -679,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
+ off_t lo = 0;
+ off_t hi = db->as_count - 1;
+
++#ifdef ENABLE_DEBUG
+ // Save start time
+ clock_t start = clock();
++#endif
+
+ while (lo <= hi) {
+ off_t i = (lo + hi) / 2;
+@@ -693,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
+ // Check if this is a match
+ uint32_t as_number = loc_as_get_number(*as);
+ if (as_number == number) {
++#ifdef ENABLE_DEBUG
+ clock_t end = clock();
+
+ // Log how fast this has been
+ DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
++#endif
+
+ return 0;
+ }
+@@ -741,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
+ return -1;
+ }
+
++#ifdef ENABLE_DEBUG
+ if (r == 0) {
+ char* string = loc_network_str(*network);
+ DEBUG(db->ctx, "Got network %s\n", string);
+ free(string);
+ }
++#endif
+
+ return r;
+ }
+@@ -840,17 +846,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
+
+ *network = NULL;
+
++#ifdef ENABLE_DEBUG
+ // Save start time
+ clock_t start = clock();
++#endif
+
+ int r = __loc_database_lookup(db, address, network, &network_address,
+ db->network_nodes_v1, 0);
+
++#ifdef ENABLE_DEBUG
+ clock_t end = clock();
+
+ // Log how fast this has been
+ DEBUG(db->ctx, "Executed network search in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
++#endif
+
+ return r;
+ }
+@@ -897,8 +907,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+ off_t lo = 0;
+ off_t hi = db->countries_count - 1;
+
++#ifdef ENABLE_DEBUG
+ // Save start time
+ clock_t start = clock();
++#endif
+
+ while (lo <= hi) {
+ off_t i = (lo + hi) / 2;
+@@ -913,11 +925,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+ int result = strcmp(code, cc);
+
+ if (result == 0) {
++#ifdef ENABLE_DEBUG
+ clock_t end = clock();
+
+ // Log how fast this has been
+ DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
++#endif
+
+ return 0;
+ }
+--
+2.20.1
+
projects / people / pmueller / ipfire-2.x.git / commitdiff
