From: Michael Tremer Date: Fri, 18 Sep 2020 10:26:15 +0000 (+0000) Subject: export: Skip writing any subnets X-Git-Tag: 0.9.4~8 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=43554dc4659cbb0d4c477fbff69cfbb5f21e4f13;p=people%2Fms%2Flibloc.git export: Skip writing any subnets It is unnecessary to write any subnets in the output, because they might already be included in a bigger parent network. Since the tree is ordered, we only need to check if the network to be written is a subnet of the previous one. If so, we skip it. Signed-off-by: Michael Tremer --- diff --git a/src/libloc.sym b/src/libloc.sym index 9a1e6f0..222a4cf 100644 --- a/src/libloc.sym +++ b/src/libloc.sym @@ -83,6 +83,7 @@ global: loc_network_get_asn; loc_network_get_country_code; loc_network_has_flag; + loc_network_is_subnet_of; loc_network_match_asn; loc_network_match_country_code; loc_network_match_flag; diff --git a/src/loc/network.h b/src/loc/network.h index 273041c..c26092f 100644 --- a/src/loc/network.h +++ b/src/loc/network.h @@ -51,6 +51,8 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); int loc_network_set_flag(struct loc_network* network, uint32_t flag); int loc_network_match_flag(struct loc_network* network, uint32_t flag); +int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + #ifdef LIBLOC_PRIVATE int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); diff --git a/src/network.c b/src/network.c index 5a8db7a..286e136 100644 --- a/src/network.c +++ b/src/network.c @@ -346,6 +346,24 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag return loc_network_has_flag(network, flag); } +LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. + if (in6_addr_cmp(&self->start_address, &other->start_address) < 0) + return 0; + + // Get the end addresses + struct in6_addr last_address_self = make_last_address(&self->start_address, self->prefix); + struct in6_addr last_address_other = make_last_address(&other->start_address, other->prefix); + + // If the end address of the other network is greater than this network, + // it cannot be a subnet. + if (in6_addr_cmp(&last_address_self, &last_address_other) > 0) + return 0; + + return 1; +} + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { // Add country code loc_country_code_copy(dbobj->country_code, network->country_code); diff --git a/src/python/export.py b/src/python/export.py index d0bbe77..180d462 100644 --- a/src/python/export.py +++ b/src/python/export.py @@ -39,8 +39,11 @@ class OutputWriter(object): suffix = "networks" mode = "w" - def __init__(self, f, prefix=None): - self.f, self.prefix = f, prefix + def __init__(self, f, prefix=None, flatten=True): + self.f, self.prefix, self.flatten = f, prefix, flatten + + # The previously written network + self._last_network = None # Immediately write the header self._write_header() @@ -57,6 +60,18 @@ class OutputWriter(object): def __repr__(self): return "<%s f=%s>" % (self.__class__.__name__, self.f) + def _flatten(self, network): + """ + Checks if the given network needs to be written to file, + or if it is a subnet of the previously written network. + """ + if self._last_network and network.is_subnet_of(self._last_network): + return True + + # Remember this network for the next call + self._last_network = network + return False + def _write_header(self): """ The header of the file @@ -69,9 +84,17 @@ class OutputWriter(object): """ pass - def write(self, network): + def _write_network(self, network): self.f.write("%s\n" % network) + def write(self, network): + if self.flatten and self._flatten(network): + log.debug("Skipping writing network %s" % network) + return + + # Write the network to file + self._write_network(network) + def finish(self): """ Called when all data has been written @@ -91,7 +114,7 @@ class IpsetOutputWriter(OutputWriter): def _write_header(self): self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix) - def write(self, network): + def _write_network(self, network): self.f.write("add %s %s\n" % (self.prefix, network)) @@ -107,7 +130,7 @@ class NftablesOutputWriter(OutputWriter): def _write_footer(self): self.f.write("}\n") - def write(self, network): + def _write_network(self, network): self.f.write(" %s,\n" % network) @@ -119,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter): suffix = "iv" mode = "wb" - def write(self, network): + def _write_network(self, network): n = ipaddress.ip_network("%s" % network) for address in (n.network_address, n.broadcast_address): diff --git a/src/python/network.c b/src/python/network.c index d1b0de8..5fbb17d 100644 --- a/src/python/network.c +++ b/src/python/network.c @@ -154,6 +154,18 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { Py_RETURN_NONE; } +static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL; + + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) + return NULL; + + if (loc_network_is_subnet_of(self->network, other->network)) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + static struct PyMethodDef Network_methods[] = { { "has_flag", @@ -161,6 +173,12 @@ static struct PyMethodDef Network_methods[] = { METH_VARARGS, NULL, }, + { + "is_subnet_of", + (PyCFunction)Network_is_subnet_of, + METH_VARARGS, + NULL, + }, { "set_flag", (PyCFunction)Network_set_flag, diff --git a/src/test-network.c b/src/test-network.c index 364099d..c49c3ef 100644 --- a/src/test-network.c +++ b/src/test-network.c @@ -95,6 +95,19 @@ int main(int argc, char** argv) { size_t nodes = loc_network_tree_count_nodes(tree); printf("The tree has %zu nodes\n", nodes); + // Check subnet function + err = loc_network_is_subnet_of(network1, network2); + if (err != 0) { + fprintf(stderr, "Subnet check 1 failed: %d\n", err); + exit(EXIT_FAILURE); + } + + err = loc_network_is_subnet_of(network2, network1); + if (err != 1) { + fprintf(stderr, "Subnet check 2 failed: %d\n", err); + exit(EXIT_FAILURE); + } + // Create a database struct loc_writer* writer; err = loc_writer_new(ctx, &writer, NULL, NULL);