From 1bf365d8fae097dfe49b776603d49199b63ab985 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 18 Mar 2024 17:12:57 +0000 Subject: [PATCH 01/16] importer: Drop even more indexes We have gone a little bit wild on this but should not need them. This patch also migrates any GIST indexes to SP-GIST. Signed-off-by: Michael Tremer --- src/scripts/location-importer.in | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in index 572f42e..34f4f05 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -197,7 +197,8 @@ class CLI(object): first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); - CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING SPGIST(network inet_ops); + CREATE INDEX IF NOT EXISTS announcements_search2 ON announcements + USING SPGIST(network inet_ops); -- autnums CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); @@ -214,7 +215,8 @@ class CLI(object): ALTER TABLE networks ADD COLUMN IF NOT EXISTS original_countries text[]; ALTER TABLE networks ADD COLUMN IF NOT EXISTS source text; CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING SPGIST(network inet_ops); + CREATE INDEX IF NOT EXISTS networks_search2 ON networks + USING SPGIST(network inet_ops); -- geofeeds CREATE TABLE IF NOT EXISTS geofeeds( @@ -235,17 +237,12 @@ class CLI(object): ); CREATE INDEX IF NOT EXISTS geofeed_networks_geofeed_id ON geofeed_networks(geofeed_id); - CREATE INDEX IF NOT EXISTS geofeed_networks_search - ON geofeed_networks USING SPGIST(network inet_ops); CREATE TABLE IF NOT EXISTS network_geofeeds(network inet, url text); ALTER TABLE network_geofeeds ADD COLUMN IF NOT EXISTS source text NOT NULL; CREATE UNIQUE INDEX IF NOT EXISTS network_geofeeds_unique2 ON network_geofeeds(network, url); - CREATE INDEX IF NOT EXISTS network_geofeeds_search - ON network_geofeeds USING SPGIST(network inet_ops); CREATE INDEX IF NOT EXISTS network_geofeeds_url ON network_geofeeds(url); - DROP INDEX IF EXISTS network_geofeeds_unique; -- feeds CREATE TABLE IF NOT EXISTS autnum_feeds( @@ -272,8 +269,6 @@ class CLI(object): ); CREATE UNIQUE INDEX IF NOT EXISTS network_feeds_unique ON network_feeds(network, source); - CREATE INDEX IF NOT EXISTS network_feeds_search - ON network_feeds USING SPGIST(network inet_ops); -- overrides CREATE TABLE IF NOT EXISTS autnum_overrides( @@ -298,15 +293,20 @@ class CLI(object): ); CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network ON network_overrides(network); - CREATE INDEX IF NOT EXISTS network_overrides_search - ON network_overrides USING SPGIST(network inet_ops); ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; ALTER TABLE network_overrides DROP COLUMN IF EXISTS source; -- Cleanup things we no longer need DROP TABLE IF EXISTS geofeed_overrides; DROP INDEX IF EXISTS announcements_family; + DROP INDEX IF EXISTS announcements_search; + DROP INDEX IF EXISTS geofeed_networks_search; DROP INDEX IF EXISTS networks_family; + DROP INDEX IF EXISTS networks_search; + DROP INDEX IF EXISTS network_feeds_search; + DROP INDEX IF EXISTS network_geofeeds_unique; + DROP INDEX IF EXISTS network_geofeeds_search; + DROP INDEX IF EXISTS network_overrides_search; """) return db -- 2.39.2 From 970d9ada7ee9292667cc9d28643c8a7a25f28add Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Tue, 19 Mar 2024 10:45:41 +0000 Subject: [PATCH 02/16] importer: Permit Geofeeds for everything instead of ignoring This seems to become the default and so we should avoid making the overrides file too verbose. Signed-off-by: Michael Tremer --- src/scripts/location-importer.in | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in index 34f4f05..edd7386 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -2099,10 +2099,12 @@ class CLI(object): networks.append(n) - # Log a warning if not networks have been permitted + # If no networks have been specified, permit for everything if not networks: - log.warning("Geofeed %s is not permitted for any networks. Ignoring." % url) - continue + networks = [ + ipaddress.ip_network("::/0"), + ipaddress.ip_network("0.0.0.0/0"), + ] # Check the URL url = self._check_geofeed_url(url) -- 2.39.2 From edbf280e6e043eaf38d4526e421cd68fcf7bc5c0 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 22 Mar 2024 14:41:44 +0000 Subject: [PATCH 03/16] address: Fix bit length calculation I have no idea what the previous version computed, but it wasn't the right thing. Signed-off-by: Michael Tremer --- src/libloc/address.h | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/libloc/address.h b/src/libloc/address.h index f4c0ee3..d6d7ca5 100644 --- a/src/libloc/address.h +++ b/src/libloc/address.h @@ -131,10 +131,30 @@ static inline struct in6_addr loc_prefix_to_bitmask(const unsigned int prefix) { } static inline unsigned int loc_address_bit_length(const struct in6_addr* address) { + unsigned int bitlength = 0; + int octet = 0; - foreach_octet_in_address(octet, address) { - if (address->s6_addr[octet]) - return (15 - octet) * 8 + 32 - __builtin_clz(address->s6_addr[octet]); + + // Initialize the bit length + if (IN6_IS_ADDR_V4MAPPED(address)) + bitlength = 32; + else + bitlength = 128; + + // Walk backwards until we find the first one + foreach_octet_in_address_reverse(octet, address) { + // Count all trailing zeroes + int trailing_zeroes = __builtin_ctz(address->s6_addr[octet]); + + // We only have one byte + if (trailing_zeroes > 8) + trailing_zeroes = 8; + + // Remove any trailing zeroes from the total length + bitlength -= trailing_zeroes; + + if (trailing_zeroes < 8) + return bitlength; } return 0; -- 2.39.2 From 8d7d3d6b635222f5a07c207859d9da554ddfd6c1 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 22 Mar 2024 14:42:30 +0000 Subject: [PATCH 04/16] network: Fix handling bit length on merge The check was kind of useless because of incorrect values from the bit length function. Signed-off-by: Michael Tremer --- src/network.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/network.c b/src/network.c index 820413c..d658709 100644 --- a/src/network.c +++ b/src/network.c @@ -608,11 +608,15 @@ int loc_network_merge(struct loc_network** n, const unsigned int prefix = loc_network_prefix(n1); // How many bits do we need to represent this address? - const size_t bitlength = loc_address_bit_length(&n1->first_address) - 1; + const size_t bitlength = loc_address_bit_length(&n1->first_address); // We cannot shorten this any more - if (bitlength < prefix) + if (bitlength >= prefix) { + DEBUG(n1->ctx, "Cannot shorten this any further because we need at least %jd bits," + " but only have %d\n", bitlength, prefix); + return 0; + } // Increment the last address of the first network address = n1->last_address; -- 2.39.2 From 8a068b791b8cc9ed4b349d546a7b6f814f14b215 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 22 Mar 2024 14:45:41 +0000 Subject: [PATCH 05/16] tests: Add tests for #13236 Signed-off-by: Michael Tremer --- tests/python/networks-dedup.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/python/networks-dedup.py b/tests/python/networks-dedup.py index 34a82d4..5b78a4b 100755 --- a/tests/python/networks-dedup.py +++ b/tests/python/networks-dedup.py @@ -139,6 +139,27 @@ class Test(unittest.TestCase): ("10.0.0.0/8",), ) + def test_bug13236(self): + self.__test( + ( + ("209.38.0.0/16", "US", None), + ("209.38.1.0/24", "US", 14061), + ("209.38.160.0/22", "US", 14061), + ("209.38.164.0/22", "US", 14061), + ("209.38.168.0/22", "US", 14061), + ("209.38.172.0/22", "US", 14061), + ("209.38.176.0/20", "US", 14061), + ("209.38.192.0/19", "US", 14061), + ("209.38.224.0/19", "US", 14061), + ), + ( + "209.38.0.0/16", + "209.38.1.0/24", + "209.38.160.0/19", + "209.38.192.0/18", + ), + ) + if __name__ == "__main__": unittest.main() -- 2.39.2 From 031bae06e70aef5449b0f0fb9650a4e97eb0e949 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sun, 31 Mar 2024 14:16:14 +0000 Subject: [PATCH 06/16] lua: Add compatibility function to compile with Lua >= 5.1 Signed-off-by: Michael Tremer --- Makefile.am | 1 + src/lua/as.c | 1 + src/lua/compat.h | 56 ++++++++++++++++++++++++++++++++++++++++++++++ src/lua/country.c | 1 + src/lua/database.c | 1 + src/lua/location.c | 1 + src/lua/location.h | 2 ++ src/lua/network.c | 1 + 8 files changed, 64 insertions(+) create mode 100644 src/lua/compat.h diff --git a/Makefile.am b/Makefile.am index 65b3a76..9afae8d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -239,6 +239,7 @@ luadir = $(LUA_INSTALL_CMOD) src_lua_location_la_SOURCES = \ src/lua/as.c \ src/lua/as.h \ + src/lua/compat.h \ src/lua/country.c \ src/lua/country.h \ src/lua/database.c \ diff --git a/src/lua/as.c b/src/lua/as.c index d36f2d8..558fcbf 100644 --- a/src/lua/as.c +++ b/src/lua/as.c @@ -25,6 +25,7 @@ #include "location.h" #include "as.h" +#include "compat.h" typedef struct as { struct loc_as* as; diff --git a/src/lua/compat.h b/src/lua/compat.h new file mode 100644 index 0000000..f0172b8 --- /dev/null +++ b/src/lua/compat.h @@ -0,0 +1,56 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2024 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LUA_LOCATION_COMPAT_H +#define LUA_LOCATION_COMPAT_H + +#include +#include + +#if LUA_VERSION_RELEASE_NUM < 502 + +static inline void luaL_setmetatable(lua_State* L, const char* name) { + luaL_checkstack(L, 1, "not enough stack slots"); + luaL_getmetatable(L, name); + lua_setmetatable(L, -2); +} + +static inline void luaL_setfuncs(lua_State* L, const luaL_Reg* l, int nup) { + int i; + + luaL_checkstack(L, nup+1, "too many upvalues"); + + for (; l->name != NULL; l++) { + lua_pushstring(L, l->name); + + for (i = 0; i < nup; i++) + lua_pushvalue(L, -(nup + 1)); + + lua_pushcclosure(L, l->func, nup); + lua_settable(L, -(nup + 3)); + } + + lua_pop(L, nup); +} + +static inline void luaL_newlib(lua_State* L, const luaL_Reg* l) { + lua_newtable(L); + luaL_setfuncs(L, l, 0); +} + +#endif /* Lua < 5.2 */ + +#endif /* LUA_LOCATION_COMPAT_H */ diff --git a/src/lua/country.c b/src/lua/country.c index 7d2a78a..816bd2f 100644 --- a/src/lua/country.c +++ b/src/lua/country.c @@ -24,6 +24,7 @@ #include #include "location.h" +#include "compat.h" #include "country.h" typedef struct country { diff --git a/src/lua/database.c b/src/lua/database.c index 9f5d177..841041d 100644 --- a/src/lua/database.c +++ b/src/lua/database.c @@ -24,6 +24,7 @@ #include "location.h" #include "as.h" +#include "compat.h" #include "country.h" #include "database.h" #include "network.h" diff --git a/src/lua/location.c b/src/lua/location.c index 6f2157c..2d22a81 100644 --- a/src/lua/location.c +++ b/src/lua/location.c @@ -26,6 +26,7 @@ #include "location.h" #include "as.h" +#include "compat.h" #include "country.h" #include "database.h" #include "network.h" diff --git a/src/lua/location.h b/src/lua/location.h index cc261e6..0708988 100644 --- a/src/lua/location.h +++ b/src/lua/location.h @@ -21,6 +21,8 @@ #include +#include "compat.h" + extern struct loc_ctx* ctx; int luaopen_location(lua_State* L); diff --git a/src/lua/network.c b/src/lua/network.c index 1d5e0af..a25166e 100644 --- a/src/lua/network.c +++ b/src/lua/network.c @@ -24,6 +24,7 @@ #include #include "location.h" +#include "compat.h" #include "network.h" typedef struct network { -- 2.39.2 From 88908fd46de5694d66eb1fe0cc3467d4a393ad93 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sun, 31 Mar 2024 16:33:34 +0000 Subject: [PATCH 07/16] lua: Don't raise an error if a network cannot be found The lookup function now returns nil which is easier to handle than catching a Lua error. Signed-off-by: Michael Tremer --- src/lua/database.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lua/database.c b/src/lua/database.c index 841041d..aada739 100644 --- a/src/lua/database.c +++ b/src/lua/database.c @@ -174,8 +174,17 @@ static int Database_lookup(lua_State* L) { // Perform lookup r = loc_database_lookup_from_string(self->db, address, &network); - if (r) - return luaL_error(L, "Could not lookup address %s: %s\n", address, strerror(errno)); + if (r) { + switch (errno) { + // Return nil if the network was not found + case ENOENT: + lua_pushnil(L); + return 1; + + default: + return luaL_error(L, "Could not lookup address %s: %s\n", address, strerror(errno)); + } + } // Create a network object r = create_network(L, network); -- 2.39.2 From 23ece648843a884f6c3cac262a0cad770ddd74a6 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 5 Apr 2024 15:31:31 +0000 Subject: [PATCH 08/16] tests: Fix bit length tests Signed-off-by: Michael Tremer --- src/test-network.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test-network.c b/src/test-network.c index 717ad3a..f6ccb62 100644 --- a/src/test-network.c +++ b/src/test-network.c @@ -308,9 +308,9 @@ int main(int argc, char** argv) { unsigned int bit_length; } bit_length_tests[] = { { "::/0", 0 }, - { "2001::/128", 126 }, - { "1.0.0.0/32", 25 }, - { "0.0.0.1/32", 1 }, + { "2001::/128", 16 }, + { "1.0.0.0/32", 8 }, + { "0.0.0.1/32", 32 }, { "255.255.255.255/32", 32 }, { NULL, 0, }, }; -- 2.39.2 From cbc4c6a7be5dc8b9ea37edbcfa395a373b429b98 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 5 Apr 2024 15:36:14 +0000 Subject: [PATCH 09/16] lua: Create a simple iterator for all networks Signed-off-by: Michael Tremer --- src/lua/database.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ tests/lua/main.lua | 11 +++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/lua/database.c b/src/lua/database.c index aada739..ae769f7 100644 --- a/src/lua/database.c +++ b/src/lua/database.c @@ -217,6 +217,50 @@ static int Database_verify(lua_State* L) { return 1; } +static int Database_next_network(lua_State* L) { + struct loc_network* network = NULL; + int r; + + struct loc_database_enumerator* e = lua_touserdata(L, lua_upvalueindex(1)); + + // Fetch the next network + r = loc_database_enumerator_next_network(e, &network); + if (r) + return luaL_error(L, "Could not fetch network: %s\n", strerror(errno)); + + // If we have received no network, we have reached the end + if (!network) { + lua_pushnil(L); + return 1; + } + + // Create a network object + r = create_network(L, network); + loc_network_unref(network); + + return r; +} + +static int Database_list_networks(lua_State* L) { + struct loc_database_enumerator* e = NULL; + int r; + + Database* self = luaL_checkdatabase(L, 1); + + // Create a new enumerator + r = loc_database_enumerator_new(&e, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); + if (r) + return luaL_error(L, "Could not create enumerator: %s\n", strerror(errno)); + + // Push the enumerator onto the stack + lua_pushlightuserdata(L, e); + + // Push the closure onto the stack + lua_pushcclosure(L, Database_next_network, 1); + + return 1; +} + static const struct luaL_Reg database_functions[] = { { "get_as", Database_get_as }, { "get_description", Database_get_description }, @@ -225,6 +269,7 @@ static const struct luaL_Reg database_functions[] = { { "get_vendor", Database_get_vendor }, { "open", Database_open }, { "lookup", Database_lookup }, + { "list_networks", Database_list_networks }, { "verify", Database_verify }, { "__gc", Database_gc }, { NULL, NULL }, diff --git a/tests/lua/main.lua b/tests/lua/main.lua index 4a7fc5d..a824513 100755 --- a/tests/lua/main.lua +++ b/tests/lua/main.lua @@ -151,4 +151,15 @@ function test_gc() print("GC: " .. collectgarbage("collect")) end +function test_list_networks() + location = require("location") + + -- Open the database + db = location.Database.open(ENV_TEST_DATABASE) + + for network in db:list_networks() do + print(network) + end +end + os.exit(luaunit.LuaUnit.run()) -- 2.39.2 From 421735739022e246fd6c84be6bb2efe43cd42bbb Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 5 Apr 2024 15:58:53 +0000 Subject: [PATCH 10/16] lua: Cleanup any database iterators Signed-off-by: Michael Tremer --- src/lua/database.c | 52 ++++++++++++++++++++++++++++++++++++++-------- src/lua/database.h | 1 + src/lua/location.c | 5 +++++ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/lua/database.c b/src/lua/database.c index ae769f7..7b6c5bf 100644 --- a/src/lua/database.c +++ b/src/lua/database.c @@ -217,14 +217,38 @@ static int Database_verify(lua_State* L) { return 1; } -static int Database_next_network(lua_State* L) { +typedef struct enumerator { + struct loc_database_enumerator* e; +} DatabaseEnumerator; + +static DatabaseEnumerator* luaL_checkdatabaseenumerator(lua_State* L, int i) { + void* userdata = luaL_checkudata(L, i, "location.DatabaseEnumerator"); + + // Throw an error if the argument doesn't match + luaL_argcheck(L, userdata, i, "DatabaseEnumerator expected"); + + return (DatabaseEnumerator*)userdata; +} + +static int DatabaseEnumerator_gc(lua_State* L) { + DatabaseEnumerator* self = luaL_checkdatabaseenumerator(L, 1); + + if (self->e) { + loc_database_enumerator_unref(self->e); + self->e = NULL; + } + + return 0; +} + +static int DatabaseEnumerator_next_network(lua_State* L) { struct loc_network* network = NULL; int r; - struct loc_database_enumerator* e = lua_touserdata(L, lua_upvalueindex(1)); + DatabaseEnumerator* self = luaL_checkdatabaseenumerator(L, lua_upvalueindex(1)); // Fetch the next network - r = loc_database_enumerator_next_network(e, &network); + r = loc_database_enumerator_next_network(self->e, &network); if (r) return luaL_error(L, "Could not fetch network: %s\n", strerror(errno)); @@ -242,21 +266,22 @@ static int Database_next_network(lua_State* L) { } static int Database_list_networks(lua_State* L) { - struct loc_database_enumerator* e = NULL; + DatabaseEnumerator* e = NULL; int r; Database* self = luaL_checkdatabase(L, 1); + // Allocate a new enumerator + e = lua_newuserdata(L, sizeof(*e)); + luaL_setmetatable(L, "location.DatabaseEnumerator"); + // Create a new enumerator - r = loc_database_enumerator_new(&e, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); + r = loc_database_enumerator_new(&e->e, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); if (r) return luaL_error(L, "Could not create enumerator: %s\n", strerror(errno)); - // Push the enumerator onto the stack - lua_pushlightuserdata(L, e); - // Push the closure onto the stack - lua_pushcclosure(L, Database_next_network, 1); + lua_pushcclosure(L, DatabaseEnumerator_next_network, 1); return 1; } @@ -278,3 +303,12 @@ static const struct luaL_Reg database_functions[] = { int register_database(lua_State* L) { return register_class(L, "location.Database", database_functions); } + +static const struct luaL_Reg database_enumerator_functions[] = { + { "__gc", DatabaseEnumerator_gc }, + { NULL, NULL }, +}; + +int register_database_enumerator(lua_State* L) { + return register_class(L, "location.DatabaseEnumerator", database_enumerator_functions); +} diff --git a/src/lua/database.h b/src/lua/database.h index 0ac8544..6a5aa4d 100644 --- a/src/lua/database.h +++ b/src/lua/database.h @@ -21,5 +21,6 @@ #include int register_database(lua_State* L); +int register_database_enumerator(lua_State* L); #endif /* LUA_LOCATION_DATABASE_H */ diff --git a/src/lua/location.c b/src/lua/location.c index 2d22a81..b7d2c0e 100644 --- a/src/lua/location.c +++ b/src/lua/location.c @@ -70,6 +70,11 @@ int luaopen_location(lua_State* L) { lua_setfield(L, -2, "Database"); + // Register DatabaseEnumerator type + register_database_enumerator(L); + + lua_setfield(L, -2, "DatabaseEnumerator"); + // Register Network type register_network(L); -- 2.39.2 From 16d3298fa5c952467903c6be6e215b612ffcd627 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 6 Apr 2024 10:33:06 +0000 Subject: [PATCH 11/16] address: Add functions to access a specific byte/nibble in an address Signed-off-by: Michael Tremer --- src/libloc/address.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libloc/address.h b/src/libloc/address.h index d6d7ca5..1c14696 100644 --- a/src/libloc/address.h +++ b/src/libloc/address.h @@ -300,6 +300,37 @@ static inline int loc_address_count_trailing_zero_bits(const struct in6_addr* ad return zeroes; } +static inline int loc_address_get_octet(const struct in6_addr* address, const unsigned int i) { + if (IN6_IS_ADDR_V4MAPPED(address)) { + if (i >= 4) + return -ERANGE; + + return (address->s6_addr32[3] >> (i * 8)) & 0xff; + + } else { + if (i >= 32) + return -ERANGE; + + return address->s6_addr[i]; + } +} + +static inline int loc_address_get_nibble(const struct in6_addr* address, const unsigned int i) { + int octet = 0; + + // Fetch the octet + octet = loc_address_get_octet(address, i / 2); + if (octet < 0) + return octet; + + // Shift if we want an uneven nibble + if (i % 2 == 0) + octet >>= 4; + + // Return the nibble + return octet & 0x0f; +} + #endif /* LIBLOC_PRIVATE */ #endif /* LIBLOC_ADDRESS_H */ -- 2.39.2 From e55b9875d26bdeb58167863db41cefdd367fab9c Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 6 Apr 2024 10:33:33 +0000 Subject: [PATCH 12/16] tests: Set LD_LIBRARY_PATH Signed-off-by: Michael Tremer --- Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.am b/Makefile.am index 9afae8d..b045e49 100644 --- a/Makefile.am +++ b/Makefile.am @@ -404,6 +404,7 @@ TESTS_LDADD = \ src/libloc-internal.la TESTS_ENVIRONMENT = \ + LD_LIBRARY_PATH="$(abs_builddir)/src/.libs" \ LUA_CPATH="$(abs_builddir)/src/lua/.libs/?.so;;" \ PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \ TEST_DATA_DIR="$(abs_top_srcdir)/data" \ -- 2.39.2 From 8da88dda6f99ca9938b81a1d75f252e42d74a2c4 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 6 Apr 2024 10:49:41 +0000 Subject: [PATCH 13/16] network: Add function to return a reverse pointer for networks Signed-off-by: Michael Tremer --- src/libloc.sym | 7 +++ src/libloc/network.h | 2 + src/lua/network.c | 31 ++++++++++++ src/network.c | 118 +++++++++++++++++++++++++++++++++++++++++++ src/python/network.c | 32 ++++++++++++ src/test-network.c | 66 ++++++++++++++++++++++++ tests/lua/main.lua | 2 +- 7 files changed, 257 insertions(+), 1 deletion(-) diff --git a/src/libloc.sym b/src/libloc.sym index 29e17f0..50734b3 100644 --- a/src/libloc.sym +++ b/src/libloc.sym @@ -146,3 +146,10 @@ global: local: *; }; + +LIBLOC_2 { +global: + loc_network_reverse_pointer; +local: + *; +} LIBLOC_1; diff --git a/src/libloc/network.h b/src/libloc/network.h index 62334ac..6f2dad2 100644 --- a/src/libloc/network.h +++ b/src/libloc/network.h @@ -66,6 +66,8 @@ struct loc_network_list* loc_network_exclude( struct loc_network_list* loc_network_exclude_list( struct loc_network* network, struct loc_network_list* list); +char* loc_network_reverse_pointer(struct loc_network* network, const char* suffix); + #ifdef LIBLOC_PRIVATE int loc_network_properties_cmp(struct loc_network* self, struct loc_network* other); diff --git a/src/lua/network.c b/src/lua/network.c index a25166e..15936f7 100644 --- a/src/lua/network.c +++ b/src/lua/network.c @@ -151,12 +151,43 @@ static int Network_has_flag(lua_State* L) { return 1; } +// Reverse Pointer + +static int Network_reverse_pointer(lua_State* L) { + char* rp = NULL; + + Network* self = luaL_checknetwork(L, 1); + + // Fetch the suffix + const char* suffix = luaL_optstring(L, 2, NULL); + + // Make the reverse pointer + rp = loc_network_reverse_pointer(self->network, suffix); + if (!rp) { + switch (errno) { + case ENOTSUP: + lua_pushnil(L); + return 1; + + default: + return luaL_error(L, "Could not create reverse pointer: %s\n", strerror(errno)); + } + } + + // Return the response + lua_pushstring(L, rp); + free(rp); + + return 1; +} + static const struct luaL_Reg Network_functions[] = { { "new", Network_new }, { "get_asn", Network_get_asn }, { "get_country_code", Network_get_country_code }, { "get_family", Network_get_family }, { "has_flag", Network_has_flag }, + { "reverse_pointer", Network_reverse_pointer }, { "__gc", Network_gc }, { "__tostring", Network_tostring }, { NULL, NULL }, diff --git a/src/network.c b/src/network.c index d658709..69306a9 100644 --- a/src/network.c +++ b/src/network.c @@ -700,3 +700,121 @@ int loc_network_new_from_database_v1(struct loc_ctx* ctx, struct loc_network** n return 0; } + +static char* loc_network_reverse_pointer6(struct loc_network* network, const char* suffix) { + char* buffer = NULL; + int r; + + unsigned int prefix = loc_network_prefix(network); + + // Must border on a nibble + if (prefix % 4) { + errno = ENOTSUP; + return NULL; + } + + if (!suffix) + suffix = "ip6.arpa."; + + // Initialize the buffer + r = asprintf(&buffer, "%s", suffix); + if (r < 0) + goto ERROR; + + for (unsigned int i = 0; i < (prefix / 4); i++) { + r = asprintf(&buffer, "%x.%s", loc_address_get_nibble(&network->first_address, i), buffer); + if (r < 0) + goto ERROR; + } + + // Add the asterisk + if (prefix < 128) { + r = asprintf(&buffer, "*.%s", buffer); + if (r < 0) + goto ERROR; + } + + return buffer; + +ERROR: + if (buffer) + free(buffer); + + return NULL; +} + +static char* loc_network_reverse_pointer4(struct loc_network* network, const char* suffix) { + char* buffer = NULL; + int r; + + unsigned int prefix = loc_network_prefix(network); + + // Must border on an octet + if (prefix % 8) { + errno = ENOTSUP; + return NULL; + } + + if (!suffix) + suffix = "in-addr.arpa."; + + switch (prefix) { + case 32: + r = asprintf(&buffer, "%d.%d.%d.%d.%s", + loc_address_get_octet(&network->first_address, 3), + loc_address_get_octet(&network->first_address, 2), + loc_address_get_octet(&network->first_address, 1), + loc_address_get_octet(&network->first_address, 0), + suffix); + break; + + case 24: + r = asprintf(&buffer, "*.%d.%d.%d.%s", + loc_address_get_octet(&network->first_address, 2), + loc_address_get_octet(&network->first_address, 1), + loc_address_get_octet(&network->first_address, 0), + suffix); + break; + + case 16: + r = asprintf(&buffer, "*.%d.%d.%s", + loc_address_get_octet(&network->first_address, 1), + loc_address_get_octet(&network->first_address, 0), + suffix); + break; + + case 8: + r = asprintf(&buffer, "*.%d.%s", + loc_address_get_octet(&network->first_address, 0), + suffix); + break; + + case 0: + r = asprintf(&buffer, "*.%s", suffix); + break; + + // To make the compiler happy + default: + return NULL; + } + + if (r < 0) + return NULL; + + return buffer; +} + +LOC_EXPORT char* loc_network_reverse_pointer(struct loc_network* network, const char* suffix) { + switch (network->family) { + case AF_INET6: + return loc_network_reverse_pointer6(network, suffix); + + case AF_INET: + return loc_network_reverse_pointer4(network, suffix); + + default: + break; + } + + return NULL; +} diff --git a/src/python/network.c b/src/python/network.c index 3400721..4bae918 100644 --- a/src/python/network.c +++ b/src/python/network.c @@ -279,6 +279,32 @@ static PyObject* Network_richcompare(NetworkObject* self, PyObject* other, int o Py_RETURN_NOTIMPLEMENTED; } +static PyObject* Network_reverse_pointer(NetworkObject* self, PyObject* args, PyObject* kwargs) { + char* kwlist[] = { "suffix", NULL }; + const char* suffix = NULL; + char* rp = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|z", kwlist, &suffix)) + return NULL; + + rp = loc_network_reverse_pointer(self->network, suffix); + if (!rp) { + switch (errno) { + case ENOTSUP: + Py_RETURN_NONE; + + default: + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + } + + PyObject* ret = PyUnicode_FromString(rp); + free(rp); + + return ret; +} + static struct PyMethodDef Network_methods[] = { { "exclude", @@ -298,6 +324,12 @@ static struct PyMethodDef Network_methods[] = { METH_VARARGS, NULL, }, + { + "reverse_pointer", + (PyCFunction)Network_reverse_pointer, + METH_VARARGS|METH_KEYWORDS, + NULL, + }, { "set_flag", (PyCFunction)Network_set_flag, diff --git a/src/test-network.c b/src/test-network.c index f6ccb62..69544e2 100644 --- a/src/test-network.c +++ b/src/test-network.c @@ -29,6 +29,67 @@ #include #include +static int test_reverse_pointers(struct loc_ctx* ctx) { + const struct test { + const char* network; + const char* rp; + } tests[] = { + // IPv6 + { "::1/128", "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." }, + { "2001:db8::/32", "*.8.b.d.0.1.0.0.2.ip6.arpa." }, + + // IPv4 + { "10.0.0.0/32", "0.0.0.10.in-addr.arpa." }, + { "10.0.0.0/24", "*.0.0.10.in-addr.arpa." }, + { "10.0.0.0/16", "*.0.10.in-addr.arpa." }, + { "10.0.0.0/8", "*.10.in-addr.arpa." }, + { "10.0.0.0/0", "*.in-addr.arpa." }, + { "10.0.0.0/1", NULL, }, + { NULL, NULL }, + }; + + struct loc_network* network = NULL; + char* rp = NULL; + int r; + + for (const struct test* test = tests; test->network; test++) { + // Create a new network + r = loc_network_new_from_string(ctx, &network, test->network); + if (r) + return r; + + // Fetch the reverse pointer + rp = loc_network_reverse_pointer(network, NULL); + + // No RP expected and got none + if (!test->rp && !rp) + continue; + + // Got a result when expecting none + else if (!test->rp && rp) { + fprintf(stderr, "Got an RP for %s when expecting none\n", test->network); + return EXIT_FAILURE; + + // Got nothing when expecting a result + } else if (test->rp && !rp) { + fprintf(stderr, "Got no RP for %s when expecting one\n", test->network); + return EXIT_FAILURE; + + // Compare values + } else if (strcmp(test->rp, rp) != 0) { + fprintf(stderr, "Got an unexpected RP for %s: Got %s, expected %s\n", + test->network, rp, test->rp); + return EXIT_FAILURE; + } + + loc_network_unref(network); + if (rp) + free(rp); + } + + return 0; +} + int main(int argc, char** argv) { int err; @@ -336,6 +397,11 @@ int main(int argc, char** argv) { loc_network_unref(network1); } + // Test reverse pointers + err = test_reverse_pointers(ctx); + if (err) + exit(err); + loc_unref(ctx); fclose(f); diff --git a/tests/lua/main.lua b/tests/lua/main.lua index a824513..b45ab1c 100755 --- a/tests/lua/main.lua +++ b/tests/lua/main.lua @@ -158,7 +158,7 @@ function test_list_networks() db = location.Database.open(ENV_TEST_DATABASE) for network in db:list_networks() do - print(network) + print(network, network:reverse_pointer()) end end -- 2.39.2 From a457e5e59fec0694893e4b19f7d91d13d637ac82 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 6 Apr 2024 11:11:28 +0000 Subject: [PATCH 14/16] lua: Add function that returns subnets of a network Signed-off-by: Michael Tremer --- src/lua/network.c | 31 +++++++++++++++++++++++++++++++ tests/lua/main.lua | 18 ++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/lua/network.c b/src/lua/network.c index 15936f7..2da6a1d 100644 --- a/src/lua/network.c +++ b/src/lua/network.c @@ -151,6 +151,36 @@ static int Network_has_flag(lua_State* L) { return 1; } +// Subnets + +static int Network_subnets(lua_State* L) { + struct loc_network* subnet1 = NULL; + struct loc_network* subnet2 = NULL; + int r; + + Network* self = luaL_checknetwork(L, 1); + + // Make subnets + r = loc_network_subnets(self->network, &subnet1, &subnet2); + if (r) + return luaL_error(L, "Could not create subnets of %s: %s\n", + loc_network_str(self->network), strerror(errno)); + + // Create a new table + lua_createtable(L, 2, 0); + + // Create the networks & push them onto the table + create_network(L, subnet1); + loc_network_unref(subnet1); + lua_rawseti(L, -2, 1); + + create_network(L, subnet2); + loc_network_unref(subnet2); + lua_rawseti(L, -2, 2); + + return 1; +} + // Reverse Pointer static int Network_reverse_pointer(lua_State* L) { @@ -188,6 +218,7 @@ static const struct luaL_Reg Network_functions[] = { { "get_family", Network_get_family }, { "has_flag", Network_has_flag }, { "reverse_pointer", Network_reverse_pointer }, + { "subnets", Network_subnets }, { "__gc", Network_gc }, { "__tostring", Network_tostring }, { NULL, NULL }, diff --git a/tests/lua/main.lua b/tests/lua/main.lua index b45ab1c..e139b2d 100755 --- a/tests/lua/main.lua +++ b/tests/lua/main.lua @@ -151,6 +151,24 @@ function test_gc() print("GC: " .. collectgarbage("collect")) end +function test_subnets() + location = require("location") + + -- Open the database + db = location.Database.open(ENV_TEST_DATABASE) + + local network = db:lookup("1.1.1.1") + + local subnets = network:subnets() + + luaunit.assertIsTable(subnets) + luaunit.assertEquals(#subnets, 2) + + for i, subnet in ipairs(subnets) do + print(subnet) + end +end + function test_list_networks() location = require("location") -- 2.39.2 From bd1087fa28665c3348529d91a3e47f5ad491b5ab Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 8 Apr 2024 10:24:21 +0000 Subject: [PATCH 15/16] lua: Add method to access database creation time Signed-off-by: Michael Tremer --- src/lua/database.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/lua/database.c b/src/lua/database.c index 7b6c5bf..fcbbad0 100644 --- a/src/lua/database.c +++ b/src/lua/database.c @@ -86,6 +86,20 @@ static int Database_gc(lua_State* L) { return 0; } +// Created At + +static int Database_created_at(lua_State* L) { + Database* self = luaL_checkdatabase(L, 1); + + // Fetch the time + time_t created_at = loc_database_created_at(self->db); + + // Push the time onto the stack + lua_pushnumber(L, created_at); + + return 1; +} + // Description static int Database_get_description(lua_State* L) { @@ -287,6 +301,7 @@ static int Database_list_networks(lua_State* L) { } static const struct luaL_Reg database_functions[] = { + { "created_at", Database_created_at }, { "get_as", Database_get_as }, { "get_description", Database_get_description }, { "get_country", Database_get_country }, -- 2.39.2 From f3637cf853126e8909f309ceab0471d98bcf47c9 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Thu, 11 Apr 2024 17:45:18 +0000 Subject: [PATCH 16/16] importer: Drop EDROP as it has been merged into DROP https://www.spamhaus.org/resource-hub/network-security/spamhaus-drop-and-edrop-to-become-a-single-list/ Signed-off-by: Michael Tremer --- src/scripts/location-importer.in | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in index edd7386..c65029b 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -2163,7 +2163,6 @@ class CLI(object): # Spamhaus DROP ("SPAMHAUS-DROP", self._import_spamhaus_drop, "https://www.spamhaus.org/drop/drop.txt"), - ("SPAMHAUS-EDROP", self._import_spamhaus_drop, "https://www.spamhaus.org/drop/edrop.txt"), ("SPAMHAUS-DROPV6", self._import_spamhaus_drop, "https://www.spamhaus.org/drop/dropv6.txt"), # Spamhaus ASNDROP -- 2.39.2