X-Git-Url: http://git.ipfire.org/?a=blobdiff_plain;f=src%2Fdatabase.c;h=b92a803d5d3db9a8be49049ee803d7a06f6180e2;hb=55664c7ef66aafced62f3427391fd5c7bc377255;hp=f484bee5efef3107bd645977a0280b74b6c82ae0;hpb=22c7b98b339093c2fc539b4842bf19a3b5a96812;p=people%2Fms%2Flibloc.git diff --git a/src/database.c b/src/database.c index f484bee..b92a803 100644 --- a/src/database.c +++ b/src/database.c @@ -38,8 +38,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -58,8 +60,11 @@ struct loc_database { off_t description; off_t license; - char* signature; - size_t signature_length; + // Signatures + char* signature1; + size_t signature1_length; + char* signature2; + size_t signature2_length; // ASes in the database struct loc_database_as_v1* as_v1; @@ -96,19 +101,28 @@ struct loc_database_enumerator { // Search string char* string; - char country_code[3]; - uint32_t asn; + struct loc_country_list* countries; + struct loc_as_list* asns; enum loc_network_flags flags; int family; + // Flatten output? + int flatten; + // Index of the AS we are looking at unsigned int as_index; + // Index of the country we are looking at + unsigned int country_index; + // Network state struct in6_addr network_address; struct loc_node_stack network_stack[MAX_STACK_DEPTH]; int network_stack_depth; unsigned int* networks_visited; + + // For subnet search + struct loc_network_list* stack; }; static int loc_database_read_magic(struct loc_database* db) { @@ -232,8 +246,30 @@ static int loc_database_read_countries_section_v1(struct loc_database* db, return 0; } +static int loc_database_read_signature(struct loc_database* db, + char** dst, char* src, size_t length) { + // Check for a plausible signature length + if (length > LOC_SIGNATURE_MAX_LENGTH) { + ERROR(db->ctx, "Signature too long: %zu\n", length); + return -EINVAL; + } + + DEBUG(db->ctx, "Reading signature of %zu bytes\n", length); + + // Allocate space + *dst = malloc(length); + if (!*dst) + return -ENOMEM; + + // Copy payload + memcpy(*dst, src, length); + + return 0; +} + static int loc_database_read_header_v1(struct loc_database* db) { struct loc_database_header_v1 header; + int r; // Read from file size_t size = fread(&header, 1, sizeof(header), db->f); @@ -249,28 +285,29 @@ static int loc_database_read_header_v1(struct loc_database* db) { db->description = be32toh(header.description); db->license = be32toh(header.license); - // Read signature - db->signature_length = be32toh(header.signature_length); - if (db->signature_length) { - // Check for a plausible signature length - if (db->signature_length > LOC_SIGNATURE_MAX_LENGTH) { - ERROR(db->ctx, "Signature too long: %ld\n", db->signature_length); - return -EINVAL; - } + db->signature1_length = be16toh(header.signature1_length); + db->signature2_length = be16toh(header.signature2_length); - DEBUG(db->ctx, "Reading signature of %ld bytes\n", - db->signature_length); + // Read signatures + if (db->signature1_length) { + r = loc_database_read_signature(db, &db->signature1, + header.signature1, db->signature1_length); + if (r) + return r; + } - db->signature = malloc(db->signature_length); - for (unsigned int i = 0; i < db->signature_length; i++) - db->signature[i] = header.signature[i]; + if (db->signature2_length) { + r = loc_database_read_signature(db, &db->signature2, + header.signature2, db->signature2_length); + if (r) + return r; } // Open pool off_t pool_offset = be32toh(header.pool_offset); size_t pool_length = be32toh(header.pool_length); - int r = loc_stringpool_open(db->ctx, &db->pool, + r = loc_stringpool_open(db->ctx, &db->pool, db->f, pool_length, pool_offset); if (r) return r; @@ -409,12 +446,21 @@ static void loc_database_free(struct loc_database* db) { ERROR(db->ctx, "Could not unmap network nodes section: %s\n", strerror(errno)); } + // Remove mapped countries section + if (db->countries_v1) { + r = munmap(db->countries_v1, db->countries_count * sizeof(*db->countries_v1)); + if (r) + ERROR(db->ctx, "Could not unmap countries section: %s\n", strerror(errno)); + } + if (db->pool) loc_stringpool_unref(db->pool); // Free signature - if (db->signature) - free(db->signature); + if (db->signature1) + free(db->signature1); + if (db->signature2) + free(db->signature2); // Close database file if (db->f) @@ -434,7 +480,7 @@ LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) { LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { // Cannot do this when no signature is available - if (!db->signature) { + if (!db->signature1 && !db->signature2) { DEBUG(db->ctx, "No signature available to verify\n"); return 1; } @@ -485,16 +531,24 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { // Read the header struct loc_database_header_v1 header_v1; + size_t bytes_read; switch (db->version) { case LOC_DATABASE_VERSION_1: - fread(&header_v1, 1, sizeof(header_v1), db->f); + bytes_read = fread(&header_v1, 1, sizeof(header_v1), db->f); + if (bytes_read < sizeof(header_v1)) { + ERROR(db->ctx, "Could not read header\n"); + r = 1; - // Clear signature - for (unsigned int i = 0; i < sizeof(header_v1.signature); i++) { - header_v1.signature[i] = '\0'; + goto CLEANUP; } + // Clear signatures + memset(header_v1.signature1, '\0', sizeof(header_v1.signature1)); + header_v1.signature1_length = 0; + memset(header_v1.signature2, '\0', sizeof(header_v1.signature2)); + header_v1.signature2_length = 0; + hexdump(db->ctx, &header_v1, sizeof(header_v1)); // Feed header into the hash @@ -518,7 +572,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { char buffer[64 * 1024]; while (!feof(db->f)) { - size_t bytes_read = fread(buffer, 1, sizeof(buffer), db->f); + bytes_read = fread(buffer, 1, sizeof(buffer), db->f); hexdump(db->ctx, buffer, bytes_read); @@ -531,27 +585,48 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { } } - // Finish - r = EVP_DigestVerifyFinal(mdctx, - (unsigned char*)db->signature, db->signature_length); + // Check first signature + if (db->signature1) { + hexdump(db->ctx, db->signature1, db->signature1_length); - if (r == 0) { - DEBUG(db->ctx, "The signature is invalid\n"); - r = 1; - } else if (r == 1) { - DEBUG(db->ctx, "The signature is valid\n"); - r = 0; - } else { - ERROR(db->ctx, "Error verifying the signature: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - r = 1; + r = EVP_DigestVerifyFinal(mdctx, + (unsigned char*)db->signature1, db->signature1_length); + + if (r == 0) { + DEBUG(db->ctx, "The first signature is invalid\n"); + r = 1; + } else if (r == 1) { + DEBUG(db->ctx, "The first signature is valid\n"); + r = 0; + } else { + ERROR(db->ctx, "Error verifying the first signature: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + r = -1; + } } - // Dump signature - hexdump(db->ctx, db->signature, db->signature_length); + // Check second signature only when the first one was invalid + if (r && db->signature2) { + hexdump(db->ctx, db->signature2, db->signature2_length); + + r = EVP_DigestVerifyFinal(mdctx, + (unsigned char*)db->signature2, db->signature2_length); + + if (r == 0) { + DEBUG(db->ctx, "The second signature is invalid\n"); + r = 1; + } else if (r == 1) { + DEBUG(db->ctx, "The second signature is valid\n"); + r = 0; + } else { + ERROR(db->ctx, "Error verifying the second signature: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + r = -1; + } + } clock_t end = clock(); - DEBUG(db->ctx, "Signature checked in %.4fms\n", + INFO(db->ctx, "Signature checked in %.4fms\n", (double)(end - start) / CLOCKS_PER_SEC * 1000); CLEANUP: @@ -611,8 +686,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, off_t lo = 0; off_t hi = db->as_count - 1; +#ifdef ENABLE_DEBUG // Save start time clock_t start = clock(); +#endif while (lo <= hi) { off_t i = (lo + hi) / 2; @@ -625,11 +702,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, // Check if this is a match uint32_t as_number = loc_as_get_number(*as); if (as_number == number) { +#ifdef ENABLE_DEBUG clock_t end = clock(); // Log how fast this has been DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, (double)(end - start) / CLOCKS_PER_SEC * 1000); +#endif return 0; } @@ -673,11 +752,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ return -1; } +#ifdef ENABLE_DEBUG if (r == 0) { char* string = loc_network_str(*network); DEBUG(db->ctx, "Got network %s\n", string); free(string); } +#endif return r; } @@ -702,8 +783,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru } // Check if the given IP address is inside the network - r = loc_network_match_address(*network, address); - if (r) { + if (!loc_network_match_address(*network, address)) { DEBUG(db->ctx, "Searched address is not part of the network\n"); loc_network_unref(*network); @@ -772,17 +852,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db, *network = NULL; +#ifdef ENABLE_DEBUG // Save start time clock_t start = clock(); +#endif int r = __loc_database_lookup(db, address, network, &network_address, db->network_nodes_v1, 0); +#ifdef ENABLE_DEBUG clock_t end = clock(); // Log how fast this has been DEBUG(db->ctx, "Executed network search in %.4fms\n", (double)(end - start) / CLOCKS_PER_SEC * 1000); +#endif return r; } @@ -829,8 +913,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, off_t lo = 0; off_t hi = db->countries_count - 1; +#ifdef ENABLE_DEBUG // Save start time clock_t start = clock(); +#endif while (lo <= hi) { off_t i = (lo + hi) / 2; @@ -845,11 +931,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, int result = strcmp(code, cc); if (result == 0) { +#ifdef ENABLE_DEBUG clock_t end = clock(); // Log how fast this has been DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, (double)(end - start) / CLOCKS_PER_SEC * 1000); +#endif return 0; } @@ -872,8 +960,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, // Enumerator +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); + + // Release all references + loc_database_unref(enumerator->db); + loc_unref(enumerator->ctx); + + if (enumerator->string) + free(enumerator->string); + + if (enumerator->countries) + loc_country_list_unref(enumerator->countries); + + if (enumerator->asns) + loc_as_list_unref(enumerator->asns); + + // Free network search + free(enumerator->networks_visited); + + // Free subnet stack + if (enumerator->stack) + loc_network_list_unref(enumerator->stack); + + free(enumerator); +} + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, - struct loc_database* db, enum loc_database_enumerator_mode mode) { + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { struct loc_database_enumerator* e = calloc(1, sizeof(*e)); if (!e) return -ENOMEM; @@ -884,11 +998,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum e->mode = mode; e->refcount = 1; + // Flatten output? + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + // Initialise graph search - //e->network_stack[++e->network_stack_depth] = 0; e->network_stack_depth = 1; e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); + // Allocate stack + int r = loc_network_list_new(e->ctx, &e->stack); + if (r) { + loc_database_enumerator_free(e); + return r; + } + DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); *enumerator = e; @@ -901,22 +1024,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo return enumerator; } -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); - - // Release all references - loc_database_unref(enumerator->db); - loc_unref(enumerator->ctx); - - if (enumerator->string) - free(enumerator->string); - - // Free network search - free(enumerator->networks_visited); - - free(enumerator); -} - LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { if (!enumerator) return NULL; @@ -938,40 +1045,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator return 0; } -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { - // Set empty country code - if (!country_code || !*country_code) { - *enumerator->country_code = '\0'; - return 0; - } +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( + struct loc_database_enumerator* enumerator) { + if (!enumerator->countries) + return NULL; - // Treat A1, A2, A3 as special country codes, - // but perform search for flags instead - if (strcmp(country_code, "A1") == 0) { - return loc_database_enumerator_set_flag(enumerator, - LOC_NETWORK_FLAG_ANONYMOUS_PROXY); - } else if (strcmp(country_code, "A2") == 0) { - return loc_database_enumerator_set_flag(enumerator, - LOC_NETWORK_FLAG_SATELLITE_PROVIDER); - } else if (strcmp(country_code, "A3") == 0) { - return loc_database_enumerator_set_flag(enumerator, - LOC_NETWORK_FLAG_ANYCAST); - } + return loc_country_list_ref(enumerator->countries); +} - // Country codes must be two characters - if (!loc_country_code_is_valid(country_code)) - return -EINVAL; +LOC_EXPORT int loc_database_enumerator_set_countries( + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { + if (enumerator->countries) + loc_country_list_unref(enumerator->countries); - for (unsigned int i = 0; i < 3; i++) { - enumerator->country_code[i] = country_code[i]; - } + enumerator->countries = loc_country_list_ref(countries); return 0; } -LOC_EXPORT int loc_database_enumerator_set_asn( - struct loc_database_enumerator* enumerator, unsigned int asn) { - enumerator->asn = asn; +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( + struct loc_database_enumerator* enumerator) { + if (!enumerator->asns) + return NULL; + + return loc_as_list_ref(enumerator->asns); +} + +LOC_EXPORT int loc_database_enumerator_set_asns( + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { + if (enumerator->asns) + loc_as_list_unref(enumerator->asns); + + enumerator->asns = loc_as_list_ref(asns); return 0; } @@ -1050,16 +1155,64 @@ static int loc_database_enumerator_stack_push_node( return 0; } -LOC_EXPORT int loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network) { - // Reset network - *network = NULL; +static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match + if (enumerator->family && loc_network_address_family(network) != enumerator->family) { + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); + return 1; + } - // Do not do anything if not in network mode - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) - return 0; + // Skip if the country code does not match + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { + const char* country_code = loc_network_get_country_code(network); - int r; + if (!loc_country_list_contains_code(enumerator->countries, country_code)) { + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); + return 1; + } + } + + // Skip if the ASN does not match + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { + uint32_t asn = loc_network_get_asn(network); + + if (!loc_as_list_contains_number(enumerator->asns, asn)) { + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); + return 1; + } + } + + // Skip if flags do not match + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); + return 1; + } + + // Do not filter + return 0; +} + +static int __loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { + // Return top element from the stack + while (1) { + *network = loc_network_list_pop_first(enumerator->stack); + + // Stack is empty + if (!*network) + break; + + // Throw away any networks by filter + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { + loc_network_unref(*network); + *network = NULL; + continue; + } + + // Return result + return 0; + } DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", enumerator->network_stack_depth); @@ -1089,7 +1242,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( enumerator->db->network_nodes_v1 + node->offset; // Add edges to stack - r = loc_database_enumerator_stack_push_node(enumerator, + int r = loc_database_enumerator_stack_push_node(enumerator, be32toh(n->one), 1, node->depth + 1); if (r) @@ -1115,50 +1268,167 @@ LOC_EXPORT int loc_database_enumerator_next_network( if (r) return r; - // Check if we are interested in this network + // Return all networks when the filter is disabled + if (!filter) + return 0; - // Skip if the family does not match - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { + // Check if we are interested in this network + if (loc_database_enumerator_filter_network(enumerator, *network)) { loc_network_unref(*network); *network = NULL; continue; } - // Skip if the country code does not match - if (*enumerator->country_code && - !loc_network_match_country_code(*network, enumerator->country_code)) { - loc_network_unref(*network); - *network = NULL; + return 0; + } + } - continue; - } + // Reached the end of the search + return 0; +} - // Skip if the ASN does not match - if (enumerator->asn && - !loc_network_match_asn(*network, enumerator->asn)) { - loc_network_unref(*network); - *network = NULL; +static int __loc_database_enumerator_next_network_flattened( + struct loc_database_enumerator* enumerator, struct loc_network** network) { + // Fetch the next network + int r = __loc_database_enumerator_next_network(enumerator, network, 1); + if (r) + return r; - continue; - } + // End if we could not read another network + if (!*network) + return 0; - // Skip if flags do not match - if (enumerator->flags && - !loc_network_match_flag(*network, enumerator->flags)) { - loc_network_unref(*network); - *network = NULL; + struct loc_network* subnet = NULL; + struct loc_network_list* subnets; + + // Create a list with all subnets + r = loc_network_list_new(enumerator->ctx, &subnets); + if (r) + return r; + + // Search all subnets from the database + while (1) { + // Fetch the next network in line + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); + if (r) { + loc_network_unref(subnet); + loc_network_list_unref(subnets); + + return r; + } + + // End if we did not receive another subnet + if (!subnet) + break; + + // Collect all subnets in a list + if (loc_network_is_subnet(*network, subnet)) { + r = loc_network_list_push(subnets, subnet); + if (r) { + loc_network_unref(subnet); + loc_network_list_unref(subnets); + + return r; } - return 0; + loc_network_unref(subnet); + continue; + } + + // If this is not a subnet, we push it back onto the stack and break + r = loc_network_list_push(enumerator->stack, subnet); + if (r) { + loc_network_unref(subnet); + loc_network_list_unref(subnets); + + return r; } + + loc_network_unref(subnet); + break; } - // Reached the end of the search + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); + + // We can abort here if the network has no subnets + if (loc_network_list_empty(subnets)) { + loc_network_list_unref(subnets); + + return 0; + } + + // If the network has any subnets, we will break it into smaller parts + // without the subnets. + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); + if (!excluded) { + loc_network_list_unref(subnets); + return -1; + } + + // Merge subnets onto the stack + r = loc_network_list_merge(enumerator->stack, subnets); + if (r) { + loc_network_list_unref(subnets); + loc_network_list_unref(excluded); + + return r; + } + + // Push excluded list onto the stack + r = loc_network_list_merge(enumerator->stack, excluded); + if (r) { + loc_network_list_unref(subnets); + loc_network_list_unref(excluded); + + return r; + } - // Mark all nodes as non-visited - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) - enumerator->networks_visited[i] = 0; + loc_network_list_unref(subnets); + loc_network_list_unref(excluded); + // Drop the network and restart the whole process again to pick the next network + loc_network_unref(*network); + + return __loc_database_enumerator_next_network_flattened(enumerator, network); +} + +LOC_EXPORT int loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network) { + // Do not do anything if not in network mode + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) + return 0; + + // Flatten output? + if (enumerator->flatten) + return __loc_database_enumerator_next_network_flattened(enumerator, network); + + return __loc_database_enumerator_next_network(enumerator, network, 1); +} + +LOC_EXPORT int loc_database_enumerator_next_country( + struct loc_database_enumerator* enumerator, struct loc_country** country) { + *country = NULL; + + // Do not do anything if not in country mode + if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES) + return 0; + + struct loc_database* db = enumerator->db; + + while (enumerator->country_index < db->countries_count) { + // Fetch the next country + int r = loc_database_fetch_country(db, country, enumerator->country_index++); + if (r) + return r; + + // We do not filter here, so it always is a match + return 0; + } + + // Reset the index + enumerator->country_index = 0; + + // We have searched through all of them return 0; }