[people/ms/libloc.git] / src / writer.c

/*
	libloc - A library to determine the location of someone on the Internet

	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
	Lesser General Public License for more details.
*/

#include <assert.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/queue.h>
#include <time.h>

#ifdef HAVE_ENDIAN_H
#  include <endian.h>
#endif

#include <openssl/bio.h>
#include <openssl/err.h>
#include <openssl/evp.h>
#include <openssl/pem.h>

#include <loc/libloc.h>
#include <loc/as.h>
#include <loc/compat.h>
#include <loc/country.h>
#include <loc/database.h>
#include <loc/format.h>
#include <loc/network.h>
#include <loc/private.h>
#include <loc/writer.h>

struct loc_writer {
	struct loc_ctx* ctx;
	int refcount;

	struct loc_stringpool* pool;
	off_t vendor;
	off_t description;
	off_t license;

	// Private key to sign any databases
	EVP_PKEY* private_key;

	struct loc_as** as;
	size_t as_count;

	struct loc_country** countries;
	size_t countries_count;

	struct loc_network_tree* networks;
};

static int parse_private_key(struct loc_writer* writer, FILE* f) {
	// Free any previously loaded keys
	if (writer->private_key)
		EVP_PKEY_free(writer->private_key);

	// Read the key
	writer->private_key = PEM_read_PrivateKey(f, NULL, NULL, NULL);

	// Log any errors
	if (!writer->private_key) {
		char* error = ERR_error_string(ERR_get_error(), NULL);
		ERROR(writer->ctx, "Could not parse private key: %s\n", error);

		return -1;
	}

	return 0;
}

LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer, FILE* fkey) {
	struct loc_writer* w = calloc(1, sizeof(*w));
	if (!w)
		return -ENOMEM;

	w->ctx = loc_ref(ctx);
	w->refcount = 1;

	int r = loc_stringpool_new(ctx, &w->pool);
	if (r) {
		loc_writer_unref(w);
		return r;
	}

	// Initialize the network tree
	r = loc_network_tree_new(ctx, &w->networks);
	if (r) {
		loc_writer_unref(w);
		return r;
	}

	// Load the private key to sign databases
	if (fkey) {
		r = parse_private_key(w, fkey);
		if (r) {
			loc_writer_unref(w);
			return r;
		}
	}

	*writer = w;
	return 0;
}

LOC_EXPORT struct loc_writer* loc_writer_ref(struct loc_writer* writer) {
	writer->refcount++;

	return writer;
}

static void loc_writer_free(struct loc_writer* writer) {
	DEBUG(writer->ctx, "Releasing writer at %p\n", writer);

	// Free private key
	if (writer->private_key)
		EVP_PKEY_free(writer->private_key);

	// Unref all AS
	for (unsigned int i = 0; i < writer->as_count; i++) {
		loc_as_unref(writer->as[i]);
	}

	// Release network tree
	if (writer->networks)
		loc_network_tree_unref(writer->networks);

	// Unref the string pool
	loc_stringpool_unref(writer->pool);

	loc_unref(writer->ctx);
	free(writer);
}

LOC_EXPORT struct loc_writer* loc_writer_unref(struct loc_writer* writer) {
	if (--writer->refcount > 0)
		return writer;

	loc_writer_free(writer);

	return NULL;
}

LOC_EXPORT const char* loc_writer_get_vendor(struct loc_writer* writer) {
	return loc_stringpool_get(writer->pool, writer->vendor);
}

LOC_EXPORT int loc_writer_set_vendor(struct loc_writer* writer, const char* vendor) {
	// Add the string to the string pool
	off_t offset = loc_stringpool_add(writer->pool, vendor);
	if (offset < 0)
		return offset;

	writer->vendor = offset;
	return 0;
}

LOC_EXPORT const char* loc_writer_get_description(struct loc_writer* writer) {
	return loc_stringpool_get(writer->pool, writer->description);
}

LOC_EXPORT int loc_writer_set_description(struct loc_writer* writer, const char* description) {
	// Add the string to the string pool
	off_t offset = loc_stringpool_add(writer->pool, description);
	if (offset < 0)
		return offset;

	writer->description = offset;
	return 0;
}

LOC_EXPORT const char* loc_writer_get_license(struct loc_writer* writer) {
	return loc_stringpool_get(writer->pool, writer->license);
}

LOC_EXPORT int loc_writer_set_license(struct loc_writer* writer, const char* license) {
	// Add the string to the string pool
	off_t offset = loc_stringpool_add(writer->pool, license);
	if (offset < 0)
		return offset;

	writer->license = offset;
	return 0;
}

static int __loc_as_cmp(const void* as1, const void* as2) {
	return loc_as_cmp(*(struct loc_as**)as1, *(struct loc_as**)as2);
}

LOC_EXPORT int loc_writer_add_as(struct loc_writer* writer, struct loc_as** as, uint32_t number) {
	int r = loc_as_new(writer->ctx, as, number);
	if (r)
		return r;

	// We have a new AS to add
	writer->as_count++;

	// Make space
	writer->as = realloc(writer->as, sizeof(*writer->as) * writer->as_count);
	if (!writer->as)
		return -ENOMEM;

	// Add as last element
	writer->as[writer->as_count - 1] = loc_as_ref(*as);

	// Sort everything
	qsort(writer->as, writer->as_count, sizeof(*writer->as), __loc_as_cmp);

	return 0;
}

LOC_EXPORT int loc_writer_add_network(struct loc_writer* writer, struct loc_network** network, const char* string) {
	int r;

	// Create a new network object
	r = loc_network_new_from_string(writer->ctx, network, string);
	if (r)
		return r;

	// Add it to the local tree
	return loc_network_tree_add_network(writer->networks, *network);
}

static int __loc_country_cmp(const void* country1, const void* country2) {
	return loc_country_cmp(*(struct loc_country**)country1, *(struct loc_country**)country2);
}

LOC_EXPORT int loc_writer_add_country(struct loc_writer* writer, struct loc_country** country, const char* country_code) {
	int r = loc_country_new(writer->ctx, country, country_code);
	if (r)
		return r;

	// We have a new country to add
	writer->countries_count++;

	// Make space
	writer->countries = realloc(writer->countries, sizeof(*writer->countries) * writer->countries_count);
	if (!writer->countries)
		return -ENOMEM;

	// Add as last element
	writer->countries[writer->countries_count - 1] = loc_country_ref(*country);

	// Sort everything
	qsort(writer->countries, writer->countries_count, sizeof(*writer->countries), __loc_country_cmp);

	return 0;
}

static void make_magic(struct loc_writer* writer, struct loc_database_magic* magic) {
	// Copy magic bytes
	for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
		magic->magic[i] = LOC_DATABASE_MAGIC[i];

	// Set version
	magic->version = htobe16(LOC_DATABASE_VERSION);
}

static void align_page_boundary(off_t* offset, FILE* f) {
	// Move to next page boundary
	while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
		*offset += fwrite("", 1, 1, f);
}

static int loc_database_write_pool(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	// Save the offset of the pool section
	DEBUG(writer->ctx, "Pool starts at %jd bytes\n", (intmax_t)*offset);
	header->pool_offset = htobe32(*offset);

	// Write the pool
	size_t pool_length = loc_stringpool_write(writer->pool, f);
	*offset += pool_length;

	DEBUG(writer->ctx, "Pool has a length of %zu bytes\n", pool_length);
	header->pool_length = htobe32(pool_length);

	return 0;
}

static int loc_database_write_as_section(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	DEBUG(writer->ctx, "AS section starts at %jd bytes\n", (intmax_t)*offset);
	header->as_offset = htobe32(*offset);

	size_t as_length = 0;

	struct loc_database_as_v0 as;
	for (unsigned int i = 0; i < writer->as_count; i++) {
		// Convert AS into database format
		loc_as_to_database_v0(writer->as[i], writer->pool, &as);

		// Write to disk
		*offset += fwrite(&as, 1, sizeof(as), f);
		as_length += sizeof(as);
	}

	DEBUG(writer->ctx, "AS section has a length of %zu bytes\n", as_length);
	header->as_length = htobe32(as_length);

	align_page_boundary(offset, f);

	return 0;
}

struct node {
	TAILQ_ENTRY(node) nodes;

	struct loc_network_tree_node* node;

	// Indices of the child nodes
	uint32_t index_zero;
	uint32_t index_one;
};

static struct node* make_node(struct loc_network_tree_node* node) {
	struct node* n = malloc(sizeof(*n));
	if (!n)
		return NULL;

	n->node  = loc_network_tree_node_ref(node);
	n->index_zero = n->index_one = 0;

	return n;
}

static void free_node(struct node* node) {
	loc_network_tree_node_unref(node->node);

	free(node);
}

struct network {
	TAILQ_ENTRY(network) networks;

	struct loc_network* network;
};

static struct network* make_network(struct loc_network* network) {
	struct network* n = malloc(sizeof(*n));
	if (!n)
		return NULL;

	n->network = loc_network_ref(network);

	return n;
}

static void free_network(struct network* network) {
	loc_network_unref(network->network);

	free(network);
}

static int loc_database_write_networks(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	// Write the network tree
	DEBUG(writer->ctx, "Network tree starts at %jd bytes\n", (intmax_t)*offset);
	header->network_tree_offset = htobe32(*offset);

	size_t network_tree_length = 0;
	size_t network_data_length = 0;

	struct node* node;
	struct node* child_node;

	uint32_t index = 0;
	uint32_t network_index = 0;

	struct loc_database_network_v0 db_network;
	struct loc_database_network_node_v0 db_node;

	// Initialize queue for nodes
	TAILQ_HEAD(node_t, node) nodes;
	TAILQ_INIT(&nodes);

	// Initialize queue for networks
	TAILQ_HEAD(network_t, network) networks;
	TAILQ_INIT(&networks);

	// Add root
	struct loc_network_tree_node* root = loc_network_tree_get_root(writer->networks);
	node = make_node(root);

	TAILQ_INSERT_TAIL(&nodes, node, nodes);

	while (!TAILQ_EMPTY(&nodes)) {
		// Pop first node in list
		node = TAILQ_FIRST(&nodes);
		TAILQ_REMOVE(&nodes, node, nodes);

		DEBUG(writer->ctx, "Processing node %p\n", node);

		// Get child nodes
		struct loc_network_tree_node* node_zero = loc_network_tree_node_get(node->node, 0);
		if (node_zero) {
			node->index_zero = ++index;

			child_node = make_node(node_zero);
			loc_network_tree_node_unref(node_zero);

			TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
		}

		struct loc_network_tree_node* node_one = loc_network_tree_node_get(node->node, 1);
		if (node_one) {
			node->index_one = ++index;

			child_node = make_node(node_one);
			loc_network_tree_node_unref(node_one);

			TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
		}

		// Prepare what we are writing to disk
		db_node.zero = htobe32(node->index_zero);
		db_node.one  = htobe32(node->index_one);

		if (loc_network_tree_node_is_leaf(node->node)) {
			struct loc_network* network = loc_network_tree_node_get_network(node->node);

			// Append network to be written out later
			struct network* nw = make_network(network);
			TAILQ_INSERT_TAIL(&networks, nw, networks);

			db_node.network = htobe32(network_index++);
			loc_network_unref(network);
		} else {
			db_node.network = htobe32(0xffffffff);
		}

		// Write the current node
		DEBUG(writer->ctx, "Writing node %p (0 = %d, 1 = %d)\n",
			node, node->index_zero, node->index_one);

		*offset += fwrite(&db_node, 1, sizeof(db_node), f);
		network_tree_length += sizeof(db_node);

		free_node(node);
	}

	loc_network_tree_node_unref(root);

	header->network_tree_length = htobe32(network_tree_length);

	align_page_boundary(offset, f);

	DEBUG(writer->ctx, "Networks data section starts at %jd bytes\n", (intmax_t)*offset);
	header->network_data_offset = htobe32(*offset);

	// We have now written the entire tree and have all networks
	// in a queue in order as they are indexed
	while (!TAILQ_EMPTY(&networks)) {
		struct network* nw = TAILQ_FIRST(&networks);
		TAILQ_REMOVE(&networks, nw, networks);

		// Prepare what we are writing to disk
		int r = loc_network_to_database_v0(nw->network, &db_network);
		if (r)
			return r;

		*offset += fwrite(&db_network, 1, sizeof(db_network), f);
		network_data_length += sizeof(db_network);

		free_network(nw);
	}

	header->network_data_length = htobe32(network_data_length);

	align_page_boundary(offset, f);

	return 0;
}

static int loc_database_write_countries(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	DEBUG(writer->ctx, "Countries section starts at %jd bytes\n", (intmax_t)*offset);
	header->countries_offset = htobe32(*offset);

	size_t countries_length = 0;

	struct loc_database_country_v0 country;
	for (unsigned int i = 0; i < writer->countries_count; i++) {
		// Convert country into database format
		loc_country_to_database_v0(writer->countries[i], writer->pool, &country);

		// Write to disk
		*offset += fwrite(&country, 1, sizeof(country), f);
		countries_length += sizeof(country);
	}

	DEBUG(writer->ctx, "Countries section has a length of %zu bytes\n", countries_length);
	header->countries_length = htobe32(countries_length);

	align_page_boundary(offset, f);

	return 0;
}

static int loc_writer_create_signature(struct loc_writer* writer,
		struct loc_database_header_v0* header, FILE* f) {
	DEBUG(writer->ctx, "Signing database...\n");

	// Read file from the beginning
	rewind(f);

	// Create a new context for signing
	EVP_MD_CTX* mdctx = EVP_MD_CTX_new();

	// Initialise the context
	int r = EVP_DigestSignInit(mdctx, NULL, NULL, NULL, writer->private_key);
	if (r != 1) {
		ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
		goto END;
	}

	// Read magic
	struct loc_database_magic magic;
	fread(&magic, 1, sizeof(magic), f);

	hexdump(writer->ctx, &magic, sizeof(magic));

	// Feed magic into the signature
	r = EVP_DigestSignUpdate(mdctx, &magic, sizeof(magic));
	if (r != 1) {
		ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
		goto END;
	}

	hexdump(writer->ctx, header, sizeof(*header));

	// Feed the header into the signature
	r = EVP_DigestSignUpdate(mdctx, header, sizeof(*header));
	if (r != 1) {
		ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
		goto END;
	}

	// Skip header
	fseek(f, sizeof(*header), SEEK_CUR);

	// Walk through the file in chunks of 64kB
	char buffer[64 * 1024];
	while (!feof(f)) {
		size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);

		if (ferror(f)) {
			ERROR(writer->ctx, "Error reading from file: %s\n", strerror(errno));
			r = errno;
			goto END;
		}

		hexdump(writer->ctx, buffer, bytes_read);

		r = EVP_DigestSignUpdate(mdctx, buffer, bytes_read);
		if (r != 1) {
			ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
			goto END;
		}
	}

	// Compute the signature
	size_t signature_length = sizeof(header->signature);

	r = EVP_DigestSignFinal(mdctx,
		(unsigned char*)header->signature, &signature_length);
	if (r != 1) {
		ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
		goto END;
	}

	// Save length of the signature
	header->signature_length = htobe32(signature_length);

	DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n",
		signature_length);
	r = 0;

	// Dump signature
	hexdump(writer->ctx, header->signature, signature_length);

END:
	EVP_MD_CTX_free(mdctx);

	return r;
}

LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f) {
	struct loc_database_magic magic;
	make_magic(writer, &magic);

	// Make the header
	struct loc_database_header_v0 header;
	header.vendor      = htobe32(writer->vendor);
	header.description = htobe32(writer->description);
	header.license     = htobe32(writer->license);

	time_t now = time(NULL);
	header.created_at = htobe64(now);

	// Clear the signature
	header.signature_length = 0;
	for (unsigned int i = 0; i < sizeof(header.signature); i++)
		header.signature[i] = '\0';

	// Clear the padding
	for (unsigned int i = 0; i < sizeof(header.padding); i++)
		header.padding[i] = '\0';

	int r;
	off_t offset = 0;

	// Start writing at the beginning of the file
	r = fseek(f, 0, SEEK_SET);
	if (r)
		return r;

	// Write the magic
	offset += fwrite(&magic, 1, sizeof(magic), f);

	// Skip the space we need to write the header later
	r = fseek(f, sizeof(header), SEEK_CUR);
	if (r) {
		DEBUG(writer->ctx, "Could not seek to position after header\n");
		return r;
	}
	offset += sizeof(header);

	align_page_boundary(&offset, f);

	// Write all ASes
	r = loc_database_write_as_section(writer, &header, &offset, f);
	if (r)
		return r;

	// Write all networks
	r = loc_database_write_networks(writer, &header, &offset, f);
	if (r)
		return r;

	// Write countries
	r = loc_database_write_countries(writer, &header, &offset, f);
	if (r)
		return r;

	// Write pool
	r = loc_database_write_pool(writer, &header, &offset, f);
	if (r)
		return r;

	// Create the signature
	if (writer->private_key) {
		r = loc_writer_create_signature(writer, &header, f);
		if (r)
			return r;
	}

	// Write the header
	r = fseek(f, sizeof(magic), SEEK_SET);
	if (r)
		return r;

	fwrite(&header, 1, sizeof(header), f);

	return r;
}
Commit	Line	Data
c182393f MT	1	/*
	2	libloc - A library to determine the location of someone on the Internet
	3
	4	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
	5
	6	This library is free software; you can redistribute it and/or
	7	modify it under the terms of the GNU Lesser General Public
	8	License as published by the Free Software Foundation; either
	9	version 2.1 of the License, or (at your option) any later version.
	10
	11	This library is distributed in the hope that it will be useful,
	12	but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15	*/
	16
726f9984	17	#include <assert.h>
c182393f MT	18	#include <errno.h>
	19	#include <stdio.h>
	20	#include <stdlib.h>
	21	#include <string.h>
438db08c	22	#include <sys/queue.h>
c182393f MT	23	#include <time.h>
c182393f MT	24
42f3ccd7 MT	25	#ifdef HAVE_ENDIAN_H
	26	# include <endian.h>
	27	#endif
	28
726f9984 MT	29	#include <openssl/bio.h>
	30	#include <openssl/err.h>
	31	#include <openssl/evp.h>
	32	#include <openssl/pem.h>
	33
9fc7f001 MT	34	#include <loc/libloc.h>
9fc7f001 MT	35	#include <loc/as.h>
42f3ccd7	36	#include <loc/compat.h>
ec684c1a	37	#include <loc/country.h>
726f9984	38	#include <loc/database.h>
c182393f	39	#include <loc/format.h>
3b5f4af2	40	#include <loc/network.h>
9fc7f001	41	#include <loc/private.h>
c182393f MT	42	#include <loc/writer.h>
c182393f MT	43
c182393f MT	44	struct loc_writer {
	45	struct loc_ctx* ctx;
	46	int refcount;
	47
	48	struct loc_stringpool* pool;
	49	off_t vendor;
	50	off_t description;
4bf49d00	51	off_t license;
c182393f	52
726f9984 MT	53	// Private key to sign any databases
	54	EVP_PKEY* private_key;
	55
c182393f MT	56	struct loc_as** as;
c182393f MT	57	size_t as_count;
3b5f4af2	58
ec684c1a MT	59	struct loc_country** countries;
	60	size_t countries_count;
	61
3b5f4af2	62	struct loc_network_tree* networks;
c182393f MT	63	};
c182393f MT	64
726f9984 MT	65	static int parse_private_key(struct loc_writer* writer, FILE* f) {
	66	// Free any previously loaded keys
	67	if (writer->private_key)
	68	EVP_PKEY_free(writer->private_key);
	69
	70	// Read the key
	71	writer->private_key = PEM_read_PrivateKey(f, NULL, NULL, NULL);
	72
	73	// Log any errors
	74	if (!writer->private_key) {
	75	char* error = ERR_error_string(ERR_get_error(), NULL);
	76	ERROR(writer->ctx, "Could not parse private key: %s\n", error);
	77
	78	return -1;
	79	}
	80
	81	return 0;
	82	}
	83
	84	LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer, FILE* fkey) {
c182393f MT	85	struct loc_writer* w = calloc(1, sizeof(*w));
	86	if (!w)
	87	return -ENOMEM;
	88
	89	w->ctx = loc_ref(ctx);
	90	w->refcount = 1;
	91
0e974d4b	92	int r = loc_stringpool_new(ctx, &w->pool);
c182393f MT	93	if (r) {
	94	loc_writer_unref(w);
	95	return r;
	96	}
	97
3b5f4af2 MT	98	// Initialize the network tree
	99	r = loc_network_tree_new(ctx, &w->networks);
	100	if (r) {
	101	loc_writer_unref(w);
	102	return r;
	103	}
	104
726f9984 MT	105	// Load the private key to sign databases
	106	if (fkey) {
	107	r = parse_private_key(w, fkey);
	108	if (r) {
	109	loc_writer_unref(w);
	110	return r;
	111	}
	112	}
	113
c182393f MT	114	*writer = w;
	115	return 0;
	116	}
	117
	118	LOC_EXPORT struct loc_writer* loc_writer_ref(struct loc_writer* writer) {
	119	writer->refcount++;
	120
	121	return writer;
	122	}
	123
	124	static void loc_writer_free(struct loc_writer* writer) {
	125	DEBUG(writer->ctx, "Releasing writer at %p\n", writer);
	126
726f9984 MT	127	// Free private key
	128	if (writer->private_key)
	129	EVP_PKEY_free(writer->private_key);
	130
c182393f MT	131	// Unref all AS
	132	for (unsigned int i = 0; i < writer->as_count; i++) {
	133	loc_as_unref(writer->as[i]);
	134	}
	135
3b5f4af2 MT	136	// Release network tree
	137	if (writer->networks)
	138	loc_network_tree_unref(writer->networks);
	139
c182393f MT	140	// Unref the string pool
	141	loc_stringpool_unref(writer->pool);
	142
	143	loc_unref(writer->ctx);
	144	free(writer);
	145	}
	146
	147	LOC_EXPORT struct loc_writer* loc_writer_unref(struct loc_writer* writer) {
	148	if (--writer->refcount > 0)
	149	return writer;
	150
	151	loc_writer_free(writer);
	152
	153	return NULL;
	154	}
	155
	156	LOC_EXPORT const char* loc_writer_get_vendor(struct loc_writer* writer) {
	157	return loc_stringpool_get(writer->pool, writer->vendor);
	158	}
	159
	160	LOC_EXPORT int loc_writer_set_vendor(struct loc_writer* writer, const char* vendor) {
	161	// Add the string to the string pool
	162	off_t offset = loc_stringpool_add(writer->pool, vendor);
	163	if (offset < 0)
	164	return offset;
	165
	166	writer->vendor = offset;
	167	return 0;
	168	}
	169
	170	LOC_EXPORT const char* loc_writer_get_description(struct loc_writer* writer) {
	171	return loc_stringpool_get(writer->pool, writer->description);
	172	}
	173
	174	LOC_EXPORT int loc_writer_set_description(struct loc_writer* writer, const char* description) {
	175	// Add the string to the string pool
	176	off_t offset = loc_stringpool_add(writer->pool, description);
	177	if (offset < 0)
	178	return offset;
	179
	180	writer->description = offset;
	181	return 0;
	182	}
	183
4bf49d00 MT	184	LOC_EXPORT const char* loc_writer_get_license(struct loc_writer* writer) {
	185	return loc_stringpool_get(writer->pool, writer->license);
	186	}
	187
	188	LOC_EXPORT int loc_writer_set_license(struct loc_writer* writer, const char* license) {
	189	// Add the string to the string pool
	190	off_t offset = loc_stringpool_add(writer->pool, license);
	191	if (offset < 0)
	192	return offset;
	193
	194	writer->license = offset;
	195	return 0;
	196	}
	197
c182393f MT	198	static int __loc_as_cmp(const void* as1, const void* as2) {
	199	return loc_as_cmp((struct loc_as)as1, (struct loc_as**)as2);
	200	}
	201
	202	LOC_EXPORT int loc_writer_add_as(struct loc_writer* writer, struct loc_as** as, uint32_t number) {
29bd8211	203	int r = loc_as_new(writer->ctx, as, number);
c182393f MT	204	if (r)
	205	return r;
	206
	207	// We have a new AS to add
	208	writer->as_count++;
	209
	210	// Make space
	211	writer->as = realloc(writer->as, sizeof(writer->as) writer->as_count);
	212	if (!writer->as)
	213	return -ENOMEM;
	214
	215	// Add as last element
	216	writer->as[writer->as_count - 1] = loc_as_ref(*as);
	217
	218	// Sort everything
	219	qsort(writer->as, writer->as_count, sizeof(*writer->as), __loc_as_cmp);
	220
	221	return 0;
	222	}
	223
3b5f4af2 MT	224	LOC_EXPORT int loc_writer_add_network(struct loc_writer* writer, struct loc_network** network, const char* string) {
	225	int r;
	226
	227	// Create a new network object
	228	r = loc_network_new_from_string(writer->ctx, network, string);
	229	if (r)
	230	return r;
	231
	232	// Add it to the local tree
	233	return loc_network_tree_add_network(writer->networks, *network);
	234	}
	235
ec684c1a MT	236	static int __loc_country_cmp(const void* country1, const void* country2) {
	237	return loc_country_cmp((struct loc_country)country1, (struct loc_country**)country2);
	238	}
	239
	240	LOC_EXPORT int loc_writer_add_country(struct loc_writer* writer, struct loc_country** country, const char* country_code) {
	241	int r = loc_country_new(writer->ctx, country, country_code);
	242	if (r)
	243	return r;
	244
	245	// We have a new country to add
	246	writer->countries_count++;
	247
	248	// Make space
	249	writer->countries = realloc(writer->countries, sizeof(writer->countries) writer->countries_count);
	250	if (!writer->countries)
	251	return -ENOMEM;
	252
	253	// Add as last element
	254	writer->countries[writer->countries_count - 1] = loc_country_ref(*country);
	255
	256	// Sort everything
	257	qsort(writer->countries, writer->countries_count, sizeof(*writer->countries), __loc_country_cmp);
	258
	259	return 0;
	260	}
	261
c182393f MT	262	static void make_magic(struct loc_writer* writer, struct loc_database_magic* magic) {
	263	// Copy magic bytes
	264	for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
	265	magic->magic[i] = LOC_DATABASE_MAGIC[i];
	266
	267	// Set version
0676cd80	268	magic->version = htobe16(LOC_DATABASE_VERSION);
c182393f MT	269	}
	270
	271	static void align_page_boundary(off_t* offset, FILE* f) {
	272	// Move to next page boundary
	273	while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
	274	*offset += fwrite("", 1, 1, f);
	275	}
	276
	277	static int loc_database_write_pool(struct loc_writer* writer,
	278	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	279	// Save the offset of the pool section
5c57de03	280	DEBUG(writer->ctx, "Pool starts at %jd bytes\n", (intmax_t)*offset);
0676cd80	281	header->pool_offset = htobe32(*offset);
c182393f MT	282
	283	// Write the pool
	284	size_t pool_length = loc_stringpool_write(writer->pool, f);
	285	*offset += pool_length;
	286
	287	DEBUG(writer->ctx, "Pool has a length of %zu bytes\n", pool_length);
0676cd80	288	header->pool_length = htobe32(pool_length);
c182393f MT	289
	290	return 0;
	291	}
	292
	293	static int loc_database_write_as_section(struct loc_writer* writer,
	294	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
5c57de03	295	DEBUG(writer->ctx, "AS section starts at %jd bytes\n", (intmax_t)*offset);
0676cd80	296	header->as_offset = htobe32(*offset);
c182393f MT	297
	298	size_t as_length = 0;
	299
	300	struct loc_database_as_v0 as;
	301	for (unsigned int i = 0; i < writer->as_count; i++) {
	302	// Convert AS into database format
29bd8211	303	loc_as_to_database_v0(writer->as[i], writer->pool, &as);
c182393f MT	304
c182393f MT	305	// Write to disk
65862405	306	*offset += fwrite(&as, 1, sizeof(as), f);
c182393f MT	307	as_length += sizeof(as);
	308	}
	309
	310	DEBUG(writer->ctx, "AS section has a length of %zu bytes\n", as_length);
0676cd80	311	header->as_length = htobe32(as_length);
c182393f	312
fda89f24 MT	313	align_page_boundary(offset, f);
fda89f24 MT	314
c182393f MT	315	return 0;
	316	}
	317
438db08c MT	318	struct node {
438db08c MT	319	TAILQ_ENTRY(node) nodes;
f3e02bc5	320
438db08c	321	struct loc_network_tree_node* node;
f3e02bc5	322
438db08c MT	323	// Indices of the child nodes
	324	uint32_t index_zero;
	325	uint32_t index_one;
	326	};
f3e02bc5	327
438db08c MT	328	static struct node* make_node(struct loc_network_tree_node* node) {
	329	struct node* n = malloc(sizeof(*n));
	330	if (!n)
	331	return NULL;
f3e02bc5	332
438db08c MT	333	n->node = loc_network_tree_node_ref(node);
	334	n->index_zero = n->index_one = 0;
	335
	336	return n;
	337	}
	338
	339	static void free_node(struct node* node) {
	340	loc_network_tree_node_unref(node->node);
	341
	342	free(node);
	343	}
	344
	345	struct network {
	346	TAILQ_ENTRY(network) networks;
	347
	348	struct loc_network* network;
	349	};
	350
	351	static struct network* make_network(struct loc_network* network) {
	352	struct network* n = malloc(sizeof(*n));
	353	if (!n)
	354	return NULL;
	355
	356	n->network = loc_network_ref(network);
	357
	358	return n;
f3e02bc5 MT	359	}
f3e02bc5 MT	360
438db08c MT	361	static void free_network(struct network* network) {
	362	loc_network_unref(network->network);
	363
	364	free(network);
	365	}
	366
	367	static int loc_database_write_networks(struct loc_writer* writer,
f3e02bc5	368	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
438db08c	369	// Write the network tree
5c57de03	370	DEBUG(writer->ctx, "Network tree starts at %jd bytes\n", (intmax_t)*offset);
438db08c	371	header->network_tree_offset = htobe32(*offset);
f3e02bc5	372
438db08c MT	373	size_t network_tree_length = 0;
438db08c MT	374	size_t network_data_length = 0;
f3e02bc5	375
438db08c MT	376	struct node* node;
	377	struct node* child_node;
	378
	379	uint32_t index = 0;
	380	uint32_t network_index = 0;
	381
	382	struct loc_database_network_v0 db_network;
	383	struct loc_database_network_node_v0 db_node;
	384
	385	// Initialize queue for nodes
	386	TAILQ_HEAD(node_t, node) nodes;
	387	TAILQ_INIT(&nodes);
	388
	389	// Initialize queue for networks
	390	TAILQ_HEAD(network_t, network) networks;
	391	TAILQ_INIT(&networks);
	392
	393	// Add root
	394	struct loc_network_tree_node* root = loc_network_tree_get_root(writer->networks);
	395	node = make_node(root);
	396
	397	TAILQ_INSERT_TAIL(&nodes, node, nodes);
	398
	399	while (!TAILQ_EMPTY(&nodes)) {
	400	// Pop first node in list
	401	node = TAILQ_FIRST(&nodes);
	402	TAILQ_REMOVE(&nodes, node, nodes);
	403
	404	DEBUG(writer->ctx, "Processing node %p\n", node);
	405
	406	// Get child nodes
	407	struct loc_network_tree_node* node_zero = loc_network_tree_node_get(node->node, 0);
	408	if (node_zero) {
	409	node->index_zero = ++index;
	410
	411	child_node = make_node(node_zero);
	412	loc_network_tree_node_unref(node_zero);
	413
	414	TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
	415	}
	416
	417	struct loc_network_tree_node* node_one = loc_network_tree_node_get(node->node, 1);
	418	if (node_one) {
	419	node->index_one = ++index;
	420
	421	child_node = make_node(node_one);
	422	loc_network_tree_node_unref(node_one);
	423
	424	TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
	425	}
	426
	427	// Prepare what we are writing to disk
39a55353 MT	428	db_node.zero = htobe32(node->index_zero);
	429	db_node.one = htobe32(node->index_one);
	430
438db08c MT	431	if (loc_network_tree_node_is_leaf(node->node)) {
	432	struct loc_network* network = loc_network_tree_node_get_network(node->node);
	433
	434	// Append network to be written out later
	435	struct network* nw = make_network(network);
	436	TAILQ_INSERT_TAIL(&networks, nw, networks);
	437
39a55353	438	db_node.network = htobe32(network_index++);
438db08c MT	439	loc_network_unref(network);
438db08c MT	440	} else {
39a55353	441	db_node.network = htobe32(0xffffffff);
438db08c MT	442	}
	443
	444	// Write the current node
	445	DEBUG(writer->ctx, "Writing node %p (0 = %d, 1 = %d)\n",
	446	node, node->index_zero, node->index_one);
	447
	448	*offset += fwrite(&db_node, 1, sizeof(db_node), f);
	449	network_tree_length += sizeof(db_node);
	450
	451	free_node(node);
	452	}
	453
	454	loc_network_tree_node_unref(root);
	455
	456	header->network_tree_length = htobe32(network_tree_length);
	457
	458	align_page_boundary(offset, f);
	459
5c57de03	460	DEBUG(writer->ctx, "Networks data section starts at %jd bytes\n", (intmax_t)*offset);
438db08c MT	461	header->network_data_offset = htobe32(*offset);
	462
	463	// We have now written the entire tree and have all networks
	464	// in a queue in order as they are indexed
	465	while (!TAILQ_EMPTY(&networks)) {
	466	struct network* nw = TAILQ_FIRST(&networks);
	467	TAILQ_REMOVE(&networks, nw, networks);
	468
	469	// Prepare what we are writing to disk
	470	int r = loc_network_to_database_v0(nw->network, &db_network);
	471	if (r)
	472	return r;
	473
	474	*offset += fwrite(&db_network, 1, sizeof(db_network), f);
	475	network_data_length += sizeof(db_network);
	476
	477	free_network(nw);
	478	}
f3e02bc5	479
438db08c	480	header->network_data_length = htobe32(network_data_length);
f3e02bc5	481
fda89f24 MT	482	align_page_boundary(offset, f);
fda89f24 MT	483
f3e02bc5 MT	484	return 0;
	485	}
	486
ec684c1a MT	487	static int loc_database_write_countries(struct loc_writer* writer,
ec684c1a MT	488	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
2e2325a9	489	DEBUG(writer->ctx, "Countries section starts at %jd bytes\n", (intmax_t)*offset);
ec684c1a MT	490	header->countries_offset = htobe32(*offset);
	491
	492	size_t countries_length = 0;
	493
	494	struct loc_database_country_v0 country;
	495	for (unsigned int i = 0; i < writer->countries_count; i++) {
	496	// Convert country into database format
	497	loc_country_to_database_v0(writer->countries[i], writer->pool, &country);
	498
	499	// Write to disk
	500	*offset += fwrite(&country, 1, sizeof(country), f);
	501	countries_length += sizeof(country);
	502	}
	503
	504	DEBUG(writer->ctx, "Countries section has a length of %zu bytes\n", countries_length);
	505	header->countries_length = htobe32(countries_length);
	506
	507	align_page_boundary(offset, f);
	508
	509	return 0;
	510	}
	511
726f9984 MT	512	static int loc_writer_create_signature(struct loc_writer* writer,
	513	struct loc_database_header_v0* header, FILE* f) {
	514	DEBUG(writer->ctx, "Signing database...\n");
	515
	516	// Read file from the beginning
	517	rewind(f);
	518
	519	// Create a new context for signing
	520	EVP_MD_CTX* mdctx = EVP_MD_CTX_new();
	521
	522	// Initialise the context
	523	int r = EVP_DigestSignInit(mdctx, NULL, NULL, NULL, writer->private_key);
	524	if (r != 1) {
	525	ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
	526	goto END;
	527	}
	528
	529	// Read magic
	530	struct loc_database_magic magic;
	531	fread(&magic, 1, sizeof(magic), f);
	532
a0cff45d MT	533	hexdump(writer->ctx, &magic, sizeof(magic));
a0cff45d MT	534
726f9984 MT	535	// Feed magic into the signature
	536	r = EVP_DigestSignUpdate(mdctx, &magic, sizeof(magic));
	537	if (r != 1) {
	538	ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
	539	goto END;
	540	}
	541
a0cff45d MT	542	hexdump(writer->ctx, header, sizeof(*header));
a0cff45d MT	543
726f9984	544	// Feed the header into the signature
72633a85	545	r = EVP_DigestSignUpdate(mdctx, header, sizeof(*header));
726f9984 MT	546	if (r != 1) {
	547	ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
	548	goto END;
	549	}
	550
	551	// Skip header
72633a85	552	fseek(f, sizeof(*header), SEEK_CUR);
726f9984 MT	553
	554	// Walk through the file in chunks of 64kB
	555	char buffer[64 * 1024];
	556	while (!feof(f)) {
	557	size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
	558
	559	if (ferror(f)) {
	560	ERROR(writer->ctx, "Error reading from file: %s\n", strerror(errno));
	561	r = errno;
	562	goto END;
	563	}
	564
a0cff45d MT	565	hexdump(writer->ctx, buffer, bytes_read);
a0cff45d MT	566
726f9984 MT	567	r = EVP_DigestSignUpdate(mdctx, buffer, bytes_read);
	568	if (r != 1) {
	569	ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
	570	goto END;
	571	}
	572	}
	573
	574	// Compute the signature
	575	size_t signature_length = sizeof(header->signature);
	576
	577	r = EVP_DigestSignFinal(mdctx,
	578	(unsigned char*)header->signature, &signature_length);
	579	if (r != 1) {
	580	ERROR(writer->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
	581	goto END;
	582	}
	583
	584	// Save length of the signature
	585	header->signature_length = htobe32(signature_length);
	586
257626f5 MT	587	DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n",
257626f5 MT	588	signature_length);
726f9984 MT	589	r = 0;
726f9984 MT	590
257626f5 MT	591	// Dump signature
	592	hexdump(writer->ctx, header->signature, signature_length);
	593
726f9984 MT	594	END:
	595	EVP_MD_CTX_free(mdctx);
	596
	597	return r;
	598	}
	599
c182393f MT	600	LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f) {
	601	struct loc_database_magic magic;
	602	make_magic(writer, &magic);
	603
	604	// Make the header
	605	struct loc_database_header_v0 header;
0676cd80 MT	606	header.vendor = htobe32(writer->vendor);
0676cd80 MT	607	header.description = htobe32(writer->description);
4bf49d00	608	header.license = htobe32(writer->license);
c182393f MT	609
	610	time_t now = time(NULL);
	611	header.created_at = htobe64(now);
	612
b1720435	613	// Clear the signature
726f9984	614	header.signature_length = 0;
b1720435 MT	615	for (unsigned int i = 0; i < sizeof(header.signature); i++)
	616	header.signature[i] = '\0';
	617
5e164830 MT	618	// Clear the padding
	619	for (unsigned int i = 0; i < sizeof(header.padding); i++)
	620	header.padding[i] = '\0';
	621
c182393f MT	622	int r;
	623	off_t offset = 0;
	624
	625	// Start writing at the beginning of the file
	626	r = fseek(f, 0, SEEK_SET);
	627	if (r)
	628	return r;
	629
	630	// Write the magic
	631	offset += fwrite(&magic, 1, sizeof(magic), f);
	632
	633	// Skip the space we need to write the header later
	634	r = fseek(f, sizeof(header), SEEK_CUR);
	635	if (r) {
	636	DEBUG(writer->ctx, "Could not seek to position after header\n");
	637	return r;
	638	}
	639	offset += sizeof(header);
	640
	641	align_page_boundary(&offset, f);
	642
	643	// Write all ASes
	644	r = loc_database_write_as_section(writer, &header, &offset, f);
	645	if (r)
	646	return r;
	647
f3e02bc5	648	// Write all networks
438db08c	649	r = loc_database_write_networks(writer, &header, &offset, f);
f3e02bc5 MT	650	if (r)
	651	return r;
	652
47bea941 MT	653	// Write countries
47bea941 MT	654	r = loc_database_write_countries(writer, &header, &offset, f);
c182393f MT	655	if (r)
	656	return r;
	657
47bea941 MT	658	// Write pool
47bea941 MT	659	r = loc_database_write_pool(writer, &header, &offset, f);
ec684c1a MT	660	if (r)
	661	return r;
	662
726f9984 MT	663	// Create the signature
	664	if (writer->private_key) {
	665	r = loc_writer_create_signature(writer, &header, f);
	666	if (r)
	667	return r;
	668	}
	669
c182393f MT	670	// Write the header
	671	r = fseek(f, sizeof(magic), SEEK_SET);
	672	if (r)
	673	return r;
	674
726f9984	675	fwrite(&header, 1, sizeof(header), f);
c182393f	676
726f9984	677	return r;
c182393f	678	}