[people/ms/libloc.git] / src / writer.c

/*
	libloc - A library to determine the location of someone on the Internet

	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
	Lesser General Public License for more details.
*/

#include <endian.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/queue.h>
#include <time.h>

#include <loc/libloc.h>
#include <loc/as.h>
#include <loc/country.h>
#include <loc/format.h>
#include <loc/network.h>
#include <loc/private.h>
#include <loc/writer.h>

struct loc_writer {
	struct loc_ctx* ctx;
	int refcount;

	struct loc_stringpool* pool;
	off_t vendor;
	off_t description;
	off_t license;

	struct loc_as** as;
	size_t as_count;

	struct loc_country** countries;
	size_t countries_count;

	struct loc_network_tree* networks;
};

LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer) {
	struct loc_writer* w = calloc(1, sizeof(*w));
	if (!w)
		return -ENOMEM;

	w->ctx = loc_ref(ctx);
	w->refcount = 1;

	int r = loc_stringpool_new(ctx, &w->pool);
	if (r) {
		loc_writer_unref(w);
		return r;
	}

	// Initialize the network tree
	r = loc_network_tree_new(ctx, &w->networks);
	if (r) {
		loc_writer_unref(w);
		return r;
	}

	*writer = w;
	return 0;
}

LOC_EXPORT struct loc_writer* loc_writer_ref(struct loc_writer* writer) {
	writer->refcount++;

	return writer;
}

static void loc_writer_free(struct loc_writer* writer) {
	DEBUG(writer->ctx, "Releasing writer at %p\n", writer);

	// Unref all AS
	for (unsigned int i = 0; i < writer->as_count; i++) {
		loc_as_unref(writer->as[i]);
	}

	// Release network tree
	if (writer->networks)
		loc_network_tree_unref(writer->networks);

	// Unref the string pool
	loc_stringpool_unref(writer->pool);

	loc_unref(writer->ctx);
	free(writer);
}

LOC_EXPORT struct loc_writer* loc_writer_unref(struct loc_writer* writer) {
	if (--writer->refcount > 0)
		return writer;

	loc_writer_free(writer);

	return NULL;
}

LOC_EXPORT const char* loc_writer_get_vendor(struct loc_writer* writer) {
	return loc_stringpool_get(writer->pool, writer->vendor);
}

LOC_EXPORT int loc_writer_set_vendor(struct loc_writer* writer, const char* vendor) {
	// Add the string to the string pool
	off_t offset = loc_stringpool_add(writer->pool, vendor);
	if (offset < 0)
		return offset;

	writer->vendor = offset;
	return 0;
}

LOC_EXPORT const char* loc_writer_get_description(struct loc_writer* writer) {
	return loc_stringpool_get(writer->pool, writer->description);
}

LOC_EXPORT int loc_writer_set_description(struct loc_writer* writer, const char* description) {
	// Add the string to the string pool
	off_t offset = loc_stringpool_add(writer->pool, description);
	if (offset < 0)
		return offset;

	writer->description = offset;
	return 0;
}

LOC_EXPORT const char* loc_writer_get_license(struct loc_writer* writer) {
	return loc_stringpool_get(writer->pool, writer->license);
}

LOC_EXPORT int loc_writer_set_license(struct loc_writer* writer, const char* license) {
	// Add the string to the string pool
	off_t offset = loc_stringpool_add(writer->pool, license);
	if (offset < 0)
		return offset;

	writer->license = offset;
	return 0;
}

static int __loc_as_cmp(const void* as1, const void* as2) {
	return loc_as_cmp(*(struct loc_as**)as1, *(struct loc_as**)as2);
}

LOC_EXPORT int loc_writer_add_as(struct loc_writer* writer, struct loc_as** as, uint32_t number) {
	int r = loc_as_new(writer->ctx, as, number);
	if (r)
		return r;

	// We have a new AS to add
	writer->as_count++;

	// Make space
	writer->as = realloc(writer->as, sizeof(*writer->as) * writer->as_count);
	if (!writer->as)
		return -ENOMEM;

	// Add as last element
	writer->as[writer->as_count - 1] = loc_as_ref(*as);

	// Sort everything
	qsort(writer->as, writer->as_count, sizeof(*writer->as), __loc_as_cmp);

	return 0;
}

LOC_EXPORT int loc_writer_add_network(struct loc_writer* writer, struct loc_network** network, const char* string) {
	int r;

	// Create a new network object
	r = loc_network_new_from_string(writer->ctx, network, string);
	if (r)
		return r;

	// Add it to the local tree
	return loc_network_tree_add_network(writer->networks, *network);
}

static int __loc_country_cmp(const void* country1, const void* country2) {
	return loc_country_cmp(*(struct loc_country**)country1, *(struct loc_country**)country2);
}

LOC_EXPORT int loc_writer_add_country(struct loc_writer* writer, struct loc_country** country, const char* country_code) {
	int r = loc_country_new(writer->ctx, country, country_code);
	if (r)
		return r;

	// We have a new country to add
	writer->countries_count++;

	// Make space
	writer->countries = realloc(writer->countries, sizeof(*writer->countries) * writer->countries_count);
	if (!writer->countries)
		return -ENOMEM;

	// Add as last element
	writer->countries[writer->countries_count - 1] = loc_country_ref(*country);

	// Sort everything
	qsort(writer->countries, writer->countries_count, sizeof(*writer->countries), __loc_country_cmp);

	return 0;
}

static void make_magic(struct loc_writer* writer, struct loc_database_magic* magic) {
	// Copy magic bytes
	for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
		magic->magic[i] = LOC_DATABASE_MAGIC[i];

	// Set version
	magic->version = htobe16(LOC_DATABASE_VERSION);
}

static void align_page_boundary(off_t* offset, FILE* f) {
	// Move to next page boundary
	while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
		*offset += fwrite("", 1, 1, f);
}

static int loc_database_write_pool(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	// Save the offset of the pool section
	DEBUG(writer->ctx, "Pool starts at %jd bytes\n", *offset);
	header->pool_offset = htobe32(*offset);

	// Write the pool
	size_t pool_length = loc_stringpool_write(writer->pool, f);
	*offset += pool_length;

	DEBUG(writer->ctx, "Pool has a length of %zu bytes\n", pool_length);
	header->pool_length = htobe32(pool_length);

	return 0;
}

static int loc_database_write_as_section(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	DEBUG(writer->ctx, "AS section starts at %jd bytes\n", *offset);
	header->as_offset = htobe32(*offset);

	size_t as_length = 0;

	struct loc_database_as_v0 as;
	for (unsigned int i = 0; i < writer->as_count; i++) {
		// Convert AS into database format
		loc_as_to_database_v0(writer->as[i], writer->pool, &as);

		// Write to disk
		*offset += fwrite(&as, 1, sizeof(as), f);
		as_length += sizeof(as);
	}

	DEBUG(writer->ctx, "AS section has a length of %zu bytes\n", as_length);
	header->as_length = htobe32(as_length);

	align_page_boundary(offset, f);

	return 0;
}

struct node {
	TAILQ_ENTRY(node) nodes;

	struct loc_network_tree_node* node;

	// Indices of the child nodes
	uint32_t index_zero;
	uint32_t index_one;
};

static struct node* make_node(struct loc_network_tree_node* node) {
	struct node* n = malloc(sizeof(*n));
	if (!n)
		return NULL;

	n->node  = loc_network_tree_node_ref(node);
	n->index_zero = n->index_one = 0;

	return n;
}

static void free_node(struct node* node) {
	loc_network_tree_node_unref(node->node);

	free(node);
}

struct network {
	TAILQ_ENTRY(network) networks;

	struct loc_network* network;
};

static struct network* make_network(struct loc_network* network) {
	struct network* n = malloc(sizeof(*n));
	if (!n)
		return NULL;

	n->network = loc_network_ref(network);

	return n;
}

static void free_network(struct network* network) {
	loc_network_unref(network->network);

	free(network);
}

static int loc_database_write_networks(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	// Write the network tree
	DEBUG(writer->ctx, "Network tree starts at %jd bytes\n", *offset);
	header->network_tree_offset = htobe32(*offset);

	size_t network_tree_length = 0;
	size_t network_data_length = 0;

	struct node* node;
	struct node* child_node;

	uint32_t index = 0;
	uint32_t network_index = 0;

	struct loc_database_network_v0 db_network;
	struct loc_database_network_node_v0 db_node;

	// Initialize queue for nodes
	TAILQ_HEAD(node_t, node) nodes;
	TAILQ_INIT(&nodes);

	// Initialize queue for networks
	TAILQ_HEAD(network_t, network) networks;
	TAILQ_INIT(&networks);

	// Add root
	struct loc_network_tree_node* root = loc_network_tree_get_root(writer->networks);
	node = make_node(root);

	TAILQ_INSERT_TAIL(&nodes, node, nodes);

	while (!TAILQ_EMPTY(&nodes)) {
		// Pop first node in list
		node = TAILQ_FIRST(&nodes);
		TAILQ_REMOVE(&nodes, node, nodes);

		DEBUG(writer->ctx, "Processing node %p\n", node);

		// Get child nodes
		struct loc_network_tree_node* node_zero = loc_network_tree_node_get(node->node, 0);
		if (node_zero) {
			node->index_zero = ++index;

			child_node = make_node(node_zero);
			loc_network_tree_node_unref(node_zero);

			TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
		}

		struct loc_network_tree_node* node_one = loc_network_tree_node_get(node->node, 1);
		if (node_one) {
			node->index_one = ++index;

			child_node = make_node(node_one);
			loc_network_tree_node_unref(node_one);

			TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
		}

		// Prepare what we are writing to disk
		db_node.zero = htobe32(node->index_zero);
		db_node.one  = htobe32(node->index_one);

		if (loc_network_tree_node_is_leaf(node->node)) {
			struct loc_network* network = loc_network_tree_node_get_network(node->node);

			// Append network to be written out later
			struct network* nw = make_network(network);
			TAILQ_INSERT_TAIL(&networks, nw, networks);

			db_node.network = htobe32(network_index++);
			loc_network_unref(network);
		} else {
			db_node.network = htobe32(0xffffffff);
		}

		// Write the current node
		DEBUG(writer->ctx, "Writing node %p (0 = %d, 1 = %d)\n",
			node, node->index_zero, node->index_one);

		*offset += fwrite(&db_node, 1, sizeof(db_node), f);
		network_tree_length += sizeof(db_node);

		free_node(node);
	}

	loc_network_tree_node_unref(root);

	header->network_tree_length = htobe32(network_tree_length);

	align_page_boundary(offset, f);

	DEBUG(writer->ctx, "Networks data section starts at %jd bytes\n", *offset);
	header->network_data_offset = htobe32(*offset);

	// We have now written the entire tree and have all networks
	// in a queue in order as they are indexed
	while (!TAILQ_EMPTY(&networks)) {
		struct network* nw = TAILQ_FIRST(&networks);
		TAILQ_REMOVE(&networks, nw, networks);

		// Prepare what we are writing to disk
		int r = loc_network_to_database_v0(nw->network, &db_network);
		if (r)
			return r;

		*offset += fwrite(&db_network, 1, sizeof(db_network), f);
		network_data_length += sizeof(db_network);

		free_network(nw);
	}

	header->network_data_length = htobe32(network_data_length);

	align_page_boundary(offset, f);

	return 0;
}

static int loc_database_write_countries(struct loc_writer* writer,
		struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	DEBUG(writer->ctx, "Countries section starts at %jd bytes\n", *offset);
	header->countries_offset = htobe32(*offset);

	size_t countries_length = 0;

	struct loc_database_country_v0 country;
	for (unsigned int i = 0; i < writer->countries_count; i++) {
		// Convert country into database format
		loc_country_to_database_v0(writer->countries[i], writer->pool, &country);

		// Write to disk
		*offset += fwrite(&country, 1, sizeof(country), f);
		countries_length += sizeof(country);
	}

	DEBUG(writer->ctx, "Countries section has a length of %zu bytes\n", countries_length);
	header->countries_length = htobe32(countries_length);

	align_page_boundary(offset, f);

	return 0;
}

LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f) {
	struct loc_database_magic magic;
	make_magic(writer, &magic);

	// Make the header
	struct loc_database_header_v0 header;
	header.vendor      = htobe32(writer->vendor);
	header.description = htobe32(writer->description);
	header.license     = htobe32(writer->license);

	time_t now = time(NULL);
	header.created_at = htobe64(now);

	int r;
	off_t offset = 0;

	// Start writing at the beginning of the file
	r = fseek(f, 0, SEEK_SET);
	if (r)
		return r;

	// Write the magic
	offset += fwrite(&magic, 1, sizeof(magic), f);

	// Skip the space we need to write the header later
	r = fseek(f, sizeof(header), SEEK_CUR);
	if (r) {
		DEBUG(writer->ctx, "Could not seek to position after header\n");
		return r;
	}
	offset += sizeof(header);

	align_page_boundary(&offset, f);

	// Write all ASes
	r = loc_database_write_as_section(writer, &header, &offset, f);
	if (r)
		return r;

	// Write all networks
	r = loc_database_write_networks(writer, &header, &offset, f);
	if (r)
		return r;

	// Write countries
	r = loc_database_write_countries(writer, &header, &offset, f);
	if (r)
		return r;

	// Write pool
	r = loc_database_write_pool(writer, &header, &offset, f);
	if (r)
		return r;

	// Write the header
	r = fseek(f, sizeof(magic), SEEK_SET);
	if (r)
		return r;

	offset += fwrite(&header, 1, sizeof(header), f);

	return 0;
}
Commit	Line	Data
c182393f MT	1	/*
	2	libloc - A library to determine the location of someone on the Internet
	3
	4	Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
	5
	6	This library is free software; you can redistribute it and/or
	7	modify it under the terms of the GNU Lesser General Public
	8	License as published by the Free Software Foundation; either
	9	version 2.1 of the License, or (at your option) any later version.
	10
	11	This library is distributed in the hope that it will be useful,
	12	but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15	*/
	16
c182393f MT	17	#include <endian.h>
	18	#include <errno.h>
	19	#include <stdio.h>
	20	#include <stdlib.h>
	21	#include <string.h>
438db08c	22	#include <sys/queue.h>
c182393f MT	23	#include <time.h>
c182393f MT	24
9fc7f001 MT	25	#include <loc/libloc.h>
9fc7f001 MT	26	#include <loc/as.h>
ec684c1a	27	#include <loc/country.h>
c182393f	28	#include <loc/format.h>
3b5f4af2	29	#include <loc/network.h>
9fc7f001	30	#include <loc/private.h>
c182393f MT	31	#include <loc/writer.h>
c182393f MT	32
c182393f MT	33	struct loc_writer {
	34	struct loc_ctx* ctx;
	35	int refcount;
	36
	37	struct loc_stringpool* pool;
	38	off_t vendor;
	39	off_t description;
4bf49d00	40	off_t license;
c182393f MT	41
	42	struct loc_as** as;
	43	size_t as_count;
3b5f4af2	44
ec684c1a MT	45	struct loc_country** countries;
	46	size_t countries_count;
	47
3b5f4af2	48	struct loc_network_tree* networks;
c182393f MT	49	};
	50
	51	LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer) {
	52	struct loc_writer* w = calloc(1, sizeof(*w));
	53	if (!w)
	54	return -ENOMEM;
	55
	56	w->ctx = loc_ref(ctx);
	57	w->refcount = 1;
	58
0e974d4b	59	int r = loc_stringpool_new(ctx, &w->pool);
c182393f MT	60	if (r) {
	61	loc_writer_unref(w);
	62	return r;
	63	}
	64
3b5f4af2 MT	65	// Initialize the network tree
	66	r = loc_network_tree_new(ctx, &w->networks);
	67	if (r) {
	68	loc_writer_unref(w);
	69	return r;
	70	}
	71
c182393f MT	72	*writer = w;
	73	return 0;
	74	}
	75
	76	LOC_EXPORT struct loc_writer* loc_writer_ref(struct loc_writer* writer) {
	77	writer->refcount++;
	78
	79	return writer;
	80	}
	81
	82	static void loc_writer_free(struct loc_writer* writer) {
	83	DEBUG(writer->ctx, "Releasing writer at %p\n", writer);
	84
	85	// Unref all AS
	86	for (unsigned int i = 0; i < writer->as_count; i++) {
	87	loc_as_unref(writer->as[i]);
	88	}
	89
3b5f4af2 MT	90	// Release network tree
	91	if (writer->networks)
	92	loc_network_tree_unref(writer->networks);
	93
c182393f MT	94	// Unref the string pool
	95	loc_stringpool_unref(writer->pool);
	96
	97	loc_unref(writer->ctx);
	98	free(writer);
	99	}
	100
	101	LOC_EXPORT struct loc_writer* loc_writer_unref(struct loc_writer* writer) {
	102	if (--writer->refcount > 0)
	103	return writer;
	104
	105	loc_writer_free(writer);
	106
	107	return NULL;
	108	}
	109
	110	LOC_EXPORT const char* loc_writer_get_vendor(struct loc_writer* writer) {
	111	return loc_stringpool_get(writer->pool, writer->vendor);
	112	}
	113
	114	LOC_EXPORT int loc_writer_set_vendor(struct loc_writer* writer, const char* vendor) {
	115	// Add the string to the string pool
	116	off_t offset = loc_stringpool_add(writer->pool, vendor);
	117	if (offset < 0)
	118	return offset;
	119
	120	writer->vendor = offset;
	121	return 0;
	122	}
	123
	124	LOC_EXPORT const char* loc_writer_get_description(struct loc_writer* writer) {
	125	return loc_stringpool_get(writer->pool, writer->description);
	126	}
	127
	128	LOC_EXPORT int loc_writer_set_description(struct loc_writer* writer, const char* description) {
	129	// Add the string to the string pool
	130	off_t offset = loc_stringpool_add(writer->pool, description);
	131	if (offset < 0)
	132	return offset;
	133
	134	writer->description = offset;
	135	return 0;
	136	}
	137
4bf49d00 MT	138	LOC_EXPORT const char* loc_writer_get_license(struct loc_writer* writer) {
	139	return loc_stringpool_get(writer->pool, writer->license);
	140	}
	141
	142	LOC_EXPORT int loc_writer_set_license(struct loc_writer* writer, const char* license) {
	143	// Add the string to the string pool
	144	off_t offset = loc_stringpool_add(writer->pool, license);
	145	if (offset < 0)
	146	return offset;
	147
	148	writer->license = offset;
	149	return 0;
	150	}
	151
c182393f MT	152	static int __loc_as_cmp(const void* as1, const void* as2) {
	153	return loc_as_cmp((struct loc_as)as1, (struct loc_as**)as2);
	154	}
	155
	156	LOC_EXPORT int loc_writer_add_as(struct loc_writer* writer, struct loc_as** as, uint32_t number) {
29bd8211	157	int r = loc_as_new(writer->ctx, as, number);
c182393f MT	158	if (r)
	159	return r;
	160
	161	// We have a new AS to add
	162	writer->as_count++;
	163
	164	// Make space
	165	writer->as = realloc(writer->as, sizeof(writer->as) writer->as_count);
	166	if (!writer->as)
	167	return -ENOMEM;
	168
	169	// Add as last element
	170	writer->as[writer->as_count - 1] = loc_as_ref(*as);
	171
	172	// Sort everything
	173	qsort(writer->as, writer->as_count, sizeof(*writer->as), __loc_as_cmp);
	174
	175	return 0;
	176	}
	177
3b5f4af2 MT	178	LOC_EXPORT int loc_writer_add_network(struct loc_writer* writer, struct loc_network** network, const char* string) {
	179	int r;
	180
	181	// Create a new network object
	182	r = loc_network_new_from_string(writer->ctx, network, string);
	183	if (r)
	184	return r;
	185
	186	// Add it to the local tree
	187	return loc_network_tree_add_network(writer->networks, *network);
	188	}
	189
ec684c1a MT	190	static int __loc_country_cmp(const void* country1, const void* country2) {
	191	return loc_country_cmp((struct loc_country)country1, (struct loc_country**)country2);
	192	}
	193
	194	LOC_EXPORT int loc_writer_add_country(struct loc_writer* writer, struct loc_country** country, const char* country_code) {
	195	int r = loc_country_new(writer->ctx, country, country_code);
	196	if (r)
	197	return r;
	198
	199	// We have a new country to add
	200	writer->countries_count++;
	201
	202	// Make space
	203	writer->countries = realloc(writer->countries, sizeof(writer->countries) writer->countries_count);
	204	if (!writer->countries)
	205	return -ENOMEM;
	206
	207	// Add as last element
	208	writer->countries[writer->countries_count - 1] = loc_country_ref(*country);
	209
	210	// Sort everything
	211	qsort(writer->countries, writer->countries_count, sizeof(*writer->countries), __loc_country_cmp);
	212
	213	return 0;
	214	}
	215
c182393f MT	216	static void make_magic(struct loc_writer* writer, struct loc_database_magic* magic) {
	217	// Copy magic bytes
	218	for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
	219	magic->magic[i] = LOC_DATABASE_MAGIC[i];
	220
	221	// Set version
0676cd80	222	magic->version = htobe16(LOC_DATABASE_VERSION);
c182393f MT	223	}
	224
	225	static void align_page_boundary(off_t* offset, FILE* f) {
	226	// Move to next page boundary
	227	while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
	228	*offset += fwrite("", 1, 1, f);
	229	}
	230
	231	static int loc_database_write_pool(struct loc_writer* writer,
	232	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	233	// Save the offset of the pool section
	234	DEBUG(writer->ctx, "Pool starts at %jd bytes\n", *offset);
0676cd80	235	header->pool_offset = htobe32(*offset);
c182393f MT	236
	237	// Write the pool
	238	size_t pool_length = loc_stringpool_write(writer->pool, f);
	239	*offset += pool_length;
	240
	241	DEBUG(writer->ctx, "Pool has a length of %zu bytes\n", pool_length);
0676cd80	242	header->pool_length = htobe32(pool_length);
c182393f MT	243
	244	return 0;
	245	}
	246
	247	static int loc_database_write_as_section(struct loc_writer* writer,
	248	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	249	DEBUG(writer->ctx, "AS section starts at %jd bytes\n", *offset);
0676cd80	250	header->as_offset = htobe32(*offset);
c182393f MT	251
	252	size_t as_length = 0;
	253
	254	struct loc_database_as_v0 as;
	255	for (unsigned int i = 0; i < writer->as_count; i++) {
	256	// Convert AS into database format
29bd8211	257	loc_as_to_database_v0(writer->as[i], writer->pool, &as);
c182393f MT	258
c182393f MT	259	// Write to disk
65862405	260	*offset += fwrite(&as, 1, sizeof(as), f);
c182393f MT	261	as_length += sizeof(as);
	262	}
	263
	264	DEBUG(writer->ctx, "AS section has a length of %zu bytes\n", as_length);
0676cd80	265	header->as_length = htobe32(as_length);
c182393f	266
fda89f24 MT	267	align_page_boundary(offset, f);
fda89f24 MT	268
c182393f MT	269	return 0;
	270	}
	271
438db08c MT	272	struct node {
438db08c MT	273	TAILQ_ENTRY(node) nodes;
f3e02bc5	274
438db08c	275	struct loc_network_tree_node* node;
f3e02bc5	276
438db08c MT	277	// Indices of the child nodes
	278	uint32_t index_zero;
	279	uint32_t index_one;
	280	};
f3e02bc5	281
438db08c MT	282	static struct node* make_node(struct loc_network_tree_node* node) {
	283	struct node* n = malloc(sizeof(*n));
	284	if (!n)
	285	return NULL;
f3e02bc5	286
438db08c MT	287	n->node = loc_network_tree_node_ref(node);
	288	n->index_zero = n->index_one = 0;
	289
	290	return n;
	291	}
	292
	293	static void free_node(struct node* node) {
	294	loc_network_tree_node_unref(node->node);
	295
	296	free(node);
	297	}
	298
	299	struct network {
	300	TAILQ_ENTRY(network) networks;
	301
	302	struct loc_network* network;
	303	};
	304
	305	static struct network* make_network(struct loc_network* network) {
	306	struct network* n = malloc(sizeof(*n));
	307	if (!n)
	308	return NULL;
	309
	310	n->network = loc_network_ref(network);
	311
	312	return n;
f3e02bc5 MT	313	}
f3e02bc5 MT	314
438db08c MT	315	static void free_network(struct network* network) {
	316	loc_network_unref(network->network);
	317
	318	free(network);
	319	}
	320
	321	static int loc_database_write_networks(struct loc_writer* writer,
f3e02bc5	322	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
438db08c MT	323	// Write the network tree
	324	DEBUG(writer->ctx, "Network tree starts at %jd bytes\n", *offset);
	325	header->network_tree_offset = htobe32(*offset);
f3e02bc5	326
438db08c MT	327	size_t network_tree_length = 0;
438db08c MT	328	size_t network_data_length = 0;
f3e02bc5	329
438db08c MT	330	struct node* node;
	331	struct node* child_node;
	332
	333	uint32_t index = 0;
	334	uint32_t network_index = 0;
	335
	336	struct loc_database_network_v0 db_network;
	337	struct loc_database_network_node_v0 db_node;
	338
	339	// Initialize queue for nodes
	340	TAILQ_HEAD(node_t, node) nodes;
	341	TAILQ_INIT(&nodes);
	342
	343	// Initialize queue for networks
	344	TAILQ_HEAD(network_t, network) networks;
	345	TAILQ_INIT(&networks);
	346
	347	// Add root
	348	struct loc_network_tree_node* root = loc_network_tree_get_root(writer->networks);
	349	node = make_node(root);
	350
	351	TAILQ_INSERT_TAIL(&nodes, node, nodes);
	352
	353	while (!TAILQ_EMPTY(&nodes)) {
	354	// Pop first node in list
	355	node = TAILQ_FIRST(&nodes);
	356	TAILQ_REMOVE(&nodes, node, nodes);
	357
	358	DEBUG(writer->ctx, "Processing node %p\n", node);
	359
	360	// Get child nodes
	361	struct loc_network_tree_node* node_zero = loc_network_tree_node_get(node->node, 0);
	362	if (node_zero) {
	363	node->index_zero = ++index;
	364
	365	child_node = make_node(node_zero);
	366	loc_network_tree_node_unref(node_zero);
	367
	368	TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
	369	}
	370
	371	struct loc_network_tree_node* node_one = loc_network_tree_node_get(node->node, 1);
	372	if (node_one) {
	373	node->index_one = ++index;
	374
	375	child_node = make_node(node_one);
	376	loc_network_tree_node_unref(node_one);
	377
	378	TAILQ_INSERT_TAIL(&nodes, child_node, nodes);
	379	}
	380
	381	// Prepare what we are writing to disk
39a55353 MT	382	db_node.zero = htobe32(node->index_zero);
	383	db_node.one = htobe32(node->index_one);
	384
438db08c MT	385	if (loc_network_tree_node_is_leaf(node->node)) {
	386	struct loc_network* network = loc_network_tree_node_get_network(node->node);
	387
	388	// Append network to be written out later
	389	struct network* nw = make_network(network);
	390	TAILQ_INSERT_TAIL(&networks, nw, networks);
	391
39a55353	392	db_node.network = htobe32(network_index++);
438db08c MT	393	loc_network_unref(network);
438db08c MT	394	} else {
39a55353	395	db_node.network = htobe32(0xffffffff);
438db08c MT	396	}
	397
	398	// Write the current node
	399	DEBUG(writer->ctx, "Writing node %p (0 = %d, 1 = %d)\n",
	400	node, node->index_zero, node->index_one);
	401
	402	*offset += fwrite(&db_node, 1, sizeof(db_node), f);
	403	network_tree_length += sizeof(db_node);
	404
	405	free_node(node);
	406	}
	407
	408	loc_network_tree_node_unref(root);
	409
	410	header->network_tree_length = htobe32(network_tree_length);
	411
	412	align_page_boundary(offset, f);
	413
	414	DEBUG(writer->ctx, "Networks data section starts at %jd bytes\n", *offset);
	415	header->network_data_offset = htobe32(*offset);
	416
	417	// We have now written the entire tree and have all networks
	418	// in a queue in order as they are indexed
	419	while (!TAILQ_EMPTY(&networks)) {
	420	struct network* nw = TAILQ_FIRST(&networks);
	421	TAILQ_REMOVE(&networks, nw, networks);
	422
	423	// Prepare what we are writing to disk
	424	int r = loc_network_to_database_v0(nw->network, &db_network);
	425	if (r)
	426	return r;
	427
	428	*offset += fwrite(&db_network, 1, sizeof(db_network), f);
	429	network_data_length += sizeof(db_network);
	430
	431	free_network(nw);
	432	}
f3e02bc5	433
438db08c	434	header->network_data_length = htobe32(network_data_length);
f3e02bc5	435
fda89f24 MT	436	align_page_boundary(offset, f);
fda89f24 MT	437
f3e02bc5 MT	438	return 0;
	439	}
	440
ec684c1a MT	441	static int loc_database_write_countries(struct loc_writer* writer,
	442	struct loc_database_header_v0* header, off_t* offset, FILE* f) {
	443	DEBUG(writer->ctx, "Countries section starts at %jd bytes\n", *offset);
	444	header->countries_offset = htobe32(*offset);
	445
	446	size_t countries_length = 0;
	447
	448	struct loc_database_country_v0 country;
	449	for (unsigned int i = 0; i < writer->countries_count; i++) {
	450	// Convert country into database format
	451	loc_country_to_database_v0(writer->countries[i], writer->pool, &country);
	452
	453	// Write to disk
	454	*offset += fwrite(&country, 1, sizeof(country), f);
	455	countries_length += sizeof(country);
	456	}
	457
	458	DEBUG(writer->ctx, "Countries section has a length of %zu bytes\n", countries_length);
	459	header->countries_length = htobe32(countries_length);
	460
	461	align_page_boundary(offset, f);
	462
	463	return 0;
	464	}
	465
c182393f MT	466	LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f) {
	467	struct loc_database_magic magic;
	468	make_magic(writer, &magic);
	469
	470	// Make the header
	471	struct loc_database_header_v0 header;
0676cd80 MT	472	header.vendor = htobe32(writer->vendor);
0676cd80 MT	473	header.description = htobe32(writer->description);
4bf49d00	474	header.license = htobe32(writer->license);
c182393f MT	475
	476	time_t now = time(NULL);
	477	header.created_at = htobe64(now);
	478
	479	int r;
	480	off_t offset = 0;
	481
	482	// Start writing at the beginning of the file
	483	r = fseek(f, 0, SEEK_SET);
	484	if (r)
	485	return r;
	486
	487	// Write the magic
	488	offset += fwrite(&magic, 1, sizeof(magic), f);
	489
	490	// Skip the space we need to write the header later
	491	r = fseek(f, sizeof(header), SEEK_CUR);
	492	if (r) {
	493	DEBUG(writer->ctx, "Could not seek to position after header\n");
	494	return r;
	495	}
	496	offset += sizeof(header);
	497
	498	align_page_boundary(&offset, f);
	499
	500	// Write all ASes
	501	r = loc_database_write_as_section(writer, &header, &offset, f);
	502	if (r)
	503	return r;
	504
f3e02bc5	505	// Write all networks
438db08c	506	r = loc_database_write_networks(writer, &header, &offset, f);
f3e02bc5 MT	507	if (r)
	508	return r;
	509
47bea941 MT	510	// Write countries
47bea941 MT	511	r = loc_database_write_countries(writer, &header, &offset, f);
c182393f MT	512	if (r)
	513	return r;
	514
47bea941 MT	515	// Write pool
47bea941 MT	516	r = loc_database_write_pool(writer, &header, &offset, f);
ec684c1a MT	517	if (r)
	518	return r;
	519
c182393f MT	520	// Write the header
	521	r = fseek(f, sizeof(magic), SEEK_SET);
	522	if (r)
	523	return r;
	524
	525	offset += fwrite(&header, 1, sizeof(header), f);
	526
	527	return 0;
	528	}