database: Move string pool to the end of the file again
[people/ms/libloc.git] / src / database.c
CommitLineData
2601e83e
MT
1/*
2 libloc - A library to determine the location of someone on the Internet
3
4 Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
5
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15*/
16
17#include <arpa/inet.h>
18#include <errno.h>
19#include <stddef.h>
20#include <stdint.h>
21#include <stdio.h>
22#include <stdlib.h>
23#include <string.h>
24#include <sys/types.h>
96ea74a5 25#include <time.h>
3f35869a 26#include <unistd.h>
2601e83e
MT
27
28#include <loc/libloc.h>
a5db3e49 29#include <loc/format.h>
2601e83e
MT
30
31#include "libloc-private.h"
a5db3e49 32#include "as.h"
2601e83e
MT
33#include "database.h"
34#include "stringpool.h"
35
36struct loc_database {
37 struct loc_ctx* ctx;
38 int refcount;
39
3f35869a 40 FILE* file;
2601e83e 41 unsigned int version;
96ea74a5 42 time_t created_at;
2601e83e
MT
43 off_t vendor;
44 off_t description;
45
a5db3e49
MT
46 // ASes in the database
47 struct loc_as** as;
48 size_t as_count;
49
2601e83e
MT
50 struct loc_stringpool* pool;
51};
52
2601e83e
MT
53LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, size_t pool_size) {
54 struct loc_database* db = calloc(1, sizeof(*db));
55 if (!db)
56 return -ENOMEM;
57
58 // Reference context
59 db->ctx = loc_ref(ctx);
60 db->refcount = 1;
61
96ea74a5
MT
62 // Save creation time
63 db->created_at = time(NULL);
64
2601e83e
MT
65 DEBUG(db->ctx, "Database allocated at %p\n", db);
66
67 // Create string pool
68 int r = loc_stringpool_new(db->ctx, &db->pool, pool_size);
69 if (r) {
70 loc_database_unref(db);
71 return r;
72 }
73
74 *database = db;
75
76 return 0;
77}
78
79LOC_EXPORT int loc_database_open(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
80 int r = loc_database_new(ctx, database, 0);
81 if (r)
82 return r;
83
84 return loc_database_read(*database, f);
85}
86
87LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
88 db->refcount++;
89
90 return db;
91}
92
93static void loc_database_free(struct loc_database* db) {
94 DEBUG(db->ctx, "Releasing database %p\n", db);
95
a5db3e49
MT
96 // Remove references to all ASes
97 if (db->as) {
98 for (unsigned int i = 0; i < db->as_count; i++) {
99 loc_as_unref(db->as[i]);
100 }
101 free(db->as);
102 }
103
2601e83e
MT
104 loc_stringpool_unref(db->pool);
105
3f35869a
MT
106 // Close file
107 if (db->file)
108 fclose(db->file);
109
2601e83e
MT
110 loc_unref(db->ctx);
111 free(db);
112}
113
114LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) {
115 if (--db->refcount > 0)
116 return NULL;
117
118 loc_database_free(db);
119 return NULL;
120}
121
96ea74a5
MT
122LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) {
123 return db->created_at;
124}
125
2601e83e
MT
126LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
127 return loc_stringpool_get(db->pool, db->vendor);
128}
129
130LOC_EXPORT int loc_database_set_vendor(struct loc_database* db, const char* vendor) {
131 // Add the string to the string pool
132 off_t offset = loc_stringpool_add(db->pool, vendor);
133 if (offset < 0)
134 return offset;
135
136 db->vendor = offset;
137 return 0;
138}
139
140LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) {
141 return loc_stringpool_get(db->pool, db->description);
142}
143
144LOC_EXPORT int loc_database_set_description(struct loc_database* db, const char* description) {
145 // Add the string to the string pool
146 off_t offset = loc_stringpool_add(db->pool, description);
147 if (offset < 0)
148 return offset;
149
150 db->description = offset;
151 return 0;
152}
153
a5db3e49
MT
154LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
155 return db->as_count;
156}
157
158static int loc_database_has_as(struct loc_database* db, struct loc_as* as) {
159 for (unsigned int i = 0; i < db->as_count; i++) {
160 if (loc_as_cmp(as, db->as[i]) == 0)
161 return i;
162 }
163
164 return -1;
165}
166
167static int __loc_as_cmp(const void* as1, const void* as2) {
168 return loc_as_cmp(*(struct loc_as**)as1, *(struct loc_as**)as2);
169}
170
171static void loc_database_sort_ases(struct loc_database* db) {
172 qsort(db->as, db->as_count, sizeof(*db->as), __loc_as_cmp);
173}
174
175static struct loc_as* __loc_database_add_as(struct loc_database* db, struct loc_as* as) {
176 // Check if AS exists already
177 int i = loc_database_has_as(db, as);
178 if (i >= 0) {
179 loc_as_unref(as);
180
181 // Select already existing AS
182 as = db->as[i];
183
184 return loc_as_ref(as);
185 }
186
187 db->as_count++;
188
189 // Make space for the new entry
190 db->as = realloc(db->as, sizeof(*db->as) * db->as_count);
191
192 // Add the new entry at the end
193 db->as[db->as_count - 1] = loc_as_ref(as);
194
195 // Sort everything
196 loc_database_sort_ases(db);
197
198 return as;
199}
200
201LOC_EXPORT struct loc_as* loc_database_add_as(struct loc_database* db, uint32_t number) {
202 struct loc_as* as;
203 int r = loc_as_new(db->ctx, db->pool, &as, number);
204 if (r)
205 return NULL;
206
207 return __loc_database_add_as(db, as);
208}
209
3f35869a 210static int loc_database_read_magic(struct loc_database* db) {
2601e83e
MT
211 struct loc_database_magic magic;
212
213 // Read from file
3f35869a 214 size_t bytes_read = fread(&magic, 1, sizeof(magic), db->file);
2601e83e
MT
215
216 // Check if we have been able to read enough data
217 if (bytes_read < sizeof(magic)) {
218 ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
219 DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
220 return -ENOMSG;
221 }
222
223 // Compare magic bytes
224 if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
225 DEBUG(db->ctx, "Magic value matches\n");
226
227 // Parse version
228 db->version = ntohs(magic.version);
229 DEBUG(db->ctx, "Database version is %u\n", db->version);
230
231 return 0;
232 }
233
234 ERROR(db->ctx, "Database format is not compatible\n");
235
236 // Return an error
237 return 1;
238}
239
a5db3e49 240static int loc_database_read_as_section_v0(struct loc_database* db,
3f35869a 241 off_t as_offset, size_t as_length) {
a5db3e49
MT
242 struct loc_database_as_v0 dbobj;
243
244 // Read from the start of the section
3f35869a 245 int r = fseek(db->file, as_offset, SEEK_SET);
a5db3e49
MT
246 if (r)
247 return r;
248
249 // Read all ASes
250 size_t as_count = as_length / sizeof(dbobj);
251 for (unsigned int i = 0; i < as_count; i++) {
3f35869a 252 size_t bytes_read = fread(&dbobj, 1, sizeof(dbobj), db->file);
a5db3e49
MT
253 if (bytes_read < sizeof(dbobj)) {
254 ERROR(db->ctx, "Could not read an AS object\n");
255 return -ENOMSG;
256 }
257
258 // Allocate a new AS
259 struct loc_as* as;
260 r = loc_as_new_from_database_v0(db->ctx, db->pool, &as, &dbobj);
261 if (r)
262 return r;
263
264 // Attach it to the database
265 as = __loc_database_add_as(db, as);
266 loc_as_unref(as);
267 }
268
269 INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
270
271 return 0;
272}
273
3f35869a 274static int loc_database_read_header_v0(struct loc_database* db) {
2601e83e
MT
275 struct loc_database_header_v0 header;
276
277 // Read from file
3f35869a 278 size_t size = fread(&header, 1, sizeof(header), db->file);
2601e83e
MT
279
280 if (size < sizeof(header)) {
281 ERROR(db->ctx, "Could not read enough data for header\n");
282 return -ENOMSG;
283 }
284
285 // Copy over data
96ea74a5 286 db->created_at = be64toh(header.created_at);
2601e83e
MT
287 db->vendor = ntohl(header.vendor);
288 db->description = ntohl(header.description);
289
290 // Open pool
291 off_t pool_offset = ntohl(header.pool_offset);
292 size_t pool_length = ntohl(header.pool_length);
293
3f35869a 294 int r = loc_stringpool_read(db->pool, db->file, pool_offset, pool_length);
2601e83e
MT
295 if (r)
296 return r;
297
a5db3e49
MT
298 // AS section
299 off_t as_offset = ntohl(header.as_offset);
300 size_t as_length = ntohl(header.as_length);
301
3f35869a 302 r = loc_database_read_as_section_v0(db, as_offset, as_length);
a5db3e49
MT
303 if (r)
304 return r;
305
2601e83e
MT
306 return 0;
307}
308
3f35869a 309static int loc_database_read_header(struct loc_database* db) {
2601e83e
MT
310 switch (db->version) {
311 case 0:
3f35869a 312 return loc_database_read_header_v0(db);
2601e83e
MT
313
314 default:
315 ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
316 return 1;
317 }
318}
319
320LOC_EXPORT int loc_database_read(struct loc_database* db, FILE* f) {
3f35869a
MT
321 // Copy the file pointer and work on that so we don't care if
322 // the calling function closes the file
323 int fd = fileno(f);
324
325 // Make a copy
326 fd = dup(fd);
327
328 // Retrieve a file pointer
329 db->file = fdopen(fd, "r");
330 if (!db->file)
331 return -errno;
332
333 int r = fseek(db->file, 0, SEEK_SET);
2601e83e
MT
334 if (r)
335 return r;
336
337 // Read magic bytes
3f35869a 338 r = loc_database_read_magic(db);
2601e83e
MT
339 if (r)
340 return r;
341
342 // Read the header
3f35869a 343 r = loc_database_read_header(db);
2601e83e
MT
344 if (r)
345 return r;
346
347 return 0;
348}
349
350static void loc_database_make_magic(struct loc_database* db, struct loc_database_magic* magic) {
351 // Copy magic bytes
352 for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
353 magic->magic[i] = LOC_DATABASE_MAGIC[i];
354
355 // Set version
356 magic->version = htons(LOC_DATABASE_VERSION);
357}
358
8f5b676a
MT
359static void loc_database_align_page_boundary(off_t* offset, FILE* f) {
360 // Move to next page boundary
361 while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
362 *offset += fwrite("", 1, 1, f);
363}
364
c34e76f1
MT
365static int loc_database_write_pool(struct loc_database* db, struct loc_database_header_v0* header, off_t* offset, FILE* f) {
366 // Save the offset of the pool section
367 DEBUG(db->ctx, "Pool starts at %jd bytes\n", *offset);
368 header->pool_offset = htonl(*offset);
369
370 // Write the pool
371 size_t pool_length = loc_stringpool_write(db->pool, f);
372 *offset += pool_length;
373
374 DEBUG(db->ctx, "Pool has a length of %zu bytes\n", pool_length);
375 header->pool_length = htonl(pool_length);
376
377 return 0;
378}
379
78ace4ed
MT
380static int loc_database_write_as_section(struct loc_database* db,
381 struct loc_database_header_v0* header, off_t* offset, FILE* f) {
382 DEBUG(db->ctx, "AS section starts at %jd bytes\n", *offset);
383 header->as_offset = htonl(*offset);
384
385 size_t as_length = 0;
386
387 struct loc_database_as_v0 dbas;
388 for (unsigned int i = 0; i < db->as_count; i++) {
389 // Convert AS into database format
390 loc_as_to_database_v0(db->as[i], &dbas);
391
392 // Write to disk
393 offset += fwrite(&dbas, 1, sizeof(dbas), f);
394 as_length += sizeof(dbas);
395 }
396
397 DEBUG(db->ctx, "AS section has a length of %zu bytes\n", as_length);
398 header->as_length = htonl(as_length);
399
400 return 0;
401}
402
2601e83e
MT
403LOC_EXPORT int loc_database_write(struct loc_database* db, FILE* f) {
404 struct loc_database_magic magic;
405 loc_database_make_magic(db, &magic);
406
407 // Make the header
408 struct loc_database_header_v0 header;
96ea74a5 409 header.created_at = htobe64(db->created_at);
2601e83e
MT
410 header.vendor = htonl(db->vendor);
411 header.description = htonl(db->description);
412
413 int r;
414 off_t offset = 0;
415
416 // Start writing at the beginning of the file
417 r = fseek(f, 0, SEEK_SET);
418 if (r)
419 return r;
420
421 // Write the magic
422 offset += fwrite(&magic, 1, sizeof(magic), f);
423
424 // Skip the space we need to write the header later
425 r = fseek(f, sizeof(header), SEEK_CUR);
426 if (r) {
427 DEBUG(db->ctx, "Could not seek to position after header\n");
428 return r;
429 }
430 offset += sizeof(header);
431
8f5b676a 432 loc_database_align_page_boundary(&offset, f);
c34e76f1 433
5df10c90
MT
434 // Write all ASes
435 r = loc_database_write_as_section(db, &header, &offset, f);
c34e76f1
MT
436 if (r)
437 return r;
438
8f5b676a
MT
439 loc_database_align_page_boundary(&offset, f);
440
5df10c90
MT
441 // Write pool
442 r = loc_database_write_pool(db, &header, &offset, f);
78ace4ed
MT
443 if (r)
444 return r;
a5db3e49 445
2601e83e
MT
446 // Write the header
447 r = fseek(f, sizeof(magic), SEEK_SET);
448 if (r)
449 return r;
450
451 offset += fwrite(&header, 1, sizeof(header), f);
452
453 return 0;
454}