database: Move string pool to the end of the file again
[people/ms/libloc.git] / src / database.c
1 /*
2 libloc - A library to determine the location of someone on the Internet
3
4 Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
5
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15 */
16
17 #include <arpa/inet.h>
18 #include <errno.h>
19 #include <stddef.h>
20 #include <stdint.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <sys/types.h>
25 #include <time.h>
26 #include <unistd.h>
27
28 #include <loc/libloc.h>
29 #include <loc/format.h>
30
31 #include "libloc-private.h"
32 #include "as.h"
33 #include "database.h"
34 #include "stringpool.h"
35
36 struct loc_database {
37 struct loc_ctx* ctx;
38 int refcount;
39
40 FILE* file;
41 unsigned int version;
42 time_t created_at;
43 off_t vendor;
44 off_t description;
45
46 // ASes in the database
47 struct loc_as** as;
48 size_t as_count;
49
50 struct loc_stringpool* pool;
51 };
52
53 LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, size_t pool_size) {
54 struct loc_database* db = calloc(1, sizeof(*db));
55 if (!db)
56 return -ENOMEM;
57
58 // Reference context
59 db->ctx = loc_ref(ctx);
60 db->refcount = 1;
61
62 // Save creation time
63 db->created_at = time(NULL);
64
65 DEBUG(db->ctx, "Database allocated at %p\n", db);
66
67 // Create string pool
68 int r = loc_stringpool_new(db->ctx, &db->pool, pool_size);
69 if (r) {
70 loc_database_unref(db);
71 return r;
72 }
73
74 *database = db;
75
76 return 0;
77 }
78
79 LOC_EXPORT int loc_database_open(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
80 int r = loc_database_new(ctx, database, 0);
81 if (r)
82 return r;
83
84 return loc_database_read(*database, f);
85 }
86
87 LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
88 db->refcount++;
89
90 return db;
91 }
92
93 static void loc_database_free(struct loc_database* db) {
94 DEBUG(db->ctx, "Releasing database %p\n", db);
95
96 // Remove references to all ASes
97 if (db->as) {
98 for (unsigned int i = 0; i < db->as_count; i++) {
99 loc_as_unref(db->as[i]);
100 }
101 free(db->as);
102 }
103
104 loc_stringpool_unref(db->pool);
105
106 // Close file
107 if (db->file)
108 fclose(db->file);
109
110 loc_unref(db->ctx);
111 free(db);
112 }
113
114 LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) {
115 if (--db->refcount > 0)
116 return NULL;
117
118 loc_database_free(db);
119 return NULL;
120 }
121
122 LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) {
123 return db->created_at;
124 }
125
126 LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
127 return loc_stringpool_get(db->pool, db->vendor);
128 }
129
130 LOC_EXPORT int loc_database_set_vendor(struct loc_database* db, const char* vendor) {
131 // Add the string to the string pool
132 off_t offset = loc_stringpool_add(db->pool, vendor);
133 if (offset < 0)
134 return offset;
135
136 db->vendor = offset;
137 return 0;
138 }
139
140 LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) {
141 return loc_stringpool_get(db->pool, db->description);
142 }
143
144 LOC_EXPORT int loc_database_set_description(struct loc_database* db, const char* description) {
145 // Add the string to the string pool
146 off_t offset = loc_stringpool_add(db->pool, description);
147 if (offset < 0)
148 return offset;
149
150 db->description = offset;
151 return 0;
152 }
153
154 LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
155 return db->as_count;
156 }
157
158 static int loc_database_has_as(struct loc_database* db, struct loc_as* as) {
159 for (unsigned int i = 0; i < db->as_count; i++) {
160 if (loc_as_cmp(as, db->as[i]) == 0)
161 return i;
162 }
163
164 return -1;
165 }
166
167 static int __loc_as_cmp(const void* as1, const void* as2) {
168 return loc_as_cmp(*(struct loc_as**)as1, *(struct loc_as**)as2);
169 }
170
171 static void loc_database_sort_ases(struct loc_database* db) {
172 qsort(db->as, db->as_count, sizeof(*db->as), __loc_as_cmp);
173 }
174
175 static struct loc_as* __loc_database_add_as(struct loc_database* db, struct loc_as* as) {
176 // Check if AS exists already
177 int i = loc_database_has_as(db, as);
178 if (i >= 0) {
179 loc_as_unref(as);
180
181 // Select already existing AS
182 as = db->as[i];
183
184 return loc_as_ref(as);
185 }
186
187 db->as_count++;
188
189 // Make space for the new entry
190 db->as = realloc(db->as, sizeof(*db->as) * db->as_count);
191
192 // Add the new entry at the end
193 db->as[db->as_count - 1] = loc_as_ref(as);
194
195 // Sort everything
196 loc_database_sort_ases(db);
197
198 return as;
199 }
200
201 LOC_EXPORT struct loc_as* loc_database_add_as(struct loc_database* db, uint32_t number) {
202 struct loc_as* as;
203 int r = loc_as_new(db->ctx, db->pool, &as, number);
204 if (r)
205 return NULL;
206
207 return __loc_database_add_as(db, as);
208 }
209
210 static int loc_database_read_magic(struct loc_database* db) {
211 struct loc_database_magic magic;
212
213 // Read from file
214 size_t bytes_read = fread(&magic, 1, sizeof(magic), db->file);
215
216 // Check if we have been able to read enough data
217 if (bytes_read < sizeof(magic)) {
218 ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
219 DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
220 return -ENOMSG;
221 }
222
223 // Compare magic bytes
224 if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
225 DEBUG(db->ctx, "Magic value matches\n");
226
227 // Parse version
228 db->version = ntohs(magic.version);
229 DEBUG(db->ctx, "Database version is %u\n", db->version);
230
231 return 0;
232 }
233
234 ERROR(db->ctx, "Database format is not compatible\n");
235
236 // Return an error
237 return 1;
238 }
239
240 static int loc_database_read_as_section_v0(struct loc_database* db,
241 off_t as_offset, size_t as_length) {
242 struct loc_database_as_v0 dbobj;
243
244 // Read from the start of the section
245 int r = fseek(db->file, as_offset, SEEK_SET);
246 if (r)
247 return r;
248
249 // Read all ASes
250 size_t as_count = as_length / sizeof(dbobj);
251 for (unsigned int i = 0; i < as_count; i++) {
252 size_t bytes_read = fread(&dbobj, 1, sizeof(dbobj), db->file);
253 if (bytes_read < sizeof(dbobj)) {
254 ERROR(db->ctx, "Could not read an AS object\n");
255 return -ENOMSG;
256 }
257
258 // Allocate a new AS
259 struct loc_as* as;
260 r = loc_as_new_from_database_v0(db->ctx, db->pool, &as, &dbobj);
261 if (r)
262 return r;
263
264 // Attach it to the database
265 as = __loc_database_add_as(db, as);
266 loc_as_unref(as);
267 }
268
269 INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
270
271 return 0;
272 }
273
274 static int loc_database_read_header_v0(struct loc_database* db) {
275 struct loc_database_header_v0 header;
276
277 // Read from file
278 size_t size = fread(&header, 1, sizeof(header), db->file);
279
280 if (size < sizeof(header)) {
281 ERROR(db->ctx, "Could not read enough data for header\n");
282 return -ENOMSG;
283 }
284
285 // Copy over data
286 db->created_at = be64toh(header.created_at);
287 db->vendor = ntohl(header.vendor);
288 db->description = ntohl(header.description);
289
290 // Open pool
291 off_t pool_offset = ntohl(header.pool_offset);
292 size_t pool_length = ntohl(header.pool_length);
293
294 int r = loc_stringpool_read(db->pool, db->file, pool_offset, pool_length);
295 if (r)
296 return r;
297
298 // AS section
299 off_t as_offset = ntohl(header.as_offset);
300 size_t as_length = ntohl(header.as_length);
301
302 r = loc_database_read_as_section_v0(db, as_offset, as_length);
303 if (r)
304 return r;
305
306 return 0;
307 }
308
309 static int loc_database_read_header(struct loc_database* db) {
310 switch (db->version) {
311 case 0:
312 return loc_database_read_header_v0(db);
313
314 default:
315 ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
316 return 1;
317 }
318 }
319
320 LOC_EXPORT int loc_database_read(struct loc_database* db, FILE* f) {
321 // Copy the file pointer and work on that so we don't care if
322 // the calling function closes the file
323 int fd = fileno(f);
324
325 // Make a copy
326 fd = dup(fd);
327
328 // Retrieve a file pointer
329 db->file = fdopen(fd, "r");
330 if (!db->file)
331 return -errno;
332
333 int r = fseek(db->file, 0, SEEK_SET);
334 if (r)
335 return r;
336
337 // Read magic bytes
338 r = loc_database_read_magic(db);
339 if (r)
340 return r;
341
342 // Read the header
343 r = loc_database_read_header(db);
344 if (r)
345 return r;
346
347 return 0;
348 }
349
350 static void loc_database_make_magic(struct loc_database* db, struct loc_database_magic* magic) {
351 // Copy magic bytes
352 for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
353 magic->magic[i] = LOC_DATABASE_MAGIC[i];
354
355 // Set version
356 magic->version = htons(LOC_DATABASE_VERSION);
357 }
358
359 static void loc_database_align_page_boundary(off_t* offset, FILE* f) {
360 // Move to next page boundary
361 while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
362 *offset += fwrite("", 1, 1, f);
363 }
364
365 static int loc_database_write_pool(struct loc_database* db, struct loc_database_header_v0* header, off_t* offset, FILE* f) {
366 // Save the offset of the pool section
367 DEBUG(db->ctx, "Pool starts at %jd bytes\n", *offset);
368 header->pool_offset = htonl(*offset);
369
370 // Write the pool
371 size_t pool_length = loc_stringpool_write(db->pool, f);
372 *offset += pool_length;
373
374 DEBUG(db->ctx, "Pool has a length of %zu bytes\n", pool_length);
375 header->pool_length = htonl(pool_length);
376
377 return 0;
378 }
379
380 static int loc_database_write_as_section(struct loc_database* db,
381 struct loc_database_header_v0* header, off_t* offset, FILE* f) {
382 DEBUG(db->ctx, "AS section starts at %jd bytes\n", *offset);
383 header->as_offset = htonl(*offset);
384
385 size_t as_length = 0;
386
387 struct loc_database_as_v0 dbas;
388 for (unsigned int i = 0; i < db->as_count; i++) {
389 // Convert AS into database format
390 loc_as_to_database_v0(db->as[i], &dbas);
391
392 // Write to disk
393 offset += fwrite(&dbas, 1, sizeof(dbas), f);
394 as_length += sizeof(dbas);
395 }
396
397 DEBUG(db->ctx, "AS section has a length of %zu bytes\n", as_length);
398 header->as_length = htonl(as_length);
399
400 return 0;
401 }
402
403 LOC_EXPORT int loc_database_write(struct loc_database* db, FILE* f) {
404 struct loc_database_magic magic;
405 loc_database_make_magic(db, &magic);
406
407 // Make the header
408 struct loc_database_header_v0 header;
409 header.created_at = htobe64(db->created_at);
410 header.vendor = htonl(db->vendor);
411 header.description = htonl(db->description);
412
413 int r;
414 off_t offset = 0;
415
416 // Start writing at the beginning of the file
417 r = fseek(f, 0, SEEK_SET);
418 if (r)
419 return r;
420
421 // Write the magic
422 offset += fwrite(&magic, 1, sizeof(magic), f);
423
424 // Skip the space we need to write the header later
425 r = fseek(f, sizeof(header), SEEK_CUR);
426 if (r) {
427 DEBUG(db->ctx, "Could not seek to position after header\n");
428 return r;
429 }
430 offset += sizeof(header);
431
432 loc_database_align_page_boundary(&offset, f);
433
434 // Write all ASes
435 r = loc_database_write_as_section(db, &header, &offset, f);
436 if (r)
437 return r;
438
439 loc_database_align_page_boundary(&offset, f);
440
441 // Write pool
442 r = loc_database_write_pool(db, &header, &offset, f);
443 if (r)
444 return r;
445
446 // Write the header
447 r = fseek(f, sizeof(magic), SEEK_SET);
448 if (r)
449 return r;
450
451 offset += fwrite(&header, 1, sizeof(header), f);
452
453 return 0;
454 }