]>
Commit | Line | Data |
---|---|---|
db9ecf05 | 1 | /* SPDX-License-Identifier: LGPL-2.1-or-later */ |
c2f1db8f | 2 | #pragma once |
87d2c1ff | 3 | |
4374d7ea | 4 | #include <fcntl.h> |
87d2c1ff | 5 | #include <inttypes.h> |
f2dc22b4 | 6 | #include <sys/uio.h> |
87d2c1ff | 7 | |
349cc4a5 | 8 | #if HAVE_GCRYPT |
a03d4359 | 9 | # include <gcrypt.h> |
7560fffc LP |
10 | #endif |
11 | ||
a03d4359 | 12 | #include "sd-event.h" |
fa6ac760 | 13 | #include "sd-id128.h" |
81527be1 | 14 | |
acc50c92 | 15 | #include "compress.h" |
71d35b6b | 16 | #include "hashmap.h" |
87d2c1ff | 17 | #include "journal-def.h" |
16e9f408 | 18 | #include "mmap-cache.h" |
71d35b6b | 19 | #include "sparse-endian.h" |
ca78ad1d | 20 | #include "time-util.h" |
de190aef | 21 | |
bc85bfee | 22 | typedef struct JournalMetrics { |
e540c5a6 | 23 | /* For all these: UINT64_MAX means "pick automatically", and 0 means "no limit enforced" */ |
8580d1f7 LP |
24 | uint64_t max_size; /* how large journal files grow at max */ |
25 | uint64_t min_size; /* how large journal files grow at least */ | |
26 | uint64_t max_use; /* how much disk space to use in total at max, keep_free permitting */ | |
27 | uint64_t min_use; /* how much disk space to use in total at least, even if keep_free says not to */ | |
28 | uint64_t keep_free; /* how much to keep free on disk */ | |
29 | uint64_t n_max_files; /* how many files to keep around at max */ | |
bc85bfee LP |
30 | } JournalMetrics; |
31 | ||
87011c25 ZJS |
32 | typedef enum direction { |
33 | DIRECTION_UP, | |
34 | DIRECTION_DOWN | |
35 | } direction_t; | |
36 | ||
99cc7653 MS |
37 | typedef enum LocationType { |
38 | /* The first and last entries, resp. */ | |
39 | LOCATION_HEAD, | |
40 | LOCATION_TAIL, | |
41 | ||
42 | /* We already read the entry we currently point to, and the | |
43 | * next one to read should probably not be this one again. */ | |
44 | LOCATION_DISCRETE, | |
45 | ||
46 | /* We should seek to the precise location specified, and | |
47 | * return it, as we haven't read it yet. */ | |
48 | LOCATION_SEEK | |
49 | } LocationType; | |
50 | ||
ac2e41f5 VC |
51 | typedef enum OfflineState { |
52 | OFFLINE_JOINED, | |
53 | OFFLINE_SYNCING, | |
54 | OFFLINE_OFFLINING, | |
55 | OFFLINE_CANCEL, | |
56 | OFFLINE_AGAIN_FROM_SYNCING, | |
57 | OFFLINE_AGAIN_FROM_OFFLINING, | |
58 | OFFLINE_DONE | |
59 | } OfflineState; | |
60 | ||
f4b47811 LP |
61 | typedef struct JournalFile { |
62 | int fd; | |
be7cdd8e | 63 | MMapFileDescriptor *cache_fd; |
87011c25 | 64 | |
0ac38b70 | 65 | mode_t mode; |
7560fffc | 66 | |
49615dbd | 67 | int open_flags; |
5d1ce257 | 68 | bool close_fd:1; |
8eb85171 | 69 | bool archive:1; |
7560fffc | 70 | |
87011c25 | 71 | direction_t last_direction; |
6573ef05 | 72 | LocationType location_type; |
6e693b42 | 73 | uint64_t last_n_entries; |
87011c25 ZJS |
74 | |
75 | char *path; | |
76 | struct stat last_stat; | |
2678031a | 77 | usec_t last_stat_usec; |
87011c25 | 78 | |
f4b47811 | 79 | Header *header; |
de190aef LP |
80 | HashItem *data_hash_table; |
81 | HashItem *field_hash_table; | |
f4b47811 | 82 | |
f4b47811 | 83 | uint64_t current_offset; |
6573ef05 MS |
84 | uint64_t current_seqnum; |
85 | uint64_t current_realtime; | |
86 | uint64_t current_monotonic; | |
87 | sd_id128_t current_boot_id; | |
88 | uint64_t current_xor_hash; | |
bc85bfee LP |
89 | |
90 | JournalMetrics metrics; | |
807e17f0 | 91 | |
7a24f3bf VC |
92 | sd_event_source *post_change_timer; |
93 | usec_t post_change_timer_period; | |
94 | ||
4743015d | 95 | OrderedHashmap *chain_cache; |
a4bcff5b | 96 | |
ac2e41f5 VC |
97 | pthread_t offline_thread; |
98 | volatile OfflineState offline_state; | |
99 | ||
858749f7 LP |
100 | unsigned last_seen_generation; |
101 | ||
57850536 | 102 | uint64_t compress_threshold_bytes; |
d80b051c | 103 | #if HAVE_COMPRESSION |
807e17f0 | 104 | void *compress_buffer; |
807e17f0 | 105 | #endif |
7560fffc | 106 | |
349cc4a5 | 107 | #if HAVE_GCRYPT |
7560fffc LP |
108 | gcry_md_hd_t hmac; |
109 | bool hmac_running; | |
110 | ||
baed47c3 LP |
111 | FSSHeader *fss_file; |
112 | size_t fss_file_size; | |
113 | ||
114 | uint64_t fss_start_usec; | |
115 | uint64_t fss_interval_usec; | |
b7c9ae91 LP |
116 | |
117 | void *fsprg_state; | |
118 | size_t fsprg_state_size; | |
119 | ||
120 | void *fsprg_seed; | |
121 | size_t fsprg_seed_size; | |
7560fffc | 122 | #endif |
f4b47811 LP |
123 | } JournalFile; |
124 | ||
49615dbd LP |
125 | typedef enum JournalFileFlags { |
126 | JOURNAL_COMPRESS = 1 << 0, | |
127 | JOURNAL_SEAL = 1 << 1, | |
128 | } JournalFileFlags; | |
129 | ||
a9089a66 DDM |
130 | typedef struct { |
131 | uint64_t object_offset; | |
132 | uint64_t hash; | |
133 | } EntryItem; | |
134 | ||
4a92baf3 | 135 | int journal_file_open( |
5d1ce257 | 136 | int fd, |
4a92baf3 | 137 | const char *fname, |
49615dbd LP |
138 | int open_flags, |
139 | JournalFileFlags file_flags, | |
4a92baf3 | 140 | mode_t mode, |
57850536 | 141 | uint64_t compress_threshold_bytes, |
4a92baf3 | 142 | JournalMetrics *metrics, |
27370278 | 143 | MMapCache *mmap_cache, |
4a92baf3 LP |
144 | JournalFile *template, |
145 | JournalFile **ret); | |
146 | ||
764721cc | 147 | int journal_file_set_offline_thread_join(JournalFile *f); |
804ae586 | 148 | JournalFile* journal_file_close(JournalFile *j); |
28ca867a | 149 | int journal_file_fstat(JournalFile *f); |
627df1dc | 150 | DEFINE_TRIVIAL_CLEANUP_FUNC(JournalFile*, journal_file_close); |
87d2c1ff | 151 | |
0284adc6 | 152 | #define ALIGN64(x) (((x) + 7ULL) & ~7ULL) |
db11ac1a | 153 | #define VALID64(x) (((x) & 7ULL) == 0ULL) |
0284adc6 | 154 | |
ae97089d ZJS |
155 | /* Use six characters to cover the offsets common in smallish journal |
156 | * files without adding too many zeros. */ | |
157 | #define OFSfmt "%06"PRIx64 | |
158 | ||
fc89a139 LP |
159 | static inline bool VALID_REALTIME(uint64_t u) { |
160 | /* This considers timestamps until the year 3112 valid. That should be plenty room... */ | |
161 | return u > 0 && u < (1ULL << 55); | |
162 | } | |
163 | ||
164 | static inline bool VALID_MONOTONIC(uint64_t u) { | |
165 | /* This considers timestamps until 1142 years of runtime valid. */ | |
166 | return u < (1ULL << 55); | |
167 | } | |
168 | ||
169 | static inline bool VALID_EPOCH(uint64_t u) { | |
170 | /* This allows changing the key for 1142 years, every usec. */ | |
171 | return u < (1ULL << 55); | |
172 | } | |
173 | ||
0284adc6 LP |
174 | #define JOURNAL_HEADER_CONTAINS(h, field) \ |
175 | (le64toh((h)->header_size) >= offsetof(Header, field) + sizeof((h)->field)) | |
176 | ||
8088cbd3 | 177 | #define JOURNAL_HEADER_SEALED(h) \ |
a7656091 | 178 | FLAGS_SET(le32toh((h)->compatible_flags), HEADER_COMPATIBLE_SEALED) |
8088cbd3 | 179 | |
d89c8fdf | 180 | #define JOURNAL_HEADER_COMPRESSED_XZ(h) \ |
a7656091 | 181 | FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_XZ) |
d89c8fdf ZJS |
182 | |
183 | #define JOURNAL_HEADER_COMPRESSED_LZ4(h) \ | |
a7656091 | 184 | FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_LZ4) |
8088cbd3 | 185 | |
8653185a LP |
186 | #define JOURNAL_HEADER_COMPRESSED_ZSTD(h) \ |
187 | FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_ZSTD) | |
188 | ||
4ce534f4 LP |
189 | #define JOURNAL_HEADER_KEYED_HASH(h) \ |
190 | FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_KEYED_HASH) | |
191 | ||
87413812 DDM |
192 | #define JOURNAL_HEADER_COMPACT(h) \ |
193 | FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPACT) | |
194 | ||
78519831 | 195 | int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret); |
e5d84733 | 196 | int journal_file_read_object_header(JournalFile *f, ObjectType type, uint64_t offset, Object *ret); |
87d2c1ff | 197 | |
e6d4a110 LP |
198 | int journal_file_tail_end_by_pread(JournalFile *f, uint64_t *ret_offset); |
199 | int journal_file_tail_end_by_mmap(JournalFile *f, uint64_t *ret_offset); | |
764721cc | 200 | |
a9089a66 DDM |
201 | static inline uint64_t journal_file_entry_item_object_offset(JournalFile *f, Object *o, size_t i) { |
202 | assert(f); | |
203 | assert(o); | |
204 | return JOURNAL_HEADER_COMPACT(f->header) ? le32toh(o->entry.items.compact[i].object_offset) : | |
205 | le64toh(o->entry.items.regular[i].object_offset); | |
206 | } | |
207 | ||
208 | static inline size_t journal_file_entry_item_size(JournalFile *f) { | |
209 | assert(f); | |
210 | return JOURNAL_HEADER_COMPACT(f->header) ? sizeof_field(Object, entry.items.compact[0]) : | |
211 | sizeof_field(Object, entry.items.regular[0]); | |
212 | } | |
213 | ||
214 | uint64_t journal_file_entry_n_items(JournalFile *f, Object *o) _pure_; | |
0e35afff DDM |
215 | |
216 | int journal_file_data_payload( | |
217 | JournalFile *f, | |
218 | Object *o, | |
219 | uint64_t offset, | |
220 | const char *field, | |
221 | size_t field_length, | |
222 | size_t data_threshold, | |
223 | void **ret_data, | |
224 | size_t *ret_size); | |
225 | ||
e81710d3 DDM |
226 | static inline size_t journal_file_data_payload_offset(JournalFile *f) { |
227 | return JOURNAL_HEADER_COMPACT(f->header) | |
228 | ? offsetof(Object, data.compact.payload) | |
229 | : offsetof(Object, data.regular.payload); | |
230 | } | |
231 | ||
232 | static inline uint8_t* journal_file_data_payload_field(JournalFile *f, Object *o) { | |
233 | return JOURNAL_HEADER_COMPACT(f->header) ? o->data.compact.payload : o->data.regular.payload; | |
234 | } | |
235 | ||
99daf3ce DDM |
236 | uint64_t journal_file_entry_array_n_items(JournalFile *f, Object *o) _pure_; |
237 | ||
238 | static inline uint64_t journal_file_entry_array_item(JournalFile *f, Object *o, size_t i) { | |
239 | assert(f); | |
240 | assert(o); | |
241 | return JOURNAL_HEADER_COMPACT(f->header) ? le32toh(o->entry_array.items.compact[i]) : | |
242 | le64toh(o->entry_array.items.regular[i]); | |
243 | } | |
244 | ||
245 | static inline size_t journal_file_entry_array_item_size(JournalFile *f) { | |
246 | assert(f); | |
247 | return JOURNAL_HEADER_COMPACT(f->header) ? sizeof(le32_t) : sizeof(le64_t); | |
248 | } | |
249 | ||
44a6b1b6 | 250 | uint64_t journal_file_hash_table_n_items(Object *o) _pure_; |
87d2c1ff | 251 | |
cc938e4a | 252 | int journal_file_append_object(JournalFile *f, ObjectType type, uint64_t size, Object **ret_object, uint64_t *ret_offset); |
d180c349 ZJS |
253 | int journal_file_append_entry( |
254 | JournalFile *f, | |
255 | const dual_timestamp *ts, | |
256 | const sd_id128_t *boot_id, | |
257 | const struct iovec iovec[], unsigned n_iovec, | |
258 | uint64_t *seqno, | |
cc938e4a | 259 | Object **ret_object, |
910eb3c0 | 260 | uint64_t *ret_offset); |
87d2c1ff | 261 | |
cc938e4a YW |
262 | int journal_file_find_data_object(JournalFile *f, const void *data, uint64_t size, Object **ret_object, uint64_t *ret_offset); |
263 | int journal_file_find_data_object_with_hash(JournalFile *f, const void *data, uint64_t size, uint64_t hash, Object **ret_object, uint64_t *ret_offset); | |
87d2c1ff | 264 | |
cc938e4a YW |
265 | int journal_file_find_field_object(JournalFile *f, const void *field, uint64_t size, Object **ret_object, uint64_t *ret_offset); |
266 | int journal_file_find_field_object_with_hash(JournalFile *f, const void *field, uint64_t size, uint64_t hash, Object **ret_object, uint64_t *ret_offset); | |
3c1668da | 267 | |
1fc605b0 | 268 | void journal_file_reset_location(JournalFile *f); |
950c07d4 | 269 | void journal_file_save_location(JournalFile *f, Object *o, uint64_t offset); |
d8ae66d7 | 270 | int journal_file_compare_locations(JournalFile *af, JournalFile *bf); |
cc938e4a | 271 | int journal_file_next_entry(JournalFile *f, uint64_t p, direction_t direction, Object **ret_object, uint64_t *ret_offset); |
87d2c1ff | 272 | |
cc938e4a | 273 | int journal_file_next_entry_for_data(JournalFile *f, Object *d, direction_t direction, Object **ret_object, uint64_t *ret_offset); |
de190aef | 274 | |
cc938e4a YW |
275 | int journal_file_move_to_entry_by_offset(JournalFile *f, uint64_t p, direction_t direction, Object **ret_object, uint64_t *ret_offset); |
276 | int journal_file_move_to_entry_by_seqnum(JournalFile *f, uint64_t seqnum, direction_t direction, Object **ret_object, uint64_t *ret_offset); | |
277 | int journal_file_move_to_entry_by_realtime(JournalFile *f, uint64_t realtime, direction_t direction, Object **ret_object, uint64_t *ret_offset); | |
278 | int journal_file_move_to_entry_by_monotonic(JournalFile *f, sd_id128_t boot_id, uint64_t monotonic, direction_t direction, Object **ret_object, uint64_t *ret_offset); | |
de190aef | 279 | |
cc938e4a YW |
280 | int journal_file_move_to_entry_by_offset_for_data(JournalFile *f, Object *d, uint64_t p, direction_t direction, Object **ret_object, uint64_t *ret_offset); |
281 | int journal_file_move_to_entry_by_seqnum_for_data(JournalFile *f, Object *d, uint64_t seqnum, direction_t direction, Object **ret_object, uint64_t *ret_offset); | |
282 | int journal_file_move_to_entry_by_realtime_for_data(JournalFile *f, Object *d, uint64_t realtime, direction_t direction, Object **ret_object, uint64_t *ret_offset); | |
283 | int journal_file_move_to_entry_by_monotonic_for_data(JournalFile *f, Object *d, sd_id128_t boot_id, uint64_t monotonic, direction_t direction, Object **ret_object, uint64_t *ret_offset); | |
87d2c1ff | 284 | |
5a271b08 | 285 | int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p); |
cf244689 | 286 | |
87d2c1ff | 287 | void journal_file_dump(JournalFile *f); |
dca6219e | 288 | void journal_file_print_header(JournalFile *f); |
87d2c1ff | 289 | |
461955ef | 290 | int journal_file_archive(JournalFile *f, char **ret_previous_path); |
596c3c7f | 291 | int journal_file_parse_uid_from_filename(const char *path, uid_t *uid); |
8b4fbbb0 | 292 | JournalFile* journal_initiate_close(JournalFile *f, Set *deferred_closes); |
0ac38b70 | 293 | |
68127658 LP |
294 | int journal_file_dispose(int dir_fd, const char *fname); |
295 | ||
cf244689 | 296 | void journal_file_post_change(JournalFile *f); |
7a24f3bf | 297 | int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t); |
cf244689 | 298 | |
8580d1f7 | 299 | void journal_reset_metrics(JournalMetrics *m); |
babfc091 | 300 | |
cc938e4a YW |
301 | int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *ret_from, usec_t *ret_to); |
302 | int journal_file_get_cutoff_monotonic_usec(JournalFile *f, sd_id128_t boot, usec_t *ret_from, usec_t *ret_to); | |
08984293 | 303 | |
c8e6e1f1 | 304 | bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec, int log_level); |
dade37d4 LP |
305 | |
306 | int journal_file_map_data_hash_table(JournalFile *f); | |
307 | int journal_file_map_field_hash_table(JournalFile *f); | |
d1afbcd2 LP |
308 | |
309 | static inline bool JOURNAL_FILE_COMPRESS(JournalFile *f) { | |
310 | assert(f); | |
4374d7ea DDM |
311 | return JOURNAL_HEADER_COMPRESSED_XZ(f->header) || JOURNAL_HEADER_COMPRESSED_LZ4(f->header) || |
312 | JOURNAL_HEADER_COMPRESSED_ZSTD(f->header); | |
d1afbcd2 | 313 | } |
4ce534f4 LP |
314 | |
315 | uint64_t journal_file_hash_data(JournalFile *f, const void *data, size_t sz); | |
adce225a YW |
316 | |
317 | bool journal_field_valid(const char *p, size_t l, bool allow_protected); | |
363b2b9a DDM |
318 | |
319 | const char* journal_object_type_to_string(ObjectType type) _const_; | |
acc50c92 LP |
320 | |
321 | static inline Compression COMPRESSION_FROM_OBJECT(const Object *o) { | |
322 | assert(o); | |
323 | ||
4d698d12 LP |
324 | switch (o->object.flags & _OBJECT_COMPRESSED_MASK) { |
325 | case 0: | |
326 | return COMPRESSION_NONE; | |
327 | case OBJECT_COMPRESSED_XZ: | |
328 | return COMPRESSION_XZ; | |
329 | case OBJECT_COMPRESSED_LZ4: | |
330 | return COMPRESSION_LZ4; | |
331 | case OBJECT_COMPRESSED_ZSTD: | |
332 | return COMPRESSION_ZSTD; | |
333 | default: | |
334 | return _COMPRESSION_INVALID; | |
335 | } | |
336 | } | |
acc50c92 | 337 | |
4374d7ea | 338 | static inline uint8_t COMPRESSION_TO_OBJECT_FLAG(Compression c) { |
4d698d12 LP |
339 | switch (c) { |
340 | case COMPRESSION_XZ: | |
341 | return OBJECT_COMPRESSED_XZ; | |
342 | case COMPRESSION_LZ4: | |
343 | return OBJECT_COMPRESSED_LZ4; | |
344 | case COMPRESSION_ZSTD: | |
345 | return OBJECT_COMPRESSED_ZSTD; | |
346 | default: | |
347 | return 0; | |
348 | } | |
acc50c92 | 349 | } |
4374d7ea DDM |
350 | |
351 | static inline uint32_t COMPRESSION_TO_HEADER_INCOMPATIBLE_FLAG(Compression c) { | |
352 | switch (c) { | |
353 | case COMPRESSION_XZ: | |
354 | return HEADER_INCOMPATIBLE_COMPRESSED_XZ; | |
355 | case COMPRESSION_LZ4: | |
356 | return HEADER_INCOMPATIBLE_COMPRESSED_LZ4; | |
357 | case COMPRESSION_ZSTD: | |
358 | return HEADER_INCOMPATIBLE_COMPRESSED_ZSTD; | |
359 | default: | |
360 | return 0; | |
361 | } | |
362 | } | |
363 | ||
364 | static inline bool journal_file_writable(JournalFile *f) { | |
365 | assert(f); | |
366 | return (f->open_flags & O_ACCMODE) != O_RDONLY; | |
367 | } |