1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2010 Lennart Poettering
7 Copyright 2014 Michal Schmidt
9 systemd is free software; you can redistribute it and/or modify it
10 under the terms of the GNU Lesser General Public License as published by
11 the Free Software Foundation; either version 2.1 of the License, or
12 (at your option) any later version.
14 systemd is distributed in the hope that it will be useful, but
15 WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 Lesser General Public License for more details.
19 You should have received a copy of the GNU Lesser General Public License
20 along with systemd; If not, see <http://www.gnu.org/licenses/>.
31 #include "siphash24.h"
34 #include "random-util.h"
36 #ifdef ENABLE_DEBUG_HASHMAP
41 * Implementation of hashmaps.
43 * - uses less RAM compared to closed addressing (chaining), because
44 * our entries are small (especially in Sets, which tend to contain
45 * the majority of entries in systemd).
46 * Collision resolution: Robin Hood
47 * - tends to equalize displacement of entries from their optimal buckets.
48 * Probe sequence: linear
49 * - though theoretically worse than random probing/uniform hashing/double
50 * hashing, it is good for cache locality.
53 * Celis, P. 1986. Robin Hood Hashing.
54 * Ph.D. Dissertation. University of Waterloo, Waterloo, Ont., Canada, Canada.
55 * https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf
56 * - The results are derived for random probing. Suggests deletion with
57 * tombstones and two mean-centered search methods. None of that works
58 * well for linear probing.
60 * Janson, S. 2005. Individual displacements for linear probing hashing with different insertion policies.
61 * ACM Trans. Algorithms 1, 2 (October 2005), 177-213.
62 * DOI=10.1145/1103963.1103964 http://doi.acm.org/10.1145/1103963.1103964
63 * http://www.math.uu.se/~svante/papers/sj157.pdf
64 * - Applies to Robin Hood with linear probing. Contains remarks on
65 * the unsuitability of mean-centered search with linear probing.
67 * Viola, A. 2005. Exact distribution of individual displacements in linear probing hashing.
68 * ACM Trans. Algorithms 1, 2 (October 2005), 214-242.
69 * DOI=10.1145/1103963.1103965 http://doi.acm.org/10.1145/1103963.1103965
70 * - Similar to Janson. Note that Viola writes about C_{m,n} (number of probes
71 * in a successful search), and Janson writes about displacement. C = d + 1.
73 * Goossaert, E. 2013. Robin Hood hashing: backward shift deletion.
74 * http://codecapsule.com/2013/11/17/robin-hood-hashing-backward-shift-deletion/
75 * - Explanation of backward shift deletion with pictures.
77 * Khuong, P. 2013. The Other Robin Hood Hashing.
78 * http://www.pvk.ca/Blog/2013/11/26/the-other-robin-hood-hashing/
79 * - Short summary of random vs. linear probing, and tombstones vs. backward shift.
83 * XXX Ideas for improvement:
84 * For unordered hashmaps, randomize iteration order, similarly to Perl:
85 * http://blog.booking.com/hardening-perls-hash-function.html
88 /* INV_KEEP_FREE = 1 / (1 - max_load_factor)
89 * e.g. 1 / (1 - 0.8) = 5 ... keep one fifth of the buckets free. */
90 #define INV_KEEP_FREE 5U
92 /* Fields common to entries of all hashmap/set types */
93 struct hashmap_base_entry
{
97 /* Entry types for specific hashmap/set types
98 * hashmap_base_entry must be at the beginning of each entry struct. */
100 struct plain_hashmap_entry
{
101 struct hashmap_base_entry b
;
105 struct ordered_hashmap_entry
{
106 struct plain_hashmap_entry p
;
107 unsigned iterate_next
, iterate_previous
;
111 struct hashmap_base_entry b
;
114 /* In several functions it is advantageous to have the hash table extended
115 * virtually by a couple of additional buckets. We reserve special index values
116 * for these "swap" buckets. */
117 #define _IDX_SWAP_BEGIN (UINT_MAX - 3)
118 #define IDX_PUT (_IDX_SWAP_BEGIN + 0)
119 #define IDX_TMP (_IDX_SWAP_BEGIN + 1)
120 #define _IDX_SWAP_END (_IDX_SWAP_BEGIN + 2)
122 #define IDX_FIRST (UINT_MAX - 1) /* special index for freshly initialized iterators */
123 #define IDX_NIL UINT_MAX /* special index value meaning "none" or "end" */
125 assert_cc(IDX_FIRST
== _IDX_SWAP_END
);
126 assert_cc(IDX_FIRST
== _IDX_ITERATOR_FIRST
);
128 /* Storage space for the "swap" buckets.
129 * All entry types can fit into a ordered_hashmap_entry. */
130 struct swap_entries
{
131 struct ordered_hashmap_entry e
[_IDX_SWAP_END
- _IDX_SWAP_BEGIN
];
134 /* Distance from Initial Bucket */
135 typedef uint8_t dib_raw_t
;
136 #define DIB_RAW_OVERFLOW ((dib_raw_t)0xfdU) /* indicates DIB value is greater than representable */
137 #define DIB_RAW_REHASH ((dib_raw_t)0xfeU) /* entry yet to be rehashed during in-place resize */
138 #define DIB_RAW_FREE ((dib_raw_t)0xffU) /* a free bucket */
139 #define DIB_RAW_INIT ((char)DIB_RAW_FREE) /* a byte to memset a DIB store with when initializing */
141 #define DIB_FREE UINT_MAX
143 #ifdef ENABLE_DEBUG_HASHMAP
144 struct hashmap_debug_info
{
145 LIST_FIELDS(struct hashmap_debug_info
, debug_list
);
146 unsigned max_entries
; /* high watermark of n_entries */
148 /* who allocated this hashmap */
153 /* fields to detect modification while iterating */
154 unsigned put_count
; /* counts puts into the hashmap */
155 unsigned rem_count
; /* counts removals from hashmap */
156 unsigned last_rem_idx
; /* remembers last removal index */
159 /* Tracks all existing hashmaps. Get at it from gdb. See sd_dump_hashmaps.py */
160 static LIST_HEAD(struct hashmap_debug_info
, hashmap_debug_list
);
161 static pthread_mutex_t hashmap_debug_list_mutex
= PTHREAD_MUTEX_INITIALIZER
;
163 #define HASHMAP_DEBUG_FIELDS struct hashmap_debug_info debug;
165 #else /* !ENABLE_DEBUG_HASHMAP */
166 #define HASHMAP_DEBUG_FIELDS
167 #endif /* ENABLE_DEBUG_HASHMAP */
171 HASHMAP_TYPE_ORDERED
,
176 struct _packed_ indirect_storage
{
177 char *storage
; /* where buckets and DIBs are stored */
178 uint8_t hash_key
[HASH_KEY_SIZE
]; /* hash key; changes during resize */
180 unsigned n_entries
; /* number of stored entries */
181 unsigned n_buckets
; /* number of buckets */
183 unsigned idx_lowest_entry
; /* Index below which all buckets are free.
184 Makes "while(hashmap_steal_first())" loops
185 O(n) instead of O(n^2) for unordered hashmaps. */
186 uint8_t _pad
[3]; /* padding for the whole HashmapBase */
187 /* The bitfields in HashmapBase complete the alignment of the whole thing. */
190 struct direct_storage
{
191 /* This gives us 39 bytes on 64bit, or 35 bytes on 32bit.
192 * That's room for 4 set_entries + 4 DIB bytes + 3 unused bytes on 64bit,
193 * or 7 set_entries + 7 DIB bytes + 0 unused bytes on 32bit. */
194 char storage
[sizeof(struct indirect_storage
)];
197 #define DIRECT_BUCKETS(entry_t) \
198 (sizeof(struct direct_storage) / (sizeof(entry_t) + sizeof(dib_raw_t)))
200 /* We should be able to store at least one entry directly. */
201 assert_cc(DIRECT_BUCKETS(struct ordered_hashmap_entry
) >= 1);
203 /* We have 3 bits for n_direct_entries. */
204 assert_cc(DIRECT_BUCKETS(struct set_entry
) < (1 << 3));
206 /* Hashmaps with directly stored entries all use this shared hash key.
207 * It's no big deal if the key is guessed, because there can be only
208 * a handful of directly stored entries in a hashmap. When a hashmap
209 * outgrows direct storage, it gets its own key for indirect storage. */
210 static uint8_t shared_hash_key
[HASH_KEY_SIZE
];
211 static bool shared_hash_key_initialized
;
213 /* Fields that all hashmap/set types must have */
215 const struct hash_ops
*hash_ops
; /* hash and compare ops to use */
218 struct indirect_storage indirect
; /* if has_indirect */
219 struct direct_storage direct
; /* if !has_indirect */
222 enum HashmapType type
:2; /* HASHMAP_TYPE_* */
223 bool has_indirect
:1; /* whether indirect storage is used */
224 unsigned n_direct_entries
:3; /* Number of entries in direct storage.
225 * Only valid if !has_indirect. */
226 bool from_pool
:1; /* whether was allocated from mempool */
227 HASHMAP_DEBUG_FIELDS
/* optional hashmap_debug_info */
230 /* Specific hash types
231 * HashmapBase must be at the beginning of each hashmap struct. */
234 struct HashmapBase b
;
237 struct OrderedHashmap
{
238 struct HashmapBase b
;
239 unsigned iterate_list_head
, iterate_list_tail
;
243 struct HashmapBase b
;
246 DEFINE_MEMPOOL(hashmap_pool
, Hashmap
, 8);
247 DEFINE_MEMPOOL(ordered_hashmap_pool
, OrderedHashmap
, 8);
248 /* No need for a separate Set pool */
249 assert_cc(sizeof(Hashmap
) == sizeof(Set
));
251 struct hashmap_type_info
{
254 struct mempool
*mempool
;
255 unsigned n_direct_buckets
;
258 static const struct hashmap_type_info hashmap_type_info
[_HASHMAP_TYPE_MAX
] = {
259 [HASHMAP_TYPE_PLAIN
] = {
260 .head_size
= sizeof(Hashmap
),
261 .entry_size
= sizeof(struct plain_hashmap_entry
),
262 .mempool
= &hashmap_pool
,
263 .n_direct_buckets
= DIRECT_BUCKETS(struct plain_hashmap_entry
),
265 [HASHMAP_TYPE_ORDERED
] = {
266 .head_size
= sizeof(OrderedHashmap
),
267 .entry_size
= sizeof(struct ordered_hashmap_entry
),
268 .mempool
= &ordered_hashmap_pool
,
269 .n_direct_buckets
= DIRECT_BUCKETS(struct ordered_hashmap_entry
),
271 [HASHMAP_TYPE_SET
] = {
272 .head_size
= sizeof(Set
),
273 .entry_size
= sizeof(struct set_entry
),
274 .mempool
= &hashmap_pool
,
275 .n_direct_buckets
= DIRECT_BUCKETS(struct set_entry
),
279 void string_hash_func(const void *p
, struct siphash
*state
) {
280 siphash24_compress(p
, strlen(p
) + 1, state
);
283 int string_compare_func(const void *a
, const void *b
) {
287 const struct hash_ops string_hash_ops
= {
288 .hash
= string_hash_func
,
289 .compare
= string_compare_func
292 void trivial_hash_func(const void *p
, struct siphash
*state
) {
293 siphash24_compress(&p
, sizeof(p
), state
);
296 int trivial_compare_func(const void *a
, const void *b
) {
297 return a
< b
? -1 : (a
> b
? 1 : 0);
300 const struct hash_ops trivial_hash_ops
= {
301 .hash
= trivial_hash_func
,
302 .compare
= trivial_compare_func
305 void uint64_hash_func(const void *p
, struct siphash
*state
) {
306 siphash24_compress(p
, sizeof(uint64_t), state
);
309 int uint64_compare_func(const void *_a
, const void *_b
) {
311 a
= *(const uint64_t*) _a
;
312 b
= *(const uint64_t*) _b
;
313 return a
< b
? -1 : (a
> b
? 1 : 0);
316 const struct hash_ops uint64_hash_ops
= {
317 .hash
= uint64_hash_func
,
318 .compare
= uint64_compare_func
321 #if SIZEOF_DEV_T != 8
322 void devt_hash_func(const void *p
, struct siphash
*state
) {
323 siphash24_compress(p
, sizeof(dev_t
), state
);
326 int devt_compare_func(const void *_a
, const void *_b
) {
328 a
= *(const dev_t
*) _a
;
329 b
= *(const dev_t
*) _b
;
330 return a
< b
? -1 : (a
> b
? 1 : 0);
333 const struct hash_ops devt_hash_ops
= {
334 .hash
= devt_hash_func
,
335 .compare
= devt_compare_func
339 static unsigned n_buckets(HashmapBase
*h
) {
340 return h
->has_indirect
? h
->indirect
.n_buckets
341 : hashmap_type_info
[h
->type
].n_direct_buckets
;
344 static unsigned n_entries(HashmapBase
*h
) {
345 return h
->has_indirect
? h
->indirect
.n_entries
346 : h
->n_direct_entries
;
349 static void n_entries_inc(HashmapBase
*h
) {
351 h
->indirect
.n_entries
++;
353 h
->n_direct_entries
++;
356 static void n_entries_dec(HashmapBase
*h
) {
358 h
->indirect
.n_entries
--;
360 h
->n_direct_entries
--;
363 static char *storage_ptr(HashmapBase
*h
) {
364 return h
->has_indirect
? h
->indirect
.storage
368 static uint8_t *hash_key(HashmapBase
*h
) {
369 return h
->has_indirect
? h
->indirect
.hash_key
373 static unsigned base_bucket_hash(HashmapBase
*h
, const void *p
) {
374 struct siphash state
;
376 siphash_init(&state
, hash_key(h
));
378 h
->hash_ops
->hash(p
, &state
);
380 return (unsigned) (siphash24_finalize(&state
) % n_buckets(h
));
382 #define bucket_hash(h, p) base_bucket_hash(HASHMAP_BASE(h), p)
384 static void get_hash_key(uint8_t hash_key
[HASH_KEY_SIZE
], bool reuse_is_ok
) {
385 static uint8_t current
[HASH_KEY_SIZE
];
386 static bool current_initialized
= false;
388 /* Returns a hash function key to use. In order to keep things
389 * fast we will not generate a new key each time we allocate a
390 * new hash table. Instead, we'll just reuse the most recently
391 * generated one, except if we never generated one or when we
392 * are rehashing an entire hash table because we reached a
395 if (!current_initialized
|| !reuse_is_ok
) {
396 random_bytes(current
, sizeof(current
));
397 current_initialized
= true;
400 memcpy(hash_key
, current
, sizeof(current
));
403 static struct hashmap_base_entry
*bucket_at(HashmapBase
*h
, unsigned idx
) {
404 return (struct hashmap_base_entry
*)
405 (storage_ptr(h
) + idx
* hashmap_type_info
[h
->type
].entry_size
);
408 static struct plain_hashmap_entry
*plain_bucket_at(Hashmap
*h
, unsigned idx
) {
409 return (struct plain_hashmap_entry
*) bucket_at(HASHMAP_BASE(h
), idx
);
412 static struct ordered_hashmap_entry
*ordered_bucket_at(OrderedHashmap
*h
, unsigned idx
) {
413 return (struct ordered_hashmap_entry
*) bucket_at(HASHMAP_BASE(h
), idx
);
416 static struct set_entry
*set_bucket_at(Set
*h
, unsigned idx
) {
417 return (struct set_entry
*) bucket_at(HASHMAP_BASE(h
), idx
);
420 static struct ordered_hashmap_entry
*bucket_at_swap(struct swap_entries
*swap
, unsigned idx
) {
421 return &swap
->e
[idx
- _IDX_SWAP_BEGIN
];
424 /* Returns a pointer to the bucket at index idx.
425 * Understands real indexes and swap indexes, hence "_virtual". */
426 static struct hashmap_base_entry
*bucket_at_virtual(HashmapBase
*h
, struct swap_entries
*swap
,
428 if (idx
< _IDX_SWAP_BEGIN
)
429 return bucket_at(h
, idx
);
431 if (idx
< _IDX_SWAP_END
)
432 return &bucket_at_swap(swap
, idx
)->p
.b
;
434 assert_not_reached("Invalid index");
437 static dib_raw_t
*dib_raw_ptr(HashmapBase
*h
) {
439 (storage_ptr(h
) + hashmap_type_info
[h
->type
].entry_size
* n_buckets(h
));
442 static unsigned bucket_distance(HashmapBase
*h
, unsigned idx
, unsigned from
) {
443 return idx
>= from
? idx
- from
444 : n_buckets(h
) + idx
- from
;
447 static unsigned bucket_calculate_dib(HashmapBase
*h
, unsigned idx
, dib_raw_t raw_dib
) {
448 unsigned initial_bucket
;
450 if (raw_dib
== DIB_RAW_FREE
)
453 if (_likely_(raw_dib
< DIB_RAW_OVERFLOW
))
457 * Having an overflow DIB value is very unlikely. The hash function
458 * would have to be bad. For example, in a table of size 2^24 filled
459 * to load factor 0.9 the maximum observed DIB is only about 60.
460 * In theory (assuming I used Maxima correctly), for an infinite size
461 * hash table with load factor 0.8 the probability of a given entry
462 * having DIB > 40 is 1.9e-8.
463 * This returns the correct DIB value by recomputing the hash value in
464 * the unlikely case. XXX Hitting this case could be a hint to rehash.
466 initial_bucket
= bucket_hash(h
, bucket_at(h
, idx
)->key
);
467 return bucket_distance(h
, idx
, initial_bucket
);
470 static void bucket_set_dib(HashmapBase
*h
, unsigned idx
, unsigned dib
) {
471 dib_raw_ptr(h
)[idx
] = dib
!= DIB_FREE
? MIN(dib
, DIB_RAW_OVERFLOW
) : DIB_RAW_FREE
;
474 static unsigned skip_free_buckets(HashmapBase
*h
, unsigned idx
) {
477 dibs
= dib_raw_ptr(h
);
479 for ( ; idx
< n_buckets(h
); idx
++)
480 if (dibs
[idx
] != DIB_RAW_FREE
)
486 static void bucket_mark_free(HashmapBase
*h
, unsigned idx
) {
487 memzero(bucket_at(h
, idx
), hashmap_type_info
[h
->type
].entry_size
);
488 bucket_set_dib(h
, idx
, DIB_FREE
);
491 static void bucket_move_entry(HashmapBase
*h
, struct swap_entries
*swap
,
492 unsigned from
, unsigned to
) {
493 struct hashmap_base_entry
*e_from
, *e_to
;
497 e_from
= bucket_at_virtual(h
, swap
, from
);
498 e_to
= bucket_at_virtual(h
, swap
, to
);
500 memcpy(e_to
, e_from
, hashmap_type_info
[h
->type
].entry_size
);
502 if (h
->type
== HASHMAP_TYPE_ORDERED
) {
503 OrderedHashmap
*lh
= (OrderedHashmap
*) h
;
504 struct ordered_hashmap_entry
*le
, *le_to
;
506 le_to
= (struct ordered_hashmap_entry
*) e_to
;
508 if (le_to
->iterate_next
!= IDX_NIL
) {
509 le
= (struct ordered_hashmap_entry
*)
510 bucket_at_virtual(h
, swap
, le_to
->iterate_next
);
511 le
->iterate_previous
= to
;
514 if (le_to
->iterate_previous
!= IDX_NIL
) {
515 le
= (struct ordered_hashmap_entry
*)
516 bucket_at_virtual(h
, swap
, le_to
->iterate_previous
);
517 le
->iterate_next
= to
;
520 if (lh
->iterate_list_head
== from
)
521 lh
->iterate_list_head
= to
;
522 if (lh
->iterate_list_tail
== from
)
523 lh
->iterate_list_tail
= to
;
527 static unsigned next_idx(HashmapBase
*h
, unsigned idx
) {
528 return (idx
+ 1U) % n_buckets(h
);
531 static unsigned prev_idx(HashmapBase
*h
, unsigned idx
) {
532 return (n_buckets(h
) + idx
- 1U) % n_buckets(h
);
535 static void *entry_value(HashmapBase
*h
, struct hashmap_base_entry
*e
) {
538 case HASHMAP_TYPE_PLAIN
:
539 case HASHMAP_TYPE_ORDERED
:
540 return ((struct plain_hashmap_entry
*)e
)->value
;
542 case HASHMAP_TYPE_SET
:
543 return (void*) e
->key
;
546 assert_not_reached("Unknown hashmap type");
550 static void base_remove_entry(HashmapBase
*h
, unsigned idx
) {
551 unsigned left
, right
, prev
, dib
;
552 dib_raw_t raw_dib
, *dibs
;
554 dibs
= dib_raw_ptr(h
);
555 assert(dibs
[idx
] != DIB_RAW_FREE
);
557 #ifdef ENABLE_DEBUG_HASHMAP
558 h
->debug
.rem_count
++;
559 h
->debug
.last_rem_idx
= idx
;
563 /* Find the stop bucket ("right"). It is either free or has DIB == 0. */
564 for (right
= next_idx(h
, left
); ; right
= next_idx(h
, right
)) {
565 raw_dib
= dibs
[right
];
566 if (raw_dib
== 0 || raw_dib
== DIB_RAW_FREE
)
569 /* The buckets are not supposed to be all occupied and with DIB > 0.
570 * That would mean we could make everyone better off by shifting them
571 * backward. This scenario is impossible. */
572 assert(left
!= right
);
575 if (h
->type
== HASHMAP_TYPE_ORDERED
) {
576 OrderedHashmap
*lh
= (OrderedHashmap
*) h
;
577 struct ordered_hashmap_entry
*le
= ordered_bucket_at(lh
, idx
);
579 if (le
->iterate_next
!= IDX_NIL
)
580 ordered_bucket_at(lh
, le
->iterate_next
)->iterate_previous
= le
->iterate_previous
;
582 lh
->iterate_list_tail
= le
->iterate_previous
;
584 if (le
->iterate_previous
!= IDX_NIL
)
585 ordered_bucket_at(lh
, le
->iterate_previous
)->iterate_next
= le
->iterate_next
;
587 lh
->iterate_list_head
= le
->iterate_next
;
590 /* Now shift all buckets in the interval (left, right) one step backwards */
591 for (prev
= left
, left
= next_idx(h
, left
); left
!= right
;
592 prev
= left
, left
= next_idx(h
, left
)) {
593 dib
= bucket_calculate_dib(h
, left
, dibs
[left
]);
595 bucket_move_entry(h
, NULL
, left
, prev
);
596 bucket_set_dib(h
, prev
, dib
- 1);
599 bucket_mark_free(h
, prev
);
602 #define remove_entry(h, idx) base_remove_entry(HASHMAP_BASE(h), idx)
604 static unsigned hashmap_iterate_in_insertion_order(OrderedHashmap
*h
, Iterator
*i
) {
605 struct ordered_hashmap_entry
*e
;
611 if (i
->idx
== IDX_NIL
)
614 if (i
->idx
== IDX_FIRST
&& h
->iterate_list_head
== IDX_NIL
)
617 if (i
->idx
== IDX_FIRST
) {
618 idx
= h
->iterate_list_head
;
619 e
= ordered_bucket_at(h
, idx
);
622 e
= ordered_bucket_at(h
, idx
);
624 * We allow removing the current entry while iterating, but removal may cause
625 * a backward shift. The next entry may thus move one bucket to the left.
626 * To detect when it happens, we remember the key pointer of the entry we were
627 * going to iterate next. If it does not match, there was a backward shift.
629 if (e
->p
.b
.key
!= i
->next_key
) {
630 idx
= prev_idx(HASHMAP_BASE(h
), idx
);
631 e
= ordered_bucket_at(h
, idx
);
633 assert(e
->p
.b
.key
== i
->next_key
);
636 #ifdef ENABLE_DEBUG_HASHMAP
640 if (e
->iterate_next
!= IDX_NIL
) {
641 struct ordered_hashmap_entry
*n
;
642 i
->idx
= e
->iterate_next
;
643 n
= ordered_bucket_at(h
, i
->idx
);
644 i
->next_key
= n
->p
.b
.key
;
655 static unsigned hashmap_iterate_in_internal_order(HashmapBase
*h
, Iterator
*i
) {
661 if (i
->idx
== IDX_NIL
)
664 if (i
->idx
== IDX_FIRST
) {
665 /* fast forward to the first occupied bucket */
666 if (h
->has_indirect
) {
667 i
->idx
= skip_free_buckets(h
, h
->indirect
.idx_lowest_entry
);
668 h
->indirect
.idx_lowest_entry
= i
->idx
;
670 i
->idx
= skip_free_buckets(h
, 0);
672 if (i
->idx
== IDX_NIL
)
675 struct hashmap_base_entry
*e
;
679 e
= bucket_at(h
, i
->idx
);
681 * We allow removing the current entry while iterating, but removal may cause
682 * a backward shift. The next entry may thus move one bucket to the left.
683 * To detect when it happens, we remember the key pointer of the entry we were
684 * going to iterate next. If it does not match, there was a backward shift.
686 if (e
->key
!= i
->next_key
)
687 e
= bucket_at(h
, --i
->idx
);
689 assert(e
->key
== i
->next_key
);
693 #ifdef ENABLE_DEBUG_HASHMAP
697 i
->idx
= skip_free_buckets(h
, i
->idx
+ 1);
698 if (i
->idx
!= IDX_NIL
)
699 i
->next_key
= bucket_at(h
, i
->idx
)->key
;
710 static unsigned hashmap_iterate_entry(HashmapBase
*h
, Iterator
*i
) {
716 #ifdef ENABLE_DEBUG_HASHMAP
717 if (i
->idx
== IDX_FIRST
) {
718 i
->put_count
= h
->debug
.put_count
;
719 i
->rem_count
= h
->debug
.rem_count
;
721 /* While iterating, must not add any new entries */
722 assert(i
->put_count
== h
->debug
.put_count
);
723 /* ... or remove entries other than the current one */
724 assert(i
->rem_count
== h
->debug
.rem_count
||
725 (i
->rem_count
== h
->debug
.rem_count
- 1 &&
726 i
->prev_idx
== h
->debug
.last_rem_idx
));
727 /* Reset our removals counter */
728 i
->rem_count
= h
->debug
.rem_count
;
732 return h
->type
== HASHMAP_TYPE_ORDERED
? hashmap_iterate_in_insertion_order((OrderedHashmap
*) h
, i
)
733 : hashmap_iterate_in_internal_order(h
, i
);
736 bool internal_hashmap_iterate(HashmapBase
*h
, Iterator
*i
, void **value
, const void **key
) {
737 struct hashmap_base_entry
*e
;
741 idx
= hashmap_iterate_entry(h
, i
);
742 if (idx
== IDX_NIL
) {
751 e
= bucket_at(h
, idx
);
752 data
= entry_value(h
, e
);
761 bool set_iterate(Set
*s
, Iterator
*i
, void **value
) {
762 return internal_hashmap_iterate(HASHMAP_BASE(s
), i
, value
, NULL
);
765 #define HASHMAP_FOREACH_IDX(idx, h, i) \
766 for ((i) = ITERATOR_FIRST, (idx) = hashmap_iterate_entry((h), &(i)); \
768 (idx) = hashmap_iterate_entry((h), &(i)))
770 static void reset_direct_storage(HashmapBase
*h
) {
771 const struct hashmap_type_info
*hi
= &hashmap_type_info
[h
->type
];
774 assert(!h
->has_indirect
);
776 p
= mempset(h
->direct
.storage
, 0, hi
->entry_size
* hi
->n_direct_buckets
);
777 memset(p
, DIB_RAW_INIT
, sizeof(dib_raw_t
) * hi
->n_direct_buckets
);
780 static struct HashmapBase
*hashmap_base_new(const struct hash_ops
*hash_ops
, enum HashmapType type HASHMAP_DEBUG_PARAMS
) {
782 const struct hashmap_type_info
*hi
= &hashmap_type_info
[type
];
785 use_pool
= is_main_thread();
787 h
= use_pool
? mempool_alloc0_tile(hi
->mempool
) : malloc0(hi
->head_size
);
793 h
->from_pool
= use_pool
;
794 h
->hash_ops
= hash_ops
? hash_ops
: &trivial_hash_ops
;
796 if (type
== HASHMAP_TYPE_ORDERED
) {
797 OrderedHashmap
*lh
= (OrderedHashmap
*)h
;
798 lh
->iterate_list_head
= lh
->iterate_list_tail
= IDX_NIL
;
801 reset_direct_storage(h
);
803 if (!shared_hash_key_initialized
) {
804 random_bytes(shared_hash_key
, sizeof(shared_hash_key
));
805 shared_hash_key_initialized
= true;
808 #ifdef ENABLE_DEBUG_HASHMAP
809 h
->debug
.func
= func
;
810 h
->debug
.file
= file
;
811 h
->debug
.line
= line
;
812 assert_se(pthread_mutex_lock(&hashmap_debug_list_mutex
) == 0);
813 LIST_PREPEND(debug_list
, hashmap_debug_list
, &h
->debug
);
814 assert_se(pthread_mutex_unlock(&hashmap_debug_list_mutex
) == 0);
820 Hashmap
*internal_hashmap_new(const struct hash_ops
*hash_ops HASHMAP_DEBUG_PARAMS
) {
821 return (Hashmap
*) hashmap_base_new(hash_ops
, HASHMAP_TYPE_PLAIN HASHMAP_DEBUG_PASS_ARGS
);
824 OrderedHashmap
*internal_ordered_hashmap_new(const struct hash_ops
*hash_ops HASHMAP_DEBUG_PARAMS
) {
825 return (OrderedHashmap
*) hashmap_base_new(hash_ops
, HASHMAP_TYPE_ORDERED HASHMAP_DEBUG_PASS_ARGS
);
828 Set
*internal_set_new(const struct hash_ops
*hash_ops HASHMAP_DEBUG_PARAMS
) {
829 return (Set
*) hashmap_base_new(hash_ops
, HASHMAP_TYPE_SET HASHMAP_DEBUG_PASS_ARGS
);
832 static int hashmap_base_ensure_allocated(HashmapBase
**h
, const struct hash_ops
*hash_ops
,
833 enum HashmapType type HASHMAP_DEBUG_PARAMS
) {
841 q
= hashmap_base_new(hash_ops
, type HASHMAP_DEBUG_PASS_ARGS
);
849 int internal_hashmap_ensure_allocated(Hashmap
**h
, const struct hash_ops
*hash_ops HASHMAP_DEBUG_PARAMS
) {
850 return hashmap_base_ensure_allocated((HashmapBase
**)h
, hash_ops
, HASHMAP_TYPE_PLAIN HASHMAP_DEBUG_PASS_ARGS
);
853 int internal_ordered_hashmap_ensure_allocated(OrderedHashmap
**h
, const struct hash_ops
*hash_ops HASHMAP_DEBUG_PARAMS
) {
854 return hashmap_base_ensure_allocated((HashmapBase
**)h
, hash_ops
, HASHMAP_TYPE_ORDERED HASHMAP_DEBUG_PASS_ARGS
);
857 int internal_set_ensure_allocated(Set
**s
, const struct hash_ops
*hash_ops HASHMAP_DEBUG_PARAMS
) {
858 return hashmap_base_ensure_allocated((HashmapBase
**)s
, hash_ops
, HASHMAP_TYPE_SET HASHMAP_DEBUG_PASS_ARGS
);
861 static void hashmap_free_no_clear(HashmapBase
*h
) {
862 assert(!h
->has_indirect
);
863 assert(!h
->n_direct_entries
);
865 #ifdef ENABLE_DEBUG_HASHMAP
866 assert_se(pthread_mutex_lock(&hashmap_debug_list_mutex
) == 0);
867 LIST_REMOVE(debug_list
, hashmap_debug_list
, &h
->debug
);
868 assert_se(pthread_mutex_unlock(&hashmap_debug_list_mutex
) == 0);
872 mempool_free_tile(hashmap_type_info
[h
->type
].mempool
, h
);
877 HashmapBase
*internal_hashmap_free(HashmapBase
*h
) {
879 /* Free the hashmap, but nothing in it */
882 internal_hashmap_clear(h
);
883 hashmap_free_no_clear(h
);
889 HashmapBase
*internal_hashmap_free_free(HashmapBase
*h
) {
891 /* Free the hashmap and all data objects in it, but not the
895 internal_hashmap_clear_free(h
);
896 hashmap_free_no_clear(h
);
902 Hashmap
*hashmap_free_free_free(Hashmap
*h
) {
904 /* Free the hashmap and all data and key objects in it */
907 hashmap_clear_free_free(h
);
908 hashmap_free_no_clear(HASHMAP_BASE(h
));
914 void internal_hashmap_clear(HashmapBase
*h
) {
918 if (h
->has_indirect
) {
919 free(h
->indirect
.storage
);
920 h
->has_indirect
= false;
923 h
->n_direct_entries
= 0;
924 reset_direct_storage(h
);
926 if (h
->type
== HASHMAP_TYPE_ORDERED
) {
927 OrderedHashmap
*lh
= (OrderedHashmap
*) h
;
928 lh
->iterate_list_head
= lh
->iterate_list_tail
= IDX_NIL
;
932 void internal_hashmap_clear_free(HashmapBase
*h
) {
938 for (idx
= skip_free_buckets(h
, 0); idx
!= IDX_NIL
;
939 idx
= skip_free_buckets(h
, idx
+ 1))
940 free(entry_value(h
, bucket_at(h
, idx
)));
942 internal_hashmap_clear(h
);
945 void hashmap_clear_free_free(Hashmap
*h
) {
951 for (idx
= skip_free_buckets(HASHMAP_BASE(h
), 0); idx
!= IDX_NIL
;
952 idx
= skip_free_buckets(HASHMAP_BASE(h
), idx
+ 1)) {
953 struct plain_hashmap_entry
*e
= plain_bucket_at(h
, idx
);
954 free((void*)e
->b
.key
);
958 internal_hashmap_clear(HASHMAP_BASE(h
));
961 static int resize_buckets(HashmapBase
*h
, unsigned entries_add
);
964 * Finds an empty bucket to put an entry into, starting the scan at 'idx'.
965 * Performs Robin Hood swaps as it goes. The entry to put must be placed
966 * by the caller into swap slot IDX_PUT.
967 * If used for in-place resizing, may leave a displaced entry in swap slot
968 * IDX_PUT. Caller must rehash it next.
969 * Returns: true if it left a displaced entry to rehash next in IDX_PUT,
972 static bool hashmap_put_robin_hood(HashmapBase
*h
, unsigned idx
,
973 struct swap_entries
*swap
) {
974 dib_raw_t raw_dib
, *dibs
;
975 unsigned dib
, distance
;
977 #ifdef ENABLE_DEBUG_HASHMAP
978 h
->debug
.put_count
++;
981 dibs
= dib_raw_ptr(h
);
983 for (distance
= 0; ; distance
++) {
985 if (raw_dib
== DIB_RAW_FREE
|| raw_dib
== DIB_RAW_REHASH
) {
986 if (raw_dib
== DIB_RAW_REHASH
)
987 bucket_move_entry(h
, swap
, idx
, IDX_TMP
);
989 if (h
->has_indirect
&& h
->indirect
.idx_lowest_entry
> idx
)
990 h
->indirect
.idx_lowest_entry
= idx
;
992 bucket_set_dib(h
, idx
, distance
);
993 bucket_move_entry(h
, swap
, IDX_PUT
, idx
);
994 if (raw_dib
== DIB_RAW_REHASH
) {
995 bucket_move_entry(h
, swap
, IDX_TMP
, IDX_PUT
);
1002 dib
= bucket_calculate_dib(h
, idx
, raw_dib
);
1004 if (dib
< distance
) {
1005 /* Found a wealthier entry. Go Robin Hood! */
1006 bucket_set_dib(h
, idx
, distance
);
1008 /* swap the entries */
1009 bucket_move_entry(h
, swap
, idx
, IDX_TMP
);
1010 bucket_move_entry(h
, swap
, IDX_PUT
, idx
);
1011 bucket_move_entry(h
, swap
, IDX_TMP
, IDX_PUT
);
1016 idx
= next_idx(h
, idx
);
1021 * Puts an entry into a hashmap, boldly - no check whether key already exists.
1022 * The caller must place the entry (only its key and value, not link indexes)
1023 * in swap slot IDX_PUT.
1024 * Caller must ensure: the key does not exist yet in the hashmap.
1025 * that resize is not needed if !may_resize.
1026 * Returns: 1 if entry was put successfully.
1027 * -ENOMEM if may_resize==true and resize failed with -ENOMEM.
1028 * Cannot return -ENOMEM if !may_resize.
1030 static int hashmap_base_put_boldly(HashmapBase
*h
, unsigned idx
,
1031 struct swap_entries
*swap
, bool may_resize
) {
1032 struct ordered_hashmap_entry
*new_entry
;
1035 assert(idx
< n_buckets(h
));
1037 new_entry
= bucket_at_swap(swap
, IDX_PUT
);
1040 r
= resize_buckets(h
, 1);
1044 idx
= bucket_hash(h
, new_entry
->p
.b
.key
);
1046 assert(n_entries(h
) < n_buckets(h
));
1048 if (h
->type
== HASHMAP_TYPE_ORDERED
) {
1049 OrderedHashmap
*lh
= (OrderedHashmap
*) h
;
1051 new_entry
->iterate_next
= IDX_NIL
;
1052 new_entry
->iterate_previous
= lh
->iterate_list_tail
;
1054 if (lh
->iterate_list_tail
!= IDX_NIL
) {
1055 struct ordered_hashmap_entry
*old_tail
;
1057 old_tail
= ordered_bucket_at(lh
, lh
->iterate_list_tail
);
1058 assert(old_tail
->iterate_next
== IDX_NIL
);
1059 old_tail
->iterate_next
= IDX_PUT
;
1062 lh
->iterate_list_tail
= IDX_PUT
;
1063 if (lh
->iterate_list_head
== IDX_NIL
)
1064 lh
->iterate_list_head
= IDX_PUT
;
1067 assert_se(hashmap_put_robin_hood(h
, idx
, swap
) == false);
1070 #ifdef ENABLE_DEBUG_HASHMAP
1071 h
->debug
.max_entries
= MAX(h
->debug
.max_entries
, n_entries(h
));
1076 #define hashmap_put_boldly(h, idx, swap, may_resize) \
1077 hashmap_base_put_boldly(HASHMAP_BASE(h), idx, swap, may_resize)
1080 * Returns 0 if resize is not needed.
1081 * 1 if successfully resized.
1082 * -ENOMEM on allocation failure.
1084 static int resize_buckets(HashmapBase
*h
, unsigned entries_add
) {
1085 struct swap_entries swap
;
1087 dib_raw_t
*old_dibs
, *new_dibs
;
1088 const struct hashmap_type_info
*hi
;
1089 unsigned idx
, optimal_idx
;
1090 unsigned old_n_buckets
, new_n_buckets
, n_rehashed
, new_n_entries
;
1096 hi
= &hashmap_type_info
[h
->type
];
1097 new_n_entries
= n_entries(h
) + entries_add
;
1100 if (_unlikely_(new_n_entries
< entries_add
))
1103 /* For direct storage we allow 100% load, because it's tiny. */
1104 if (!h
->has_indirect
&& new_n_entries
<= hi
->n_direct_buckets
)
1108 * Load factor = n/m = 1 - (1/INV_KEEP_FREE).
1109 * From it follows: m = n + n/(INV_KEEP_FREE - 1)
1111 new_n_buckets
= new_n_entries
+ new_n_entries
/ (INV_KEEP_FREE
- 1);
1113 if (_unlikely_(new_n_buckets
< new_n_entries
))
1116 if (_unlikely_(new_n_buckets
> UINT_MAX
/ (hi
->entry_size
+ sizeof(dib_raw_t
))))
1119 old_n_buckets
= n_buckets(h
);
1121 if (_likely_(new_n_buckets
<= old_n_buckets
))
1124 new_shift
= log2u_round_up(MAX(
1125 new_n_buckets
* (hi
->entry_size
+ sizeof(dib_raw_t
)),
1126 2 * sizeof(struct direct_storage
)));
1128 /* Realloc storage (buckets and DIB array). */
1129 new_storage
= realloc(h
->has_indirect
? h
->indirect
.storage
: NULL
,
1134 /* Must upgrade direct to indirect storage. */
1135 if (!h
->has_indirect
) {
1136 memcpy(new_storage
, h
->direct
.storage
,
1137 old_n_buckets
* (hi
->entry_size
+ sizeof(dib_raw_t
)));
1138 h
->indirect
.n_entries
= h
->n_direct_entries
;
1139 h
->indirect
.idx_lowest_entry
= 0;
1140 h
->n_direct_entries
= 0;
1143 /* Get a new hash key. If we've just upgraded to indirect storage,
1144 * allow reusing a previously generated key. It's still a different key
1145 * from the shared one that we used for direct storage. */
1146 get_hash_key(h
->indirect
.hash_key
, !h
->has_indirect
);
1148 h
->has_indirect
= true;
1149 h
->indirect
.storage
= new_storage
;
1150 h
->indirect
.n_buckets
= (1U << new_shift
) /
1151 (hi
->entry_size
+ sizeof(dib_raw_t
));
1153 old_dibs
= (dib_raw_t
*)(new_storage
+ hi
->entry_size
* old_n_buckets
);
1154 new_dibs
= dib_raw_ptr(h
);
1157 * Move the DIB array to the new place, replacing valid DIB values with
1158 * DIB_RAW_REHASH to indicate all of the used buckets need rehashing.
1159 * Note: Overlap is not possible, because we have at least doubled the
1160 * number of buckets and dib_raw_t is smaller than any entry type.
1162 for (idx
= 0; idx
< old_n_buckets
; idx
++) {
1163 assert(old_dibs
[idx
] != DIB_RAW_REHASH
);
1164 new_dibs
[idx
] = old_dibs
[idx
] == DIB_RAW_FREE
? DIB_RAW_FREE
1168 /* Zero the area of newly added entries (including the old DIB area) */
1169 memzero(bucket_at(h
, old_n_buckets
),
1170 (n_buckets(h
) - old_n_buckets
) * hi
->entry_size
);
1172 /* The upper half of the new DIB array needs initialization */
1173 memset(&new_dibs
[old_n_buckets
], DIB_RAW_INIT
,
1174 (n_buckets(h
) - old_n_buckets
) * sizeof(dib_raw_t
));
1176 /* Rehash entries that need it */
1178 for (idx
= 0; idx
< old_n_buckets
; idx
++) {
1179 if (new_dibs
[idx
] != DIB_RAW_REHASH
)
1182 optimal_idx
= bucket_hash(h
, bucket_at(h
, idx
)->key
);
1185 * Not much to do if by luck the entry hashes to its current
1186 * location. Just set its DIB.
1188 if (optimal_idx
== idx
) {
1194 new_dibs
[idx
] = DIB_RAW_FREE
;
1195 bucket_move_entry(h
, &swap
, idx
, IDX_PUT
);
1196 /* bucket_move_entry does not clear the source */
1197 memzero(bucket_at(h
, idx
), hi
->entry_size
);
1201 * Find the new bucket for the current entry. This may make
1202 * another entry homeless and load it into IDX_PUT.
1204 rehash_next
= hashmap_put_robin_hood(h
, optimal_idx
, &swap
);
1207 /* Did the current entry displace another one? */
1209 optimal_idx
= bucket_hash(h
, bucket_at_swap(&swap
, IDX_PUT
)->p
.b
.key
);
1210 } while (rehash_next
);
1213 assert(n_rehashed
== n_entries(h
));
1219 * Finds an entry with a matching key
1220 * Returns: index of the found entry, or IDX_NIL if not found.
1222 static unsigned base_bucket_scan(HashmapBase
*h
, unsigned idx
, const void *key
) {
1223 struct hashmap_base_entry
*e
;
1224 unsigned dib
, distance
;
1225 dib_raw_t
*dibs
= dib_raw_ptr(h
);
1227 assert(idx
< n_buckets(h
));
1229 for (distance
= 0; ; distance
++) {
1230 if (dibs
[idx
] == DIB_RAW_FREE
)
1233 dib
= bucket_calculate_dib(h
, idx
, dibs
[idx
]);
1237 if (dib
== distance
) {
1238 e
= bucket_at(h
, idx
);
1239 if (h
->hash_ops
->compare(e
->key
, key
) == 0)
1243 idx
= next_idx(h
, idx
);
1246 #define bucket_scan(h, idx, key) base_bucket_scan(HASHMAP_BASE(h), idx, key)
1248 int hashmap_put(Hashmap
*h
, const void *key
, void *value
) {
1249 struct swap_entries swap
;
1250 struct plain_hashmap_entry
*e
;
1255 hash
= bucket_hash(h
, key
);
1256 idx
= bucket_scan(h
, hash
, key
);
1257 if (idx
!= IDX_NIL
) {
1258 e
= plain_bucket_at(h
, idx
);
1259 if (e
->value
== value
)
1264 e
= &bucket_at_swap(&swap
, IDX_PUT
)->p
;
1267 return hashmap_put_boldly(h
, hash
, &swap
, true);
1270 int set_put(Set
*s
, const void *key
) {
1271 struct swap_entries swap
;
1272 struct hashmap_base_entry
*e
;
1277 hash
= bucket_hash(s
, key
);
1278 idx
= bucket_scan(s
, hash
, key
);
1282 e
= &bucket_at_swap(&swap
, IDX_PUT
)->p
.b
;
1284 return hashmap_put_boldly(s
, hash
, &swap
, true);
1287 int hashmap_replace(Hashmap
*h
, const void *key
, void *value
) {
1288 struct swap_entries swap
;
1289 struct plain_hashmap_entry
*e
;
1294 hash
= bucket_hash(h
, key
);
1295 idx
= bucket_scan(h
, hash
, key
);
1296 if (idx
!= IDX_NIL
) {
1297 e
= plain_bucket_at(h
, idx
);
1298 #ifdef ENABLE_DEBUG_HASHMAP
1299 /* Although the key is equal, the key pointer may have changed,
1300 * and this would break our assumption for iterating. So count
1301 * this operation as incompatible with iteration. */
1302 if (e
->b
.key
!= key
) {
1303 h
->b
.debug
.put_count
++;
1304 h
->b
.debug
.rem_count
++;
1305 h
->b
.debug
.last_rem_idx
= idx
;
1313 e
= &bucket_at_swap(&swap
, IDX_PUT
)->p
;
1316 return hashmap_put_boldly(h
, hash
, &swap
, true);
1319 int hashmap_update(Hashmap
*h
, const void *key
, void *value
) {
1320 struct plain_hashmap_entry
*e
;
1325 hash
= bucket_hash(h
, key
);
1326 idx
= bucket_scan(h
, hash
, key
);
1330 e
= plain_bucket_at(h
, idx
);
1335 void *internal_hashmap_get(HashmapBase
*h
, const void *key
) {
1336 struct hashmap_base_entry
*e
;
1342 hash
= bucket_hash(h
, key
);
1343 idx
= bucket_scan(h
, hash
, key
);
1347 e
= bucket_at(h
, idx
);
1348 return entry_value(h
, e
);
1351 void *hashmap_get2(Hashmap
*h
, const void *key
, void **key2
) {
1352 struct plain_hashmap_entry
*e
;
1358 hash
= bucket_hash(h
, key
);
1359 idx
= bucket_scan(h
, hash
, key
);
1363 e
= plain_bucket_at(h
, idx
);
1365 *key2
= (void*) e
->b
.key
;
1370 bool internal_hashmap_contains(HashmapBase
*h
, const void *key
) {
1376 hash
= bucket_hash(h
, key
);
1377 return bucket_scan(h
, hash
, key
) != IDX_NIL
;
1380 void *internal_hashmap_remove(HashmapBase
*h
, const void *key
) {
1381 struct hashmap_base_entry
*e
;
1388 hash
= bucket_hash(h
, key
);
1389 idx
= bucket_scan(h
, hash
, key
);
1393 e
= bucket_at(h
, idx
);
1394 data
= entry_value(h
, e
);
1395 remove_entry(h
, idx
);
1400 void *hashmap_remove2(Hashmap
*h
, const void *key
, void **rkey
) {
1401 struct plain_hashmap_entry
*e
;
1411 hash
= bucket_hash(h
, key
);
1412 idx
= bucket_scan(h
, hash
, key
);
1413 if (idx
== IDX_NIL
) {
1419 e
= plain_bucket_at(h
, idx
);
1422 *rkey
= (void*) e
->b
.key
;
1424 remove_entry(h
, idx
);
1429 int hashmap_remove_and_put(Hashmap
*h
, const void *old_key
, const void *new_key
, void *value
) {
1430 struct swap_entries swap
;
1431 struct plain_hashmap_entry
*e
;
1432 unsigned old_hash
, new_hash
, idx
;
1437 old_hash
= bucket_hash(h
, old_key
);
1438 idx
= bucket_scan(h
, old_hash
, old_key
);
1442 new_hash
= bucket_hash(h
, new_key
);
1443 if (bucket_scan(h
, new_hash
, new_key
) != IDX_NIL
)
1446 remove_entry(h
, idx
);
1448 e
= &bucket_at_swap(&swap
, IDX_PUT
)->p
;
1451 assert_se(hashmap_put_boldly(h
, new_hash
, &swap
, false) == 1);
1456 int set_remove_and_put(Set
*s
, const void *old_key
, const void *new_key
) {
1457 struct swap_entries swap
;
1458 struct hashmap_base_entry
*e
;
1459 unsigned old_hash
, new_hash
, idx
;
1464 old_hash
= bucket_hash(s
, old_key
);
1465 idx
= bucket_scan(s
, old_hash
, old_key
);
1469 new_hash
= bucket_hash(s
, new_key
);
1470 if (bucket_scan(s
, new_hash
, new_key
) != IDX_NIL
)
1473 remove_entry(s
, idx
);
1475 e
= &bucket_at_swap(&swap
, IDX_PUT
)->p
.b
;
1477 assert_se(hashmap_put_boldly(s
, new_hash
, &swap
, false) == 1);
1482 int hashmap_remove_and_replace(Hashmap
*h
, const void *old_key
, const void *new_key
, void *value
) {
1483 struct swap_entries swap
;
1484 struct plain_hashmap_entry
*e
;
1485 unsigned old_hash
, new_hash
, idx_old
, idx_new
;
1490 old_hash
= bucket_hash(h
, old_key
);
1491 idx_old
= bucket_scan(h
, old_hash
, old_key
);
1492 if (idx_old
== IDX_NIL
)
1495 old_key
= bucket_at(HASHMAP_BASE(h
), idx_old
)->key
;
1497 new_hash
= bucket_hash(h
, new_key
);
1498 idx_new
= bucket_scan(h
, new_hash
, new_key
);
1499 if (idx_new
!= IDX_NIL
)
1500 if (idx_old
!= idx_new
) {
1501 remove_entry(h
, idx_new
);
1502 /* Compensate for a possible backward shift. */
1503 if (old_key
!= bucket_at(HASHMAP_BASE(h
), idx_old
)->key
)
1504 idx_old
= prev_idx(HASHMAP_BASE(h
), idx_old
);
1505 assert(old_key
== bucket_at(HASHMAP_BASE(h
), idx_old
)->key
);
1508 remove_entry(h
, idx_old
);
1510 e
= &bucket_at_swap(&swap
, IDX_PUT
)->p
;
1513 assert_se(hashmap_put_boldly(h
, new_hash
, &swap
, false) == 1);
1518 void *hashmap_remove_value(Hashmap
*h
, const void *key
, void *value
) {
1519 struct plain_hashmap_entry
*e
;
1525 hash
= bucket_hash(h
, key
);
1526 idx
= bucket_scan(h
, hash
, key
);
1530 e
= plain_bucket_at(h
, idx
);
1531 if (e
->value
!= value
)
1534 remove_entry(h
, idx
);
1539 static unsigned find_first_entry(HashmapBase
*h
) {
1540 Iterator i
= ITERATOR_FIRST
;
1542 if (!h
|| !n_entries(h
))
1545 return hashmap_iterate_entry(h
, &i
);
1548 void *internal_hashmap_first(HashmapBase
*h
) {
1551 idx
= find_first_entry(h
);
1555 return entry_value(h
, bucket_at(h
, idx
));
1558 void *internal_hashmap_first_key(HashmapBase
*h
) {
1559 struct hashmap_base_entry
*e
;
1562 idx
= find_first_entry(h
);
1566 e
= bucket_at(h
, idx
);
1567 return (void*) e
->key
;
1570 void *internal_hashmap_steal_first(HashmapBase
*h
) {
1571 struct hashmap_base_entry
*e
;
1575 idx
= find_first_entry(h
);
1579 e
= bucket_at(h
, idx
);
1580 data
= entry_value(h
, e
);
1581 remove_entry(h
, idx
);
1586 void *internal_hashmap_steal_first_key(HashmapBase
*h
) {
1587 struct hashmap_base_entry
*e
;
1591 idx
= find_first_entry(h
);
1595 e
= bucket_at(h
, idx
);
1596 key
= (void*) e
->key
;
1597 remove_entry(h
, idx
);
1602 unsigned internal_hashmap_size(HashmapBase
*h
) {
1607 return n_entries(h
);
1610 unsigned internal_hashmap_buckets(HashmapBase
*h
) {
1615 return n_buckets(h
);
1618 int internal_hashmap_merge(Hashmap
*h
, Hashmap
*other
) {
1624 HASHMAP_FOREACH_IDX(idx
, HASHMAP_BASE(other
), i
) {
1625 struct plain_hashmap_entry
*pe
= plain_bucket_at(other
, idx
);
1628 r
= hashmap_put(h
, pe
->b
.key
, pe
->value
);
1629 if (r
< 0 && r
!= -EEXIST
)
1636 int set_merge(Set
*s
, Set
*other
) {
1642 HASHMAP_FOREACH_IDX(idx
, HASHMAP_BASE(other
), i
) {
1643 struct set_entry
*se
= set_bucket_at(other
, idx
);
1646 r
= set_put(s
, se
->b
.key
);
1654 int internal_hashmap_reserve(HashmapBase
*h
, unsigned entries_add
) {
1659 r
= resize_buckets(h
, entries_add
);
1667 * The same as hashmap_merge(), but every new item from other is moved to h.
1668 * Keys already in h are skipped and stay in other.
1669 * Returns: 0 on success.
1670 * -ENOMEM on alloc failure, in which case no move has been done.
1672 int internal_hashmap_move(HashmapBase
*h
, HashmapBase
*other
) {
1673 struct swap_entries swap
;
1674 struct hashmap_base_entry
*e
, *n
;
1684 assert(other
->type
== h
->type
);
1687 * This reserves buckets for the worst case, where none of other's
1688 * entries are yet present in h. This is preferable to risking
1689 * an allocation failure in the middle of the moving and having to
1690 * rollback or return a partial result.
1692 r
= resize_buckets(h
, n_entries(other
));
1696 HASHMAP_FOREACH_IDX(idx
, other
, i
) {
1699 e
= bucket_at(other
, idx
);
1700 h_hash
= bucket_hash(h
, e
->key
);
1701 if (bucket_scan(h
, h_hash
, e
->key
) != IDX_NIL
)
1704 n
= &bucket_at_swap(&swap
, IDX_PUT
)->p
.b
;
1706 if (h
->type
!= HASHMAP_TYPE_SET
)
1707 ((struct plain_hashmap_entry
*) n
)->value
=
1708 ((struct plain_hashmap_entry
*) e
)->value
;
1709 assert_se(hashmap_put_boldly(h
, h_hash
, &swap
, false) == 1);
1711 remove_entry(other
, idx
);
1717 int internal_hashmap_move_one(HashmapBase
*h
, HashmapBase
*other
, const void *key
) {
1718 struct swap_entries swap
;
1719 unsigned h_hash
, other_hash
, idx
;
1720 struct hashmap_base_entry
*e
, *n
;
1725 h_hash
= bucket_hash(h
, key
);
1726 if (bucket_scan(h
, h_hash
, key
) != IDX_NIL
)
1732 assert(other
->type
== h
->type
);
1734 other_hash
= bucket_hash(other
, key
);
1735 idx
= bucket_scan(other
, other_hash
, key
);
1739 e
= bucket_at(other
, idx
);
1741 n
= &bucket_at_swap(&swap
, IDX_PUT
)->p
.b
;
1743 if (h
->type
!= HASHMAP_TYPE_SET
)
1744 ((struct plain_hashmap_entry
*) n
)->value
=
1745 ((struct plain_hashmap_entry
*) e
)->value
;
1746 r
= hashmap_put_boldly(h
, h_hash
, &swap
, true);
1750 remove_entry(other
, idx
);
1754 HashmapBase
*internal_hashmap_copy(HashmapBase
*h
) {
1760 copy
= hashmap_base_new(h
->hash_ops
, h
->type HASHMAP_DEBUG_SRC_ARGS
);
1765 case HASHMAP_TYPE_PLAIN
:
1766 case HASHMAP_TYPE_ORDERED
:
1767 r
= hashmap_merge((Hashmap
*)copy
, (Hashmap
*)h
);
1769 case HASHMAP_TYPE_SET
:
1770 r
= set_merge((Set
*)copy
, (Set
*)h
);
1773 assert_not_reached("Unknown hashmap type");
1777 internal_hashmap_free(copy
);
1784 char **internal_hashmap_get_strv(HashmapBase
*h
) {
1789 sv
= new(char*, n_entries(h
)+1);
1794 HASHMAP_FOREACH_IDX(idx
, h
, i
)
1795 sv
[n
++] = entry_value(h
, bucket_at(h
, idx
));
1801 void *ordered_hashmap_next(OrderedHashmap
*h
, const void *key
) {
1802 struct ordered_hashmap_entry
*e
;
1808 hash
= bucket_hash(h
, key
);
1809 idx
= bucket_scan(h
, hash
, key
);
1813 e
= ordered_bucket_at(h
, idx
);
1814 if (e
->iterate_next
== IDX_NIL
)
1816 return ordered_bucket_at(h
, e
->iterate_next
)->p
.value
;
1819 int set_consume(Set
*s
, void *value
) {
1822 r
= set_put(s
, value
);
1829 int set_put_strdup(Set
*s
, const char *p
) {
1840 r
= set_consume(s
, c
);
1847 int set_put_strdupv(Set
*s
, char **l
) {
1851 STRV_FOREACH(i
, l
) {
1852 r
= set_put_strdup(s
, *i
);