]>
Commit | Line | Data |
---|---|---|
959ef981 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
d11cc69e DW |
2 | /* |
3 | * Copyright (C) 2018 Oracle. All Rights Reserved. | |
d11cc69e | 4 | * Author: Darrick J. Wong <darrick.wong@oracle.com> |
d11cc69e DW |
5 | */ |
6 | #include <stdint.h> | |
7 | #include <stdlib.h> | |
8 | #include <stdbool.h> | |
9 | #include <string.h> | |
10 | #include <assert.h> | |
11 | #include <pthread.h> | |
12 | #include <unistd.h> | |
13 | #include "platform_defs.h" | |
14 | #include "ptvar.h" | |
15 | ||
16 | /* | |
17 | * Per-thread Variables | |
18 | * | |
19 | * This data structure manages a lockless per-thread variable. We | |
20 | * implement this by allocating an array of memory regions, and as each | |
21 | * thread tries to acquire its own region, we hand out the array | |
22 | * elements to each thread. This way, each thread gets its own | |
23 | * cacheline and (after the first access) doesn't have to contend for a | |
24 | * lock for each access. | |
25 | */ | |
26 | struct ptvar { | |
27 | pthread_key_t key; | |
28 | pthread_mutex_t lock; | |
29 | size_t nr_used; | |
30 | size_t nr_counters; | |
31 | size_t data_size; | |
32 | unsigned char data[0]; | |
33 | }; | |
34 | #define PTVAR_SIZE(nr, sz) (sizeof(struct ptvar) + ((nr) * (size))) | |
35 | ||
36 | /* Initialize per-thread counter. */ | |
37 | struct ptvar * | |
38 | ptvar_init( | |
39 | size_t nr, | |
40 | size_t size) | |
41 | { | |
42 | struct ptvar *ptv; | |
43 | int ret; | |
44 | ||
45 | #ifdef _SC_LEVEL1_DCACHE_LINESIZE | |
46 | /* Try to prevent cache pingpong by aligning to cacheline size. */ | |
47 | size = max(size, sysconf(_SC_LEVEL1_DCACHE_LINESIZE)); | |
48 | #endif | |
49 | ||
50 | ptv = malloc(PTVAR_SIZE(nr, size)); | |
51 | if (!ptv) | |
52 | return NULL; | |
53 | ptv->data_size = size; | |
54 | ptv->nr_counters = nr; | |
55 | ptv->nr_used = 0; | |
56 | memset(ptv->data, 0, nr * size); | |
57 | ret = pthread_mutex_init(&ptv->lock, NULL); | |
58 | if (ret) | |
59 | goto out; | |
60 | ret = pthread_key_create(&ptv->key, NULL); | |
61 | if (ret) | |
62 | goto out_mutex; | |
63 | return ptv; | |
64 | ||
65 | out_mutex: | |
66 | pthread_mutex_destroy(&ptv->lock); | |
67 | out: | |
68 | free(ptv); | |
69 | return NULL; | |
70 | } | |
71 | ||
72 | /* Free per-thread counter. */ | |
73 | void | |
74 | ptvar_free( | |
75 | struct ptvar *ptv) | |
76 | { | |
77 | pthread_key_delete(ptv->key); | |
78 | pthread_mutex_destroy(&ptv->lock); | |
79 | free(ptv); | |
80 | } | |
81 | ||
82 | /* Get a reference to this thread's variable. */ | |
83 | void * | |
84 | ptvar_get( | |
85 | struct ptvar *ptv) | |
86 | { | |
87 | void *p; | |
88 | ||
89 | p = pthread_getspecific(ptv->key); | |
90 | if (!p) { | |
91 | pthread_mutex_lock(&ptv->lock); | |
92 | assert(ptv->nr_used < ptv->nr_counters); | |
93 | p = &ptv->data[(ptv->nr_used++) * ptv->data_size]; | |
94 | pthread_setspecific(ptv->key, p); | |
95 | pthread_mutex_unlock(&ptv->lock); | |
96 | } | |
97 | return p; | |
98 | } | |
99 | ||
100 | /* Iterate all of the per-thread variables. */ | |
101 | bool | |
102 | ptvar_foreach( | |
103 | struct ptvar *ptv, | |
104 | ptvar_iter_fn fn, | |
105 | void *foreach_arg) | |
106 | { | |
107 | size_t i; | |
108 | bool ret = true; | |
109 | ||
110 | pthread_mutex_lock(&ptv->lock); | |
111 | for (i = 0; i < ptv->nr_used; i++) { | |
112 | ret = fn(ptv, &ptv->data[i * ptv->data_size], foreach_arg); | |
113 | if (!ret) | |
114 | break; | |
115 | } | |
116 | pthread_mutex_unlock(&ptv->lock); | |
117 | ||
118 | return ret; | |
119 | } |