]>
Commit | Line | Data |
---|---|---|
959ef981 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
d11cc69e DW |
2 | /* |
3 | * Copyright (C) 2018 Oracle. All Rights Reserved. | |
d11cc69e | 4 | * Author: Darrick J. Wong <darrick.wong@oracle.com> |
d11cc69e DW |
5 | */ |
6 | #include <stdint.h> | |
7 | #include <stdlib.h> | |
8 | #include <stdbool.h> | |
9 | #include <string.h> | |
10 | #include <assert.h> | |
11 | #include <pthread.h> | |
12 | #include <unistd.h> | |
13 | #include "platform_defs.h" | |
14 | #include "ptvar.h" | |
15 | ||
16 | /* | |
17 | * Per-thread Variables | |
18 | * | |
19 | * This data structure manages a lockless per-thread variable. We | |
20 | * implement this by allocating an array of memory regions, and as each | |
21 | * thread tries to acquire its own region, we hand out the array | |
22 | * elements to each thread. This way, each thread gets its own | |
23 | * cacheline and (after the first access) doesn't have to contend for a | |
24 | * lock for each access. | |
25 | */ | |
26 | struct ptvar { | |
27 | pthread_key_t key; | |
28 | pthread_mutex_t lock; | |
29 | size_t nr_used; | |
30 | size_t nr_counters; | |
31 | size_t data_size; | |
32 | unsigned char data[0]; | |
33 | }; | |
34 | #define PTVAR_SIZE(nr, sz) (sizeof(struct ptvar) + ((nr) * (size))) | |
35 | ||
cb321a39 DW |
36 | /* Allocate a new per-thread counter. */ |
37 | int | |
38 | ptvar_alloc( | |
d11cc69e | 39 | size_t nr, |
cb321a39 DW |
40 | size_t size, |
41 | struct ptvar **pptv) | |
d11cc69e DW |
42 | { |
43 | struct ptvar *ptv; | |
44 | int ret; | |
45 | ||
46 | #ifdef _SC_LEVEL1_DCACHE_LINESIZE | |
336c4824 DW |
47 | long l1_dcache; |
48 | ||
d11cc69e | 49 | /* Try to prevent cache pingpong by aligning to cacheline size. */ |
336c4824 DW |
50 | l1_dcache = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); |
51 | if (l1_dcache > 0) | |
52 | size = roundup(size, l1_dcache); | |
d11cc69e DW |
53 | #endif |
54 | ||
55 | ptv = malloc(PTVAR_SIZE(nr, size)); | |
56 | if (!ptv) | |
cb321a39 | 57 | return errno; |
d11cc69e DW |
58 | ptv->data_size = size; |
59 | ptv->nr_counters = nr; | |
60 | ptv->nr_used = 0; | |
61 | memset(ptv->data, 0, nr * size); | |
62 | ret = pthread_mutex_init(&ptv->lock, NULL); | |
63 | if (ret) | |
64 | goto out; | |
65 | ret = pthread_key_create(&ptv->key, NULL); | |
66 | if (ret) | |
67 | goto out_mutex; | |
d11cc69e | 68 | |
cb321a39 DW |
69 | *pptv = ptv; |
70 | return 0; | |
d11cc69e DW |
71 | out_mutex: |
72 | pthread_mutex_destroy(&ptv->lock); | |
73 | out: | |
74 | free(ptv); | |
cb321a39 | 75 | return ret; |
d11cc69e DW |
76 | } |
77 | ||
78 | /* Free per-thread counter. */ | |
79 | void | |
80 | ptvar_free( | |
81 | struct ptvar *ptv) | |
82 | { | |
83 | pthread_key_delete(ptv->key); | |
84 | pthread_mutex_destroy(&ptv->lock); | |
85 | free(ptv); | |
86 | } | |
87 | ||
88 | /* Get a reference to this thread's variable. */ | |
89 | void * | |
90 | ptvar_get( | |
cb321a39 DW |
91 | struct ptvar *ptv, |
92 | int *retp) | |
d11cc69e DW |
93 | { |
94 | void *p; | |
336c4824 | 95 | int ret; |
d11cc69e DW |
96 | |
97 | p = pthread_getspecific(ptv->key); | |
98 | if (!p) { | |
99 | pthread_mutex_lock(&ptv->lock); | |
100 | assert(ptv->nr_used < ptv->nr_counters); | |
101 | p = &ptv->data[(ptv->nr_used++) * ptv->data_size]; | |
336c4824 DW |
102 | ret = pthread_setspecific(ptv->key, p); |
103 | if (ret) | |
104 | goto out_unlock; | |
d11cc69e DW |
105 | pthread_mutex_unlock(&ptv->lock); |
106 | } | |
cb321a39 | 107 | *retp = 0; |
d11cc69e | 108 | return p; |
336c4824 DW |
109 | |
110 | out_unlock: | |
111 | ptv->nr_used--; | |
112 | pthread_mutex_unlock(&ptv->lock); | |
113 | *retp = ret; | |
114 | return NULL; | |
d11cc69e DW |
115 | } |
116 | ||
117 | /* Iterate all of the per-thread variables. */ | |
cb321a39 | 118 | int |
d11cc69e DW |
119 | ptvar_foreach( |
120 | struct ptvar *ptv, | |
121 | ptvar_iter_fn fn, | |
122 | void *foreach_arg) | |
123 | { | |
124 | size_t i; | |
cb321a39 | 125 | int ret = 0; |
d11cc69e DW |
126 | |
127 | pthread_mutex_lock(&ptv->lock); | |
128 | for (i = 0; i < ptv->nr_used; i++) { | |
129 | ret = fn(ptv, &ptv->data[i * ptv->data_size], foreach_arg); | |
cb321a39 | 130 | if (ret) |
d11cc69e DW |
131 | break; |
132 | } | |
133 | pthread_mutex_unlock(&ptv->lock); | |
134 | ||
135 | return ret; | |
136 | } |