]>
Commit | Line | Data |
---|---|---|
1 | // SPDX-License-Identifier: GPL-2.0+ | |
2 | /* | |
3 | * Copyright (C) 2018 Oracle. All Rights Reserved. | |
4 | * Author: Darrick J. Wong <darrick.wong@oracle.com> | |
5 | */ | |
6 | #include <stdint.h> | |
7 | #include <stdlib.h> | |
8 | #include <stdbool.h> | |
9 | #include <string.h> | |
10 | #include <assert.h> | |
11 | #include <pthread.h> | |
12 | #include <unistd.h> | |
13 | #include "platform_defs.h" | |
14 | #include "ptvar.h" | |
15 | ||
16 | /* | |
17 | * Per-thread Variables | |
18 | * | |
19 | * This data structure manages a lockless per-thread variable. We | |
20 | * implement this by allocating an array of memory regions, and as each | |
21 | * thread tries to acquire its own region, we hand out the array | |
22 | * elements to each thread. This way, each thread gets its own | |
23 | * cacheline and (after the first access) doesn't have to contend for a | |
24 | * lock for each access. | |
25 | */ | |
26 | struct ptvar { | |
27 | pthread_key_t key; | |
28 | pthread_mutex_t lock; | |
29 | size_t nr_used; | |
30 | size_t nr_counters; | |
31 | size_t data_size; | |
32 | unsigned char data[0]; | |
33 | }; | |
34 | #define PTVAR_SIZE(nr, sz) (sizeof(struct ptvar) + ((nr) * (size))) | |
35 | ||
36 | /* Allocate a new per-thread counter. */ | |
37 | int | |
38 | ptvar_alloc( | |
39 | size_t nr, | |
40 | size_t size, | |
41 | struct ptvar **pptv) | |
42 | { | |
43 | struct ptvar *ptv; | |
44 | int ret; | |
45 | ||
46 | #ifdef _SC_LEVEL1_DCACHE_LINESIZE | |
47 | long l1_dcache; | |
48 | ||
49 | /* Try to prevent cache pingpong by aligning to cacheline size. */ | |
50 | l1_dcache = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); | |
51 | if (l1_dcache > 0) | |
52 | size = roundup(size, l1_dcache); | |
53 | #endif | |
54 | ||
55 | ptv = malloc(PTVAR_SIZE(nr, size)); | |
56 | if (!ptv) | |
57 | return errno; | |
58 | ptv->data_size = size; | |
59 | ptv->nr_counters = nr; | |
60 | ptv->nr_used = 0; | |
61 | memset(ptv->data, 0, nr * size); | |
62 | ret = pthread_mutex_init(&ptv->lock, NULL); | |
63 | if (ret) | |
64 | goto out; | |
65 | ret = pthread_key_create(&ptv->key, NULL); | |
66 | if (ret) | |
67 | goto out_mutex; | |
68 | ||
69 | *pptv = ptv; | |
70 | return 0; | |
71 | out_mutex: | |
72 | pthread_mutex_destroy(&ptv->lock); | |
73 | out: | |
74 | free(ptv); | |
75 | return ret; | |
76 | } | |
77 | ||
78 | /* Free per-thread counter. */ | |
79 | void | |
80 | ptvar_free( | |
81 | struct ptvar *ptv) | |
82 | { | |
83 | pthread_key_delete(ptv->key); | |
84 | pthread_mutex_destroy(&ptv->lock); | |
85 | free(ptv); | |
86 | } | |
87 | ||
88 | /* Get a reference to this thread's variable. */ | |
89 | void * | |
90 | ptvar_get( | |
91 | struct ptvar *ptv, | |
92 | int *retp) | |
93 | { | |
94 | void *p; | |
95 | int ret; | |
96 | ||
97 | p = pthread_getspecific(ptv->key); | |
98 | if (!p) { | |
99 | pthread_mutex_lock(&ptv->lock); | |
100 | assert(ptv->nr_used < ptv->nr_counters); | |
101 | p = &ptv->data[(ptv->nr_used++) * ptv->data_size]; | |
102 | ret = pthread_setspecific(ptv->key, p); | |
103 | if (ret) | |
104 | goto out_unlock; | |
105 | pthread_mutex_unlock(&ptv->lock); | |
106 | } | |
107 | *retp = 0; | |
108 | return p; | |
109 | ||
110 | out_unlock: | |
111 | ptv->nr_used--; | |
112 | pthread_mutex_unlock(&ptv->lock); | |
113 | *retp = ret; | |
114 | return NULL; | |
115 | } | |
116 | ||
117 | /* Iterate all of the per-thread variables. */ | |
118 | int | |
119 | ptvar_foreach( | |
120 | struct ptvar *ptv, | |
121 | ptvar_iter_fn fn, | |
122 | void *foreach_arg) | |
123 | { | |
124 | size_t i; | |
125 | int ret = 0; | |
126 | ||
127 | pthread_mutex_lock(&ptv->lock); | |
128 | for (i = 0; i < ptv->nr_used; i++) { | |
129 | ret = fn(ptv, &ptv->data[i * ptv->data_size], foreach_arg); | |
130 | if (ret) | |
131 | break; | |
132 | } | |
133 | pthread_mutex_unlock(&ptv->lock); | |
134 | ||
135 | return ret; | |
136 | } |