1 // SPDX-License-Identifier: GPL-2.0+
3 * Copyright (C) 2018 Oracle. All Rights Reserved.
4 * Author: Darrick J. Wong <darrick.wong@oracle.com>
13 #include "platform_defs.h"
17 * Per-thread Variables
19 * This data structure manages a lockless per-thread variable. We
20 * implement this by allocating an array of memory regions, and as each
21 * thread tries to acquire its own region, we hand out the array
22 * elements to each thread. This way, each thread gets its own
23 * cacheline and (after the first access) doesn't have to contend for a
24 * lock for each access.
32 unsigned char data
[0];
34 #define PTVAR_SIZE(nr, sz) (sizeof(struct ptvar) + ((nr) * (size)))
36 /* Allocate a new per-thread counter. */
46 #ifdef _SC_LEVEL1_DCACHE_LINESIZE
49 /* Try to prevent cache pingpong by aligning to cacheline size. */
50 l1_dcache
= sysconf(_SC_LEVEL1_DCACHE_LINESIZE
);
52 size
= roundup(size
, l1_dcache
);
55 ptv
= malloc(PTVAR_SIZE(nr
, size
));
58 ptv
->data_size
= size
;
59 ptv
->nr_counters
= nr
;
61 memset(ptv
->data
, 0, nr
* size
);
62 ret
= pthread_mutex_init(&ptv
->lock
, NULL
);
65 ret
= pthread_key_create(&ptv
->key
, NULL
);
72 pthread_mutex_destroy(&ptv
->lock
);
78 /* Free per-thread counter. */
83 pthread_key_delete(ptv
->key
);
84 pthread_mutex_destroy(&ptv
->lock
);
88 /* Get a reference to this thread's variable. */
97 p
= pthread_getspecific(ptv
->key
);
99 pthread_mutex_lock(&ptv
->lock
);
100 assert(ptv
->nr_used
< ptv
->nr_counters
);
101 p
= &ptv
->data
[(ptv
->nr_used
++) * ptv
->data_size
];
102 ret
= pthread_setspecific(ptv
->key
, p
);
105 pthread_mutex_unlock(&ptv
->lock
);
112 pthread_mutex_unlock(&ptv
->lock
);
117 /* Iterate all of the per-thread variables. */
127 pthread_mutex_lock(&ptv
->lock
);
128 for (i
= 0; i
< ptv
->nr_used
; i
++) {
129 ret
= fn(ptv
, &ptv
->data
[i
* ptv
->data_size
], foreach_arg
);
133 pthread_mutex_unlock(&ptv
->lock
);