libphobos/libdruntime/gcc/emutls.d

   1 // GNU D Compiler emulated TLS routines.
   2 // Copyright (C) 2019-2022 Free Software Foundation, Inc.
   3
   4 // GCC is free software; you can redistribute it and/or modify it under
   5 // the terms of the GNU General Public License as published by the Free
   6 // Software Foundation; either version 3, or (at your option) any later
   7 // version.
   8
   9 // GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  10 // WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // for more details.
  13
  14 // Under Section 7 of GPL version 3, you are granted additional
  15 // permissions described in the GCC Runtime Library Exception, version
  16 // 3.1, as published by the Free Software Foundation.
  17
  18 // You should have received a copy of the GNU General Public License and
  19 // a copy of the GCC Runtime Library Exception along with this program;
  20 // see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  21 // <http://www.gnu.org/licenses/>.
  22
  23 // This code is based on the libgcc emutls.c emulated TLS support.
  24
  25 module gcc.emutls;
  26
  27 import core.atomic, core.stdc.stdlib, core.stdc.string, core.sync.mutex;
  28 import core.internal.container.array;
  29 import core.internal.container.hashtab;
  30 import core.internal.traits : classInstanceAlignment;
  31 import gcc.builtins, gcc.gthread;
  32
  33 version (GNU_EMUTLS): private:
  34
  35 alias word = __builtin_machine_uint;
  36 alias pointer = __builtin_pointer_uint;
  37 alias TlsArray = Array!(void**);
  38
  39 /*
  40  * TLS control data emitted by GCC for every TLS variable.
  41  */
  42 struct __emutls_object
  43 {
  44     word size;
  45     word align_;
  46     union
  47     {
  48         pointer offset;
  49         void* ptr;
  50     }
  51
  52     ubyte* templ;
  53 }
  54
  55 // Per-thread key to obtain the per-thread TLS variable array
  56 __gshared __gthread_key_t emutlsKey;
  57 // Largest, currently assigned TLS variable offset
  58 __gshared pointer emutlsMaxOffset = 0;
  59 // Contains the size of the TLS variables (for GC)
  60 __gshared Array!word emutlsSizes;
  61 // Contains the TLS variable array for single-threaded apps
  62 __gshared TlsArray singleArray;
  63 // List of all currently alive TlsArrays (for GC)
  64 __gshared HashTab!(TlsArray*, TlsArray*) emutlsArrays;
  65
  66 // emutlsMutex Mutex + @nogc handling
  67 enum mutexAlign = classInstanceAlignment!Mutex;
  68 enum mutexClassInstanceSize = __traits(classInstanceSize, Mutex);
  69 __gshared align(mutexAlign) void[mutexClassInstanceSize] _emutlsMutex;
  70
  71 @property Mutex emutlsMutex() nothrow @nogc
  72 {
  73     return cast(Mutex) _emutlsMutex.ptr;
  74 }
  75
  76 /*
  77  * Global (de)initialization functions
  78  */
  79 extern (C) void _d_emutls_init() nothrow @nogc
  80 {
  81     memcpy(_emutlsMutex.ptr, typeid(Mutex).initializer.ptr, _emutlsMutex.length);
  82     (cast(Mutex) _emutlsMutex.ptr).__ctor();
  83
  84     if (__gthread_key_create(&emutlsKey, &emutlsDestroyThread) != 0)
  85         abort();
  86 }
  87
  88 __gshared __gthread_once_t initOnce = GTHREAD_ONCE_INIT;
  89
  90 /*
  91  * emutls main entrypoint, called by GCC for each TLS variable access.
  92  */
  93 extern (C) void* __emutls_get_address(shared __emutls_object* obj) nothrow @nogc
  94 {
  95     pointer offset;
  96     if (__gthread_active_p())
  97     {
  98         // Obtain the offset index into the TLS array (same for all-threads)
  99         // for requested var. If it is unset, obtain a new offset index.
 100         offset = atomicLoad!(MemoryOrder.acq, pointer)(obj.offset);
 101         if (__builtin_expect(offset == 0, 0))
 102         {
 103             __gthread_once(&initOnce, &_d_emutls_init);
 104             emutlsMutex.lock_nothrow();
 105
 106             offset = obj.offset;
 107             if (offset == 0)
 108             {
 109                 offset = ++emutlsMaxOffset;
 110
 111                 emutlsSizes.ensureLength(offset);
 112                 // Note: it's important that we copy any data from obj and
 113                 // do not keep an reference to obj itself: If a library is
 114                 // unloaded, its tls variables are not removed from the arrays
 115                 // and the GC will still scan these. If we then try to reference
 116                 // a pointer to the data segment of an unloaded library, this
 117                 // will crash.
 118                 emutlsSizes[offset - 1] = obj.size;
 119
 120                 atomicStore!(MemoryOrder.rel, pointer)(obj.offset, offset);
 121             }
 122             emutlsMutex.unlock_nothrow();
 123         }
 124     }
 125     // For single-threaded systems, don't synchronize
 126     else
 127     {
 128         if (__builtin_expect(obj.offset == 0, 0))
 129         {
 130             offset = ++emutlsMaxOffset;
 131
 132             emutlsSizes.ensureLength(offset);
 133             emutlsSizes[offset - 1] = obj.size;
 134
 135             obj.offset = offset;
 136         }
 137     }
 138
 139     TlsArray* arr;
 140     if (__gthread_active_p())
 141         arr = cast(TlsArray*) __gthread_getspecific(emutlsKey);
 142     else
 143         arr = &singleArray;
 144
 145     // This will always be false for singleArray
 146     if (__builtin_expect(arr == null, 0))
 147     {
 148         arr = mallocTlsArray(offset);
 149         __gthread_setspecific(emutlsKey, arr);
 150         emutlsMutex.lock_nothrow();
 151         emutlsArrays[arr] = arr;
 152         emutlsMutex.unlock_nothrow();
 153     }
 154     // Check if we have to grow the per-thread array
 155     else if (__builtin_expect(offset > arr.length, 0))
 156     {
 157         (*arr).ensureLength(offset);
 158     }
 159
 160     // Offset 0 is used as a not-initialized marker above. In the
 161     // TLS array, we start at 0.
 162     auto index = offset - 1;
 163
 164     // Get the per-thread pointer from the TLS array
 165     void** ret = (*arr)[index];
 166     if (__builtin_expect(ret == null, 0))
 167     {
 168         // Initial access, have to allocate the storage
 169         ret = emutlsAlloc(obj);
 170         (*arr)[index] = ret;
 171     }
 172
 173     return ret;
 174 }
 175
 176 // 1:1 copy from libgcc emutls.c
 177 extern (C) void __emutls_register_common(__emutls_object* obj, word size, word align_, ubyte* templ) nothrow @nogc
 178 {
 179     if (obj.size < size)
 180     {
 181         obj.size = size;
 182         obj.templ = null;
 183     }
 184     if (obj.align_ < align_)
 185         obj.align_ = align_;
 186     if (templ && size == obj.size)
 187         obj.templ = templ;
 188 }
 189
 190 // 1:1 copy from libgcc emutls.c
 191 void** emutlsAlloc(shared __emutls_object* obj) nothrow @nogc
 192 {
 193     void* ptr;
 194     void* ret;
 195     enum pointerSize = (void*).sizeof;
 196
 197     /* We could use here posix_memalign if available and adjust
 198      emutls_destroy accordingly.  */
 199     if ((cast() obj).align_ <= pointerSize)
 200     {
 201         ptr = malloc((cast() obj).size + pointerSize);
 202         if (ptr == null)
 203             abort();
 204         (cast(void**) ptr)[0] = ptr;
 205         ret = ptr + pointerSize;
 206     }
 207     else
 208     {
 209         ptr = malloc(obj.size + pointerSize + obj.align_ - 1);
 210         if (ptr == null)
 211             abort();
 212         ret = cast(void*)((cast(pointer)(ptr + pointerSize + obj.align_ - 1)) & ~cast(
 213                 pointer)(obj.align_ - 1));
 214         (cast(void**) ret)[-1] = ptr;
 215     }
 216
 217     if (obj.templ)
 218         memcpy(ret, cast(ubyte*) obj.templ, cast() obj.size);
 219     else
 220         memset(ret, 0, cast() obj.size);
 221
 222     return cast(void**) ret;
 223 }
 224
 225 /*
 226  * When a thread has finished, remove the TLS array from the GC
 227  * scan list emutlsArrays, free all allocated TLS variables and
 228  * finally free the array.
 229  */
 230 extern (C) void emutlsDestroyThread(void* ptr) nothrow @nogc
 231 {
 232     auto arr = cast(TlsArray*) ptr;
 233
 234     foreach (entry; *arr)
 235     {
 236         if (entry)
 237             free(entry[-1]);
 238     }
 239
 240     free(arr);
 241 }
 242
 243 /*
 244  * Allocate a new TLS array, set length according to offset.
 245  */
 246 TlsArray* mallocTlsArray(pointer offset = 0) nothrow @nogc
 247 {
 248     static assert(TlsArray.alignof == (void*).alignof);
 249     void[] data = malloc(TlsArray.sizeof)[0 .. TlsArray.sizeof];
 250     if (data.ptr == null)
 251         abort();
 252
 253     static immutable TlsArray init = TlsArray.init;
 254     memcpy(data.ptr, &init, data.length);
 255     (cast(TlsArray*) data).length = 32;
 256     return cast(TlsArray*) data.ptr;
 257 }
 258
 259 /*
 260  * Make sure array is large enough to hold an entry for offset.
 261  * Note: the array index will be offset - 1!
 262  */
 263 void ensureLength(Value)(ref Array!(Value) arr, size_t offset) nothrow @nogc
 264 {
 265     // index is offset-1
 266     if (offset > arr.length)
 267     {
 268         auto newSize = arr.length * 2;
 269         if (offset > newSize)
 270             newSize = offset + 32;
 271         arr.length = newSize;
 272     }
 273 }
 274
 275 // Public interface
 276 public:
 277 void _d_emutls_scan(scope void delegate(void* pbeg, void* pend) nothrow cb) nothrow
 278 {
 279     void scanArray(scope TlsArray* arr) nothrow
 280     {
 281         foreach (index, entry; *arr)
 282         {
 283             auto ptr = cast(void*) entry;
 284             if (ptr)
 285                 cb(ptr, ptr + emutlsSizes[index]);
 286         }
 287     }
 288
 289     __gthread_once(&initOnce, &_d_emutls_init);
 290     emutlsMutex.lock_nothrow();
 291     // this code is effectively nothrow
 292     try
 293     {
 294         foreach (arr, value; emutlsArrays)
 295         {
 296             scanArray(arr);
 297         }
 298     }
 299     catch (Exception)
 300     {
 301     }
 302     emutlsMutex.unlock_nothrow();
 303     scanArray(&singleArray);
 304 }
 305
 306 // Call this after druntime has been unloaded
 307 void _d_emutls_destroy() nothrow @nogc
 308 {
 309     (cast(Mutex) _emutlsMutex.ptr).__dtor();
 310     destroy(emutlsArrays);
 311 }