include/internal/refcount.h

   1 /*
   2  * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9 #ifndef OSSL_INTERNAL_REFCOUNT_H
  10 # define OSSL_INTERNAL_REFCOUNT_H
  11 # pragma once
  12
  13 # include <openssl/e_os2.h>
  14 # include <openssl/trace.h>
  15 # include <openssl/err.h>
  16
  17 # if defined(OPENSSL_THREADS) && !defined(OPENSSL_DEV_NO_ATOMICS)
  18 #  if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
  19       && !defined(__STDC_NO_ATOMICS__)
  20 #   include <stdatomic.h>
  21 #   define HAVE_C11_ATOMICS
  22 #  endif
  23
  24 #  if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \
  25       && ATOMIC_INT_LOCK_FREE > 0
  26
  27 #   define HAVE_ATOMICS 1
  28
  29 typedef struct {
  30     _Atomic int val;
  31 } CRYPTO_REF_COUNT;
  32
  33 static inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  34 {
  35     *ret = atomic_fetch_add_explicit(&refcnt->val, 1, memory_order_relaxed) + 1;
  36     return 1;
  37 }
  38
  39 /*
  40  * Changes to shared structure other than reference counter have to be
  41  * serialized. And any kind of serialization implies a release fence. This
  42  * means that by the time reference counter is decremented all other
  43  * changes are visible on all processors. Hence decrement itself can be
  44  * relaxed. In case it hits zero, object will be destructed. Since it's
  45  * last use of the object, destructor programmer might reason that access
  46  * to mutable members doesn't have to be serialized anymore, which would
  47  * otherwise imply an acquire fence. Hence conditional acquire fence...
  48  */
  49 static inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  50 {
  51     *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_relaxed) - 1;
  52     if (*ret == 0)
  53         atomic_thread_fence(memory_order_acquire);
  54     return 1;
  55 }
  56
  57 static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  58 {
  59     *ret = atomic_load_explicit(&refcnt->val, memory_order_relaxed);
  60     return 1;
  61 }
  62
  63 #  elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0
  64
  65 #   define HAVE_ATOMICS 1
  66
  67 typedef struct {
  68     int val;
  69 } CRYPTO_REF_COUNT;
  70
  71 static __inline__ int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  72 {
  73     *ret = __atomic_fetch_add(&refcnt->val, 1, __ATOMIC_RELAXED) + 1;
  74     return 1;
  75 }
  76
  77 static __inline__ int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  78 {
  79     *ret = __atomic_fetch_sub(&refcnt->val, 1, __ATOMIC_RELAXED) - 1;
  80     if (*ret == 0)
  81         __atomic_thread_fence(__ATOMIC_ACQUIRE);
  82     return 1;
  83 }
  84
  85 static __inline__ int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  86 {
  87     *ret = __atomic_load_n(&refcnt->val, __ATOMIC_RELAXED);
  88     return 1;
  89 }
  90
  91 #  elif defined(__ICL) && defined(_WIN32)
  92 #   define HAVE_ATOMICS 1
  93
  94 typedef struct {
  95     volatile int val;
  96 } CRYPTO_REF_COUNT;
  97
  98 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
  99 {
 100     *ret = _InterlockedExchangeAdd((void *)&refcnt->val, 1) + 1;
 101     return 1;
 102 }
 103
 104 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *val, int *refcnt)
 105 {
 106     *ret = _InterlockedExchangeAdd((void *)&refcnt->val, -1) - 1;
 107     return 1;
 108 }
 109
 110 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 111 {
 112     *ret = _InterlockedOr((void *)&refcnt->val, 0);
 113     return 1;
 114 }
 115
 116 #  elif defined(_MSC_VER) && _MSC_VER>=1200
 117
 118 #   define HAVE_ATOMICS 1
 119
 120 typedef struct {
 121     volatile int val;
 122 } CRYPTO_REF_COUNT;
 123
 124 #   if (defined(_M_ARM) && _M_ARM>=7 && !defined(_WIN32_WCE)) || defined(_M_ARM64)
 125 #    include <intrin.h>
 126 #    if defined(_M_ARM64) && !defined(_ARM_BARRIER_ISH)
 127 #     define _ARM_BARRIER_ISH _ARM64_BARRIER_ISH
 128 #    endif
 129
 130 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 131 {
 132     *ret = _InterlockedExchangeAdd_nf(&refcnt->val, 1) + 1;
 133     return 1;
 134 }
 135
 136 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 137 {
 138     *ret = _InterlockedExchangeAdd_nf(&refcnt->val, -1) - 1;
 139     if (*ret == 0)
 140         __dmb(_ARM_BARRIER_ISH);
 141     return 1;
 142 }
 143
 144 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 145 {
 146     *ret = _InterlockedOr_nf((void *)&refcnt->val, 0);
 147     return 1;
 148 }
 149
 150 #   else
 151 #    if !defined(_WIN32_WCE)
 152 #     pragma intrinsic(_InterlockedExchangeAdd)
 153 #    else
 154 #     if _WIN32_WCE >= 0x600
 155        extern long __cdecl _InterlockedExchangeAdd(long volatile*, long);
 156 #     else
 157        /* under Windows CE we still have old-style Interlocked* functions */
 158        extern long __cdecl InterlockedExchangeAdd(long volatile*, long);
 159 #      define _InterlockedExchangeAdd InterlockedExchangeAdd
 160 #     endif
 161 #    endif
 162
 163 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 164 {
 165     *ret = _InterlockedExchangeAdd(&refcnt->val, 1) + 1;
 166     return 1;
 167 }
 168
 169 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 170 {
 171     *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1;
 172     return 1;
 173 }
 174
 175 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
 176 {
 177     *ret = _InterlockedExchangeAdd(&refcnt->val, 0);
 178     return 1;
 179 }
 180
 181 #   endif
 182
 183 #  endif
 184 # endif  /* !OPENSSL_DEV_NO_ATOMICS */
 185
 186 /*
 187  * All the refcounting implementations above define HAVE_ATOMICS, so if it's
 188  * still undefined here (such as when OPENSSL_DEV_NO_ATOMICS is defined), it
 189  * means we need to implement a fallback.  This fallback uses locks.
 190  */
 191 # ifndef HAVE_ATOMICS
 192
 193 typedef struct {
 194     int val;
 195 #  ifdef OPENSSL_THREADS
 196     CRYPTO_RWLOCK *lock;
 197 #  endif
 198 } CRYPTO_REF_COUNT;
 199
 200 #  ifdef OPENSSL_THREADS
 201
 202 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt,
 203                                                  int *ret)
 204 {
 205     return CRYPTO_atomic_add(&refcnt->val, 1, ret, refcnt->lock);
 206 }
 207
 208 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt,
 209                                                    int *ret)
 210 {
 211     return CRYPTO_atomic_add(&refcnt->val, -1, ret, refcnt->lock);
 212 }
 213
 214 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt,
 215                                                    int *ret)
 216 {
 217     return CRYPTO_atomic_load_int(&refcnt->val, ret, refcnt->lock);
 218 }
 219
 220 #   define CRYPTO_NEW_FREE_DEFINED  1
 221 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n)
 222 {
 223     refcnt->val = n;
 224     refcnt->lock = CRYPTO_THREAD_lock_new();
 225     if (refcnt->lock == NULL) {
 226         ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB);
 227         return 0;
 228     }
 229     return 1;
 230 }
 231
 232 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt)                                  \
 233 {
 234     if (refcnt != NULL)
 235         CRYPTO_THREAD_lock_free(refcnt->lock);
 236 }
 237
 238 #  else     /* OPENSSL_THREADS */
 239
 240 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt,
 241                                                  int *ret)
 242 {
 243     refcnt->val++;
 244     *ret = refcnt->val;
 245     return 1;
 246 }
 247
 248 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt,
 249                                                    int *ret)
 250 {
 251     refcnt->val--;
 252     *ret = refcnt->val;
 253     return 1;
 254 }
 255
 256 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt,
 257                                                    int *ret)
 258 {
 259     *ret = refcnt->val;
 260     return 1;
 261 }
 262
 263 #  endif    /* OPENSSL_THREADS */
 264 # endif
 265
 266 # ifndef CRYPTO_NEW_FREE_DEFINED
 267 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n)
 268 {
 269     refcnt->val = n;
 270     return 1;
 271 }
 272
 273 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt)                                  \
 274 {
 275 }
 276 # endif /* CRYPTO_NEW_FREE_DEFINED */
 277 #undef CRYPTO_NEW_FREE_DEFINED
 278
 279 # if !defined(NDEBUG) && !defined(OPENSSL_NO_STDIO)
 280 #  define REF_ASSERT_ISNT(test) \
 281     (void)((test) ? (OPENSSL_die("refcount error", __FILE__, __LINE__), 1) : 0)
 282 # else
 283 #  define REF_ASSERT_ISNT(i)
 284 # endif
 285
 286 # define REF_PRINT_EX(text, count, object) \
 287     OSSL_TRACE3(REF_COUNT, "%p:%4d:%s\n", (object), (count), (text));
 288 # define REF_PRINT_COUNT(text, object) \
 289     REF_PRINT_EX(text, object->references.val, (void *)object)
 290
 291 #endif