GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/qdk

Public

mirrored from https://github.com/microsoft/qdkAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
82b4ff844af944291d836b7950ff682d76f4786a

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

qdk/source/allocator/mimalloc-sys/mimalloc/include/mimalloc

source/allocator/mimalloc-sys/mimalloc/include/mimalloc/atomic.h

557lines · modecode

RawDownload
Latest commit unavailable.
1/* ----------------------------------------------------------------------------
2Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#pragma once
8#ifndef MIMALLOC_ATOMIC_H
9#define MIMALLOC_ATOMIC_H
10
11// include windows.h or pthreads.h
12#if defined(_WIN32)
13#ifndef WIN32_LEAN_AND_MEAN
14#define WIN32_LEAN_AND_MEAN
15#endif
16#include <windows.h>
17#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
18#define MI_USE_PTHREADS
19#include <pthread.h>
20#endif
21
22// --------------------------------------------------------------------------------------------
23// Atomics
24// We need to be portable between C, C++, and MSVC.
25// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.
26// This is why we try to use only `uintptr_t` and `<type>*` as atomic types.
27// To gain better insight in the range of used atomics, we use explicitly named memory order operations
28// instead of passing the memory order as a parameter.
29// -----------------------------------------------------------------------------------------------
30
31#if defined(__cplusplus)
32// Use C++ atomics
33#include <atomic>
34#define _Atomic(tp) std::atomic<tp>
35#define mi_atomic(name) std::atomic_##name
36#define mi_memory_order(name) std::memory_order_##name
37#if (__cplusplus >= 202002L) // c++20, see issue #571
38 #define MI_ATOMIC_VAR_INIT(x) x
39#elif !defined(ATOMIC_VAR_INIT)
40 #define MI_ATOMIC_VAR_INIT(x) x
41#else
42 #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
43#endif
44#elif defined(_MSC_VER)
45// Use MSVC C wrapper for C11 atomics
46#define _Atomic(tp) tp
47#define MI_ATOMIC_VAR_INIT(x) x
48#define mi_atomic(name) mi_atomic_##name
49#define mi_memory_order(name) mi_memory_order_##name
50#else
51// Use C11 atomics
52#include <stdatomic.h>
53#define mi_atomic(name) atomic_##name
54#define mi_memory_order(name) memory_order_##name
55#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735
56 #define MI_ATOMIC_VAR_INIT(x) x
57#elif !defined(ATOMIC_VAR_INIT)
58 #define MI_ATOMIC_VAR_INIT(x) x
59#else
60 #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)
61#endif
62#endif
63
64// Various defines for all used memory orders in mimalloc
65#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \
66 mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)
67
68#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \
69 mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)
70
71#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
72#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
73#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
74#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
75#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))
76#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))
77#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))
78#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
79#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
80#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
81#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
82
83#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))
84#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))
85#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))
86#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))
87#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))
88#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))
89
90#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1)
91#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1)
92#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1)
93#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1)
94
95static inline void mi_atomic_yield(void);
96static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
97static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
98
99
100#if defined(__cplusplus) || !defined(_MSC_VER)
101
102// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)
103// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well
104#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p)
105#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p)
106
107// In C++ we need to add casts to help resolve templates if NULL is passed
108#if defined(__cplusplus)
109#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x)
110#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x)
111#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des)
112#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)
113#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des)
114#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)
115#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x)
116#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x)
117#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x)
118#else
119#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x)
120#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x)
121#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des)
122#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des)
123#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des)
124#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,des)
125#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x)
126#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x)
127#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x)
128#endif
129
130// These are used by the statistics
131static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {
132 return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
133}
134static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
135 const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);
136 if (add != 0) {
137 mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
138 }
139}
140static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
141 int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);
142 while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, &current, x)) { /* nothing */ };
143}
144
145// Used by timers
146#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))
147#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
148#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
149#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
150
151#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d)
152#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i)
153
154
155#elif defined(_MSC_VER)
156
157// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
158#include <intrin.h>
159#ifdef _WIN64
160typedef LONG64 msc_intptr_t;
161#define MI_64(f) f##64
162#else
163typedef LONG msc_intptr_t;
164#define MI_64(f) f
165#endif
166
167typedef enum mi_memory_order_e {
168 mi_memory_order_relaxed,
169 mi_memory_order_consume,
170 mi_memory_order_acquire,
171 mi_memory_order_release,
172 mi_memory_order_acq_rel,
173 mi_memory_order_seq_cst
174} mi_memory_order;
175
176static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {
177 (void)(mo);
178 return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
179}
180static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {
181 (void)(mo);
182 return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
183}
184static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
185 (void)(mo);
186 return (uintptr_t)MI_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
187}
188static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
189 (void)(mo);
190 return (uintptr_t)MI_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
191}
192static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
193 (void)(mo1); (void)(mo2);
194 uintptr_t read = (uintptr_t)MI_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
195 if (read == *expected) {
196 return true;
197 }
198 else {
199 *expected = read;
200 return false;
201 }
202}
203static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
204 return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);
205}
206static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {
207 (void)(mo);
208 return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
209}
210static inline void mi_atomic_thread_fence(mi_memory_order mo) {
211 (void)(mo);
212 _Atomic(uintptr_t) x = 0;
213 mi_atomic_exchange_explicit(&x, 1, mo);
214}
215static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
216 (void)(mo);
217#if defined(_M_IX86) || defined(_M_X64)
218 return *p;
219#else
220 uintptr_t x = *p;
221 if (mo > mi_memory_order_relaxed) {
222 while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
223 }
224 return x;
225#endif
226}
227static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
228 (void)(mo);
229#if defined(_M_IX86) || defined(_M_X64)
230 *p = x;
231#else
232 mi_atomic_exchange_explicit(p, x, mo);
233#endif
234}
235static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {
236 (void)(mo);
237#if defined(_M_X64)
238 return *p;
239#else
240 int64_t old = *p;
241 int64_t x = old;
242 while ((old = InterlockedCompareExchange64(p, x, old)) != x) {
243 x = old;
244 }
245 return x;
246#endif
247}
248static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
249 (void)(mo);
250#if defined(x_M_IX86) || defined(_M_X64)
251 *p = x;
252#else
253 InterlockedExchange64(p, x);
254#endif
255}
256
257// These are used by the statistics
258static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {
259#ifdef _WIN64
260 return (int64_t)mi_atomic_addi((int64_t*)p, add);
261#else
262 int64_t current;
263 int64_t sum;
264 do {
265 current = *p;
266 sum = current + add;
267 } while (_InterlockedCompareExchange64(p, sum, current) != current);
268 return current;
269#endif
270}
271static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
272 const int64_t add = *padd;
273 if (add != 0) {
274 mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);
275 }
276}
277
278static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {
279 int64_t current;
280 do {
281 current = *p;
282 } while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
283}
284
285static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
286 mi_atomic_addi64_relaxed(p, i);
287}
288
289static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
290 int64_t read = _InterlockedCompareExchange64(p, des, *exp);
291 if (read == *exp) {
292 return true;
293 }
294 else {
295 *exp = read;
296 return false;
297 }
298}
299
300// The pointer macros cast to `uintptr_t`.
301#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
302#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
303#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
304#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
305#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
306#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
307#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
308#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
309#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
310#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
311#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
312
313#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))
314#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))
315#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))
316#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))
317
318
319#endif
320
321
322// Atomically add a signed value; returns the previous value.
323static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {
324 return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);
325}
326
327// Atomically subtract a signed value; returns the previous value.
328static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
329 return (intptr_t)mi_atomic_addi(p, -sub);
330}
331
332
333// ----------------------------------------------------------------------
334// Once and Guard
335// ----------------------------------------------------------------------
336
337typedef _Atomic(uintptr_t) mi_atomic_once_t;
338
339// Returns true only on the first invocation
340static inline bool mi_atomic_once( mi_atomic_once_t* once ) {
341 if (mi_atomic_load_relaxed(once) != 0) return false; // quick test
342 uintptr_t expected = 0;
343 return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1
344}
345
346typedef _Atomic(uintptr_t) mi_atomic_guard_t;
347
348// Allows only one thread to execute at a time
349#define mi_atomic_guard(guard) \
350 uintptr_t _mi_guard_expected = 0; \
351 for(bool _mi_guard_once = true; \
352 _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \
353 (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
354
355
356
357// ----------------------------------------------------------------------
358// Yield
359// ----------------------------------------------------------------------
360
361#if defined(__cplusplus)
362#include <thread>
363static inline void mi_atomic_yield(void) {
364 std::this_thread::yield();
365}
366#elif defined(_WIN32)
367static inline void mi_atomic_yield(void) {
368 YieldProcessor();
369}
370#elif defined(__SSE2__)
371#include <emmintrin.h>
372static inline void mi_atomic_yield(void) {
373 _mm_pause();
374}
375#elif (defined(__GNUC__) || defined(__clang__)) && \
376 (defined(__x86_64__) || defined(__i386__) || \
377 defined(__aarch64__) || defined(__arm__) || \
378 defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__))
379#if defined(__x86_64__) || defined(__i386__)
380static inline void mi_atomic_yield(void) {
381 __asm__ volatile ("pause" ::: "memory");
382}
383#elif defined(__aarch64__)
384static inline void mi_atomic_yield(void) {
385 __asm__ volatile("wfe");
386}
387#elif defined(__arm__)
388#if __ARM_ARCH >= 7
389static inline void mi_atomic_yield(void) {
390 __asm__ volatile("yield" ::: "memory");
391}
392#else
393static inline void mi_atomic_yield(void) {
394 __asm__ volatile ("nop" ::: "memory");
395}
396#endif
397#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__)
398#ifdef __APPLE__
399static inline void mi_atomic_yield(void) {
400 __asm__ volatile ("or r27,r27,r27" ::: "memory");
401}
402#else
403static inline void mi_atomic_yield(void) {
404 __asm__ __volatile__ ("or 27,27,27" ::: "memory");
405}
406#endif
407#endif
408#elif defined(__sun)
409// Fallback for other archs
410#include <synch.h>
411static inline void mi_atomic_yield(void) {
412 smt_pause();
413}
414#elif defined(__wasi__)
415#include <sched.h>
416static inline void mi_atomic_yield(void) {
417 sched_yield();
418}
419#else
420#include <unistd.h>
421static inline void mi_atomic_yield(void) {
422 sleep(0);
423}
424#endif
425
426
427// ----------------------------------------------------------------------
428// Locks
429// These do not have to be recursive and should be light-weight
430// in-process only locks. Only used for reserving arena's and to
431// maintain the abandoned list.
432// ----------------------------------------------------------------------
433#if _MSC_VER
434#pragma warning(disable:26110) // unlock with holding lock
435#endif
436
437#define mi_lock(lock) for(bool _go = (mi_lock_acquire(lock),true); _go; (mi_lock_release(lock), _go=false) )
438
439#if defined(_WIN32)
440
441#if 1
442#define mi_lock_t SRWLOCK // slim reader-writer lock
443
444static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
445 return TryAcquireSRWLockExclusive(lock);
446}
447static inline void mi_lock_acquire(mi_lock_t* lock) {
448 AcquireSRWLockExclusive(lock);
449}
450static inline void mi_lock_release(mi_lock_t* lock) {
451 ReleaseSRWLockExclusive(lock);
452}
453static inline void mi_lock_init(mi_lock_t* lock) {
454 InitializeSRWLock(lock);
455}
456static inline void mi_lock_done(mi_lock_t* lock) {
457 (void)(lock);
458}
459
460#else
461#define mi_lock_t CRITICAL_SECTION
462
463static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
464 return TryEnterCriticalSection(lock);
465}
466static inline void mi_lock_acquire(mi_lock_t* lock) {
467 EnterCriticalSection(lock);
468}
469static inline void mi_lock_release(mi_lock_t* lock) {
470 LeaveCriticalSection(lock);
471}
472static inline void mi_lock_init(mi_lock_t* lock) {
473 InitializeCriticalSection(lock);
474}
475static inline void mi_lock_done(mi_lock_t* lock) {
476 DeleteCriticalSection(lock);
477}
478
479#endif
480
481#elif defined(MI_USE_PTHREADS)
482
483void _mi_error_message(int err, const char* fmt, ...);
484
485#define mi_lock_t pthread_mutex_t
486
487static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
488 return (pthread_mutex_trylock(lock) == 0);
489}
490static inline void mi_lock_acquire(mi_lock_t* lock) {
491 const int err = pthread_mutex_lock(lock);
492 if (err != 0) {
493 _mi_error_message(err, "internal error: lock cannot be acquired\n");
494 }
495}
496static inline void mi_lock_release(mi_lock_t* lock) {
497 pthread_mutex_unlock(lock);
498}
499static inline void mi_lock_init(mi_lock_t* lock) {
500 pthread_mutex_init(lock, NULL);
501}
502static inline void mi_lock_done(mi_lock_t* lock) {
503 pthread_mutex_destroy(lock);
504}
505
506#elif defined(__cplusplus)
507
508#include <mutex>
509#define mi_lock_t std::mutex
510
511static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
512 return lock->try_lock();
513}
514static inline void mi_lock_acquire(mi_lock_t* lock) {
515 lock->lock();
516}
517static inline void mi_lock_release(mi_lock_t* lock) {
518 lock->unlock();
519}
520static inline void mi_lock_init(mi_lock_t* lock) {
521 (void)(lock);
522}
523static inline void mi_lock_done(mi_lock_t* lock) {
524 (void)(lock);
525}
526
527#else
528
529// fall back to poor man's locks.
530// this should only be the case in a single-threaded environment (like __wasi__)
531
532#define mi_lock_t _Atomic(uintptr_t)
533
534static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
535 uintptr_t expected = 0;
536 return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1);
537}
538static inline void mi_lock_acquire(mi_lock_t* lock) {
539 for (int i = 0; i < 1000; i++) { // for at most 1000 tries?
540 if (mi_lock_try_acquire(lock)) return;
541 mi_atomic_yield();
542 }
543}
544static inline void mi_lock_release(mi_lock_t* lock) {
545 mi_atomic_store_release(lock, (uintptr_t)0);
546}
547static inline void mi_lock_init(mi_lock_t* lock) {
548 mi_lock_release(lock);
549}
550static inline void mi_lock_done(mi_lock_t* lock) {
551 (void)(lock);
552}
553
554#endif
555
556
557#endif // __MIMALLOC_ATOMIC_H