GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/qdk

Public

mirrored from https://github.com/microsoft/qdkAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
ec2026db2019aec0d91ed79e76b96e7bf21db842

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

qdk/allocator/mimalloc-sys/mimalloc/include/mimalloc

allocator/mimalloc-sys/mimalloc/include/mimalloc/prim.h

323lines · modecode

RawDownload
Latest commit unavailable.
1/* ----------------------------------------------------------------------------
2Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#pragma once
8#ifndef MIMALLOC_PRIM_H
9#define MIMALLOC_PRIM_H
10
11
12// --------------------------------------------------------------------------
13// This file specifies the primitive portability API.
14// Each OS/host needs to implement these primitives, see `src/prim`
15// for implementations on Window, macOS, WASI, and Linux/Unix.
16//
17// note: on all primitive functions, we always have result parameters != NUL, and:
18// addr != NULL and page aligned
19// size > 0 and page aligned
20// return value is an error code an int where 0 is success.
21// --------------------------------------------------------------------------
22
23// OS memory configuration
24typedef struct mi_os_mem_config_s {
25 size_t page_size; // 4KiB
26 size_t large_page_size; // 2MiB
27 size_t alloc_granularity; // smallest allocation size (on Windows 64KiB)
28 bool has_overcommit; // can we reserve more memory than can be actually committed?
29 bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc)
30 bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
31} mi_os_mem_config_t;
32
33// Initialize
34void _mi_prim_mem_init( mi_os_mem_config_t* config );
35
36// Free OS memory
37int _mi_prim_free(void* addr, size_t size );
38
39// Allocate OS memory. Return NULL on error.
40// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.
41// If `commit` is false, the virtual memory range only needs to be reserved (with no access)
42// which will later be committed explicitly using `_mi_prim_commit`.
43// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
44// pre: !commit => !allow_large
45// try_alignment >= _mi_os_page_size() and a power of 2
46int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
47
48// Commit memory. Returns error code or 0 on success.
49// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
50// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)
51int _mi_prim_commit(void* addr, size_t size, bool* is_zero);
52
53// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true
54// if the memory would need to be re-committed. For example, on Windows this is always true,
55// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.
56// pre: needs_recommit != NULL
57int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);
58
59// Reset memory. The range keeps being accessible but the content might be reset.
60// Returns error code or 0 on success.
61int _mi_prim_reset(void* addr, size_t size);
62
63// Protect memory. Returns error code or 0 on success.
64int _mi_prim_protect(void* addr, size_t size, bool protect);
65
66// Allocate huge (1GiB) pages possibly associated with a NUMA node.
67// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
68// pre: size > 0 and a multiple of 1GiB.
69// numa_node is either negative (don't care), or a numa node number.
70int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
71
72// Return the current NUMA node
73size_t _mi_prim_numa_node(void);
74
75// Return the number of logical NUMA nodes
76size_t _mi_prim_numa_node_count(void);
77
78// Clock ticks
79mi_msecs_t _mi_prim_clock_now(void);
80
81// Return process information (only for statistics)
82typedef struct mi_process_info_s {
83 mi_msecs_t elapsed;
84 mi_msecs_t utime;
85 mi_msecs_t stime;
86 size_t current_rss;
87 size_t peak_rss;
88 size_t current_commit;
89 size_t peak_commit;
90 size_t page_faults;
91} mi_process_info_t;
92
93void _mi_prim_process_info(mi_process_info_t* pinfo);
94
95// Default stderr output. (only for warnings etc. with verbose enabled)
96// msg != NULL && _mi_strlen(msg) > 0
97void _mi_prim_out_stderr( const char* msg );
98
99// Get an environment variable. (only for options)
100// name != NULL, result != NULL, result_size >= 64
101bool _mi_prim_getenv(const char* name, char* result, size_t result_size);
102
103
104// Fill a buffer with strong randomness; return `false` on error or if
105// there is no strong randomization available.
106bool _mi_prim_random_buf(void* buf, size_t buf_len);
107
108// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.
109void _mi_prim_thread_init_auto_done(void);
110
111// Called on process exit and may take action to clean up resources associated with the thread auto done.
112void _mi_prim_thread_done_auto_done(void);
113
114// Called when the default heap for a thread changes
115void _mi_prim_thread_associate_default_heap(mi_heap_t* heap);
116
117
118//-------------------------------------------------------------------
119// Thread id: `_mi_prim_thread_id()`
120//
121// Getting the thread id should be performant as it is called in the
122// fast path of `_mi_free` and we specialize for various platforms as
123// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
124// We only require _mi_prim_thread_id() to return a unique id
125// for each thread (unequal to zero).
126//-------------------------------------------------------------------
127
128// defined in `init.c`; do not use these directly
129extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
130extern bool _mi_process_is_initialized; // has mi_process_init been called?
131
132static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept;
133
134#if defined(_WIN32)
135
136#define WIN32_LEAN_AND_MEAN
137#include <windows.h>
138static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
139 // Windows: works on Intel and ARM in both 32- and 64-bit
140 return (uintptr_t)NtCurrentTeb();
141}
142
143// We use assembly for a fast thread id on the main platforms. The TLS layout depends on
144// both the OS and libc implementation so we use specific tests for each main platform.
145// If you test on another platform and it works please send a PR :-)
146// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
147#elif defined(__GNUC__) && ( \
148 (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
149 || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \
150 || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
151 || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
152 || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
153 )
154
155static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
156 void* res;
157 const size_t ofs = (slot*sizeof(void*));
158 #if defined(__i386__)
159 __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
160 #elif defined(__APPLE__) && defined(__x86_64__)
161 __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
162 #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
163 __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
164 #elif defined(__x86_64__)
165 __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
166 #elif defined(__arm__)
167 void** tcb; MI_UNUSED(ofs);
168 __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
169 res = tcb[slot];
170 #elif defined(__aarch64__)
171 void** tcb; MI_UNUSED(ofs);
172 #if defined(__APPLE__) // M1, issue #343
173 __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
174 #else
175 __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
176 #endif
177 res = tcb[slot];
178 #endif
179 return res;
180}
181
182// setting a tls slot is only used on macOS for now
183static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
184 const size_t ofs = (slot*sizeof(void*));
185 #if defined(__i386__)
186 __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
187 #elif defined(__APPLE__) && defined(__x86_64__)
188 __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
189 #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
190 __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
191 #elif defined(__x86_64__)
192 __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
193 #elif defined(__arm__)
194 void** tcb; MI_UNUSED(ofs);
195 __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
196 tcb[slot] = value;
197 #elif defined(__aarch64__)
198 void** tcb; MI_UNUSED(ofs);
199 #if defined(__APPLE__) // M1, issue #343
200 __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
201 #else
202 __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
203 #endif
204 tcb[slot] = value;
205 #endif
206}
207
208static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
209 #if defined(__BIONIC__)
210 // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
211 // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
212 return (uintptr_t)mi_prim_tls_slot(1);
213 #else
214 // in all our other targets, slot 0 is the thread id
215 // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
216 // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
217 return (uintptr_t)mi_prim_tls_slot(0);
218 #endif
219}
220
221#else
222
223// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
224static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
225 return (uintptr_t)&_mi_heap_default;
226}
227
228#endif
229
230
231
232/* ----------------------------------------------------------------------------------------
233The thread local default heap: `_mi_prim_get_default_heap()`
234This is inlined here as it is on the fast path for allocation functions.
235
236On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a
237__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures
238that the storage will always be available (allocated on the thread stacks).
239
240On some platforms though we cannot use that when overriding `malloc` since the underlying
241TLS implementation (or the loader) will call itself `malloc` on a first access and recurse.
242We try to circumvent this in an efficient way:
243- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the
244 loader itself calls `malloc` even before the modules are initialized.
245- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS).
246- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323)
247------------------------------------------------------------------------------------------- */
248
249static inline mi_heap_t* mi_prim_get_default_heap(void);
250
251#if defined(MI_MALLOC_OVERRIDE)
252#if defined(__APPLE__) // macOS
253 #define MI_TLS_SLOT 89 // seems unused?
254 // #define MI_TLS_RECURSE_GUARD 1
255 // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89)
256 // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
257#elif defined(__OpenBSD__)
258 // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16)
259 // see <https://github.com/openbsd/src/blob/master/lib/libc/include/thread_private.h#L371>
260 #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24)
261 // #elif defined(__DragonFly__)
262 // #warning "mimalloc is not working correctly on DragonFly yet."
263 // #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) <https://github.com/DragonFlyBSD/DragonFlyBSD/blob/master/lib/libthread_xu/thread/thr_private.h#L458>
264#elif defined(__ANDROID__)
265 // See issue #381
266 #define MI_TLS_PTHREAD
267#endif
268#endif
269
270
271#if defined(MI_TLS_SLOT)
272
273static inline mi_heap_t* mi_prim_get_default_heap(void) {
274 mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT);
275 if mi_unlikely(heap == NULL) {
276 #ifdef __GNUC__
277 __asm(""); // prevent conditional load of the address of _mi_heap_empty
278 #endif
279 heap = (mi_heap_t*)&_mi_heap_empty;
280 }
281 return heap;
282}
283
284#elif defined(MI_TLS_PTHREAD_SLOT_OFS)
285
286static inline mi_heap_t** mi_prim_tls_pthread_heap_slot(void) {
287 pthread_t self = pthread_self();
288 #if defined(__DragonFly__)
289 if (self==NULL) return NULL;
290 #endif
291 return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS);
292}
293
294static inline mi_heap_t* mi_prim_get_default_heap(void) {
295 mi_heap_t** pheap = mi_prim_tls_pthread_heap_slot();
296 if mi_unlikely(pheap == NULL) return _mi_heap_main_get();
297 mi_heap_t* heap = *pheap;
298 if mi_unlikely(heap == NULL) return (mi_heap_t*)&_mi_heap_empty;
299 return heap;
300}
301
302#elif defined(MI_TLS_PTHREAD)
303
304extern pthread_key_t _mi_heap_default_key;
305static inline mi_heap_t* mi_prim_get_default_heap(void) {
306 mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key));
307 return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
308}
309
310#else // default using a thread local variable; used on most platforms.
311
312static inline mi_heap_t* mi_prim_get_default_heap(void) {
313 #if defined(MI_TLS_RECURSE_GUARD)
314 if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get();
315 #endif
316 return _mi_heap_default;
317}
318
319#endif // mi_prim_get_default_heap()
320
321
322
323#endif // MIMALLOC_PRIM_H
324