GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/qdk

Public

mirrored from https://github.com/microsoft/qdkAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
8829ede154fe072649e1bb87bf1cbb81b7f96db0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

qdk/source/allocator/mimalloc-sys/mimalloc/src

source/allocator/mimalloc-sys/mimalloc/src/alloc.c

692lines · modeblame

RawDownload
Latest commit unavailable.
91589c3aIan Davis2 years ago1/* ----------------------------------------------------------------------------
ce21e681Ian Davis1 years ago2Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
91589c3aIan Davis2 years ago3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#ifndef _DEFAULT_SOURCE
8#define _DEFAULT_SOURCE // for realpath() on Linux
9#endif
10
11#include "mimalloc.h"
12#include "mimalloc/internal.h"
13#include "mimalloc/atomic.h"
14#include "mimalloc/prim.h" // _mi_prim_thread_id()
15
16#include <string.h> // memset, strlen (for mi_strdup)
17#include <stdlib.h> // malloc, abort
18
19#define MI_IN_ALLOC_C
20#include "alloc-override.c"
ce21e681Ian Davis1 years ago21#include "free.c"
91589c3aIan Davis2 years ago22#undef MI_IN_ALLOC_C
23
24// ------------------------------------------------------
25// Allocation
26// ------------------------------------------------------
27
28// Fast allocation in a page: just pop from the free list.
29// Fall back to generic allocation only if the list is empty.
ce21e681Ian Davis1 years ago30// Note: in release mode the (inlined) routine is about 7 instructions with a single test.
31extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept
32{
33mi_assert_internal(size >= MI_PADDING_SIZE);
34mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size);
35
36// check the free list
91589c3aIan Davis2 years ago37mi_block_t* const block = page->free;
38if mi_unlikely(block == NULL) {
39return _mi_malloc_generic(heap, size, zero, 0);
40}
41mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
ce21e681Ian Davis1 years ago42
91589c3aIan Davis2 years ago43// pop from the free list
44page->free = mi_block_next(page, block);
ce21e681Ian Davis1 years ago45page->used++;
91589c3aIan Davis2 years ago46mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
ce21e681Ian Davis1 years ago47mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));
48
91589c3aIan Davis2 years ago49#if MI_DEBUG>3
ce21e681Ian Davis1 years ago50if (page->free_is_zero && size > sizeof(*block)) {
91589c3aIan Davis2 years ago51mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));
52}
53#endif
54
55// allow use of the block internally
56// note: when tracking we need to avoid ever touching the MI_PADDING since
57// that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`)
58mi_track_mem_undefined(block, mi_page_usable_block_size(page));
59
60// zero the block? note: we need to zero the full block size (issue #63)
61if mi_unlikely(zero) {
ce21e681Ian Davis1 years ago62mi_assert_internal(page->block_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic)
63mi_assert_internal(!mi_page_is_huge(page));
64#if MI_PADDING
65mi_assert_internal(page->block_size >= MI_PADDING_SIZE);
66#endif
91589c3aIan Davis2 years ago67if (page->free_is_zero) {
68block->next = 0;
ce21e681Ian Davis1 years ago69mi_track_mem_defined(block, page->block_size - MI_PADDING_SIZE);
91589c3aIan Davis2 years ago70}
71else {
ce21e681Ian Davis1 years ago72_mi_memzero_aligned(block, page->block_size - MI_PADDING_SIZE);
73}
91589c3aIan Davis2 years ago74}
75
ce21e681Ian Davis1 years ago76#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
91589c3aIan Davis2 years ago77if (!zero && !mi_page_is_huge(page)) {
78memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page));
79}
ce21e681Ian Davis1 years ago80#elif (MI_SECURE!=0)
91589c3aIan Davis2 years ago81if (!zero) { block->next = 0; } // don't leak internal data
ce21e681Ian Davis1 years ago82#endif
91589c3aIan Davis2 years ago83
ce21e681Ian Davis1 years ago84#if (MI_STAT>0)
91589c3aIan Davis2 years ago85const size_t bsize = mi_page_usable_block_size(page);
86if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) {
ce21e681Ian Davis1 years ago87mi_heap_stat_increase(heap, malloc_normal, bsize);
88mi_heap_stat_counter_increase(heap, malloc_normal_count, 1);
89#if (MI_STAT>1)
91589c3aIan Davis2 years ago90const size_t bin = _mi_bin(bsize);
ce21e681Ian Davis1 years ago91mi_heap_stat_increase(heap, malloc_bins[bin], 1);
92mi_heap_stat_increase(heap, malloc_requested, size - MI_PADDING_SIZE);
93#endif
91589c3aIan Davis2 years ago94}
95#endif
ce21e681Ian Davis1 years ago96
97#if MI_PADDING // && !MI_TRACK_ENABLED
98mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page));
99ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
100#if (MI_DEBUG>=2)
101mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta));
102#endif
103mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess
104padding->canary = mi_ptr_encode_canary(page,block,page->keys);
105padding->delta = (uint32_t)(delta);
106#if MI_PADDING_CHECK
107if (!mi_page_is_huge(page)) {
108uint8_t* fill = (uint8_t*)padding - delta;
109const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
110for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
111}
112#endif
91589c3aIan Davis2 years ago113#endif
114
115return block;
116}
117
ce21e681Ian Davis1 years ago118// extra entries for improved efficiency in `alloc-aligned.c`.
119extern void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept {
120return _mi_page_malloc_zero(heap,page,size,false);
121}
122extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept {
123return _mi_page_malloc_zero(heap,page,size,true);
124}
125
126#if MI_GUARDED
127mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
128#endif
129
91589c3aIan Davis2 years ago130static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
131mi_assert(heap != NULL);
ce21e681Ian Davis1 years ago132mi_assert(size <= MI_SMALL_SIZE_MAX);
91589c3aIan Davis2 years ago133#if MI_DEBUG
134const uintptr_t tid = _mi_thread_id();
135mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local
136#endif
ce21e681Ian Davis1 years ago137#if (MI_PADDING || MI_GUARDED)
91589c3aIan Davis2 years ago138if (size == 0) { size = sizeof(void*); }
139#endif
ce21e681Ian Davis1 years ago140#if MI_GUARDED
141if (mi_heap_malloc_use_guarded(heap,size)) {
142return _mi_heap_malloc_guarded(heap, size, zero);
91589c3aIan Davis2 years ago143}
144#endif
ce21e681Ian Davis1 years ago145
146// get page in constant time, and allocate from it
147mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE);
148void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero);
149mi_track_malloc(p,size,zero);
150
91589c3aIan Davis2 years ago151#if MI_DEBUG>3
152if (p != NULL && zero) {
153mi_assert_expensive(mi_mem_is_zero(p, size));
154}
155#endif
156return p;
157}
158
159// allocate a small block
160mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
161return mi_heap_malloc_small_zero(heap, size, false);
162}
163
164mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {
165return mi_heap_malloc_small(mi_prim_get_default_heap(), size);
166}
167
168// The main allocation function
169extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept {
ce21e681Ian Davis1 years ago170// fast path for small objects
91589c3aIan Davis2 years ago171if mi_likely(size <= MI_SMALL_SIZE_MAX) {
172mi_assert_internal(huge_alignment == 0);
173return mi_heap_malloc_small_zero(heap, size, zero);
174}
ce21e681Ian Davis1 years ago175#if MI_GUARDED
176else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) {
177return _mi_heap_malloc_guarded(heap, size, zero);
178}
179#endif
91589c3aIan Davis2 years ago180else {
ce21e681Ian Davis1 years ago181// regular allocation
91589c3aIan Davis2 years ago182mi_assert(heap!=NULL);
183mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
184void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic
185mi_track_malloc(p,size,zero);
ce21e681Ian Davis1 years ago186
91589c3aIan Davis2 years ago187#if MI_DEBUG>3
188if (p != NULL && zero) {
189mi_assert_expensive(mi_mem_is_zero(p, size));
190}
191#endif
192return p;
193}
194}
195
196extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
197return _mi_heap_malloc_zero_ex(heap, size, zero, 0);
198}
199
200mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
201return _mi_heap_malloc_zero(heap, size, false);
202}
203
204mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {
205return mi_heap_malloc(mi_prim_get_default_heap(), size);
206}
207
208// zero initialized small block
209mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
210return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, true);
211}
212
213mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
214return _mi_heap_malloc_zero(heap, size, true);
215}
216
217mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {
218return mi_heap_zalloc(mi_prim_get_default_heap(),size);
219}
220
221
222mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
223size_t total;
224if (mi_count_size_overflow(count,size,&total)) return NULL;
225return mi_heap_zalloc(heap,total);
226}
227
228mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
229return mi_heap_calloc(mi_prim_get_default_heap(),count,size);
230}
231
232// Uninitialized `calloc`
233mi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
234size_t total;
235if (mi_count_size_overflow(count, size, &total)) return NULL;
236return mi_heap_malloc(heap, total);
237}
238
239mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
240return mi_heap_mallocn(mi_prim_get_default_heap(),count,size);
241}
242
243// Expand (or shrink) in place (or fail)
244void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
245#if MI_PADDING
246// we do not shrink/expand with padding enabled
247MI_UNUSED(p); MI_UNUSED(newsize);
248return NULL;
249#else
250if (p == NULL) return NULL;
251const size_t size = _mi_usable_size(p,"mi_expand");
252if (newsize > size) return NULL;
253return p; // it fits
254#endif
255}
256
257void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept {
258// if p == NULL then behave as malloc.
259// else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
260// (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
261const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0)
262if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
263mi_assert_internal(p!=NULL);
264// todo: do not track as the usable size is still the same in the free; adjust potential padding?
265// mi_track_resize(p,size,newsize)
266// if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
267return p; // reallocation still fits and not more than 50% waste
268}
269void* newp = mi_heap_malloc(heap,newsize);
270if mi_likely(newp != NULL) {
271if (zero && newsize > size) {
272// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
273const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
274_mi_memzero((uint8_t*)newp + start, newsize - start);
275}
276else if (newsize == 0) {
277((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)
278}
279if mi_likely(p != NULL) {
280const size_t copysize = (newsize > size ? size : newsize);
281mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking..
282_mi_memcpy(newp, p, copysize);
283mi_free(p); // only free the original pointer if successful
284}
285}
286return newp;
287}
288
289mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
290return _mi_heap_realloc_zero(heap, p, newsize, false);
291}
292
293mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
294size_t total;
295if (mi_count_size_overflow(count, size, &total)) return NULL;
296return mi_heap_realloc(heap, p, total);
297}
298
299
300// Reallocate but free `p` on errors
301mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
302void* newp = mi_heap_realloc(heap, p, newsize);
303if (newp==NULL && p!=NULL) mi_free(p);
304return newp;
305}
306
307mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
308return _mi_heap_realloc_zero(heap, p, newsize, true);
309}
310
311mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
312size_t total;
313if (mi_count_size_overflow(count, size, &total)) return NULL;
314return mi_heap_rezalloc(heap, p, total);
315}
316
317
318mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
319return mi_heap_realloc(mi_prim_get_default_heap(),p,newsize);
320}
321
322mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
323return mi_heap_reallocn(mi_prim_get_default_heap(),p,count,size);
324}
325
326// Reallocate but free `p` on errors
327mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
328return mi_heap_reallocf(mi_prim_get_default_heap(),p,newsize);
329}
330
331mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {
332return mi_heap_rezalloc(mi_prim_get_default_heap(), p, newsize);
333}
334
335mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {
336return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size);
337}
338
339
340
341// ------------------------------------------------------
342// strdup, strndup, and realpath
343// ------------------------------------------------------
344
345// `strdup` using mi_malloc
346mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {
347if (s == NULL) return NULL;
ce21e681Ian Davis1 years ago348size_t len = _mi_strlen(s);
349char* t = (char*)mi_heap_malloc(heap,len+1);
91589c3aIan Davis2 years ago350if (t == NULL) return NULL;
ce21e681Ian Davis1 years ago351_mi_memcpy(t, s, len);
352t[len] = 0;
91589c3aIan Davis2 years ago353return t;
354}
355
356mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept {
357return mi_heap_strdup(mi_prim_get_default_heap(), s);
358}
359
360// `strndup` using mi_malloc
361mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept {
362if (s == NULL) return NULL;
ce21e681Ian Davis1 years ago363const size_t len = _mi_strnlen(s,n); // len <= n
364char* t = (char*)mi_heap_malloc(heap, len+1);
91589c3aIan Davis2 years ago365if (t == NULL) return NULL;
ce21e681Ian Davis1 years ago366_mi_memcpy(t, s, len);
367t[len] = 0;
91589c3aIan Davis2 years ago368return t;
369}
370
371mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {
372return mi_heap_strndup(mi_prim_get_default_heap(),s,n);
373}
374
375#ifndef __wasi__
376// `realpath` using mi_malloc
377#ifdef _WIN32
378#ifndef PATH_MAX
379#define PATH_MAX MAX_PATH
380#endif
ce21e681Ian Davis1 years ago381
91589c3aIan Davis2 years ago382mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {
383// todo: use GetFullPathNameW to allow longer file names
384char buf[PATH_MAX];
385DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);
386if (res == 0) {
387errno = GetLastError(); return NULL;
388}
389else if (res > PATH_MAX) {
390errno = EINVAL; return NULL;
391}
392else if (resolved_name != NULL) {
393return resolved_name;
394}
395else {
396return mi_heap_strndup(heap, buf, PATH_MAX);
397}
398}
399#else
400/*
401#include <unistd.h> // pathconf
402static size_t mi_path_max(void) {
403static size_t path_max = 0;
404if (path_max <= 0) {
405long m = pathconf("/",_PC_PATH_MAX);
406if (m <= 0) path_max = 4096; // guess
407else if (m < 256) path_max = 256; // at least 256
408else path_max = m;
409}
410return path_max;
411}
412*/
413char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {
414if (resolved_name != NULL) {
415return realpath(fname,resolved_name);
416}
417else {
418char* rname = realpath(fname, NULL);
419if (rname == NULL) return NULL;
420char* result = mi_heap_strdup(heap, rname);
ce21e681Ian Davis1 years ago421mi_cfree(rname); // use checked free (which may be redirected to our free but that's ok)
422// note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(
91589c3aIan Davis2 years ago423return result;
424}
425/*
426const size_t n = mi_path_max();
427char* buf = (char*)mi_malloc(n+1);
428if (buf == NULL) {
429errno = ENOMEM;
430return NULL;
431}
432char* rname = realpath(fname,buf);
433char* result = mi_heap_strndup(heap,rname,n); // ok if `rname==NULL`
434mi_free(buf);
435return result;
436}
437*/
438}
439#endif
440
441mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {
442return mi_heap_realpath(mi_prim_get_default_heap(),fname,resolved_name);
443}
444#endif
445
446/*-------------------------------------------------------
447C++ new and new_aligned
448The standard requires calling into `get_new_handler` and
449throwing the bad_alloc exception on failure. If we compile
450with a C++ compiler we can implement this precisely. If we
451use a C compiler we cannot throw a `bad_alloc` exception
452but we call `exit` instead (i.e. not returning).
453-------------------------------------------------------*/
454
455#ifdef __cplusplus
456#include <new>
457static bool mi_try_new_handler(bool nothrow) {
458#if defined(_MSC_VER) || (__cplusplus >= 201103L)
459std::new_handler h = std::get_new_handler();
460#else
461std::new_handler h = std::set_new_handler();
462std::set_new_handler(h);
463#endif
464if (h==NULL) {
465_mi_error_message(ENOMEM, "out of memory in 'new'");
ce21e681Ian Davis1 years ago466#if defined(_CPPUNWIND) || defined(__cpp_exceptions) // exceptions are not always enabled
91589c3aIan Davis2 years ago467if (!nothrow) {
468throw std::bad_alloc();
469}
ce21e681Ian Davis1 years ago470#else
471MI_UNUSED(nothrow);
472#endif
91589c3aIan Davis2 years ago473return false;
474}
475else {
476h();
477return true;
478}
479}
480#else
481typedef void (*std_new_handler_t)(void);
482
483#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631
484std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) {
485return NULL;
486}
487static std_new_handler_t mi_get_new_handler(void) {
488return _ZSt15get_new_handlerv();
489}
490#else
491// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`.
492static std_new_handler_t mi_get_new_handler() {
493return NULL;
494}
495#endif
496
497static bool mi_try_new_handler(bool nothrow) {
498std_new_handler_t h = mi_get_new_handler();
499if (h==NULL) {
500_mi_error_message(ENOMEM, "out of memory in 'new'");
501if (!nothrow) {
502abort(); // cannot throw in plain C, use abort
503}
504return false;
505}
506else {
507h();
508return true;
509}
510}
511#endif
512
513mi_decl_export mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) {
514void* p = NULL;
515while(p == NULL && mi_try_new_handler(nothrow)) {
516p = mi_heap_malloc(heap,size);
517}
518return p;
519}
520
521static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
522return mi_heap_try_new(mi_prim_get_default_heap(), size, nothrow);
523}
524
525
526mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) {
527void* p = mi_heap_malloc(heap,size);
528if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false);
529return p;
530}
531
532mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) {
533return mi_heap_alloc_new(mi_prim_get_default_heap(), size);
534}
535
536
537mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) {
538size_t total;
539if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
540mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
541return NULL;
542}
543else {
544return mi_heap_alloc_new(heap,total);
545}
546}
547
548mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) {
ce21e681Ian Davis1 years ago549return mi_heap_alloc_new_n(mi_prim_get_default_heap(), count, size);
91589c3aIan Davis2 years ago550}
551
552
553mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
554void* p = mi_malloc(size);
555if mi_unlikely(p == NULL) return mi_try_new(size, true);
556return p;
557}
558
559mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {
560void* p;
561do {
562p = mi_malloc_aligned(size, alignment);
563}
564while(p == NULL && mi_try_new_handler(false));
565return p;
566}
567
568mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {
569void* p;
570do {
571p = mi_malloc_aligned(size, alignment);
572}
573while(p == NULL && mi_try_new_handler(true));
574return p;
575}
576
577mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {
578void* q;
579do {
580q = mi_realloc(p, newsize);
581} while (q == NULL && mi_try_new_handler(false));
582return q;
583}
584
585mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
586size_t total;
587if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {
588mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
589return NULL;
590}
591else {
592return mi_new_realloc(p, total);
593}
594}
595
ce21e681Ian Davis1 years ago596#if MI_GUARDED
597// We always allocate a guarded allocation at an offset (`mi_page_has_aligned` will be true).
598// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
599// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
600
601static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {
602// TODO: we can still make padding work by moving it out of the guard page area
603mi_page_t* const page = _mi_ptr_page(block);
604mi_page_set_has_aligned(page, true);
605block->next = MI_BLOCK_TAG_GUARDED;
606
607// set guard page at the end of the block
608mi_segment_t* const segment = _mi_page_segment(page);
609const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`
610const size_t os_page_size = _mi_os_page_size();
611mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
612if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
613// should never happen
614mi_free(block);
615return NULL;
616}
617uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
618mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
619if (segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) {
620_mi_os_protect(guard_page, os_page_size);
621}
622else {
623_mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
624}
625
626// align pointer just in front of the guard page
627size_t offset = block_size - os_page_size - obj_size;
628mi_assert_internal(offset > sizeof(mi_block_t));
629if (offset > MI_BLOCK_ALIGNMENT_MAX) {
630// give up to place it right in front of the guard page if the offset is too large for unalignment
631offset = MI_BLOCK_ALIGNMENT_MAX;
632}
633void* p = (uint8_t*)block + offset;
634mi_track_align(block, p, offset, obj_size);
635mi_track_mem_defined(block, sizeof(mi_block_t));
636return p;
637}
638
639mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept
640{
641#if defined(MI_PADDING_SIZE)
642mi_assert(MI_PADDING_SIZE==0);
643#endif
644// allocate multiple of page size ending in a guard page
645// ensure minimal alignment requirement?
646const size_t os_page_size = _mi_os_page_size();
647const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
648const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
649const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);
650mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */);
651if (block==NULL) return NULL;
652void* const p = mi_block_ptr_set_guarded(block, obj_size);
653
654// stats
655mi_track_malloc(p, size, zero);
656if (p != NULL) {
657if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }
658#if MI_STAT>1
659mi_heap_stat_adjust_decrease(heap, malloc_requested, req_size);
660mi_heap_stat_increase(heap, malloc_requested, size);
661#endif
662_mi_stat_counter_increase(&heap->tld->stats.malloc_guarded_count, 1);
663}
664#if MI_DEBUG>3
665if (p != NULL && zero) {
666mi_assert_expensive(mi_mem_is_zero(p, size));
667}
668#endif
669return p;
670}
671#endif
672
91589c3aIan Davis2 years ago673// ------------------------------------------------------
674// ensure explicit external inline definitions are emitted!
675// ------------------------------------------------------
676
677#ifdef __cplusplus
678void* _mi_externs[] = {
679(void*)&_mi_page_malloc,
ce21e681Ian Davis1 years ago680(void*)&_mi_page_malloc_zero,
91589c3aIan Davis2 years ago681(void*)&_mi_heap_malloc_zero,
682(void*)&_mi_heap_malloc_zero_ex,
683(void*)&mi_malloc,
684(void*)&mi_malloc_small,
685(void*)&mi_zalloc_small,
686(void*)&mi_heap_malloc,
687(void*)&mi_heap_zalloc,
688(void*)&mi_heap_malloc_small,
689// (void*)&mi_heap_alloc_new,
690// (void*)&mi_heap_alloc_new_n
691};
692#endif