microsoft/qdk
Publicmirrored fromhttps://github.com/microsoft/qdkAvailable
allocator/mimalloc-sys/mimalloc/src/alloc.c
1060lines · modecode
| 1 | /* ---------------------------------------------------------------------------- |
| 2 | Copyright (c) 2018-2022, Microsoft Research, Daan Leijen |
| 3 | This is free software; you can redistribute it and/or modify it under the |
| 4 | terms of the MIT license. A copy of the license can be found in the file |
| 5 | "LICENSE" at the root of this distribution. |
| 6 | -----------------------------------------------------------------------------*/ |
| 7 | #ifndef _DEFAULT_SOURCE |
| 8 | #define _DEFAULT_SOURCE // for realpath() on Linux |
| 9 | #endif |
| 10 | |
| 11 | #include "mimalloc.h" |
| 12 | #include "mimalloc/internal.h" |
| 13 | #include "mimalloc/atomic.h" |
| 14 | #include "mimalloc/prim.h" // _mi_prim_thread_id() |
| 15 | |
| 16 | #include <string.h> // memset, strlen (for mi_strdup) |
| 17 | #include <stdlib.h> // malloc, abort |
| 18 | |
| 19 | #define MI_IN_ALLOC_C |
| 20 | #include "alloc-override.c" |
| 21 | #undef MI_IN_ALLOC_C |
| 22 | |
| 23 | // ------------------------------------------------------ |
| 24 | // Allocation |
| 25 | // ------------------------------------------------------ |
| 26 | |
| 27 | // Fast allocation in a page: just pop from the free list. |
| 28 | // Fall back to generic allocation only if the list is empty. |
| 29 | extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { |
| 30 | mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size); |
| 31 | mi_block_t* const block = page->free; |
| 32 | if mi_unlikely(block == NULL) { |
| 33 | return _mi_malloc_generic(heap, size, zero, 0); |
| 34 | } |
| 35 | mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); |
| 36 | // pop from the free list |
| 37 | page->used++; |
| 38 | page->free = mi_block_next(page, block); |
| 39 | mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); |
| 40 | #if MI_DEBUG>3 |
| 41 | if (page->free_is_zero) { |
| 42 | mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); |
| 43 | } |
| 44 | #endif |
| 45 | |
| 46 | // allow use of the block internally |
| 47 | // note: when tracking we need to avoid ever touching the MI_PADDING since |
| 48 | // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`) |
| 49 | mi_track_mem_undefined(block, mi_page_usable_block_size(page)); |
| 50 | |
| 51 | // zero the block? note: we need to zero the full block size (issue #63) |
| 52 | if mi_unlikely(zero) { |
| 53 | mi_assert_internal(page->xblock_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic) |
| 54 | mi_assert_internal(page->xblock_size >= MI_PADDING_SIZE); |
| 55 | if (page->free_is_zero) { |
| 56 | block->next = 0; |
| 57 | mi_track_mem_defined(block, page->xblock_size - MI_PADDING_SIZE); |
| 58 | } |
| 59 | else { |
| 60 | _mi_memzero_aligned(block, page->xblock_size - MI_PADDING_SIZE); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN |
| 65 | if (!zero && !mi_page_is_huge(page)) { |
| 66 | memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); |
| 67 | } |
| 68 | #elif (MI_SECURE!=0) |
| 69 | if (!zero) { block->next = 0; } // don't leak internal data |
| 70 | #endif |
| 71 | |
| 72 | #if (MI_STAT>0) |
| 73 | const size_t bsize = mi_page_usable_block_size(page); |
| 74 | if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) { |
| 75 | mi_heap_stat_increase(heap, normal, bsize); |
| 76 | mi_heap_stat_counter_increase(heap, normal_count, 1); |
| 77 | #if (MI_STAT>1) |
| 78 | const size_t bin = _mi_bin(bsize); |
| 79 | mi_heap_stat_increase(heap, normal_bins[bin], 1); |
| 80 | #endif |
| 81 | } |
| 82 | #endif |
| 83 | |
| 84 | #if MI_PADDING // && !MI_TRACK_ENABLED |
| 85 | mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); |
| 86 | ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); |
| 87 | #if (MI_DEBUG>=2) |
| 88 | mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); |
| 89 | #endif |
| 90 | mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess |
| 91 | padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); |
| 92 | padding->delta = (uint32_t)(delta); |
| 93 | #if MI_PADDING_CHECK |
| 94 | if (!mi_page_is_huge(page)) { |
| 95 | uint8_t* fill = (uint8_t*)padding - delta; |
| 96 | const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes |
| 97 | for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } |
| 98 | } |
| 99 | #endif |
| 100 | #endif |
| 101 | |
| 102 | return block; |
| 103 | } |
| 104 | |
| 105 | static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { |
| 106 | mi_assert(heap != NULL); |
| 107 | #if MI_DEBUG |
| 108 | const uintptr_t tid = _mi_thread_id(); |
| 109 | mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local |
| 110 | #endif |
| 111 | mi_assert(size <= MI_SMALL_SIZE_MAX); |
| 112 | #if (MI_PADDING) |
| 113 | if (size == 0) { size = sizeof(void*); } |
| 114 | #endif |
| 115 | mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); |
| 116 | void* const p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE, zero); |
| 117 | mi_track_malloc(p,size,zero); |
| 118 | #if MI_STAT>1 |
| 119 | if (p != NULL) { |
| 120 | if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } |
| 121 | mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); |
| 122 | } |
| 123 | #endif |
| 124 | #if MI_DEBUG>3 |
| 125 | if (p != NULL && zero) { |
| 126 | mi_assert_expensive(mi_mem_is_zero(p, size)); |
| 127 | } |
| 128 | #endif |
| 129 | return p; |
| 130 | } |
| 131 | |
| 132 | // allocate a small block |
| 133 | mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { |
| 134 | return mi_heap_malloc_small_zero(heap, size, false); |
| 135 | } |
| 136 | |
| 137 | mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { |
| 138 | return mi_heap_malloc_small(mi_prim_get_default_heap(), size); |
| 139 | } |
| 140 | |
| 141 | // The main allocation function |
| 142 | extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept { |
| 143 | if mi_likely(size <= MI_SMALL_SIZE_MAX) { |
| 144 | mi_assert_internal(huge_alignment == 0); |
| 145 | return mi_heap_malloc_small_zero(heap, size, zero); |
| 146 | } |
| 147 | else { |
| 148 | mi_assert(heap!=NULL); |
| 149 | mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local |
| 150 | void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic |
| 151 | mi_track_malloc(p,size,zero); |
| 152 | #if MI_STAT>1 |
| 153 | if (p != NULL) { |
| 154 | if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } |
| 155 | mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); |
| 156 | } |
| 157 | #endif |
| 158 | #if MI_DEBUG>3 |
| 159 | if (p != NULL && zero) { |
| 160 | mi_assert_expensive(mi_mem_is_zero(p, size)); |
| 161 | } |
| 162 | #endif |
| 163 | return p; |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { |
| 168 | return _mi_heap_malloc_zero_ex(heap, size, zero, 0); |
| 169 | } |
| 170 | |
| 171 | mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { |
| 172 | return _mi_heap_malloc_zero(heap, size, false); |
| 173 | } |
| 174 | |
| 175 | mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { |
| 176 | return mi_heap_malloc(mi_prim_get_default_heap(), size); |
| 177 | } |
| 178 | |
| 179 | // zero initialized small block |
| 180 | mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { |
| 181 | return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, true); |
| 182 | } |
| 183 | |
| 184 | mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { |
| 185 | return _mi_heap_malloc_zero(heap, size, true); |
| 186 | } |
| 187 | |
| 188 | mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { |
| 189 | return mi_heap_zalloc(mi_prim_get_default_heap(),size); |
| 190 | } |
| 191 | |
| 192 | |
| 193 | // ------------------------------------------------------ |
| 194 | // Check for double free in secure and debug mode |
| 195 | // This is somewhat expensive so only enabled for secure mode 4 |
| 196 | // ------------------------------------------------------ |
| 197 | |
| 198 | #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) |
| 199 | // linear check if the free list contains a specific element |
| 200 | static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) { |
| 201 | while (list != NULL) { |
| 202 | if (elem==list) return true; |
| 203 | list = mi_block_next(page, list); |
| 204 | } |
| 205 | return false; |
| 206 | } |
| 207 | |
| 208 | static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { |
| 209 | // The decoded value is in the same page (or NULL). |
| 210 | // Walk the free lists to verify positively if it is already freed |
| 211 | if (mi_list_contains(page, page->free, block) || |
| 212 | mi_list_contains(page, page->local_free, block) || |
| 213 | mi_list_contains(page, mi_page_thread_free(page), block)) |
| 214 | { |
| 215 | _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); |
| 216 | return true; |
| 217 | } |
| 218 | return false; |
| 219 | } |
| 220 | |
| 221 | #define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); } |
| 222 | |
| 223 | static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { |
| 224 | bool is_double_free = false; |
| 225 | mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field |
| 226 | if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? |
| 227 | (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? |
| 228 | { |
| 229 | // Suspicous: decoded value a in block is in the same page (or NULL) -- maybe a double free? |
| 230 | // (continue in separate function to improve code generation) |
| 231 | is_double_free = mi_check_is_double_freex(page, block); |
| 232 | } |
| 233 | return is_double_free; |
| 234 | } |
| 235 | #else |
| 236 | static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { |
| 237 | MI_UNUSED(page); |
| 238 | MI_UNUSED(block); |
| 239 | return false; |
| 240 | } |
| 241 | #endif |
| 242 | |
| 243 | // --------------------------------------------------------------------------- |
| 244 | // Check for heap block overflow by setting up padding at the end of the block |
| 245 | // --------------------------------------------------------------------------- |
| 246 | |
| 247 | #if MI_PADDING // && !MI_TRACK_ENABLED |
| 248 | static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { |
| 249 | *bsize = mi_page_usable_block_size(page); |
| 250 | const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); |
| 251 | mi_track_mem_defined(padding,sizeof(mi_padding_t)); |
| 252 | *delta = padding->delta; |
| 253 | uint32_t canary = padding->canary; |
| 254 | uintptr_t keys[2]; |
| 255 | keys[0] = page->keys[0]; |
| 256 | keys[1] = page->keys[1]; |
| 257 | bool ok = ((uint32_t)mi_ptr_encode(page,block,keys) == canary && *delta <= *bsize); |
| 258 | mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); |
| 259 | return ok; |
| 260 | } |
| 261 | |
| 262 | // Return the exact usable size of a block. |
| 263 | static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { |
| 264 | size_t bsize; |
| 265 | size_t delta; |
| 266 | bool ok = mi_page_decode_padding(page, block, &delta, &bsize); |
| 267 | mi_assert_internal(ok); mi_assert_internal(delta <= bsize); |
| 268 | return (ok ? bsize - delta : 0); |
| 269 | } |
| 270 | |
| 271 | // When a non-thread-local block is freed, it becomes part of the thread delayed free |
| 272 | // list that is freed later by the owning heap. If the exact usable size is too small to |
| 273 | // contain the pointer for the delayed list, then shrink the padding (by decreasing delta) |
| 274 | // so it will later not trigger an overflow error in `mi_free_block`. |
| 275 | void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { |
| 276 | size_t bsize; |
| 277 | size_t delta; |
| 278 | bool ok = mi_page_decode_padding(page, block, &delta, &bsize); |
| 279 | mi_assert_internal(ok); |
| 280 | if (!ok || (bsize - delta) >= min_size) return; // usually already enough space |
| 281 | mi_assert_internal(bsize >= min_size); |
| 282 | if (bsize < min_size) return; // should never happen |
| 283 | size_t new_delta = (bsize - min_size); |
| 284 | mi_assert_internal(new_delta < bsize); |
| 285 | mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize); |
| 286 | mi_track_mem_defined(padding,sizeof(mi_padding_t)); |
| 287 | padding->delta = (uint32_t)new_delta; |
| 288 | mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); |
| 289 | } |
| 290 | #else |
| 291 | static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { |
| 292 | MI_UNUSED(block); |
| 293 | return mi_page_usable_block_size(page); |
| 294 | } |
| 295 | |
| 296 | void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { |
| 297 | MI_UNUSED(page); |
| 298 | MI_UNUSED(block); |
| 299 | MI_UNUSED(min_size); |
| 300 | } |
| 301 | #endif |
| 302 | |
| 303 | #if MI_PADDING && MI_PADDING_CHECK |
| 304 | |
| 305 | static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) { |
| 306 | size_t bsize; |
| 307 | size_t delta; |
| 308 | bool ok = mi_page_decode_padding(page, block, &delta, &bsize); |
| 309 | *size = *wrong = bsize; |
| 310 | if (!ok) return false; |
| 311 | mi_assert_internal(bsize >= delta); |
| 312 | *size = bsize - delta; |
| 313 | if (!mi_page_is_huge(page)) { |
| 314 | uint8_t* fill = (uint8_t*)block + bsize - delta; |
| 315 | const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes |
| 316 | mi_track_mem_defined(fill, maxpad); |
| 317 | for (size_t i = 0; i < maxpad; i++) { |
| 318 | if (fill[i] != MI_DEBUG_PADDING) { |
| 319 | *wrong = bsize - delta + i; |
| 320 | ok = false; |
| 321 | break; |
| 322 | } |
| 323 | } |
| 324 | mi_track_mem_noaccess(fill, maxpad); |
| 325 | } |
| 326 | return ok; |
| 327 | } |
| 328 | |
| 329 | static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { |
| 330 | size_t size; |
| 331 | size_t wrong; |
| 332 | if (!mi_verify_padding(page,block,&size,&wrong)) { |
| 333 | _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, size, wrong ); |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | #else |
| 338 | |
| 339 | static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { |
| 340 | MI_UNUSED(page); |
| 341 | MI_UNUSED(block); |
| 342 | } |
| 343 | |
| 344 | #endif |
| 345 | |
| 346 | // only maintain stats for smaller objects if requested |
| 347 | #if (MI_STAT>0) |
| 348 | static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { |
| 349 | #if (MI_STAT < 2) |
| 350 | MI_UNUSED(block); |
| 351 | #endif |
| 352 | mi_heap_t* const heap = mi_heap_get_default(); |
| 353 | const size_t bsize = mi_page_usable_block_size(page); |
| 354 | #if (MI_STAT>1) |
| 355 | const size_t usize = mi_page_usable_size_of(page, block); |
| 356 | mi_heap_stat_decrease(heap, malloc, usize); |
| 357 | #endif |
| 358 | if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) { |
| 359 | mi_heap_stat_decrease(heap, normal, bsize); |
| 360 | #if (MI_STAT > 1) |
| 361 | mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1); |
| 362 | #endif |
| 363 | } |
| 364 | else if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { |
| 365 | mi_heap_stat_decrease(heap, large, bsize); |
| 366 | } |
| 367 | else { |
| 368 | mi_heap_stat_decrease(heap, huge, bsize); |
| 369 | } |
| 370 | } |
| 371 | #else |
| 372 | static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { |
| 373 | MI_UNUSED(page); MI_UNUSED(block); |
| 374 | } |
| 375 | #endif |
| 376 | |
| 377 | #if MI_HUGE_PAGE_ABANDON |
| 378 | #if (MI_STAT>0) |
| 379 | // maintain stats for huge objects |
| 380 | static void mi_stat_huge_free(const mi_page_t* page) { |
| 381 | mi_heap_t* const heap = mi_heap_get_default(); |
| 382 | const size_t bsize = mi_page_block_size(page); // to match stats in `page.c:mi_page_huge_alloc` |
| 383 | if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { |
| 384 | mi_heap_stat_decrease(heap, large, bsize); |
| 385 | } |
| 386 | else { |
| 387 | mi_heap_stat_decrease(heap, huge, bsize); |
| 388 | } |
| 389 | } |
| 390 | #else |
| 391 | static void mi_stat_huge_free(const mi_page_t* page) { |
| 392 | MI_UNUSED(page); |
| 393 | } |
| 394 | #endif |
| 395 | #endif |
| 396 | |
| 397 | // ------------------------------------------------------ |
| 398 | // Free |
| 399 | // ------------------------------------------------------ |
| 400 | |
| 401 | // multi-threaded free (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON) |
| 402 | static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) |
| 403 | { |
| 404 | // The padding check may access the non-thread-owned page for the key values. |
| 405 | // that is safe as these are constant and the page won't be freed (as the block is not freed yet). |
| 406 | mi_check_padding(page, block); |
| 407 | _mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection |
| 408 | |
| 409 | // huge page segments are always abandoned and can be freed immediately |
| 410 | mi_segment_t* segment = _mi_page_segment(page); |
| 411 | if (segment->kind == MI_SEGMENT_HUGE) { |
| 412 | #if MI_HUGE_PAGE_ABANDON |
| 413 | // huge page segments are always abandoned and can be freed immediately |
| 414 | mi_stat_huge_free(page); |
| 415 | _mi_segment_huge_page_free(segment, page, block); |
| 416 | return; |
| 417 | #else |
| 418 | // huge pages are special as they occupy the entire segment |
| 419 | // as these are large we reset the memory occupied by the page so it is available to other threads |
| 420 | // (as the owning thread needs to actually free the memory later). |
| 421 | _mi_segment_huge_page_reset(segment, page, block); |
| 422 | #endif |
| 423 | } |
| 424 | |
| 425 | #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading |
| 426 | if (segment->kind != MI_SEGMENT_HUGE) { // not for huge segments as we just reset the content |
| 427 | memset(block, MI_DEBUG_FREED, mi_usable_size(block)); |
| 428 | } |
| 429 | #endif |
| 430 | |
| 431 | // Try to put the block on either the page-local thread free list, or the heap delayed free list. |
| 432 | mi_thread_free_t tfreex; |
| 433 | bool use_delayed; |
| 434 | mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); |
| 435 | do { |
| 436 | use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE); |
| 437 | if mi_unlikely(use_delayed) { |
| 438 | // unlikely: this only happens on the first concurrent free in a page that is in the full list |
| 439 | tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING); |
| 440 | } |
| 441 | else { |
| 442 | // usual: directly add to page thread_free list |
| 443 | mi_block_set_next(page, block, mi_tf_block(tfree)); |
| 444 | tfreex = mi_tf_set_block(tfree,block); |
| 445 | } |
| 446 | } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); |
| 447 | |
| 448 | if mi_unlikely(use_delayed) { |
| 449 | // racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`) |
| 450 | mi_heap_t* const heap = (mi_heap_t*)(mi_atomic_load_acquire(&page->xheap)); //mi_page_heap(page); |
| 451 | mi_assert_internal(heap != NULL); |
| 452 | if (heap != NULL) { |
| 453 | // add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity) |
| 454 | mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); |
| 455 | do { |
| 456 | mi_block_set_nextx(heap,block,dfree, heap->keys); |
| 457 | } while (!mi_atomic_cas_ptr_weak_release(mi_block_t,&heap->thread_delayed_free, &dfree, block)); |
| 458 | } |
| 459 | |
| 460 | // and reset the MI_DELAYED_FREEING flag |
| 461 | tfree = mi_atomic_load_relaxed(&page->xthread_free); |
| 462 | do { |
| 463 | tfreex = tfree; |
| 464 | mi_assert_internal(mi_tf_delayed(tfree) == MI_DELAYED_FREEING); |
| 465 | tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE); |
| 466 | } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); |
| 467 | } |
| 468 | } |
| 469 | |
| 470 | // regular free |
| 471 | static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block) |
| 472 | { |
| 473 | // and push it on the free list |
| 474 | //const size_t bsize = mi_page_block_size(page); |
| 475 | if mi_likely(local) { |
| 476 | // owning thread can free a block directly |
| 477 | if mi_unlikely(mi_check_is_double_free(page, block)) return; |
| 478 | mi_check_padding(page, block); |
| 479 | #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN |
| 480 | if (!mi_page_is_huge(page)) { // huge page content may be already decommitted |
| 481 | memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); |
| 482 | } |
| 483 | #endif |
| 484 | mi_block_set_next(page, block, page->local_free); |
| 485 | page->local_free = block; |
| 486 | page->used--; |
| 487 | if mi_unlikely(mi_page_all_free(page)) { |
| 488 | _mi_page_retire(page); |
| 489 | } |
| 490 | else if mi_unlikely(mi_page_is_in_full(page)) { |
| 491 | _mi_page_unfull(page); |
| 492 | } |
| 493 | } |
| 494 | else { |
| 495 | _mi_free_block_mt(page,block); |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | |
| 500 | // Adjust a block that was allocated aligned, to the actual start of the block in the page. |
| 501 | mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) { |
| 502 | mi_assert_internal(page!=NULL && p!=NULL); |
| 503 | const size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL); |
| 504 | const size_t adjust = (diff % mi_page_block_size(page)); |
| 505 | return (mi_block_t*)((uintptr_t)p - adjust); |
| 506 | } |
| 507 | |
| 508 | |
| 509 | void mi_decl_noinline _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept { |
| 510 | mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); |
| 511 | mi_stat_free(page, block); // stat_free may access the padding |
| 512 | mi_track_free_size(block, mi_page_usable_size_of(page,block)); |
| 513 | _mi_free_block(page, is_local, block); |
| 514 | } |
| 515 | |
| 516 | // Get the segment data belonging to a pointer |
| 517 | // This is just a single `and` in assembly but does further checks in debug mode |
| 518 | // (and secure mode) if this was a valid pointer. |
| 519 | static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg) |
| 520 | { |
| 521 | MI_UNUSED(msg); |
| 522 | mi_assert(p != NULL); |
| 523 | |
| 524 | #if (MI_DEBUG>0) |
| 525 | if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) { |
| 526 | _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); |
| 527 | return NULL; |
| 528 | } |
| 529 | #endif |
| 530 | |
| 531 | mi_segment_t* const segment = _mi_ptr_segment(p); |
| 532 | mi_assert_internal(segment != NULL); |
| 533 | |
| 534 | #if (MI_DEBUG>0) |
| 535 | if mi_unlikely(!mi_is_in_heap_region(p)) { |
| 536 | #if (MI_INTPTR_SIZE == 8 && defined(__linux__)) |
| 537 | if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640) |
| 538 | #else |
| 539 | { |
| 540 | #endif |
| 541 | _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" |
| 542 | "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); |
| 543 | if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) { |
| 544 | _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); |
| 545 | } |
| 546 | } |
| 547 | } |
| 548 | #endif |
| 549 | #if (MI_DEBUG>0 || MI_SECURE>=4) |
| 550 | if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) { |
| 551 | _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); |
| 552 | return NULL; |
| 553 | } |
| 554 | #endif |
| 555 | |
| 556 | return segment; |
| 557 | } |
| 558 | |
| 559 | // Free a block |
| 560 | // fast path written carefully to prevent spilling on the stack |
| 561 | void mi_free(void* p) mi_attr_noexcept |
| 562 | { |
| 563 | if mi_unlikely(p == NULL) return; |
| 564 | mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); |
| 565 | const bool is_local= (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); |
| 566 | mi_page_t* const page = _mi_segment_page_of(segment, p); |
| 567 | |
| 568 | if mi_likely(is_local) { // thread-local free? |
| 569 | if mi_likely(page->flags.full_aligned == 0) // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned) |
| 570 | { |
| 571 | mi_block_t* const block = (mi_block_t*)p; |
| 572 | if mi_unlikely(mi_check_is_double_free(page, block)) return; |
| 573 | mi_check_padding(page, block); |
| 574 | mi_stat_free(page, block); |
| 575 | #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN |
| 576 | memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); |
| 577 | #endif |
| 578 | mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned |
| 579 | mi_block_set_next(page, block, page->local_free); |
| 580 | page->local_free = block; |
| 581 | if mi_unlikely(--page->used == 0) { // using this expression generates better code than: page->used--; if (mi_page_all_free(page)) |
| 582 | _mi_page_retire(page); |
| 583 | } |
| 584 | } |
| 585 | else { |
| 586 | // page is full or contains (inner) aligned blocks; use generic path |
| 587 | _mi_free_generic(segment, page, true, p); |
| 588 | } |
| 589 | } |
| 590 | else { |
| 591 | // not thread-local; use generic path |
| 592 | _mi_free_generic(segment, page, false, p); |
| 593 | } |
| 594 | } |
| 595 | |
| 596 | // return true if successful |
| 597 | bool _mi_free_delayed_block(mi_block_t* block) { |
| 598 | // get segment and page |
| 599 | const mi_segment_t* const segment = _mi_ptr_segment(block); |
| 600 | mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); |
| 601 | mi_assert_internal(_mi_thread_id() == segment->thread_id); |
| 602 | mi_page_t* const page = _mi_segment_page_of(segment, block); |
| 603 | |
| 604 | // Clear the no-delayed flag so delayed freeing is used again for this page. |
| 605 | // This must be done before collecting the free lists on this page -- otherwise |
| 606 | // some blocks may end up in the page `thread_free` list with no blocks in the |
| 607 | // heap `thread_delayed_free` list which may cause the page to be never freed! |
| 608 | // (it would only be freed if we happen to scan it in `mi_page_queue_find_free_ex`) |
| 609 | if (!_mi_page_try_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */)) { |
| 610 | return false; |
| 611 | } |
| 612 | |
| 613 | // collect all other non-local frees to ensure up-to-date `used` count |
| 614 | _mi_page_free_collect(page, false); |
| 615 | |
| 616 | // and free the block (possibly freeing the page as well since used is updated) |
| 617 | _mi_free_block(page, true, block); |
| 618 | return true; |
| 619 | } |
| 620 | |
| 621 | // Bytes available in a block |
| 622 | mi_decl_noinline static size_t mi_page_usable_aligned_size_of(const mi_segment_t* segment, const mi_page_t* page, const void* p) mi_attr_noexcept { |
| 623 | const mi_block_t* block = _mi_page_ptr_unalign(segment, page, p); |
| 624 | const size_t size = mi_page_usable_size_of(page, block); |
| 625 | const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block; |
| 626 | mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); |
| 627 | return (size - adjust); |
| 628 | } |
| 629 | |
| 630 | static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { |
| 631 | if (p == NULL) return 0; |
| 632 | const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); |
| 633 | const mi_page_t* const page = _mi_segment_page_of(segment, p); |
| 634 | if mi_likely(!mi_page_has_aligned(page)) { |
| 635 | const mi_block_t* block = (const mi_block_t*)p; |
| 636 | return mi_page_usable_size_of(page, block); |
| 637 | } |
| 638 | else { |
| 639 | // split out to separate routine for improved code generation |
| 640 | return mi_page_usable_aligned_size_of(segment, page, p); |
| 641 | } |
| 642 | } |
| 643 | |
| 644 | mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { |
| 645 | return _mi_usable_size(p, "mi_usable_size"); |
| 646 | } |
| 647 | |
| 648 | |
| 649 | // ------------------------------------------------------ |
| 650 | // Allocation extensions |
| 651 | // ------------------------------------------------------ |
| 652 | |
| 653 | void mi_free_size(void* p, size_t size) mi_attr_noexcept { |
| 654 | MI_UNUSED_RELEASE(size); |
| 655 | mi_assert(p == NULL || size <= _mi_usable_size(p,"mi_free_size")); |
| 656 | mi_free(p); |
| 657 | } |
| 658 | |
| 659 | void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { |
| 660 | MI_UNUSED_RELEASE(alignment); |
| 661 | mi_assert(((uintptr_t)p % alignment) == 0); |
| 662 | mi_free_size(p,size); |
| 663 | } |
| 664 | |
| 665 | void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { |
| 666 | MI_UNUSED_RELEASE(alignment); |
| 667 | mi_assert(((uintptr_t)p % alignment) == 0); |
| 668 | mi_free(p); |
| 669 | } |
| 670 | |
| 671 | mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { |
| 672 | size_t total; |
| 673 | if (mi_count_size_overflow(count,size,&total)) return NULL; |
| 674 | return mi_heap_zalloc(heap,total); |
| 675 | } |
| 676 | |
| 677 | mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { |
| 678 | return mi_heap_calloc(mi_prim_get_default_heap(),count,size); |
| 679 | } |
| 680 | |
| 681 | // Uninitialized `calloc` |
| 682 | mi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { |
| 683 | size_t total; |
| 684 | if (mi_count_size_overflow(count, size, &total)) return NULL; |
| 685 | return mi_heap_malloc(heap, total); |
| 686 | } |
| 687 | |
| 688 | mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { |
| 689 | return mi_heap_mallocn(mi_prim_get_default_heap(),count,size); |
| 690 | } |
| 691 | |
| 692 | // Expand (or shrink) in place (or fail) |
| 693 | void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { |
| 694 | #if MI_PADDING |
| 695 | // we do not shrink/expand with padding enabled |
| 696 | MI_UNUSED(p); MI_UNUSED(newsize); |
| 697 | return NULL; |
| 698 | #else |
| 699 | if (p == NULL) return NULL; |
| 700 | const size_t size = _mi_usable_size(p,"mi_expand"); |
| 701 | if (newsize > size) return NULL; |
| 702 | return p; // it fits |
| 703 | #endif |
| 704 | } |
| 705 | |
| 706 | void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept { |
| 707 | // if p == NULL then behave as malloc. |
| 708 | // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). |
| 709 | // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) |
| 710 | const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0) |
| 711 | if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) |
| 712 | mi_assert_internal(p!=NULL); |
| 713 | // todo: do not track as the usable size is still the same in the free; adjust potential padding? |
| 714 | // mi_track_resize(p,size,newsize) |
| 715 | // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } |
| 716 | return p; // reallocation still fits and not more than 50% waste |
| 717 | } |
| 718 | void* newp = mi_heap_malloc(heap,newsize); |
| 719 | if mi_likely(newp != NULL) { |
| 720 | if (zero && newsize > size) { |
| 721 | // also set last word in the previous allocation to zero to ensure any padding is zero-initialized |
| 722 | const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); |
| 723 | _mi_memzero((uint8_t*)newp + start, newsize - start); |
| 724 | } |
| 725 | else if (newsize == 0) { |
| 726 | ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) |
| 727 | } |
| 728 | if mi_likely(p != NULL) { |
| 729 | const size_t copysize = (newsize > size ? size : newsize); |
| 730 | mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking.. |
| 731 | _mi_memcpy(newp, p, copysize); |
| 732 | mi_free(p); // only free the original pointer if successful |
| 733 | } |
| 734 | } |
| 735 | return newp; |
| 736 | } |
| 737 | |
| 738 | mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { |
| 739 | return _mi_heap_realloc_zero(heap, p, newsize, false); |
| 740 | } |
| 741 | |
| 742 | mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { |
| 743 | size_t total; |
| 744 | if (mi_count_size_overflow(count, size, &total)) return NULL; |
| 745 | return mi_heap_realloc(heap, p, total); |
| 746 | } |
| 747 | |
| 748 | |
| 749 | // Reallocate but free `p` on errors |
| 750 | mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { |
| 751 | void* newp = mi_heap_realloc(heap, p, newsize); |
| 752 | if (newp==NULL && p!=NULL) mi_free(p); |
| 753 | return newp; |
| 754 | } |
| 755 | |
| 756 | mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { |
| 757 | return _mi_heap_realloc_zero(heap, p, newsize, true); |
| 758 | } |
| 759 | |
| 760 | mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { |
| 761 | size_t total; |
| 762 | if (mi_count_size_overflow(count, size, &total)) return NULL; |
| 763 | return mi_heap_rezalloc(heap, p, total); |
| 764 | } |
| 765 | |
| 766 | |
| 767 | mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { |
| 768 | return mi_heap_realloc(mi_prim_get_default_heap(),p,newsize); |
| 769 | } |
| 770 | |
| 771 | mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { |
| 772 | return mi_heap_reallocn(mi_prim_get_default_heap(),p,count,size); |
| 773 | } |
| 774 | |
| 775 | // Reallocate but free `p` on errors |
| 776 | mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { |
| 777 | return mi_heap_reallocf(mi_prim_get_default_heap(),p,newsize); |
| 778 | } |
| 779 | |
| 780 | mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { |
| 781 | return mi_heap_rezalloc(mi_prim_get_default_heap(), p, newsize); |
| 782 | } |
| 783 | |
| 784 | mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { |
| 785 | return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size); |
| 786 | } |
| 787 | |
| 788 | |
| 789 | |
| 790 | // ------------------------------------------------------ |
| 791 | // strdup, strndup, and realpath |
| 792 | // ------------------------------------------------------ |
| 793 | |
| 794 | // `strdup` using mi_malloc |
| 795 | mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { |
| 796 | if (s == NULL) return NULL; |
| 797 | size_t n = strlen(s); |
| 798 | char* t = (char*)mi_heap_malloc(heap,n+1); |
| 799 | if (t == NULL) return NULL; |
| 800 | _mi_memcpy(t, s, n); |
| 801 | t[n] = 0; |
| 802 | return t; |
| 803 | } |
| 804 | |
| 805 | mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { |
| 806 | return mi_heap_strdup(mi_prim_get_default_heap(), s); |
| 807 | } |
| 808 | |
| 809 | // `strndup` using mi_malloc |
| 810 | mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { |
| 811 | if (s == NULL) return NULL; |
| 812 | const char* end = (const char*)memchr(s, 0, n); // find end of string in the first `n` characters (returns NULL if not found) |
| 813 | const size_t m = (end != NULL ? (size_t)(end - s) : n); // `m` is the minimum of `n` or the end-of-string |
| 814 | mi_assert_internal(m <= n); |
| 815 | char* t = (char*)mi_heap_malloc(heap, m+1); |
| 816 | if (t == NULL) return NULL; |
| 817 | _mi_memcpy(t, s, m); |
| 818 | t[m] = 0; |
| 819 | return t; |
| 820 | } |
| 821 | |
| 822 | mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { |
| 823 | return mi_heap_strndup(mi_prim_get_default_heap(),s,n); |
| 824 | } |
| 825 | |
| 826 | #ifndef __wasi__ |
| 827 | // `realpath` using mi_malloc |
| 828 | #ifdef _WIN32 |
| 829 | #ifndef PATH_MAX |
| 830 | #define PATH_MAX MAX_PATH |
| 831 | #endif |
| 832 | #include <windows.h> |
| 833 | mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { |
| 834 | // todo: use GetFullPathNameW to allow longer file names |
| 835 | char buf[PATH_MAX]; |
| 836 | DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); |
| 837 | if (res == 0) { |
| 838 | errno = GetLastError(); return NULL; |
| 839 | } |
| 840 | else if (res > PATH_MAX) { |
| 841 | errno = EINVAL; return NULL; |
| 842 | } |
| 843 | else if (resolved_name != NULL) { |
| 844 | return resolved_name; |
| 845 | } |
| 846 | else { |
| 847 | return mi_heap_strndup(heap, buf, PATH_MAX); |
| 848 | } |
| 849 | } |
| 850 | #else |
| 851 | /* |
| 852 | #include <unistd.h> // pathconf |
| 853 | static size_t mi_path_max(void) { |
| 854 | static size_t path_max = 0; |
| 855 | if (path_max <= 0) { |
| 856 | long m = pathconf("/",_PC_PATH_MAX); |
| 857 | if (m <= 0) path_max = 4096; // guess |
| 858 | else if (m < 256) path_max = 256; // at least 256 |
| 859 | else path_max = m; |
| 860 | } |
| 861 | return path_max; |
| 862 | } |
| 863 | */ |
| 864 | char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { |
| 865 | if (resolved_name != NULL) { |
| 866 | return realpath(fname,resolved_name); |
| 867 | } |
| 868 | else { |
| 869 | char* rname = realpath(fname, NULL); |
| 870 | if (rname == NULL) return NULL; |
| 871 | char* result = mi_heap_strdup(heap, rname); |
| 872 | free(rname); // use regular free! (which may be redirected to our free but that's ok) |
| 873 | return result; |
| 874 | } |
| 875 | /* |
| 876 | const size_t n = mi_path_max(); |
| 877 | char* buf = (char*)mi_malloc(n+1); |
| 878 | if (buf == NULL) { |
| 879 | errno = ENOMEM; |
| 880 | return NULL; |
| 881 | } |
| 882 | char* rname = realpath(fname,buf); |
| 883 | char* result = mi_heap_strndup(heap,rname,n); // ok if `rname==NULL` |
| 884 | mi_free(buf); |
| 885 | return result; |
| 886 | } |
| 887 | */ |
| 888 | } |
| 889 | #endif |
| 890 | |
| 891 | mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { |
| 892 | return mi_heap_realpath(mi_prim_get_default_heap(),fname,resolved_name); |
| 893 | } |
| 894 | #endif |
| 895 | |
| 896 | /*------------------------------------------------------- |
| 897 | C++ new and new_aligned |
| 898 | The standard requires calling into `get_new_handler` and |
| 899 | throwing the bad_alloc exception on failure. If we compile |
| 900 | with a C++ compiler we can implement this precisely. If we |
| 901 | use a C compiler we cannot throw a `bad_alloc` exception |
| 902 | but we call `exit` instead (i.e. not returning). |
| 903 | -------------------------------------------------------*/ |
| 904 | |
| 905 | #ifdef __cplusplus |
| 906 | #include <new> |
| 907 | static bool mi_try_new_handler(bool nothrow) { |
| 908 | #if defined(_MSC_VER) || (__cplusplus >= 201103L) |
| 909 | std::new_handler h = std::get_new_handler(); |
| 910 | #else |
| 911 | std::new_handler h = std::set_new_handler(); |
| 912 | std::set_new_handler(h); |
| 913 | #endif |
| 914 | if (h==NULL) { |
| 915 | _mi_error_message(ENOMEM, "out of memory in 'new'"); |
| 916 | if (!nothrow) { |
| 917 | throw std::bad_alloc(); |
| 918 | } |
| 919 | return false; |
| 920 | } |
| 921 | else { |
| 922 | h(); |
| 923 | return true; |
| 924 | } |
| 925 | } |
| 926 | #else |
| 927 | typedef void (*std_new_handler_t)(void); |
| 928 | |
| 929 | #if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631 |
| 930 | std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) { |
| 931 | return NULL; |
| 932 | } |
| 933 | static std_new_handler_t mi_get_new_handler(void) { |
| 934 | return _ZSt15get_new_handlerv(); |
| 935 | } |
| 936 | #else |
| 937 | // note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`. |
| 938 | static std_new_handler_t mi_get_new_handler() { |
| 939 | return NULL; |
| 940 | } |
| 941 | #endif |
| 942 | |
| 943 | static bool mi_try_new_handler(bool nothrow) { |
| 944 | std_new_handler_t h = mi_get_new_handler(); |
| 945 | if (h==NULL) { |
| 946 | _mi_error_message(ENOMEM, "out of memory in 'new'"); |
| 947 | if (!nothrow) { |
| 948 | abort(); // cannot throw in plain C, use abort |
| 949 | } |
| 950 | return false; |
| 951 | } |
| 952 | else { |
| 953 | h(); |
| 954 | return true; |
| 955 | } |
| 956 | } |
| 957 | #endif |
| 958 | |
| 959 | mi_decl_export mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) { |
| 960 | void* p = NULL; |
| 961 | while(p == NULL && mi_try_new_handler(nothrow)) { |
| 962 | p = mi_heap_malloc(heap,size); |
| 963 | } |
| 964 | return p; |
| 965 | } |
| 966 | |
| 967 | static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { |
| 968 | return mi_heap_try_new(mi_prim_get_default_heap(), size, nothrow); |
| 969 | } |
| 970 | |
| 971 | |
| 972 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { |
| 973 | void* p = mi_heap_malloc(heap,size); |
| 974 | if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); |
| 975 | return p; |
| 976 | } |
| 977 | |
| 978 | mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { |
| 979 | return mi_heap_alloc_new(mi_prim_get_default_heap(), size); |
| 980 | } |
| 981 | |
| 982 | |
| 983 | mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { |
| 984 | size_t total; |
| 985 | if mi_unlikely(mi_count_size_overflow(count, size, &total)) { |
| 986 | mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc |
| 987 | return NULL; |
| 988 | } |
| 989 | else { |
| 990 | return mi_heap_alloc_new(heap,total); |
| 991 | } |
| 992 | } |
| 993 | |
| 994 | mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { |
| 995 | return mi_heap_alloc_new_n(mi_prim_get_default_heap(), size, count); |
| 996 | } |
| 997 | |
| 998 | |
| 999 | mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { |
| 1000 | void* p = mi_malloc(size); |
| 1001 | if mi_unlikely(p == NULL) return mi_try_new(size, true); |
| 1002 | return p; |
| 1003 | } |
| 1004 | |
| 1005 | mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { |
| 1006 | void* p; |
| 1007 | do { |
| 1008 | p = mi_malloc_aligned(size, alignment); |
| 1009 | } |
| 1010 | while(p == NULL && mi_try_new_handler(false)); |
| 1011 | return p; |
| 1012 | } |
| 1013 | |
| 1014 | mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { |
| 1015 | void* p; |
| 1016 | do { |
| 1017 | p = mi_malloc_aligned(size, alignment); |
| 1018 | } |
| 1019 | while(p == NULL && mi_try_new_handler(true)); |
| 1020 | return p; |
| 1021 | } |
| 1022 | |
| 1023 | mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { |
| 1024 | void* q; |
| 1025 | do { |
| 1026 | q = mi_realloc(p, newsize); |
| 1027 | } while (q == NULL && mi_try_new_handler(false)); |
| 1028 | return q; |
| 1029 | } |
| 1030 | |
| 1031 | mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { |
| 1032 | size_t total; |
| 1033 | if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { |
| 1034 | mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc |
| 1035 | return NULL; |
| 1036 | } |
| 1037 | else { |
| 1038 | return mi_new_realloc(p, total); |
| 1039 | } |
| 1040 | } |
| 1041 | |
| 1042 | // ------------------------------------------------------ |
| 1043 | // ensure explicit external inline definitions are emitted! |
| 1044 | // ------------------------------------------------------ |
| 1045 | |
| 1046 | #ifdef __cplusplus |
| 1047 | void* _mi_externs[] = { |
| 1048 | (void*)&_mi_page_malloc, |
| 1049 | (void*)&_mi_heap_malloc_zero, |
| 1050 | (void*)&_mi_heap_malloc_zero_ex, |
| 1051 | (void*)&mi_malloc, |
| 1052 | (void*)&mi_malloc_small, |
| 1053 | (void*)&mi_zalloc_small, |
| 1054 | (void*)&mi_heap_malloc, |
| 1055 | (void*)&mi_heap_zalloc, |
| 1056 | (void*)&mi_heap_malloc_small, |
| 1057 | // (void*)&mi_heap_alloc_new, |
| 1058 | // (void*)&mi_heap_alloc_new_n |
| 1059 | }; |
| 1060 | #endif |
| 1061 | |