GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/qdk

Public

mirrored from https://github.com/microsoft/qdkAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.19.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

qdk/source/allocator/mimalloc-sys/mimalloc/src

source/allocator/mimalloc-sys/mimalloc/src/free.c

572lines · modeblame

RawDownload
Latest commit unavailable.
ce21e681Ian Davis11 months ago1/* ----------------------------------------------------------------------------
2Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#if !defined(MI_IN_ALLOC_C)
8#error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)"
9// add includes help an IDE
10#include "mimalloc.h"
11#include "mimalloc/internal.h"
12#include "mimalloc/prim.h" // _mi_prim_thread_id()
13#endif
14
15// forward declarations
16static void mi_check_padding(const mi_page_t* page, const mi_block_t* block);
17static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);
18static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block);
19static void mi_stat_free(const mi_page_t* page, const mi_block_t* block);
20
21
22// ------------------------------------------------------
23// Free
24// ------------------------------------------------------
25
26// forward declaration of multi-threaded free (`_mt`) (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON)
27static mi_decl_noinline void mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block);
28
29// regular free of a (thread local) block pointer
30// fast path written carefully to prevent spilling on the stack
31static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool track_stats, bool check_full)
32{
33// checks
34if mi_unlikely(mi_check_is_double_free(page, block)) return;
35mi_check_padding(page, block);
36if (track_stats) { mi_stat_free(page, block); }
37#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN && !MI_GUARDED
38if (!mi_page_is_huge(page)) { // huge page content may be already decommitted
39memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
40}
41#endif
42if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block)); } // faster then mi_usable_size as we already know the page and that p is unaligned
43
44// actual free: push on the local free list
45mi_block_set_next(page, block, page->local_free);
46page->local_free = block;
47if mi_unlikely(--page->used == 0) {
48_mi_page_retire(page);
49}
50else if mi_unlikely(check_full && mi_page_is_in_full(page)) {
51_mi_page_unfull(page);
52}
53}
54
55// Adjust a block that was allocated aligned, to the actual start of the block in the page.
56// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
57// `page_start` and `block_size` fields; however these are constant and the page won't be
58// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.
59mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
60mi_assert_internal(page!=NULL && p!=NULL);
61
62size_t diff = (uint8_t*)p - page->page_start;
63size_t adjust;
64if mi_likely(page->block_size_shift != 0) {
65adjust = diff & (((size_t)1 << page->block_size_shift) - 1);
66}
67else {
68adjust = diff % mi_page_block_size(page);
69}
70
71return (mi_block_t*)((uintptr_t)p - adjust);
72}
73
74// forward declaration for a MI_GUARDED build
75#if MI_GUARDED
76static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
77static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
78if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard(page, block, p); }
79}
80#else
81static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
82MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
83}
84#endif
85
86// free a local pointer (page parameter comes first for better codegen)
87static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept {
88MI_UNUSED(segment);
89mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p);
90mi_block_check_unguard(page, block, p);
91mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */);
92}
93
94// free a pointer owned by another thread (page parameter comes first for better codegen)
95static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept {
96mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865)
97mi_block_check_unguard(page, block, p);
98mi_free_block_mt(page, segment, block);
99}
100
101// generic free (for runtime integration)
102void mi_decl_noinline _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
103if (is_local) mi_free_generic_local(page,segment,p);
104else mi_free_generic_mt(page,segment,p);
105}
106
107// Get the segment data belonging to a pointer
108// This is just a single `and` in release mode but does further checks in debug mode
109// (and secure mode) to see if this was a valid pointer.
110static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg)
111{
112MI_UNUSED(msg);
113
114#if (MI_DEBUG>0)
115if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {
116_mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
117return NULL;
118}
119#endif
120
121mi_segment_t* const segment = _mi_ptr_segment(p);
122if mi_unlikely(segment==NULL) return segment;
123
124#if (MI_DEBUG>0)
125if mi_unlikely(!mi_is_in_heap_region(p)) {
126#if (MI_INTPTR_SIZE == 8 && defined(__linux__))
127if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640)
128#else
129{
130#endif
131_mi_warning_message("%s: pointer might not point to a valid heap region: %p\n"
132"(this may still be a valid very large allocation (over 64MiB))\n", msg, p);
133if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) {
134_mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p);
135}
136}
137}
138#endif
139#if (MI_DEBUG>0 || MI_SECURE>=4)
140if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) {
141_mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p);
142return NULL;
143}
144#endif
145
146return segment;
147}
148
149// Free a block
150// Fast path written carefully to prevent register spilling on the stack
151void mi_free(void* p) mi_attr_noexcept
152{
153mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free");
154if mi_unlikely(segment==NULL) return;
155
156const bool is_local = (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));
157mi_page_t* const page = _mi_segment_page_of(segment, p);
158
159if mi_likely(is_local) { // thread-local free?
160if mi_likely(page->flags.full_aligned == 0) { // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned)
161// thread-local, aligned, and not a full page
162mi_block_t* const block = (mi_block_t*)p;
163mi_free_block_local(page, block, true /* track stats */, false /* no need to check if the page is full */);
164}
165else {
166// page is full or contains (inner) aligned blocks; use generic path
167mi_free_generic_local(page, segment, p);
168}
169}
170else {
171// not thread-local; use generic path
172mi_free_generic_mt(page, segment, p);
173}
174}
175
176// return true if successful
177bool _mi_free_delayed_block(mi_block_t* block) {
178// get segment and page
179mi_assert_internal(block!=NULL);
180const mi_segment_t* const segment = _mi_ptr_segment(block);
181mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
182mi_assert_internal(_mi_thread_id() == segment->thread_id);
183mi_page_t* const page = _mi_segment_page_of(segment, block);
184
185// Clear the no-delayed flag so delayed freeing is used again for this page.
186// This must be done before collecting the free lists on this page -- otherwise
187// some blocks may end up in the page `thread_free` list with no blocks in the
188// heap `thread_delayed_free` list which may cause the page to be never freed!
189// (it would only be freed if we happen to scan it in `mi_page_queue_find_free_ex`)
190if (!_mi_page_try_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */)) {
191return false;
192}
193
194// collect all other non-local frees (move from `thread_free` to `free`) to ensure up-to-date `used` count
195_mi_page_free_collect(page, false);
196
197// and free the block (possibly freeing the page as well since `used` is updated)
198mi_free_block_local(page, block, false /* stats have already been adjusted */, true /* check for a full page */);
199return true;
200}
201
202// ------------------------------------------------------
203// Multi-threaded Free (`_mt`)
204// ------------------------------------------------------
205
206// Push a block that is owned by another thread on its page-local thread free
207// list or it's heap delayed free list. Such blocks are later collected by
208// the owning thread in `_mi_free_delayed_block`.
209static void mi_decl_noinline mi_free_block_delayed_mt( mi_page_t* page, mi_block_t* block )
210{
211// Try to put the block on either the page-local thread free list,
212// or the heap delayed free list (if this is the first non-local free in that page)
213mi_thread_free_t tfreex;
214bool use_delayed;
215mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);
216do {
217use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE);
218if mi_unlikely(use_delayed) {
219// unlikely: this only happens on the first concurrent free in a page that is in the full list
220tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING);
221}
222else {
223// usual: directly add to page thread_free list
224mi_block_set_next(page, block, mi_tf_block(tfree));
225tfreex = mi_tf_set_block(tfree,block);
226}
227} while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex));
228
229// If this was the first non-local free, we need to push it on the heap delayed free list instead
230if mi_unlikely(use_delayed) {
231// racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`)
232mi_heap_t* const heap = (mi_heap_t*)(mi_atomic_load_acquire(&page->xheap)); //mi_page_heap(page);
233mi_assert_internal(heap != NULL);
234if (heap != NULL) {
235// add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity)
236mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);
237do {
238mi_block_set_nextx(heap,block,dfree, heap->keys);
239} while (!mi_atomic_cas_ptr_weak_release(mi_block_t,&heap->thread_delayed_free, &dfree, block));
240}
241
242// and reset the MI_DELAYED_FREEING flag
243tfree = mi_atomic_load_relaxed(&page->xthread_free);
244do {
245tfreex = tfree;
246mi_assert_internal(mi_tf_delayed(tfree) == MI_DELAYED_FREEING);
247tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE);
248} while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex));
249}
250}
251
252// Multi-threaded free (`_mt`) (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON)
253static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block)
254{
255// first see if the segment was abandoned and if we can reclaim it into our thread
256if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 &&
257#if MI_HUGE_PAGE_ABANDON
258segment->page_kind != MI_PAGE_HUGE &&
259#endif
260mi_atomic_load_relaxed(&segment->thread_id) == 0 && // segment is abandoned?
261mi_prim_get_default_heap() != (mi_heap_t*)&_mi_heap_empty) // and we did not already exit this thread (without this check, a fresh heap will be initalized (issue #944))
262{
263// the segment is abandoned, try to reclaim it into our heap
264if (_mi_segment_attempt_reclaim(mi_heap_get_default(), segment)) {
265mi_assert_internal(_mi_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));
266mi_assert_internal(mi_heap_get_default()->tld->segments.subproc == segment->subproc);
267mi_free(block); // recursively free as now it will be a local free in our heap
268return;
269}
270}
271
272// The padding check may access the non-thread-owned page for the key values.
273// that is safe as these are constant and the page won't be freed (as the block is not freed yet).
274mi_check_padding(page, block);
275
276// adjust stats (after padding check and potentially recursive `mi_free` above)
277mi_stat_free(page, block); // stat_free may access the padding
278mi_track_free_size(block, mi_page_usable_size_of(page,block));
279
280// for small size, ensure we can fit the delayed thread pointers without triggering overflow detection
281_mi_padding_shrink(page, block, sizeof(mi_block_t));
282
283if (segment->kind == MI_SEGMENT_HUGE) {
284#if MI_HUGE_PAGE_ABANDON
285// huge page segments are always abandoned and can be freed immediately
286_mi_segment_huge_page_free(segment, page, block);
287return;
288#else
289// huge pages are special as they occupy the entire segment
290// as these are large we reset the memory occupied by the page so it is available to other threads
291// (as the owning thread needs to actually free the memory later).
292_mi_segment_huge_page_reset(segment, page, block);
293#endif
294}
295else {
296#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading
297memset(block, MI_DEBUG_FREED, mi_usable_size(block));
298#endif
299}
300
301// and finally free the actual block by pushing it on the owning heap
302// thread_delayed free list (or heap delayed free list)
303mi_free_block_delayed_mt(page,block);
304}
305
306
307// ------------------------------------------------------
308// Usable size
309// ------------------------------------------------------
310
311// Bytes available in a block
312static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {
313const mi_block_t* block = _mi_page_ptr_unalign(page, p);
314const size_t size = mi_page_usable_size_of(page, block);
315const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
316mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
317const size_t aligned_size = (size - adjust);
318#if MI_GUARDED
319if (mi_block_ptr_is_guarded(block, p)) {
320return aligned_size - _mi_os_page_size();
321}
322#endif
323return aligned_size;
324}
325
326static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept {
327const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg);
328if mi_unlikely(segment==NULL) return 0;
329const mi_page_t* const page = _mi_segment_page_of(segment, p);
330if mi_likely(!mi_page_has_aligned(page)) {
331const mi_block_t* block = (const mi_block_t*)p;
332return mi_page_usable_size_of(page, block);
333}
334else {
335// split out to separate routine for improved code generation
336return mi_page_usable_aligned_size_of(page, p);
337}
338}
339
340mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {
341return _mi_usable_size(p, "mi_usable_size");
342}
343
344
345// ------------------------------------------------------
346// Free variants
347// ------------------------------------------------------
348
349void mi_free_size(void* p, size_t size) mi_attr_noexcept {
350MI_UNUSED_RELEASE(size);
351#if MI_DEBUG
352const size_t available = _mi_usable_size(p,"mi_free_size");
353mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );
354#endif
355mi_free(p);
356}
357
358void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
359MI_UNUSED_RELEASE(alignment);
360mi_assert(((uintptr_t)p % alignment) == 0);
361mi_free_size(p,size);
362}
363
364void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
365MI_UNUSED_RELEASE(alignment);
366mi_assert(((uintptr_t)p % alignment) == 0);
367mi_free(p);
368}
369
370
371// ------------------------------------------------------
372// Check for double free in secure and debug mode
373// This is somewhat expensive so only enabled for secure mode 4
374// ------------------------------------------------------
375
376#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))
377// linear check if the free list contains a specific element
378static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {
379while (list != NULL) {
380if (elem==list) return true;
381list = mi_block_next(page, list);
382}
383return false;
384}
385
386static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {
387// The decoded value is in the same page (or NULL).
388// Walk the free lists to verify positively if it is already freed
389if (mi_list_contains(page, page->free, block) ||
390mi_list_contains(page, page->local_free, block) ||
391mi_list_contains(page, mi_page_thread_free(page), block))
392{
393_mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
394return true;
395}
396return false;
397}
398
399#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }
400
401static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
402bool is_double_free = false;
403mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field
404if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer?
405(n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?
406{
407// Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?
408// (continue in separate function to improve code generation)
409is_double_free = mi_check_is_double_freex(page, block);
410}
411return is_double_free;
412}
413#else
414static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
415MI_UNUSED(page);
416MI_UNUSED(block);
417return false;
418}
419#endif
420
421
422// ---------------------------------------------------------------------------
423// Check for heap block overflow by setting up padding at the end of the block
424// ---------------------------------------------------------------------------
425
426#if MI_PADDING // && !MI_TRACK_ENABLED
427static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {
428*bsize = mi_page_usable_block_size(page);
429const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
430mi_track_mem_defined(padding,sizeof(mi_padding_t));
431*delta = padding->delta;
432uint32_t canary = padding->canary;
433uintptr_t keys[2];
434keys[0] = page->keys[0];
435keys[1] = page->keys[1];
436bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);
437mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
438return ok;
439}
440
441// Return the exact usable size of a block.
442static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) {
443size_t bsize;
444size_t delta;
445bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
446mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
447return (ok ? bsize - delta : 0);
448}
449
450// When a non-thread-local block is freed, it becomes part of the thread delayed free
451// list that is freed later by the owning heap. If the exact usable size is too small to
452// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)
453// so it will later not trigger an overflow error in `mi_free_block`.
454void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
455size_t bsize;
456size_t delta;
457bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
458mi_assert_internal(ok);
459if (!ok || (bsize - delta) >= min_size) return; // usually already enough space
460mi_assert_internal(bsize >= min_size);
461if (bsize < min_size) return; // should never happen
462size_t new_delta = (bsize - min_size);
463mi_assert_internal(new_delta < bsize);
464mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);
465mi_track_mem_defined(padding,sizeof(mi_padding_t));
466padding->delta = (uint32_t)new_delta;
467mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
468}
469#else
470static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) {
471MI_UNUSED(block);
472return mi_page_usable_block_size(page);
473}
474
475void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
476MI_UNUSED(page);
477MI_UNUSED(block);
478MI_UNUSED(min_size);
479}
480#endif
481
482#if MI_PADDING && MI_PADDING_CHECK
483
484static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {
485size_t bsize;
486size_t delta;
487bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
488*size = *wrong = bsize;
489if (!ok) return false;
490mi_assert_internal(bsize >= delta);
491*size = bsize - delta;
492if (!mi_page_is_huge(page)) {
493uint8_t* fill = (uint8_t*)block + bsize - delta;
494const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes
495mi_track_mem_defined(fill, maxpad);
496for (size_t i = 0; i < maxpad; i++) {
497if (fill[i] != MI_DEBUG_PADDING) {
498*wrong = bsize - delta + i;
499ok = false;
500break;
501}
502}
503mi_track_mem_noaccess(fill, maxpad);
504}
505return ok;
506}
507
508static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
509size_t size;
510size_t wrong;
511if (!mi_verify_padding(page,block,&size,&wrong)) {
512_mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, size, wrong );
513}
514}
515
516#else
517
518static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
519MI_UNUSED(page);
520MI_UNUSED(block);
521}
522
523#endif
524
525// only maintain stats for smaller objects if requested
526#if (MI_STAT>0)
527static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
528MI_UNUSED(block);
529mi_heap_t* const heap = mi_heap_get_default();
530const size_t bsize = mi_page_usable_block_size(page);
531// #if (MI_STAT>1)
532// const size_t usize = mi_page_usable_size_of(page, block);
533// mi_heap_stat_decrease(heap, malloc_requested, usize);
534// #endif
535if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) {
536mi_heap_stat_decrease(heap, malloc_normal, bsize);
537#if (MI_STAT > 1)
538mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], 1);
539#endif
540}
541//else if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
542// mi_heap_stat_decrease(heap, malloc_large, bsize);
543//}
544else {
545mi_heap_stat_decrease(heap, malloc_huge, bsize);
546}
547}
548#else
549static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
550MI_UNUSED(page); MI_UNUSED(block);
551}
552#endif
553
554
555// Remove guard page when building with MI_GUARDED
556#if MI_GUARDED
557static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
558MI_UNUSED(p);
559mi_assert_internal(mi_block_ptr_is_guarded(block, p));
560mi_assert_internal(mi_page_has_aligned(page));
561mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
562mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
563
564const size_t bsize = mi_page_block_size(page);
565const size_t psize = _mi_os_page_size();
566mi_assert_internal(bsize > psize);
567mi_assert_internal(_mi_page_segment(page)->allow_decommit);
568void* gpage = (uint8_t*)block + bsize - psize;
569mi_assert_internal(_mi_is_aligned(gpage, psize));
570_mi_os_unprotect(gpage, psize);
571}
572#endif