microsoft/qdk
Publicmirrored fromhttps://github.com/microsoft/qdkAvailable
allocator/mimalloc-sys/mimalloc/include/mimalloc.h
565lines · modecode
| 1 | /* ---------------------------------------------------------------------------- |
| 2 | Copyright (c) 2018-2023, Microsoft Research, Daan Leijen |
| 3 | This is free software; you can redistribute it and/or modify it under the |
| 4 | terms of the MIT license. A copy of the license can be found in the file |
| 5 | "LICENSE" at the root of this distribution. |
| 6 | -----------------------------------------------------------------------------*/ |
| 7 | #pragma once |
| 8 | #ifndef MIMALLOC_H |
| 9 | #define MIMALLOC_H |
| 10 | |
| 11 | #define MI_MALLOC_VERSION 212 // major + 2 digits minor |
| 12 | |
| 13 | // ------------------------------------------------------ |
| 14 | // Compiler specific attributes |
| 15 | // ------------------------------------------------------ |
| 16 | |
| 17 | #ifdef __cplusplus |
| 18 | #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 |
| 19 | #define mi_attr_noexcept noexcept |
| 20 | #else |
| 21 | #define mi_attr_noexcept throw() |
| 22 | #endif |
| 23 | #else |
| 24 | #define mi_attr_noexcept |
| 25 | #endif |
| 26 | |
| 27 | #if defined(__cplusplus) && (__cplusplus >= 201703) |
| 28 | #define mi_decl_nodiscard [[nodiscard]] |
| 29 | #elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl |
| 30 | #define mi_decl_nodiscard __attribute__((warn_unused_result)) |
| 31 | #elif defined(_HAS_NODISCARD) |
| 32 | #define mi_decl_nodiscard _NODISCARD |
| 33 | #elif (_MSC_VER >= 1700) |
| 34 | #define mi_decl_nodiscard _Check_return_ |
| 35 | #else |
| 36 | #define mi_decl_nodiscard |
| 37 | #endif |
| 38 | |
| 39 | #if defined(_MSC_VER) || defined(__MINGW32__) |
| 40 | #if !defined(MI_SHARED_LIB) |
| 41 | #define mi_decl_export |
| 42 | #elif defined(MI_SHARED_LIB_EXPORT) |
| 43 | #define mi_decl_export __declspec(dllexport) |
| 44 | #else |
| 45 | #define mi_decl_export __declspec(dllimport) |
| 46 | #endif |
| 47 | #if defined(__MINGW32__) |
| 48 | #define mi_decl_restrict |
| 49 | #define mi_attr_malloc __attribute__((malloc)) |
| 50 | #else |
| 51 | #if (_MSC_VER >= 1900) && !defined(__EDG__) |
| 52 | #define mi_decl_restrict __declspec(allocator) __declspec(restrict) |
| 53 | #else |
| 54 | #define mi_decl_restrict __declspec(restrict) |
| 55 | #endif |
| 56 | #define mi_attr_malloc |
| 57 | #endif |
| 58 | #define mi_cdecl __cdecl |
| 59 | #define mi_attr_alloc_size(s) |
| 60 | #define mi_attr_alloc_size2(s1,s2) |
| 61 | #define mi_attr_alloc_align(p) |
| 62 | #elif defined(__GNUC__) // includes clang and icc |
| 63 | #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) |
| 64 | #define mi_decl_export __attribute__((visibility("default"))) |
| 65 | #else |
| 66 | #define mi_decl_export |
| 67 | #endif |
| 68 | #define mi_cdecl // leads to warnings... __attribute__((cdecl)) |
| 69 | #define mi_decl_restrict |
| 70 | #define mi_attr_malloc __attribute__((malloc)) |
| 71 | #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) |
| 72 | #define mi_attr_alloc_size(s) |
| 73 | #define mi_attr_alloc_size2(s1,s2) |
| 74 | #define mi_attr_alloc_align(p) |
| 75 | #elif defined(__INTEL_COMPILER) |
| 76 | #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) |
| 77 | #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) |
| 78 | #define mi_attr_alloc_align(p) |
| 79 | #else |
| 80 | #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) |
| 81 | #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) |
| 82 | #define mi_attr_alloc_align(p) __attribute__((alloc_align(p))) |
| 83 | #endif |
| 84 | #else |
| 85 | #define mi_cdecl |
| 86 | #define mi_decl_export |
| 87 | #define mi_decl_restrict |
| 88 | #define mi_attr_malloc |
| 89 | #define mi_attr_alloc_size(s) |
| 90 | #define mi_attr_alloc_size2(s1,s2) |
| 91 | #define mi_attr_alloc_align(p) |
| 92 | #endif |
| 93 | |
| 94 | // ------------------------------------------------------ |
| 95 | // Includes |
| 96 | // ------------------------------------------------------ |
| 97 | |
| 98 | #include <stddef.h> // size_t |
| 99 | #include <stdbool.h> // bool |
| 100 | #include <stdint.h> // INTPTR_MAX |
| 101 | |
| 102 | #ifdef __cplusplus |
| 103 | extern "C" { |
| 104 | #endif |
| 105 | |
| 106 | // ------------------------------------------------------ |
| 107 | // Standard malloc interface |
| 108 | // ------------------------------------------------------ |
| 109 | |
| 110 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 111 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); |
| 112 | mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); |
| 113 | mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); |
| 114 | |
| 115 | mi_decl_export void mi_free(void* p) mi_attr_noexcept; |
| 116 | mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc; |
| 117 | mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; |
| 118 | mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; |
| 119 | |
| 120 | // ------------------------------------------------------ |
| 121 | // Extended functionality |
| 122 | // ------------------------------------------------------ |
| 123 | #define MI_SMALL_WSIZE_MAX (128) |
| 124 | #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) |
| 125 | |
| 126 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 127 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 128 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 129 | |
| 130 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); |
| 131 | mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); |
| 132 | mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); |
| 133 | |
| 134 | mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; |
| 135 | mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; |
| 136 | |
| 137 | |
| 138 | // ------------------------------------------------------ |
| 139 | // Internals |
| 140 | // ------------------------------------------------------ |
| 141 | |
| 142 | typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg); |
| 143 | mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept; |
| 144 | |
| 145 | typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg); |
| 146 | mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept; |
| 147 | |
| 148 | typedef void (mi_cdecl mi_error_fun)(int err, void* arg); |
| 149 | mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); |
| 150 | |
| 151 | mi_decl_export void mi_collect(bool force) mi_attr_noexcept; |
| 152 | mi_decl_export int mi_version(void) mi_attr_noexcept; |
| 153 | mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; |
| 154 | mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; |
| 155 | mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL |
| 156 | mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; |
| 157 | |
| 158 | mi_decl_export void mi_process_init(void) mi_attr_noexcept; |
| 159 | mi_decl_export void mi_thread_init(void) mi_attr_noexcept; |
| 160 | mi_decl_export void mi_thread_done(void) mi_attr_noexcept; |
| 161 | mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; |
| 162 | |
| 163 | mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, |
| 164 | size_t* current_rss, size_t* peak_rss, |
| 165 | size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept; |
| 166 | |
| 167 | // ------------------------------------------------------------------------------------- |
| 168 | // Aligned allocation |
| 169 | // Note that `alignment` always follows `size` for consistency with unaligned |
| 170 | // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. |
| 171 | // ------------------------------------------------------------------------------------- |
| 172 | |
| 173 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); |
| 174 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 175 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); |
| 176 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 177 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); |
| 178 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); |
| 179 | mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); |
| 180 | mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); |
| 181 | |
| 182 | |
| 183 | // ------------------------------------------------------------------------------------- |
| 184 | // Heaps: first-class, but can only allocate from the same thread that created it. |
| 185 | // ------------------------------------------------------------------------------------- |
| 186 | |
| 187 | struct mi_heap_s; |
| 188 | typedef struct mi_heap_s mi_heap_t; |
| 189 | |
| 190 | mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); |
| 191 | mi_decl_export void mi_heap_delete(mi_heap_t* heap); |
| 192 | mi_decl_export void mi_heap_destroy(mi_heap_t* heap); |
| 193 | mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap); |
| 194 | mi_decl_export mi_heap_t* mi_heap_get_default(void); |
| 195 | mi_decl_export mi_heap_t* mi_heap_get_backing(void); |
| 196 | mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; |
| 197 | |
| 198 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); |
| 199 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); |
| 200 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); |
| 201 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); |
| 202 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); |
| 203 | |
| 204 | mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); |
| 205 | mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); |
| 206 | mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); |
| 207 | |
| 208 | mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc; |
| 209 | mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; |
| 210 | mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; |
| 211 | |
| 212 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); |
| 213 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); |
| 214 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); |
| 215 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); |
| 216 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); |
| 217 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); |
| 218 | mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); |
| 219 | mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); |
| 220 | |
| 221 | |
| 222 | // -------------------------------------------------------------------------------- |
| 223 | // Zero initialized re-allocation. |
| 224 | // Only valid on memory that was originally allocated with zero initialization too. |
| 225 | // e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc. |
| 226 | // see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992> |
| 227 | // -------------------------------------------------------------------------------- |
| 228 | |
| 229 | mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); |
| 230 | mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); |
| 231 | |
| 232 | mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); |
| 233 | mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); |
| 234 | mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); |
| 235 | mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3); |
| 236 | |
| 237 | mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); |
| 238 | mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); |
| 239 | |
| 240 | mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); |
| 241 | mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); |
| 242 | mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); |
| 243 | mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4); |
| 244 | |
| 245 | |
| 246 | // ------------------------------------------------------ |
| 247 | // Analysis |
| 248 | // ------------------------------------------------------ |
| 249 | |
| 250 | mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p); |
| 251 | mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p); |
| 252 | mi_decl_export bool mi_check_owned(const void* p); |
| 253 | |
| 254 | // An area of heap space contains blocks of a single size. |
| 255 | typedef struct mi_heap_area_s { |
| 256 | void* blocks; // start of the area containing heap blocks |
| 257 | size_t reserved; // bytes reserved for this area (virtual) |
| 258 | size_t committed; // current available bytes for this area |
| 259 | size_t used; // number of allocated blocks |
| 260 | size_t block_size; // size in bytes of each block |
| 261 | size_t full_block_size; // size in bytes of a full block including padding and metadata. |
| 262 | } mi_heap_area_t; |
| 263 | |
| 264 | typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); |
| 265 | |
| 266 | mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); |
| 267 | |
| 268 | // Experimental |
| 269 | mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; |
| 270 | mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; |
| 271 | |
| 272 | mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; |
| 273 | mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; |
| 274 | |
| 275 | mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; |
| 276 | mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; |
| 277 | |
| 278 | mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; |
| 279 | |
| 280 | // Experimental: heaps associated with specific memory arena's |
| 281 | typedef int mi_arena_id_t; |
| 282 | mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); |
| 283 | mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; |
| 284 | mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; |
| 285 | mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; |
| 286 | |
| 287 | #if MI_MALLOC_VERSION >= 182 |
| 288 | // Create a heap that only allocates in the specified arena |
| 289 | mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); |
| 290 | #endif |
| 291 | |
| 292 | // deprecated |
| 293 | mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; |
| 294 | |
| 295 | |
| 296 | // ------------------------------------------------------ |
| 297 | // Convenience |
| 298 | // ------------------------------------------------------ |
| 299 | |
| 300 | #define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) |
| 301 | #define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) |
| 302 | #define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) |
| 303 | #define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) |
| 304 | #define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) |
| 305 | #define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) |
| 306 | |
| 307 | #define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) |
| 308 | #define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) |
| 309 | #define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp))) |
| 310 | #define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp))) |
| 311 | #define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp))) |
| 312 | #define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp))) |
| 313 | |
| 314 | |
| 315 | // ------------------------------------------------------ |
| 316 | // Options |
| 317 | // ------------------------------------------------------ |
| 318 | |
| 319 | typedef enum mi_option_e { |
| 320 | // stable options |
| 321 | mi_option_show_errors, // print error messages |
| 322 | mi_option_show_stats, // print statistics on termination |
| 323 | mi_option_verbose, // print verbose messages |
| 324 | // the following options are experimental (see src/options.h) |
| 325 | mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1) |
| 326 | mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) |
| 327 | mi_option_purge_decommits, // should a memory purge decommit (or only reset) (=1) |
| 328 | mi_option_allow_large_os_pages, // allow large (2MiB) OS pages, implies eager commit |
| 329 | mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB/page) at startup |
| 330 | mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node |
| 331 | mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup |
| 332 | mi_option_deprecated_segment_cache, |
| 333 | mi_option_deprecated_page_reset, |
| 334 | mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination |
| 335 | mi_option_deprecated_segment_reset, |
| 336 | mi_option_eager_commit_delay, |
| 337 | mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. |
| 338 | mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. |
| 339 | mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) |
| 340 | mi_option_os_tag, // tag used for OS logging (macOS only for now) |
| 341 | mi_option_max_errors, // issue at most N error messages |
| 342 | mi_option_max_warnings, // issue at most N warning messages |
| 343 | mi_option_max_segment_reclaim, |
| 344 | mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe. |
| 345 | mi_option_arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit) |
| 346 | mi_option_arena_purge_mult, |
| 347 | mi_option_purge_extend_delay, |
| 348 | _mi_option_last, |
| 349 | // legacy option names |
| 350 | mi_option_large_os_pages = mi_option_allow_large_os_pages, |
| 351 | mi_option_eager_region_commit = mi_option_arena_eager_commit, |
| 352 | mi_option_reset_decommits = mi_option_purge_decommits, |
| 353 | mi_option_reset_delay = mi_option_purge_delay, |
| 354 | mi_option_abandoned_page_reset = mi_option_abandoned_page_purge |
| 355 | } mi_option_t; |
| 356 | |
| 357 | |
| 358 | mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); |
| 359 | mi_decl_export void mi_option_enable(mi_option_t option); |
| 360 | mi_decl_export void mi_option_disable(mi_option_t option); |
| 361 | mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); |
| 362 | mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); |
| 363 | |
| 364 | mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); |
| 365 | mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); |
| 366 | mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); |
| 367 | mi_decl_export void mi_option_set(mi_option_t option, long value); |
| 368 | mi_decl_export void mi_option_set_default(mi_option_t option, long value); |
| 369 | |
| 370 | |
| 371 | // ------------------------------------------------------------------------------------------------------- |
| 372 | // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. |
| 373 | // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) |
| 374 | // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing. |
| 375 | // ------------------------------------------------------------------------------------------------------- |
| 376 | |
| 377 | mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; |
| 378 | mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; |
| 379 | mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; |
| 380 | mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; |
| 381 | mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; |
| 382 | |
| 383 | mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; |
| 384 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); |
| 385 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 386 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 387 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); |
| 388 | |
| 389 | mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); |
| 390 | mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept; |
| 391 | mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; |
| 392 | mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; |
| 393 | |
| 394 | mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc; |
| 395 | mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; |
| 396 | mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; |
| 397 | mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; |
| 398 | |
| 399 | mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; |
| 400 | mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; |
| 401 | mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; |
| 402 | |
| 403 | // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. |
| 404 | // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). |
| 405 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); |
| 406 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); |
| 407 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); |
| 408 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); |
| 409 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); |
| 410 | mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); |
| 411 | mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); |
| 412 | |
| 413 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); |
| 414 | mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); |
| 415 | |
| 416 | #ifdef __cplusplus |
| 417 | } |
| 418 | #endif |
| 419 | |
| 420 | // --------------------------------------------------------------------------------------------- |
| 421 | // Implement the C++ std::allocator interface for use in STL containers. |
| 422 | // (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally) |
| 423 | // --------------------------------------------------------------------------------------------- |
| 424 | #ifdef __cplusplus |
| 425 | |
| 426 | #include <cstddef> // std::size_t |
| 427 | #include <cstdint> // PTRDIFF_MAX |
| 428 | #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 |
| 429 | #include <type_traits> // std::true_type |
| 430 | #include <utility> // std::forward |
| 431 | #endif |
| 432 | |
| 433 | template<class T> struct _mi_stl_allocator_common { |
| 434 | typedef T value_type; |
| 435 | typedef std::size_t size_type; |
| 436 | typedef std::ptrdiff_t difference_type; |
| 437 | typedef value_type& reference; |
| 438 | typedef value_type const& const_reference; |
| 439 | typedef value_type* pointer; |
| 440 | typedef value_type const* const_pointer; |
| 441 | |
| 442 | #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 |
| 443 | using propagate_on_container_copy_assignment = std::true_type; |
| 444 | using propagate_on_container_move_assignment = std::true_type; |
| 445 | using propagate_on_container_swap = std::true_type; |
| 446 | template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); } |
| 447 | template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); } |
| 448 | #else |
| 449 | void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } |
| 450 | void destroy(pointer p) { p->~value_type(); } |
| 451 | #endif |
| 452 | |
| 453 | size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } |
| 454 | pointer address(reference x) const { return &x; } |
| 455 | const_pointer address(const_reference x) const { return &x; } |
| 456 | }; |
| 457 | |
| 458 | template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> { |
| 459 | using typename _mi_stl_allocator_common<T>::size_type; |
| 460 | using typename _mi_stl_allocator_common<T>::value_type; |
| 461 | using typename _mi_stl_allocator_common<T>::pointer; |
| 462 | template <class U> struct rebind { typedef mi_stl_allocator<U> other; }; |
| 463 | |
| 464 | mi_stl_allocator() mi_attr_noexcept = default; |
| 465 | mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default; |
| 466 | template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { } |
| 467 | mi_stl_allocator select_on_container_copy_construction() const { return *this; } |
| 468 | void deallocate(T* p, size_type) { mi_free(p); } |
| 469 | |
| 470 | #if (__cplusplus >= 201703L) // C++17 |
| 471 | mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); } |
| 472 | mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } |
| 473 | #else |
| 474 | mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); } |
| 475 | #endif |
| 476 | |
| 477 | #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 |
| 478 | using is_always_equal = std::true_type; |
| 479 | #endif |
| 480 | }; |
| 481 | |
| 482 | template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; } |
| 483 | template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; } |
| 484 | |
| 485 | |
| 486 | #if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11 |
| 487 | #define MI_HAS_HEAP_STL_ALLOCATOR 1 |
| 488 | |
| 489 | #include <memory> // std::shared_ptr |
| 490 | |
| 491 | // Common base class for STL allocators in a specific heap |
| 492 | template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> { |
| 493 | using typename _mi_stl_allocator_common<T>::size_type; |
| 494 | using typename _mi_stl_allocator_common<T>::value_type; |
| 495 | using typename _mi_stl_allocator_common<T>::pointer; |
| 496 | |
| 497 | _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp) { } /* will not delete nor destroy the passed in heap */ |
| 498 | |
| 499 | #if (__cplusplus >= 201703L) // C++17 |
| 500 | mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } |
| 501 | mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } |
| 502 | #else |
| 503 | mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } |
| 504 | #endif |
| 505 | |
| 506 | #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 |
| 507 | using is_always_equal = std::false_type; |
| 508 | #endif |
| 509 | |
| 510 | void collect(bool force) { mi_heap_collect(this->heap.get(), force); } |
| 511 | template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); } |
| 512 | |
| 513 | protected: |
| 514 | std::shared_ptr<mi_heap_t> heap; |
| 515 | template<class U, bool D> friend struct _mi_heap_stl_allocator_common; |
| 516 | |
| 517 | _mi_heap_stl_allocator_common() { |
| 518 | mi_heap_t* hp = mi_heap_new(); |
| 519 | this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ |
| 520 | } |
| 521 | _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } |
| 522 | template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { } |
| 523 | |
| 524 | private: |
| 525 | static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } |
| 526 | static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } } |
| 527 | }; |
| 528 | |
| 529 | // STL allocator allocation in a specific heap |
| 530 | template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> { |
| 531 | using typename _mi_heap_stl_allocator_common<T, false>::size_type; |
| 532 | mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called |
| 533 | mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap |
| 534 | template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { } |
| 535 | |
| 536 | mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } |
| 537 | void deallocate(T* p, size_type) { mi_free(p); } |
| 538 | template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; }; |
| 539 | }; |
| 540 | |
| 541 | template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); } |
| 542 | template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); } |
| 543 | |
| 544 | |
| 545 | // STL allocator allocation in a specific heap, where `free` does nothing and |
| 546 | // the heap is destroyed in one go on destruction -- use with care! |
| 547 | template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> { |
| 548 | using typename _mi_heap_stl_allocator_common<T, true>::size_type; |
| 549 | mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called |
| 550 | mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap |
| 551 | template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { } |
| 552 | |
| 553 | mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } |
| 554 | void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ } |
| 555 | template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; }; |
| 556 | }; |
| 557 | |
| 558 | template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); } |
| 559 | template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); } |
| 560 | |
| 561 | #endif // C++11 |
| 562 | |
| 563 | #endif // __cplusplus |
| 564 | |
| 565 | #endif |