[6db6fd1] | 1 | /*
|
---|
| 2 | * Copyright (c) 2009 Martin Decky
|
---|
| 3 | * Copyright (c) 2009 Petr Tuma
|
---|
| 4 | * All rights reserved.
|
---|
| 5 | *
|
---|
| 6 | * Redistribution and use in source and binary forms, with or without
|
---|
| 7 | * modification, are permitted provided that the following conditions
|
---|
| 8 | * are met:
|
---|
| 9 | *
|
---|
| 10 | * - Redistributions of source code must retain the above copyright
|
---|
| 11 | * notice, this list of conditions and the following disclaimer.
|
---|
| 12 | * - Redistributions in binary form must reproduce the above copyright
|
---|
| 13 | * notice, this list of conditions and the following disclaimer in the
|
---|
| 14 | * documentation and/or other materials provided with the distribution.
|
---|
| 15 | * - The name of the author may not be used to endorse or promote products
|
---|
| 16 | * derived from this software without specific prior written permission.
|
---|
| 17 | *
|
---|
| 18 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
---|
| 19 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
---|
| 20 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
---|
| 21 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
---|
| 22 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
---|
| 23 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
---|
| 24 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
---|
| 25 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
---|
| 26 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
---|
| 27 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
---|
| 28 | */
|
---|
| 29 |
|
---|
| 30 | /** @addtogroup libc
|
---|
| 31 | * @{
|
---|
| 32 | */
|
---|
| 33 | /** @file
|
---|
| 34 | */
|
---|
| 35 |
|
---|
| 36 | #include <malloc.h>
|
---|
| 37 | #include <bool.h>
|
---|
| 38 | #include <as.h>
|
---|
| 39 | #include <align.h>
|
---|
| 40 | #include <macros.h>
|
---|
| 41 | #include <assert.h>
|
---|
| 42 | #include <errno.h>
|
---|
| 43 | #include <bitops.h>
|
---|
| 44 | #include <mem.h>
|
---|
[3292623] | 45 | #include <futex.h>
|
---|
[d161715] | 46 | #include <stdlib.h>
|
---|
[6db6fd1] | 47 | #include <adt/gcdlcm.h>
|
---|
[47b7006] | 48 | #include "private/malloc.h"
|
---|
[6db6fd1] | 49 |
|
---|
[0b37882] | 50 | /** Magic used in heap headers. */
|
---|
| 51 | #define HEAP_BLOCK_HEAD_MAGIC UINT32_C(0xBEEF0101)
|
---|
[6db6fd1] | 52 |
|
---|
[0b37882] | 53 | /** Magic used in heap footers. */
|
---|
| 54 | #define HEAP_BLOCK_FOOT_MAGIC UINT32_C(0xBEEF0202)
|
---|
[6db6fd1] | 55 |
|
---|
[0b37882] | 56 | /** Magic used in heap descriptor. */
|
---|
| 57 | #define HEAP_AREA_MAGIC UINT32_C(0xBEEFCAFE)
|
---|
| 58 |
|
---|
| 59 | /** Allocation alignment.
|
---|
| 60 | *
|
---|
| 61 | * This also covers the alignment of fields
|
---|
| 62 | * in the heap header and footer.
|
---|
| 63 | *
|
---|
| 64 | */
|
---|
[6db6fd1] | 65 | #define BASE_ALIGN 16
|
---|
| 66 |
|
---|
[013a5d7] | 67 | /** Heap shrink granularity
|
---|
| 68 | *
|
---|
| 69 | * Try not to pump and stress the heap to much
|
---|
| 70 | * by shrinking and enlarging it too often.
|
---|
| 71 | * A heap area won't shrunk if it the released
|
---|
| 72 | * free block is smaller than this constant.
|
---|
| 73 | *
|
---|
| 74 | */
|
---|
| 75 | #define SHRINK_GRANULARITY (64 * PAGE_SIZE)
|
---|
| 76 |
|
---|
[0b37882] | 77 | /** Overhead of each heap block. */
|
---|
| 78 | #define STRUCT_OVERHEAD \
|
---|
| 79 | (sizeof(heap_block_head_t) + sizeof(heap_block_foot_t))
|
---|
| 80 |
|
---|
| 81 | /** Calculate real size of a heap block.
|
---|
| 82 | *
|
---|
| 83 | * Add header and footer size.
|
---|
| 84 | *
|
---|
[6db6fd1] | 85 | */
|
---|
[0b37882] | 86 | #define GROSS_SIZE(size) ((size) + STRUCT_OVERHEAD)
|
---|
[6db6fd1] | 87 |
|
---|
[0b37882] | 88 | /** Calculate net size of a heap block.
|
---|
| 89 | *
|
---|
| 90 | * Subtract header and footer size.
|
---|
[6db6fd1] | 91 | *
|
---|
| 92 | */
|
---|
[0b37882] | 93 | #define NET_SIZE(size) ((size) - STRUCT_OVERHEAD)
|
---|
[6db6fd1] | 94 |
|
---|
[0b37882] | 95 | /** Get first block in heap area.
|
---|
| 96 | *
|
---|
[6db6fd1] | 97 | */
|
---|
[013a5d7] | 98 | #define AREA_FIRST_BLOCK_HEAD(area) \
|
---|
[0b37882] | 99 | (ALIGN_UP(((uintptr_t) (area)) + sizeof(heap_area_t), BASE_ALIGN))
|
---|
[6db6fd1] | 100 |
|
---|
[013a5d7] | 101 | /** Get last block in heap area.
|
---|
| 102 | *
|
---|
| 103 | */
|
---|
| 104 | #define AREA_LAST_BLOCK_FOOT(area) \
|
---|
| 105 | (((uintptr_t) (area)->end) - sizeof(heap_block_foot_t))
|
---|
| 106 |
|
---|
| 107 | /** Get header in heap block.
|
---|
| 108 | *
|
---|
| 109 | */
|
---|
| 110 | #define BLOCK_HEAD(foot) \
|
---|
| 111 | ((heap_block_head_t *) \
|
---|
| 112 | (((uintptr_t) (foot)) + sizeof(heap_block_foot_t) - (foot)->size))
|
---|
| 113 |
|
---|
[0b37882] | 114 | /** Get footer in heap block.
|
---|
| 115 | *
|
---|
[6db6fd1] | 116 | */
|
---|
[0b37882] | 117 | #define BLOCK_FOOT(head) \
|
---|
| 118 | ((heap_block_foot_t *) \
|
---|
[013a5d7] | 119 | (((uintptr_t) (head)) + (head)->size - sizeof(heap_block_foot_t)))
|
---|
[0b37882] | 120 |
|
---|
| 121 | /** Heap area.
|
---|
| 122 | *
|
---|
| 123 | * The memory managed by the heap allocator is divided into
|
---|
| 124 | * multiple discontinuous heaps. Each heap is represented
|
---|
| 125 | * by a separate address space area which has this structure
|
---|
| 126 | * at its very beginning.
|
---|
| 127 | *
|
---|
| 128 | */
|
---|
| 129 | typedef struct heap_area {
|
---|
| 130 | /** Start of the heap area (including this structure)
|
---|
| 131 | *
|
---|
| 132 | * Aligned on page boundary.
|
---|
| 133 | *
|
---|
| 134 | */
|
---|
| 135 | void *start;
|
---|
| 136 |
|
---|
| 137 | /** End of the heap area (aligned on page boundary) */
|
---|
| 138 | void *end;
|
---|
| 139 |
|
---|
[013a5d7] | 140 | /** Previous heap area */
|
---|
| 141 | struct heap_area *prev;
|
---|
| 142 |
|
---|
[0b37882] | 143 | /** Next heap area */
|
---|
| 144 | struct heap_area *next;
|
---|
| 145 |
|
---|
| 146 | /** A magic value */
|
---|
| 147 | uint32_t magic;
|
---|
| 148 | } heap_area_t;
|
---|
[6db6fd1] | 149 |
|
---|
| 150 | /** Header of a heap block
|
---|
| 151 | *
|
---|
| 152 | */
|
---|
| 153 | typedef struct {
|
---|
| 154 | /* Size of the block (including header and footer) */
|
---|
| 155 | size_t size;
|
---|
| 156 |
|
---|
| 157 | /* Indication of a free block */
|
---|
| 158 | bool free;
|
---|
| 159 |
|
---|
[0b37882] | 160 | /** Heap area this block belongs to */
|
---|
| 161 | heap_area_t *area;
|
---|
| 162 |
|
---|
[6db6fd1] | 163 | /* A magic value to detect overwrite of heap header */
|
---|
| 164 | uint32_t magic;
|
---|
| 165 | } heap_block_head_t;
|
---|
| 166 |
|
---|
| 167 | /** Footer of a heap block
|
---|
| 168 | *
|
---|
| 169 | */
|
---|
| 170 | typedef struct {
|
---|
| 171 | /* Size of the block (including header and footer) */
|
---|
| 172 | size_t size;
|
---|
| 173 |
|
---|
| 174 | /* A magic value to detect overwrite of heap footer */
|
---|
| 175 | uint32_t magic;
|
---|
| 176 | } heap_block_foot_t;
|
---|
| 177 |
|
---|
[0b37882] | 178 | /** First heap area */
|
---|
| 179 | static heap_area_t *first_heap_area = NULL;
|
---|
[6db6fd1] | 180 |
|
---|
[0b37882] | 181 | /** Last heap area */
|
---|
| 182 | static heap_area_t *last_heap_area = NULL;
|
---|
[6db6fd1] | 183 |
|
---|
[0b37882] | 184 | /** Next heap block to examine (next fit algorithm) */
|
---|
| 185 | static heap_block_head_t *next = NULL;
|
---|
[6db6fd1] | 186 |
|
---|
[0b37882] | 187 | /** Futex for thread-safe heap manipulation */
|
---|
| 188 | static futex_t malloc_futex = FUTEX_INITIALIZER;
|
---|
[6db6fd1] | 189 |
|
---|
| 190 | /** Initialize a heap block
|
---|
| 191 | *
|
---|
[0b37882] | 192 | * Fill in the structures related to a heap block.
|
---|
[3292623] | 193 | * Should be called only inside the critical section.
|
---|
[6db6fd1] | 194 | *
|
---|
| 195 | * @param addr Address of the block.
|
---|
| 196 | * @param size Size of the block including the header and the footer.
|
---|
| 197 | * @param free Indication of a free block.
|
---|
[0b37882] | 198 | * @param area Heap area the block belongs to.
|
---|
[6db6fd1] | 199 | *
|
---|
| 200 | */
|
---|
[0b37882] | 201 | static void block_init(void *addr, size_t size, bool free, heap_area_t *area)
|
---|
[6db6fd1] | 202 | {
|
---|
| 203 | /* Calculate the position of the header and the footer */
|
---|
| 204 | heap_block_head_t *head = (heap_block_head_t *) addr;
|
---|
| 205 |
|
---|
| 206 | head->size = size;
|
---|
| 207 | head->free = free;
|
---|
[0b37882] | 208 | head->area = area;
|
---|
[6db6fd1] | 209 | head->magic = HEAP_BLOCK_HEAD_MAGIC;
|
---|
| 210 |
|
---|
[0b37882] | 211 | heap_block_foot_t *foot = BLOCK_FOOT(head);
|
---|
| 212 |
|
---|
[6db6fd1] | 213 | foot->size = size;
|
---|
| 214 | foot->magic = HEAP_BLOCK_FOOT_MAGIC;
|
---|
| 215 | }
|
---|
| 216 |
|
---|
| 217 | /** Check a heap block
|
---|
| 218 | *
|
---|
| 219 | * Verifies that the structures related to a heap block still contain
|
---|
| 220 | * the magic constants. This helps detect heap corruption early on.
|
---|
[3292623] | 221 | * Should be called only inside the critical section.
|
---|
[6db6fd1] | 222 | *
|
---|
| 223 | * @param addr Address of the block.
|
---|
| 224 | *
|
---|
| 225 | */
|
---|
| 226 | static void block_check(void *addr)
|
---|
| 227 | {
|
---|
| 228 | heap_block_head_t *head = (heap_block_head_t *) addr;
|
---|
| 229 |
|
---|
| 230 | assert(head->magic == HEAP_BLOCK_HEAD_MAGIC);
|
---|
| 231 |
|
---|
[0b37882] | 232 | heap_block_foot_t *foot = BLOCK_FOOT(head);
|
---|
[6db6fd1] | 233 |
|
---|
| 234 | assert(foot->magic == HEAP_BLOCK_FOOT_MAGIC);
|
---|
| 235 | assert(head->size == foot->size);
|
---|
| 236 | }
|
---|
| 237 |
|
---|
[0b37882] | 238 | /** Check a heap area structure
|
---|
[013a5d7] | 239 | *
|
---|
| 240 | * Should be called only inside the critical section.
|
---|
[3292623] | 241 | *
|
---|
[0b37882] | 242 | * @param addr Address of the heap area.
|
---|
[3292623] | 243 | *
|
---|
[0b37882] | 244 | */
|
---|
| 245 | static void area_check(void *addr)
|
---|
| 246 | {
|
---|
| 247 | heap_area_t *area = (heap_area_t *) addr;
|
---|
| 248 |
|
---|
| 249 | assert(area->magic == HEAP_AREA_MAGIC);
|
---|
[013a5d7] | 250 | assert(addr == area->start);
|
---|
[0b37882] | 251 | assert(area->start < area->end);
|
---|
| 252 | assert(((uintptr_t) area->start % PAGE_SIZE) == 0);
|
---|
| 253 | assert(((uintptr_t) area->end % PAGE_SIZE) == 0);
|
---|
| 254 | }
|
---|
| 255 |
|
---|
| 256 | /** Create new heap area
|
---|
| 257 | *
|
---|
[013a5d7] | 258 | * Should be called only inside the critical section.
|
---|
| 259 | *
|
---|
| 260 | * @param size Size of the area.
|
---|
[3292623] | 261 | *
|
---|
| 262 | */
|
---|
[0b37882] | 263 | static bool area_create(size_t size)
|
---|
[6db6fd1] | 264 | {
|
---|
[0b37882] | 265 | void *start = as_get_mappable_page(size);
|
---|
| 266 | if (start == NULL)
|
---|
[6db6fd1] | 267 | return false;
|
---|
[0b37882] | 268 |
|
---|
| 269 | /* Align the heap area on page boundary */
|
---|
| 270 | void *astart = (void *) ALIGN_UP((uintptr_t) start, PAGE_SIZE);
|
---|
| 271 | size_t asize = ALIGN_UP(size, PAGE_SIZE);
|
---|
| 272 |
|
---|
| 273 | astart = as_area_create(astart, asize, AS_AREA_WRITE | AS_AREA_READ);
|
---|
| 274 | if (astart == (void *) -1)
|
---|
[e70edd1] | 275 | return false;
|
---|
[6db6fd1] | 276 |
|
---|
[0b37882] | 277 | heap_area_t *area = (heap_area_t *) astart;
|
---|
| 278 |
|
---|
| 279 | area->start = astart;
|
---|
[013a5d7] | 280 | area->end = (void *) ((uintptr_t) astart + asize);
|
---|
| 281 | area->prev = NULL;
|
---|
[0b37882] | 282 | area->next = NULL;
|
---|
| 283 | area->magic = HEAP_AREA_MAGIC;
|
---|
| 284 |
|
---|
[013a5d7] | 285 | void *block = (void *) AREA_FIRST_BLOCK_HEAD(area);
|
---|
[0b37882] | 286 | size_t bsize = (size_t) (area->end - block);
|
---|
| 287 |
|
---|
| 288 | block_init(block, bsize, true, area);
|
---|
| 289 |
|
---|
| 290 | if (last_heap_area == NULL) {
|
---|
| 291 | first_heap_area = area;
|
---|
| 292 | last_heap_area = area;
|
---|
| 293 | } else {
|
---|
[013a5d7] | 294 | area->prev = last_heap_area;
|
---|
[0b37882] | 295 | last_heap_area->next = area;
|
---|
| 296 | last_heap_area = area;
|
---|
| 297 | }
|
---|
| 298 |
|
---|
| 299 | return true;
|
---|
| 300 | }
|
---|
| 301 |
|
---|
| 302 | /** Try to enlarge a heap area
|
---|
[013a5d7] | 303 | *
|
---|
| 304 | * Should be called only inside the critical section.
|
---|
[0b37882] | 305 | *
|
---|
| 306 | * @param area Heap area to grow.
|
---|
[013a5d7] | 307 | * @param size Gross size to grow (bytes).
|
---|
| 308 | *
|
---|
| 309 | * @return True if successful.
|
---|
[0b37882] | 310 | *
|
---|
| 311 | */
|
---|
| 312 | static bool area_grow(heap_area_t *area, size_t size)
|
---|
| 313 | {
|
---|
| 314 | if (size == 0)
|
---|
| 315 | return true;
|
---|
| 316 |
|
---|
| 317 | area_check(area);
|
---|
| 318 |
|
---|
| 319 | /* New heap area size */
|
---|
[013a5d7] | 320 | size_t gross_size = (size_t) (area->end - area->start) + size;
|
---|
| 321 | size_t asize = ALIGN_UP(gross_size, PAGE_SIZE);
|
---|
| 322 | void *end = (void *) ((uintptr_t) area->start + asize);
|
---|
[0b37882] | 323 |
|
---|
| 324 | /* Check for overflow */
|
---|
| 325 | if (end < area->start)
|
---|
| 326 | return false;
|
---|
[6db6fd1] | 327 |
|
---|
[0b37882] | 328 | /* Resize the address space area */
|
---|
| 329 | int ret = as_area_resize(area->start, asize, 0);
|
---|
| 330 | if (ret != EOK)
|
---|
[6db6fd1] | 331 | return false;
|
---|
| 332 |
|
---|
[0b37882] | 333 | /* Add new free block */
|
---|
[013a5d7] | 334 | size_t net_size = (size_t) (end - area->end);
|
---|
| 335 | if (net_size > 0)
|
---|
| 336 | block_init(area->end, net_size, true, area);
|
---|
[6db6fd1] | 337 |
|
---|
[0b37882] | 338 | /* Update heap area parameters */
|
---|
| 339 | area->end = end;
|
---|
| 340 |
|
---|
| 341 | return true;
|
---|
| 342 | }
|
---|
| 343 |
|
---|
| 344 | /** Try to enlarge any of the heap areas
|
---|
[013a5d7] | 345 | *
|
---|
| 346 | * Should be called only inside the critical section.
|
---|
[0b37882] | 347 | *
|
---|
| 348 | * @param size Gross size of item to allocate (bytes).
|
---|
| 349 | *
|
---|
| 350 | */
|
---|
| 351 | static bool heap_grow(size_t size)
|
---|
| 352 | {
|
---|
| 353 | if (size == 0)
|
---|
[6db6fd1] | 354 | return true;
|
---|
[0b37882] | 355 |
|
---|
| 356 | /* First try to enlarge some existing area */
|
---|
[013a5d7] | 357 | for (heap_area_t *area = first_heap_area; area != NULL;
|
---|
| 358 | area = area->next) {
|
---|
[0b37882] | 359 | if (area_grow(area, size))
|
---|
| 360 | return true;
|
---|
[6db6fd1] | 361 | }
|
---|
| 362 |
|
---|
[0b37882] | 363 | /* Eventually try to create a new area */
|
---|
[013a5d7] | 364 | return area_create(AREA_FIRST_BLOCK_HEAD(size));
|
---|
[6db6fd1] | 365 | }
|
---|
| 366 |
|
---|
[013a5d7] | 367 | /** Try to shrink heap
|
---|
[3292623] | 368 | *
|
---|
[013a5d7] | 369 | * Should be called only inside the critical section.
|
---|
[0b37882] | 370 | * In all cases the next pointer is reset.
|
---|
[3292623] | 371 | *
|
---|
[013a5d7] | 372 | * @param area Last modified heap area.
|
---|
| 373 | *
|
---|
[3292623] | 374 | */
|
---|
[013a5d7] | 375 | static void heap_shrink(heap_area_t *area)
|
---|
[6db6fd1] | 376 | {
|
---|
[013a5d7] | 377 | area_check(area);
|
---|
| 378 |
|
---|
| 379 | heap_block_foot_t *last_foot =
|
---|
| 380 | (heap_block_foot_t *) AREA_LAST_BLOCK_FOOT(area);
|
---|
| 381 | heap_block_head_t *last_head = BLOCK_HEAD(last_foot);
|
---|
| 382 |
|
---|
| 383 | block_check((void *) last_head);
|
---|
| 384 | assert(last_head->area == area);
|
---|
| 385 |
|
---|
| 386 | if (last_head->free) {
|
---|
| 387 | /*
|
---|
| 388 | * The last block of the heap area is
|
---|
| 389 | * unused. The area might be potentially
|
---|
| 390 | * shrunk.
|
---|
| 391 | */
|
---|
| 392 |
|
---|
| 393 | heap_block_head_t *first_head =
|
---|
| 394 | (heap_block_head_t *) AREA_FIRST_BLOCK_HEAD(area);
|
---|
| 395 |
|
---|
| 396 | block_check((void *) first_head);
|
---|
| 397 | assert(first_head->area == area);
|
---|
| 398 |
|
---|
[e6eee2b] | 399 | size_t shrink_size = ALIGN_DOWN(last_head->size, PAGE_SIZE);
|
---|
| 400 |
|
---|
[013a5d7] | 401 | if (first_head == last_head) {
|
---|
| 402 | /*
|
---|
| 403 | * The entire heap area consists of a single
|
---|
| 404 | * free heap block. This means we can get rid
|
---|
| 405 | * of it entirely.
|
---|
| 406 | */
|
---|
| 407 |
|
---|
| 408 | heap_area_t *prev = area->prev;
|
---|
| 409 | heap_area_t *next = area->next;
|
---|
| 410 |
|
---|
| 411 | if (prev != NULL) {
|
---|
| 412 | area_check(prev);
|
---|
| 413 | prev->next = next;
|
---|
| 414 | } else
|
---|
| 415 | first_heap_area = next;
|
---|
| 416 |
|
---|
| 417 | if (next != NULL) {
|
---|
| 418 | area_check(next);
|
---|
| 419 | next->prev = prev;
|
---|
| 420 | } else
|
---|
| 421 | last_heap_area = prev;
|
---|
| 422 |
|
---|
| 423 | as_area_destroy(area->start);
|
---|
[e6eee2b] | 424 | } else if (shrink_size >= SHRINK_GRANULARITY) {
|
---|
[013a5d7] | 425 | /*
|
---|
| 426 | * Make sure that we always shrink the area
|
---|
| 427 | * by a multiple of page size and update
|
---|
| 428 | * the block layout accordingly.
|
---|
| 429 | */
|
---|
| 430 |
|
---|
| 431 | size_t asize = (size_t) (area->end - area->start) - shrink_size;
|
---|
| 432 | void *end = (void *) ((uintptr_t) area->start + asize);
|
---|
| 433 |
|
---|
| 434 | /* Resize the address space area */
|
---|
| 435 | int ret = as_area_resize(area->start, asize, 0);
|
---|
| 436 | if (ret != EOK)
|
---|
| 437 | abort();
|
---|
| 438 |
|
---|
| 439 | /* Update heap area parameters */
|
---|
| 440 | area->end = end;
|
---|
| 441 |
|
---|
| 442 | /* Update block layout */
|
---|
| 443 | void *last = (void *) last_head;
|
---|
| 444 | size_t excess = (size_t) (area->end - last);
|
---|
| 445 |
|
---|
| 446 | if (excess > 0) {
|
---|
| 447 | if (excess >= STRUCT_OVERHEAD) {
|
---|
| 448 | /*
|
---|
| 449 | * The previous block cannot be free and there
|
---|
| 450 | * is enough free space left in the area to
|
---|
| 451 | * create a new free block.
|
---|
| 452 | */
|
---|
| 453 | block_init(last, excess, true, area);
|
---|
| 454 | } else {
|
---|
| 455 | /*
|
---|
| 456 | * The excess is small. Therefore just enlarge
|
---|
| 457 | * the previous block.
|
---|
| 458 | */
|
---|
| 459 | heap_block_foot_t *prev_foot = (heap_block_foot_t *)
|
---|
| 460 | (((uintptr_t) last_head) - sizeof(heap_block_foot_t));
|
---|
| 461 | heap_block_head_t *prev_head = BLOCK_HEAD(prev_foot);
|
---|
| 462 |
|
---|
| 463 | block_check((void *) prev_head);
|
---|
| 464 |
|
---|
| 465 | block_init(prev_head, prev_head->size + excess,
|
---|
| 466 | prev_head->free, area);
|
---|
| 467 | }
|
---|
| 468 | }
|
---|
| 469 | }
|
---|
| 470 | }
|
---|
| 471 |
|
---|
[0b37882] | 472 | next = NULL;
|
---|
[6db6fd1] | 473 | }
|
---|
| 474 |
|
---|
| 475 | /** Initialize the heap allocator
|
---|
| 476 | *
|
---|
[47b7006] | 477 | * Create initial heap memory area. This routine is
|
---|
| 478 | * only called from libc initialization, thus we do not
|
---|
| 479 | * take any locks.
|
---|
[6db6fd1] | 480 | *
|
---|
| 481 | */
|
---|
[47b7006] | 482 | void __malloc_init(void)
|
---|
[6db6fd1] | 483 | {
|
---|
[0b37882] | 484 | if (!area_create(PAGE_SIZE))
|
---|
[47b7006] | 485 | abort();
|
---|
[6db6fd1] | 486 | }
|
---|
| 487 |
|
---|
[3292623] | 488 | /** Split heap block and mark it as used.
|
---|
| 489 | *
|
---|
| 490 | * Should be called only inside the critical section.
|
---|
| 491 | *
|
---|
| 492 | * @param cur Heap block to split.
|
---|
| 493 | * @param size Number of bytes to split and mark from the beginning
|
---|
| 494 | * of the block.
|
---|
| 495 | *
|
---|
| 496 | */
|
---|
[6db6fd1] | 497 | static void split_mark(heap_block_head_t *cur, const size_t size)
|
---|
| 498 | {
|
---|
| 499 | assert(cur->size >= size);
|
---|
| 500 |
|
---|
| 501 | /* See if we should split the block. */
|
---|
| 502 | size_t split_limit = GROSS_SIZE(size);
|
---|
| 503 |
|
---|
| 504 | if (cur->size > split_limit) {
|
---|
| 505 | /* Block big enough -> split. */
|
---|
| 506 | void *next = ((void *) cur) + size;
|
---|
[0b37882] | 507 | block_init(next, cur->size - size, true, cur->area);
|
---|
| 508 | block_init(cur, size, false, cur->area);
|
---|
[6db6fd1] | 509 | } else {
|
---|
| 510 | /* Block too small -> use as is. */
|
---|
| 511 | cur->free = false;
|
---|
| 512 | }
|
---|
| 513 | }
|
---|
| 514 |
|
---|
[0b37882] | 515 | /** Allocate memory from heap area starting from given block
|
---|
[3292623] | 516 | *
|
---|
| 517 | * Should be called only inside the critical section.
|
---|
[0b37882] | 518 | * As a side effect this function also sets the current
|
---|
| 519 | * pointer on successful allocation.
|
---|
[6db6fd1] | 520 | *
|
---|
[0b37882] | 521 | * @param area Heap area where to allocate from.
|
---|
| 522 | * @param first_block Starting heap block.
|
---|
| 523 | * @param final_block Heap block where to finish the search
|
---|
| 524 | * (may be NULL).
|
---|
| 525 | * @param real_size Gross number of bytes to allocate.
|
---|
| 526 | * @param falign Physical alignment of the block.
|
---|
[6db6fd1] | 527 | *
|
---|
[0b37882] | 528 | * @return Address of the allocated block or NULL on not enough memory.
|
---|
[6db6fd1] | 529 | *
|
---|
| 530 | */
|
---|
[0b37882] | 531 | static void *malloc_area(heap_area_t *area, heap_block_head_t *first_block,
|
---|
| 532 | heap_block_head_t *final_block, size_t real_size, size_t falign)
|
---|
[6db6fd1] | 533 | {
|
---|
[0b37882] | 534 | area_check((void *) area);
|
---|
[013a5d7] | 535 | assert((void *) first_block >= (void *) AREA_FIRST_BLOCK_HEAD(area));
|
---|
[0b37882] | 536 | assert((void *) first_block < area->end);
|
---|
[6db6fd1] | 537 |
|
---|
[013a5d7] | 538 | for (heap_block_head_t *cur = first_block; (void *) cur < area->end;
|
---|
[0b37882] | 539 | cur = (heap_block_head_t *) (((void *) cur) + cur->size)) {
|
---|
[6db6fd1] | 540 | block_check(cur);
|
---|
| 541 |
|
---|
[0b37882] | 542 | /* Finish searching on the final block */
|
---|
| 543 | if ((final_block != NULL) && (cur == final_block))
|
---|
| 544 | break;
|
---|
| 545 |
|
---|
[6db6fd1] | 546 | /* Try to find a block that is free and large enough. */
|
---|
| 547 | if ((cur->free) && (cur->size >= real_size)) {
|
---|
[0b37882] | 548 | /*
|
---|
| 549 | * We have found a suitable block.
|
---|
| 550 | * Check for alignment properties.
|
---|
| 551 | */
|
---|
| 552 | void *addr = (void *)
|
---|
| 553 | ((uintptr_t) cur + sizeof(heap_block_head_t));
|
---|
| 554 | void *aligned = (void *)
|
---|
| 555 | ALIGN_UP((uintptr_t) addr, falign);
|
---|
[6db6fd1] | 556 |
|
---|
| 557 | if (addr == aligned) {
|
---|
| 558 | /* Exact block start including alignment. */
|
---|
| 559 | split_mark(cur, real_size);
|
---|
[0b37882] | 560 |
|
---|
| 561 | next = cur;
|
---|
| 562 | return addr;
|
---|
[6db6fd1] | 563 | } else {
|
---|
[d851f597] | 564 | /* Block start has to be aligned */
|
---|
[6db6fd1] | 565 | size_t excess = (size_t) (aligned - addr);
|
---|
| 566 |
|
---|
| 567 | if (cur->size >= real_size + excess) {
|
---|
[0b37882] | 568 | /*
|
---|
| 569 | * The current block is large enough to fit
|
---|
| 570 | * data in (including alignment).
|
---|
| 571 | */
|
---|
[013a5d7] | 572 | if ((void *) cur > (void *) AREA_FIRST_BLOCK_HEAD(area)) {
|
---|
[0b37882] | 573 | /*
|
---|
| 574 | * There is a block before the current block.
|
---|
| 575 | * This previous block can be enlarged to
|
---|
| 576 | * compensate for the alignment excess.
|
---|
| 577 | */
|
---|
| 578 | heap_block_foot_t *prev_foot = (heap_block_foot_t *)
|
---|
| 579 | ((void *) cur - sizeof(heap_block_foot_t));
|
---|
[6db6fd1] | 580 |
|
---|
[0b37882] | 581 | heap_block_head_t *prev_head = (heap_block_head_t *)
|
---|
| 582 | ((void *) cur - prev_foot->size);
|
---|
[6db6fd1] | 583 |
|
---|
| 584 | block_check(prev_head);
|
---|
| 585 |
|
---|
| 586 | size_t reduced_size = cur->size - excess;
|
---|
[d851f597] | 587 | heap_block_head_t *next_head = ((void *) cur) + excess;
|
---|
[6db6fd1] | 588 |
|
---|
[0b37882] | 589 | if ((!prev_head->free) &&
|
---|
| 590 | (excess >= STRUCT_OVERHEAD)) {
|
---|
| 591 | /*
|
---|
| 592 | * The previous block is not free and there
|
---|
| 593 | * is enough free space left to fill in
|
---|
| 594 | * a new free block between the previous
|
---|
| 595 | * and current block.
|
---|
| 596 | */
|
---|
| 597 | block_init(cur, excess, true, area);
|
---|
[d851f597] | 598 | } else {
|
---|
[0b37882] | 599 | /*
|
---|
| 600 | * The previous block is free (thus there
|
---|
| 601 | * is no need to induce additional
|
---|
| 602 | * fragmentation to the heap) or the
|
---|
| 603 | * excess is small. Therefore just enlarge
|
---|
| 604 | * the previous block.
|
---|
| 605 | */
|
---|
| 606 | block_init(prev_head, prev_head->size + excess,
|
---|
| 607 | prev_head->free, area);
|
---|
[d851f597] | 608 | }
|
---|
| 609 |
|
---|
[0b37882] | 610 | block_init(next_head, reduced_size, true, area);
|
---|
[d851f597] | 611 | split_mark(next_head, real_size);
|
---|
[0b37882] | 612 |
|
---|
| 613 | next = next_head;
|
---|
| 614 | return aligned;
|
---|
[6db6fd1] | 615 | } else {
|
---|
[0b37882] | 616 | /*
|
---|
| 617 | * The current block is the first block
|
---|
| 618 | * in the heap area. We have to make sure
|
---|
| 619 | * that the alignment excess is large enough
|
---|
| 620 | * to fit a new free block just before the
|
---|
| 621 | * current block.
|
---|
| 622 | */
|
---|
[6db6fd1] | 623 | while (excess < STRUCT_OVERHEAD) {
|
---|
| 624 | aligned += falign;
|
---|
| 625 | excess += falign;
|
---|
| 626 | }
|
---|
| 627 |
|
---|
[d851f597] | 628 | /* Check for current block size again */
|
---|
[6db6fd1] | 629 | if (cur->size >= real_size + excess) {
|
---|
| 630 | size_t reduced_size = cur->size - excess;
|
---|
[0b37882] | 631 | cur = (heap_block_head_t *)
|
---|
[013a5d7] | 632 | (AREA_FIRST_BLOCK_HEAD(area) + excess);
|
---|
[6db6fd1] | 633 |
|
---|
[013a5d7] | 634 | block_init((void *) AREA_FIRST_BLOCK_HEAD(area),
|
---|
| 635 | excess, true, area);
|
---|
[0b37882] | 636 | block_init(cur, reduced_size, true, area);
|
---|
[6db6fd1] | 637 | split_mark(cur, real_size);
|
---|
[0b37882] | 638 |
|
---|
| 639 | next = cur;
|
---|
| 640 | return aligned;
|
---|
[6db6fd1] | 641 | }
|
---|
| 642 | }
|
---|
| 643 | }
|
---|
| 644 | }
|
---|
| 645 | }
|
---|
[0b37882] | 646 | }
|
---|
| 647 |
|
---|
| 648 | return NULL;
|
---|
| 649 | }
|
---|
| 650 |
|
---|
| 651 | /** Allocate a memory block
|
---|
| 652 | *
|
---|
| 653 | * Should be called only inside the critical section.
|
---|
| 654 | *
|
---|
| 655 | * @param size The size of the block to allocate.
|
---|
| 656 | * @param align Memory address alignment.
|
---|
| 657 | *
|
---|
| 658 | * @return Address of the allocated block or NULL on not enough memory.
|
---|
| 659 | *
|
---|
| 660 | */
|
---|
| 661 | static void *malloc_internal(const size_t size, const size_t align)
|
---|
| 662 | {
|
---|
| 663 | assert(first_heap_area != NULL);
|
---|
| 664 |
|
---|
| 665 | if (align == 0)
|
---|
| 666 | return NULL;
|
---|
| 667 |
|
---|
| 668 | size_t falign = lcm(align, BASE_ALIGN);
|
---|
| 669 | size_t real_size = GROSS_SIZE(ALIGN_UP(size, falign));
|
---|
| 670 |
|
---|
| 671 | bool retry = false;
|
---|
| 672 | heap_block_head_t *split;
|
---|
| 673 |
|
---|
| 674 | loop:
|
---|
| 675 |
|
---|
| 676 | /* Try the next fit approach */
|
---|
| 677 | split = next;
|
---|
| 678 |
|
---|
| 679 | if (split != NULL) {
|
---|
| 680 | void *addr = malloc_area(split->area, split, NULL, real_size,
|
---|
| 681 | falign);
|
---|
[6db6fd1] | 682 |
|
---|
[0b37882] | 683 | if (addr != NULL)
|
---|
| 684 | return addr;
|
---|
[6db6fd1] | 685 | }
|
---|
| 686 |
|
---|
[0b37882] | 687 | /* Search the entire heap */
|
---|
[013a5d7] | 688 | for (heap_area_t *area = first_heap_area; area != NULL;
|
---|
| 689 | area = area->next) {
|
---|
[0b37882] | 690 | heap_block_head_t *first = (heap_block_head_t *)
|
---|
[013a5d7] | 691 | AREA_FIRST_BLOCK_HEAD(area);
|
---|
[0b37882] | 692 |
|
---|
| 693 | void *addr = malloc_area(area, first, split, real_size,
|
---|
| 694 | falign);
|
---|
| 695 |
|
---|
| 696 | if (addr != NULL)
|
---|
| 697 | return addr;
|
---|
| 698 | }
|
---|
| 699 |
|
---|
| 700 | if (!retry) {
|
---|
| 701 | /* Try to grow the heap space */
|
---|
| 702 | if (heap_grow(real_size)) {
|
---|
| 703 | retry = true;
|
---|
[6db6fd1] | 704 | goto loop;
|
---|
| 705 | }
|
---|
| 706 | }
|
---|
| 707 |
|
---|
[0b37882] | 708 | return NULL;
|
---|
[6db6fd1] | 709 | }
|
---|
| 710 |
|
---|
[3292623] | 711 | /** Allocate memory by number of elements
|
---|
| 712 | *
|
---|
| 713 | * @param nmemb Number of members to allocate.
|
---|
| 714 | * @param size Size of one member in bytes.
|
---|
| 715 | *
|
---|
| 716 | * @return Allocated memory or NULL.
|
---|
| 717 | *
|
---|
| 718 | */
|
---|
[15b8e495] | 719 | void *calloc(const size_t nmemb, const size_t size)
|
---|
| 720 | {
|
---|
[e6b73ad0] | 721 | void *block = malloc(nmemb * size);
|
---|
| 722 | if (block == NULL)
|
---|
| 723 | return NULL;
|
---|
[3292623] | 724 |
|
---|
[e6b73ad0] | 725 | memset(block, 0, nmemb * size);
|
---|
| 726 | return block;
|
---|
[15b8e495] | 727 | }
|
---|
| 728 |
|
---|
[3292623] | 729 | /** Allocate memory
|
---|
| 730 | *
|
---|
| 731 | * @param size Number of bytes to allocate.
|
---|
| 732 | *
|
---|
| 733 | * @return Allocated memory or NULL.
|
---|
| 734 | *
|
---|
| 735 | */
|
---|
[6db6fd1] | 736 | void *malloc(const size_t size)
|
---|
| 737 | {
|
---|
[3292623] | 738 | futex_down(&malloc_futex);
|
---|
| 739 | void *block = malloc_internal(size, BASE_ALIGN);
|
---|
| 740 | futex_up(&malloc_futex);
|
---|
| 741 |
|
---|
| 742 | return block;
|
---|
[6db6fd1] | 743 | }
|
---|
| 744 |
|
---|
[3292623] | 745 | /** Allocate memory with specified alignment
|
---|
| 746 | *
|
---|
| 747 | * @param align Alignment in byes.
|
---|
| 748 | * @param size Number of bytes to allocate.
|
---|
| 749 | *
|
---|
| 750 | * @return Allocated memory or NULL.
|
---|
| 751 | *
|
---|
| 752 | */
|
---|
[6db6fd1] | 753 | void *memalign(const size_t align, const size_t size)
|
---|
| 754 | {
|
---|
| 755 | if (align == 0)
|
---|
| 756 | return NULL;
|
---|
| 757 |
|
---|
| 758 | size_t palign =
|
---|
| 759 | 1 << (fnzb(max(sizeof(void *), align) - 1) + 1);
|
---|
| 760 |
|
---|
[3292623] | 761 | futex_down(&malloc_futex);
|
---|
| 762 | void *block = malloc_internal(size, palign);
|
---|
| 763 | futex_up(&malloc_futex);
|
---|
| 764 |
|
---|
| 765 | return block;
|
---|
[6db6fd1] | 766 | }
|
---|
| 767 |
|
---|
[3292623] | 768 | /** Reallocate memory block
|
---|
| 769 | *
|
---|
| 770 | * @param addr Already allocated memory or NULL.
|
---|
| 771 | * @param size New size of the memory block.
|
---|
| 772 | *
|
---|
| 773 | * @return Reallocated memory or NULL.
|
---|
| 774 | *
|
---|
| 775 | */
|
---|
[6db6fd1] | 776 | void *realloc(const void *addr, const size_t size)
|
---|
| 777 | {
|
---|
| 778 | if (addr == NULL)
|
---|
| 779 | return malloc(size);
|
---|
| 780 |
|
---|
[3292623] | 781 | futex_down(&malloc_futex);
|
---|
| 782 |
|
---|
[6db6fd1] | 783 | /* Calculate the position of the header. */
|
---|
| 784 | heap_block_head_t *head =
|
---|
| 785 | (heap_block_head_t *) (addr - sizeof(heap_block_head_t));
|
---|
| 786 |
|
---|
| 787 | block_check(head);
|
---|
| 788 | assert(!head->free);
|
---|
| 789 |
|
---|
[0b37882] | 790 | heap_area_t *area = head->area;
|
---|
| 791 |
|
---|
| 792 | area_check(area);
|
---|
[013a5d7] | 793 | assert((void *) head >= (void *) AREA_FIRST_BLOCK_HEAD(area));
|
---|
[0b37882] | 794 | assert((void *) head < area->end);
|
---|
| 795 |
|
---|
[6db6fd1] | 796 | void *ptr = NULL;
|
---|
[3292623] | 797 | bool reloc = false;
|
---|
[f450280] | 798 | size_t real_size = GROSS_SIZE(ALIGN_UP(size, BASE_ALIGN));
|
---|
[6db6fd1] | 799 | size_t orig_size = head->size;
|
---|
| 800 |
|
---|
| 801 | if (orig_size > real_size) {
|
---|
| 802 | /* Shrink */
|
---|
| 803 | if (orig_size - real_size >= STRUCT_OVERHEAD) {
|
---|
[0b37882] | 804 | /*
|
---|
| 805 | * Split the original block to a full block
|
---|
| 806 | * and a trailing free block.
|
---|
| 807 | */
|
---|
| 808 | block_init((void *) head, real_size, false, area);
|
---|
[6db6fd1] | 809 | block_init((void *) head + real_size,
|
---|
[0b37882] | 810 | orig_size - real_size, true, area);
|
---|
[013a5d7] | 811 | heap_shrink(area);
|
---|
[6db6fd1] | 812 | }
|
---|
| 813 |
|
---|
| 814 | ptr = ((void *) head) + sizeof(heap_block_head_t);
|
---|
| 815 | } else {
|
---|
[0b37882] | 816 | /*
|
---|
| 817 | * Look at the next block. If it is free and the size is
|
---|
| 818 | * sufficient then merge the two. Otherwise just allocate
|
---|
| 819 | * a new block, copy the original data into it and
|
---|
| 820 | * free the original block.
|
---|
| 821 | */
|
---|
[6db6fd1] | 822 | heap_block_head_t *next_head =
|
---|
| 823 | (heap_block_head_t *) (((void *) head) + head->size);
|
---|
| 824 |
|
---|
[0b37882] | 825 | if (((void *) next_head < area->end) &&
|
---|
[7d88587] | 826 | (head->size + next_head->size >= real_size) &&
|
---|
| 827 | (next_head->free)) {
|
---|
[6db6fd1] | 828 | block_check(next_head);
|
---|
[0b37882] | 829 | block_init(head, head->size + next_head->size, false, area);
|
---|
[ba0aa6f] | 830 | split_mark(head, real_size);
|
---|
[6db6fd1] | 831 |
|
---|
| 832 | ptr = ((void *) head) + sizeof(heap_block_head_t);
|
---|
[0b37882] | 833 | next = NULL;
|
---|
[3292623] | 834 | } else
|
---|
| 835 | reloc = true;
|
---|
| 836 | }
|
---|
| 837 |
|
---|
| 838 | futex_up(&malloc_futex);
|
---|
| 839 |
|
---|
| 840 | if (reloc) {
|
---|
| 841 | ptr = malloc(size);
|
---|
| 842 | if (ptr != NULL) {
|
---|
| 843 | memcpy(ptr, addr, NET_SIZE(orig_size));
|
---|
| 844 | free(addr);
|
---|
[6db6fd1] | 845 | }
|
---|
| 846 | }
|
---|
| 847 |
|
---|
| 848 | return ptr;
|
---|
| 849 | }
|
---|
| 850 |
|
---|
| 851 | /** Free a memory block
|
---|
| 852 | *
|
---|
| 853 | * @param addr The address of the block.
|
---|
[3292623] | 854 | *
|
---|
[6db6fd1] | 855 | */
|
---|
| 856 | void free(const void *addr)
|
---|
| 857 | {
|
---|
[3292623] | 858 | futex_down(&malloc_futex);
|
---|
| 859 |
|
---|
[6db6fd1] | 860 | /* Calculate the position of the header. */
|
---|
| 861 | heap_block_head_t *head
|
---|
| 862 | = (heap_block_head_t *) (addr - sizeof(heap_block_head_t));
|
---|
| 863 |
|
---|
| 864 | block_check(head);
|
---|
| 865 | assert(!head->free);
|
---|
| 866 |
|
---|
[0b37882] | 867 | heap_area_t *area = head->area;
|
---|
| 868 |
|
---|
| 869 | area_check(area);
|
---|
[013a5d7] | 870 | assert((void *) head >= (void *) AREA_FIRST_BLOCK_HEAD(area));
|
---|
[0b37882] | 871 | assert((void *) head < area->end);
|
---|
| 872 |
|
---|
[6db6fd1] | 873 | /* Mark the block itself as free. */
|
---|
| 874 | head->free = true;
|
---|
| 875 |
|
---|
| 876 | /* Look at the next block. If it is free, merge the two. */
|
---|
| 877 | heap_block_head_t *next_head
|
---|
| 878 | = (heap_block_head_t *) (((void *) head) + head->size);
|
---|
| 879 |
|
---|
[0b37882] | 880 | if ((void *) next_head < area->end) {
|
---|
[6db6fd1] | 881 | block_check(next_head);
|
---|
| 882 | if (next_head->free)
|
---|
[0b37882] | 883 | block_init(head, head->size + next_head->size, true, area);
|
---|
[6db6fd1] | 884 | }
|
---|
| 885 |
|
---|
| 886 | /* Look at the previous block. If it is free, merge the two. */
|
---|
[013a5d7] | 887 | if ((void *) head > (void *) AREA_FIRST_BLOCK_HEAD(area)) {
|
---|
[6db6fd1] | 888 | heap_block_foot_t *prev_foot =
|
---|
| 889 | (heap_block_foot_t *) (((void *) head) - sizeof(heap_block_foot_t));
|
---|
| 890 |
|
---|
| 891 | heap_block_head_t *prev_head =
|
---|
| 892 | (heap_block_head_t *) (((void *) head) - prev_foot->size);
|
---|
| 893 |
|
---|
| 894 | block_check(prev_head);
|
---|
| 895 |
|
---|
| 896 | if (prev_head->free)
|
---|
[0b37882] | 897 | block_init(prev_head, prev_head->size + head->size, true,
|
---|
| 898 | area);
|
---|
[6db6fd1] | 899 | }
|
---|
| 900 |
|
---|
[013a5d7] | 901 | heap_shrink(area);
|
---|
[3292623] | 902 |
|
---|
| 903 | futex_up(&malloc_futex);
|
---|
[6db6fd1] | 904 | }
|
---|
| 905 |
|
---|
| 906 | /** @}
|
---|
| 907 | */
|
---|