Index: kernel/generic/include/bitops.h =================================================================== --- kernel/generic/include/bitops.h (revision e394c1962a6fec2218e561da2994f4070e2be503) +++ kernel/generic/include/bitops.h (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) @@ -36,4 +36,5 @@ #define KERN_BITOPS_H_ +#include #include Index: kernel/generic/include/main/main.h =================================================================== --- kernel/generic/include/main/main.h (revision e394c1962a6fec2218e561da2994f4070e2be503) +++ kernel/generic/include/main/main.h (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) @@ -48,4 +48,6 @@ extern void main_ap(void); +extern void malloc_init(void); + #endif Index: kernel/generic/include/mm/slab.h =================================================================== --- kernel/generic/include/mm/slab.h (revision e394c1962a6fec2218e561da2994f4070e2be503) +++ kernel/generic/include/mm/slab.h (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) @@ -40,10 +40,4 @@ #include #include - -/** Minimum size to be allocated by malloc */ -#define SLAB_MIN_MALLOC_W 4 - -/** Maximum size to be allocated by malloc */ -#define SLAB_MAX_MALLOC_W 22 /** Initial Magazine size (TODO: dynamically growing magazines) */ Index: kernel/generic/src/main/main.c =================================================================== --- kernel/generic/src/main/main.c (revision e394c1962a6fec2218e561da2994f4070e2be503) +++ kernel/generic/src/main/main.c (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) @@ -246,4 +246,5 @@ frame_init(); slab_cache_init(); + malloc_init(); ra_init(); sysinfo_init(); Index: kernel/generic/src/mm/malloc.c =================================================================== --- kernel/generic/src/mm/malloc.c (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) +++ kernel/generic/src/mm/malloc.c (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2006 Ondrej Palkovsky + * Copyright (c) 2018 Jiří Zárevúcky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include
// malloc_init() + +/** Minimum size to be allocated by malloc */ +#define SLAB_MIN_MALLOC_W 4 + +/** Maximum size to be allocated by malloc */ +#define SLAB_MAX_MALLOC_W 22 + +/** Caches for malloc */ +static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1]; + +static const char *malloc_names[] = { + "malloc-16", + "malloc-32", + "malloc-64", + "malloc-128", + "malloc-256", + "malloc-512", + "malloc-1K", + "malloc-2K", + "malloc-4K", + "malloc-8K", + "malloc-16K", + "malloc-32K", + "malloc-64K", + "malloc-128K", + "malloc-256K", + "malloc-512K", + "malloc-1M", + "malloc-2M", + "malloc-4M" +}; + +void malloc_init(void) +{ + /* Initialize structures for malloc */ + size_t i; + size_t size; + + for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); + i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); + i++, size <<= 1) { + malloc_caches[i] = slab_cache_create(malloc_names[i], size, 0, + NULL, NULL, SLAB_CACHE_MAGDEFERRED); + } +} + +static void _check_sizes(size_t *size, size_t *alignment) +{ + assert(size); + assert(alignment); + + assert(*size > 0); + + /* Alignment must be a power of 2. */ + assert(__builtin_popcountl(*alignment) <= 1); + assert(*alignment <= PAGE_SIZE); + + if (*alignment < alignof(max_align_t)) + *alignment = alignof(max_align_t); + + *size = ALIGN_UP(*size, *alignment); + + if (*size < (1 << SLAB_MIN_MALLOC_W)) + *size = (1 << SLAB_MIN_MALLOC_W); +} + +static slab_cache_t *cache_for_size(size_t size) +{ + assert(size > 0); + assert(size <= (1 << SLAB_MAX_MALLOC_W)); + + size_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; + + assert(idx < sizeof(malloc_caches) / sizeof(malloc_caches[0])); + + slab_cache_t *cache = malloc_caches[idx]; + + assert(cache != NULL); + return cache; +} + +// TODO: Expose publicly and use mem_alloc() and mem_free() instead of malloc() + +static void *mem_alloc(size_t, size_t) __attribute__((malloc)); + +static void *mem_alloc(size_t size, size_t alignment) +{ + _check_sizes(&size, &alignment); + + if (size > (1 << SLAB_MAX_MALLOC_W)) { + // TODO: Allocate big objects directly from coarse allocator. + assert(size <= (1 << SLAB_MAX_MALLOC_W)); + } + + /* We assume that slab objects are aligned naturally */ + return slab_alloc(cache_for_size(size), FRAME_ATOMIC); +} + +/** + * Free memory allocated using mem_alloc(). + * + * @param ptr Pointer returned by mem_alloc(). + * @param size Size used to call mem_alloc(). + * @param alignment Alignment used to call mem_alloc(). + */ +static void mem_free(void *ptr, size_t size, size_t alignment) +{ + if (!ptr) + return; + + _check_sizes(&size, &alignment); + + if (size > (1 << SLAB_MAX_MALLOC_W)) { + // TODO: Allocate big objects directly from coarse allocator. + assert(size <= (1 << SLAB_MAX_MALLOC_W)); + } + + return slab_free(cache_for_size(size), ptr); +} + +static const size_t _offset = ALIGN_UP(sizeof(size_t), alignof(max_align_t)); + +void *malloc(size_t size) +{ + void *obj = mem_alloc(size + _offset, alignof(max_align_t)) + _offset; + + /* Remember the allocation size just before the object. */ + ((size_t *) obj)[-1] = size; + return obj; +} + +void free(void *obj) +{ + /* + * We don't check integrity of size, so buffer over/underruns can + * corrupt it. That's ok, it ultimately only serves as a hint to + * select the correct slab cache. If the selected cache is not correct, + * slab_free() will detect it and panic. + */ + size_t size = ((size_t *) obj)[-1]; + mem_free(obj - _offset, size + _offset, alignof(max_align_t)); +} + +void *realloc(void *old_obj, size_t new_size) +{ + if (!old_obj) + return malloc(new_size); + + size_t old_size = ((size_t *) old_obj)[-1]; + + if (cache_for_size(old_size + _offset) == + cache_for_size(new_size + _offset)) + return old_obj; + + void *new_obj = malloc(new_size); + if (!new_obj) + return NULL; + + memcpy(new_obj, old_obj, min(old_size, new_size)); + free(old_obj); + return new_obj; +} Index: kernel/generic/src/mm/slab.c =================================================================== --- kernel/generic/src/mm/slab.c (revision e394c1962a6fec2218e561da2994f4070e2be503) +++ kernel/generic/src/mm/slab.c (revision b60615bd9d3afe6d075036e1b1ebdc9437912cfc) @@ -138,29 +138,4 @@ static slab_cache_t *slab_extern_cache; -/** Caches for malloc */ -static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1]; - -static const char *malloc_names[] = { - "malloc-16", - "malloc-32", - "malloc-64", - "malloc-128", - "malloc-256", - "malloc-512", - "malloc-1K", - "malloc-2K", - "malloc-4K", - "malloc-8K", - "malloc-16K", - "malloc-32K", - "malloc-64K", - "malloc-128K", - "malloc-256K", - "malloc-512K", - "malloc-1M", - "malloc-2M", - "malloc-4M" -}; - /** Slab descriptor */ typedef struct { @@ -775,7 +750,6 @@ panic("Destroying cache that is not empty."); - if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) { - slab_t *mag_slab = obj2slab(cache->mag_cache); - _slab_free(mag_slab->cache, cache->mag_cache, mag_slab); + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE) && cache->mag_cache) { + slab_free(&slab_mag_cache, cache->mag_cache); } @@ -914,15 +888,4 @@ NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); - /* Initialize structures for malloc */ - size_t i; - size_t size; - - for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); - i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); - i++, size <<= 1) { - malloc_caches[i] = slab_cache_create(malloc_names[i], size, 0, - NULL, NULL, SLAB_CACHE_MAGDEFERRED); - } - #ifdef CONFIG_DEBUG _slab_initialized = 1; @@ -961,53 +924,4 @@ } -void *malloc(size_t size) -{ - assert(_slab_initialized); - assert(size <= (1 << SLAB_MAX_MALLOC_W)); - - if (size < (1 << SLAB_MIN_MALLOC_W)) - size = (1 << SLAB_MIN_MALLOC_W); - - uint8_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; - - return slab_alloc(malloc_caches[idx], FRAME_ATOMIC); -} - -void *realloc(void *ptr, size_t size) -{ - assert(_slab_initialized); - assert(size <= (1 << SLAB_MAX_MALLOC_W)); - - void *new_ptr; - - if (size > 0) { - if (size < (1 << SLAB_MIN_MALLOC_W)) - size = (1 << SLAB_MIN_MALLOC_W); - uint8_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; - - new_ptr = slab_alloc(malloc_caches[idx], FRAME_ATOMIC); - } else - new_ptr = NULL; - - if ((new_ptr != NULL) && (ptr != NULL)) { - slab_t *slab = obj2slab(ptr); - memcpy(new_ptr, ptr, min(size, slab->cache->size)); - } - - if (ptr != NULL) - free(ptr); - - return new_ptr; -} - -void free(void *ptr) -{ - if (!ptr) - return; - - slab_t *slab = obj2slab(ptr); - _slab_free(slab->cache, ptr, slab); -} - /** @} */