Index: kernel/generic/src/mm/as.c =================================================================== --- kernel/generic/src/mm/as.c (revision c96452180f7af3adf60e8b6f88ac2ad6ec1683e6) +++ kernel/generic/src/mm/as.c (revision da1bafb8cf9a3b3be8ef21bc114daaa476a85190) @@ -33,5 +33,5 @@ /** * @file - * @brief Address space related functions. + * @brief Address space related functions. * * This file contains address space manipulation functions. @@ -86,4 +86,5 @@ * Each architecture decides what functions will be used to carry out * address space operations such as creating or locking page tables. + * */ as_operations_t *as_operations = NULL; @@ -91,4 +92,5 @@ /** * Slab for as_t objects. + * */ static slab_cache_t *as_slab; @@ -100,4 +102,5 @@ * - as->asid for each as of the as_t type * - asids_allocated counter + * */ SPINLOCK_INITIALIZE(asidlock); @@ -106,4 +109,5 @@ * This list contains address spaces that are not active on any * processor and that have valid ASID. + * */ LIST_INITIALIZE(inactive_as_with_asid_head); @@ -112,26 +116,24 @@ as_t *AS_KERNEL = NULL; -static int area_flags_to_page_flags(int); +static unsigned int area_flags_to_page_flags(unsigned int); static as_area_t *find_area_and_lock(as_t *, uintptr_t); static bool check_area_conflicts(as_t *, uintptr_t, size_t, as_area_t *); static void sh_info_remove_reference(share_info_t *); -static int as_constructor(void *obj, int flags) +static int as_constructor(void *obj, unsigned int flags) { as_t *as = (as_t *) obj; - int rc; - + link_initialize(&as->inactive_as_with_asid_link); mutex_initialize(&as->lock, MUTEX_PASSIVE); - rc = as_constructor_arch(as, flags); + int rc = as_constructor_arch(as, flags); return rc; } -static int as_destructor(void *obj) +static size_t as_destructor(void *obj) { as_t *as = (as_t *) obj; - return as_destructor_arch(as); } @@ -141,5 +143,5 @@ { as_arch_init(); - + as_slab = slab_cache_create("as_slab", sizeof(as_t), 0, as_constructor, as_destructor, SLAB_CACHE_MAGDEFERRED); @@ -157,12 +159,11 @@ /** Create address space. * - * @param flags Flags that influence the way in wich the address space - * is created. - */ -as_t *as_create(int flags) -{ - as_t *as; - - as = (as_t *) slab_alloc(as_slab, 0); + * @param flags Flags that influence the way in wich the address + * space is created. + * + */ +as_t *as_create(unsigned int flags) +{ + as_t *as = (as_t *) slab_alloc(as_slab, 0); (void) as_create_arch(as, 0); @@ -176,4 +177,5 @@ atomic_set(&as->refcount, 0); as->cpu_refcount = 0; + #ifdef AS_PAGE_TABLE as->genarch.page_table = page_table_create(flags); @@ -192,10 +194,9 @@ * We know that we don't hold any spinlock. * - * @param as Address space to be destroyed. + * @param as Address space to be destroyed. + * */ void as_destroy(as_t *as) { - ipl_t ipl; - bool cond; DEADLOCK_PROBE_INIT(p_asidlock); @@ -214,7 +215,9 @@ * disabled to prevent nested context switches. We also depend on the * fact that so far no spinlocks are held. + * */ preemption_disable(); - ipl = interrupts_read(); + ipl_t ipl = interrupts_read(); + retry: interrupts_disable(); @@ -224,30 +227,37 @@ goto retry; } - preemption_enable(); /* Interrupts disabled, enable preemption */ - if (as->asid != ASID_INVALID && as != AS_KERNEL) { + + /* Interrupts disabled, enable preemption */ + preemption_enable(); + + if ((as->asid != ASID_INVALID) && (as != AS_KERNEL)) { if (as->cpu_refcount == 0) list_remove(&as->inactive_as_with_asid_link); + asid_put(as->asid); } + spinlock_unlock(&asidlock); - + /* * Destroy address space areas of the address space. * The B+tree must be walked carefully because it is * also being destroyed. - */ - for (cond = true; cond; ) { - btree_node_t *node; - + * + */ + bool cond = true; + while (cond) { ASSERT(!list_empty(&as->as_area_btree.leaf_head)); - node = list_get_instance(as->as_area_btree.leaf_head.next, + + btree_node_t *node = + list_get_instance(as->as_area_btree.leaf_head.next, btree_node_t, leaf_link); - - if ((cond = node->keys)) { + + if ((cond = node->keys)) as_area_destroy(as, node->key[0]); - } - } - + } + btree_destroy(&as->as_area_btree); + #ifdef AS_PAGE_TABLE page_table_destroy(as->genarch.page_table); @@ -255,7 +265,7 @@ page_table_destroy(NULL); #endif - + interrupts_restore(ipl); - + slab_free(as_slab, as); } @@ -266,5 +276,6 @@ * space. * - * @param a Address space to be held. + * @param as Address space to be held. + * */ void as_hold(as_t *as) @@ -278,5 +289,6 @@ * space. * - * @param a Address space to be released. + * @param asAddress space to be released. + * */ void as_release(as_t *as) @@ -290,32 +302,30 @@ * The created address space area is added to the target address space. * - * @param as Target address space. - * @param flags Flags of the area memory. - * @param size Size of area. - * @param base Base address of area. - * @param attrs Attributes of the area. - * @param backend Address space area backend. NULL if no backend is used. - * @param backend_data NULL or a pointer to an array holding two void *. - * - * @return Address space area on success or NULL on failure. - */ -as_area_t * -as_area_create(as_t *as, int flags, size_t size, uintptr_t base, int attrs, - mem_backend_t *backend, mem_backend_data_t *backend_data) -{ - ipl_t ipl; - as_area_t *a; - + * @param as Target address space. + * @param flags Flags of the area memory. + * @param size Size of area. + * @param base Base address of area. + * @param attrs Attributes of the area. + * @param backend Address space area backend. NULL if no backend is used. + * @param backend_data NULL or a pointer to an array holding two void *. + * + * @return Address space area on success or NULL on failure. + * + */ +as_area_t *as_area_create(as_t *as, unsigned int flags, size_t size, + uintptr_t base, unsigned int attrs, mem_backend_t *backend, + mem_backend_data_t *backend_data) +{ if (base % PAGE_SIZE) return NULL; - + if (!size) return NULL; - + /* Writeable executable areas are not supported. */ if ((flags & AS_AREA_EXEC) && (flags & AS_AREA_WRITE)) return NULL; - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); @@ -326,54 +336,52 @@ } - a = (as_area_t *) malloc(sizeof(as_area_t), 0); - - mutex_initialize(&a->lock, MUTEX_PASSIVE); - - a->as = as; - a->flags = flags; - a->attributes = attrs; - a->pages = SIZE2FRAMES(size); - a->base = base; - a->sh_info = NULL; - a->backend = backend; + as_area_t *area = (as_area_t *) malloc(sizeof(as_area_t), 0); + + mutex_initialize(&area->lock, MUTEX_PASSIVE); + + area->as = as; + area->flags = flags; + area->attributes = attrs; + area->pages = SIZE2FRAMES(size); + area->base = base; + area->sh_info = NULL; + area->backend = backend; + if (backend_data) - a->backend_data = *backend_data; + area->backend_data = *backend_data; else - memsetb(&a->backend_data, sizeof(a->backend_data), 0); - - btree_create(&a->used_space); - - btree_insert(&as->as_area_btree, base, (void *) a, NULL); - + memsetb(&area->backend_data, sizeof(area->backend_data), 0); + + btree_create(&area->used_space); + btree_insert(&as->as_area_btree, base, (void *) area, NULL); + mutex_unlock(&as->lock); interrupts_restore(ipl); - - return a; + + return area; } /** Find address space area and change it. * - * @param as Address space. - * @param address Virtual address belonging to the area to be changed. - * Must be page-aligned. - * @param size New size of the virtual memory block starting at - * address. - * @param flags Flags influencing the remap operation. Currently unused. - * - * @return Zero on success or a value from @ref errno.h otherwise. - */ -int as_area_resize(as_t *as, uintptr_t address, size_t size, int flags) -{ - as_area_t *area; - ipl_t ipl; - size_t pages; - - ipl = interrupts_disable(); + * @param as Address space. + * @param address Virtual address belonging to the area to be changed. + * Must be page-aligned. + * @param size New size of the virtual memory block starting at + * address. + * @param flags Flags influencing the remap operation. Currently unused. + * + * @return Zero on success or a value from @ref errno.h otherwise. + * + */ +int as_area_resize(as_t *as, uintptr_t address, size_t size, unsigned int flags) +{ + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); /* * Locate the area. - */ - area = find_area_and_lock(as, address); + * + */ + as_area_t *area = find_area_and_lock(as, address); if (!area) { mutex_unlock(&as->lock); @@ -381,9 +389,10 @@ return ENOENT; } - + if (area->backend == &phys_backend) { /* * Remapping of address space areas associated * with memory mapped devices is not supported. + * */ mutex_unlock(&area->lock); @@ -392,8 +401,10 @@ return ENOTSUP; } + if (area->sh_info) { /* - * Remapping of shared address space areas + * Remapping of shared address space areas * is not supported. + * */ mutex_unlock(&area->lock); @@ -402,9 +413,10 @@ return ENOTSUP; } - - pages = SIZE2FRAMES((address - area->base) + size); + + size_t pages = SIZE2FRAMES((address - area->base) + size); if (!pages) { /* * Zero size address space areas are not allowed. + * */ mutex_unlock(&area->lock); @@ -415,20 +427,21 @@ if (pages < area->pages) { - bool cond; uintptr_t start_free = area->base + pages * PAGE_SIZE; - + /* * Shrinking the area. * No need to check for overlaps. - */ - + * + */ + page_table_lock(as, false); - + /* * Start TLB shootdown sequence. + * */ tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base + pages * PAGE_SIZE, area->pages - pages); - + /* * Remove frames belonging to used space starting from @@ -437,42 +450,46 @@ * is also the right way to remove part of the used_space * B+tree leaf list. - */ - for (cond = true; cond;) { - btree_node_t *node; - + * + */ + bool cond = true; + while (cond) { ASSERT(!list_empty(&area->used_space.leaf_head)); - node = + + btree_node_t *node = list_get_instance(area->used_space.leaf_head.prev, btree_node_t, leaf_link); + if ((cond = (bool) node->keys)) { - uintptr_t b = node->key[node->keys - 1]; - size_t c = + uintptr_t ptr = node->key[node->keys - 1]; + size_t size = (size_t) node->value[node->keys - 1]; - unsigned int i = 0; - - if (overlaps(b, c * PAGE_SIZE, area->base, + size_t i = 0; + + if (overlaps(ptr, size * PAGE_SIZE, area->base, pages * PAGE_SIZE)) { - if (b + c * PAGE_SIZE <= start_free) { + if (ptr + size * PAGE_SIZE <= start_free) { /* * The whole interval fits * completely in the resized * address space area. + * */ break; } - + /* * Part of the interval corresponding * to b and c overlaps with the resized * address space area. + * */ - - cond = false; /* we are almost done */ - i = (start_free - b) >> PAGE_WIDTH; + + /* We are almost done */ + cond = false; + i = (start_free - ptr) >> PAGE_WIDTH; if (!used_space_remove(area, start_free, - c - i)) - panic("Cannot remove used " - "space."); + size - i)) + panic("Cannot remove used space."); } else { /* @@ -480,58 +497,61 @@ * completely removed. */ - if (!used_space_remove(area, b, c)) - panic("Cannot remove used " - "space."); + if (!used_space_remove(area, ptr, size)) + panic("Cannot remove used space."); } - - for (; i < c; i++) { - pte_t *pte; - - pte = page_mapping_find(as, b + + + for (; i < size; i++) { + pte_t *pte = page_mapping_find(as, ptr + i * PAGE_SIZE); - ASSERT(pte && PTE_VALID(pte) && - PTE_PRESENT(pte)); - if (area->backend && - area->backend->frame_free) { + + ASSERT(pte); + ASSERT(PTE_VALID(pte)); + ASSERT(PTE_PRESENT(pte)); + + if ((area->backend) && + (area->backend->frame_free)) { area->backend->frame_free(area, - b + i * PAGE_SIZE, + ptr + i * PAGE_SIZE, PTE_GET_FRAME(pte)); } - page_mapping_remove(as, b + + + page_mapping_remove(as, ptr + i * PAGE_SIZE); } } } - + /* * Finish TLB shootdown sequence. - */ - + * + */ + tlb_invalidate_pages(as->asid, area->base + pages * PAGE_SIZE, area->pages - pages); - + /* * Invalidate software translation caches (e.g. TSB on sparc64). + * */ as_invalidate_translation_cache(as, area->base + pages * PAGE_SIZE, area->pages - pages); tlb_shootdown_finalize(); - + page_table_unlock(as, false); - } else { /* * Growing the area. * Check for overlaps with other address space areas. + * */ if (!check_area_conflicts(as, address, pages * PAGE_SIZE, area)) { mutex_unlock(&area->lock); - mutex_unlock(&as->lock); + mutex_unlock(&as->lock); interrupts_restore(ipl); return EADDRNOTAVAIL; } - } - + } + area->pages = pages; @@ -539,5 +559,5 @@ mutex_unlock(&as->lock); interrupts_restore(ipl); - + return 0; } @@ -545,20 +565,16 @@ /** Destroy address space area. * - * @param as Address space. - * @param address Address within the area to be deleted. - * - * @return Zero on success or a value from @ref errno.h on failure. + * @param as Address space. + * @param address Address within the area to be deleted. + * + * @return Zero on success or a value from @ref errno.h on failure. + * */ int as_area_destroy(as_t *as, uintptr_t address) { - as_area_t *area; - uintptr_t base; - link_t *cur; - ipl_t ipl; - - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); - - area = find_area_and_lock(as, address); + + as_area_t *area = find_area_and_lock(as, address); if (!area) { mutex_unlock(&as->lock); @@ -566,68 +582,75 @@ return ENOENT; } - - base = area->base; - + + uintptr_t base = area->base; + page_table_lock(as, false); - + /* * Start TLB shootdown sequence. */ tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base, area->pages); - + /* * Visit only the pages mapped by used_space B+tree. */ + link_t *cur; for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { btree_node_t *node; - unsigned int i; + btree_key_t i; node = list_get_instance(cur, btree_node_t, leaf_link); for (i = 0; i < node->keys; i++) { - uintptr_t b = node->key[i]; - size_t j; - pte_t *pte; + uintptr_t ptr = node->key[i]; + size_t size; - for (j = 0; j < (size_t) node->value[i]; j++) { - pte = page_mapping_find(as, b + j * PAGE_SIZE); - ASSERT(pte && PTE_VALID(pte) && - PTE_PRESENT(pte)); - if (area->backend && - area->backend->frame_free) { - area->backend->frame_free(area, b + - j * PAGE_SIZE, PTE_GET_FRAME(pte)); + for (size = 0; size < (size_t) node->value[i]; size++) { + pte_t *pte = page_mapping_find(as, ptr + size * PAGE_SIZE); + + ASSERT(pte); + ASSERT(PTE_VALID(pte)); + ASSERT(PTE_PRESENT(pte)); + + if ((area->backend) && + (area->backend->frame_free)) { + area->backend->frame_free(area, + ptr + size * PAGE_SIZE, PTE_GET_FRAME(pte)); } - page_mapping_remove(as, b + j * PAGE_SIZE); + + page_mapping_remove(as, ptr + size * PAGE_SIZE); } } } - + /* * Finish TLB shootdown sequence. - */ - + * + */ + tlb_invalidate_pages(as->asid, area->base, area->pages); - + /* * Invalidate potential software translation caches (e.g. TSB on * sparc64). + * */ as_invalidate_translation_cache(as, area->base, area->pages); tlb_shootdown_finalize(); - + page_table_unlock(as, false); btree_destroy(&area->used_space); - + area->attributes |= AS_AREA_ATTR_PARTIAL; if (area->sh_info) sh_info_remove_reference(area->sh_info); - + mutex_unlock(&area->lock); - + /* * Remove the empty area from address space. + * */ btree_remove(&as->as_area_btree, base, NULL); @@ -647,36 +670,31 @@ * sh_info of the source area. The process of duplicating the * mapping is done through the backend share function. - * - * @param src_as Pointer to source address space. - * @param src_base Base address of the source address space area. - * @param acc_size Expected size of the source area. - * @param dst_as Pointer to destination address space. - * @param dst_base Target base address. + * + * @param src_as Pointer to source address space. + * @param src_base Base address of the source address space area. + * @param acc_size Expected size of the source area. + * @param dst_as Pointer to destination address space. + * @param dst_base Target base address. * @param dst_flags_mask Destination address space area flags mask. * - * @return Zero on success or ENOENT if there is no such task or if - * there is no such address space area, EPERM if there was - * a problem in accepting the area or ENOMEM if there was a - * problem in allocating destination address space area. - * ENOTSUP is returned if the address space area backend - * does not support sharing. + * @return Zero on success. + * @return ENOENT if there is no such task or such address space. + * @return EPERM if there was a problem in accepting the area. + * @return ENOMEM if there was a problem in allocating destination + * address space area. + * @return ENOTSUP if the address space area backend does not support + * sharing. + * */ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, - as_t *dst_as, uintptr_t dst_base, int dst_flags_mask) -{ - ipl_t ipl; - int src_flags; - size_t src_size; - as_area_t *src_area, *dst_area; - share_info_t *sh_info; - mem_backend_t *src_backend; - mem_backend_data_t src_backend_data; - - ipl = interrupts_disable(); + as_t *dst_as, uintptr_t dst_base, unsigned int dst_flags_mask) +{ + ipl_t ipl = interrupts_disable(); mutex_lock(&src_as->lock); - src_area = find_area_and_lock(src_as, src_base); + as_area_t *src_area = find_area_and_lock(src_as, src_base); if (!src_area) { /* * Could not find the source address space area. + * */ mutex_unlock(&src_as->lock); @@ -684,9 +702,10 @@ return ENOENT; } - - if (!src_area->backend || !src_area->backend->share) { + + if ((!src_area->backend) || (!src_area->backend->share)) { /* * There is no backend or the backend does not * know how to share the area. + * */ mutex_unlock(&src_area->lock); @@ -696,15 +715,15 @@ } - src_size = src_area->pages * PAGE_SIZE; - src_flags = src_area->flags; - src_backend = src_area->backend; - src_backend_data = src_area->backend_data; - + size_t src_size = src_area->pages * PAGE_SIZE; + unsigned int src_flags = src_area->flags; + mem_backend_t *src_backend = src_area->backend; + mem_backend_data_t src_backend_data = src_area->backend_data; + /* Share the cacheable flag from the original mapping */ if (src_flags & AS_AREA_CACHEABLE) dst_flags_mask |= AS_AREA_CACHEABLE; - - if (src_size != acc_size || - (src_flags & dst_flags_mask) != dst_flags_mask) { + + if ((src_size != acc_size) || + ((src_flags & dst_flags_mask) != dst_flags_mask)) { mutex_unlock(&src_area->lock); mutex_unlock(&src_as->lock); @@ -712,11 +731,12 @@ return EPERM; } - + /* * Now we are committed to sharing the area. * First, prepare the area for sharing. * Then it will be safe to unlock it. - */ - sh_info = src_area->sh_info; + * + */ + share_info_t *sh_info = src_area->sh_info; if (!sh_info) { sh_info = (share_info_t *) malloc(sizeof(share_info_t), 0); @@ -725,6 +745,8 @@ btree_create(&sh_info->pagemap); src_area->sh_info = sh_info; + /* * Call the backend to setup sharing. + * */ src_area->backend->share(src_area); @@ -734,8 +756,8 @@ mutex_unlock(&sh_info->lock); } - + mutex_unlock(&src_area->lock); mutex_unlock(&src_as->lock); - + /* * Create copy of the source address space area. @@ -745,7 +767,8 @@ * The flags of the source area are masked against dst_flags_mask * to support sharing in less privileged mode. - */ - dst_area = as_area_create(dst_as, dst_flags_mask, src_size, dst_base, - AS_AREA_ATTR_PARTIAL, src_backend, &src_backend_data); + * + */ + as_area_t *dst_area = as_area_create(dst_as, dst_flags_mask, src_size, + dst_base, AS_AREA_ATTR_PARTIAL, src_backend, &src_backend_data); if (!dst_area) { /* @@ -757,17 +780,18 @@ return ENOMEM; } - + /* * Now the destination address space area has been * fully initialized. Clear the AS_AREA_ATTR_PARTIAL * attribute and set the sh_info. - */ - mutex_lock(&dst_as->lock); + * + */ + mutex_lock(&dst_as->lock); mutex_lock(&dst_area->lock); dst_area->attributes &= ~AS_AREA_ATTR_PARTIAL; dst_area->sh_info = sh_info; mutex_unlock(&dst_area->lock); - mutex_unlock(&dst_as->lock); - + mutex_unlock(&dst_as->lock); + interrupts_restore(ipl); @@ -779,9 +803,10 @@ * The address space area must be locked prior to this call. * - * @param area Address space area. - * @param access Access mode. - * - * @return False if access violates area's permissions, true - * otherwise. + * @param area Address space area. + * @param access Access mode. + * + * @return False if access violates area's permissions, true + * otherwise. + * */ bool as_area_check_access(as_area_t *area, pf_access_t access) @@ -792,5 +817,5 @@ [PF_ACCESS_EXEC] = AS_AREA_EXEC }; - + if (!(area->flags & flagmap[access])) return false; @@ -813,21 +838,13 @@ * */ -int as_area_change_flags(as_t *as, int flags, uintptr_t address) -{ - as_area_t *area; - link_t *cur; - ipl_t ipl; - int page_flags; - uintptr_t *old_frame; - size_t frame_idx; - size_t used_pages; - +int as_area_change_flags(as_t *as, unsigned int flags, uintptr_t address) +{ /* Flags for the new memory mapping */ - page_flags = area_flags_to_page_flags(flags); - - ipl = interrupts_disable(); + unsigned int page_flags = area_flags_to_page_flags(flags); + + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); - - area = find_area_and_lock(as, address); + + as_area_t *area = find_area_and_lock(as, address); if (!area) { mutex_unlock(&as->lock); @@ -835,5 +852,5 @@ return ENOENT; } - + if ((area->sh_info) || (area->backend != &anon_backend)) { /* Copying shared areas not supported yet */ @@ -844,64 +861,70 @@ return ENOTSUP; } - + /* * Compute total number of used pages in the used_space B+tree - */ - used_pages = 0; - + * + */ + size_t used_pages = 0; + link_t *cur; + for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; - - node = list_get_instance(cur, btree_node_t, leaf_link); - for (i = 0; i < node->keys; i++) { + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + + for (i = 0; i < node->keys; i++) used_pages += (size_t) node->value[i]; - } - } - + } + /* An array for storing frame numbers */ - old_frame = malloc(used_pages * sizeof(uintptr_t), 0); - + uintptr_t *old_frame = malloc(used_pages * sizeof(uintptr_t), 0); + page_table_lock(as, false); - + /* * Start TLB shootdown sequence. + * */ tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base, area->pages); - + /* * Remove used pages from page tables and remember their frame * numbers. - */ - frame_idx = 0; - + * + */ + size_t frame_idx = 0; + for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; - - node = list_get_instance(cur, btree_node_t, leaf_link); + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { - uintptr_t b = node->key[i]; - size_t j; - pte_t *pte; + uintptr_t ptr = node->key[i]; + size_t size; - for (j = 0; j < (size_t) node->value[i]; j++) { - pte = page_mapping_find(as, b + j * PAGE_SIZE); - ASSERT(pte && PTE_VALID(pte) && - PTE_PRESENT(pte)); + for (size = 0; size < (size_t) node->value[i]; size++) { + pte_t *pte = page_mapping_find(as, ptr + size * PAGE_SIZE); + + ASSERT(pte); + ASSERT(PTE_VALID(pte)); + ASSERT(PTE_PRESENT(pte)); + old_frame[frame_idx++] = PTE_GET_FRAME(pte); - + /* Remove old mapping */ - page_mapping_remove(as, b + j * PAGE_SIZE); + page_mapping_remove(as, ptr + size * PAGE_SIZE); } } } - + /* * Finish TLB shootdown sequence. - */ - + * + */ + tlb_invalidate_pages(as->asid, area->base, area->pages); @@ -909,15 +932,16 @@ * Invalidate potential software translation caches (e.g. TSB on * sparc64). + * */ as_invalidate_translation_cache(as, area->base, area->pages); tlb_shootdown_finalize(); - + page_table_unlock(as, false); - + /* * Set the new flags. */ area->flags = flags; - + /* * Map pages back in with new flags. This step is kept separate @@ -926,36 +950,35 @@ */ frame_idx = 0; - + for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; - - node = list_get_instance(cur, btree_node_t, leaf_link); + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { - uintptr_t b = node->key[i]; - size_t j; + uintptr_t ptr = node->key[i]; + size_t size; - for (j = 0; j < (size_t) node->value[i]; j++) { + for (size = 0; size < (size_t) node->value[i]; size++) { page_table_lock(as, false); - + /* Insert the new mapping */ - page_mapping_insert(as, b + j * PAGE_SIZE, + page_mapping_insert(as, ptr + size * PAGE_SIZE, old_frame[frame_idx++], page_flags); - + page_table_unlock(as, false); } } } - + free(old_frame); - + mutex_unlock(&area->lock); mutex_unlock(&as->lock); interrupts_restore(ipl); - + return 0; } - /** Handle page fault within the current address space. @@ -967,18 +990,17 @@ * Interrupts are assumed disabled. * - * @param page Faulting page. - * @param access Access mode that caused the page fault (i.e. - * read/write/exec). - * @param istate Pointer to the interrupted state. - * - * @return AS_PF_FAULT on page fault, AS_PF_OK on success or - * AS_PF_DEFER if the fault was caused by copy_to_uspace() - * or copy_from_uspace(). + * @param page Faulting page. + * @param access Access mode that caused the page fault (i.e. + * read/write/exec). + * @param istate Pointer to the interrupted state. + * + * @return AS_PF_FAULT on page fault. + * @return AS_PF_OK on success. + * @return AS_PF_DEFER if the fault was caused by copy_to_uspace() + * or copy_from_uspace(). + * */ int as_page_fault(uintptr_t page, pf_access_t access, istate_t *istate) { - pte_t *pte; - as_area_t *area; - if (!THREAD) return AS_PF_FAULT; @@ -988,14 +1010,15 @@ mutex_lock(&AS->lock); - area = find_area_and_lock(AS, page); + as_area_t *area = find_area_and_lock(AS, page); if (!area) { /* * No area contained mapping for 'page'. * Signal page fault to low-level handler. + * */ mutex_unlock(&AS->lock); goto page_fault; } - + if (area->attributes & AS_AREA_ATTR_PARTIAL) { /* @@ -1005,17 +1028,18 @@ mutex_unlock(&area->lock); mutex_unlock(&AS->lock); - goto page_fault; - } - - if (!area->backend || !area->backend->page_fault) { + goto page_fault; + } + + if ((!area->backend) || (!area->backend->page_fault)) { /* * The address space area is not backed by any backend * or the backend cannot handle page faults. + * */ mutex_unlock(&area->lock); mutex_unlock(&AS->lock); - goto page_fault; - } - + goto page_fault; + } + page_table_lock(AS, false); @@ -1023,5 +1047,7 @@ * To avoid race condition between two page faults on the same address, * we need to make sure the mapping has not been already inserted. - */ + * + */ + pte_t *pte; if ((pte = page_mapping_find(AS, page))) { if (PTE_PRESENT(pte)) { @@ -1039,4 +1065,5 @@ /* * Resort to the backend page fault handler. + * */ if (area->backend->page_fault(area, page, access) != AS_PF_OK) { @@ -1051,5 +1078,5 @@ mutex_unlock(&AS->lock); return AS_PF_OK; - + page_fault: if (THREAD->in_copy_from_uspace) { @@ -1064,5 +1091,5 @@ return AS_PF_FAULT; } - + return AS_PF_DEFER; } @@ -1076,6 +1103,7 @@ * When this function is enetered, no spinlocks may be held. * - * @param old Old address space or NULL. - * @param new New address space. + * @param old Old address space or NULL. + * @param new New address space. + * */ void as_switch(as_t *old_as, as_t *new_as) @@ -1083,12 +1111,14 @@ DEADLOCK_PROBE_INIT(p_asidlock); preemption_disable(); + retry: (void) interrupts_disable(); if (!spinlock_trylock(&asidlock)) { - /* + /* * Avoid deadlock with TLB shootdown. * We can enable interrupts here because * preemption is disabled. We should not be * holding any other lock. + * */ (void) interrupts_enable(); @@ -1097,11 +1127,12 @@ } preemption_enable(); - + /* * First, take care of the old address space. - */ + */ if (old_as) { ASSERT(old_as->cpu_refcount); - if((--old_as->cpu_refcount == 0) && (old_as != AS_KERNEL)) { + + if ((--old_as->cpu_refcount == 0) && (old_as != AS_KERNEL)) { /* * The old address space is no longer active on @@ -1109,19 +1140,23 @@ * list of inactive address spaces with assigned * ASID. + * */ ASSERT(old_as->asid != ASID_INVALID); + list_append(&old_as->inactive_as_with_asid_link, &inactive_as_with_asid_head); } - + /* * Perform architecture-specific tasks when the address space * is being removed from the CPU. + * */ as_deinstall_arch(old_as); } - + /* * Second, prepare the new address space. + * */ if ((new_as->cpu_refcount++ == 0) && (new_as != AS_KERNEL)) { @@ -1131,4 +1166,5 @@ new_as->asid = asid_get(); } + #ifdef AS_PAGE_TABLE SET_PTL0_ADDRESS(new_as->genarch.page_table); @@ -1138,7 +1174,8 @@ * Perform architecture-specific steps. * (e.g. write ASID to hardware register etc.) + * */ as_install_arch(new_as); - + spinlock_unlock(&asidlock); @@ -1148,13 +1185,12 @@ /** Convert address space area flags to page flags. * - * @param aflags Flags of some address space area. - * - * @return Flags to be passed to page_mapping_insert(). - */ -int area_flags_to_page_flags(int aflags) -{ - int flags; - - flags = PAGE_USER | PAGE_PRESENT; + * @param aflags Flags of some address space area. + * + * @return Flags to be passed to page_mapping_insert(). + * + */ +unsigned int area_flags_to_page_flags(unsigned int aflags) +{ + unsigned int flags = PAGE_USER | PAGE_PRESENT; if (aflags & AS_AREA_READ) @@ -1169,5 +1205,5 @@ if (aflags & AS_AREA_CACHEABLE) flags |= PAGE_CACHEABLE; - + return flags; } @@ -1178,11 +1214,12 @@ * Interrupts must be disabled. * - * @param a Address space area. - * - * @return Flags to be used in page_mapping_insert(). - */ -int as_area_get_flags(as_area_t *a) -{ - return area_flags_to_page_flags(a->flags); + * @param area Address space area. + * + * @return Flags to be used in page_mapping_insert(). + * + */ +unsigned int as_area_get_flags(as_area_t *area) +{ + return area_flags_to_page_flags(area->flags); } @@ -1192,10 +1229,11 @@ * table. * - * @param flags Flags saying whether the page table is for the kernel - * address space. - * - * @return First entry of the page table. - */ -pte_t *page_table_create(int flags) + * @param flags Flags saying whether the page table is for the kernel + * address space. + * + * @return First entry of the page table. + * + */ +pte_t *page_table_create(unsigned int flags) { ASSERT(as_operations); @@ -1209,5 +1247,6 @@ * Destroy page table in architecture specific way. * - * @param page_table Physical address of PTL0. + * @param page_table Physical address of PTL0. + * */ void page_table_destroy(pte_t *page_table) @@ -1223,11 +1262,12 @@ * This function should be called before any page_mapping_insert(), * page_mapping_remove() and page_mapping_find(). - * + * * Locking order is such that address space areas must be locked * prior to this call. Address space can be locked prior to this * call in which case the lock argument is false. * - * @param as Address space. - * @param lock If false, do not attempt to lock as->lock. + * @param as Address space. + * @param lock If false, do not attempt to lock as->lock. + * */ void page_table_lock(as_t *as, bool lock) @@ -1241,6 +1281,7 @@ /** Unlock page table. * - * @param as Address space. - * @param unlock If false, do not attempt to unlock as->lock. + * @param as Address space. + * @param unlock If false, do not attempt to unlock as->lock. + * */ void page_table_unlock(as_t *as, bool unlock) @@ -1257,21 +1298,19 @@ * The address space must be locked and interrupts must be disabled. * - * @param as Address space. - * @param va Virtual address. - * - * @return Locked address space area containing va on success or - * NULL on failure. + * @param as Address space. + * @param va Virtual address. + * + * @return Locked address space area containing va on success or + * NULL on failure. + * */ as_area_t *find_area_and_lock(as_t *as, uintptr_t va) { - as_area_t *a; - btree_node_t *leaf, *lnode; - unsigned int i; - - a = (as_area_t *) btree_search(&as->as_area_btree, va, &leaf); - if (a) { + btree_node_t *leaf; + as_area_t *area = (as_area_t *) btree_search(&as->as_area_btree, va, &leaf); + if (area) { /* va is the base address of an address space area */ - mutex_lock(&a->lock); - return a; + mutex_lock(&area->lock); + return area; } @@ -1280,30 +1319,38 @@ * to find out whether this is a miss or va belongs to an address * space area found there. + * */ /* First, search the leaf node itself. */ + btree_key_t i; + for (i = 0; i < leaf->keys; i++) { - a = (as_area_t *) leaf->value[i]; - mutex_lock(&a->lock); - if ((a->base <= va) && (va < a->base + a->pages * PAGE_SIZE)) { - return a; - } - mutex_unlock(&a->lock); - } - + area = (as_area_t *) leaf->value[i]; + + mutex_lock(&area->lock); + + if ((area->base <= va) && (va < area->base + area->pages * PAGE_SIZE)) + return area; + + mutex_unlock(&area->lock); + } + /* * Second, locate the left neighbour and test its last record. * Because of its position in the B+tree, it must have base < va. - */ - lnode = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); + * + */ + btree_node_t *lnode = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); if (lnode) { - a = (as_area_t *) lnode->value[lnode->keys - 1]; - mutex_lock(&a->lock); - if (va < a->base + a->pages * PAGE_SIZE) { - return a; - } - mutex_unlock(&a->lock); - } - + area = (as_area_t *) lnode->value[lnode->keys - 1]; + + mutex_lock(&area->lock); + + if (va < area->base + area->pages * PAGE_SIZE) + return area; + + mutex_unlock(&area->lock); + } + return NULL; } @@ -1313,20 +1360,18 @@ * The address space must be locked and interrupts must be disabled. * - * @param as Address space. - * @param va Starting virtual address of the area being tested. - * @param size Size of the area being tested. - * @param avoid_area Do not touch this area. - * - * @return True if there is no conflict, false otherwise. - */ -bool -check_area_conflicts(as_t *as, uintptr_t va, size_t size, as_area_t *avoid_area) -{ - as_area_t *a; - btree_node_t *leaf, *node; - unsigned int i; - + * @param as Address space. + * @param va Starting virtual address of the area being tested. + * @param size Size of the area being tested. + * @param avoid_area Do not touch this area. + * + * @return True if there is no conflict, false otherwise. + * + */ +bool check_area_conflicts(as_t *as, uintptr_t va, size_t size, + as_area_t *avoid_area) +{ /* * We don't want any area to have conflicts with NULL page. + * */ if (overlaps(va, size, NULL, PAGE_SIZE)) @@ -1339,57 +1384,73 @@ * record in the left neighbour, the leftmost record in the right * neighbour and all records in the leaf node itself. - */ - - if ((a = (as_area_t *) btree_search(&as->as_area_btree, va, &leaf))) { - if (a != avoid_area) + * + */ + btree_node_t *leaf; + as_area_t *area = + (as_area_t *) btree_search(&as->as_area_btree, va, &leaf); + if (area) { + if (area != avoid_area) return false; } /* First, check the two border cases. */ - if ((node = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf))) { - a = (as_area_t *) node->value[node->keys - 1]; - mutex_lock(&a->lock); - if (overlaps(va, size, a->base, a->pages * PAGE_SIZE)) { - mutex_unlock(&a->lock); + btree_node_t *node = + btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); + if (node) { + area = (as_area_t *) node->value[node->keys - 1]; + + mutex_lock(&area->lock); + + if (overlaps(va, size, area->base, area->pages * PAGE_SIZE)) { + mutex_unlock(&area->lock); return false; } - mutex_unlock(&a->lock); - } + + mutex_unlock(&area->lock); + } + node = btree_leaf_node_right_neighbour(&as->as_area_btree, leaf); if (node) { - a = (as_area_t *) node->value[0]; - mutex_lock(&a->lock); - if (overlaps(va, size, a->base, a->pages * PAGE_SIZE)) { - mutex_unlock(&a->lock); + area = (as_area_t *) node->value[0]; + + mutex_lock(&area->lock); + + if (overlaps(va, size, area->base, area->pages * PAGE_SIZE)) { + mutex_unlock(&area->lock); return false; } - mutex_unlock(&a->lock); + + mutex_unlock(&area->lock); } /* Second, check the leaf node. */ + btree_key_t i; for (i = 0; i < leaf->keys; i++) { - a = (as_area_t *) leaf->value[i]; - - if (a == avoid_area) + area = (as_area_t *) leaf->value[i]; + + if (area == avoid_area) continue; - - mutex_lock(&a->lock); - if (overlaps(va, size, a->base, a->pages * PAGE_SIZE)) { - mutex_unlock(&a->lock); + + mutex_lock(&area->lock); + + if (overlaps(va, size, area->base, area->pages * PAGE_SIZE)) { + mutex_unlock(&area->lock); return false; } - mutex_unlock(&a->lock); - } - + + mutex_unlock(&area->lock); + } + /* * So far, the area does not conflict with other areas. * Check if it doesn't conflict with kernel address space. - */ + * + */ if (!KERNEL_ADDRESS_SPACE_SHADOWED) { - return !overlaps(va, size, + return !overlaps(va, size, KERNEL_ADDRESS_SPACE_START, KERNEL_ADDRESS_SPACE_END - KERNEL_ADDRESS_SPACE_START); } - + return true; } @@ -1397,23 +1458,23 @@ /** Return size of the address space area with given base. * - * @param base Arbitrary address insede the address space area. - * - * @return Size of the address space area in bytes or zero if it - * does not exist. + * @param base Arbitrary address insede the address space area. + * + * @return Size of the address space area in bytes or zero if it + * does not exist. + * */ size_t as_area_get_size(uintptr_t base) { - ipl_t ipl; - as_area_t *src_area; size_t size; - - ipl = interrupts_disable(); - src_area = find_area_and_lock(AS, base); + + ipl_t ipl = interrupts_disable(); + as_area_t *src_area = find_area_and_lock(AS, base); + if (src_area) { size = src_area->pages * PAGE_SIZE; mutex_unlock(&src_area->lock); - } else { + } else size = 0; - } + interrupts_restore(ipl); return size; @@ -1424,33 +1485,32 @@ * The address space area must be already locked. * - * @param a Address space area. - * @param page First page to be marked. - * @param count Number of page to be marked. - * - * @return Zero on failure and non-zero on success. - */ -int used_space_insert(as_area_t *a, uintptr_t page, size_t count) -{ - btree_node_t *leaf, *node; - size_t pages; - unsigned int i; - + * @param area Address space area. + * @param page First page to be marked. + * @param count Number of page to be marked. + * + * @return Zero on failure and non-zero on success. + * + */ +int used_space_insert(as_area_t *area, uintptr_t page, size_t count) +{ ASSERT(page == ALIGN_DOWN(page, PAGE_SIZE)); ASSERT(count); - - pages = (size_t) btree_search(&a->used_space, page, &leaf); + + btree_node_t *leaf; + size_t pages = (size_t) btree_search(&area->used_space, page, &leaf); if (pages) { /* * We hit the beginning of some used space. + * */ return 0; } - + if (!leaf->keys) { - btree_insert(&a->used_space, page, (void *) count, leaf); + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; } - - node = btree_leaf_node_left_neighbour(&a->used_space, leaf); + + btree_node_t *node = btree_leaf_node_left_neighbour(&area->used_space, leaf); if (node) { uintptr_t left_pg = node->key[node->keys - 1]; @@ -1463,6 +1523,7 @@ * somewhere between the rightmost interval of * the left neigbour and the first interval of the leaf. - */ - + * + */ + if (page >= right_pg) { /* Do nothing. */ @@ -1474,5 +1535,5 @@ right_cnt * PAGE_SIZE)) { /* The interval intersects with the right interval. */ - return 0; + return 0; } else if ((page == left_pg + left_cnt * PAGE_SIZE) && (page + count * PAGE_SIZE == right_pg)) { @@ -1480,12 +1541,14 @@ * The interval can be added by merging the two already * present intervals. + * */ node->value[node->keys - 1] += count + right_cnt; - btree_remove(&a->used_space, right_pg, leaf); - return 1; + btree_remove(&area->used_space, right_pg, leaf); + return 1; } else if (page == left_pg + left_cnt * PAGE_SIZE) { - /* + /* * The interval can be added by simply growing the left * interval. + * */ node->value[node->keys - 1] += count; @@ -1496,4 +1559,5 @@ * the right interval down and increasing its size * accordingly. + * */ leaf->value[0] += count; @@ -1504,6 +1568,7 @@ * The interval is between both neigbouring intervals, * but cannot be merged with any of them. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1512,10 +1577,11 @@ uintptr_t right_pg = leaf->key[0]; size_t right_cnt = (size_t) leaf->value[0]; - + /* * Investigate the border case in which the left neighbour does * not exist but the interval fits from the left. - */ - + * + */ + if (overlaps(page, count * PAGE_SIZE, right_pg, right_cnt * PAGE_SIZE)) { @@ -1527,4 +1593,5 @@ * right interval down and increasing its size * accordingly. + * */ leaf->key[0] = page; @@ -1535,12 +1602,13 @@ * The interval doesn't adjoin with the right interval. * It must be added individually. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; } } - - node = btree_leaf_node_right_neighbour(&a->used_space, leaf); + + node = btree_leaf_node_right_neighbour(&area->used_space, leaf); if (node) { uintptr_t left_pg = leaf->key[leaf->keys - 1]; @@ -1553,6 +1621,7 @@ * somewhere between the leftmost interval of * the right neigbour and the last interval of the leaf. - */ - + * + */ + if (page < left_pg) { /* Do nothing. */ @@ -1564,5 +1633,5 @@ right_cnt * PAGE_SIZE)) { /* The interval intersects with the right interval. */ - return 0; + return 0; } else if ((page == left_pg + left_cnt * PAGE_SIZE) && (page + count * PAGE_SIZE == right_pg)) { @@ -1570,13 +1639,15 @@ * The interval can be added by merging the two already * present intervals. - * */ + * + */ leaf->value[leaf->keys - 1] += count + right_cnt; - btree_remove(&a->used_space, right_pg, node); - return 1; + btree_remove(&area->used_space, right_pg, node); + return 1; } else if (page == left_pg + left_cnt * PAGE_SIZE) { /* * The interval can be added by simply growing the left * interval. - * */ + * + */ leaf->value[leaf->keys - 1] += count; return 1; @@ -1586,4 +1657,5 @@ * the right interval down and increasing its size * accordingly. + * */ node->value[0] += count; @@ -1594,6 +1666,7 @@ * The interval is between both neigbouring intervals, * but cannot be merged with any of them. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1602,10 +1675,11 @@ uintptr_t left_pg = leaf->key[leaf->keys - 1]; size_t left_cnt = (size_t) leaf->value[leaf->keys - 1]; - + /* * Investigate the border case in which the right neighbour * does not exist but the interval fits from the right. - */ - + * + */ + if (overlaps(page, count * PAGE_SIZE, left_pg, left_cnt * PAGE_SIZE)) { @@ -1616,4 +1690,5 @@ * The interval can be added by growing the left * interval. + * */ leaf->value[leaf->keys - 1] += count; @@ -1623,6 +1698,7 @@ * The interval doesn't adjoin with the left interval. * It must be added individually. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1634,5 +1710,7 @@ * only between two other intervals of the leaf. The two border cases * were already resolved. - */ + * + */ + btree_key_t i; for (i = 1; i < leaf->keys; i++) { if (page < leaf->key[i]) { @@ -1641,9 +1719,10 @@ size_t left_cnt = (size_t) leaf->value[i - 1]; size_t right_cnt = (size_t) leaf->value[i]; - + /* * The interval fits between left_pg and right_pg. + * */ - + if (overlaps(page, count * PAGE_SIZE, left_pg, left_cnt * PAGE_SIZE)) { @@ -1651,4 +1730,5 @@ * The interval intersects with the left * interval. + * */ return 0; @@ -1658,6 +1738,7 @@ * The interval intersects with the right * interval. + * */ - return 0; + return 0; } else if ((page == left_pg + left_cnt * PAGE_SIZE) && (page + count * PAGE_SIZE == right_pg)) { @@ -1665,12 +1746,14 @@ * The interval can be added by merging the two * already present intervals. + * */ leaf->value[i - 1] += count + right_cnt; - btree_remove(&a->used_space, right_pg, leaf); - return 1; + btree_remove(&area->used_space, right_pg, leaf); + return 1; } else if (page == left_pg + left_cnt * PAGE_SIZE) { /* * The interval can be added by simply growing * the left interval. + * */ leaf->value[i - 1] += count; @@ -1678,8 +1761,9 @@ } else if (page + count * PAGE_SIZE == right_pg) { /* - * The interval can be addded by simply moving + * The interval can be addded by simply moving * base of the right interval down and * increasing its size accordingly. - */ + * + */ leaf->value[i] += count; leaf->key[i] = page; @@ -1690,6 +1774,7 @@ * intervals, but cannot be merged with any of * them. + * */ - btree_insert(&a->used_space, page, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1697,5 +1782,5 @@ } } - + panic("Inconsistency detected while adding %" PRIs " pages of used " "space at %p.", count, page); @@ -1706,28 +1791,27 @@ * The address space area must be already locked. * - * @param a Address space area. - * @param page First page to be marked. - * @param count Number of page to be marked. - * - * @return Zero on failure and non-zero on success. - */ -int used_space_remove(as_area_t *a, uintptr_t page, size_t count) -{ - btree_node_t *leaf, *node; - size_t pages; - unsigned int i; - + * @param area Address space area. + * @param page First page to be marked. + * @param count Number of page to be marked. + * + * @return Zero on failure and non-zero on success. + * + */ +int used_space_remove(as_area_t *area, uintptr_t page, size_t count) +{ ASSERT(page == ALIGN_DOWN(page, PAGE_SIZE)); ASSERT(count); - - pages = (size_t) btree_search(&a->used_space, page, &leaf); + + btree_node_t *leaf; + size_t pages = (size_t) btree_search(&area->used_space, page, &leaf); if (pages) { /* * We are lucky, page is the beginning of some interval. + * */ if (count > pages) { return 0; } else if (count == pages) { - btree_remove(&a->used_space, page, leaf); + btree_remove(&area->used_space, page, leaf); return 1; } else { @@ -1735,5 +1819,7 @@ * Find the respective interval. * Decrease its size and relocate its start address. + * */ + btree_key_t i; for (i = 0; i < leaf->keys; i++) { if (leaf->key[i] == page) { @@ -1746,10 +1832,10 @@ } } - - node = btree_leaf_node_left_neighbour(&a->used_space, leaf); - if (node && page < leaf->key[0]) { + + btree_node_t *node = btree_leaf_node_left_neighbour(&area->used_space, leaf); + if ((node) && (page < leaf->key[0])) { uintptr_t left_pg = node->key[node->keys - 1]; size_t left_cnt = (size_t) node->value[node->keys - 1]; - + if (overlaps(left_pg, left_cnt * PAGE_SIZE, page, count * PAGE_SIZE)) { @@ -1761,4 +1847,5 @@ * removed by updating the size of the bigger * interval. + * */ node->value[node->keys - 1] -= count; @@ -1766,6 +1853,4 @@ } else if (page + count * PAGE_SIZE < left_pg + left_cnt*PAGE_SIZE) { - size_t new_cnt; - /* * The interval is contained in the rightmost @@ -1774,9 +1859,10 @@ * the original interval and also inserting a * new interval. + * */ - new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - + size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - (page + count*PAGE_SIZE)) >> PAGE_WIDTH; node->value[node->keys - 1] -= count + new_cnt; - btree_insert(&a->used_space, page + + btree_insert(&area->used_space, page + count * PAGE_SIZE, (void *) new_cnt, leaf); return 1; @@ -1784,15 +1870,14 @@ } return 0; - } else if (page < leaf->key[0]) { + } else if (page < leaf->key[0]) return 0; - } if (page > leaf->key[leaf->keys - 1]) { uintptr_t left_pg = leaf->key[leaf->keys - 1]; size_t left_cnt = (size_t) leaf->value[leaf->keys - 1]; - + if (overlaps(left_pg, left_cnt * PAGE_SIZE, page, count * PAGE_SIZE)) { - if (page + count * PAGE_SIZE == + if (page + count * PAGE_SIZE == left_pg + left_cnt * PAGE_SIZE) { /* @@ -1800,4 +1885,5 @@ * interval of the leaf and can be removed by * updating the size of the bigger interval. + * */ leaf->value[leaf->keys - 1] -= count; @@ -1805,6 +1891,4 @@ } else if (page + count * PAGE_SIZE < left_pg + left_cnt * PAGE_SIZE) { - size_t new_cnt; - /* * The interval is contained in the rightmost @@ -1813,9 +1897,10 @@ * original interval and also inserting a new * interval. + * */ - new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - + size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - (page + count * PAGE_SIZE)) >> PAGE_WIDTH; leaf->value[leaf->keys - 1] -= count + new_cnt; - btree_insert(&a->used_space, page + + btree_insert(&area->used_space, page + count * PAGE_SIZE, (void *) new_cnt, leaf); return 1; @@ -1823,5 +1908,5 @@ } return 0; - } + } /* @@ -1829,9 +1914,10 @@ * Now the interval can be only between intervals of the leaf. */ + btree_key_t i; for (i = 1; i < leaf->keys - 1; i++) { if (page < leaf->key[i]) { uintptr_t left_pg = leaf->key[i - 1]; size_t left_cnt = (size_t) leaf->value[i - 1]; - + /* * Now the interval is between intervals corresponding @@ -1847,4 +1933,5 @@ * be removed by updating the size of * the bigger interval. + * */ leaf->value[i - 1] -= count; @@ -1852,6 +1939,4 @@ } else if (page + count * PAGE_SIZE < left_pg + left_cnt * PAGE_SIZE) { - size_t new_cnt; - /* * The interval is contained in the @@ -1861,10 +1946,10 @@ * also inserting a new interval. */ - new_cnt = ((left_pg + + size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - (page + count * PAGE_SIZE)) >> PAGE_WIDTH; leaf->value[i - 1] -= count + new_cnt; - btree_insert(&a->used_space, page + + btree_insert(&area->used_space, page + count * PAGE_SIZE, (void *) new_cnt, leaf); @@ -1875,5 +1960,5 @@ } } - + error: panic("Inconsistency detected while removing %" PRIs " pages of used " @@ -1885,12 +1970,14 @@ * If the reference count drops to 0, the sh_info is deallocated. * - * @param sh_info Pointer to address space area share info. + * @param sh_info Pointer to address space area share info. + * */ void sh_info_remove_reference(share_info_t *sh_info) { bool dealloc = false; - + mutex_lock(&sh_info->lock); ASSERT(sh_info->refcount); + if (--sh_info->refcount == 0) { dealloc = true; @@ -1903,9 +1990,9 @@ for (cur = sh_info->pagemap.leaf_head.next; cur != &sh_info->pagemap.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; - node = list_get_instance(cur, btree_node_t, leaf_link); - for (i = 0; i < node->keys; i++) + for (i = 0; i < node->keys; i++) frame_free((uintptr_t) node->value[i]); } @@ -1925,5 +2012,5 @@ /** Wrapper for as_area_create(). */ -unative_t sys_as_area_create(uintptr_t address, size_t size, int flags) +unative_t sys_as_area_create(uintptr_t address, size_t size, unsigned int flags) { if (as_area_create(AS, flags | AS_AREA_CACHEABLE, size, address, @@ -1935,5 +2022,5 @@ /** Wrapper for as_area_resize(). */ -unative_t sys_as_area_resize(uintptr_t address, size_t size, int flags) +unative_t sys_as_area_resize(uintptr_t address, size_t size, unsigned int flags) { return (unative_t) as_area_resize(AS, address, size, 0); @@ -1941,5 +2028,5 @@ /** Wrapper for as_area_change_flags(). */ -unative_t sys_as_area_change_flags(uintptr_t address, int flags) +unative_t sys_as_area_change_flags(uintptr_t address, unsigned int flags) { return (unative_t) as_area_change_flags(AS, flags, address); @@ -1954,77 +2041,69 @@ /** Get list of adress space areas. * - * @param as Address space. - * @param obuf Place to save pointer to returned buffer. - * @param osize Place to save size of returned buffer. + * @param as Address space. + * @param obuf Place to save pointer to returned buffer. + * @param osize Place to save size of returned buffer. + * */ void as_get_area_info(as_t *as, as_area_info_t **obuf, size_t *osize) { - ipl_t ipl; - size_t area_cnt, area_idx, i; + ipl_t ipl = interrupts_disable(); + mutex_lock(&as->lock); + + /* First pass, count number of areas. */ + + size_t area_cnt = 0; link_t *cur; - - as_area_info_t *info; - size_t isize; - - ipl = interrupts_disable(); - mutex_lock(&as->lock); - - /* First pass, count number of areas. */ - - area_cnt = 0; - + for (cur = as->as_area_btree.leaf_head.next; cur != &as->as_area_btree.leaf_head; cur = cur->next) { - btree_node_t *node; - - node = list_get_instance(cur, btree_node_t, leaf_link); + btree_node_t *node = + list_get_instance(cur, btree_node_t, leaf_link); area_cnt += node->keys; } - - isize = area_cnt * sizeof(as_area_info_t); - info = malloc(isize, 0); - + + size_t isize = area_cnt * sizeof(as_area_info_t); + as_area_info_t *info = malloc(isize, 0); + /* Second pass, record data. */ - - area_idx = 0; - + + size_t area_idx = 0; + for (cur = as->as_area_btree.leaf_head.next; cur != &as->as_area_btree.leaf_head; cur = cur->next) { - btree_node_t *node; - - node = list_get_instance(cur, btree_node_t, leaf_link); - + btree_node_t *node = + list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { as_area_t *area = node->value[i]; - + ASSERT(area_idx < area_cnt); mutex_lock(&area->lock); - + info[area_idx].start_addr = area->base; info[area_idx].size = FRAMES2SIZE(area->pages); info[area_idx].flags = area->flags; ++area_idx; - + mutex_unlock(&area->lock); } } - + mutex_unlock(&as->lock); interrupts_restore(ipl); - + *obuf = info; *osize = isize; } - /** Print out information about address space. * - * @param as Address space. + * @param as Address space. + * */ void as_print(as_t *as) { - ipl_t ipl; - - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); @@ -2033,12 +2112,11 @@ for (cur = as->as_area_btree.leaf_head.next; cur != &as->as_area_btree.leaf_head; cur = cur->next) { - btree_node_t *node; - - node = list_get_instance(cur, btree_node_t, leaf_link); - - unsigned int i; + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { as_area_t *area = node->value[i]; - + mutex_lock(&area->lock); printf("as_area: %p, base=%p, pages=%" PRIs Index: kernel/generic/src/mm/frame.c =================================================================== --- kernel/generic/src/mm/frame.c (revision c96452180f7af3adf60e8b6f88ac2ad6ec1683e6) +++ kernel/generic/src/mm/frame.c (revision da1bafb8cf9a3b3be8ef21bc114daaa476a85190) @@ -66,8 +66,8 @@ * available. */ -mutex_t mem_avail_mtx; -condvar_t mem_avail_cv; -size_t mem_avail_req = 0; /**< Number of frames requested. */ -size_t mem_avail_gen = 0; /**< Generation counter. */ +static mutex_t mem_avail_mtx; +static condvar_t mem_avail_cv; +static size_t mem_avail_req = 0; /**< Number of frames requested. */ +static size_t mem_avail_gen = 0; /**< Generation counter. */ /********************/ @@ -171,5 +171,5 @@ return total; } -#endif +#endif /* CONFIG_DEBUG */ /** Find a zone with a given frames. @@ -199,4 +199,5 @@ if (i >= zones.count) i = 0; + } while (i != hint); @@ -242,4 +243,5 @@ if (i >= zones.count) i = 0; + } while (i != hint); @@ -296,5 +298,5 @@ index = (frame_index(zone, frame)) + (1 << frame->buddy_order); - } else { /* is_right */ + } else { /* is_right */ index = (frame_index(zone, frame)) - (1 << frame->buddy_order); @@ -673,6 +675,5 @@ bool zone_merge(size_t z1, size_t z2) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); bool ret = true; @@ -744,6 +745,5 @@ errout: - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return ret; @@ -777,5 +777,6 @@ * */ -static void zone_construct(zone_t *zone, buddy_system_t *buddy, pfn_t start, size_t count, zone_flags_t flags) +static void zone_construct(zone_t *zone, buddy_system_t *buddy, pfn_t start, + size_t count, zone_flags_t flags) { zone->base = start; @@ -841,8 +842,8 @@ * */ -size_t zone_create(pfn_t start, size_t count, pfn_t confframe, zone_flags_t flags) -{ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); +size_t zone_create(pfn_t start, size_t count, pfn_t confframe, + zone_flags_t flags) +{ + irq_spinlock_lock(&zones.lock, true); if (zone_flags_available(flags)) { /* Create available zone */ @@ -889,6 +890,5 @@ size_t znum = zones_insert_zone(start, count); if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return (size_t) -1; } @@ -905,6 +905,5 @@ } - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return znum; @@ -914,12 +913,10 @@ size_t znum = zones_insert_zone(start, count); if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return (size_t) -1; } zone_construct(&zones.info[znum], NULL, start, count, flags); - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return znum; @@ -933,6 +930,5 @@ void frame_set_parent(pfn_t pfn, void *data, size_t hint) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = find_zone(pfn, 1, hint); @@ -943,12 +939,10 @@ pfn - zones.info[znum].base)->parent = data; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); } void *frame_get_parent(pfn_t pfn, size_t hint) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = find_zone(pfn, 1, hint); @@ -959,6 +953,5 @@ pfn - zones.info[znum].base)->parent; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return res; @@ -977,10 +970,8 @@ { size_t size = ((size_t) 1) << order; - ipl_t ipl; size_t hint = pzone ? (*pzone) : 0; loop: - ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); /* @@ -993,11 +984,7 @@ if it does not help, reclaim all */ if ((znum == (size_t) -1) && (!(flags & FRAME_NO_RECLAIM))) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&zones.lock, true); size_t freed = slab_reclaim(0); - - ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); if (freed > 0) @@ -1006,11 +993,7 @@ if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&zones.lock, true); freed = slab_reclaim(SLAB_RECLAIM_ALL); - - ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); if (freed > 0) @@ -1022,6 +1005,5 @@ if (znum == (size_t) -1) { if (flags & FRAME_ATOMIC) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return NULL; } @@ -1031,7 +1013,6 @@ #endif - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&zones.lock, true); + if (!THREAD) panic("Cannot wait for memory to become available."); @@ -1069,6 +1050,5 @@ + zones.info[znum].base; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); if (pzone) @@ -1092,6 +1072,5 @@ void frame_free(uintptr_t frame) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); /* @@ -1105,6 +1084,5 @@ zone_frame_free(&zones.info[znum], pfn - zones.info[znum].base); - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); /* @@ -1132,6 +1110,5 @@ void frame_reference_add(pfn_t pfn) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); /* @@ -1144,13 +1121,13 @@ zones.info[znum].frames[pfn - zones.info[znum].base].refcount++; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); -} - -/** Mark given range unavailable in frame zones. */ + irq_spinlock_unlock(&zones.lock, true); +} + +/** Mark given range unavailable in frame zones. + * + */ void frame_mark_unavailable(pfn_t start, size_t count) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t i; @@ -1164,14 +1141,15 @@ } - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); -} - -/** Initialize physical memory management. */ + irq_spinlock_unlock(&zones.lock, true); +} + +/** Initialize physical memory management. + * + */ void frame_init(void) { if (config.cpu_active == 1) { zones.count = 0; - spinlock_initialize(&zones.lock, "zones.lock"); + irq_spinlock_initialize(&zones.lock, "frame.zones.lock"); mutex_initialize(&mem_avail_mtx, MUTEX_ACTIVE); condvar_initialize(&mem_avail_cv); @@ -1204,9 +1182,10 @@ } -/** Return total size of all zones. */ +/** Return total size of all zones. + * + */ uint64_t zones_total_size(void) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); uint64_t total = 0; @@ -1215,6 +1194,5 @@ total += (uint64_t) FRAMES2SIZE(zones.info[i].count); - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return total; @@ -1229,6 +1207,5 @@ ASSERT(free != NULL); - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); *total = 0; @@ -1248,9 +1225,10 @@ } - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); -} - -/** Prints list of zones. */ + irq_spinlock_unlock(&zones.lock, true); +} + +/** Prints list of zones. + * + */ void zones_print_list(void) { @@ -1278,10 +1256,8 @@ size_t i; for (i = 0;; i++) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); if (i >= zones.count) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); break; } @@ -1293,6 +1269,5 @@ size_t busy_count = zones.info[i].busy_count; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); bool available = zone_flags_available(flags); @@ -1328,6 +1303,5 @@ void zone_print_one(size_t num) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = (size_t) -1; @@ -1341,6 +1315,5 @@ if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); printf("Zone not found.\n"); return; @@ -1353,6 +1326,5 @@ size_t busy_count = zones.info[i].busy_count; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); bool available = zone_flags_available(flags); Index: kernel/generic/src/mm/page.c =================================================================== --- kernel/generic/src/mm/page.c (revision c96452180f7af3adf60e8b6f88ac2ad6ec1683e6) +++ kernel/generic/src/mm/page.c (revision da1bafb8cf9a3b3be8ef21bc114daaa476a85190) @@ -33,5 +33,5 @@ /** * @file - * @brief Virtual Address Translation subsystem. + * @brief Virtual Address Translation subsystem. * * This file contains code for creating, destroying and searching @@ -39,4 +39,5 @@ * Functions here are mere wrappers that call the real implementation. * They however, define the single interface. + * */ @@ -55,4 +56,5 @@ * will do an implicit serialization by virtue of running the TLB shootdown * interrupt handler. + * */ @@ -83,18 +85,18 @@ * of page boundaries. * - * @param s Address of the structure. - * @param size Size of the structure. + * @param addr Address of the structure. + * @param size Size of the structure. + * */ -void map_structure(uintptr_t s, size_t size) +void map_structure(uintptr_t addr, size_t size) { - int i, cnt, length; - - length = size + (s - (s & ~(PAGE_SIZE - 1))); - cnt = length / PAGE_SIZE + (length % PAGE_SIZE > 0); - + size_t length = size + (addr - (addr & ~(PAGE_SIZE - 1))); + size_t cnt = length / PAGE_SIZE + (length % PAGE_SIZE > 0); + + size_t i; for (i = 0; i < cnt; i++) - page_mapping_insert(AS_KERNEL, s + i * PAGE_SIZE, - s + i * PAGE_SIZE, PAGE_NOT_CACHEABLE | PAGE_WRITE); - + page_mapping_insert(AS_KERNEL, addr + i * PAGE_SIZE, + addr + i * PAGE_SIZE, PAGE_NOT_CACHEABLE | PAGE_WRITE); + /* Repel prefetched accesses to the old mapping. */ memory_barrier(); @@ -108,11 +110,13 @@ * The page table must be locked and interrupts must be disabled. * - * @param as Address space to wich page belongs. - * @param page Virtual address of the page to be mapped. - * @param frame Physical address of memory frame to which the mapping is - * done. - * @param flags Flags to be used for mapping. + * @param as Address space to wich page belongs. + * @param page Virtual address of the page to be mapped. + * @param frame Physical address of memory frame to which the mapping is + * done. + * @param flags Flags to be used for mapping. + * */ -void page_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, int flags) +void page_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, + unsigned int flags) { ASSERT(page_mapping_operations); @@ -133,6 +137,7 @@ * The page table must be locked and interrupts must be disabled. * - * @param as Address space to wich page belongs. - * @param page Virtual address of the page to be demapped. + * @param as Address space to wich page belongs. + * @param page Virtual address of the page to be demapped. + * */ void page_mapping_remove(as_t *as, uintptr_t page) @@ -142,5 +147,5 @@ page_mapping_operations->mapping_remove(as, page); - + /* Repel prefetched accesses to the old mapping. */ memory_barrier(); @@ -153,9 +158,10 @@ * The page table must be locked and interrupts must be disabled. * - * @param as Address space to wich page belongs. - * @param page Virtual page. + * @param as Address space to wich page belongs. + * @param page Virtual page. * - * @return NULL if there is no such mapping; requested mapping - * otherwise. + * @return NULL if there is no such mapping; requested mapping + * otherwise. + * */ pte_t *page_mapping_find(as_t *as, uintptr_t page) @@ -163,5 +169,5 @@ ASSERT(page_mapping_operations); ASSERT(page_mapping_operations->mapping_find); - + return page_mapping_operations->mapping_find(as, page); } Index: kernel/generic/src/mm/slab.c =================================================================== --- kernel/generic/src/mm/slab.c (revision c96452180f7af3adf60e8b6f88ac2ad6ec1683e6) +++ kernel/generic/src/mm/slab.c (revision da1bafb8cf9a3b3be8ef21bc114daaa476a85190) @@ -33,5 +33,5 @@ /** * @file - * @brief Slab allocator. + * @brief Slab allocator. * * The slab allocator is closely modelled after OpenSolaris slab allocator. @@ -50,5 +50,5 @@ * * The slab allocator supports per-CPU caches ('magazines') to facilitate - * good SMP scaling. + * good SMP scaling. * * When a new object is being allocated, it is first checked, if it is @@ -65,9 +65,9 @@ * thrashing when somebody is allocating/deallocating 1 item at the magazine * size boundary. LIFO order is enforced, which should avoid fragmentation - * as much as possible. - * + * as much as possible. + * * Every cache contains list of full slabs and list of partially full slabs. * Empty slabs are immediately freed (thrashing will be avoided because - * of magazines). + * of magazines). * * The slab information structure is kept inside the data area, if possible. @@ -95,7 +95,8 @@ * * @todo - * it might be good to add granularity of locks even to slab level, + * It might be good to add granularity of locks even to slab level, * we could then try_spinlock over all partial slabs and thus improve - * scalability even on slab level + * scalability even on slab level. + * */ @@ -114,11 +115,13 @@ #include -SPINLOCK_INITIALIZE(slab_cache_lock); +IRQ_SPINLOCK_STATIC_INITIALIZE(slab_cache_lock); static LIST_INITIALIZE(slab_cache_list); /** Magazine cache */ static slab_cache_t mag_cache; + /** Cache for cache descriptors */ static slab_cache_t slab_cache_cache; + /** Cache for external slab descriptors * This time we want per-cpu cache, so do not make it static @@ -128,6 +131,8 @@ */ static slab_cache_t *slab_extern_cache; + /** Caches for malloc */ static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1]; + static const char *malloc_names[] = { "malloc-16", @@ -154,34 +159,36 @@ /** Slab descriptor */ typedef struct { - slab_cache_t *cache; /**< Pointer to parent cache. */ - link_t link; /**< List of full/partial slabs. */ - void *start; /**< Start address of first available item. */ - size_t available; /**< Count of available items in this slab. */ - size_t nextavail; /**< The index of next available item. */ + slab_cache_t *cache; /**< Pointer to parent cache. */ + link_t link; /**< List of full/partial slabs. */ + void *start; /**< Start address of first available item. */ + size_t available; /**< Count of available items in this slab. */ + size_t nextavail; /**< The index of next available item. */ } slab_t; #ifdef CONFIG_DEBUG -static int _slab_initialized = 0; +static unsigned int _slab_initialized = 0; #endif /**************************************/ /* Slab allocation functions */ - -/** - * Allocate frames for slab space and initialize - * - */ -static slab_t *slab_space_alloc(slab_cache_t *cache, int flags) -{ - void *data; +/**************************************/ + +/** Allocate frames for slab space and initialize + * + */ +static slab_t *slab_space_alloc(slab_cache_t *cache, unsigned int flags) +{ + + + size_t zone = 0; + + void *data = frame_alloc_generic(cache->order, FRAME_KA | flags, &zone); + if (!data) { + return NULL; + } + slab_t *slab; size_t fsize; - unsigned int i; - size_t zone = 0; - - data = frame_alloc_generic(cache->order, FRAME_KA | flags, &zone); - if (!data) { - return NULL; - } + if (!(cache->flags & SLAB_CACHE_SLINSIDE)) { slab = slab_alloc(slab_extern_cache, flags); @@ -196,61 +203,62 @@ /* Fill in slab structures */ - for (i = 0; i < ((unsigned int) 1 << cache->order); i++) + size_t i; + for (i = 0; i < ((size_t) 1 << cache->order); i++) frame_set_parent(ADDR2PFN(KA2PA(data)) + i, slab, zone); - + slab->start = data; slab->available = cache->objects; slab->nextavail = 0; slab->cache = cache; - + for (i = 0; i < cache->objects; i++) - *((int *) (slab->start + i*cache->size)) = i + 1; - + *((size_t *) (slab->start + i * cache->size)) = i + 1; + atomic_inc(&cache->allocated_slabs); return slab; } -/** - * Deallocate space associated with slab +/** Deallocate space associated with slab * * @return number of freed frames + * */ static size_t slab_space_free(slab_cache_t *cache, slab_t *slab) { frame_free(KA2PA(slab->start)); - if (! (cache->flags & SLAB_CACHE_SLINSIDE)) + if (!(cache->flags & SLAB_CACHE_SLINSIDE)) slab_free(slab_extern_cache, slab); - + atomic_dec(&cache->allocated_slabs); - return 1 << cache->order; + return (1 << cache->order); } /** Map object to slab structure */ -static slab_t * obj2slab(void *obj) +static slab_t *obj2slab(void *obj) { return (slab_t *) frame_get_parent(ADDR2PFN(KA2PA(obj)), 0); } -/**************************************/ +/******************/ /* Slab functions */ - - -/** - * Return object to slab and call a destructor +/******************/ + +/** Return object to slab and call a destructor * * @param slab If the caller knows directly slab of the object, otherwise NULL * * @return Number of freed pages + * */ static size_t slab_obj_destroy(slab_cache_t *cache, void *obj, slab_t *slab) { - int freed = 0; - if (!slab) slab = obj2slab(obj); - + ASSERT(slab->cache == cache); - + + size_t freed = 0; + if (cache->destructor) freed = cache->destructor(obj); @@ -258,9 +266,9 @@ spinlock_lock(&cache->slablock); ASSERT(slab->available < cache->objects); - - *((int *)obj) = slab->nextavail; + + *((size_t *) obj) = slab->nextavail; slab->nextavail = (obj - slab->start) / cache->size; slab->available++; - + /* Move it to correct list */ if (slab->available == cache->objects) { @@ -268,7 +276,6 @@ list_remove(&slab->link); spinlock_unlock(&cache->slablock); - + return freed + slab_space_free(cache, slab); - } else if (slab->available == 1) { /* It was in full, move to partial */ @@ -276,26 +283,28 @@ list_prepend(&slab->link, &cache->partial_slabs); } + spinlock_unlock(&cache->slablock); return freed; } -/** - * Take new object from slab or create new if needed +/** Take new object from slab or create new if needed * * @return Object address or null + * */ static void *slab_obj_create(slab_cache_t *cache, int flags) { + spinlock_lock(&cache->slablock); + slab_t *slab; - void *obj; - - spinlock_lock(&cache->slablock); - + if (list_empty(&cache->partial_slabs)) { - /* Allow recursion and reclaiming + /* + * Allow recursion and reclaiming * - this should work, as the slab control structures * are small and do not need to allocate with anything * other than frame_alloc when they are allocating, * that's why we should get recursion at most 1-level deep + * */ spinlock_unlock(&cache->slablock); @@ -303,4 +312,5 @@ if (!slab) return NULL; + spinlock_lock(&cache->slablock); } else { @@ -309,37 +319,39 @@ list_remove(&slab->link); } - obj = slab->start + slab->nextavail * cache->size; - slab->nextavail = *((int *)obj); + + void *obj = slab->start + slab->nextavail * cache->size; + slab->nextavail = *((size_t *) obj); slab->available--; - + if (!slab->available) list_prepend(&slab->link, &cache->full_slabs); else list_prepend(&slab->link, &cache->partial_slabs); - + spinlock_unlock(&cache->slablock); - - if (cache->constructor && cache->constructor(obj, flags)) { + + if ((cache->constructor) && (cache->constructor(obj, flags))) { /* Bad, bad, construction failed */ slab_obj_destroy(cache, obj, slab); return NULL; } + return obj; } -/**************************************/ +/****************************/ /* CPU-Cache slab functions */ - -/** - * Finds a full magazine in cache, takes it from list - * and returns it - * - * @param first If true, return first, else last mag - */ -static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, int first) +/****************************/ + +/** Find a full magazine in cache, take it from list and return it + * + * @param first If true, return first, else last mag. + * + */ +static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, bool first) { slab_magazine_t *mag = NULL; link_t *cur; - + spinlock_lock(&cache->maglock); if (!list_empty(&cache->magazines)) { @@ -348,17 +360,21 @@ else cur = cache->magazines.prev; + mag = list_get_instance(cur, slab_magazine_t, link); list_remove(&mag->link); atomic_dec(&cache->magazine_counter); } + spinlock_unlock(&cache->maglock); return mag; } -/** Prepend magazine to magazine list in cache */ +/** Prepend magazine to magazine list in cache + * + */ static void put_mag_to_cache(slab_cache_t *cache, slab_magazine_t *mag) { spinlock_lock(&cache->maglock); - + list_prepend(&mag->link, &cache->magazines); atomic_inc(&cache->magazine_counter); @@ -367,14 +383,14 @@ } -/** - * Free all objects in magazine and free memory associated with magazine +/** Free all objects in magazine and free memory associated with magazine * * @return Number of freed pages + * */ static size_t magazine_destroy(slab_cache_t *cache, slab_magazine_t *mag) { - unsigned int i; + size_t i; size_t frames = 0; - + for (i = 0; i < mag->busy; i++) { frames += slab_obj_destroy(cache, mag->objs[i], NULL); @@ -383,24 +399,23 @@ slab_free(&mag_cache, mag); - + return frames; } -/** - * Find full magazine, set it as current and return it +/** Find full magazine, set it as current and return it * * Assume cpu_magazine lock is held + * */ static slab_magazine_t *get_full_current_mag(slab_cache_t *cache) { - slab_magazine_t *cmag, *lastmag, *newmag; - - cmag = cache->mag_cache[CPU->id].current; - lastmag = cache->mag_cache[CPU->id].last; + slab_magazine_t *cmag = cache->mag_cache[CPU->id].current; + slab_magazine_t *lastmag = cache->mag_cache[CPU->id].last; + if (cmag) { /* First try local CPU magazines */ if (cmag->busy) return cmag; - - if (lastmag && lastmag->busy) { + + if ((lastmag) && (lastmag->busy)) { cache->mag_cache[CPU->id].current = lastmag; cache->mag_cache[CPU->id].last = cmag; @@ -408,39 +423,40 @@ } } + /* Local magazines are empty, import one from magazine list */ - newmag = get_mag_from_cache(cache, 1); + slab_magazine_t *newmag = get_mag_from_cache(cache, 1); if (!newmag) return NULL; - + if (lastmag) magazine_destroy(cache, lastmag); - + cache->mag_cache[CPU->id].last = cmag; cache->mag_cache[CPU->id].current = newmag; + return newmag; } -/** - * Try to find object in CPU-cache magazines +/** Try to find object in CPU-cache magazines * * @return Pointer to object or NULL if not available + * */ static void *magazine_obj_get(slab_cache_t *cache) { - slab_magazine_t *mag; - void *obj; - if (!CPU) return NULL; - + spinlock_lock(&cache->mag_cache[CPU->id].lock); - - mag = get_full_current_mag(cache); + + slab_magazine_t *mag = get_full_current_mag(cache); if (!mag) { spinlock_unlock(&cache->mag_cache[CPU->id].lock); return NULL; } - obj = mag->objs[--mag->busy]; + + void *obj = mag->objs[--mag->busy]; spinlock_unlock(&cache->mag_cache[CPU->id].lock); + atomic_dec(&cache->cached_objs); @@ -448,28 +464,25 @@ } -/** - * Assure that the current magazine is empty, return pointer to it, or NULL if - * no empty magazine is available and cannot be allocated +/** Assure that the current magazine is empty, return pointer to it, + * or NULL if no empty magazine is available and cannot be allocated * * Assume mag_cache[CPU->id].lock is held * - * We have 2 magazines bound to processor. - * First try the current. - * If full, try the last. - * If full, put to magazines list. - * allocate new, exchange last & current + * We have 2 magazines bound to processor. + * First try the current. + * If full, try the last. + * If full, put to magazines list. * */ static slab_magazine_t *make_empty_current_mag(slab_cache_t *cache) { - slab_magazine_t *cmag,*lastmag,*newmag; - - cmag = cache->mag_cache[CPU->id].current; - lastmag = cache->mag_cache[CPU->id].last; - + slab_magazine_t *cmag = cache->mag_cache[CPU->id].current; + slab_magazine_t *lastmag = cache->mag_cache[CPU->id].last; + if (cmag) { if (cmag->busy < cmag->size) return cmag; - if (lastmag && lastmag->busy < lastmag->size) { + + if ((lastmag) && (lastmag->busy < lastmag->size)) { cache->mag_cache[CPU->id].last = cmag; cache->mag_cache[CPU->id].current = lastmag; @@ -477,40 +490,45 @@ } } + /* current | last are full | nonexistent, allocate new */ - /* We do not want to sleep just because of caching */ - /* Especially we do not want reclaiming to start, as - * this would deadlock */ - newmag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM); + + /* + * We do not want to sleep just because of caching, + * especially we do not want reclaiming to start, as + * this would deadlock. + * + */ + slab_magazine_t *newmag = slab_alloc(&mag_cache, + FRAME_ATOMIC | FRAME_NO_RECLAIM); if (!newmag) return NULL; + newmag->size = SLAB_MAG_SIZE; newmag->busy = 0; - + /* Flush last to magazine list */ if (lastmag) put_mag_to_cache(cache, lastmag); - + /* Move current as last, save new as current */ - cache->mag_cache[CPU->id].last = cmag; - cache->mag_cache[CPU->id].current = newmag; - + cache->mag_cache[CPU->id].last = cmag; + cache->mag_cache[CPU->id].current = newmag; + return newmag; } -/** - * Put object into CPU-cache magazine - * - * @return 0 - success, -1 - could not get memory +/** Put object into CPU-cache magazine + * + * @return 0 on success, -1 on no memory + * */ static int magazine_obj_put(slab_cache_t *cache, void *obj) { - slab_magazine_t *mag; - if (!CPU) return -1; - + spinlock_lock(&cache->mag_cache[CPU->id].lock); - - mag = make_empty_current_mag(cache); + + slab_magazine_t *mag = make_empty_current_mag(cache); if (!mag) { spinlock_unlock(&cache->mag_cache[CPU->id].lock); @@ -519,94 +537,101 @@ mag->objs[mag->busy++] = obj; - + spinlock_unlock(&cache->mag_cache[CPU->id].lock); + atomic_inc(&cache->cached_objs); + return 0; } - -/**************************************/ +/************************/ /* Slab cache functions */ - -/** Return number of objects that fit in certain cache size */ -static unsigned int comp_objects(slab_cache_t *cache) +/************************/ + +/** Return number of objects that fit in certain cache size + * + */ +static size_t comp_objects(slab_cache_t *cache) { if (cache->flags & SLAB_CACHE_SLINSIDE) - return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / - cache->size; - else + return ((PAGE_SIZE << cache->order) + - sizeof(slab_t)) / cache->size; + else return (PAGE_SIZE << cache->order) / cache->size; } -/** Return wasted space in slab */ -static unsigned int badness(slab_cache_t *cache) -{ - unsigned int objects; - unsigned int ssize; - - objects = comp_objects(cache); - ssize = PAGE_SIZE << cache->order; +/** Return wasted space in slab + * + */ +static size_t badness(slab_cache_t *cache) +{ + size_t objects = comp_objects(cache); + size_t ssize = PAGE_SIZE << cache->order; + if (cache->flags & SLAB_CACHE_SLINSIDE) ssize -= sizeof(slab_t); + return ssize - objects * cache->size; } -/** - * Initialize mag_cache structure in slab cache +/** Initialize mag_cache structure in slab cache + * */ static bool make_magcache(slab_cache_t *cache) { - unsigned int i; - ASSERT(_slab_initialized >= 2); - + cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count, FRAME_ATOMIC); if (!cache->mag_cache) return false; - + + size_t i; for (i = 0; i < config.cpu_count; i++) { memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); spinlock_initialize(&cache->mag_cache[i].lock, - "slab_maglock_cpu"); - } + "slab.cache.mag_cache[].lock"); + } + return true; } -/** Initialize allocated memory as a slab cache */ +/** Initialize allocated memory as a slab cache + * + */ static void _slab_cache_create(slab_cache_t *cache, const char *name, - size_t size, size_t align, int (*constructor)(void *obj, int kmflag), - int (*destructor)(void *obj), int flags) -{ - int pages; - ipl_t ipl; - + size_t size, size_t align, int (*constructor)(void *obj, + unsigned int kmflag), size_t (*destructor)(void *obj), unsigned int flags) +{ memsetb(cache, sizeof(*cache), 0); cache->name = name; - + if (align < sizeof(unative_t)) align = sizeof(unative_t); + size = ALIGN_UP(size, align); - + cache->size = size; - cache->constructor = constructor; cache->destructor = destructor; cache->flags = flags; - + list_initialize(&cache->full_slabs); list_initialize(&cache->partial_slabs); list_initialize(&cache->magazines); - spinlock_initialize(&cache->slablock, "slab_lock"); - spinlock_initialize(&cache->maglock, "slab_maglock"); + + spinlock_initialize(&cache->slablock, "slab.cache.slablock"); + spinlock_initialize(&cache->maglock, "slab.cache.maglock"); + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) (void) make_magcache(cache); - + /* Compute slab sizes, object counts in slabs etc. */ if (cache->size < SLAB_INSIDE_SIZE) cache->flags |= SLAB_CACHE_SLINSIDE; - + /* Minimum slab order */ - pages = SIZE2FRAMES(cache->size); + size_t pages = SIZE2FRAMES(cache->size); + /* We need the 2^order >= pages */ if (pages == 1) @@ -614,59 +639,58 @@ else cache->order = fnzb(pages - 1) + 1; - - while (badness(cache) > SLAB_MAX_BADNESS(cache)) { + + while (badness(cache) > SLAB_MAX_BADNESS(cache)) cache->order += 1; - } + cache->objects = comp_objects(cache); + /* If info fits in, put it inside */ if (badness(cache) > sizeof(slab_t)) cache->flags |= SLAB_CACHE_SLINSIDE; - + /* Add cache to cache list */ - ipl = interrupts_disable(); - spinlock_lock(&slab_cache_lock); - + irq_spinlock_lock(&slab_cache_lock, true); list_append(&cache->link, &slab_cache_list); - - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); -} - -/** Create slab cache */ + irq_spinlock_unlock(&slab_cache_lock, true); +} + +/** Create slab cache + * + */ slab_cache_t *slab_cache_create(const char *name, size_t size, size_t align, - int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), - int flags) -{ - slab_cache_t *cache; - - cache = slab_alloc(&slab_cache_cache, 0); + int (*constructor)(void *obj, unsigned int kmflag), + size_t (*destructor)(void *obj), unsigned int flags) +{ + slab_cache_t *cache = slab_alloc(&slab_cache_cache, 0); _slab_cache_create(cache, name, size, align, constructor, destructor, flags); + return cache; } -/** - * Reclaim space occupied by objects that are already free +/** Reclaim space occupied by objects that are already free * * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing + * * @return Number of freed pages - */ -static size_t _slab_reclaim(slab_cache_t *cache, int flags) -{ - unsigned int i; + * + */ +static size_t _slab_reclaim(slab_cache_t *cache, unsigned int flags) +{ + if (cache->flags & SLAB_CACHE_NOMAGAZINE) + return 0; /* Nothing to do */ + + /* + * We count up to original magazine count to avoid + * endless loop + */ + atomic_count_t magcount = atomic_get(&cache->magazine_counter); + slab_magazine_t *mag; size_t frames = 0; - int magcount; - - if (cache->flags & SLAB_CACHE_NOMAGAZINE) - return 0; /* Nothing to do */ - - /* We count up to original magazine count to avoid - * endless loop - */ - magcount = atomic_get(&cache->magazine_counter); - while (magcount-- && (mag=get_mag_from_cache(cache, 0))) { - frames += magazine_destroy(cache,mag); - if (!(flags & SLAB_RECLAIM_ALL) && frames) + + while ((magcount--) && (mag = get_mag_from_cache(cache, 0))) { + frames += magazine_destroy(cache, mag); + if ((!(flags & SLAB_RECLAIM_ALL)) && (frames)) break; } @@ -675,7 +699,8 @@ /* Free cpu-bound magazines */ /* Destroy CPU magazines */ + size_t i; for (i = 0; i < config.cpu_count; i++) { spinlock_lock(&cache->mag_cache[i].lock); - + mag = cache->mag_cache[i].current; if (mag) @@ -687,85 +712,88 @@ frames += magazine_destroy(cache, mag); cache->mag_cache[i].last = NULL; - + spinlock_unlock(&cache->mag_cache[i].lock); } } - + return frames; } -/** Check that there are no slabs and remove cache from system */ +/** Check that there are no slabs and remove cache from system + * + */ void slab_cache_destroy(slab_cache_t *cache) { - ipl_t ipl; - - /* First remove cache from link, so that we don't need + /* + * First remove cache from link, so that we don't need * to disable interrupts later + * */ - - ipl = interrupts_disable(); - spinlock_lock(&slab_cache_lock); - + irq_spinlock_lock(&slab_cache_lock, true); list_remove(&cache->link); - - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); - - /* Do not lock anything, we assume the software is correct and - * does not touch the cache when it decides to destroy it */ + irq_spinlock_unlock(&slab_cache_lock, true); + + /* + * Do not lock anything, we assume the software is correct and + * does not touch the cache when it decides to destroy it + * + */ /* Destroy all magazines */ _slab_reclaim(cache, SLAB_RECLAIM_ALL); - + /* All slabs must be empty */ - if (!list_empty(&cache->full_slabs) || - !list_empty(&cache->partial_slabs)) + if ((!list_empty(&cache->full_slabs)) || + (!list_empty(&cache->partial_slabs))) panic("Destroying cache that is not empty."); - + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) free(cache->mag_cache); + slab_free(&slab_cache_cache, cache); } -/** Allocate new object from cache - if no flags given, always returns memory */ -void *slab_alloc(slab_cache_t *cache, int flags) -{ - ipl_t ipl; +/** Allocate new object from cache - if no flags given, always returns memory + * + */ +void *slab_alloc(slab_cache_t *cache, unsigned int flags) +{ + /* Disable interrupts to avoid deadlocks with interrupt handlers */ + ipl_t ipl = interrupts_disable(); + void *result = NULL; - /* Disable interrupts to avoid deadlocks with interrupt handlers */ - ipl = interrupts_disable(); - - if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) { + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) result = magazine_obj_get(cache); - } + if (!result) result = slab_obj_create(cache, flags); - + interrupts_restore(ipl); - + if (result) atomic_inc(&cache->allocated_objs); - + return result; } -/** Return object to cache, use slab if known */ +/** Return object to cache, use slab if known + * + */ static void _slab_free(slab_cache_t *cache, void *obj, slab_t *slab) { - ipl_t ipl; - - ipl = interrupts_disable(); - + ipl_t ipl = interrupts_disable(); + if ((cache->flags & SLAB_CACHE_NOMAGAZINE) || - magazine_obj_put(cache, obj)) { + (magazine_obj_put(cache, obj))) slab_obj_destroy(cache, obj, slab); - - } + interrupts_restore(ipl); atomic_dec(&cache->allocated_objs); } -/** Return slab object to cache */ +/** Return slab object to cache + * + */ void slab_free(slab_cache_t *cache, void *obj) { @@ -773,45 +801,39 @@ } -/* Go through all caches and reclaim what is possible */ -size_t slab_reclaim(int flags) -{ - slab_cache_t *cache; +/** Go through all caches and reclaim what is possible + * + * Interrupts must be disabled before calling this function, + * otherwise memory allocation from interrupts can deadlock. + * + */ +size_t slab_reclaim(unsigned int flags) +{ + irq_spinlock_lock(&slab_cache_lock, false); + + size_t frames = 0; link_t *cur; - size_t frames = 0; - - spinlock_lock(&slab_cache_lock); - - /* TODO: Add assert, that interrupts are disabled, otherwise - * memory allocation from interrupts can deadlock. - */ - for (cur = slab_cache_list.next; cur != &slab_cache_list; cur = cur->next) { - cache = list_get_instance(cur, slab_cache_t, link); + slab_cache_t *cache = list_get_instance(cur, slab_cache_t, link); frames += _slab_reclaim(cache, flags); } - - spinlock_unlock(&slab_cache_lock); - + + irq_spinlock_unlock(&slab_cache_lock, false); + return frames; } - -/* Print list of slabs */ +/* Print list of slabs + * + */ void slab_print_list(void) { - int skip = 0; - - printf("slab name size pages obj/pg slabs cached allocated" + printf("slab name size pages obj/pg slabs cached allocated" " ctl\n"); - printf("---------------- -------- ------ ------ ------ ------ ---------" + printf("---------------- -------- ------ -------- ------ ------ ---------" " ---\n"); - + + size_t skip = 0; while (true) { - slab_cache_t *cache; - link_t *cur; - ipl_t ipl; - int i; - /* * We must not hold the slab_cache_lock spinlock when printing @@ -836,36 +858,34 @@ * statistics. */ - - ipl = interrupts_disable(); - spinlock_lock(&slab_cache_lock); - + + irq_spinlock_lock(&slab_cache_lock, true); + + link_t *cur; + size_t i; for (i = 0, cur = slab_cache_list.next; - i < skip && cur != &slab_cache_list; - i++, cur = cur->next) - ; - + (i < skip) && (cur != &slab_cache_list); + i++, cur = cur->next); + if (cur == &slab_cache_list) { - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&slab_cache_lock, true); break; } - + skip++; - - cache = list_get_instance(cur, slab_cache_t, link); - + + slab_cache_t *cache = list_get_instance(cur, slab_cache_t, link); + const char *name = cache->name; uint8_t order = cache->order; size_t size = cache->size; - unsigned int objects = cache->objects; + size_t objects = cache->objects; long allocated_slabs = atomic_get(&cache->allocated_slabs); long cached_objs = atomic_get(&cache->cached_objs); long allocated_objs = atomic_get(&cache->allocated_objs); - int flags = cache->flags; - - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); - - printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", + unsigned int flags = cache->flags; + + irq_spinlock_unlock(&slab_cache_lock, true); + + printf("%-16s %8" PRIs " %6u %8" PRIs " %6ld %6ld %9ld %-3s\n", name, size, (1 << order), objects, allocated_slabs, cached_objs, allocated_objs, @@ -876,6 +896,4 @@ void slab_cache_init(void) { - int i, size; - /* Initialize magazine cache */ _slab_cache_create(&mag_cache, "slab_magazine", @@ -883,13 +901,18 @@ sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); + /* Initialize slab_cache cache */ _slab_cache_create(&slab_cache_cache, "slab_cache", sizeof(slab_cache_cache), sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); + /* Initialize external slab cache */ slab_extern_cache = slab_cache_create("slab_extern", sizeof(slab_t), 0, NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); - + /* Initialize structures for malloc */ + size_t i; + size_t size; + for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); @@ -898,4 +921,5 @@ NULL, NULL, SLAB_CACHE_MAGDEFERRED); } + #ifdef CONFIG_DEBUG _slab_initialized = 1; @@ -906,35 +930,32 @@ * * Kernel calls this function, when it knows the real number of - * processors. - * Allocate slab for cpucache and enable it on all existing - * slabs that are SLAB_CACHE_MAGDEFERRED + * processors. Allocate slab for cpucache and enable it on all + * existing slabs that are SLAB_CACHE_MAGDEFERRED + * */ void slab_enable_cpucache(void) { - link_t *cur; - slab_cache_t *s; - #ifdef CONFIG_DEBUG _slab_initialized = 2; #endif - - spinlock_lock(&slab_cache_lock); - + + irq_spinlock_lock(&slab_cache_lock, false); + + link_t *cur; for (cur = slab_cache_list.next; cur != &slab_cache_list; - cur = cur->next){ - s = list_get_instance(cur, slab_cache_t, link); - if ((s->flags & SLAB_CACHE_MAGDEFERRED) != + cur = cur->next) { + slab_cache_t *slab = list_get_instance(cur, slab_cache_t, link); + if ((slab->flags & SLAB_CACHE_MAGDEFERRED) != SLAB_CACHE_MAGDEFERRED) continue; - (void) make_magcache(s); - s->flags &= ~SLAB_CACHE_MAGDEFERRED; - } - - spinlock_unlock(&slab_cache_lock); -} - -/**************************************/ -/* kalloc/kfree functions */ -void *malloc(unsigned int size, int flags) + + (void) make_magcache(slab); + slab->flags &= ~SLAB_CACHE_MAGDEFERRED; + } + + irq_spinlock_unlock(&slab_cache_lock, false); +} + +void *malloc(size_t size, unsigned int flags) { ASSERT(_slab_initialized); @@ -943,11 +964,11 @@ if (size < (1 << SLAB_MIN_MALLOC_W)) size = (1 << SLAB_MIN_MALLOC_W); - - int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; - + + uint8_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; + return slab_alloc(malloc_caches[idx], flags); } -void *realloc(void *ptr, unsigned int size, int flags) +void *realloc(void *ptr, size_t size, unsigned int flags) { ASSERT(_slab_initialized); @@ -959,5 +980,5 @@ if (size < (1 << SLAB_MIN_MALLOC_W)) size = (1 << SLAB_MIN_MALLOC_W); - int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; + uint8_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; new_ptr = slab_alloc(malloc_caches[idx], flags); @@ -980,5 +1001,5 @@ if (!ptr) return; - + slab_t *slab = obj2slab(ptr); _slab_free(slab->cache, ptr, slab); Index: kernel/generic/src/mm/tlb.c =================================================================== --- kernel/generic/src/mm/tlb.c (revision c96452180f7af3adf60e8b6f88ac2ad6ec1683e6) +++ kernel/generic/src/mm/tlb.c (revision da1bafb8cf9a3b3be8ef21bc114daaa476a85190) @@ -33,5 +33,5 @@ /** * @file - * @brief Generic TLB shootdown algorithm. + * @brief Generic TLB shootdown algorithm. * * The algorithm implemented here is based on the CMU TLB shootdown @@ -53,11 +53,4 @@ #include -/** - * This lock is used for synchronisation between sender and - * recipients of TLB shootdown message. It must be acquired - * before CPU structure lock. - */ -SPINLOCK_INITIALIZE(tlblock); - void tlb_init(void) { @@ -66,4 +59,12 @@ #ifdef CONFIG_SMP + +/** + * This lock is used for synchronisation between sender and + * recipients of TLB shootdown message. It must be acquired + * before CPU structure lock. + * + */ +IRQ_SPINLOCK_STATIC_INITIALIZE(tlblock); /** Send TLB shootdown message. @@ -78,13 +79,13 @@ * @param page Virtual page address, if required by type. * @param count Number of pages, if required by type. + * */ void tlb_shootdown_start(tlb_invalidate_type_t type, asid_t asid, uintptr_t page, size_t count) { - unsigned int i; - - CPU->tlb_active = 0; - spinlock_lock(&tlblock); + CPU->tlb_active = false; + irq_spinlock_lock(&tlblock, false); + size_t i; for (i = 0; i < config.cpu_count; i++) { cpu_t *cpu; @@ -92,7 +93,7 @@ if (i == CPU->id) continue; - + cpu = &cpus[i]; - spinlock_lock(&cpu->lock); + irq_spinlock_lock(&cpu->lock, false); if (cpu->tlb_messages_count == TLB_MESSAGE_QUEUE_LEN) { /* @@ -115,10 +116,10 @@ cpu->tlb_messages[idx].count = count; } - spinlock_unlock(&cpu->lock); + irq_spinlock_unlock(&cpu->lock, false); } tlb_shootdown_ipi_send(); - -busy_wait: + +busy_wait: for (i = 0; i < config.cpu_count; i++) if (cpus[i].tlb_active) @@ -126,9 +127,11 @@ } -/** Finish TLB shootdown sequence. */ +/** Finish TLB shootdown sequence. + * + */ void tlb_shootdown_finalize(void) { - spinlock_unlock(&tlblock); - CPU->tlb_active = 1; + irq_spinlock_unlock(&tlblock, false); + CPU->tlb_active = true; } @@ -138,28 +141,25 @@ } -/** Receive TLB shootdown message. */ +/** Receive TLB shootdown message. + * + */ void tlb_shootdown_ipi_recv(void) { - tlb_invalidate_type_t type; - asid_t asid; - uintptr_t page; - size_t count; - unsigned int i; - ASSERT(CPU); - CPU->tlb_active = 0; - spinlock_lock(&tlblock); - spinlock_unlock(&tlblock); + CPU->tlb_active = false; + irq_spinlock_lock(&tlblock, false); + irq_spinlock_unlock(&tlblock, false); - spinlock_lock(&CPU->lock); + irq_spinlock_lock(&CPU->lock, false); ASSERT(CPU->tlb_messages_count <= TLB_MESSAGE_QUEUE_LEN); - + + size_t i; for (i = 0; i < CPU->tlb_messages_count; CPU->tlb_messages_count--) { - type = CPU->tlb_messages[i].type; - asid = CPU->tlb_messages[i].asid; - page = CPU->tlb_messages[i].page; - count = CPU->tlb_messages[i].count; - + tlb_invalidate_type_t type = CPU->tlb_messages[i].type; + asid_t asid = CPU->tlb_messages[i].asid; + uintptr_t page = CPU->tlb_messages[i].page; + size_t count = CPU->tlb_messages[i].count; + switch (type) { case TLB_INVL_ALL: @@ -170,5 +170,5 @@ break; case TLB_INVL_PAGES: - ASSERT(count); + ASSERT(count); tlb_invalidate_pages(asid, page, count); break; @@ -177,10 +177,11 @@ break; } + if (type == TLB_INVL_ALL) break; } - spinlock_unlock(&CPU->lock); - CPU->tlb_active = 1; + irq_spinlock_unlock(&CPU->lock, false); + CPU->tlb_active = true; }