Index: kernel/generic/include/adt/cht.h =================================================================== --- kernel/generic/include/adt/cht.h (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) +++ kernel/generic/include/adt/cht.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -36,4 +36,5 @@ #define KERN_CONC_HASH_TABLE_H_ +#include #include #include Index: kernel/generic/include/lib/refcount.h =================================================================== --- kernel/generic/include/lib/refcount.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) +++ kernel/generic/include/lib/refcount.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2018 CZ.NIC, z.s.p.o. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Authors: + * Jiří Zárevúcky (jzr) + */ + +/* + * Using atomics for reference counting efficiently is a little tricky, + * so we define a unified API for this. + */ + +#ifndef LIBC_REFCOUNT_H_ +#define LIBC_REFCOUNT_H_ + +#include +#include +#include + +/* Wrapped in a structure to prevent direct manipulation. */ +typedef struct atomic_refcount { + volatile atomic_int __cnt; +} atomic_refcount_t; + +static inline void refcount_init(atomic_refcount_t *rc) +{ + atomic_store_explicit(&rc->__cnt, 0, memory_order_relaxed); +} + +/** + * Increment a reference count. + * + * Calling this without already owning a reference is undefined behavior. + * E.g. acquiring a reference through a shared mutable pointer requires that + * the caller first locks the pointer itself (thereby acquiring the reference + * inherent to the shared variable), and only then may call refcount_up(). + */ +static inline void refcount_up(atomic_refcount_t *rc) +{ + // XXX: We can use relaxed operation because acquiring a reference + // implies no ordering relationships. A reference-counted object + // still needs to be synchronized independently of the refcount. + + int old = atomic_fetch_add_explicit(&rc->__cnt, 1, + memory_order_relaxed); + + /* old < 0 indicates that the function is used incorrectly. */ + assert(old >= 0); +} + +static inline bool refcount_unique(atomic_refcount_t *rc) +{ + int val = atomic_load_explicit(&rc->__cnt, memory_order_acquire); + if (val < 0) { + assert(val == -1); + } + + return val <= 0; +} + +/** + * Decrement a reference count. Caller must own the reference. + * + * If the function returns `false`, the caller no longer owns the reference and + * must not access the reference counted object. + * + * If the function returns `true`, the caller is now the sole owner of the + * reference counted object, and must deallocate it. + */ +static inline bool refcount_down(atomic_refcount_t *rc) +{ + // XXX: The decrementers don't need to synchronize with each other, + // but they do need to synchronize with the one doing deallocation. + int old = atomic_fetch_sub_explicit(&rc->__cnt, 1, + memory_order_release); + + assert(old >= 0); + + if (old == 0) { + // XXX: We are holding the last reference, so we must now + // synchronize with all the other decrementers. + + int val = atomic_load_explicit(&rc->__cnt, + memory_order_acquire); + assert(val == -1); + + /* + * The compiler probably wouldn't optimize the memory barrier + * away, but better safe than sorry. + */ + return val < 0; + } + + return false; +} + +#endif + Index: kernel/generic/include/mm/as.h =================================================================== --- kernel/generic/include/mm/as.h (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) +++ kernel/generic/include/mm/as.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -48,4 +48,5 @@ #include #include +#include #define AS THE->as @@ -111,5 +112,5 @@ /** Number of references (i.e. tasks that reference this as). */ - atomic_t refcount; + atomic_refcount_t refcount; mutex_t lock; Index: kernel/generic/include/synch/spinlock.h =================================================================== --- kernel/generic/include/synch/spinlock.h (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) +++ kernel/generic/include/synch/spinlock.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -36,9 +36,8 @@ #define KERN_SPINLOCK_H_ +#include +#include #include -#include -#include #include -#include #include @@ -46,5 +45,5 @@ typedef struct spinlock { - atomic_t val; + atomic_flag flag; #ifdef CONFIG_DEBUG_SPINLOCK @@ -70,5 +69,5 @@ spinlock_t lock_name = { \ .name = desc_name, \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ } @@ -76,5 +75,5 @@ static spinlock_t lock_name = { \ .name = desc_name, \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ } @@ -89,10 +88,10 @@ #define SPINLOCK_INITIALIZE_NAME(lock_name, desc_name) \ spinlock_t lock_name = { \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ } #define SPINLOCK_STATIC_INITIALIZE_NAME(lock_name, desc_name) \ static spinlock_t lock_name = { \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ } @@ -126,10 +125,5 @@ NO_TRACE static inline void spinlock_unlock_nondebug(spinlock_t *lock) { - /* - * Prevent critical section code from bleeding out this way down. - */ - CS_LEAVE_BARRIER(); - - atomic_set(&lock->val, 0); + atomic_flag_clear_explicit(&lock->flag, memory_order_release); preemption_enable(); } @@ -215,5 +209,5 @@ .lock = { \ .name = desc_name, \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ }, \ .guard = false, \ @@ -225,5 +219,5 @@ .lock = { \ .name = desc_name, \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ }, \ .guard = false, \ @@ -236,5 +230,5 @@ irq_spinlock_t lock_name = { \ .lock = { \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ }, \ .guard = false, \ @@ -245,5 +239,5 @@ static irq_spinlock_t lock_name = { \ .lock = { \ - .val = { 0 } \ + .flag = ATOMIC_FLAG_INIT \ }, \ .guard = false, \ Index: kernel/generic/src/mm/as.c =================================================================== --- kernel/generic/src/mm/as.c (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) +++ kernel/generic/src/mm/as.c (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -163,5 +163,5 @@ as->asid = ASID_INVALID; - atomic_set(&as->refcount, 0); + refcount_init(&as->refcount); as->cpu_refcount = 0; @@ -190,5 +190,5 @@ assert(as != AS); - assert(atomic_get(&as->refcount) == 0); + assert(refcount_unique(&as->refcount)); /* @@ -267,5 +267,5 @@ NO_TRACE void as_hold(as_t *as) { - atomic_inc(&as->refcount); + refcount_up(&as->refcount); } @@ -275,10 +275,10 @@ * destroys the address space. * - * @param asAddress space to be released. + * @param as Address space to be released. * */ NO_TRACE void as_release(as_t *as) { - if (atomic_predec(&as->refcount) == 0) + if (refcount_down(&as->refcount)) as_destroy(as); } Index: kernel/generic/src/synch/spinlock.c =================================================================== --- kernel/generic/src/synch/spinlock.c (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) +++ kernel/generic/src/synch/spinlock.c (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -56,5 +56,5 @@ void spinlock_initialize(spinlock_t *lock, const char *name) { - atomic_set(&lock->val, 0); + atomic_flag_clear_explicit(&lock->flag, memory_order_relaxed); #ifdef CONFIG_DEBUG_SPINLOCK lock->name = name; @@ -79,5 +79,5 @@ preemption_disable(); - while (test_and_set(&lock->val)) { + while (atomic_flag_test_and_set_explicit(&lock->flag, memory_order_acquire)) { /* * We need to be careful about particular locks @@ -115,9 +115,4 @@ if (deadlock_reported) printf("cpu%u: not deadlocked\n", CPU->id); - - /* - * Prevent critical section code from bleeding out this way up. - */ - CS_ENTER_BARRIER(); } @@ -132,10 +127,5 @@ ASSERT_SPINLOCK(spinlock_locked(lock), lock); - /* - * Prevent critical section code from bleeding out this way down. - */ - CS_LEAVE_BARRIER(); - - atomic_set(&lock->val, 0); + atomic_flag_clear_explicit(&lock->flag, memory_order_release); preemption_enable(); } @@ -156,10 +146,5 @@ { preemption_disable(); - bool ret = !test_and_set(&lock->val); - - /* - * Prevent critical section code from bleeding out this way up. - */ - CS_ENTER_BARRIER(); + bool ret = !atomic_flag_test_and_set_explicit(&lock->flag, memory_order_acquire); if (!ret) @@ -176,5 +161,13 @@ bool spinlock_locked(spinlock_t *lock) { - return atomic_get(&lock->val) != 0; + // XXX: Atomic flag doesn't support simple atomic read (by design), + // so instead we test_and_set and then clear if necessary. + // This function is only used inside assert, so we don't need + // any preemption_disable/enable here. + + bool ret = atomic_flag_test_and_set_explicit(&lock->flag, memory_order_relaxed); + if (!ret) + atomic_flag_clear_explicit(&lock->flag, memory_order_relaxed); + return ret; }