Index: kernel/arch/abs32le/include/arch/atomic.h =================================================================== --- kernel/arch/abs32le/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,135 +1,0 @@ -/* - * Copyright (c) 2010 Martin Decky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup abs32le - * @{ - */ -/** @file - */ - -#ifndef KERN_abs32le_ATOMIC_H_ -#define KERN_abs32le_ATOMIC_H_ - -#include -#include -#include -#include -#include - -NO_TRACE ATOMIC static inline void atomic_inc(atomic_t *val) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) - REQUIRES(val->count < ATOMIC_COUNT_MAX) -{ - /* - * On real hardware the increment has to be done - * as an atomic action. - */ - - val->count++; -} - -NO_TRACE ATOMIC static inline void atomic_dec(atomic_t *val) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) - REQUIRES(val->count > ATOMIC_COUNT_MIN) -{ - /* - * On real hardware the decrement has to be done - * as an atomic action. - */ - - val->count--; -} - -NO_TRACE ATOMIC static inline atomic_count_t atomic_postinc(atomic_t *val) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) - REQUIRES(val->count < ATOMIC_COUNT_MAX) -{ - /* - * On real hardware both the storing of the previous - * value and the increment have to be done as a single - * atomic action. - */ - - atomic_count_t prev = val->count; - - val->count++; - return prev; -} - -NO_TRACE ATOMIC static inline atomic_count_t atomic_postdec(atomic_t *val) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) - REQUIRES(val->count > ATOMIC_COUNT_MIN) -{ - /* - * On real hardware both the storing of the previous - * value and the decrement have to be done as a single - * atomic action. - */ - - atomic_count_t prev = val->count; - - val->count--; - return prev; -} - -#define atomic_preinc(val) (atomic_postinc(val) + 1) -#define atomic_predec(val) (atomic_postdec(val) - 1) - -NO_TRACE ATOMIC static inline atomic_count_t test_and_set(atomic_t *val) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) -{ - /* - * On real hardware the retrieving of the original - * value and storing 1 have to be done as a single - * atomic action. - */ - - atomic_count_t prev = val->count; - val->count = 1; - return prev; -} - -NO_TRACE static inline void atomic_lock_arch(atomic_t *val) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) -{ - do { - while (val->count) - ; - } while (test_and_set(val)); -} - -#endif - -/** @} - */ Index: kernel/arch/amd64/include/arch/atomic.h =================================================================== --- kernel/arch/amd64/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,263 +1,0 @@ -/* - * Copyright (c) 2001-2004 Jakub Jermar - * Copyright (c) 2012 Adam Hraska - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup amd64 - * @{ - */ -/** @file - */ - -#ifndef KERN_amd64_ATOMIC_H_ -#define KERN_amd64_ATOMIC_H_ - -#include -#include -#include - -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ -#ifdef CONFIG_SMP - asm volatile ( - "lock incq %[count]\n" - : [count] "+m" (val->count) - ); -#else - asm volatile ( - "incq %[count]\n" - : [count] "+m" (val->count) - ); -#endif /* CONFIG_SMP */ -} - -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ -#ifdef CONFIG_SMP - asm volatile ( - "lock decq %[count]\n" - : [count] "+m" (val->count) - ); -#else - asm volatile ( - "decq %[count]\n" - : [count] "+m" (val->count) - ); -#endif /* CONFIG_SMP */ -} - -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - atomic_count_t r = 1; - - asm volatile ( - "lock xaddq %[r], %[count]\n" - : [count] "+m" (val->count), - [r] "+r" (r) - ); - - return r; -} - -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - atomic_count_t r = -1; - - asm volatile ( - "lock xaddq %[r], %[count]\n" - : [count] "+m" (val->count), - [r] "+r" (r) - ); - - return r; -} - -#define atomic_preinc(val) (atomic_postinc(val) + 1) -#define atomic_predec(val) (atomic_postdec(val) - 1) - -NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) -{ - atomic_count_t v = 1; - - asm volatile ( - "xchgq %[v], %[count]\n" - : [v] "+r" (v), - [count] "+m" (val->count) - ); - - return v; -} - -/** amd64 specific fast spinlock */ -NO_TRACE static inline void atomic_lock_arch(atomic_t *val) -{ - atomic_count_t tmp; - - preemption_disable(); - asm volatile ( - "0:\n" - " pause\n" - " mov %[count], %[tmp]\n" - " testq %[tmp], %[tmp]\n" - " jnz 0b\n" /* lightweight looping on locked spinlock */ - - " incq %[tmp]\n" /* now use the atomic operation */ - " xchgq %[count], %[tmp]\n" - " testq %[tmp], %[tmp]\n" - " jnz 0b\n" - : [count] "+m" (val->count), - [tmp] "=&r" (tmp) - ); - - /* - * Prevent critical section code from bleeding out this way up. - */ - CS_ENTER_BARRIER(); -} - - -#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \ -({ \ - switch (sizeof(typeof(*(pptr)))) { \ - case 1: \ - asm volatile ( \ - prefix " cmpxchgb %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - case 2: \ - asm volatile ( \ - prefix " cmpxchgw %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - case 4: \ - asm volatile ( \ - prefix " cmpxchgl %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - case 8: \ - asm volatile ( \ - prefix " cmpxchgq %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - } \ -}) - - -#ifndef local_atomic_cas - -#define local_atomic_cas(pptr, exp_val, new_val) \ -({ \ - /* Use proper types and avoid name clashes */ \ - typeof(*(pptr)) _old_val_cas; \ - typeof(*(pptr)) _exp_val_cas = exp_val; \ - typeof(*(pptr)) _new_val_cas = new_val; \ - _atomic_cas_impl(pptr, _exp_val_cas, _new_val_cas, _old_val_cas, ""); \ - \ - _old_val_cas; \ -}) - -#else -/* Check if arch/atomic.h does not accidentally include /atomic.h .*/ -#error Architecture specific cpu local atomics already defined! Check your includes. -#endif - - -#ifndef local_atomic_exchange -/* - * Issuing a xchg instruction always implies lock prefix semantics. - * Therefore, it is cheaper to use a cmpxchg without a lock prefix - * in a loop. - */ -#define local_atomic_exchange(pptr, new_val) \ -({ \ - /* Use proper types and avoid name clashes */ \ - typeof(*(pptr)) _exp_val_x; \ - typeof(*(pptr)) _old_val_x; \ - typeof(*(pptr)) _new_val_x = new_val; \ - \ - do { \ - _exp_val_x = *pptr; \ - _old_val_x = local_atomic_cas(pptr, _exp_val_x, _new_val_x); \ - } while (_old_val_x != _exp_val_x); \ - \ - _old_val_x; \ -}) - -#else -/* Check if arch/atomic.h does not accidentally include /atomic.h .*/ -#error Architecture specific cpu local atomics already defined! Check your includes. -#endif - - -#endif - -/** @} - */ Index: kernel/arch/arm32/include/arch/atomic.h =================================================================== --- kernel/arch/arm32/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,132 +1,0 @@ -/* - * Copyright (c) 2007 Michal Kebrt - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup arm32 - * @{ - */ -/** @file - * @brief Atomic operations. - */ - -#ifndef KERN_arm32_ATOMIC_H_ -#define KERN_arm32_ATOMIC_H_ - -#include -#include - -/** Atomic addition. - * - * @param val Where to add. - * @param i Value to be added. - * - * @return Value after addition. - * - */ -NO_TRACE static inline atomic_count_t atomic_add(atomic_t *val, - atomic_count_t i) -{ - /* - * This implementation is for UP pre-ARMv6 systems where we do not have - * the LDREX and STREX instructions. - */ - ipl_t ipl = interrupts_disable(); - val->count += i; - atomic_count_t ret = val->count; - interrupts_restore(ipl); - - return ret; -} - -/** Atomic increment. - * - * @param val Variable to be incremented. - * - */ -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ - atomic_add(val, 1); -} - -/** Atomic decrement. - * - * @param val Variable to be decremented. - * - */ -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ - atomic_add(val, -1); -} - -/** Atomic pre-increment. - * - * @param val Variable to be incremented. - * @return Value after incrementation. - * - */ -NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) -{ - return atomic_add(val, 1); -} - -/** Atomic pre-decrement. - * - * @param val Variable to be decremented. - * @return Value after decrementation. - * - */ -NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) -{ - return atomic_add(val, -1); -} - -/** Atomic post-increment. - * - * @param val Variable to be incremented. - * @return Value before incrementation. - * - */ -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - return atomic_add(val, 1) - 1; -} - -/** Atomic post-decrement. - * - * @param val Variable to be decremented. - * @return Value before decrementation. - * - */ -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - return atomic_add(val, -1) + 1; -} - -#endif - -/** @} - */ Index: kernel/arch/arm32/src/atomic.c =================================================================== --- kernel/arch/arm32/src/atomic.c (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ kernel/arch/arm32/src/atomic.c (revision 4621d2311994bf63dea425ed923239d4ca1babc9) @@ -36,5 +36,27 @@ #include #include +#include +unsigned __atomic_fetch_add_4(volatile unsigned *mem, unsigned val, int model) +{ + /* + * This implementation is for UP pre-ARMv6 systems where we do not have + * the LDREX and STREX instructions. + */ + ipl_t ipl = interrupts_disable(); + unsigned ret = *mem; + *mem += val; + interrupts_restore(ipl); + return ret; +} + +unsigned __atomic_fetch_sub_4(volatile unsigned *mem, unsigned val, int model) +{ + ipl_t ipl = interrupts_disable(); + unsigned ret = *mem; + *mem -= val; + interrupts_restore(ipl); + return ret; +} IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(cas_lock, "arm-cas-lock"); Index: kernel/arch/ia32/include/arch/atomic.h =================================================================== --- kernel/arch/ia32/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,251 +1,0 @@ -/* - * Copyright (c) 2001-2004 Jakub Jermar - * Copyright (c) 2012 Adam Hraska - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ia32 - * @{ - */ -/** @file - */ - -#ifndef KERN_ia32_ATOMIC_H_ -#define KERN_ia32_ATOMIC_H_ - -#include -#include -#include -#include - -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ -#ifdef CONFIG_SMP - asm volatile ( - "lock incl %[count]\n" - : [count] "+m" (val->count) - ); -#else - asm volatile ( - "incl %[count]\n" - : [count] "+m" (val->count) - ); -#endif /* CONFIG_SMP */ -} - -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ -#ifdef CONFIG_SMP - asm volatile ( - "lock decl %[count]\n" - : [count] "+m" (val->count) - ); -#else - asm volatile ( - "decl %[count]\n" - : [count] "+m" (val->count) - ); -#endif /* CONFIG_SMP */ -} - -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - atomic_count_t r = 1; - - asm volatile ( - "lock xaddl %[r], %[count]\n" - : [count] "+m" (val->count), - [r] "+r" (r) - ); - - return r; -} - -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - atomic_count_t r = -1; - - asm volatile ( - "lock xaddl %[r], %[count]\n" - : [count] "+m" (val->count), - [r] "+r" (r) - ); - - return r; -} - -#define atomic_preinc(val) (atomic_postinc(val) + 1) -#define atomic_predec(val) (atomic_postdec(val) - 1) - -NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) -{ - atomic_count_t v = 1; - - asm volatile ( - "xchgl %[v], %[count]\n" - : [v] "+r" (v), - [count] "+m" (val->count) - ); - - return v; -} - - -/** ia32 specific fast spinlock */ -NO_TRACE static inline void atomic_lock_arch(atomic_t *val) -{ - atomic_count_t tmp; - - preemption_disable(); - asm volatile ( - "0:\n" -#ifndef PROCESSOR_i486 - "pause\n" /* Pentium 4's HT love this instruction */ -#endif - "mov %[count], %[tmp]\n" - "testl %[tmp], %[tmp]\n" - "jnz 0b\n" /* lightweight looping on locked spinlock */ - - "incl %[tmp]\n" /* now use the atomic operation */ - "xchgl %[count], %[tmp]\n" - "testl %[tmp], %[tmp]\n" - "jnz 0b\n" - : [count] "+m" (val->count), - [tmp] "=&r" (tmp) - ); - - /* - * Prevent critical section code from bleeding out this way up. - */ - CS_ENTER_BARRIER(); -} - - -#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \ -({ \ - switch (sizeof(typeof(*(pptr)))) { \ - case 1: \ - asm volatile ( \ - prefix " cmpxchgb %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - case 2: \ - asm volatile ( \ - prefix " cmpxchgw %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - case 4: \ - asm volatile ( \ - prefix " cmpxchgl %[newval], %[ptr]\n" \ - : /* Output operands. */ \ - /* Old/current value is returned in eax. */ \ - [oldval] "=a" (old_val), \ - /* (*ptr) will be read and written to, hence "+" */ \ - [ptr] "+m" (*pptr) \ - : /* Input operands. */ \ - /* Expected value must be in eax. */ \ - [expval] "a" (exp_val), \ - /* The new value may be in any register. */ \ - [newval] "r" (new_val) \ - : "memory" \ - ); \ - break; \ - } \ -}) - - -#ifndef local_atomic_cas - -#define local_atomic_cas(pptr, exp_val, new_val) \ -({ \ - /* Use proper types and avoid name clashes */ \ - typeof(*(pptr)) _old_val_cas; \ - typeof(*(pptr)) _exp_val_cas = exp_val; \ - typeof(*(pptr)) _new_val_cas = new_val; \ - _atomic_cas_impl(pptr, _exp_val_cas, _new_val_cas, _old_val_cas, ""); \ - \ - _old_val_cas; \ -}) - -#else -/* Check if arch/atomic.h does not accidentally include /atomic.h .*/ -#error Architecture specific cpu local atomics already defined! Check your includes. -#endif - - -#ifndef local_atomic_exchange -/* - * Issuing a xchg instruction always implies lock prefix semantics. - * Therefore, it is cheaper to use a cmpxchg without a lock prefix - * in a loop. - */ -#define local_atomic_exchange(pptr, new_val) \ -({ \ - /* Use proper types and avoid name clashes */ \ - typeof(*(pptr)) _exp_val_x; \ - typeof(*(pptr)) _old_val_x; \ - typeof(*(pptr)) _new_val_x = new_val; \ - \ - do { \ - _exp_val_x = *pptr; \ - _old_val_x = local_atomic_cas(pptr, _exp_val_x, _new_val_x); \ - } while (_old_val_x != _exp_val_x); \ - \ - _old_val_x; \ -}) - -#else -/* Check if arch/atomic.h does not accidentally include /atomic.h .*/ -#error Architecture specific cpu local atomics already defined! Check your includes. -#endif - - -#endif - -/** @} - */ Index: kernel/arch/ia64/include/arch/atomic.h =================================================================== --- kernel/arch/ia64/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,139 +1,0 @@ -/* - * Copyright (c) 2005 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ia64 - * @{ - */ -/** @file - */ - -#ifndef KERN_ia64_ATOMIC_H_ -#define KERN_ia64_ATOMIC_H_ - -#include - -NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "movl %[v] = 0x1;;\n" - "xchg8 %[v] = %[count], %[v];;\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); - - return v; -} - -NO_TRACE static inline void atomic_lock_arch(atomic_t *val) -{ - do { - while (val->count) - ; - } while (test_and_set(val)); -} - -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "fetchadd8.rel %[v] = %[count], 1\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); -} - -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "fetchadd8.rel %[v] = %[count], -1\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); -} - -NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "fetchadd8.rel %[v] = %[count], 1\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); - - return (v + 1); -} - -NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "fetchadd8.rel %[v] = %[count], -1\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); - - return (v - 1); -} - -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "fetchadd8.rel %[v] = %[count], 1\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); - - return v; -} - -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - atomic_count_t v; - - asm volatile ( - "fetchadd8.rel %[v] = %[count], -1\n" - : [v] "=r" (v), - [count] "+m" (val->count) - ); - - return v; -} - -#endif - -/** @} - */ Index: kernel/arch/mips32/include/arch/atomic.h =================================================================== --- kernel/arch/mips32/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,114 +1,0 @@ -/* - * Copyright (c) 2005 Ondrej Palkovsky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup mips32 - * @{ - */ -/** @file - */ - -#ifndef KERN_mips32_ATOMIC_H_ -#define KERN_mips32_ATOMIC_H_ - -#include - -#define atomic_inc(x) ((void) atomic_add(x, 1)) -#define atomic_dec(x) ((void) atomic_add(x, -1)) - -#define atomic_postinc(x) (atomic_add(x, 1) - 1) -#define atomic_postdec(x) (atomic_add(x, -1) + 1) - -#define atomic_preinc(x) atomic_add(x, 1) -#define atomic_predec(x) atomic_add(x, -1) - -/** Atomic addition of immediate value. - * - * @param val Memory location to which will be the immediate value added. - * @param i Signed immediate that will be added to *val. - * - * @return Value after addition. - */ -NO_TRACE static inline atomic_count_t atomic_add(atomic_t *val, - atomic_count_t i) -{ - atomic_count_t tmp; - atomic_count_t v; - - asm volatile ( - "1:\n" - " ll %0, %1\n" - " addu %0, %0, %3\n" /* same as addi, but never traps on overflow */ - " move %2, %0\n" - " sc %0, %1\n" - " beq %0, %4, 1b\n" /* if the atomic operation failed, try again */ - " nop\n" - : "=&r" (tmp), - "+m" (val->count), - "=&r" (v) - : "r" (i), - "i" (0) - ); - - return v; -} - -NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) -{ - atomic_count_t tmp; - atomic_count_t v; - - asm volatile ( - "1:\n" - " ll %2, %1\n" - " bnez %2, 2f\n" - " li %0, %3\n" - " sc %0, %1\n" - " beqz %0, 1b\n" - " nop\n" - "2:\n" - : "=&r" (tmp), - "+m" (val->count), - "=&r" (v) - : "i" (1) - ); - - return v; -} - -NO_TRACE static inline void atomic_lock_arch(atomic_t *val) -{ - do { - while (val->count) - ; - } while (test_and_set(val)); -} - -#endif - -/** @} - */ Index: kernel/arch/ppc32/include/arch/atomic.h =================================================================== --- kernel/arch/ppc32/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,103 +1,0 @@ -/* - * Copyright (c) 2005 Martin Decky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ppc32 - * @{ - */ -/** @file - */ - -#ifndef KERN_ppc32_ATOMIC_H_ -#define KERN_ppc32_ATOMIC_H_ - -#include - -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ - atomic_count_t tmp; - - asm volatile ( - "1:\n" - " lwarx %[tmp], 0, %[count_ptr]\n" - " addic %[tmp], %[tmp], 1\n" - " stwcx. %[tmp], 0, %[count_ptr]\n" - " bne- 1b" - : [tmp] "=&r" (tmp), - "=m" (val->count) - : [count_ptr] "r" (&val->count), - "m" (val->count) - : "cc" - ); -} - -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ - atomic_count_t tmp; - - asm volatile ( - "1:\n" - " lwarx %[tmp], 0, %[count_ptr]\n" - " addic %[tmp], %[tmp], -1\n" - " stwcx. %[tmp], 0, %[count_ptr]\n" - " bne- 1b" - : [tmp] "=&r" (tmp), - "=m" (val->count) - : [count_ptr] "r" (&val->count), - "m" (val->count) - : "cc" - ); -} - -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - atomic_inc(val); - return val->count - 1; -} - -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - atomic_dec(val); - return val->count + 1; -} - -NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) -{ - atomic_inc(val); - return val->count; -} - -NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) -{ - atomic_dec(val); - return val->count; -} - -#endif - -/** @} - */ Index: kernel/arch/riscv64/include/arch/atomic.h =================================================================== --- kernel/arch/riscv64/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,113 +1,0 @@ -/* - * Copyright (c) 2016 Martin Decky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup riscv64 - * @{ - */ -/** @file - */ - -#ifndef KERN_riscv64_ATOMIC_H_ -#define KERN_riscv64_ATOMIC_H_ - -#include - -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ - asm volatile ( - "amoadd.d zero, %[inc], %[addr]\n" - : [addr] "+A" (val->count) - : [inc] "r" (1) - ); -} - -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ - asm volatile ( - "amoadd.d zero, %[inc], %[addr]\n" - : [addr] "+A" (val->count) - : [inc] "r" (-1) - ); -} - -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - atomic_count_t orig; - - asm volatile ( - "amoadd.d %[orig], %[inc], %[addr]\n" - : [orig] "=r" (orig), [addr] "+A" (val->count) - : [inc] "r" (1) - ); - - return orig; -} - -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - atomic_count_t orig; - - asm volatile ( - "amoadd.d %[orig], %[inc], %[addr]\n" - : [orig] "=r" (orig), [addr] "+A" (val->count) - : [inc] "r" (-1) - ); - - return orig; -} - -NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) -{ - atomic_count_t orig; - - asm volatile ( - "amoadd.d %[orig], %[inc], %[addr]\n" - : [orig] "=r" (orig), [addr] "+A" (val->count) - : [inc] "r" (1) - ); - - return orig - 1; -} - -NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) -{ - atomic_count_t orig; - - asm volatile ( - "amoadd.d %[orig], %[inc], %[addr]\n" - : [orig] "=r" (orig), [addr] "+A" (val->count) - : [inc] "r" (-1) - ); - - return orig + 1; -} - -#endif - -/** @} - */ Index: kernel/arch/sparc64/include/arch/atomic.h =================================================================== --- kernel/arch/sparc64/include/arch/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ (revision ) @@ -1,156 +1,0 @@ -/* - * Copyright (c) 2005 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup sparc64 - * @{ - */ -/** @file - */ - -#ifndef KERN_sparc64_ATOMIC_H_ -#define KERN_sparc64_ATOMIC_H_ - -#include -#include -#include -#include - -/** Atomic add operation. - * - * Use atomic compare and swap operation to atomically add signed value. - * - * @param val Atomic variable. - * @param i Signed value to be added. - * - * @return Value of the atomic variable as it existed before addition. - * - */ -NO_TRACE static inline atomic_count_t atomic_add(atomic_t *val, - atomic_count_t i) -{ - atomic_count_t a; - atomic_count_t b; - - do { - volatile uintptr_t ptr = (uintptr_t) &val->count; - - a = *((atomic_count_t *) ptr); - b = a + i; - - asm volatile ( - "casx %0, %2, %1\n" - : "+m" (*((atomic_count_t *) ptr)), - "+r" (b) - : "r" (a) - ); - } while (a != b); - - return a; -} - -NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) -{ - return atomic_add(val, 1) + 1; -} - -NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - return atomic_add(val, 1); -} - -NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) -{ - return atomic_add(val, -1) - 1; -} - -NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - return atomic_add(val, -1); -} - -NO_TRACE static inline void atomic_inc(atomic_t *val) -{ - (void) atomic_add(val, 1); -} - -NO_TRACE static inline void atomic_dec(atomic_t *val) -{ - (void) atomic_add(val, -1); -} - -NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) -{ - atomic_count_t v = 1; - volatile uintptr_t ptr = (uintptr_t) &val->count; - - asm volatile ( - "casx %0, %2, %1\n" - : "+m" (*((atomic_count_t *) ptr)), - "+r" (v) - : "r" (0) - ); - - return v; -} - -NO_TRACE static inline void atomic_lock_arch(atomic_t *val) -{ - atomic_count_t tmp1 = 1; - atomic_count_t tmp2 = 0; - - volatile uintptr_t ptr = (uintptr_t) &val->count; - - preemption_disable(); - - asm volatile ( - "0:\n" - "casx %0, %3, %1\n" - "brz %1, 2f\n" - "nop\n" - "1:\n" - "ldx %0, %2\n" - "brz %2, 0b\n" - "nop\n" - "ba,a %%xcc, 1b\n" - "2:\n" - : "+m" (*((atomic_count_t *) ptr)), - "+r" (tmp1), - "+r" (tmp2) - : "r" (0) - ); - - /* - * Prevent critical section code from bleeding out this way up. - */ - CS_ENTER_BARRIER(); -} - -#endif - -/** @} - */ Index: kernel/generic/include/atomic.h =================================================================== --- kernel/generic/include/atomic.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ kernel/generic/include/atomic.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) @@ -36,33 +36,64 @@ #define KERN_ATOMIC_H_ +#include #include -#include -#include +#include -NO_TRACE ATOMIC static inline void atomic_set(atomic_t *val, atomic_count_t i) - WRITES(&val->count) - REQUIRES_EXTENT_MUTABLE(val) +typedef size_t atomic_count_t; +typedef ssize_t atomic_signed_t; + +#define PRIua "zu" /**< Format for atomic_count_t. */ + +typedef struct { + volatile atomic_size_t count; +} atomic_t; + +static inline void atomic_set(atomic_t *val, atomic_count_t i) { - val->count = i; + atomic_store(&val->count, i); } -NO_TRACE ATOMIC static inline atomic_count_t atomic_get(atomic_t *val) - REQUIRES_EXTENT_MUTABLE(val) +static inline atomic_count_t atomic_get(atomic_t *val) { - return val->count; + return atomic_load(&val->count); } +static inline size_t atomic_predec(atomic_t *val) +{ + return atomic_fetch_sub(&val->count, 1) - 1; +} -/* - * If the architecture does not provide operations that are atomic - * only with respect to the local cpu (eg exception handlers) and - * not other cpus, implement these cpu local atomic operations with - * full blown smp-safe atomics. - */ -#ifndef local_atomic_exchange +static inline size_t atomic_preinc(atomic_t *val) +{ + return atomic_fetch_add(&val->count, 1) + 1; +} + +static inline size_t atomic_postdec(atomic_t *val) +{ + return atomic_fetch_sub(&val->count, 1); +} + +static inline size_t atomic_postinc(atomic_t *val) +{ + return atomic_fetch_add(&val->count, 1); +} + +static inline void atomic_dec(atomic_t *val) +{ + atomic_fetch_sub(&val->count, 1); +} + +static inline void atomic_inc(atomic_t *val) +{ + atomic_fetch_add(&val->count, 1); +} + #define local_atomic_exchange(var_addr, new_val) \ - __atomic_exchange_n((var_addr), (new_val), __ATOMIC_RELAXED) -#endif + atomic_exchange_explicit(var_addr, new_val, memory_order_relaxed) +static inline bool test_and_set(atomic_t *val) +{ + return atomic_exchange(&val->count, 1); +} Index: kernel/generic/include/typedefs.h =================================================================== --- kernel/generic/include/typedefs.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ kernel/generic/include/typedefs.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) @@ -39,8 +39,4 @@ #include <_bits/errno.h> -typedef struct { - volatile atomic_count_t count; -} atomic_t; - typedef void (*function)(void); Index: kernel/generic/include/verify.h =================================================================== --- kernel/generic/include/verify.h (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ kernel/generic/include/verify.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) @@ -39,6 +39,4 @@ #ifdef CONFIG_VERIFY_VCC -#define ATOMIC __specification_attr("atomic_inline", "") - #define READS(ptr) __specification(reads(ptr)) #define WRITES(ptr) __specification(writes(ptr)) @@ -55,6 +53,4 @@ #else /* CONFIG_VERIFY_VCC */ - -#define ATOMIC #define READS(ptr) Index: kernel/generic/src/proc/scheduler.c =================================================================== --- kernel/generic/src/proc/scheduler.c (revision 7328ff41dcbe15332d62c013417ec72bbb613ee6) +++ kernel/generic/src/proc/scheduler.c (revision 4621d2311994bf63dea425ed923239d4ca1babc9) @@ -40,4 +40,5 @@ #include +#include #include #include