Changes in kernel/arch/amd64/include/atomic.h [9d58539:b17518e] in mainline

File:

• kernel/arch/amd64/include/atomic.h (modified) (1 diff)

kernel/arch/amd64/include/atomic.h

@@ -140 +140 @@
 }
 
+
+#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \
+        asm volatile ( \
+                prefix " cmpxchgq %[newval], %[ptr]\n" \
+                : /* Output operands. */ \
+                /* Old/current value is returned in eax. */ \
+                [oldval] "=a" (old_val), \
+                /* (*ptr) will be read and written to, hence "+" */ \
+                [ptr] "+m" (*pptr) \
+                : /* Input operands. */ \
+                /* Expected value must be in eax. */ \
+                [expval] "a" (exp_val), \
+                /* The new value may be in any register. */ \
+                [newval] "r" (new_val) \
+                : "memory" \
+        )
+        
+/** Atomically compares and swaps the pointer at pptr. */
+NO_TRACE static inline void * atomic_cas_ptr(void **pptr, 
+        void *exp_val, void *new_val)
+{
+        void *old_val;
+        _atomic_cas_impl(pptr, exp_val, new_val, old_val, "lock\n");
+        return old_val;
+}
+
+/** Compare-and-swap of a pointer that is atomic wrt to local cpu's interrupts.
+ * 
+ * This function is NOT smp safe and is not atomic with respect to other cpus.
+ */
+NO_TRACE static inline void * atomic_cas_ptr_local(void **pptr, 
+        void *exp_val, void *new_val)
+{
+        void *old_val;
+        _atomic_cas_impl(pptr, exp_val, new_val, old_val, "");
+        return old_val;
+}
+
+
+#define _atomic_swap_impl(pptr, new_val) \
+({ \
+        typeof(*(pptr)) new_in_old_out = new_val; \
+        asm volatile ( \
+                "xchgq %[val], %[p_ptr]\n" \
+                : [val] "+r" (new_in_old_out), \
+                  [p_ptr] "+m" (*pptr) \
+        ); \
+        \
+        new_in_old_out; \
+})
+
+/* 
+ * Issuing a xchg instruction always implies lock prefix semantics.
+ * Therefore, it is cheaper to use a cmpxchg without a lock prefix 
+ * in a loop.
+ */
+#define _atomic_swap_local_impl(pptr, new_val) \
+({ \
+        typeof(*(pptr)) exp_val; \
+        typeof(*(pptr)) old_val; \
+        \
+        do { \
+                exp_val = *pptr; \
+                _atomic_cas_impl(pptr, exp_val, new_val, old_val, ""); \
+        } while (old_val != exp_val); \
+        \
+        old_val; \
+})
+
+
+/** Atomicaly sets *ptr to val and returns the previous value. */
+NO_TRACE static inline void * atomic_set_return_ptr(void **pptr, void *val)
+{
+        return _atomic_swap_impl(pptr, val);
+}
+
+/** Sets *ptr to new_val and returns the previous value. NOT smp safe.
+ * 
+ * This function is only atomic wrt to local interrupts and it is
+ * NOT atomic wrt to other cpus.
+ */
+NO_TRACE static inline void * atomic_set_return_ptr_local(
+        void **pptr, void *new_val)
+{
+        return _atomic_swap_local_impl(pptr, new_val);
+}
+
+/** Atomicaly sets *ptr to val and returns the previous value. */
+NO_TRACE static inline native_t atomic_set_return_native_t(
+        native_t *p, native_t val)
+{
+        return _atomic_swap_impl(p, val);
+}
+
+/** Sets *ptr to new_val and returns the previous value. NOT smp safe.
+ * 
+ * This function is only atomic wrt to local interrupts and it is
+ * NOT atomic wrt to other cpus.
+ */
+NO_TRACE static inline native_t atomic_set_return_native_t_local(
+        native_t *p, native_t new_val)
+{
+        return _atomic_swap_local_impl(p, new_val);
+}
+
+
+#undef _atomic_cas_ptr_impl
+#undef _atomic_swap_impl
+#undef _atomic_swap_local_impl
+
 #endif