[5753fbb] | 1 | /*
|
---|
[df4ed85] | 2 | * Copyright (c) 2001-2004 Jakub Jermar
|
---|
[5753fbb] | 3 | * All rights reserved.
|
---|
| 4 | *
|
---|
| 5 | * Redistribution and use in source and binary forms, with or without
|
---|
| 6 | * modification, are permitted provided that the following conditions
|
---|
| 7 | * are met:
|
---|
| 8 | *
|
---|
| 9 | * - Redistributions of source code must retain the above copyright
|
---|
| 10 | * notice, this list of conditions and the following disclaimer.
|
---|
| 11 | * - Redistributions in binary form must reproduce the above copyright
|
---|
| 12 | * notice, this list of conditions and the following disclaimer in the
|
---|
| 13 | * documentation and/or other materials provided with the distribution.
|
---|
| 14 | * - The name of the author may not be used to endorse or promote products
|
---|
| 15 | * derived from this software without specific prior written permission.
|
---|
| 16 | *
|
---|
| 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
---|
| 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
---|
| 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
---|
| 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
---|
| 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
---|
| 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
---|
| 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
---|
| 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
---|
| 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
---|
| 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
---|
| 27 | */
|
---|
| 28 |
|
---|
[06e1e95] | 29 | /** @addtogroup amd64
|
---|
[b45c443] | 30 | * @{
|
---|
| 31 | */
|
---|
| 32 | /** @file
|
---|
| 33 | */
|
---|
| 34 |
|
---|
[06e1e95] | 35 | #ifndef KERN_amd64_ATOMIC_H_
|
---|
| 36 | #define KERN_amd64_ATOMIC_H_
|
---|
[5753fbb] | 37 |
|
---|
| 38 | #include <arch/types.h>
|
---|
[53f9821] | 39 | #include <arch/barrier.h>
|
---|
| 40 | #include <preemption.h>
|
---|
[5753fbb] | 41 |
|
---|
| 42 | static inline void atomic_inc(atomic_t *val) {
|
---|
| 43 | #ifdef CONFIG_SMP
|
---|
[e7b7be3f] | 44 | asm volatile ("lock incq %0\n" : "=m" (val->count));
|
---|
[5753fbb] | 45 | #else
|
---|
[e7b7be3f] | 46 | asm volatile ("incq %0\n" : "=m" (val->count));
|
---|
[5753fbb] | 47 | #endif /* CONFIG_SMP */
|
---|
| 48 | }
|
---|
| 49 |
|
---|
| 50 | static inline void atomic_dec(atomic_t *val) {
|
---|
| 51 | #ifdef CONFIG_SMP
|
---|
[e7b7be3f] | 52 | asm volatile ("lock decq %0\n" : "=m" (val->count));
|
---|
[5753fbb] | 53 | #else
|
---|
[e7b7be3f] | 54 | asm volatile ("decq %0\n" : "=m" (val->count));
|
---|
[5753fbb] | 55 | #endif /* CONFIG_SMP */
|
---|
| 56 | }
|
---|
| 57 |
|
---|
[23684b7] | 58 | static inline long atomic_postinc(atomic_t *val)
|
---|
[5753fbb] | 59 | {
|
---|
[0b917dd] | 60 | long r = 1;
|
---|
[5753fbb] | 61 |
|
---|
[e7b7be3f] | 62 | asm volatile (
|
---|
[0b917dd] | 63 | "lock xaddq %1, %0\n"
|
---|
[8abbcc9] | 64 | : "=m" (val->count), "+r" (r)
|
---|
[5753fbb] | 65 | );
|
---|
| 66 |
|
---|
| 67 | return r;
|
---|
| 68 | }
|
---|
| 69 |
|
---|
[23684b7] | 70 | static inline long atomic_postdec(atomic_t *val)
|
---|
[5753fbb] | 71 | {
|
---|
[0b917dd] | 72 | long r = -1;
|
---|
[5753fbb] | 73 |
|
---|
[e7b7be3f] | 74 | asm volatile (
|
---|
[0b917dd] | 75 | "lock xaddq %1, %0\n"
|
---|
[8abbcc9] | 76 | : "=m" (val->count), "+r" (r)
|
---|
[5753fbb] | 77 | );
|
---|
| 78 |
|
---|
| 79 | return r;
|
---|
| 80 | }
|
---|
| 81 |
|
---|
[9a2d6e1] | 82 | #define atomic_preinc(val) (atomic_postinc(val)+1)
|
---|
| 83 | #define atomic_predec(val) (atomic_postdec(val)-1)
|
---|
[5753fbb] | 84 |
|
---|
[7f1c620] | 85 | static inline uint64_t test_and_set(atomic_t *val) {
|
---|
| 86 | uint64_t v;
|
---|
[5753fbb] | 87 |
|
---|
[e7b7be3f] | 88 | asm volatile (
|
---|
[5753fbb] | 89 | "movq $1, %0\n"
|
---|
| 90 | "xchgq %0, %1\n"
|
---|
| 91 | : "=r" (v),"=m" (val->count)
|
---|
| 92 | );
|
---|
| 93 |
|
---|
| 94 | return v;
|
---|
| 95 | }
|
---|
| 96 |
|
---|
| 97 |
|
---|
[23684b7] | 98 | /** amd64 specific fast spinlock */
|
---|
[53f9821] | 99 | static inline void atomic_lock_arch(atomic_t *val)
|
---|
| 100 | {
|
---|
[7f1c620] | 101 | uint64_t tmp;
|
---|
[53f9821] | 102 |
|
---|
| 103 | preemption_disable();
|
---|
[e7b7be3f] | 104 | asm volatile (
|
---|
[53f9821] | 105 | "0:;"
|
---|
| 106 | #ifdef CONFIG_HT
|
---|
[e1be3b6] | 107 | "pause;"
|
---|
[53f9821] | 108 | #endif
|
---|
| 109 | "mov %0, %1;"
|
---|
| 110 | "testq %1, %1;"
|
---|
[23684b7] | 111 | "jnz 0b;" /* Lightweight looping on locked spinlock */
|
---|
[53f9821] | 112 |
|
---|
| 113 | "incq %1;" /* now use the atomic operation */
|
---|
| 114 | "xchgq %0, %1;"
|
---|
| 115 | "testq %1, %1;"
|
---|
| 116 | "jnz 0b;"
|
---|
| 117 | : "=m"(val->count),"=r"(tmp)
|
---|
| 118 | );
|
---|
| 119 | /*
|
---|
| 120 | * Prevent critical section code from bleeding out this way up.
|
---|
| 121 | */
|
---|
| 122 | CS_ENTER_BARRIER();
|
---|
| 123 | }
|
---|
[5753fbb] | 124 |
|
---|
| 125 | #endif
|
---|
[b45c443] | 126 |
|
---|
[06e1e95] | 127 | /** @}
|
---|
[b45c443] | 128 | */
|
---|