source: mainline/kernel/arch/amd64/include/atomic.h@ 5e3d959

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 5e3d959 was a1ef4c0, checked in by Adam Hraska <adam.hraska+hos@…>, 13 years ago

Fixed type mismatch in local_atomic_exchange() on amd64, ia32. Fixes build of amd64.
  • Property mode set to 100644
File size: 6.6 KB
Line 
1/*
2 * Copyright (c) 2001-2004 Jakub Jermar
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup amd64
30 * @{
31 */
32/** @file
33 */
34
35#ifndef KERN_amd64_ATOMIC_H_
36#define KERN_amd64_ATOMIC_H_
37
38#include <typedefs.h>
39#include <arch/barrier.h>
40#include <preemption.h>
41#include <trace.h>
42
43NO_TRACE static inline void atomic_inc(atomic_t *val)
44{
45#ifdef CONFIG_SMP
46 asm volatile (
47 "lock incq %[count]\n"
48 : [count] "+m" (val->count)
49 );
50#else
51 asm volatile (
52 "incq %[count]\n"
53 : [count] "+m" (val->count)
54 );
55#endif /* CONFIG_SMP */
56}
57
58NO_TRACE static inline void atomic_dec(atomic_t *val)
59{
60#ifdef CONFIG_SMP
61 asm volatile (
62 "lock decq %[count]\n"
63 : [count] "+m" (val->count)
64 );
65#else
66 asm volatile (
67 "decq %[count]\n"
68 : [count] "+m" (val->count)
69 );
70#endif /* CONFIG_SMP */
71}
72
73NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val)
74{
75 atomic_count_t r = 1;
76
77 asm volatile (
78 "lock xaddq %[r], %[count]\n"
79 : [count] "+m" (val->count),
80 [r] "+r" (r)
81 );
82
83 return r;
84}
85
86NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val)
87{
88 atomic_count_t r = -1;
89
90 asm volatile (
91 "lock xaddq %[r], %[count]\n"
92 : [count] "+m" (val->count),
93 [r] "+r" (r)
94 );
95
96 return r;
97}
98
99#define atomic_preinc(val) (atomic_postinc(val) + 1)
100#define atomic_predec(val) (atomic_postdec(val) - 1)
101
102NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val)
103{
104 atomic_count_t v = 1;
105
106 asm volatile (
107 "xchgq %[v], %[count]\n"
108 : [v] "+r" (v),
109 [count] "+m" (val->count)
110 );
111
112 return v;
113}
114
115/** amd64 specific fast spinlock */
116NO_TRACE static inline void atomic_lock_arch(atomic_t *val)
117{
118 atomic_count_t tmp;
119
120 preemption_disable();
121 asm volatile (
122 "0:\n"
123 " pause\n"
124 " mov %[count], %[tmp]\n"
125 " testq %[tmp], %[tmp]\n"
126 " jnz 0b\n" /* lightweight looping on locked spinlock */
127
128 " incq %[tmp]\n" /* now use the atomic operation */
129 " xchgq %[count], %[tmp]\n"
130 " testq %[tmp], %[tmp]\n"
131 " jnz 0b\n"
132 : [count] "+m" (val->count),
133 [tmp] "=&r" (tmp)
134 );
135
136 /*
137 * Prevent critical section code from bleeding out this way up.
138 */
139 CS_ENTER_BARRIER();
140}
141
142
143#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \
144({ \
145 switch (sizeof(typeof(*(pptr)))) { \
146 case 1: \
147 asm volatile ( \
148 prefix " cmpxchgb %[newval], %[ptr]\n" \
149 : /* Output operands. */ \
150 /* Old/current value is returned in eax. */ \
151 [oldval] "=a" (old_val), \
152 /* (*ptr) will be read and written to, hence "+" */ \
153 [ptr] "+m" (*pptr) \
154 : /* Input operands. */ \
155 /* Expected value must be in eax. */ \
156 [expval] "a" (exp_val), \
157 /* The new value may be in any register. */ \
158 [newval] "r" (new_val) \
159 : "memory" \
160 ); \
161 break; \
162 case 2: \
163 asm volatile ( \
164 prefix " cmpxchgw %[newval], %[ptr]\n" \
165 : /* Output operands. */ \
166 /* Old/current value is returned in eax. */ \
167 [oldval] "=a" (old_val), \
168 /* (*ptr) will be read and written to, hence "+" */ \
169 [ptr] "+m" (*pptr) \
170 : /* Input operands. */ \
171 /* Expected value must be in eax. */ \
172 [expval] "a" (exp_val), \
173 /* The new value may be in any register. */ \
174 [newval] "r" (new_val) \
175 : "memory" \
176 ); \
177 break; \
178 case 4: \
179 asm volatile ( \
180 prefix " cmpxchgl %[newval], %[ptr]\n" \
181 : /* Output operands. */ \
182 /* Old/current value is returned in eax. */ \
183 [oldval] "=a" (old_val), \
184 /* (*ptr) will be read and written to, hence "+" */ \
185 [ptr] "+m" (*pptr) \
186 : /* Input operands. */ \
187 /* Expected value must be in eax. */ \
188 [expval] "a" (exp_val), \
189 /* The new value may be in any register. */ \
190 [newval] "r" (new_val) \
191 : "memory" \
192 ); \
193 break; \
194 case 8: \
195 asm volatile ( \
196 prefix " cmpxchgq %[newval], %[ptr]\n" \
197 : /* Output operands. */ \
198 /* Old/current value is returned in eax. */ \
199 [oldval] "=a" (old_val), \
200 /* (*ptr) will be read and written to, hence "+" */ \
201 [ptr] "+m" (*pptr) \
202 : /* Input operands. */ \
203 /* Expected value must be in eax. */ \
204 [expval] "a" (exp_val), \
205 /* The new value may be in any register. */ \
206 [newval] "r" (new_val) \
207 : "memory" \
208 ); \
209 break; \
210 } \
211})
212
213
214#ifndef local_atomic_cas
215
216#define local_atomic_cas(pptr, exp_val, new_val) \
217({ \
218 /* Use proper types and avoid name clashes */ \
219 typeof(*(pptr)) _old_val_cas; \
220 typeof(*(pptr)) _exp_val_cas = exp_val; \
221 typeof(*(pptr)) _new_val_cas = new_val; \
222 _atomic_cas_impl(pptr, _exp_val_cas, _new_val_cas, _old_val_cas, ""); \
223 \
224 _old_val_cas; \
225})
226
227#else
228/* Check if arch/atomic.h does not accidentally include /atomic.h .*/
229#error Architecture specific cpu local atomics already defined! Check your includes.
230#endif
231
232
233#ifndef local_atomic_exchange
234/*
235 * Issuing a xchg instruction always implies lock prefix semantics.
236 * Therefore, it is cheaper to use a cmpxchg without a lock prefix
237 * in a loop.
238 */
239#define local_atomic_exchange(pptr, new_val) \
240({ \
241 /* Use proper types and avoid name clashes */ \
242 typeof(*(pptr)) _exp_val_x; \
243 typeof(*(pptr)) _old_val_x; \
244 typeof(*(pptr)) _new_val_x = new_val; \
245 \
246 do { \
247 _exp_val_x = *pptr; \
248 _old_val_x = local_atomic_cas(pptr, _exp_val_x, _new_val_x); \
249 } while (_old_val_x != _exp_val_x); \
250 \
251 _old_val_x; \
252})
253
254#else
255/* Check if arch/atomic.h does not accidentally include /atomic.h .*/
256#error Architecture specific cpu local atomics already defined! Check your includes.
257#endif
258
259
260#endif
261
262/** @}
263 */
Note: See TracBrowser for help on using the repository browser.