source: mainline/uspace/lib/c/arch/arm32/src/atomic.c@ bb4d0b5

Last change on this file since bb4d0b5 was 25fdb2d, checked in by Jiří Zárevúcky <zarevucky.jiri@…>, 8 months ago

Change atomic_load/store_8 to use single word read/writes

Turns out strd/ldrd aren't supported on some older processors.
  • Property mode set to 100644
File size: 10.1 KB
Line 
1/*
2 * Copyright (c) 2007 Michal Kebrt
3 * Copyright (c) 2018 CZ.NIC, z.s.p.o.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * - Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * - Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * - The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30/*
31 * Older ARMs don't have atomic instructions, so we need to define a bunch
32 * of symbols for GCC to use.
33 */
34
35#include <stdbool.h>
36#include "ras_page.h"
37
38volatile unsigned *ras_page;
39
40unsigned long long __atomic_load_8(const volatile void *mem0, int model)
41{
42 const volatile unsigned *mem = mem0;
43
44 (void) model;
45
46 union {
47 unsigned long long a;
48 unsigned b[2];
49 } ret;
50
51 /*
52 * The following instructions between labels 1 and 2 constitute a
53 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
54 * the kernel will restart it.
55 */
56 asm volatile (
57 "1:\n"
58 " adr %[ret0], 1b\n"
59 " str %[ret0], %[rp0]\n"
60 " adr %[ret0], 2f\n"
61 " str %[ret0], %[rp1]\n"
62
63 " ldr %[ret0], %[addr0]\n"
64 " ldr %[ret1], %[addr1]\n"
65 "2:\n"
66 : [ret0] "=&r" (ret.b[0]),
67 [ret1] "=&r" (ret.b[1]),
68 [rp0] "=m" (ras_page[0]),
69 [rp1] "=m" (ras_page[1])
70 : [addr0] "m" (mem[0]),
71 [addr1] "m" (mem[1])
72 );
73
74 ras_page[0] = 0;
75 ras_page[1] = 0xffffffff;
76
77 return ret.a;
78}
79
80void __atomic_store_8(volatile void *mem0, unsigned long long val, int model)
81{
82 volatile unsigned *mem = mem0;
83
84 (void) model;
85
86 union {
87 unsigned long long a;
88 unsigned b[2];
89 } v;
90
91 v.a = val;
92
93 /* scratch register */
94 unsigned tmp;
95
96 /*
97 * The following instructions between labels 1 and 2 constitute a
98 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
99 * the kernel will restart it.
100 */
101 asm volatile (
102 "1:\n"
103 " adr %[tmp], 1b\n"
104 " str %[tmp], %[rp0]\n"
105 " adr %[tmp], 2f\n"
106 " str %[tmp], %[rp1]\n"
107
108 " str %[val0], %[addr0]\n"
109 " str %[val1], %[addr1]\n"
110 "2:\n"
111 : [tmp] "=&r" (tmp),
112 [rp0] "=m" (ras_page[0]),
113 [rp1] "=m" (ras_page[1]),
114 [addr0] "=m" (mem[0]),
115 [addr1] "=m" (mem[1])
116 : [val0] "r" (v.b[0]),
117 [val1] "r" (v.b[1])
118 );
119
120 ras_page[0] = 0;
121 ras_page[1] = 0xffffffff;
122}
123
124bool __atomic_compare_exchange_1(volatile void *mem0, void *expected0,
125 unsigned char desired, bool weak, int success, int failure)
126{
127 volatile unsigned char *mem = mem0;
128 unsigned char *expected = expected0;
129
130 (void) success;
131 (void) failure;
132 (void) weak;
133
134 unsigned char ov = *expected;
135 unsigned ret;
136
137 /*
138 * The following instructions between labels 1 and 2 constitute a
139 * Restartable Atomic Sequence. Should the sequence be non-atomic,
140 * the kernel will restart it.
141 */
142 asm volatile (
143 "1:\n"
144 " adr %[ret], 1b\n"
145 " str %[ret], %[rp0]\n"
146 " adr %[ret], 2f\n"
147 " str %[ret], %[rp1]\n"
148
149 " ldrb %[ret], %[addr]\n"
150 " cmp %[ret], %[ov]\n"
151 " streqb %[nv], %[addr]\n"
152 "2:\n"
153 : [ret] "=&r" (ret),
154 [rp0] "=m" (ras_page[0]),
155 [rp1] "=m" (ras_page[1]),
156 [addr] "+m" (*mem)
157 : [ov] "r" (ov),
158 [nv] "r" (desired)
159 );
160
161 ras_page[0] = 0;
162 ras_page[1] = 0xffffffff;
163
164 if (ret == ov)
165 return true;
166
167 *expected = ret;
168 return false;
169}
170
171bool __atomic_compare_exchange_4(volatile void *mem0, void *expected0,
172 unsigned desired, bool weak, int success, int failure)
173{
174 volatile unsigned *mem = mem0;
175 unsigned *expected = expected0;
176
177 (void) success;
178 (void) failure;
179 (void) weak;
180
181 unsigned ov = *expected;
182 unsigned ret;
183
184 /*
185 * The following instructions between labels 1 and 2 constitute a
186 * Restartable Atomic Sequence. Should the sequence be non-atomic,
187 * the kernel will restart it.
188 */
189 asm volatile (
190 "1:\n"
191 " adr %[ret], 1b\n"
192 " str %[ret], %[rp0]\n"
193 " adr %[ret], 2f\n"
194 " str %[ret], %[rp1]\n"
195
196 " ldr %[ret], %[addr]\n"
197 " cmp %[ret], %[ov]\n"
198 " streq %[nv], %[addr]\n"
199 "2:\n"
200 : [ret] "=&r" (ret),
201 [rp0] "=m" (ras_page[0]),
202 [rp1] "=m" (ras_page[1]),
203 [addr] "+m" (*mem)
204 : [ov] "r" (ov),
205 [nv] "r" (desired)
206 : "memory"
207 );
208
209 ras_page[0] = 0;
210 ras_page[1] = 0xffffffff;
211
212 if (ret == ov)
213 return true;
214
215 *expected = ret;
216 return false;
217}
218
219unsigned char __atomic_exchange_1(volatile void *mem0, unsigned char val,
220 int model)
221{
222 volatile unsigned char *mem = mem0;
223
224 (void) model;
225
226 unsigned ret;
227
228 /*
229 * The following instructions between labels 1 and 2 constitute a
230 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
231 * the kernel will restart it.
232 */
233 asm volatile (
234 "1:\n"
235 " adr %[ret], 1b\n"
236 " str %[ret], %[rp0]\n"
237 " adr %[ret], 2f\n"
238 " str %[ret], %[rp1]\n"
239 " ldrb %[ret], %[addr]\n"
240 " strb %[imm], %[addr]\n"
241 "2:\n"
242 : [ret] "=&r" (ret),
243 [rp0] "=m" (ras_page[0]),
244 [rp1] "=m" (ras_page[1]),
245 [addr] "+m" (*mem)
246 : [imm] "r" (val)
247 );
248
249 ras_page[0] = 0;
250 ras_page[1] = 0xffffffff;
251
252 return ret;
253}
254
255unsigned short __atomic_exchange_2(volatile void *mem0, unsigned short val,
256 int model)
257{
258 volatile unsigned short *mem = mem0;
259
260 (void) model;
261
262 unsigned ret;
263
264 /*
265 * The following instructions between labels 1 and 2 constitute a
266 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
267 * the kernel will restart it.
268 */
269 asm volatile (
270 "1:\n"
271 " adr %[ret], 1b\n"
272 " str %[ret], %[rp0]\n"
273 " adr %[ret], 2f\n"
274 " str %[ret], %[rp1]\n"
275 " ldrh %[ret], %[addr]\n"
276 " strh %[imm], %[addr]\n"
277 "2:\n"
278 : [ret] "=&r" (ret),
279 [rp0] "=m" (ras_page[0]),
280 [rp1] "=m" (ras_page[1]),
281 [addr] "+m" (*mem)
282 : [imm] "r" (val)
283 );
284
285 ras_page[0] = 0;
286 ras_page[1] = 0xffffffff;
287
288 return ret;
289}
290
291unsigned __atomic_exchange_4(volatile void *mem0, unsigned val, int model)
292{
293 volatile unsigned *mem = mem0;
294
295 (void) model;
296
297 unsigned ret;
298
299 /*
300 * The following instructions between labels 1 and 2 constitute a
301 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
302 * the kernel will restart it.
303 */
304 asm volatile (
305 "1:\n"
306 " adr %[ret], 1b\n"
307 " str %[ret], %[rp0]\n"
308 " adr %[ret], 2f\n"
309 " str %[ret], %[rp1]\n"
310 " ldr %[ret], %[addr]\n"
311 " str %[imm], %[addr]\n"
312 "2:\n"
313 : [ret] "=&r" (ret),
314 [rp0] "=m" (ras_page[0]),
315 [rp1] "=m" (ras_page[1]),
316 [addr] "+m" (*mem)
317 : [imm] "r" (val)
318 );
319
320 ras_page[0] = 0;
321 ras_page[1] = 0xffffffff;
322
323 return ret;
324}
325
326unsigned short __atomic_fetch_add_2(volatile void *mem0, unsigned short val,
327 int model)
328{
329 volatile unsigned short *mem = mem0;
330
331 (void) model;
332
333 unsigned short ret;
334
335 /*
336 * The following instructions between labels 1 and 2 constitute a
337 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
338 * the kernel will restart it.
339 */
340 asm volatile (
341 "1:\n"
342 " adr %[ret], 1b\n"
343 " str %[ret], %[rp0]\n"
344 " adr %[ret], 2f\n"
345 " str %[ret], %[rp1]\n"
346 " ldrh %[ret], %[addr]\n"
347 " add %[ret], %[ret], %[imm]\n"
348 " strh %[ret], %[addr]\n"
349 "2:\n"
350 : [ret] "=&r" (ret),
351 [rp0] "=m" (ras_page[0]),
352 [rp1] "=m" (ras_page[1]),
353 [addr] "+m" (*mem)
354 : [imm] "r" (val)
355 );
356
357 ras_page[0] = 0;
358 ras_page[1] = 0xffffffff;
359
360 return ret - val;
361}
362
363unsigned __atomic_fetch_add_4(volatile void *mem0, unsigned val, int model)
364{
365 volatile unsigned *mem = mem0;
366
367 (void) model;
368
369 unsigned ret;
370
371 /*
372 * The following instructions between labels 1 and 2 constitute a
373 * Restartable Atomic Seqeunce. Should the sequence be non-atomic,
374 * the kernel will restart it.
375 */
376 asm volatile (
377 "1:\n"
378 " adr %[ret], 1b\n"
379 " str %[ret], %[rp0]\n"
380 " adr %[ret], 2f\n"
381 " str %[ret], %[rp1]\n"
382 " ldr %[ret], %[addr]\n"
383 " add %[ret], %[ret], %[imm]\n"
384 " str %[ret], %[addr]\n"
385 "2:\n"
386 : [ret] "=&r" (ret),
387 [rp0] "=m" (ras_page[0]),
388 [rp1] "=m" (ras_page[1]),
389 [addr] "+m" (*mem)
390 : [imm] "r" (val)
391 );
392
393 ras_page[0] = 0;
394 ras_page[1] = 0xffffffff;
395
396 return ret - val;
397}
398
399unsigned __atomic_fetch_sub_4(volatile void *mem, unsigned val, int model)
400{
401 return __atomic_fetch_add_4(mem, -val, model);
402}
403
404bool __atomic_test_and_set(volatile void *ptr, int memorder)
405{
406 volatile unsigned char *b = ptr;
407
408 unsigned char orig = __atomic_exchange_n(b, (unsigned char) true, memorder);
409 return orig != 0;
410}
411
412void __sync_synchronize(void)
413{
414 // FIXME: Full memory barrier. We might need a syscall for this.
415}
416
417unsigned __sync_add_and_fetch_4(volatile void *vptr, unsigned val)
418{
419 return __atomic_fetch_add_4(vptr, val, __ATOMIC_SEQ_CST) + val;
420}
421
422unsigned __sync_sub_and_fetch_4(volatile void *vptr, unsigned val)
423{
424 return __atomic_fetch_sub_4(vptr, val, __ATOMIC_SEQ_CST) - val;
425}
426
427bool __sync_bool_compare_and_swap_4(volatile void *ptr, unsigned old_val, unsigned new_val)
428{
429 return __atomic_compare_exchange_4(ptr, &old_val, new_val, false,
430 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
431}
432
433unsigned __sync_val_compare_and_swap_4(volatile void *ptr, unsigned old_val, unsigned new_val)
434{
435 __atomic_compare_exchange_4(ptr, &old_val, new_val, false,
436 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
437 return old_val;
438}
Note: See TracBrowser for help on using the repository browser.