Branch data Line data Source code
1 : : // This file is a part of Julia. License is MIT: https://julialang.org/license
2 : :
3 : : #ifndef JL_ATOMICS_H
4 : : #define JL_ATOMICS_H
5 : :
6 : : #if defined(__i386__) && defined(__GNUC__) && !defined(__SSE2__)
7 : : # error Julia can only be built for architectures above Pentium 4. Pass -march=pentium4, or set MARCH=pentium4 and ensure that -march is not passed separately with an older architecture.
8 : : #endif
9 : :
10 : : // Low-level atomic operations
11 : : #ifdef __cplusplus
12 : : #include <atomic>
13 : : using std::memory_order_relaxed;
14 : : using std::memory_order_consume;
15 : : using std::memory_order_acquire;
16 : : using std::memory_order_release;
17 : : using std::memory_order_acq_rel;
18 : : using std::memory_order_seq_cst;
19 : : using std::atomic_thread_fence;
20 : : using std::atomic_signal_fence;
21 : : using std::atomic_load;
22 : : using std::atomic_load_explicit;
23 : : using std::atomic_store;
24 : : using std::atomic_store_explicit;
25 : : using std::atomic_fetch_add;
26 : : using std::atomic_fetch_add_explicit;
27 : : using std::atomic_fetch_and;
28 : : using std::atomic_fetch_and_explicit;
29 : : using std::atomic_fetch_or;
30 : : using std::atomic_fetch_or_explicit;
31 : : using std::atomic_compare_exchange_strong;
32 : : using std::atomic_compare_exchange_strong_explicit;
33 : : using std::atomic_exchange;
34 : : using std::atomic_exchange_explicit;
35 : : extern "C" {
36 : : #define _Atomic(T) std::atomic<T>
37 : : #else
38 : : #include <stdatomic.h>
39 : : #endif
40 : : #include <signal.h> // for sig_atomic_t
41 : :
42 : : #if defined(_CPU_X86_64_) || defined(_CPU_X86_)
43 : : # include <immintrin.h>
44 : : #endif
45 : :
46 : : enum jl_memory_order {
47 : : jl_memory_order_unspecified = -2,
48 : : jl_memory_order_invalid = -1,
49 : : jl_memory_order_notatomic = 0,
50 : : jl_memory_order_unordered,
51 : : jl_memory_order_monotonic,
52 : : jl_memory_order_consume,
53 : : jl_memory_order_acquire,
54 : : jl_memory_order_release,
55 : : jl_memory_order_acq_rel,
56 : : jl_memory_order_seq_cst
57 : : };
58 : :
59 : : /**
60 : : * Thread synchronization primitives:
61 : : *
62 : : * These roughly follows the c11/c++11 memory model and the act as memory
63 : : * barriers at both the compiler level and the hardware level.
64 : : * The only exception is the GC safepoint and GC state transitions for which
65 : : * we use only a compiler (signal) barrier and use the signal handler to do the
66 : : * synchronization in order to lower the mutator overhead as much as possible.
67 : : *
68 : : * We use the compiler intrinsics to implement a similar API to the c11/c++11
69 : : * one instead of using it directly because, we need interoperability between
70 : : * code written in different languages. The current c++ standard (c++14) does
71 : : * not allow using c11 atomic functions or types and there's currently no
72 : : * guarantee that the two types are compatible (although most of them probably
73 : : * are). We also need to access these atomic variables from the LLVM JIT code
74 : : * which is very hard unless the layout of the object is fully specified.
75 : : */
76 : : #define jl_fence() atomic_thread_fence(memory_order_seq_cst)
77 : : #define jl_fence_release() atomic_thread_fence(memory_order_release)
78 : : #define jl_signal_fence() atomic_signal_fence(memory_order_seq_cst)
79 : :
80 : : #ifdef __cplusplus
81 : : }
82 : : // implicit conversion wasn't correctly specified 2017, so many compilers get
83 : : // this wrong thus we include the correct definitions here (with implicit
84 : : // conversion), instead of using the macro version
85 : : template<class T>
86 : : T jl_atomic_load(std::atomic<T> *ptr)
87 : : {
88 : : return std::atomic_load<T>(ptr);
89 : : }
90 : : template<class T>
91 : 287341193 : T jl_atomic_load_explicit(std::atomic<T> *ptr, std::memory_order order)
92 : : {
93 : 287341193 : return std::atomic_load_explicit<T>(ptr, order);
94 : : }
95 : : #define jl_atomic_load_relaxed(ptr) jl_atomic_load_explicit(ptr, memory_order_relaxed)
96 : : #define jl_atomic_load_acquire(ptr) jl_atomic_load_explicit(ptr, memory_order_acquire)
97 : : template<class T, class S>
98 : : void jl_atomic_store(std::atomic<T> *ptr, S desired)
99 : : {
100 : : std::atomic_store<T>(ptr, desired);
101 : : }
102 : : template<class T, class S>
103 : 137761 : void jl_atomic_store_explicit(std::atomic<T> *ptr, S desired, std::memory_order order)
104 : : {
105 : 137761 : std::atomic_store_explicit<T>(ptr, desired, order);
106 : 137761 : }
107 : : #define jl_atomic_store_relaxed(ptr, val) jl_atomic_store_explicit(ptr, val, memory_order_relaxed)
108 : : #define jl_atomic_store_release(ptr, val) jl_atomic_store_explicit(ptr, val, memory_order_release)
109 : : template<class T, class S>
110 : 0 : T jl_atomic_fetch_add(std::atomic<T> *ptr, S val)
111 : : {
112 : 0 : return std::atomic_fetch_add<T>(ptr, val);
113 : : }
114 : : template<class T, class S>
115 : 0 : T jl_atomic_fetch_add_explicit(std::atomic<T> *ptr, S val, std::memory_order order)
116 : : {
117 : 0 : return std::atomic_fetch_add_explicit<T>(ptr, val, order);
118 : : }
119 : : #define jl_atomic_fetch_add_relaxed(ptr, val) jl_atomic_fetch_add_explicit(ptr, val, memory_order_relaxed)
120 : : template<class T, class S>
121 : : T jl_atomic_fetch_and(std::atomic<T> *ptr, S val)
122 : : {
123 : : return std::atomic_fetch_and<T>(ptr, val);
124 : : }
125 : : template<class T, class S>
126 : : T jl_atomic_fetch_and_explicit(std::atomic<T> *ptr, S val, std::memory_order order)
127 : : {
128 : : return std::atomic_fetch_and_explicit<T>(ptr, val, order);
129 : : }
130 : : #define jl_atomic_fetch_and_relaxed(ptr, val) jl_atomic_fetch_and_explicit(ptr, val, memory_order_relaxed)
131 : : template<class T, class S>
132 : : T jl_atomic_fetch_or(std::atomic<T> *ptr, S val)
133 : : {
134 : : return std::atomic_fetch_or<T>(ptr, val);
135 : : }
136 : : template<class T, class S>
137 : : T jl_atomic_fetch_or_explicit(std::atomic<T> *ptr, S val, std::memory_order order)
138 : : {
139 : : return std::atomic_fetch_or_explicit<T>(ptr, val, order);
140 : : }
141 : : #define jl_atomic_fetch_or_relaxed(ptr, val) jl_atomic_fetch_or_explicit(ptr, val, memory_order_relaxed)
142 : : template<class T, class S>
143 : : bool jl_atomic_cmpswap(std::atomic<T> *ptr, T *expected, S val)
144 : : {
145 : : return std::atomic_compare_exchange_strong<T>(ptr, expected, val);
146 : : }
147 : : template<class T, class S>
148 : : bool jl_atomic_cmpswap_explicit(std::atomic<T> *ptr, T *expected, S val, std::memory_order order)
149 : : {
150 : : return std::atomic_compare_exchange_strong_explicit<T>(ptr, expected, val, order, order);
151 : : }
152 : : #define jl_atomic_cmpswap_relaxed(ptr, expected, val) jl_atomic_cmpswap_explicit(ptr, expected, val, memory_order_relaxed)
153 : : template<class T, class S>
154 : : T jl_atomic_exchange(std::atomic<T> *ptr, S desired)
155 : : {
156 : : return std::atomic_exchange<T>(ptr, desired);
157 : : }
158 : : template<class T, class S>
159 : : T jl_atomic_exchange_explicit(std::atomic<T> *ptr, S desired, std::memory_order order)
160 : : {
161 : : return std::atomic_exchange_explicit<T>(ptr, desired, order);
162 : : }
163 : : #define jl_atomic_exchange_relaxed(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_relaxed)
164 : : extern "C" {
165 : : #else
166 : :
167 : : # define jl_atomic_fetch_add_relaxed(obj, arg) \
168 : : atomic_fetch_add_explicit(obj, arg, memory_order_relaxed)
169 : : # define jl_atomic_fetch_add(obj, arg) \
170 : : atomic_fetch_add(obj, arg)
171 : : # define jl_atomic_fetch_and_relaxed(obj, arg) \
172 : : atomic_fetch_and_explicit(obj, arg, memory_order_relaxed)
173 : : # define jl_atomic_fetch_and(obj, arg) \
174 : : atomic_fetch_and(obj, arg)
175 : : # define jl_atomic_fetch_or_relaxed(obj, arg) \
176 : : atomic_fetch_or_explicit(obj, arg, __ATOMIC_RELAXED)
177 : : # define jl_atomic_fetch_or(obj, arg) \
178 : : atomic_fetch_or(obj, arg)
179 : : # define jl_atomic_cmpswap(obj, expected, desired) \
180 : : atomic_compare_exchange_strong(obj, expected, desired)
181 : : # define jl_atomic_cmpswap_relaxed(obj, expected, desired) \
182 : : atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_relaxed, memory_order_relaxed)
183 : : // TODO: Maybe add jl_atomic_cmpswap_weak for spin lock
184 : : # define jl_atomic_exchange(obj, desired) \
185 : : atomic_exchange(obj, desired)
186 : : # define jl_atomic_exchange_relaxed(obj, desired) \
187 : : atomic_exchange_explicit(obj, desired, memory_order_relaxed)
188 : : # define jl_atomic_store(obj, val) \
189 : : atomic_store(obj, val)
190 : : # define jl_atomic_store_relaxed(obj, val) \
191 : : atomic_store_explicit(obj, val, memory_order_relaxed)
192 : :
193 : : # if defined(__clang__) || !(defined(_CPU_X86_) || defined(_CPU_X86_64_))
194 : : // Clang doesn't have this bug...
195 : : # define jl_atomic_store_release(obj, val) \
196 : : atomic_store_explicit(obj, val, memory_order_release)
197 : : # else
198 : : // Workaround a GCC bug when using store with release order by using the
199 : : // stronger version instead.
200 : : // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67458
201 : : // fixed in https://gcc.gnu.org/git/?p=gcc.git&a=commit;h=d8c40eff56f69877b33c697ded756d50fde90c27
202 : : # define jl_atomic_store_release(obj, val) do { \
203 : : jl_signal_fence(); \
204 : : atomic_store_explicit(obj, val, memory_order_release); \
205 : : } while (0)
206 : : # endif
207 : : # define jl_atomic_load(obj) \
208 : : atomic_load(obj)
209 : : # define jl_atomic_load_acquire(obj) \
210 : : atomic_load_explicit(obj, memory_order_acquire)
211 : : #ifdef _COMPILER_TSAN_ENABLED_
212 : : // For the sake of tsan, call these loads consume ordering since they will act
213 : : // as such on the processors we support while normally, the compiler would
214 : : // upgrade this to acquire ordering, which is strong (and slower) than we want.
215 : : # define jl_atomic_load_relaxed(obj) \
216 : : atomic_load_explicit(obj, memory_order_consume)
217 : : #else
218 : : # define jl_atomic_load_relaxed(obj) \
219 : : atomic_load_explicit(obj, memory_order_relaxed)
220 : : #endif
221 : : #endif
222 : :
223 : : #ifdef __clang_gcanalyzer__
224 : : // for the purposes of the GC analyzer, we can turn these into non-atomic
225 : : // expressions with similar properties (for the sake of the analyzer, we don't
226 : : // care if it is an exact match for behavior)
227 : :
228 : : #undef _Atomic
229 : : #define _Atomic(T) T
230 : :
231 : : #undef jl_atomic_exchange
232 : : #undef jl_atomic_exchange_relaxed
233 : : #define jl_atomic_exchange(obj, desired) \
234 : : (__extension__({ \
235 : : __typeof__((obj)) p__analyzer__ = (obj); \
236 : : __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \
237 : : *p__analyzer__ = (desired); \
238 : : temp__analyzer__; \
239 : : }))
240 : : #define jl_atomic_exchange_relaxed jl_atomic_exchange
241 : :
242 : : #undef jl_atomic_cmpswap
243 : : #undef jl_atomic_cmpswap_relaxed
244 : : #define jl_atomic_cmpswap(obj, expected, desired) \
245 : : (__extension__({ \
246 : : __typeof__((obj)) p__analyzer__ = (obj); \
247 : : __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \
248 : : __typeof__((expected)) x__analyzer__ = (expected); \
249 : : int eq__analyzer__ = memcmp(&temp__analyzer__, x__analyzer__, sizeof(temp__analyzer__)) == 0; \
250 : : if (eq__analyzer__) \
251 : : *p__analyzer__ = (desired); \
252 : : else \
253 : : *x__analyzer__ = temp__analyzer__; \
254 : : eq__analyzer__; \
255 : : }))
256 : : #define jl_atomic_cmpswap_relaxed jl_atomic_cmpswap
257 : :
258 : : #undef jl_atomic_store
259 : : #undef jl_atomic_store_release
260 : : #undef jl_atomic_store_relaxed
261 : : #define jl_atomic_store(obj, val) (*(obj) = (val))
262 : : #define jl_atomic_store_release jl_atomic_store
263 : : #define jl_atomic_store_relaxed jl_atomic_store
264 : :
265 : : #undef jl_atomic_load
266 : : #undef jl_atomic_load_acquire
267 : : #undef jl_atomic_load_relaxed
268 : : #define jl_atomic_load(obj) (*(obj))
269 : : #define jl_atomic_load_acquire jl_atomic_load
270 : : #define jl_atomic_load_relaxed jl_atomic_load
271 : :
272 : : #undef jl_atomic_fetch_add
273 : : #undef jl_atomic_fetch_and
274 : : #undef jl_atomic_fetch_or
275 : : #undef jl_atomic_fetch_add_relaxed
276 : : #undef jl_atomic_fetch_and_relaxed
277 : : #undef jl_atomic_fetch_or_relaxed
278 : : #define jl_atomic_fetch_add(obj, val) \
279 : : (__extension__({ \
280 : : __typeof__((obj)) p__analyzer__ = (obj); \
281 : : __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \
282 : : *(p__analyzer__) = temp__analyzer__ + (val); \
283 : : temp__analyzer__; \
284 : : }))
285 : : #define jl_atomic_fetch_and(obj, val) \
286 : : (__extension__({ \
287 : : __typeof__((obj)) p__analyzer__ = (obj); \
288 : : __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \
289 : : *(p__analyzer__) = temp__analyzer__ & (val); \
290 : : temp__analyzer__; \
291 : : }))
292 : : #define jl_atomic_fetch_or(obj, val) \
293 : : (__extension__({ \
294 : : __typeof__((obj)) p__analyzer__ = (obj); \
295 : : __typeof__(*p__analyzer__) temp__analyzer__ = *p__analyzer__; \
296 : : *(p__analyzer__) = temp__analyzer__ | (val); \
297 : : temp__analyzer__; \
298 : : }))
299 : : #define jl_atomic_fetch_add_relaxed jl_atomic_fetch_add
300 : : #define jl_atomic_fetch_and_relaxed jl_atomic_fetch_and
301 : : #define jl_atomic_fetch_or_relaxed jl_atomic_fetch_or
302 : :
303 : : #endif
304 : :
305 : :
306 : : #ifdef __cplusplus
307 : : }
308 : : #endif
309 : :
310 : : #endif // JL_ATOMICS_H
|
