Branch data Line data Source code
1 : : // This file is a part of Julia. License is MIT: https://julialang.org/license
2 : :
3 : : #ifndef JL_INTERNAL_H
4 : : #define JL_INTERNAL_H
5 : :
6 : : #include "options.h"
7 : : #include "julia_locks.h"
8 : : #include "julia_threads.h"
9 : : #include "support/utils.h"
10 : : #include "support/hashing.h"
11 : : #include "support/ptrhash.h"
12 : : #include "support/strtod.h"
13 : : #include "gc-alloc-profiler.h"
14 : : #include "support/rle.h"
15 : : #include <uv.h>
16 : : #include <llvm-c/Types.h>
17 : : #include <llvm-c/Orc.h>
18 : : #if !defined(_WIN32)
19 : : #include <unistd.h>
20 : : #else
21 : : #define sleep(x) Sleep(1000*x)
22 : : #endif
23 : :
24 : : #ifdef __cplusplus
25 : : extern "C" {
26 : : #endif
27 : : #ifdef _COMPILER_ASAN_ENABLED_
28 : : void __sanitizer_start_switch_fiber(void**, const void*, size_t);
29 : : void __sanitizer_finish_switch_fiber(void*, const void**, size_t*);
30 : : #endif
31 : : #ifdef _COMPILER_TSAN_ENABLED_
32 : : void *__tsan_create_fiber(unsigned flags);
33 : : void *__tsan_get_current_fiber(void);
34 : : void __tsan_destroy_fiber(void *fiber);
35 : : void __tsan_switch_to_fiber(void *fiber, unsigned flags);
36 : : #endif
37 : : #ifdef __cplusplus
38 : : }
39 : : #endif
40 : :
41 : : // Remove when C11 is required for C code.
42 : : #ifndef static_assert
43 : : # ifndef __cplusplus
44 : : // C11 should already have `static_assert` from `<assert.h>` so there's no need
45 : : // to check C version.
46 : : # ifdef __GNUC__
47 : : # define static_assert _Static_assert
48 : : # else
49 : : # define static_assert(...)
50 : : # endif
51 : : # endif
52 : : // For C++, C++11 or MSVC is required. Both provide `static_assert`.
53 : : #endif
54 : :
55 : : #ifndef alignof
56 : : # ifndef __cplusplus
57 : : # ifdef __GNUC__
58 : : # define alignof _Alignof
59 : : # else
60 : : # define alignof(...) 1
61 : : # endif
62 : : # endif
63 : : #endif
64 : :
65 : : #if defined(__GLIBC__) && defined(JULIA_HAS_IFUNC_SUPPORT)
66 : : // Make sure both the compiler and the glibc supports it.
67 : : // Only enable this on known working glibc versions.
68 : : # if (defined(_CPU_X86_) || defined(_CPU_X86_64_)) && __GLIBC_PREREQ(2, 12)
69 : : # define JL_USE_IFUNC 1
70 : : # elif (defined(_CPU_ARM_) || defined(_CPU_AARCH64_)) && __GLIBC_PREREQ(2, 18)
71 : : // This is the oldest tested version that supports ifunc.
72 : : # define JL_USE_IFUNC 1
73 : : # endif
74 : : // TODO: PPC probably supports ifunc on some glibc versions too
75 : : #endif
76 : : // Make sure JL_USE_IFUNC is always defined to catch include errors.
77 : : #ifndef JL_USE_IFUNC
78 : : # define JL_USE_IFUNC 0
79 : : #endif
80 : :
81 : : // If we've smashed the stack, (and not just normal NORETURN)
82 : : // this will smash stack-unwind too
83 : : #ifdef _OS_WINDOWS_
84 : : #if defined(_CPU_X86_64_)
85 : : // install the unhandled exception handler at the top of our stack
86 : : // to call directly into our personality handler
87 : : #define CFI_NORETURN \
88 : : asm volatile ("\t.seh_handler __julia_personality, @except\n\t.text");
89 : : #else
90 : : #define CFI_NORETURN
91 : : #endif
92 : : #else
93 : : // wipe out the call-stack unwind capability beyond this function
94 : : // (we are noreturn, so it is not a total lie)
95 : : #if defined(_CPU_X86_64_)
96 : : // per nongnu libunwind: "x86_64 ABI specifies that end of call-chain is marked with a NULL RBP or undefined return address"
97 : : // so we do all 3, to be extra certain of it
98 : : #define CFI_NORETURN \
99 : : asm volatile ("\t.cfi_undefined rip"); \
100 : : asm volatile ("\t.cfi_undefined rbp"); \
101 : : asm volatile ("\t.cfi_return_column rbp");
102 : : #else
103 : : // per nongnu libunwind: "DWARF spec says undefined return address location means end of stack"
104 : : // we use whatever happens to be register 1 on this platform for this
105 : : #define CFI_NORETURN \
106 : : asm volatile ("\t.cfi_undefined 1"); \
107 : : asm volatile ("\t.cfi_return_column 1");
108 : : #endif
109 : : #endif
110 : :
111 : : extern JL_DLLEXPORT uintptr_t __stack_chk_guard;
112 : :
113 : : // If this is detected in a backtrace of segfault, it means the functions
114 : : // that use this value must be reworked into their async form with cb arg
115 : : // provided and with JL_UV_LOCK used around the calls
116 : : static uv_loop_t *const unused_uv_loop_arg = (uv_loop_t *)0xBAD10;
117 : :
118 : : extern jl_mutex_t jl_uv_mutex;
119 : : extern _Atomic(int) jl_uv_n_waiters;
120 : : void JL_UV_LOCK(void);
121 : : #define JL_UV_UNLOCK() JL_UNLOCK(&jl_uv_mutex)
122 : :
123 : : #ifdef __cplusplus
124 : : extern "C" {
125 : : #endif
126 : :
127 : : int jl_running_under_rr(int recheck) JL_NOTSAFEPOINT;
128 : :
129 : : //--------------------------------------------------
130 : : // timers
131 : : // Returns time in nanosec
132 : : JL_DLLEXPORT uint64_t jl_hrtime(void) JL_NOTSAFEPOINT;
133 : :
134 : : JL_DLLEXPORT void jl_set_peek_cond(uintptr_t);
135 : : JL_DLLEXPORT double jl_get_profile_peek_duration(void);
136 : : JL_DLLEXPORT void jl_set_profile_peek_duration(double);
137 : :
138 : : JL_DLLEXPORT void jl_init_profile_lock(void);
139 : : JL_DLLEXPORT uintptr_t jl_lock_profile_rd_held(void) JL_NOTSAFEPOINT;
140 : : JL_DLLEXPORT void jl_lock_profile(void) JL_NOTSAFEPOINT;
141 : : JL_DLLEXPORT void jl_unlock_profile(void) JL_NOTSAFEPOINT;
142 : : JL_DLLEXPORT void jl_lock_profile_wr(void) JL_NOTSAFEPOINT;
143 : : JL_DLLEXPORT void jl_unlock_profile_wr(void) JL_NOTSAFEPOINT;
144 : :
145 : : // number of cycles since power-on
146 : 90 : static inline uint64_t cycleclock(void) JL_NOTSAFEPOINT
147 : : {
148 : : #if defined(_CPU_X86_64_)
149 : : uint64_t low, high;
150 : 90 : __asm__ volatile("rdtsc" : "=a"(low), "=d"(high));
151 : 90 : return (high << 32) | low;
152 : : #elif defined(_CPU_X86_)
153 : : int64_t ret;
154 : : __asm__ volatile("rdtsc" : "=A"(ret));
155 : : return ret;
156 : : #elif defined(_CPU_AARCH64_)
157 : : // System timer of ARMv8 runs at a different frequency than the CPU's.
158 : : // The frequency is fixed, typically in the range 1-50MHz. It can be
159 : : // read at CNTFRQ special register. We assume the OS has set up
160 : : // the virtual timer properly.
161 : : int64_t virtual_timer_value;
162 : : __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value));
163 : : return virtual_timer_value;
164 : : #elif defined(_CPU_PPC64_)
165 : : // This returns a time-base, which is not always precisely a cycle-count.
166 : : // https://reviews.llvm.org/D78084
167 : : int64_t tb;
168 : : asm volatile("mfspr %0, 268" : "=r" (tb));
169 : : return tb;
170 : : #else
171 : : #warning No cycleclock() definition for your platform
172 : : // copy from https://github.com/google/benchmark/blob/v1.5.0/src/cycleclock.h
173 : : return 0;
174 : : #endif
175 : : }
176 : :
177 : : #include "timing.h"
178 : :
179 : : // Global *atomic* integers controlling *process-wide* measurement of compilation time.
180 : : extern JL_DLLEXPORT _Atomic(uint8_t) jl_measure_compile_time_enabled;
181 : : extern JL_DLLEXPORT _Atomic(uint64_t) jl_cumulative_compile_time;
182 : : extern JL_DLLEXPORT _Atomic(uint64_t) jl_cumulative_recompile_time;
183 : :
184 : : #define jl_return_address() ((uintptr_t)__builtin_return_address(0))
185 : :
186 : 414845000 : STATIC_INLINE uint32_t jl_int32hash_fast(uint32_t a)
187 : : {
188 : : // a = (a+0x7ed55d16) + (a<<12);
189 : : // a = (a^0xc761c23c) ^ (a>>19);
190 : : // a = (a+0x165667b1) + (a<<5);
191 : : // a = (a+0xd3a2646c) ^ (a<<9);
192 : : // a = (a+0xfd7046c5) + (a<<3);
193 : : // a = (a^0xb55a4f09) ^ (a>>16);
194 : 414845000 : return a; // identity hashing seems to work well enough here
195 : : }
196 : :
197 : :
198 : : // this is a version of memcpy that preserves atomic memory ordering
199 : : // which makes it safe to use for objects that can contain memory references
200 : : // without risk of creating pointers out of thin air
201 : : // TODO: replace with LLVM's llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32
202 : : // aka `__llvm_memmove_element_unordered_atomic_8` (for 64 bit)
203 : 7942262 : static inline void memmove_refs(void **dstp, void *const *srcp, size_t n) JL_NOTSAFEPOINT
204 : : {
205 : : size_t i;
206 : 7942262 : _Atomic(void*) *srcpa = (_Atomic(void*)*)srcp;
207 : 7942262 : _Atomic(void*) *dstpa = (_Atomic(void*)*)dstp;
208 [ + + + + ]: 7942262 : if (dstp < srcp || dstp > srcp + n) {
209 [ + + ]: 27242670 : for (i = 0; i < n; i++) {
210 : 19301933 : jl_atomic_store_release(dstpa + i, jl_atomic_load_relaxed(srcpa + i));
211 : : }
212 : : }
213 : : else {
214 [ + + ]: 20501 : for (i = 0; i < n; i++) {
215 : 18972 : jl_atomic_store_release(dstpa + n - i - 1, jl_atomic_load_relaxed(srcpa + n - i - 1));
216 : : }
217 : : }
218 : 7942262 : }
219 : :
220 : : // -- gc.c -- //
221 : :
222 : : #define GC_CLEAN 0 // freshly allocated
223 : : #define GC_MARKED 1 // reachable and young
224 : : #define GC_OLD 2 // if it is reachable it will be marked as old
225 : : #define GC_OLD_MARKED (GC_OLD | GC_MARKED) // reachable and old
226 : :
227 : : // useful constants
228 : : extern jl_methtable_t *jl_type_type_mt JL_GLOBALLY_ROOTED;
229 : : extern jl_methtable_t *jl_nonfunction_mt JL_GLOBALLY_ROOTED;
230 : : extern JL_DLLEXPORT _Atomic(size_t) jl_world_counter;
231 : :
232 : : typedef void (*tracer_cb)(jl_value_t *tracee);
233 : : extern tracer_cb jl_newmeth_tracer;
234 : : void jl_call_tracer(tracer_cb callback, jl_value_t *tracee);
235 : : void print_func_loc(JL_STREAM *s, jl_method_t *m);
236 : : extern jl_array_t *_jl_debug_method_invalidation JL_GLOBALLY_ROOTED;
237 : : void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_instance_t *replaced_mi, size_t max_world, const char *why);
238 : :
239 : : extern JL_DLLEXPORT size_t jl_page_size;
240 : : extern jl_function_t *jl_typeinf_func;
241 : : extern JL_DLLEXPORT size_t jl_typeinf_world;
242 : : extern _Atomic(jl_typemap_entry_t*) call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED;
243 : : extern jl_array_t *jl_all_methods JL_GLOBALLY_ROOTED;
244 : :
245 : : JL_DLLEXPORT extern int jl_lineno;
246 : : JL_DLLEXPORT extern const char *jl_filename;
247 : :
248 : : jl_value_t *jl_gc_pool_alloc_noinline(jl_ptls_t ptls, int pool_offset,
249 : : int osize);
250 : : jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz);
251 : : JL_DLLEXPORT int jl_gc_classify_pools(size_t sz, int *osize);
252 : : extern uv_mutex_t gc_perm_lock;
253 : : void *jl_gc_perm_alloc_nolock(size_t sz, int zero,
254 : : unsigned align, unsigned offset) JL_NOTSAFEPOINT;
255 : : void *jl_gc_perm_alloc(size_t sz, int zero,
256 : : unsigned align, unsigned offset) JL_NOTSAFEPOINT;
257 : : void jl_gc_force_mark_old(jl_ptls_t ptls, jl_value_t *v);
258 : : void gc_sweep_sysimg(void);
259 : :
260 : :
261 : : // pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET)
262 : : static const int jl_gc_sizeclasses[] = {
263 : : #ifdef _P64
264 : : 8,
265 : : #elif MAX_ALIGN > 4
266 : : // ARM and PowerPC have max alignment larger than pointer,
267 : : // make sure allocation of size 8 has that alignment.
268 : : 4, 8,
269 : : #else
270 : : 4, 8, 12,
271 : : #endif
272 : :
273 : : // 16 pools at 8-byte spacing
274 : : // the 8-byte aligned pools are only used for Strings
275 : : 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136,
276 : : // 8 pools at 16-byte spacing
277 : : 144, 160, 176, 192, 208, 224, 240, 256,
278 : :
279 : : // the following tables are computed for maximum packing efficiency via the formula:
280 : : // pg = 2^14
281 : : // sz = (div.(pg-8, rng).÷16)*16; hcat(sz, (pg-8).÷sz, pg .- (pg-8).÷sz.*sz)'
282 : :
283 : : // rng = 60:-4:32 (8 pools)
284 : : 272, 288, 304, 336, 368, 400, 448, 496,
285 : : // 60, 56, 53, 48, 44, 40, 36, 33, /pool
286 : : // 64, 256, 272, 256, 192, 384, 256, 16, bytes lost
287 : :
288 : : // rng = 30:-2:16 (8 pools)
289 : : 544, 576, 624, 672, 736, 816, 896, 1008,
290 : : // 30, 28, 26, 24, 22, 20, 18, 16, /pool
291 : : // 64, 256, 160, 256, 192, 64, 256, 256, bytes lost
292 : :
293 : : // rng = 15:-1:8 (8 pools)
294 : : 1088, 1168, 1248, 1360, 1488, 1632, 1808, 2032
295 : : // 15, 14, 13, 12, 11, 10, 9, 8, /pool
296 : : // 64, 32, 160, 64, 16, 64, 112, 128, bytes lost
297 : : };
298 : : static_assert(sizeof(jl_gc_sizeclasses) / sizeof(jl_gc_sizeclasses[0]) == JL_GC_N_POOLS, "");
299 : :
300 : 56 : STATIC_INLINE int jl_gc_alignment(size_t sz)
301 : : {
302 [ - + ]: 56 : if (sz == 0)
303 : 0 : return sizeof(void*);
304 : : #ifdef _P64
305 : : (void)sz;
306 : 56 : return 16;
307 : : #elif MAX_ALIGN == 8
308 : : return sz <= 4 ? 8 : 16;
309 : : #else
310 : : // szclass 8
311 : : if (sz <= 4)
312 : : return 8;
313 : : // szclass 12
314 : : if (sz <= 8)
315 : : return 4;
316 : : // szclass 16+
317 : : return 16;
318 : : #endif
319 : : }
320 : : JL_DLLEXPORT int jl_alignment(size_t sz);
321 : :
322 : : // the following table is computed as:
323 : : // [searchsortedfirst(jl_gc_sizeclasses, i) - 1 for i = 0:16:jl_gc_sizeclasses[end]]
324 : : static const uint8_t szclass_table[] = {0, 1, 3, 5, 7, 9, 11, 13, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 31, 32, 32, 32, 33, 33, 33, 34, 34, 35, 35, 35, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 43, 43, 43, 43, 43, 44, 44, 44, 44, 44, 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48};
325 : : static_assert(sizeof(szclass_table) == 128, "");
326 : :
327 : 636315193 : STATIC_INLINE uint8_t JL_CONST_FUNC jl_gc_szclass(unsigned sz)
328 : : {
329 [ - + ]: 636315193 : assert(sz <= 2032);
330 : : #ifdef _P64
331 [ + + ]: 636315193 : if (sz <= 8)
332 : 2980 : return 0;
333 : 636312193 : const int N = 0;
334 : : #elif MAX_ALIGN == 8
335 : : if (sz <= 8)
336 : : return (sz >= 4 ? 1 : 0);
337 : : const int N = 1;
338 : : #else
339 : : if (sz <= 12)
340 : : return (sz >= 8 ? 2 : (sz >= 4 ? 1 : 0));
341 : : const int N = 2;
342 : : #endif
343 : 636312193 : uint8_t klass = szclass_table[(sz + 15) / 16];
344 : 636312193 : return klass + N;
345 : : }
346 : :
347 : 330021 : STATIC_INLINE uint8_t JL_CONST_FUNC jl_gc_szclass_align8(unsigned sz)
348 : : {
349 [ + - + + ]: 330021 : if (sz >= 16 && sz <= 152) {
350 : : #ifdef _P64
351 : 314692 : const int N = 0;
352 : : #elif MAX_ALIGN == 8
353 : : const int N = 1;
354 : : #else
355 : : const int N = 2;
356 : : #endif
357 : 314692 : return (sz + 7)/8 - 1 + N;
358 : : }
359 : 15329 : return jl_gc_szclass(sz);
360 : : }
361 : :
362 : : #define JL_SMALL_BYTE_ALIGNMENT 16
363 : : #define JL_CACHE_BYTE_ALIGNMENT 64
364 : : // JL_HEAP_ALIGNMENT is the maximum alignment that the GC can provide
365 : : #define JL_HEAP_ALIGNMENT JL_SMALL_BYTE_ALIGNMENT
366 : : #define GC_MAX_SZCLASS (2032-sizeof(void*))
367 : : static_assert(ARRAY_CACHE_ALIGN_THRESHOLD > GC_MAX_SZCLASS, "");
368 : :
369 : 636773864 : STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)
370 : : {
371 : : jl_value_t *v;
372 : 636773864 : const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
373 [ + + ]: 636773864 : if (sz <= GC_MAX_SZCLASS) {
374 : 636235864 : int pool_id = jl_gc_szclass(allocsz);
375 : 636235864 : jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id];
376 : 636235864 : int osize = jl_gc_sizeclasses[pool_id];
377 : : // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in
378 : : // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.)
379 : 636235864 : v = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize);
380 : : }
381 : : else {
382 [ - + ]: 538299 : if (allocsz < sz) // overflow in adding offs, size was "negative"
383 : 0 : jl_throw(jl_memory_exception);
384 : 538299 : v = jl_gc_big_alloc_noinline(ptls, allocsz);
385 : : }
386 : 636773864 : jl_set_typeof(v, ty);
387 : 636773864 : maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty);
388 : 636773864 : return v;
389 : : }
390 : :
391 : : /* Programming style note: When using jl_gc_alloc, do not JL_GC_PUSH it into a
392 : : * gc frame, until it has been fully initialized. An uninitialized value in a
393 : : * gc frame can crash upon encountering the first safepoint. By delaying use of
394 : : * the JL_GC_PUSH macro until the value has been initialized, any accidental
395 : : * safepoints will be caught by the GC analyzer.
396 : : */
397 : : JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty);
398 : : // On GCC, only inline when sz is constant
399 : : #ifdef __GNUC__
400 : : # define jl_gc_alloc(ptls, sz, ty) \
401 : : (__builtin_constant_p(sz) ? \
402 : : jl_gc_alloc_(ptls, sz, ty) : \
403 : : (jl_gc_alloc)(ptls, sz, ty))
404 : : #else
405 : : # define jl_gc_alloc(ptls, sz, ty) jl_gc_alloc_(ptls, sz, ty)
406 : : #endif
407 : :
408 : : // jl_buff_tag must be a multiple of GC_PAGE_SZ so that it can't be
409 : : // confused for an actual type reference.
410 : : #define jl_buff_tag ((uintptr_t)0x4eadc000)
411 : : typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer
412 : 59374857 : STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz)
413 : : {
414 : 59374857 : return jl_gc_alloc(ptls, sz, (void*)jl_buff_tag);
415 : : }
416 : :
417 : 155782 : STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT
418 : : {
419 : 155782 : const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
420 [ + + ]: 155782 : unsigned align = (sz == 0 ? sizeof(void*) : (allocsz <= sizeof(void*) * 2 ?
421 : : sizeof(void*) * 2 : 16));
422 : 155782 : jl_taggedvalue_t *o = (jl_taggedvalue_t*)jl_gc_perm_alloc(allocsz, 0, align,
423 : 0 : sizeof(void*) % align);
424 : 155782 : uintptr_t tag = (uintptr_t)ty;
425 : 155782 : o->header = tag | GC_OLD_MARKED;
426 : 155782 : return jl_valueof(o);
427 : : }
428 : : jl_value_t *jl_permbox8(jl_datatype_t *t, int8_t x);
429 : : jl_value_t *jl_permbox16(jl_datatype_t *t, int16_t x);
430 : : jl_value_t *jl_permbox32(jl_datatype_t *t, int32_t x);
431 : : jl_value_t *jl_permbox64(jl_datatype_t *t, int64_t x);
432 : : jl_svec_t *jl_perm_symsvec(size_t n, ...);
433 : :
434 : : // this sizeof(__VA_ARGS__) trick can't be computed until C11, but that only matters to Clang in some situations
435 : : #if !defined(__clang_analyzer__) && !(defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_))
436 : : #ifdef _COMPILER_GCC_
437 : : #define jl_perm_symsvec(n, ...) \
438 : : (jl_perm_symsvec)(__extension__({ \
439 : : static_assert( \
440 : : n == sizeof((char *[]){ __VA_ARGS__ })/sizeof(char *), \
441 : : "Number of passed arguments does not match expected number"); \
442 : : n; \
443 : : }), __VA_ARGS__)
444 : : #ifdef jl_svec
445 : : #undef jl_svec
446 : : #define jl_svec(n, ...) \
447 : : (ijl_svec)(__extension__({ \
448 : : static_assert( \
449 : : n == sizeof((void *[]){ __VA_ARGS__ })/sizeof(void *), \
450 : : "Number of passed arguments does not match expected number"); \
451 : : n; \
452 : : }), __VA_ARGS__)
453 : : #else
454 : : #define jl_svec(n, ...) \
455 : : (jl_svec)(__extension__({ \
456 : : static_assert( \
457 : : n == sizeof((void *[]){ __VA_ARGS__ })/sizeof(void *), \
458 : : "Number of passed arguments does not match expected number"); \
459 : : n; \
460 : : }), __VA_ARGS__)
461 : : #endif
462 : : #endif
463 : : #endif
464 : :
465 : : jl_value_t *jl_gc_realloc_string(jl_value_t *s, size_t sz);
466 : : JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz);
467 : :
468 : : JL_DLLEXPORT void JL_NORETURN jl_throw_out_of_memory_error(void);
469 : :
470 : :
471 : : JL_DLLEXPORT int64_t jl_gc_diff_total_bytes(void) JL_NOTSAFEPOINT;
472 : : JL_DLLEXPORT int64_t jl_gc_sync_total_bytes(int64_t offset) JL_NOTSAFEPOINT;
473 : : void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT;
474 : : void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT;
475 : : void jl_gc_run_all_finalizers(jl_task_t *ct);
476 : : void jl_release_task_stack(jl_ptls_t ptls, jl_task_t *task);
477 : : void jl_gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) JL_NOTSAFEPOINT;
478 : :
479 : : // Set GC memory trigger in bytes for greedy memory collecting
480 : : void jl_gc_set_max_memory(uint64_t max_mem);
481 : :
482 : : JL_DLLEXPORT void jl_gc_queue_binding(jl_binding_t *bnd) JL_NOTSAFEPOINT;
483 : : void gc_setmark_buf(jl_ptls_t ptls, void *buf, uint8_t, size_t) JL_NOTSAFEPOINT;
484 : :
485 : 30676 : STATIC_INLINE void jl_gc_wb_binding(jl_binding_t *bnd, void *val) JL_NOTSAFEPOINT // val isa jl_value_t*
486 : : {
487 [ + + + + ]: 30676 : if (__unlikely(jl_astaggedvalue(bnd)->bits.gc == 3 &&
488 : : (jl_astaggedvalue(val)->bits.gc & 1) == 0))
489 : 6518 : jl_gc_queue_binding(bnd);
490 : 30676 : }
491 : :
492 : 59374676 : STATIC_INLINE void jl_gc_wb_buf(void *parent, void *bufptr, size_t minsz) JL_NOTSAFEPOINT // parent isa jl_value_t*
493 : : {
494 : : // if parent is marked and buf is not
495 [ + + ]: 59374676 : if (__unlikely(jl_astaggedvalue(parent)->bits.gc & 1)) {
496 : 107156 : jl_task_t *ct = jl_current_task;
497 : 107156 : gc_setmark_buf(ct->ptls, bufptr, 3, minsz);
498 : : }
499 : 59374676 : }
500 : :
501 : : void jl_gc_debug_print_status(void);
502 : : JL_DLLEXPORT void jl_gc_debug_critical_error(void);
503 : : void jl_print_gc_stats(JL_STREAM *s);
504 : : void jl_gc_reset_alloc_count(void);
505 : : uint32_t jl_get_gs_ctr(void);
506 : : void jl_set_gs_ctr(uint32_t ctr);
507 : :
508 : 129545790 : STATIC_INLINE jl_value_t *undefref_check(jl_datatype_t *dt, jl_value_t *v) JL_NOTSAFEPOINT
509 : : {
510 [ + + ]: 129545790 : if (dt->layout->first_ptr >= 0) {
511 : 10427810 : jl_value_t *nullp = ((jl_value_t**)v)[dt->layout->first_ptr];
512 [ - + ]: 10427810 : if (__unlikely(nullp == NULL))
513 : 0 : return NULL;
514 : : }
515 : 129545790 : return v;
516 : : }
517 : :
518 : : // -- helper types -- //
519 : :
520 : : typedef struct {
521 : : uint8_t pure:1;
522 : : uint8_t propagate_inbounds:1;
523 : : uint8_t inlineable:1;
524 : : uint8_t inferred:1;
525 : : uint8_t constprop:2; // 0 = use heuristic; 1 = aggressive; 2 = none
526 : : } jl_code_info_flags_bitfield_t;
527 : :
528 : : typedef union {
529 : : jl_code_info_flags_bitfield_t bits;
530 : : uint8_t packed;
531 : : } jl_code_info_flags_t;
532 : :
533 : : // -- functions -- //
534 : :
535 : : // jl_code_info_flag_t code_info_flags(uint8_t pure, uint8_t propagate_inbounds, uint8_t inlineable, uint8_t inferred, uint8_t constprop);
536 : : JL_DLLEXPORT jl_code_info_t *jl_type_infer(jl_method_instance_t *li, size_t world, int force);
537 : : JL_DLLEXPORT jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *meth JL_PROPAGATES_ROOT, size_t world);
538 : : jl_code_instance_t *jl_generate_fptr(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world);
539 : : void jl_generate_fptr_for_unspecialized(jl_code_instance_t *unspec);
540 : : JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred(
541 : : jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_value_t *rettype,
542 : : size_t min_world, size_t max_world);
543 : : jl_method_instance_t *jl_get_unspecialized_from_mi(jl_method_instance_t *method JL_PROPAGATES_ROOT);
544 : : jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT);
545 : :
546 : : JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types);
547 : : jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *lam JL_PROPAGATES_ROOT);
548 : : int jl_code_requires_compiler(jl_code_info_t *src);
549 : : jl_code_info_t *jl_new_code_info_from_ir(jl_expr_t *ast);
550 : : JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void);
551 : : void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals,
552 : : int binding_effects);
553 : :
554 : : JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root);
555 : : void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots);
556 : : int get_root_reference(rle_reference *rr, jl_method_t *m, size_t i);
557 : : jl_value_t *lookup_root(jl_method_t *m, uint64_t key, int index);
558 : : int nroots_with_key(jl_method_t *m, uint64_t key);
559 : :
560 : : int jl_valid_type_param(jl_value_t *v);
561 : :
562 : : JL_DLLEXPORT jl_value_t *jl_apply_2va(jl_value_t *f, jl_value_t **args, uint32_t nargs);
563 : :
564 : : void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na, size_t world);
565 : : JL_DLLEXPORT jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...);
566 : :
567 : : JL_DLLEXPORT jl_value_t *jl_get_keyword_sorter(jl_value_t *f);
568 : : JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t);
569 : :
570 : : #define JL_CALLABLE(name) \
571 : : JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs)
572 : :
573 : : JL_CALLABLE(jl_f_tuple);
574 : : JL_CALLABLE(jl_f_intrinsic_call);
575 : : JL_CALLABLE(jl_f_opaque_closure_call);
576 : : void jl_install_default_signal_handlers(void);
577 : : void restore_signals(void);
578 : : void jl_install_thread_signal_handler(jl_ptls_t ptls);
579 : :
580 : : JL_DLLEXPORT jl_fptr_args_t jl_get_builtin_fptr(jl_value_t *b);
581 : :
582 : : extern uv_loop_t *jl_io_loop;
583 : : void jl_uv_flush(uv_stream_t *stream);
584 : :
585 : : typedef struct jl_typeenv_t {
586 : : jl_tvar_t *var;
587 : : jl_value_t *val;
588 : : struct jl_typeenv_t *prev;
589 : : } jl_typeenv_t;
590 : :
591 : : int jl_tuple_isa(jl_value_t **child, size_t cl, jl_datatype_t *pdt);
592 : : int jl_tuple1_isa(jl_value_t *child1, jl_value_t **child, size_t cl, jl_datatype_t *pdt);
593 : :
594 : : JL_DLLEXPORT int jl_has_intersect_type_not_kind(jl_value_t *t);
595 : : int jl_subtype_invariant(jl_value_t *a, jl_value_t *b, int ta);
596 : : int jl_has_concrete_subtype(jl_value_t *typ);
597 : : jl_tupletype_t *jl_inst_arg_tuple_type(jl_value_t *arg1, jl_value_t **args, size_t nargs, int leaf);
598 : : jl_tupletype_t *jl_lookup_arg_tuple_type(jl_value_t *arg1 JL_PROPAGATES_ROOT, jl_value_t **args, size_t nargs, int leaf);
599 : : JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype);
600 : : jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED;
601 : : int jl_obviously_unequal(jl_value_t *a, jl_value_t *b);
602 : : JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v);
603 : : int jl_has_fixed_layout(jl_datatype_t *t);
604 : : JL_DLLEXPORT int jl_struct_try_layout(jl_datatype_t *dt);
605 : : JL_DLLEXPORT int jl_type_mappable_to_c(jl_value_t *ty);
606 : : jl_svec_t *jl_outer_unionall_vars(jl_value_t *u);
607 : : jl_value_t *jl_type_intersection_env_s(jl_value_t *a, jl_value_t *b, jl_svec_t **penv, int *issubty);
608 : : jl_value_t *jl_type_intersection_env(jl_value_t *a, jl_value_t *b, jl_svec_t **penv);
609 : : int jl_subtype_matching(jl_value_t *a, jl_value_t *b, jl_svec_t **penv);
610 : : JL_DLLEXPORT int jl_types_egal(jl_value_t *a, jl_value_t *b);
611 : : // specificity comparison assuming !(a <: b) and !(b <: a)
612 : : JL_DLLEXPORT int jl_type_morespecific_no_subtype(jl_value_t *a, jl_value_t *b);
613 : : jl_value_t *jl_instantiate_type_with(jl_value_t *t, jl_value_t **env, size_t n);
614 : : JL_DLLEXPORT jl_value_t *jl_instantiate_type_in_env(jl_value_t *ty, jl_unionall_t *env, jl_value_t **vals);
615 : : jl_value_t *jl_substitute_var(jl_value_t *t, jl_tvar_t *var, jl_value_t *val);
616 : : JL_DLLEXPORT jl_value_t *jl_unwrap_unionall(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
617 : : JL_DLLEXPORT jl_value_t *jl_rewrap_unionall(jl_value_t *t, jl_value_t *u);
618 : : int jl_count_union_components(jl_value_t *v);
619 : : JL_DLLEXPORT jl_value_t *jl_nth_union_component(jl_value_t *v JL_PROPAGATES_ROOT, int i) JL_NOTSAFEPOINT;
620 : : int jl_find_union_component(jl_value_t *haystack, jl_value_t *needle, unsigned *nth) JL_NOTSAFEPOINT;
621 : : jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_module_t *module,
622 : : jl_datatype_t *super, jl_svec_t *parameters);
623 : : jl_datatype_t *jl_new_uninitialized_datatype(void);
624 : : void jl_precompute_memoized_dt(jl_datatype_t *dt, int cacheable);
625 : : JL_DLLEXPORT jl_datatype_t *jl_wrap_Type(jl_value_t *t); // x -> Type{x}
626 : : jl_vararg_t *jl_wrap_vararg(jl_value_t *t, jl_value_t *n);
627 : : void jl_reinstantiate_inner_types(jl_datatype_t *t);
628 : : jl_datatype_t *jl_lookup_cache_type_(jl_datatype_t *type);
629 : : void jl_cache_type_(jl_datatype_t *type);
630 : : void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic) JL_NOTSAFEPOINT;
631 : : jl_value_t *swap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *rhs, int isatomic);
632 : : jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *op, jl_value_t *rhs, int isatomic);
633 : : jl_value_t *replace_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t *expected, jl_value_t *rhs, int isatomic);
634 : : jl_expr_t *jl_exprn(jl_sym_t *head, size_t n);
635 : : jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module);
636 : : jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_t *module, jl_datatype_t *st);
637 : : int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), void *env);
638 : : void jl_init_main_module(void);
639 : : JL_DLLEXPORT int jl_is_submodule(jl_module_t *child, jl_module_t *parent) JL_NOTSAFEPOINT;
640 : : jl_array_t *jl_get_loaded_modules(void);
641 : : JL_DLLEXPORT int jl_datatype_isinlinealloc(jl_datatype_t *ty, int pointerfree);
642 : :
643 : : void jl_eval_global_expr(jl_module_t *m, jl_expr_t *ex, int set_type);
644 : : jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded);
645 : :
646 : : jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e);
647 : : jl_value_t *jl_interpret_opaque_closure(jl_opaque_closure_t *clos, jl_value_t **args, size_t nargs);
648 : : jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src);
649 : : jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e,
650 : : jl_code_info_t *src,
651 : : jl_svec_t *sparam_vals);
652 : : JL_DLLEXPORT int jl_is_toplevel_only_expr(jl_value_t *e) JL_NOTSAFEPOINT;
653 : : jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule);
654 : :
655 : : jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, size_t world);
656 : :
657 : : jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value_t **args, size_t nargs);
658 : : jl_value_t *jl_gf_invoke(jl_value_t *types, jl_value_t *f, jl_value_t **args, size_t nargs);
659 : : JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, jl_value_t *mt, int lim, int include_ambiguous,
660 : : size_t world, size_t *min_valid, size_t *max_valid, int *ambig);
661 : :
662 : : JL_DLLEXPORT jl_datatype_t *jl_first_argument_datatype(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
663 : : JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
664 : : JL_DLLEXPORT jl_methtable_t *jl_method_table_for(
665 : : jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
666 : : JL_DLLEXPORT jl_methtable_t *jl_method_get_table(
667 : : jl_method_t *method) JL_NOTSAFEPOINT;
668 : : jl_methtable_t *jl_argument_method_table(jl_value_t *argt JL_PROPAGATES_ROOT);
669 : :
670 : : JL_DLLEXPORT int jl_pointer_egal(jl_value_t *t);
671 : : JL_DLLEXPORT jl_value_t *jl_nth_slot_type(jl_value_t *sig JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT;
672 : : void jl_compute_field_offsets(jl_datatype_t *st);
673 : : jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
674 : : int isunboxed, int hasptr, int isunion, int elsz);
675 : : void jl_module_run_initializer(jl_module_t *m);
676 : : jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var);
677 : : JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b);
678 : : extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED;
679 : : extern htable_t jl_current_modules JL_GLOBALLY_ROOTED;
680 : : extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED;
681 : : int jl_compile_extern_c(LLVMOrcThreadSafeModuleRef llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt);
682 : :
683 : : jl_opaque_closure_t *jl_new_opaque_closure(jl_tupletype_t *argt, jl_value_t *rt_lb, jl_value_t *rt_ub,
684 : : jl_value_t *source, jl_value_t **env, size_t nenv);
685 : : JL_DLLEXPORT int jl_is_valid_oc_argtype(jl_tupletype_t *argt, jl_method_t *source);
686 : :
687 : : // Each tuple can exist in one of 4 Vararg states:
688 : : // NONE: no vararg Tuple{Int,Float32}
689 : : // INT: vararg with integer length Tuple{Int,Vararg{Float32,2}}
690 : : // BOUND: vararg with bound TypeVar length Tuple{Int,Vararg{Float32,N}}
691 : : // UNBOUND: vararg with unbound length Tuple{Int,Vararg{Float32}}
692 : : typedef enum {
693 : : JL_VARARG_NONE = 0,
694 : : JL_VARARG_INT = 1,
695 : : JL_VARARG_BOUND = 2,
696 : : JL_VARARG_UNBOUND = 3
697 : : } jl_vararg_kind_t;
698 : :
699 : 3494759925 : STATIC_INLINE int jl_is_vararg(jl_value_t *v) JL_NOTSAFEPOINT
700 : : {
701 : 3494759925 : return jl_typeof(v) == (jl_value_t*)jl_vararg_type;
702 : : }
703 : :
704 : 96219476 : STATIC_INLINE jl_value_t *jl_unwrap_vararg(jl_vararg_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
705 : : {
706 [ - + ]: 96218334 : assert(jl_is_vararg((jl_value_t*)v));
707 : 96219476 : jl_value_t *T = ((jl_vararg_t*)v)->T;
708 [ + + ]: 96219476 : return T ? T : (jl_value_t*)jl_any_type;
709 : : }
710 : : #define jl_unwrap_vararg(v) (jl_unwrap_vararg)((jl_vararg_t *)v)
711 : :
712 : 26679900 : STATIC_INLINE jl_value_t *jl_unwrap_vararg_num(jl_vararg_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
713 : : {
714 [ - + ]: 26679900 : assert(jl_is_vararg((jl_value_t*)v));
715 : 26679900 : return ((jl_vararg_t*)v)->N;
716 : : }
717 : : #define jl_unwrap_vararg_num(v) (jl_unwrap_vararg_num)((jl_vararg_t *)v)
718 : :
719 : 302077288 : STATIC_INLINE jl_vararg_kind_t jl_vararg_kind(jl_value_t *v) JL_NOTSAFEPOINT
720 : : {
721 [ + + ]: 302077288 : if (!jl_is_vararg(v))
722 : 216407696 : return JL_VARARG_NONE;
723 : 85669892 : jl_vararg_t *vm = (jl_vararg_t *)v;
724 [ + + ]: 85669892 : if (!vm->N)
725 : 81686657 : return JL_VARARG_UNBOUND;
726 [ - + ]: 3983175 : if (jl_is_long(vm->N))
727 : 0 : return JL_VARARG_INT;
728 : 3983175 : return JL_VARARG_BOUND;
729 : : }
730 : :
731 : 230536153 : STATIC_INLINE int jl_is_va_tuple(jl_datatype_t *t) JL_NOTSAFEPOINT
732 : : {
733 [ - + ]: 230536153 : assert(jl_is_tuple_type(t));
734 : 230536153 : size_t l = jl_svec_len(t->parameters);
735 [ + + + + ]: 230536153 : return (l>0 && jl_is_vararg(jl_tparam(t,l-1)));
736 : : }
737 : :
738 : 0 : STATIC_INLINE size_t jl_vararg_length(jl_value_t *v) JL_NOTSAFEPOINT
739 : : {
740 [ # # ]: 0 : assert(jl_is_vararg(v));
741 : 0 : jl_value_t *len = jl_unwrap_vararg_num(v);
742 [ # # ]: 0 : assert(jl_is_long(len));
743 : 0 : return jl_unbox_long(len);
744 : : }
745 : :
746 : 3901538 : STATIC_INLINE jl_vararg_kind_t jl_va_tuple_kind(jl_datatype_t *t) JL_NOTSAFEPOINT
747 : : {
748 : 3901538 : t = (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)t);
749 [ - + ]: 3901538 : assert(jl_is_tuple_type(t));
750 : 3901538 : size_t l = jl_svec_len(t->parameters);
751 [ + + ]: 3901538 : if (l == 0)
752 : 31634 : return JL_VARARG_NONE;
753 : 3869898 : return jl_vararg_kind(jl_tparam(t,l-1));
754 : : }
755 : :
756 : : // -- init.c -- //
757 : :
758 : : void jl_init_types(void) JL_GC_DISABLED;
759 : : void jl_init_box_caches(void);
760 : : void jl_init_flisp(void);
761 : : void jl_init_common_symbols(void);
762 : : void jl_init_primitives(void) JL_GC_DISABLED;
763 : : void jl_init_llvm(void);
764 : : void jl_init_codegen(void);
765 : : void jl_init_runtime_ccall(void);
766 : : void jl_init_intrinsic_functions(void);
767 : : void jl_init_intrinsic_properties(void);
768 : : void jl_init_tasks(void) JL_GC_DISABLED;
769 : : void jl_init_stack_limits(int ismaster, void **stack_hi, void **stack_lo);
770 : : jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi);
771 : : void jl_init_serializer(void);
772 : : void jl_gc_init(void);
773 : : void jl_init_uv(void);
774 : : void jl_init_thread_heap(jl_ptls_t ptls);
775 : : void jl_init_int32_int64_cache(void);
776 : : JL_DLLEXPORT void jl_init_options(void);
777 : :
778 : : void jl_teardown_codegen(void);
779 : :
780 : : void jl_set_base_ctx(char *__stk);
781 : :
782 : : extern JL_DLLEXPORT ssize_t jl_tls_offset;
783 : : extern JL_DLLEXPORT const int jl_tls_elf_support;
784 : : void jl_init_threading(void);
785 : : void jl_start_threads(void);
786 : : int jl_effective_threads(void);
787 : :
788 : : // Whether the GC is running
789 : : extern char *jl_safepoint_pages;
790 : 0 : STATIC_INLINE int jl_addr_is_safepoint(uintptr_t addr)
791 : : {
792 : 0 : uintptr_t safepoint_addr = (uintptr_t)jl_safepoint_pages;
793 [ # # # # ]: 0 : return addr >= safepoint_addr && addr < safepoint_addr + jl_page_size * 3;
794 : : }
795 : : extern _Atomic(uint32_t) jl_gc_running;
796 : : // All the functions are safe to be called from within a signal handler
797 : : // provided that the thread will not be interrupted by another asynchronous
798 : : // signal.
799 : : // Initialize the safepoint
800 : : void jl_safepoint_init(void);
801 : : // Start the GC, return `1` if the thread should be running the GC.
802 : : // Otherwise, the thread will wait in this function until the GC finishes on
803 : : // another thread and return `0`.
804 : : // The caller should have saved the `gc_state` and set it to `WAITING`
805 : : // before calling this function. If the calling thread is to run the GC,
806 : : // it should also wait for the mutator threads to hit a safepoint **AFTER**
807 : : // this function returns
808 : : int jl_safepoint_start_gc(void);
809 : : // Can only be called by the thread that have got a `1` return value from
810 : : // `jl_safepoint_start_gc()`. This disables the safepoint (for GC,
811 : : // the `mprotect` may not be removed if there's pending SIGINT) and wake
812 : : // up waiting threads if there's any.
813 : : // The caller should restore `gc_state` **AFTER** calling this function.
814 : : void jl_safepoint_end_gc(void);
815 : : // Wait for the GC to finish
816 : : // This function does **NOT** modify the `gc_state` to inform the GC thread
817 : : // The caller should set it **BEFORE** calling this function.
818 : : void jl_safepoint_wait_gc(void);
819 : :
820 : : // Set pending sigint and enable the mechanisms to deliver the sigint.
821 : : void jl_safepoint_enable_sigint(void);
822 : : // If the safepoint is enabled to deliver sigint, disable it
823 : : // so that the thread won't repeatedly trigger it in a sigatomic region
824 : : // while not being able to actually throw the exception.
825 : : void jl_safepoint_defer_sigint(void);
826 : : // Clear the sigint pending flag and disable the mechanism to deliver sigint.
827 : : // Return `1` if the sigint should be delivered and `0` if there's no sigint
828 : : // to be delivered.
829 : : int jl_safepoint_consume_sigint(void);
830 : : void jl_wake_libuv(void);
831 : :
832 : : void jl_set_pgcstack(jl_gcframe_t **) JL_NOTSAFEPOINT;
833 : : #if defined(_OS_DARWIN_)
834 : : typedef pthread_key_t jl_pgcstack_key_t;
835 : : #elif defined(_OS_WINDOWS_)
836 : : typedef DWORD jl_pgcstack_key_t;
837 : : #else
838 : : typedef jl_gcframe_t ***(*jl_pgcstack_key_t)(void) JL_NOTSAFEPOINT;
839 : : #endif
840 : : JL_DLLEXPORT void jl_pgcstack_getkey(jl_get_pgcstack_func **f, jl_pgcstack_key_t *k);
841 : :
842 : : #if !defined(_OS_WINDOWS_) && !defined(__APPLE__) && !defined(JL_DISABLE_LIBUNWIND)
843 : : extern pthread_mutex_t in_signal_lock;
844 : : #endif
845 : :
846 : : #if !defined(__clang_gcanalyzer__) && !defined(_OS_DARWIN_)
847 : 0 : static inline void jl_set_gc_and_wait(void)
848 : : {
849 : 0 : jl_task_t *ct = jl_current_task;
850 : : // reading own gc state doesn't need atomic ops since no one else
851 : : // should store to it.
852 : 0 : int8_t state = jl_atomic_load_relaxed(&ct->ptls->gc_state);
853 : 0 : jl_atomic_store_release(&ct->ptls->gc_state, JL_GC_STATE_WAITING);
854 : 0 : jl_safepoint_wait_gc();
855 : 0 : jl_atomic_store_release(&ct->ptls->gc_state, state);
856 : 0 : }
857 : : #endif
858 : : void jl_gc_set_permalloc_region(void *start, void *end);
859 : :
860 : : typedef struct {
861 : : LLVMOrcThreadSafeModuleRef TSM;
862 : : LLVMValueRef F;
863 : : } jl_llvmf_dump_t;
864 : :
865 : : JL_DLLEXPORT jl_value_t *jl_dump_method_asm(jl_method_instance_t *linfo, size_t world,
866 : : char raw_mc, char getwrapper, const char* asm_variant, const char *debuginfo, char binary);
867 : : JL_DLLEXPORT void jl_get_llvmf_defn(jl_llvmf_dump_t* dump, jl_method_instance_t *linfo, size_t world, char getwrapper, char optimize, const jl_cgparams_t params);
868 : : JL_DLLEXPORT jl_value_t *jl_dump_fptr_asm(uint64_t fptr, char raw_mc, const char* asm_variant, const char *debuginfo, char binary);
869 : : JL_DLLEXPORT jl_value_t *jl_dump_function_ir(jl_llvmf_dump_t *dump, char strip_ir_metadata, char dump_module, const char *debuginfo);
870 : : JL_DLLEXPORT jl_value_t *jl_dump_function_asm(jl_llvmf_dump_t *dump, char raw_mc, const char* asm_variant, const char *debuginfo, char binary);
871 : :
872 : : void *jl_create_native(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int policy);
873 : : void jl_dump_native(void *native_code,
874 : : const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname,
875 : : const char *sysimg_data, size_t sysimg_len);
876 : : int32_t jl_get_llvm_gv(void *native_code, jl_value_t *p) JL_NOTSAFEPOINT;
877 : : JL_DLLEXPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode,
878 : : int32_t *func_idx, int32_t *specfunc_idx);
879 : :
880 : : // the first argument to jl_idtable_rehash is used to return a value
881 : : // make sure it is rooted if it is used after the function returns
882 : : JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz);
883 : : _Atomic(jl_value_t*) *jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT;
884 : :
885 : : JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*);
886 : :
887 : : JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module);
888 : : JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache);
889 : : jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp);
890 : : JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world);
891 : : JL_DLLEXPORT jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world);
892 : : JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *type, size_t world);
893 : : JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(
894 : : jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams);
895 : : jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins);
896 : : JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_method_instance_t *caller);
897 : : JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller);
898 : :
899 : : uint32_t jl_module_next_counter(jl_module_t *m) JL_NOTSAFEPOINT;
900 : : jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs);
901 : :
902 : : JL_DLLEXPORT int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT;
903 : :
904 : : jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t *filename,
905 : : size_t lineno, size_t offset, jl_value_t *options);
906 : :
907 : : //--------------------------------------------------
908 : : // Backtraces
909 : :
910 : : // Backtrace buffers:
911 : : //
912 : : // A backtrace buffer conceptually contains a stack of instruction pointers
913 : : // ordered from the inner-most frame to the outermost. We store them in a
914 : : // special raw format for two reasons:
915 : : //
916 : : // * Efficiency: Every `throw()` must populate the trace so it must be as
917 : : // efficient as possible.
918 : : // * Signal safety: For signal-based exceptions such as StackOverflowError
919 : : // the trace buffer needs to be filled from a signal handler where most
920 : : // operations are not allowed (including malloc) so we choose a flat
921 : : // preallocated buffer.
922 : : //
923 : : // The raw buffer layout contains "frame entries" composed of one or several
924 : : // values of type `jl_bt_element_t`. From the point of view of the GC, an entry
925 : : // is either:
926 : : //
927 : : // 1. A single instruction pointer to native code, not GC-managed.
928 : : // 2. An "extended entry": a mixture of raw data and pointers to julia objects
929 : : // which must be treated as GC roots.
930 : : //
931 : : // A single extended entry is seralized using multiple elements from the raw
932 : : // buffer; if `e` is the pointer to the first slot we have:
933 : : //
934 : : // e[0] JL_BT_NON_PTR_ENTRY - Special marker to distinguish extended entries
935 : : // e[1] descriptor - A bit packed uintptr_t containing a tag and
936 : : // the number of GC- managed and non-managed values
937 : : // e[2+j] - GC managed data
938 : : // e[2+ngc+i] - Non-GC-managed data
939 : : //
940 : : // The format of `descriptor` is, from LSB to MSB:
941 : : // 0:2 ngc Number of GC-managed pointers for this frame entry
942 : : // 3:5 nptr Number of non-GC-managed buffer elements
943 : : // 6:9 tag Entry type
944 : : // 10:... header Entry-specific header data
945 : : typedef struct _jl_bt_element_t {
946 : : union {
947 : : uintptr_t uintptr; // Metadata or native instruction ptr
948 : : jl_value_t* jlvalue; // Pointer to GC-managed value
949 : : };
950 : : } jl_bt_element_t;
951 : :
952 : : #define JL_BT_NON_PTR_ENTRY (((uintptr_t)0)-1)
953 : : // Maximum size for an extended backtrace entry (likely significantly larger
954 : : // than the actual size of 3-4 for an interpreter frame)
955 : : #define JL_BT_MAX_ENTRY_SIZE 16
956 : :
957 : 352 : STATIC_INLINE int jl_bt_is_native(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT
958 : : {
959 : 352 : return bt_entry[0].uintptr != JL_BT_NON_PTR_ENTRY;
960 : : }
961 : :
962 : : // Extended backtrace entry header packing; the bit packing is done manually
963 : : // for precise layout control for interop with julia side.
964 : 8773 : STATIC_INLINE uintptr_t jl_bt_entry_descriptor(int ngc, int nptr,
965 : : int tag, uintptr_t header) JL_NOTSAFEPOINT
966 : : {
967 [ + - + - : 8773 : assert(((ngc & 0x7) == ngc) && ((nptr & 0x7) == nptr) && ((tag & 0xf) == tag));
+ - ]
968 : 8773 : return (ngc & 0x7) | (nptr & 0x7) << 3 | (tag & 0xf) << 6 | header << 10;
969 : : }
970 : :
971 : : // Unpacking of extended backtrace entry data
972 : 8 : STATIC_INLINE size_t jl_bt_num_jlvals(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT
973 : : {
974 [ - + ]: 8 : assert(!jl_bt_is_native(bt_entry));
975 : 8 : return bt_entry[1].uintptr & 0x7;
976 : : }
977 : 4 : STATIC_INLINE size_t jl_bt_num_uintvals(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT
978 : : {
979 [ - + ]: 4 : assert(!jl_bt_is_native(bt_entry));
980 : 4 : return (bt_entry[1].uintptr >> 3) & 0x7;
981 : : }
982 : 0 : STATIC_INLINE int jl_bt_entry_tag(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT
983 : : {
984 [ # # ]: 0 : assert(!jl_bt_is_native(bt_entry));
985 : 0 : return (bt_entry[1].uintptr >> 6) & 0xf;
986 : : }
987 : 0 : STATIC_INLINE uintptr_t jl_bt_entry_header(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT
988 : : {
989 [ # # ]: 0 : assert(!jl_bt_is_native(bt_entry));
990 : 0 : return bt_entry[1].uintptr >> 10;
991 : : }
992 : :
993 : : // Return `i`th GC-managed pointer for extended backtrace entry
994 : : // The returned value is rooted for the lifetime of the parent exception stack.
995 : 8 : STATIC_INLINE jl_value_t *jl_bt_entry_jlvalue(jl_bt_element_t *bt_entry, size_t i) JL_NOTSAFEPOINT
996 : : {
997 : 8 : return bt_entry[2 + i].jlvalue;
998 : : }
999 : :
1000 : : #define JL_BT_INTERP_FRAME_TAG 1 // An interpreter frame
1001 : :
1002 : : // Number of bt elements in frame.
1003 : 170 : STATIC_INLINE size_t jl_bt_entry_size(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT
1004 : : {
1005 : 170 : return jl_bt_is_native(bt_entry) ?
1006 [ + + ]: 170 : 1 : 2 + jl_bt_num_jlvals(bt_entry) + jl_bt_num_uintvals(bt_entry);
1007 : : }
1008 : :
1009 : : //------------------------------
1010 : : // Stack walking and debug info lookup
1011 : :
1012 : : // Function metadata arising from debug info lookup of instruction pointer
1013 : : typedef struct {
1014 : : char *func_name;
1015 : : char *file_name;
1016 : : int line;
1017 : : jl_method_instance_t *linfo;
1018 : : int fromC;
1019 : : int inlined;
1020 : : } jl_frame_t;
1021 : :
1022 : : // Might be called from unmanaged thread
1023 : : uint64_t jl_getUnwindInfo(uint64_t dwBase);
1024 : : #ifdef _OS_WINDOWS_
1025 : : #include <dbghelp.h>
1026 : : JL_DLLEXPORT EXCEPTION_DISPOSITION NTAPI __julia_personality(
1027 : : PEXCEPTION_RECORD ExceptionRecord, void *EstablisherFrame, PCONTEXT ContextRecord, void *DispatcherContext);
1028 : : extern HANDLE hMainThread;
1029 : : typedef CONTEXT bt_context_t;
1030 : : #if defined(_CPU_X86_64_)
1031 : : typedef CONTEXT bt_cursor_t;
1032 : : #else
1033 : : typedef struct {
1034 : : STACKFRAME64 stackframe;
1035 : : CONTEXT context;
1036 : : } bt_cursor_t;
1037 : : #endif
1038 : : extern JL_DLLEXPORT uv_mutex_t jl_in_stackwalk;
1039 : : #elif !defined(JL_DISABLE_LIBUNWIND)
1040 : : // This gives unwind only local unwinding options ==> faster code
1041 : : # define UNW_LOCAL_ONLY
1042 : : # include <libunwind.h>
1043 : : typedef unw_context_t bt_context_t;
1044 : : typedef unw_cursor_t bt_cursor_t;
1045 : : # if (!defined(SYSTEM_LIBUNWIND) || UNW_VERSION_MAJOR > 1 || \
1046 : : (UNW_VERSION_MAJOR == 1 && UNW_VERSION_MINOR != 0 && UNW_VERSION_MINOR != 1))
1047 : : // Enable our memory manager only for libunwind with our patch or
1048 : : // on a newer release
1049 : : # define JL_UNW_HAS_FORMAT_IP 1
1050 : : # endif
1051 : : #else
1052 : : // Unwinding is disabled
1053 : : typedef int bt_context_t;
1054 : : typedef int bt_cursor_t;
1055 : : #endif
1056 : : size_t rec_backtrace(jl_bt_element_t *bt_data, size_t maxsize, int skip) JL_NOTSAFEPOINT;
1057 : : // Record backtrace from a signal handler. `ctx` is the context of the code
1058 : : // which was asynchronously interrupted.
1059 : : size_t rec_backtrace_ctx(jl_bt_element_t *bt_data, size_t maxsize, bt_context_t *ctx,
1060 : : jl_gcframe_t *pgcstack) JL_NOTSAFEPOINT;
1061 : : #ifdef LLVMLIBUNWIND
1062 : : size_t rec_backtrace_ctx_dwarf(jl_bt_element_t *bt_data, size_t maxsize, bt_context_t *ctx, jl_gcframe_t *pgcstack) JL_NOTSAFEPOINT;
1063 : : #endif
1064 : : JL_DLLEXPORT jl_value_t *jl_get_backtrace(void);
1065 : : void jl_critical_error(int sig, bt_context_t *context, jl_task_t *ct);
1066 : : JL_DLLEXPORT void jl_raise_debugger(void);
1067 : : int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int noInline) JL_NOTSAFEPOINT;
1068 : : JL_DLLEXPORT void jl_gdblookup(void* ip) JL_NOTSAFEPOINT;
1069 : : void jl_print_native_codeloc(uintptr_t ip) JL_NOTSAFEPOINT;
1070 : : void jl_print_bt_entry_codeloc(jl_bt_element_t *bt_data) JL_NOTSAFEPOINT;
1071 : : #ifdef _OS_WINDOWS_
1072 : : JL_DLLEXPORT void jl_refresh_dbg_module_list(void);
1073 : : #endif
1074 : : // *to is NULL or malloc'd pointer, from is allowed to be NULL
1075 : 1314 : STATIC_INLINE char *jl_copy_str(char **to, const char *from) JL_NOTSAFEPOINT
1076 : : {
1077 [ + + ]: 1314 : if (!from) {
1078 : 163 : free(*to);
1079 : 163 : *to = NULL;
1080 : 163 : return NULL;
1081 : : }
1082 : 1151 : size_t len = strlen(from) + 1;
1083 : 1151 : *to = (char*)realloc_s(*to, len);
1084 : 1151 : memcpy(*to, from, len);
1085 : 1151 : return *to;
1086 : : }
1087 : :
1088 : : JL_DLLEXPORT size_t jl_capture_interp_frame(jl_bt_element_t *bt_data,
1089 : : void *frameend, size_t space_remaining) JL_NOTSAFEPOINT;
1090 : :
1091 : : //--------------------------------------------------
1092 : : // Exception stack access and manipulation
1093 : :
1094 : : // Exception stack: a stack of pairs of (exception,raw_backtrace).
1095 : : // The stack may be traversed and accessed with the functions below.
1096 : : struct _jl_excstack_t { // typedef in julia.h
1097 : : size_t top;
1098 : : size_t reserved_size;
1099 : : // Pack all stack entries into a growable buffer to amortize allocation
1100 : : // across repeated exception handling.
1101 : : // Layout: [bt_data1... bt_size1 exc1 bt_data2... bt_size2 exc2 ..]
1102 : : // jl_bt_element_t data[]; // Access with jl_excstack_raw
1103 : : };
1104 : :
1105 : 9318 : STATIC_INLINE jl_bt_element_t *jl_excstack_raw(jl_excstack_t *stack) JL_NOTSAFEPOINT
1106 : : {
1107 : 9318 : return (jl_bt_element_t*)(stack + 1);
1108 : : }
1109 : :
1110 : : // Exception stack access
1111 : 66 : STATIC_INLINE jl_value_t *jl_excstack_exception(jl_excstack_t *stack JL_PROPAGATES_ROOT,
1112 : : size_t itr) JL_NOTSAFEPOINT
1113 : : {
1114 : 66 : return jl_excstack_raw(stack)[itr-1].jlvalue;
1115 : : }
1116 : 12 : STATIC_INLINE size_t jl_excstack_bt_size(jl_excstack_t *stack, size_t itr) JL_NOTSAFEPOINT
1117 : : {
1118 : 12 : return jl_excstack_raw(stack)[itr-2].uintptr;
1119 : : }
1120 : 4 : STATIC_INLINE jl_bt_element_t *jl_excstack_bt_data(jl_excstack_t *stack, size_t itr) JL_NOTSAFEPOINT
1121 : : {
1122 : 4 : return jl_excstack_raw(stack) + itr-2 - jl_excstack_bt_size(stack, itr);
1123 : : }
1124 : : // Exception stack iteration (start at itr=stack->top, stop at itr=0)
1125 : 4 : STATIC_INLINE size_t jl_excstack_next(jl_excstack_t *stack, size_t itr) JL_NOTSAFEPOINT
1126 : : {
1127 : 4 : return itr-2 - jl_excstack_bt_size(stack, itr);
1128 : : }
1129 : : // Exception stack manipulation
1130 : : void jl_push_excstack(jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_ARGUMENT,
1131 : : jl_value_t *exception JL_ROOTED_ARGUMENT,
1132 : : jl_bt_element_t *bt_data, size_t bt_size);
1133 : :
1134 : : //--------------------------------------------------
1135 : : // congruential random number generator
1136 : : // for a small amount of thread-local randomness
1137 : : STATIC_INLINE void unbias_cong(uint64_t max, uint64_t *unbias) JL_NOTSAFEPOINT
1138 : : {
1139 : : *unbias = UINT64_MAX - ((UINT64_MAX % max) + 1);
1140 : : }
1141 : 77 : STATIC_INLINE uint64_t cong(uint64_t max, uint64_t unbias, uint64_t *seed) JL_NOTSAFEPOINT
1142 : : {
1143 [ - + ]: 77 : while ((*seed = 69069 * (*seed) + 362437) > unbias)
1144 : : ;
1145 : 77 : return *seed % max;
1146 : : }
1147 : : JL_DLLEXPORT uint64_t jl_rand(void) JL_NOTSAFEPOINT;
1148 : : JL_DLLEXPORT void jl_srand(uint64_t) JL_NOTSAFEPOINT;
1149 : : JL_DLLEXPORT void jl_init_rand(void);
1150 : :
1151 : : JL_DLLEXPORT extern void *jl_libjulia_internal_handle;
1152 : : JL_DLLEXPORT extern void *jl_RTLD_DEFAULT_handle;
1153 : : #if defined(_OS_WINDOWS_)
1154 : : JL_DLLEXPORT extern void *jl_exe_handle;
1155 : : JL_DLLEXPORT extern void *jl_libjulia_handle;
1156 : : JL_DLLEXPORT extern const char *jl_crtdll_basename;
1157 : : extern void *jl_ntdll_handle;
1158 : : extern void *jl_kernel32_handle;
1159 : : extern void *jl_crtdll_handle;
1160 : : extern void *jl_winsock_handle;
1161 : : #endif
1162 : :
1163 : : JL_DLLEXPORT void *jl_get_library_(const char *f_lib, int throw_err);
1164 : : #define jl_get_library(f_lib) jl_get_library_(f_lib, 1)
1165 : : JL_DLLEXPORT void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *hnd);
1166 : : JL_DLLEXPORT void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name);
1167 : : JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline(
1168 : : jl_value_t *fobj, jl_datatype_t *result, htable_t *cache, jl_svec_t *fill,
1169 : : void *(*init_trampoline)(void *tramp, void **nval),
1170 : : jl_unionall_t *env, jl_value_t **vals);
1171 : :
1172 : :
1173 : : // Windows only
1174 : : #define JL_EXE_LIBNAME ((const char*)1)
1175 : : #define JL_LIBJULIA_DL_LIBNAME ((const char*)2)
1176 : : #define JL_LIBJULIA_INTERNAL_DL_LIBNAME ((const char*)3)
1177 : : JL_DLLEXPORT const char *jl_dlfind_win32(const char *name);
1178 : :
1179 : : // libuv wrappers:
1180 : : JL_DLLEXPORT int jl_fs_rename(const char *src_path, const char *dst_path);
1181 : :
1182 : : #ifdef SEGV_EXCEPTION
1183 : : extern JL_DLLEXPORT jl_value_t *jl_segv_exception;
1184 : : #endif
1185 : :
1186 : : // -- Runtime intrinsics -- //
1187 : : JL_DLLEXPORT const char *jl_intrinsic_name(int f) JL_NOTSAFEPOINT;
1188 : : JL_DLLEXPORT unsigned jl_intrinsic_nargs(int f) JL_NOTSAFEPOINT;
1189 : :
1190 : 19747 : STATIC_INLINE int is_valid_intrinsic_elptr(jl_value_t *ety)
1191 : : {
1192 [ + - + - : 19747 : return ety == (jl_value_t*)jl_any_type || (jl_is_concrete_type(ety) && !jl_is_layout_opaque(((jl_datatype_t*)ety)->layout));
+ - ]
1193 : : }
1194 : : JL_DLLEXPORT jl_value_t *jl_bitcast(jl_value_t *ty, jl_value_t *v);
1195 : : JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align);
1196 : : JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *align, jl_value_t *i);
1197 : : JL_DLLEXPORT jl_value_t *jl_atomic_fence(jl_value_t *order);
1198 : : JL_DLLEXPORT jl_value_t *jl_atomic_pointerref(jl_value_t *p, jl_value_t *order);
1199 : : JL_DLLEXPORT jl_value_t *jl_atomic_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *order);
1200 : : JL_DLLEXPORT jl_value_t *jl_atomic_pointerswap(jl_value_t *p, jl_value_t *x, jl_value_t *order);
1201 : : JL_DLLEXPORT jl_value_t *jl_atomic_pointermodify(jl_value_t *p, jl_value_t *f, jl_value_t *x, jl_value_t *order);
1202 : : JL_DLLEXPORT jl_value_t *jl_atomic_pointerreplace(jl_value_t *p, jl_value_t *x, jl_value_t *expected, jl_value_t *success_order, jl_value_t *failure_order);
1203 : : JL_DLLEXPORT jl_value_t *jl_cglobal(jl_value_t *v, jl_value_t *ty);
1204 : : JL_DLLEXPORT jl_value_t *jl_cglobal_auto(jl_value_t *v);
1205 : :
1206 : : JL_DLLEXPORT jl_value_t *jl_neg_int(jl_value_t *a);
1207 : : JL_DLLEXPORT jl_value_t *jl_add_int(jl_value_t *a, jl_value_t *b);
1208 : : JL_DLLEXPORT jl_value_t *jl_sub_int(jl_value_t *a, jl_value_t *b);
1209 : : JL_DLLEXPORT jl_value_t *jl_mul_int(jl_value_t *a, jl_value_t *b);
1210 : : JL_DLLEXPORT jl_value_t *jl_sdiv_int(jl_value_t *a, jl_value_t *b);
1211 : : JL_DLLEXPORT jl_value_t *jl_udiv_int(jl_value_t *a, jl_value_t *b);
1212 : : JL_DLLEXPORT jl_value_t *jl_srem_int(jl_value_t *a, jl_value_t *b);
1213 : : JL_DLLEXPORT jl_value_t *jl_urem_int(jl_value_t *a, jl_value_t *b);
1214 : :
1215 : : JL_DLLEXPORT jl_value_t *jl_add_ptr(jl_value_t *a, jl_value_t *b);
1216 : : JL_DLLEXPORT jl_value_t *jl_sub_ptr(jl_value_t *a, jl_value_t *b);
1217 : :
1218 : : JL_DLLEXPORT jl_value_t *jl_neg_float(jl_value_t *a);
1219 : : JL_DLLEXPORT jl_value_t *jl_add_float(jl_value_t *a, jl_value_t *b);
1220 : : JL_DLLEXPORT jl_value_t *jl_sub_float(jl_value_t *a, jl_value_t *b);
1221 : : JL_DLLEXPORT jl_value_t *jl_mul_float(jl_value_t *a, jl_value_t *b);
1222 : : JL_DLLEXPORT jl_value_t *jl_div_float(jl_value_t *a, jl_value_t *b);
1223 : : JL_DLLEXPORT jl_value_t *jl_rem_float(jl_value_t *a, jl_value_t *b);
1224 : : JL_DLLEXPORT jl_value_t *jl_fma_float(jl_value_t *a, jl_value_t *b, jl_value_t *c);
1225 : : JL_DLLEXPORT jl_value_t *jl_muladd_float(jl_value_t *a, jl_value_t *b, jl_value_t *c);
1226 : :
1227 : : JL_DLLEXPORT jl_value_t *jl_eq_int(jl_value_t *a, jl_value_t *b);
1228 : : JL_DLLEXPORT jl_value_t *jl_ne_int(jl_value_t *a, jl_value_t *b);
1229 : : JL_DLLEXPORT jl_value_t *jl_slt_int(jl_value_t *a, jl_value_t *b);
1230 : : JL_DLLEXPORT jl_value_t *jl_ult_int(jl_value_t *a, jl_value_t *b);
1231 : : JL_DLLEXPORT jl_value_t *jl_sle_int(jl_value_t *a, jl_value_t *b);
1232 : : JL_DLLEXPORT jl_value_t *jl_ule_int(jl_value_t *a, jl_value_t *b);
1233 : :
1234 : : JL_DLLEXPORT jl_value_t *jl_eq_float(jl_value_t *a, jl_value_t *b);
1235 : : JL_DLLEXPORT jl_value_t *jl_ne_float(jl_value_t *a, jl_value_t *b);
1236 : : JL_DLLEXPORT jl_value_t *jl_lt_float(jl_value_t *a, jl_value_t *b);
1237 : : JL_DLLEXPORT jl_value_t *jl_le_float(jl_value_t *a, jl_value_t *b);
1238 : : JL_DLLEXPORT jl_value_t *jl_fpiseq(jl_value_t *a, jl_value_t *b);
1239 : :
1240 : : JL_DLLEXPORT jl_value_t *jl_not_int(jl_value_t *a);
1241 : : JL_DLLEXPORT jl_value_t *jl_and_int(jl_value_t *a, jl_value_t *b);
1242 : : JL_DLLEXPORT jl_value_t *jl_or_int(jl_value_t *a, jl_value_t *b);
1243 : : JL_DLLEXPORT jl_value_t *jl_xor_int(jl_value_t *a, jl_value_t *b);
1244 : : JL_DLLEXPORT jl_value_t *jl_shl_int(jl_value_t *a, jl_value_t *b);
1245 : : JL_DLLEXPORT jl_value_t *jl_lshr_int(jl_value_t *a, jl_value_t *b);
1246 : : JL_DLLEXPORT jl_value_t *jl_ashr_int(jl_value_t *a, jl_value_t *b);
1247 : : JL_DLLEXPORT jl_value_t *jl_bswap_int(jl_value_t *a);
1248 : : JL_DLLEXPORT jl_value_t *jl_ctpop_int(jl_value_t *a);
1249 : : JL_DLLEXPORT jl_value_t *jl_ctlz_int(jl_value_t *a);
1250 : : JL_DLLEXPORT jl_value_t *jl_cttz_int(jl_value_t *a);
1251 : :
1252 : : JL_DLLEXPORT jl_value_t *jl_sext_int(jl_value_t *ty, jl_value_t *a);
1253 : : JL_DLLEXPORT jl_value_t *jl_zext_int(jl_value_t *ty, jl_value_t *a);
1254 : : JL_DLLEXPORT jl_value_t *jl_trunc_int(jl_value_t *ty, jl_value_t *a);
1255 : : JL_DLLEXPORT jl_value_t *jl_sitofp(jl_value_t *ty, jl_value_t *a);
1256 : : JL_DLLEXPORT jl_value_t *jl_uitofp(jl_value_t *ty, jl_value_t *a);
1257 : :
1258 : : JL_DLLEXPORT jl_value_t *jl_fptoui(jl_value_t *ty, jl_value_t *a);
1259 : : JL_DLLEXPORT jl_value_t *jl_fptosi(jl_value_t *ty, jl_value_t *a);
1260 : : JL_DLLEXPORT jl_value_t *jl_fptrunc(jl_value_t *ty, jl_value_t *a);
1261 : : JL_DLLEXPORT jl_value_t *jl_fpext(jl_value_t *ty, jl_value_t *a);
1262 : :
1263 : : JL_DLLEXPORT jl_value_t *jl_checked_sadd_int(jl_value_t *a, jl_value_t *b);
1264 : : JL_DLLEXPORT jl_value_t *jl_checked_uadd_int(jl_value_t *a, jl_value_t *b);
1265 : : JL_DLLEXPORT jl_value_t *jl_checked_ssub_int(jl_value_t *a, jl_value_t *b);
1266 : : JL_DLLEXPORT jl_value_t *jl_checked_usub_int(jl_value_t *a, jl_value_t *b);
1267 : : JL_DLLEXPORT jl_value_t *jl_checked_smul_int(jl_value_t *a, jl_value_t *b);
1268 : : JL_DLLEXPORT jl_value_t *jl_checked_umul_int(jl_value_t *a, jl_value_t *b);
1269 : : JL_DLLEXPORT jl_value_t *jl_checked_sdiv_int(jl_value_t *a, jl_value_t *b);
1270 : : JL_DLLEXPORT jl_value_t *jl_checked_udiv_int(jl_value_t *a, jl_value_t *b);
1271 : : JL_DLLEXPORT jl_value_t *jl_checked_srem_int(jl_value_t *a, jl_value_t *b);
1272 : : JL_DLLEXPORT jl_value_t *jl_checked_urem_int(jl_value_t *a, jl_value_t *b);
1273 : :
1274 : : JL_DLLEXPORT jl_value_t *jl_ceil_llvm(jl_value_t *a);
1275 : : JL_DLLEXPORT jl_value_t *jl_floor_llvm(jl_value_t *a);
1276 : : JL_DLLEXPORT jl_value_t *jl_trunc_llvm(jl_value_t *a);
1277 : : JL_DLLEXPORT jl_value_t *jl_rint_llvm(jl_value_t *a);
1278 : : JL_DLLEXPORT jl_value_t *jl_sqrt_llvm(jl_value_t *a);
1279 : : JL_DLLEXPORT jl_value_t *jl_sqrt_llvm_fast(jl_value_t *a);
1280 : : JL_DLLEXPORT jl_value_t *jl_abs_float(jl_value_t *a);
1281 : : JL_DLLEXPORT jl_value_t *jl_copysign_float(jl_value_t *a, jl_value_t *b);
1282 : : JL_DLLEXPORT jl_value_t *jl_flipsign_int(jl_value_t *a, jl_value_t *b);
1283 : :
1284 : : JL_DLLEXPORT jl_value_t *jl_arraylen(jl_value_t *a);
1285 : : JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *a);
1286 : : JL_DLLEXPORT int jl_stored_inline(jl_value_t *el_type);
1287 : : JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a);
1288 : : JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i);
1289 : : JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary);
1290 : :
1291 : : JL_DLLEXPORT uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) JL_NOTSAFEPOINT;
1292 : : JL_DLLEXPORT void jl_set_next_task(jl_task_t *task) JL_NOTSAFEPOINT;
1293 : :
1294 : : // -- synchronization utilities -- //
1295 : :
1296 : : extern jl_mutex_t typecache_lock;
1297 : : extern JL_DLLEXPORT jl_mutex_t jl_codegen_lock;
1298 : :
1299 : : #if defined(__APPLE__)
1300 : : void jl_mach_gc_end(void);
1301 : : #endif
1302 : :
1303 : : // -- smallintset.c -- //
1304 : :
1305 : : typedef uint_t (*smallintset_hash)(size_t val, jl_svec_t *data);
1306 : : typedef int (*smallintset_eq)(size_t val, const void *key, jl_svec_t *data, uint_t hv);
1307 : : ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv);
1308 : : void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data);
1309 : :
1310 : : // -- typemap.c -- //
1311 : :
1312 : : void jl_typemap_insert(_Atomic(jl_typemap_t*) *cache, jl_value_t *parent,
1313 : : jl_typemap_entry_t *newrec, int8_t offs);
1314 : : jl_typemap_entry_t *jl_typemap_alloc(
1315 : : jl_tupletype_t *type, jl_tupletype_t *simpletype, jl_svec_t *guardsigs,
1316 : : jl_value_t *newvalue, size_t min_world, size_t max_world);
1317 : :
1318 : : struct jl_typemap_assoc {
1319 : : // inputs
1320 : : jl_value_t *const types;
1321 : : size_t const world;
1322 : : // outputs
1323 : : jl_svec_t *env; // subtype env (initialize to null to perform intersection without an environment)
1324 : : size_t min_valid;
1325 : : size_t max_valid;
1326 : : };
1327 : :
1328 : : jl_typemap_entry_t *jl_typemap_assoc_by_type(
1329 : : jl_typemap_t *ml_or_cache JL_PROPAGATES_ROOT,
1330 : : struct jl_typemap_assoc *search,
1331 : : int8_t offs, uint8_t subtype);
1332 : :
1333 : : jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world);
1334 : : jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *mn, jl_value_t *arg1, jl_value_t **args, size_t n, size_t world);
1335 : 159152400 : STATIC_INLINE jl_typemap_entry_t *jl_typemap_assoc_exact(
1336 : : jl_typemap_t *ml_or_cache JL_PROPAGATES_ROOT,
1337 : : jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world)
1338 : : {
1339 : : // NOTE: This function is a huge performance hot spot!!
1340 [ + + ]: 159152400 : if (jl_typeof(ml_or_cache) == (jl_value_t *)jl_typemap_entry_type) {
1341 : 126811400 : return jl_typemap_entry_assoc_exact(
1342 : : (jl_typemap_entry_t *)ml_or_cache, arg1, args, n, world);
1343 : : }
1344 [ + + ]: 32341000 : else if (jl_typeof(ml_or_cache) == (jl_value_t*)jl_typemap_level_type) {
1345 : 14069160 : return jl_typemap_level_assoc_exact(
1346 : : (jl_typemap_level_t *)ml_or_cache, arg1, args, n, offs, world);
1347 : : }
1348 : 18271950 : return NULL;
1349 : : }
1350 : : typedef int (*jl_typemap_visitor_fptr)(jl_typemap_entry_t *l, void *closure);
1351 : : int jl_typemap_visitor(jl_typemap_t *a, jl_typemap_visitor_fptr fptr, void *closure);
1352 : :
1353 : : struct typemap_intersection_env;
1354 : : typedef int (*jl_typemap_intersection_visitor_fptr)(jl_typemap_entry_t *l, struct typemap_intersection_env *closure);
1355 : : struct typemap_intersection_env {
1356 : : // input values
1357 : : jl_typemap_intersection_visitor_fptr const fptr; // fptr to call on a match
1358 : : jl_value_t *const type; // type to match
1359 : : jl_value_t *const va; // the tparam0 for the vararg in type, if applicable (or NULL)
1360 : : // output values
1361 : : jl_value_t *ti; // intersection type
1362 : : jl_svec_t *env; // intersection env (initialize to null to perform intersection without an environment)
1363 : : int issubty; // if `a <: b` is true in `intersect(a,b)`
1364 : : };
1365 : : int jl_typemap_intersection_visitor(jl_typemap_t *a, int offs, struct typemap_intersection_env *closure);
1366 : :
1367 : : // -- simplevector.c -- //
1368 : :
1369 : : // For codegen only.
1370 : : JL_DLLEXPORT size_t (jl_svec_len)(jl_svec_t *t) JL_NOTSAFEPOINT;
1371 : : JL_DLLEXPORT int8_t jl_svec_isassigned(jl_svec_t *t JL_PROPAGATES_ROOT, ssize_t i) JL_NOTSAFEPOINT;
1372 : : JL_DLLEXPORT jl_value_t *jl_svec_ref(jl_svec_t *t JL_PROPAGATES_ROOT, ssize_t i);
1373 : :
1374 : :
1375 : : JL_DLLEXPORT unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *field_type);
1376 : :
1377 : : void register_eh_frames(uint8_t *Addr, size_t Size);
1378 : : void deregister_eh_frames(uint8_t *Addr, size_t Size);
1379 : :
1380 : 0 : STATIC_INLINE void *jl_get_frame_addr(void)
1381 : : {
1382 : : #ifdef __GNUC__
1383 : 0 : return __builtin_frame_address(0);
1384 : : #else
1385 : : void *dummy = NULL;
1386 : : // The mask is to suppress the compiler warning about returning
1387 : : // address of local variable
1388 : : return (void*)((uintptr_t)&dummy & ~(uintptr_t)15);
1389 : : #endif
1390 : : }
1391 : :
1392 : : JL_DLLEXPORT jl_array_t *jl_array_cconvert_cstring(jl_array_t *a);
1393 : :
1394 : : // Log `msg` to the current logger by calling CoreLogging.logmsg_shim() on the
1395 : : // julia side. If any of module, group, id, file or line are NULL, these will
1396 : : // be passed to the julia side as `nothing`. If `kwargs` is NULL an empty set
1397 : : // of keyword arguments will be passed.
1398 : : void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id,
1399 : : jl_value_t *file, jl_value_t *line, jl_value_t *kwargs,
1400 : : jl_value_t *msg);
1401 : :
1402 : : JL_DLLEXPORT int jl_isabspath(const char *in) JL_NOTSAFEPOINT;
1403 : :
1404 : : extern JL_DLLEXPORT jl_sym_t *jl_call_sym;
1405 : : extern JL_DLLEXPORT jl_sym_t *jl_invoke_sym;
1406 : : extern JL_DLLEXPORT jl_sym_t *jl_invoke_modify_sym;
1407 : : extern JL_DLLEXPORT jl_sym_t *jl_empty_sym;
1408 : : extern JL_DLLEXPORT jl_sym_t *jl_top_sym;
1409 : : extern JL_DLLEXPORT jl_sym_t *jl_module_sym;
1410 : : extern JL_DLLEXPORT jl_sym_t *jl_slot_sym;
1411 : : extern JL_DLLEXPORT jl_sym_t *jl_export_sym;
1412 : : extern JL_DLLEXPORT jl_sym_t *jl_import_sym;
1413 : : extern JL_DLLEXPORT jl_sym_t *jl_toplevel_sym;
1414 : : extern JL_DLLEXPORT jl_sym_t *jl_quote_sym;
1415 : : extern JL_DLLEXPORT jl_sym_t *jl_line_sym;
1416 : : extern JL_DLLEXPORT jl_sym_t *jl_incomplete_sym;
1417 : : extern JL_DLLEXPORT jl_sym_t *jl_goto_sym;
1418 : : extern JL_DLLEXPORT jl_sym_t *jl_goto_ifnot_sym;
1419 : : extern JL_DLLEXPORT jl_sym_t *jl_return_sym;
1420 : : extern JL_DLLEXPORT jl_sym_t *jl_lineinfo_sym;
1421 : : extern JL_DLLEXPORT jl_sym_t *jl_lambda_sym;
1422 : : extern JL_DLLEXPORT jl_sym_t *jl_assign_sym;
1423 : : extern JL_DLLEXPORT jl_sym_t *jl_globalref_sym;
1424 : : extern JL_DLLEXPORT jl_sym_t *jl_do_sym;
1425 : : extern JL_DLLEXPORT jl_sym_t *jl_method_sym;
1426 : : extern JL_DLLEXPORT jl_sym_t *jl_core_sym;
1427 : : extern JL_DLLEXPORT jl_sym_t *jl_enter_sym;
1428 : : extern JL_DLLEXPORT jl_sym_t *jl_leave_sym;
1429 : : extern JL_DLLEXPORT jl_sym_t *jl_pop_exception_sym;
1430 : : extern JL_DLLEXPORT jl_sym_t *jl_exc_sym;
1431 : : extern JL_DLLEXPORT jl_sym_t *jl_error_sym;
1432 : : extern JL_DLLEXPORT jl_sym_t *jl_new_sym;
1433 : : extern JL_DLLEXPORT jl_sym_t *jl_using_sym;
1434 : : extern JL_DLLEXPORT jl_sym_t *jl_splatnew_sym;
1435 : : extern JL_DLLEXPORT jl_sym_t *jl_block_sym;
1436 : : extern JL_DLLEXPORT jl_sym_t *jl_new_opaque_closure_sym;
1437 : : extern JL_DLLEXPORT jl_sym_t *jl_opaque_closure_method_sym;
1438 : : extern JL_DLLEXPORT jl_sym_t *jl_const_sym;
1439 : : extern JL_DLLEXPORT jl_sym_t *jl_thunk_sym;
1440 : : extern JL_DLLEXPORT jl_sym_t *jl_foreigncall_sym;
1441 : : extern JL_DLLEXPORT jl_sym_t *jl_as_sym;
1442 : : extern JL_DLLEXPORT jl_sym_t *jl_global_sym;
1443 : : extern JL_DLLEXPORT jl_sym_t *jl_list_sym;
1444 : : extern JL_DLLEXPORT jl_sym_t *jl_dot_sym;
1445 : : extern JL_DLLEXPORT jl_sym_t *jl_newvar_sym;
1446 : : extern JL_DLLEXPORT jl_sym_t *jl_boundscheck_sym;
1447 : : extern JL_DLLEXPORT jl_sym_t *jl_inbounds_sym;
1448 : : extern JL_DLLEXPORT jl_sym_t *jl_copyast_sym;
1449 : : extern JL_DLLEXPORT jl_sym_t *jl_cfunction_sym;
1450 : : extern JL_DLLEXPORT jl_sym_t *jl_pure_sym;
1451 : : extern JL_DLLEXPORT jl_sym_t *jl_loopinfo_sym;
1452 : : extern JL_DLLEXPORT jl_sym_t *jl_meta_sym;
1453 : : extern JL_DLLEXPORT jl_sym_t *jl_inert_sym;
1454 : : extern JL_DLLEXPORT jl_sym_t *jl_polly_sym;
1455 : : extern JL_DLLEXPORT jl_sym_t *jl_unused_sym;
1456 : : extern JL_DLLEXPORT jl_sym_t *jl_static_parameter_sym;
1457 : : extern JL_DLLEXPORT jl_sym_t *jl_inline_sym;
1458 : : extern JL_DLLEXPORT jl_sym_t *jl_noinline_sym;
1459 : : extern JL_DLLEXPORT jl_sym_t *jl_generated_sym;
1460 : : extern JL_DLLEXPORT jl_sym_t *jl_generated_only_sym;
1461 : : extern JL_DLLEXPORT jl_sym_t *jl_isdefined_sym;
1462 : : extern JL_DLLEXPORT jl_sym_t *jl_propagate_inbounds_sym;
1463 : : extern JL_DLLEXPORT jl_sym_t *jl_specialize_sym;
1464 : : extern JL_DLLEXPORT jl_sym_t *jl_aggressive_constprop_sym;
1465 : : extern JL_DLLEXPORT jl_sym_t *jl_no_constprop_sym;
1466 : : extern JL_DLLEXPORT jl_sym_t *jl_purity_sym;
1467 : : extern JL_DLLEXPORT jl_sym_t *jl_nospecialize_sym;
1468 : : extern JL_DLLEXPORT jl_sym_t *jl_macrocall_sym;
1469 : : extern JL_DLLEXPORT jl_sym_t *jl_colon_sym;
1470 : : extern JL_DLLEXPORT jl_sym_t *jl_hygienicscope_sym;
1471 : : extern JL_DLLEXPORT jl_sym_t *jl_throw_undef_if_not_sym;
1472 : : extern JL_DLLEXPORT jl_sym_t *jl_getfield_undefref_sym;
1473 : : extern JL_DLLEXPORT jl_sym_t *jl_gc_preserve_begin_sym;
1474 : : extern JL_DLLEXPORT jl_sym_t *jl_gc_preserve_end_sym;
1475 : : extern JL_DLLEXPORT jl_sym_t *jl_coverageeffect_sym;
1476 : : extern JL_DLLEXPORT jl_sym_t *jl_escape_sym;
1477 : : extern JL_DLLEXPORT jl_sym_t *jl_aliasscope_sym;
1478 : : extern JL_DLLEXPORT jl_sym_t *jl_popaliasscope_sym;
1479 : : extern JL_DLLEXPORT jl_sym_t *jl_optlevel_sym;
1480 : : extern JL_DLLEXPORT jl_sym_t *jl_thismodule_sym;
1481 : : extern JL_DLLEXPORT jl_sym_t *jl_atom_sym;
1482 : : extern JL_DLLEXPORT jl_sym_t *jl_statement_sym;
1483 : : extern JL_DLLEXPORT jl_sym_t *jl_all_sym;
1484 : : extern JL_DLLEXPORT jl_sym_t *jl_compile_sym;
1485 : : extern JL_DLLEXPORT jl_sym_t *jl_force_compile_sym;
1486 : : extern JL_DLLEXPORT jl_sym_t *jl_infer_sym;
1487 : : extern JL_DLLEXPORT jl_sym_t *jl_max_methods_sym;
1488 : : extern JL_DLLEXPORT jl_sym_t *jl_atomic_sym;
1489 : : extern JL_DLLEXPORT jl_sym_t *jl_not_atomic_sym;
1490 : : extern JL_DLLEXPORT jl_sym_t *jl_unordered_sym;
1491 : : extern JL_DLLEXPORT jl_sym_t *jl_monotonic_sym;
1492 : : extern JL_DLLEXPORT jl_sym_t *jl_acquire_sym;
1493 : : extern JL_DLLEXPORT jl_sym_t *jl_release_sym;
1494 : : extern JL_DLLEXPORT jl_sym_t *jl_acquire_release_sym;
1495 : : extern JL_DLLEXPORT jl_sym_t *jl_sequentially_consistent_sym;
1496 : :
1497 : : JL_DLLEXPORT enum jl_memory_order jl_get_atomic_order(jl_sym_t *order, char loading, char storing);
1498 : : JL_DLLEXPORT enum jl_memory_order jl_get_atomic_order_checked(jl_sym_t *order, char loading, char storing);
1499 : :
1500 : : struct _jl_sysimg_fptrs_t;
1501 : :
1502 : : void jl_register_fptrs(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs,
1503 : : jl_method_instance_t **linfos, size_t n);
1504 : : void jl_write_coverage_data(const char*);
1505 : : void jl_write_malloc_log(void);
1506 : : void jl_write_compiler_output(void);
1507 : :
1508 : : #if jl_has_builtin(__builtin_unreachable) || defined(_COMPILER_GCC_) || defined(_COMPILER_INTEL_)
1509 : : # define jl_unreachable() __builtin_unreachable()
1510 : : #else
1511 : : # define jl_unreachable() ((void)jl_assume(0))
1512 : : #endif
1513 : :
1514 : : jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT;
1515 : :
1516 : : // Tools for locally disabling spurious compiler warnings
1517 : : //
1518 : : // Particular calls which are used elsewhere in the code include:
1519 : : //
1520 : : // * JL_GCC_IGNORE_START(-Wclobbered) - gcc misidentifies some variables which
1521 : : // are used inside a JL_TRY as being "clobbered" if JL_CATCH is entered. This
1522 : : // warning is spurious if the variable is not modified inside the JL_TRY.
1523 : : // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65041
1524 : : #ifdef _COMPILER_GCC_
1525 : : #define JL_DO_PRAGMA(s) _Pragma(#s)
1526 : : #define JL_GCC_IGNORE_START(warning) \
1527 : : JL_DO_PRAGMA(GCC diagnostic push) \
1528 : : JL_DO_PRAGMA(GCC diagnostic ignored warning)
1529 : : #define JL_GCC_IGNORE_STOP \
1530 : : JL_DO_PRAGMA(GCC diagnostic pop)
1531 : : #else
1532 : : #define JL_GCC_IGNORE_START(w)
1533 : : #define JL_GCC_IGNORE_STOP
1534 : : #endif // _COMPILER_GCC_
1535 : :
1536 : : #ifdef __clang_gcanalyzer__
1537 : : // Not a safepoint (so it dosn't free other values), but an artificial use.
1538 : : // Usually this is unnecessary because the analyzer can see all real uses,
1539 : : // but sometimes real uses are harder for the analyzer to see, or it may
1540 : : // give up before it sees it, so this can be helpful to be explicit.
1541 : : void JL_GC_ASSERT_LIVE(jl_value_t *v) JL_NOTSAFEPOINT;
1542 : : #else
1543 : : #define JL_GC_ASSERT_LIVE(x) (void)(x)
1544 : : #endif
1545 : :
1546 : : float __gnu_h2f_ieee(uint16_t param) JL_NOTSAFEPOINT;
1547 : : uint16_t __gnu_f2h_ieee(float param) JL_NOTSAFEPOINT;
1548 : :
1549 : : #ifdef __cplusplus
1550 : : }
1551 : : #endif
1552 : :
1553 : : #ifdef USE_DTRACE
1554 : : #include "uprobes.h.gen"
1555 : :
1556 : : // uprobes.h.gen on systems with DTrace, is auto-generated to include
1557 : : // `JL_PROBE_{PROBE}` and `JL_PROBE_{PROBE}_ENABLED()` macros for every probe
1558 : : // defined in uprobes.d
1559 : : //
1560 : : // If the arguments to `JL_PROBE_{PROBE}` are expensive to compute, the call to
1561 : : // these functions must be guarded by a JL_PROBE_{PROBE}_ENABLED() check, to
1562 : : // minimize performance impact when probing is off. As an example:
1563 : : //
1564 : : // if (JL_PROBE_GC_STOP_THE_WORLD_ENABLED())
1565 : : // JL_PROBE_GC_STOP_THE_WORLD();
1566 : :
1567 : : #else
1568 : : // define a dummy version of the probe functions
1569 : : #define JL_PROBE_GC_BEGIN(collection) do ; while (0)
1570 : : #define JL_PROBE_GC_STOP_THE_WORLD() do ; while (0)
1571 : : #define JL_PROBE_GC_MARK_BEGIN() do ; while (0)
1572 : : #define JL_PROBE_GC_MARK_END(scanned_bytes, perm_scanned_bytes) do ; while (0)
1573 : : #define JL_PROBE_GC_SWEEP_BEGIN(full) do ; while (0)
1574 : : #define JL_PROBE_GC_SWEEP_END() do ; while (0)
1575 : : #define JL_PROBE_GC_END() do ; while (0)
1576 : : #define JL_PROBE_GC_FINALIZER() do ; while (0)
1577 : : #define JL_PROBE_RT_RUN_TASK(task) do ; while (0)
1578 : : #define JL_PROBE_RT_PAUSE_TASK(task) do ; while (0)
1579 : : #define JL_PROBE_RT_NEW_TASK(parent, child) do ; while (0)
1580 : : #define JL_PROBE_RT_START_TASK(task) do ; while (0)
1581 : : #define JL_PROBE_RT_FINISH_TASK(task) do ; while (0)
1582 : : #define JL_PROBE_RT_START_PROCESS_EVENTS(task) do ; while (0)
1583 : : #define JL_PROBE_RT_FINISH_PROCESS_EVENTS(task) do ; while (0)
1584 : : #define JL_PROBE_RT_TASKQ_INSERT(ptls, task) do ; while (0)
1585 : : #define JL_PROBE_RT_TASKQ_GET(ptls, task) do ; while (0)
1586 : : #define JL_PROBE_RT_SLEEP_CHECK_WAKE(other, old_state) do ; while (0)
1587 : : #define JL_PROBE_RT_SLEEP_CHECK_WAKEUP(ptls) do ; while (0)
1588 : : #define JL_PROBE_RT_SLEEP_CHECK_SLEEP(ptls) do ; while (0)
1589 : : #define JL_PROBE_RT_SLEEP_CHECK_TASKQ_WAKE(ptls) do ; while (0)
1590 : : #define JL_PROBE_RT_SLEEP_CHECK_TASK_WAKE(ptls) do ; while (0)
1591 : : #define JL_PROBE_RT_SLEEP_CHECK_UV_WAKE(ptls) do ; while (0)
1592 : :
1593 : : #define JL_PROBE_GC_BEGIN_ENABLED() (0)
1594 : : #define JL_PROBE_GC_STOP_THE_WORLD_ENABLED() (0)
1595 : : #define JL_PROBE_GC_MARK_BEGIN_ENABLED() (0)
1596 : : #define JL_PROBE_GC_MARK_END_ENABLED() (0)
1597 : : #define JL_PROBE_GC_SWEEP_BEGIN_ENABLED() (0)
1598 : : #define JL_PROBE_GC_SWEEP_END_ENABLED() (0)
1599 : : #define JL_PROBE_GC_END_ENABLED() (0)
1600 : : #define JL_PROBE_GC_FINALIZER_ENABLED() (0)
1601 : : #define JL_PROBE_RT_RUN_TASK_ENABLED() (0)
1602 : : #define JL_PROBE_RT_PAUSE_TASK_ENABLED() (0)
1603 : : #define JL_PROBE_RT_NEW_TASK_ENABLED() (0)
1604 : : #define JL_PROBE_RT_START_TASK_ENABLED() (0)
1605 : : #define JL_PROBE_RT_FINISH_TASK_ENABLED() (0)
1606 : : #define JL_PROBE_RT_START_PROCESS_EVENTS_ENABLED() (0)
1607 : : #define JL_PROBE_RT_FINISH_PROCESS_EVENTS_ENABLED() (0)
1608 : : #define JL_PROBE_RT_TASKQ_INSERT_ENABLED() (0)
1609 : : #define JL_PROBE_RT_TASKQ_GET_ENABLED() (0)
1610 : : #define JL_PROBE_RT_SLEEP_CHECK_WAKE_ENABLED() (0)
1611 : : #define JL_PROBE_RT_SLEEP_CHECK_WAKEUP_ENABLED() (0)
1612 : : #define JL_PROBE_RT_SLEEP_CHECK_SLEEP_ENABLED() (0)
1613 : : #define JL_PROBE_RT_SLEEP_CHECK_TASKQ_WAKE_ENABLED() (0)
1614 : : #define JL_PROBE_RT_SLEEP_CHECK_TASK_WAKE_ENABLED() (0)
1615 : : #define JL_PROBE_RT_SLEEP_CHECK_UV_WAKE_ENABLED() (0)
1616 : : #endif
1617 : :
1618 : : #endif
|
