Branch data     Line data    Source code

       1                 :            : // This file is a part of Julia. License is MIT: https://julialang.org/license
       2                 :            : 
       3                 :            : // Meant to be included in <julia.h>
       4                 :            : #ifndef JL_THREADS_H
       5                 :            : #define JL_THREADS_H
       6                 :            : 
       7                 :            : #include "julia_atomics.h"
       8                 :            : #ifndef _OS_WINDOWS_
       9                 :            : #include "pthread.h"
      10                 :            : #endif
      11                 :            : // threading ------------------------------------------------------------------
      12                 :            : 
      13                 :            : #ifdef __cplusplus
      14                 :            : extern "C" {
      15                 :            : #endif
      16                 :            : 
      17                 :            : 
      18                 :            : JL_DLLEXPORT int16_t jl_threadid(void);
      19                 :            : JL_DLLEXPORT int8_t jl_threadpoolid(int16_t tid) JL_NOTSAFEPOINT;
      20                 :            : 
      21                 :            : // JULIA_ENABLE_THREADING may be controlled by altering JULIA_THREADS in Make.user
      22                 :            : 
      23                 :            : // When running into scheduler issues, this may help provide information on the
      24                 :            : // sequence of events that led to the issue. Normally, it is empty.
      25                 :            : //#define JULIA_DEBUG_SLEEPWAKE(x) x
      26                 :            : #define JULIA_DEBUG_SLEEPWAKE(x)
      27                 :            : 
      28                 :            : //  Options for task switching algorithm (in order of preference):
      29                 :            : // JL_HAVE_ASM -- mostly setjmp
      30                 :            : // JL_HAVE_ASM && JL_HAVE_UNW_CONTEXT -- libunwind-based
      31                 :            : // JL_HAVE_UNW_CONTEXT -- libunwind-based
      32                 :            : // JL_HAVE_ASYNCIFY -- task switching based on the binary asyncify transform
      33                 :            : // JL_HAVE_UCONTEXT -- posix standard API, requires syscall for resume
      34                 :            : // JL_HAVE_SIGALTSTACK -- requires several syscall for start, setjmp for resume
      35                 :            : 
      36                 :            : #ifdef _OS_WINDOWS_
      37                 :            : #define JL_HAVE_UCONTEXT
      38                 :            : typedef win32_ucontext_t jl_stack_context_t;
      39                 :            : typedef jl_stack_context_t _jl_ucontext_t;
      40                 :            : #else
      41                 :            : typedef struct {
      42                 :            :     jl_jmp_buf uc_mcontext;
      43                 :            : } jl_stack_context_t;
      44                 :            : #if !defined(JL_HAVE_UCONTEXT) && \
      45                 :            :     !defined(JL_HAVE_ASM) && \
      46                 :            :     !defined(JL_HAVE_UNW_CONTEXT) && \
      47                 :            :     !defined(JL_HAVE_SIGALTSTACK) && \
      48                 :            :     !defined(JL_HAVE_ASYNCIFY)
      49                 :            : #if (defined(_CPU_X86_64_) || defined(_CPU_X86_) || defined(_CPU_AARCH64_) ||  \
      50                 :            :      defined(_CPU_ARM_) || defined(_CPU_PPC64_))
      51                 :            : #define JL_HAVE_ASM
      52                 :            : #endif
      53                 :            : #if 0
      54                 :            : // very slow, but more debugging
      55                 :            : //#elif defined(_OS_DARWIN_)
      56                 :            : //#define JL_HAVE_UNW_CONTEXT
      57                 :            : //#elif defined(_OS_LINUX_)
      58                 :            : //#define JL_HAVE_UNW_CONTEXT
      59                 :            : #elif defined(_OS_EMSCRIPTEN_)
      60                 :            : #define JL_HAVE_ASYNCIFY
      61                 :            : #elif !defined(JL_HAVE_ASM)
      62                 :            : #define JL_HAVE_UNW_CONTEXT // optimistically?
      63                 :            : #endif
      64                 :            : #endif
      65                 :            : 
      66                 :            : #if (!defined(JL_HAVE_UNW_CONTEXT) && defined(JL_HAVE_ASM)) || defined(JL_HAVE_SIGALTSTACK)
      67                 :            : typedef jl_stack_context_t _jl_ucontext_t;
      68                 :            : #endif
      69                 :            : #if defined(JL_HAVE_ASYNCIFY)
      70                 :            : #if defined(_COMPILER_TSAN_ENABLED_)
      71                 :            : #error TSAN not currently supported with asyncify
      72                 :            : #endif
      73                 :            : typedef struct {
      74                 :            :     // This is the extent of the asyncify stack, but because the top of the
      75                 :            :     // asyncify stack (stacktop) is also the bottom of the C stack, we can
      76                 :            :     // reuse stacktop for both. N.B.: This matches the layout of the
      77                 :            :     // __asyncify_data struct.
      78                 :            :     void *stackbottom;
      79                 :            :     void *stacktop;
      80                 :            : } _jl_ucontext_t;
      81                 :            : #endif
      82                 :            : #if defined(JL_HAVE_UNW_CONTEXT)
      83                 :            : #define UNW_LOCAL_ONLY
      84                 :            : #include <libunwind.h>
      85                 :            : typedef unw_context_t _jl_ucontext_t;
      86                 :            : #endif
      87                 :            : #if defined(JL_HAVE_UCONTEXT)
      88                 :            : #include <ucontext.h>
      89                 :            : typedef ucontext_t _jl_ucontext_t;
      90                 :            : #endif
      91                 :            : #endif
      92                 :            : 
      93                 :            : typedef struct {
      94                 :            :     union {
      95                 :            :         _jl_ucontext_t ctx;
      96                 :            :         jl_stack_context_t copy_ctx;
      97                 :            :     };
      98                 :            : #if defined(_COMPILER_TSAN_ENABLED_)
      99                 :            :     void *tsan_state;
     100                 :            : #endif
     101                 :            : } jl_ucontext_t;
     102                 :            : 
     103                 :            : 
     104                 :            : // handle to reference an OS thread
     105                 :            : #ifdef _OS_WINDOWS_
     106                 :            : typedef DWORD jl_thread_t;
     107                 :            : #else
     108                 :            : typedef pthread_t jl_thread_t;
     109                 :            : #endif
     110                 :            : 
     111                 :            : struct _jl_task_t;
     112                 :            : 
     113                 :            : // Recursive spin lock
     114                 :            : typedef struct {
     115                 :            :     _Atomic(struct _jl_task_t*) owner;
     116                 :            :     uint32_t count;
     117                 :            : } jl_mutex_t;
     118                 :            : 
     119                 :            : typedef struct {
     120                 :            :     jl_taggedvalue_t *freelist;   // root of list of free objects
     121                 :            :     jl_taggedvalue_t *newpages;   // root of list of chunks of free objects
     122                 :            :     uint16_t osize;      // size of objects in this pool
     123                 :            : } jl_gc_pool_t;
     124                 :            : 
     125                 :            : typedef struct {
     126                 :            :     _Atomic(int64_t) allocd;
     127                 :            :     _Atomic(int64_t) freed;
     128                 :            :     _Atomic(uint64_t) malloc;
     129                 :            :     _Atomic(uint64_t) realloc;
     130                 :            :     _Atomic(uint64_t) poolalloc;
     131                 :            :     _Atomic(uint64_t) bigalloc;
     132                 :            :     _Atomic(uint64_t) freecall;
     133                 :            : } jl_thread_gc_num_t;
     134                 :            : 
     135                 :            : typedef struct {
     136                 :            :     // variable for tracking weak references
     137                 :            :     arraylist_t weak_refs;
     138                 :            :     // live tasks started on this thread
     139                 :            :     // that are holding onto a stack from the pool
     140                 :            :     arraylist_t live_tasks;
     141                 :            : 
     142                 :            :     // variables for tracking malloc'd arrays
     143                 :            :     struct _mallocarray_t *mallocarrays;
     144                 :            :     struct _mallocarray_t *mafreelist;
     145                 :            : 
     146                 :            :     // variables for tracking big objects
     147                 :            :     struct _bigval_t *big_objects;
     148                 :            : 
     149                 :            :     // variables for tracking "remembered set"
     150                 :            :     arraylist_t rem_bindings;
     151                 :            :     arraylist_t _remset[2]; // contains jl_value_t*
     152                 :            :     // lower bound of the number of pointers inside remembered values
     153                 :            :     int remset_nptr;
     154                 :            :     arraylist_t *remset;
     155                 :            :     arraylist_t *last_remset;
     156                 :            : 
     157                 :            :     // variables for allocating objects from pools
     158                 :            : #ifdef _P64
     159                 :            : #  define JL_GC_N_POOLS 49
     160                 :            : #elif MAX_ALIGN == 8
     161                 :            : #  define JL_GC_N_POOLS 50
     162                 :            : #else
     163                 :            : #  define JL_GC_N_POOLS 51
     164                 :            : #endif
     165                 :            :     jl_gc_pool_t norm_pools[JL_GC_N_POOLS];
     166                 :            : 
     167                 :            : #define JL_N_STACK_POOLS 16
     168                 :            :     arraylist_t free_stacks[JL_N_STACK_POOLS];
     169                 :            : } jl_thread_heap_t;
     170                 :            : 
     171                 :            : // Cache of thread local change to global metadata during GC
     172                 :            : // This is sync'd after marking.
     173                 :            : typedef union _jl_gc_mark_data jl_gc_mark_data_t;
     174                 :            : 
     175                 :            : typedef struct {
     176                 :            :     void **pc; // Current stack address for the pc (up growing)
     177                 :            :     jl_gc_mark_data_t *data; // Current stack address for the data (up growing)
     178                 :            :     void **pc_start; // Cached value of `gc_cache->pc_stack`
     179                 :            :     void **pc_end; // Cached value of `gc_cache->pc_stack_end`
     180                 :            : } jl_gc_mark_sp_t;
     181                 :            : 
     182                 :            : typedef struct {
     183                 :            :     // thread local increment of `perm_scanned_bytes`
     184                 :            :     size_t perm_scanned_bytes;
     185                 :            :     // thread local increment of `scanned_bytes`
     186                 :            :     size_t scanned_bytes;
     187                 :            :     // Number of queued big objects (<= 1024)
     188                 :            :     size_t nbig_obj;
     189                 :            :     // Array of queued big objects to be moved between the young list
     190                 :            :     // and the old list.
     191                 :            :     // A set low bit means that the object should be moved from the old list
     192                 :            :     // to the young list (`mark_reset_age`).
     193                 :            :     // Objects can only be put into this list when the mark bit is flipped to
     194                 :            :     // `1` (atomically). Combining with the sync after marking,
     195                 :            :     // this makes sure that a single objects can only appear once in
     196                 :            :     // the lists (the mark bit cannot be flipped to `0` without sweeping)
     197                 :            :     void *big_obj[1024];
     198                 :            :     void **pc_stack;
     199                 :            :     void **pc_stack_end;
     200                 :            :     jl_gc_mark_data_t *data_stack;
     201                 :            : } jl_gc_mark_cache_t;
     202                 :            : 
     203                 :            : struct _jl_bt_element_t;
     204                 :            : 
     205                 :            : // This includes all the thread local states we care about for a thread.
     206                 :            : // Changes to TLS field types must be reflected in codegen.
     207                 :            : #define JL_MAX_BT_SIZE 80000
     208                 :            : typedef struct _jl_tls_states_t {
     209                 :            :     int16_t tid;
     210                 :            :     int8_t threadpoolid;
     211                 :            :     uint64_t rngseed;
     212                 :            :     volatile size_t *safepoint;
     213                 :            :     _Atomic(int8_t) sleep_check_state; // read/write from foreign threads
     214                 :            :     // Whether it is safe to execute GC at the same time.
     215                 :            : #define JL_GC_STATE_WAITING 1
     216                 :            :     // gc_state = 1 means the thread is doing GC or is waiting for the GC to
     217                 :            :     //              finish.
     218                 :            : #define JL_GC_STATE_SAFE 2
     219                 :            :     // gc_state = 2 means the thread is running unmanaged code that can be
     220                 :            :     //              execute at the same time with the GC.
     221                 :            :     _Atomic(int8_t) gc_state; // read from foreign threads
     222                 :            :     // execution of certain certain impure
     223                 :            :     // statements is prohibited from certain
     224                 :            :     // callbacks (such as generated functions)
     225                 :            :     // as it may make compilation undecidable
     226                 :            :     int8_t in_pure_callback;
     227                 :            :     int8_t in_finalizer;
     228                 :            :     int8_t disable_gc;
     229                 :            :     // Counter to disable finalizer **on the current thread**
     230                 :            :     int finalizers_inhibited;
     231                 :            :     jl_thread_heap_t heap; // this is very large, and the offset is baked into codegen
     232                 :            :     jl_thread_gc_num_t gc_num;
     233                 :            :     volatile sig_atomic_t defer_signal;
     234                 :            :     _Atomic(struct _jl_task_t*) current_task;
     235                 :            :     struct _jl_task_t *next_task;
     236                 :            :     struct _jl_task_t *previous_task;
     237                 :            :     struct _jl_task_t *root_task;
     238                 :            :     struct _jl_timing_block_t *timing_stack;
     239                 :            :     void *stackbase;
     240                 :            :     size_t stacksize;
     241                 :            :     union {
     242                 :            :         _jl_ucontext_t base_ctx; // base context of stack
     243                 :            :         // This hack is needed to support always_copy_stacks:
     244                 :            :         jl_stack_context_t copy_stack_ctx;
     245                 :            :     };
     246                 :            :     // Temp storage for exception thrown in signal handler. Not rooted.
     247                 :            :     struct _jl_value_t *sig_exception;
     248                 :            :     // Temporary backtrace buffer. Scanned for gc roots when bt_size > 0.
     249                 :            :     struct _jl_bt_element_t *bt_data; // JL_MAX_BT_SIZE + 1 elements long
     250                 :            :     size_t bt_size;    // Size for backtrace in transit in bt_data
     251                 :            :     // Temporary backtrace buffer used only for allocations profiler.
     252                 :            :     struct _jl_bt_element_t *profiling_bt_buffer;
     253                 :            :     // Atomically set by the sender, reset by the handler.
     254                 :            :     volatile _Atomic(sig_atomic_t) signal_request; // TODO: no actual reason for this to be _Atomic
     255                 :            :     // Allow the sigint to be raised asynchronously
     256                 :            :     // this is limited to the few places we do synchronous IO
     257                 :            :     // we can make this more general (similar to defer_signal) if necessary
     258                 :            :     volatile sig_atomic_t io_wait;
     259                 :            : #ifdef _OS_WINDOWS_
     260                 :            :     int needs_resetstkoflw;
     261                 :            : #else
     262                 :            :     void *signal_stack;
     263                 :            : #endif
     264                 :            :     jl_thread_t system_id;
     265                 :            :     arraylist_t finalizers;
     266                 :            :     jl_gc_mark_cache_t gc_cache;
     267                 :            :     arraylist_t sweep_objs;
     268                 :            :     jl_gc_mark_sp_t gc_mark_sp;
     269                 :            :     // Saved exception for previous *external* API call or NULL if cleared.
     270                 :            :     // Access via jl_exception_occurred().
     271                 :            :     struct _jl_value_t *previous_exception;
     272                 :            : 
     273                 :            :     // currently-held locks, to be released when an exception is thrown
     274                 :            :     small_arraylist_t locks;
     275                 :            : 
     276                 :            :     JULIA_DEBUG_SLEEPWAKE(
     277                 :            :         uint64_t uv_run_enter;
     278                 :            :         uint64_t uv_run_leave;
     279                 :            :         uint64_t sleep_enter;
     280                 :            :         uint64_t sleep_leave;
     281                 :            :     )
     282                 :            : 
     283                 :            :     // some hidden state (usually just because we don't have the type's size declaration)
     284                 :            : #ifdef LIBRARY_EXPORTS
     285                 :            :     uv_mutex_t sleep_lock;
     286                 :            :     uv_cond_t wake_signal;
     287                 :            : #endif
     288                 :            : } jl_tls_states_t;
     289                 :            : 
     290                 :            : #ifndef LIBRARY_EXPORTS
     291                 :            : // deprecated (only for external consumers)
     292                 :            : JL_DLLEXPORT void *jl_get_ptls_states(void);
     293                 :            : #endif
     294                 :            : 
     295                 :            : // Update codegen version in `ccall.cpp` after changing either `pause` or `wake`
     296                 :            : #ifdef __MIC__
     297                 :            : #  define jl_cpu_pause() _mm_delay_64(100)
     298                 :            : #  define jl_cpu_wake() ((void)0)
     299                 :            : #  define JL_CPU_WAKE_NOOP 1
     300                 :            : #elif defined(_CPU_X86_64_) || defined(_CPU_X86_)  /* !__MIC__ */
     301                 :            : #  define jl_cpu_pause() _mm_pause()
     302                 :            : #  define jl_cpu_wake() ((void)0)
     303                 :            : #  define JL_CPU_WAKE_NOOP 1
     304                 :            : #elif defined(_CPU_AARCH64_) || (defined(_CPU_ARM_) && __ARM_ARCH >= 7)
     305                 :            : #  define jl_cpu_pause() __asm__ volatile ("wfe" ::: "memory")
     306                 :            : #  define jl_cpu_wake() __asm__ volatile ("sev" ::: "memory")
     307                 :            : #  define JL_CPU_WAKE_NOOP 0
     308                 :            : #else
     309                 :            : #  define jl_cpu_pause() ((void)0)
     310                 :            : #  define jl_cpu_wake() ((void)0)
     311                 :            : #  define JL_CPU_WAKE_NOOP 1
     312                 :            : #endif
     313                 :            : 
     314                 :            : JL_DLLEXPORT void (jl_cpu_pause)(void);
     315                 :            : JL_DLLEXPORT void (jl_cpu_wake)(void);
     316                 :            : 
     317                 :            : #ifdef __clang_gcanalyzer__
     318                 :            : // Note that the sigint safepoint can also trigger GC, albeit less likely
     319                 :            : void jl_gc_safepoint_(jl_ptls_t tls);
     320                 :            : void jl_sigint_safepoint(jl_ptls_t tls);
     321                 :            : #else
     322                 :            : // gc safepoint and gc states
     323                 :            : // This triggers a SegFault when we are in GC
     324                 :            : // Assign it to a variable to make sure the compiler emit the load
     325                 :            : // and to avoid Clang warning for -Wunused-volatile-lvalue
     326                 :            : #define jl_gc_safepoint_(ptls) do {                     \
     327                 :            :         jl_signal_fence();                              \
     328                 :            :         size_t safepoint_load = *ptls->safepoint;       \
     329                 :            :         jl_signal_fence();                              \
     330                 :            :         (void)safepoint_load;                           \
     331                 :            :     } while (0)
     332                 :            : #define jl_sigint_safepoint(ptls) do {                  \
     333                 :            :         jl_signal_fence();                              \
     334                 :            :         size_t safepoint_load = ptls->safepoint[-1];    \
     335                 :            :         jl_signal_fence();                              \
     336                 :            :         (void)safepoint_load;                           \
     337                 :            :     } while (0)
     338                 :            : #endif
     339                 :      19998 : STATIC_INLINE int8_t jl_gc_state_set(jl_ptls_t ptls, int8_t state,
     340                 :            :                                      int8_t old_state)
     341                 :            : {
     342                 :      19998 :     jl_atomic_store_release(&ptls->gc_state, state);
     343                 :            :     // A safe point is required if we transition from GC-safe region to
     344                 :            :     // non GC-safe region.
     345   [ +  +  +  - ]:      19998 :     if (old_state && !state)
     346                 :       1540 :         jl_gc_safepoint_(ptls);
     347                 :      19998 :     return old_state;
     348                 :            : }
     349                 :      15509 : STATIC_INLINE int8_t jl_gc_state_save_and_set(jl_ptls_t ptls,
     350                 :            :                                               int8_t state)
     351                 :            : {
     352                 :      15509 :     return jl_gc_state_set(ptls, state, jl_atomic_load_relaxed(&ptls->gc_state));
     353                 :            : }
     354                 :            : #ifdef __clang_gcanalyzer__
     355                 :            : int8_t jl_gc_unsafe_enter(jl_ptls_t ptls); // Can be a safepoint
     356                 :            : int8_t jl_gc_unsafe_leave(jl_ptls_t ptls, int8_t state) JL_NOTSAFEPOINT;
     357                 :            : int8_t jl_gc_safe_enter(jl_ptls_t ptls) JL_NOTSAFEPOINT;
     358                 :            : int8_t jl_gc_safe_leave(jl_ptls_t ptls, int8_t state); // Can be a safepoint
     359                 :            : #else
     360                 :            : #define jl_gc_unsafe_enter(ptls) jl_gc_state_save_and_set(ptls, 0)
     361                 :            : #define jl_gc_unsafe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), 0))
     362                 :            : #define jl_gc_safe_enter(ptls) jl_gc_state_save_and_set(ptls, JL_GC_STATE_SAFE)
     363                 :            : #define jl_gc_safe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), JL_GC_STATE_SAFE))
     364                 :            : #endif
     365                 :            : 
     366                 :            : JL_DLLEXPORT void jl_gc_enable_finalizers(struct _jl_task_t *ct, int on);
     367                 :            : JL_DLLEXPORT void jl_gc_disable_finalizers_internal(void);
     368                 :            : JL_DLLEXPORT void jl_gc_enable_finalizers_internal(void);
     369                 :            : JL_DLLEXPORT void jl_gc_run_pending_finalizers(struct _jl_task_t *ct);
     370                 :            : extern JL_DLLEXPORT _Atomic(int) jl_gc_have_pending_finalizers;
     371                 :            : 
     372                 :            : JL_DLLEXPORT void jl_wakeup_thread(int16_t tid);
     373                 :            : 
     374                 :            : #ifdef __cplusplus
     375                 :            : }
     376                 :            : #endif
     377                 :            : 
     378                 :            : #endif