Branch data     Line data    Source code

       1                 :            : // This file is a part of Julia. License is MIT: https://julialang.org/license
       2                 :            : 
       3                 :            : #include "julia.h"
       4                 :            : #include "julia_internal.h"
       5                 :            : #include "threading.h"
       6                 :            : #ifndef _OS_WINDOWS_
       7                 :            : #include <sys/mman.h>
       8                 :            : #if defined(_OS_DARWIN_) && !defined(MAP_ANONYMOUS)
       9                 :            : #define MAP_ANONYMOUS MAP_ANON
      10                 :            : #endif
      11                 :            : #endif
      12                 :            : #include "julia_assert.h"
      13                 :            : 
      14                 :            : #ifdef __cplusplus
      15                 :            : extern "C" {
      16                 :            : #endif
      17                 :            : 
      18                 :            : // 0: no sigint is pending
      19                 :            : // 1: at least one sigint is pending, only the sigint page is enabled.
      20                 :            : // 2: at least one sigint is pending, both safepoint pages are enabled.
      21                 :            : JL_DLLEXPORT sig_atomic_t jl_signal_pending = 0;
      22                 :            : _Atomic(uint32_t) jl_gc_running = 0;
      23                 :            : char *jl_safepoint_pages = NULL;
      24                 :            : // The number of safepoints enabled on the three pages.
      25                 :            : // The first page, is the SIGINT page, only used by the master thread.
      26                 :            : // The second page, is the GC page for the master thread, this is where
      27                 :            : // the `safepoint` tls pointer points to for the master thread.
      28                 :            : // The third page is the GC page for the other threads. The thread's
      29                 :            : // `safepoint` tls pointer points the beginning of this page + `sizeof(size_t)`
      30                 :            : // so that both safepoint load and pending signal load falls in this page.
      31                 :            : // The initialization of the `safepoint` pointer is done `ti_initthread`
      32                 :            : // in `threading.c`.
      33                 :            : uint8_t jl_safepoint_enable_cnt[3] = {0, 0, 0};
      34                 :            : 
      35                 :            : // This lock should be acquired before enabling/disabling the safepoint
      36                 :            : // or accessing one of the following variables:
      37                 :            : //
      38                 :            : // * jl_gc_running
      39                 :            : // * jl_signal_pending
      40                 :            : // * jl_safepoint_enable_cnt
      41                 :            : //
      42                 :            : // Additionally accessing `jl_gc_running` should use acquire/release
      43                 :            : // load/store so that threads waiting for the GC doesn't have to also
      44                 :            : // fight on the safepoint lock...
      45                 :            : uv_mutex_t safepoint_lock;
      46                 :            : uv_cond_t safepoint_cond;
      47                 :            : 
      48                 :          0 : static void jl_safepoint_enable(int idx) JL_NOTSAFEPOINT
      49                 :            : {
      50                 :            :     // safepoint_lock should be held
      51   [ #  #  #  # ]:          0 :     assert(0 <= idx && idx < 3);
      52         [ #  # ]:          0 :     if (jl_safepoint_enable_cnt[idx]++ != 0) {
      53                 :            :         // We expect this to be enabled at most twice
      54                 :            :         // one for the GC, one for SIGINT.
      55                 :            :         // Update this if this is not the case anymore in the future.
      56         [ #  # ]:          0 :         assert(jl_safepoint_enable_cnt[idx] <= 2);
      57                 :          0 :         return;
      58                 :            :     }
      59                 :            :     // Now that we are requested to mprotect the page and it wasn't already.
      60                 :          0 :     char *pageaddr = jl_safepoint_pages + jl_page_size * idx;
      61                 :            : #ifdef _OS_WINDOWS_
      62                 :            :     DWORD old_prot;
      63                 :            :     VirtualProtect(pageaddr, jl_page_size, PAGE_NOACCESS, &old_prot);
      64                 :            : #else
      65                 :          0 :     mprotect(pageaddr, jl_page_size, PROT_NONE);
      66                 :            : #endif
      67                 :            : }
      68                 :            : 
      69                 :          0 : static void jl_safepoint_disable(int idx) JL_NOTSAFEPOINT
      70                 :            : {
      71                 :            :     // safepoint_lock should be held
      72   [ #  #  #  # ]:          0 :     assert(0 <= idx && idx < 3);
      73         [ #  # ]:          0 :     if (--jl_safepoint_enable_cnt[idx] != 0) {
      74         [ #  # ]:          0 :         assert(jl_safepoint_enable_cnt[idx] > 0);
      75                 :          0 :         return;
      76                 :            :     }
      77                 :            :     // Now that we are requested to un-mprotect the page and no one else
      78                 :            :     // want it to be kept protected.
      79                 :          0 :     char *pageaddr = jl_safepoint_pages + jl_page_size * idx;
      80                 :            : #ifdef _OS_WINDOWS_
      81                 :            :     DWORD old_prot;
      82                 :            :     VirtualProtect(pageaddr, jl_page_size, PAGE_READONLY, &old_prot);
      83                 :            : #else
      84                 :          0 :     mprotect(pageaddr, jl_page_size, PROT_READ);
      85                 :            : #endif
      86                 :            : }
      87                 :            : 
      88                 :         15 : void jl_safepoint_init(void)
      89                 :            : {
      90                 :         15 :     uv_mutex_init(&safepoint_lock);
      91                 :         15 :     uv_cond_init(&safepoint_cond);
      92                 :            :     // jl_page_size isn't available yet.
      93                 :         15 :     size_t pgsz = jl_getpagesize();
      94                 :            : #ifdef _OS_WINDOWS_
      95                 :            :     char *addr = (char*)VirtualAlloc(NULL, pgsz * 3, MEM_COMMIT, PAGE_READONLY);
      96                 :            : #else
      97                 :         15 :     char *addr = (char*)mmap(0, pgsz * 3, PROT_READ,
      98                 :            :                              MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
      99         [ -  + ]:         15 :     if (addr == MAP_FAILED)
     100                 :          0 :         addr = NULL;
     101                 :            : #endif
     102         [ -  + ]:         15 :     if (addr == NULL) {
     103                 :          0 :         jl_printf(JL_STDERR, "could not allocate GC synchronization page\n");
     104                 :          0 :         jl_gc_debug_critical_error();
     105                 :          0 :         abort();
     106                 :            :     }
     107                 :            :     // The signal page is for the gc safepoint.
     108                 :            :     // The page before it is the sigint pending flag.
     109                 :         15 :     jl_safepoint_pages = addr;
     110                 :         15 : }
     111                 :            : 
     112                 :       1174 : int jl_safepoint_start_gc(void)
     113                 :            : {
     114         [ +  - ]:       1174 :     if (jl_n_threads == 1) {
     115                 :       1174 :         jl_atomic_store_relaxed(&jl_gc_running, 1);
     116                 :       1174 :         return 1;
     117                 :            :     }
     118                 :            :     // The thread should have set this already
     119         [ #  # ]:          0 :     assert(jl_atomic_load_relaxed(&jl_current_task->ptls->gc_state) == JL_GC_STATE_WAITING);
     120                 :          0 :     uv_mutex_lock(&safepoint_lock);
     121                 :            :     // In case multiple threads enter the GC at the same time, only allow
     122                 :            :     // one of them to actually run the collection. We can't just let the
     123                 :            :     // master thread do the GC since it might be running unmanaged code
     124                 :            :     // and can take arbitrarily long time before hitting a safe point.
     125                 :          0 :     uint32_t running = 0;
     126         [ #  # ]:          0 :     if (!jl_atomic_cmpswap(&jl_gc_running, &running, 1)) {
     127                 :          0 :         uv_mutex_unlock(&safepoint_lock);
     128                 :          0 :         jl_safepoint_wait_gc();
     129                 :          0 :         return 0;
     130                 :            :     }
     131                 :          0 :     jl_safepoint_enable(1);
     132                 :          0 :     jl_safepoint_enable(2);
     133                 :          0 :     uv_mutex_unlock(&safepoint_lock);
     134                 :          0 :     return 1;
     135                 :            : }
     136                 :            : 
     137                 :       1174 : void jl_safepoint_end_gc(void)
     138                 :            : {
     139         [ -  + ]:       1174 :     assert(jl_atomic_load_relaxed(&jl_gc_running));
     140         [ +  - ]:       1174 :     if (jl_n_threads == 1) {
     141                 :       1174 :         jl_atomic_store_relaxed(&jl_gc_running, 0);
     142                 :       1174 :         return;
     143                 :            :     }
     144                 :          0 :     uv_mutex_lock(&safepoint_lock);
     145                 :            :     // Need to reset the page protection before resetting the flag since
     146                 :            :     // the thread will trigger a segfault immediately after returning from
     147                 :            :     // the signal handler.
     148                 :          0 :     jl_safepoint_disable(2);
     149                 :          0 :     jl_safepoint_disable(1);
     150                 :          0 :     jl_atomic_store_release(&jl_gc_running, 0);
     151                 :            : #  ifdef __APPLE__
     152                 :            :     // This wakes up other threads on mac.
     153                 :            :     jl_mach_gc_end();
     154                 :            : #  endif
     155                 :          0 :     uv_mutex_unlock(&safepoint_lock);
     156                 :          0 :     uv_cond_broadcast(&safepoint_cond);
     157                 :            : }
     158                 :            : 
     159                 :          0 : void jl_safepoint_wait_gc(void)
     160                 :            : {
     161                 :            :     // The thread should have set this is already
     162         [ #  # ]:          0 :     assert(jl_atomic_load_relaxed(&jl_current_task->ptls->gc_state) != 0);
     163                 :            :     // Use normal volatile load in the loop for speed until GC finishes.
     164                 :            :     // Then use an acquire load to make sure the GC result is visible on this thread.
     165   [ #  #  #  # ]:          0 :     while (jl_atomic_load_relaxed(&jl_gc_running) || jl_atomic_load_acquire(&jl_gc_running)) {
     166                 :            :         // Use system mutexes rather than spin locking to minimize wasted CPU
     167                 :            :         // time on the idle cores while we wait for the GC to finish.
     168                 :            :         // This is particularly important when run under rr.
     169                 :          0 :         uv_mutex_lock(&safepoint_lock);
     170         [ #  # ]:          0 :         if (jl_atomic_load_relaxed(&jl_gc_running))
     171                 :          0 :             uv_cond_wait(&safepoint_cond, &safepoint_lock);
     172                 :          0 :         uv_mutex_unlock(&safepoint_lock);
     173                 :            :     }
     174                 :          0 : }
     175                 :            : 
     176                 :          0 : void jl_safepoint_enable_sigint(void)
     177                 :            : {
     178                 :          0 :     uv_mutex_lock(&safepoint_lock);
     179                 :            :     // Make sure both safepoints are enabled exactly once for SIGINT.
     180   [ #  #  #  # ]:          0 :     switch (jl_signal_pending) {
     181                 :          0 :     default:
     182                 :          0 :         assert(0 && "Shouldn't happen.");
     183                 :            :     case 0:
     184                 :            :         // Enable SIGINT page
     185                 :          0 :         jl_safepoint_enable(0);
     186                 :            :         // fall through
     187                 :          0 :     case 1:
     188                 :            :         // SIGINT page is enabled, enable GC page
     189                 :          0 :         jl_safepoint_enable(1);
     190                 :            :         // fall through
     191                 :          0 :     case 2:
     192                 :          0 :         jl_signal_pending = 2;
     193                 :            :     }
     194                 :          0 :     uv_mutex_unlock(&safepoint_lock);
     195                 :          0 : }
     196                 :            : 
     197                 :          0 : void jl_safepoint_defer_sigint(void)
     198                 :            : {
     199                 :          0 :     uv_mutex_lock(&safepoint_lock);
     200                 :            :     // Make sure the GC safepoint is disabled for SIGINT.
     201         [ #  # ]:          0 :     if (jl_signal_pending == 2) {
     202                 :          0 :         jl_safepoint_disable(1);
     203                 :          0 :         jl_signal_pending = 1;
     204                 :            :     }
     205                 :          0 :     uv_mutex_unlock(&safepoint_lock);
     206                 :          0 : }
     207                 :            : 
     208                 :          0 : int jl_safepoint_consume_sigint(void)
     209                 :            : {
     210                 :          0 :     int has_signal = 0;
     211                 :          0 :     uv_mutex_lock(&safepoint_lock);
     212                 :            :     // Make sure both safepoints are disabled for SIGINT.
     213   [ #  #  #  # ]:          0 :     switch (jl_signal_pending) {
     214                 :          0 :     default:
     215                 :          0 :         assert(0 && "Shouldn't happen.");
     216                 :            :     case 2:
     217                 :            :         // Disable gc page
     218                 :          0 :         jl_safepoint_disable(1);
     219                 :            :         // fall through
     220                 :          0 :     case 1:
     221                 :            :         // GC page is disabled, disable SIGINT page
     222                 :          0 :         jl_safepoint_disable(0);
     223                 :          0 :         has_signal = 1;
     224                 :            :         // fall through
     225                 :          0 :     case 0:
     226                 :          0 :         jl_signal_pending = 0;
     227                 :            :     }
     228                 :          0 :     uv_mutex_unlock(&safepoint_lock);
     229                 :          0 :     return has_signal;
     230                 :            : }
     231                 :            : 
     232                 :            : #ifdef __cplusplus
     233                 :            : }
     234                 :            : #endif