2 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
5 * Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
7 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
8 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 * Permission is hereby granted to use or copy this program
11 * for any purpose, provided the above notices are retained on all copies.
12 * Permission to modify the code and to distribute modified code is granted,
13 * provided the above notices are retained, and a notice that the code was
14 * modified is included with the above copyright notice.
17 #include "private/pthread_support.h"
20 * Support code originally for LinuxThreads, the clone()-based kernel
21 * thread package for Linux which is included in libc6.
23 * This code no doubt makes some assumptions beyond what is
24 * guaranteed by the pthread standard, though it now does
25 * very little of that. It now also supports NPTL, and many
26 * other Posix thread implementations. We are trying to merge
27 * all flavors of pthread support code into this file.
30 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
38 # if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
39 # if !defined(GC_RTEMS_PTHREADS)
40 # include <sys/mman.h>
42 # include <sys/time.h>
43 # include <sys/types.h>
44 # include <sys/stat.h>
49 # include "gc_inline.h"
51 #if defined(GC_DARWIN_THREADS)
52 # include "private/darwin_semaphore.h"
54 # include <semaphore.h>
55 #endif /* !GC_DARWIN_THREADS */
57 #if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
58 # include <sys/sysctl.h>
59 #endif /* GC_DARWIN_THREADS */
61 #if defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
62 # include <sys/param.h>
63 # include <sys/sysctl.h>
64 #endif /* GC_NETBSD_THREADS */
66 /* Allocator lock definitions. */
67 #if !defined(USE_SPIN_LOCK)
68 GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
72 GC_INNER unsigned long GC_lock_holder = NO_THREAD;
73 /* Used only for assertions. */
76 #if defined(GC_DGUX386_THREADS)
77 # include <sys/dg_sys_info.h>
78 # include <sys/_int_psem.h>
79 /* sem_t is an uint in DG/UX */
80 typedef unsigned int sem_t;
81 #endif /* GC_DGUX386_THREADS */
83 /* Undefine macros used to redirect pthread primitives. */
84 # undef pthread_create
85 # ifndef GC_NO_PTHREAD_SIGMASK
86 # undef pthread_sigmask
88 # ifndef GC_NO_PTHREAD_CANCEL
89 # undef pthread_cancel
91 # ifdef GC_HAVE_PTHREAD_EXIT
95 # undef pthread_detach
96 # if defined(GC_OSF1_THREADS) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
97 && !defined(_PTHREAD_USE_PTDNAM_)
98 /* Restore the original mangled names on Tru64 UNIX. */
99 # define pthread_create __pthread_create
100 # define pthread_join __pthread_join
101 # define pthread_detach __pthread_detach
102 # ifndef GC_NO_PTHREAD_CANCEL
103 # define pthread_cancel __pthread_cancel
105 # ifdef GC_HAVE_PTHREAD_EXIT
106 # define pthread_exit __pthread_exit
110 #ifdef GC_USE_LD_WRAP
111 # define WRAP_FUNC(f) __wrap_##f
112 # define REAL_FUNC(f) __real_##f
113 int REAL_FUNC(pthread_create)(pthread_t *,
114 GC_PTHREAD_CREATE_CONST pthread_attr_t *,
115 void *(*start_routine)(void *), void *);
116 int REAL_FUNC(pthread_join)(pthread_t, void **);
117 int REAL_FUNC(pthread_detach)(pthread_t);
118 # ifndef GC_NO_PTHREAD_SIGMASK
119 int REAL_FUNC(pthread_sigmask)(int, const sigset_t *, sigset_t *);
121 # ifndef GC_NO_PTHREAD_CANCEL
122 int REAL_FUNC(pthread_cancel)(pthread_t);
124 # ifdef GC_HAVE_PTHREAD_EXIT
125 void REAL_FUNC(pthread_exit)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
128 # ifdef GC_USE_DLOPEN_WRAP
130 # define WRAP_FUNC(f) f
131 # define REAL_FUNC(f) GC_real_##f
132 /* We define both GC_f and plain f to be the wrapped function. */
133 /* In that way plain calls work, as do calls from files that */
134 /* included gc.h, which redefined f to GC_f. */
135 /* FIXME: Needs work for DARWIN and True64 (OSF1) */
136 typedef int (* GC_pthread_create_t)(pthread_t *,
137 GC_PTHREAD_CREATE_CONST pthread_attr_t *,
138 void * (*)(void *), void *);
139 static GC_pthread_create_t REAL_FUNC(pthread_create);
140 # ifndef GC_NO_PTHREAD_SIGMASK
141 typedef int (* GC_pthread_sigmask_t)(int, const sigset_t *,
143 static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask);
145 typedef int (* GC_pthread_join_t)(pthread_t, void **);
146 static GC_pthread_join_t REAL_FUNC(pthread_join);
147 typedef int (* GC_pthread_detach_t)(pthread_t);
148 static GC_pthread_detach_t REAL_FUNC(pthread_detach);
149 # ifndef GC_NO_PTHREAD_CANCEL
150 typedef int (* GC_pthread_cancel_t)(pthread_t);
151 static GC_pthread_cancel_t REAL_FUNC(pthread_cancel);
153 # ifdef GC_HAVE_PTHREAD_EXIT
154 typedef void (* GC_pthread_exit_t)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
155 static GC_pthread_exit_t REAL_FUNC(pthread_exit);
158 # define WRAP_FUNC(f) GC_##f
159 # if !defined(GC_DGUX386_THREADS)
160 # define REAL_FUNC(f) f
161 # else /* GC_DGUX386_THREADS */
162 # define REAL_FUNC(f) __d10_##f
163 # endif /* GC_DGUX386_THREADS */
167 #if defined(GC_USE_LD_WRAP) || defined(GC_USE_DLOPEN_WRAP)
168 /* Define GC_ functions as aliases for the plain ones, which will */
169 /* be intercepted. This allows files which include gc.h, and hence */
170 /* generate references to the GC_ symbols, to see the right symbols. */
171 GC_API int GC_pthread_create(pthread_t * t,
172 GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
173 void * (* fn)(void *), void * arg)
175 return pthread_create(t, a, fn, arg);
178 # ifndef GC_NO_PTHREAD_SIGMASK
179 GC_API int GC_pthread_sigmask(int how, const sigset_t *mask,
182 return pthread_sigmask(how, mask, old);
184 # endif /* !GC_NO_PTHREAD_SIGMASK */
186 GC_API int GC_pthread_join(pthread_t t, void **res)
188 return pthread_join(t, res);
191 GC_API int GC_pthread_detach(pthread_t t)
193 return pthread_detach(t);
196 # ifndef GC_NO_PTHREAD_CANCEL
197 GC_API int GC_pthread_cancel(pthread_t t)
199 return pthread_cancel(t);
201 # endif /* !GC_NO_PTHREAD_CANCEL */
203 # ifdef GC_HAVE_PTHREAD_EXIT
204 GC_API GC_PTHREAD_EXIT_ATTRIBUTE void GC_pthread_exit(void *retval)
206 pthread_exit(retval);
209 #endif /* Linker-based interception. */
211 #ifdef GC_USE_DLOPEN_WRAP
212 STATIC GC_bool GC_syms_initialized = FALSE;
214 STATIC void GC_init_real_syms(void)
218 if (GC_syms_initialized) return;
220 dl_handle = RTLD_NEXT;
222 dl_handle = dlopen("libpthread.so.0", RTLD_LAZY);
223 if (NULL == dl_handle) {
224 dl_handle = dlopen("libpthread.so", RTLD_LAZY); /* without ".0" */
226 if (NULL == dl_handle) ABORT("Couldn't open libpthread");
228 REAL_FUNC(pthread_create) = (GC_pthread_create_t)(word)
229 dlsym(dl_handle, "pthread_create");
231 if (REAL_FUNC(pthread_create) == 0)
232 ABORT("pthread_create not found"
233 " (probably -lgc is specified after -lpthread)");
235 # ifndef GC_NO_PTHREAD_SIGMASK
236 REAL_FUNC(pthread_sigmask) = (GC_pthread_sigmask_t)(word)
237 dlsym(dl_handle, "pthread_sigmask");
239 REAL_FUNC(pthread_join) = (GC_pthread_join_t)(word)
240 dlsym(dl_handle, "pthread_join");
241 REAL_FUNC(pthread_detach) = (GC_pthread_detach_t)(word)
242 dlsym(dl_handle, "pthread_detach");
243 # ifndef GC_NO_PTHREAD_CANCEL
244 REAL_FUNC(pthread_cancel) = (GC_pthread_cancel_t)(word)
245 dlsym(dl_handle, "pthread_cancel");
247 # ifdef GC_HAVE_PTHREAD_EXIT
248 REAL_FUNC(pthread_exit) = (GC_pthread_exit_t)(word)
249 dlsym(dl_handle, "pthread_exit");
251 GC_syms_initialized = TRUE;
254 # define INIT_REAL_SYMS() if (EXPECT(GC_syms_initialized, TRUE)) {} \
255 else GC_init_real_syms()
256 # define ASSERT_SYMS_INITIALIZED() GC_ASSERT(GC_syms_initialized)
258 # define INIT_REAL_SYMS() (void)0
259 # define ASSERT_SYMS_INITIALIZED() GC_ASSERT(parallel_initialized)
262 static GC_bool parallel_initialized = FALSE;
264 #ifndef GC_ALWAYS_MULTITHREADED
265 GC_INNER GC_bool GC_need_to_lock = FALSE;
268 STATIC int GC_nprocs = 1;
269 /* Number of processors. We may not have */
270 /* access to all of them, but this is as good */
271 /* a guess as any ... */
273 #ifdef THREAD_LOCAL_ALLOC
274 /* We must explicitly mark ptrfree and gcj free lists, since the free */
275 /* list links wouldn't otherwise be found. We also set them in the */
276 /* normal free lists, since that involves touching less memory than */
277 /* if we scanned them normally. */
278 GC_INNER void GC_mark_thread_local_free_lists(void)
283 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
284 for (p = GC_threads[i]; 0 != p; p = p -> next) {
285 if (!(p -> flags & FINISHED))
286 GC_mark_thread_local_fls_for(&(p->tlfs));
291 # if defined(GC_ASSERTIONS)
292 /* Check that all thread-local free-lists are completely marked. */
293 /* Also check that thread-specific-data structures are marked. */
294 void GC_check_tls(void)
299 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
300 for (p = GC_threads[i]; 0 != p; p = p -> next) {
301 if (!(p -> flags & FINISHED))
302 GC_check_tls_for(&(p->tlfs));
305 # if defined(USE_CUSTOM_SPECIFIC)
306 if (GC_thread_key != 0)
307 GC_check_tsd_marks(GC_thread_key);
310 # endif /* GC_ASSERTIONS */
312 #endif /* THREAD_LOCAL_ALLOC */
317 # define MAX_MARKERS 16
320 static ptr_t marker_sp[MAX_MARKERS - 1] = {0};
322 static ptr_t marker_bsp[MAX_MARKERS - 1] = {0};
325 #if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
326 static mach_port_t marker_mach_threads[MAX_MARKERS - 1] = {0};
328 /* Used only by GC_suspend_thread_list(). */
329 GC_INNER GC_bool GC_is_mach_marker(thread_act_t thread)
332 for (i = 0; i < GC_markers_m1; i++) {
333 if (marker_mach_threads[i] == thread)
338 #endif /* GC_DARWIN_THREADS */
340 STATIC void * GC_mark_thread(void * id)
343 IF_CANCEL(int cancel_state;)
345 if ((word)id == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
346 DISABLE_CANCEL(cancel_state);
347 /* Mark threads are not cancellable; they */
348 /* should be invisible to client. */
349 marker_sp[(word)id] = GC_approx_sp();
351 marker_bsp[(word)id] = GC_save_regs_in_stack();
353 # if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
354 marker_mach_threads[(word)id] = mach_thread_self();
357 /* Inform GC_start_mark_threads about completion of marker data init. */
358 GC_acquire_mark_lock();
359 if (0 == --GC_fl_builder_count) /* count may have a negative value */
360 GC_notify_all_builder();
362 for (;; ++my_mark_no) {
363 /* GC_mark_no is passed only to allow GC_help_marker to terminate */
364 /* promptly. This is important if it were called from the signal */
365 /* handler or from the GC lock acquisition code. Under Linux, it's */
366 /* not safe to call it from a signal handler, since it uses mutexes */
367 /* and condition variables. Since it is called only here, the */
368 /* argument is unnecessary. */
369 if (my_mark_no < GC_mark_no || my_mark_no > GC_mark_no + 2) {
370 /* resynchronize if we get far off, e.g. because GC_mark_no */
372 my_mark_no = GC_mark_no;
374 # ifdef DEBUG_THREADS
375 GC_log_printf("Starting mark helper for mark number %lu\n",
376 (unsigned long)my_mark_no);
378 GC_help_marker(my_mark_no);
382 STATIC pthread_t GC_mark_threads[MAX_MARKERS];
384 #ifdef CAN_HANDLE_FORK
385 static int available_markers_m1 = 0;
386 static pthread_cond_t mark_cv;
387 /* initialized by GC_start_mark_threads_inner */
389 # define available_markers_m1 GC_markers_m1
390 static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
393 GC_INNER void GC_start_mark_threads_inner(void)
397 # ifndef NO_MARKER_SPECIAL_SIGMASK
398 sigset_t set, oldset;
401 GC_ASSERT(I_DONT_HOLD_LOCK());
402 if (available_markers_m1 <= 0) return;
403 /* Skip if parallel markers disabled or already started. */
404 # ifdef CAN_HANDLE_FORK
405 if (GC_parallel) return;
407 /* Initialize mark_cv (for the first time), or cleanup its value */
408 /* after forking in the child process. All the marker threads in */
409 /* the parent process were blocked on this variable at fork, so */
410 /* pthread_cond_wait() malfunction (hang) is possible in the */
411 /* child process without such a cleanup. */
412 /* TODO: This is not portable, it is better to shortly unblock */
413 /* all marker threads in the parent process at fork. */
415 pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
416 BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
420 GC_ASSERT(GC_fl_builder_count == 0);
421 INIT_REAL_SYMS(); /* for pthread_create */
422 if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
423 if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
424 ABORT("pthread_attr_setdetachstate failed");
426 # ifdef DEFAULT_STACK_MAYBE_SMALL
427 /* Default stack size is usually too small: increase it. */
428 /* Otherwise marker threads or GC may run out of space. */
432 if (pthread_attr_getstacksize(&attr, &old_size) != 0)
433 ABORT("pthread_attr_getstacksize failed");
434 if (old_size < MIN_STACK_SIZE
435 && old_size != 0 /* stack size is known */) {
436 if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
437 ABORT("pthread_attr_setstacksize failed");
440 # endif /* DEFAULT_STACK_MAYBE_SMALL */
442 # ifndef NO_MARKER_SPECIAL_SIGMASK
443 /* Apply special signal mask to GC marker threads, and don't drop */
444 /* user defined signals by GC marker threads. */
445 if (sigfillset(&set) != 0)
446 ABORT("sigfillset failed");
448 # if !defined(GC_DARWIN_THREADS) && !defined(GC_OPENBSD_UTHREADS) \
450 /* These are used by GC to stop and restart the world. */
451 if (sigdelset(&set, GC_get_suspend_signal()) != 0
452 || sigdelset(&set, GC_get_thr_restart_signal()) != 0)
453 ABORT("sigdelset failed");
456 if (REAL_FUNC(pthread_sigmask)(SIG_BLOCK, &set, &oldset) < 0) {
457 WARN("pthread_sigmask set failed, no markers started,"
458 " errno = %" WARN_PRIdPTR "\n", errno);
460 (void)pthread_attr_destroy(&attr);
463 # endif /* !NO_MARKER_SPECIAL_SIGMASK */
465 # ifdef CAN_HANDLE_FORK
466 /* To have proper GC_parallel value in GC_help_marker. */
467 GC_markers_m1 = available_markers_m1;
469 for (i = 0; i < available_markers_m1; ++i) {
470 if (0 != REAL_FUNC(pthread_create)(GC_mark_threads + i, &attr,
471 GC_mark_thread, (void *)(word)i)) {
472 WARN("Marker thread creation failed, errno = %" WARN_PRIdPTR "\n",
474 /* Don't try to create other marker threads. */
480 # ifndef NO_MARKER_SPECIAL_SIGMASK
481 /* Restore previous signal mask. */
482 if (REAL_FUNC(pthread_sigmask)(SIG_SETMASK, &oldset, NULL) < 0) {
483 WARN("pthread_sigmask restore failed, errno = %" WARN_PRIdPTR "\n",
488 (void)pthread_attr_destroy(&attr);
489 GC_wait_for_markers_init();
490 GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
493 #endif /* PARALLEL_MARK */
495 GC_INNER GC_bool GC_thr_initialized = FALSE;
497 GC_INNER volatile GC_thread GC_threads[THREAD_TABLE_SZ] = {0};
499 void GC_push_thread_structures(void)
501 GC_ASSERT(I_HOLD_LOCK());
502 GC_PUSH_ALL_SYM(GC_threads);
503 # if defined(THREAD_LOCAL_ALLOC)
504 GC_PUSH_ALL_SYM(GC_thread_key);
509 STATIC int GC_count_threads(void)
513 GC_ASSERT(I_HOLD_LOCK());
514 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
515 GC_thread th = GC_threads[i];
517 if (!(th->flags & FINISHED))
524 #endif /* DEBUG_THREADS */
526 /* It may not be safe to allocate when we register the first thread. */
527 /* As "next" and "status" fields are unused, no need to push this. */
528 static struct GC_Thread_Rep first_thread;
530 /* Add a thread to GC_threads. We assume it wasn't already there. */
531 /* Caller holds allocation lock. */
532 STATIC GC_thread GC_new_thread(pthread_t id)
534 int hv = THREAD_TABLE_INDEX(id);
536 static GC_bool first_thread_used = FALSE;
538 # ifdef DEBUG_THREADS
539 GC_log_printf("Creating thread %p\n", (void *)id);
540 for (result = GC_threads[hv]; result != NULL; result = result->next)
541 if (!THREAD_EQUAL(result->id, id)) {
542 GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
546 GC_ASSERT(I_HOLD_LOCK());
547 if (!EXPECT(first_thread_used, TRUE)) {
548 result = &first_thread;
549 first_thread_used = TRUE;
550 GC_ASSERT(NULL == GC_threads[hv]);
551 # if defined(THREAD_SANITIZER) && defined(CPPCHECK)
552 GC_noop1(result->dummy[0]);
555 result = (struct GC_Thread_Rep *)
556 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
557 if (result == 0) return(0);
560 # ifdef USE_TKILL_ON_ANDROID
561 result -> kernel_id = gettid();
563 result -> next = GC_threads[hv];
564 GC_threads[hv] = result;
566 GC_nacl_gc_thread_self = result;
567 GC_nacl_initialize_gc_thread();
569 GC_ASSERT(result -> flags == 0 && result -> thread_blocked == 0);
570 if (EXPECT(result != &first_thread, TRUE))
575 /* Delete a thread from GC_threads. We assume it is there. */
576 /* (The code intentionally traps if it wasn't.) */
577 /* It is safe to delete the main thread. */
578 STATIC void GC_delete_thread(pthread_t id)
580 int hv = THREAD_TABLE_INDEX(id);
581 GC_thread p = GC_threads[hv];
582 GC_thread prev = NULL;
584 # ifdef DEBUG_THREADS
585 GC_log_printf("Deleting thread %p, n_threads = %d\n",
586 (void *)id, GC_count_threads());
590 GC_nacl_shutdown_gc_thread();
591 GC_nacl_gc_thread_self = NULL;
594 GC_ASSERT(I_HOLD_LOCK());
595 while (!THREAD_EQUAL(p -> id, id)) {
600 GC_threads[hv] = p -> next;
602 GC_ASSERT(prev != &first_thread);
603 prev -> next = p -> next;
606 if (p != &first_thread) {
607 # ifdef GC_DARWIN_THREADS
608 mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
614 /* If a thread has been joined, but we have not yet */
615 /* been notified, then there may be more than one thread */
616 /* in the table with the same pthread id. */
617 /* This is OK, but we need a way to delete a specific one. */
618 STATIC void GC_delete_gc_thread(GC_thread t)
620 pthread_t id = t -> id;
621 int hv = THREAD_TABLE_INDEX(id);
622 GC_thread p = GC_threads[hv];
623 GC_thread prev = NULL;
625 GC_ASSERT(I_HOLD_LOCK());
631 GC_threads[hv] = p -> next;
633 GC_ASSERT(prev != &first_thread);
634 prev -> next = p -> next;
637 # ifdef GC_DARWIN_THREADS
638 mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
642 # ifdef DEBUG_THREADS
643 GC_log_printf("Deleted thread %p, n_threads = %d\n",
644 (void *)id, GC_count_threads());
648 /* Return a GC_thread corresponding to a given pthread_t. */
649 /* Returns 0 if it's not there. */
650 /* Caller holds allocation lock or otherwise inhibits */
652 /* If there is more than one thread with the given id we */
653 /* return the most recent one. */
654 GC_INNER GC_thread GC_lookup_thread(pthread_t id)
656 GC_thread p = GC_threads[THREAD_TABLE_INDEX(id)];
658 while (p != 0 && !THREAD_EQUAL(p -> id, id)) p = p -> next;
662 /* Called by GC_finalize() (in case of an allocation failure observed). */
663 GC_INNER void GC_reset_finalizer_nested(void)
665 GC_thread me = GC_lookup_thread(pthread_self());
666 me->finalizer_nested = 0;
669 /* Checks and updates the thread-local level of finalizers recursion. */
670 /* Returns NULL if GC_invoke_finalizers() should not be called by the */
671 /* collector (to minimize the risk of a deep finalizers recursion), */
672 /* otherwise returns a pointer to the thread-local finalizer_nested. */
673 /* Called by GC_notify_or_invoke_finalizers() only (the lock is held). */
674 GC_INNER unsigned char *GC_check_finalizer_nested(void)
676 GC_thread me = GC_lookup_thread(pthread_self());
677 unsigned nesting_level = me->finalizer_nested;
679 /* We are inside another GC_invoke_finalizers(). */
680 /* Skip some implicitly-called GC_invoke_finalizers() */
681 /* depending on the nesting (recursion) level. */
682 if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
683 me->finalizer_skipped = 0;
685 me->finalizer_nested = (unsigned char)(nesting_level + 1);
686 return &me->finalizer_nested;
689 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
690 /* This is called from thread-local GC_malloc(). */
691 GC_bool GC_is_thread_tsd_valid(void *tsd)
697 me = GC_lookup_thread(pthread_self());
699 return (word)tsd >= (word)(&me->tlfs)
700 && (word)tsd < (word)(&me->tlfs) + sizeof(me->tlfs);
702 #endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
704 GC_API int GC_CALL GC_thread_is_registered(void)
706 pthread_t self = pthread_self();
711 me = GC_lookup_thread(self);
716 static pthread_t main_pthread_id;
717 static void *main_stack, *main_altstack;
718 static word main_stack_size, main_altstack_size;
720 GC_API void GC_CALL GC_register_altstack(void *stack, GC_word stack_size,
722 GC_word altstack_size)
725 pthread_t self = pthread_self();
729 me = GC_lookup_thread(self);
731 me->stack = (ptr_t)stack;
732 me->stack_size = stack_size;
733 me->altstack = (ptr_t)altstack;
734 me->altstack_size = altstack_size;
736 /* This happens if we are called before GC_thr_init. */
737 main_pthread_id = self;
739 main_stack_size = stack_size;
740 main_altstack = altstack;
741 main_altstack_size = altstack_size;
746 #ifdef CAN_HANDLE_FORK
748 /* Prevent TSan false positive about the race during items removal */
749 /* from GC_threads. (The race cannot happen since only one thread */
750 /* survives in the child.) */
751 # ifdef CAN_CALL_ATFORK
752 GC_ATTR_NO_SANITIZE_THREAD
754 static void store_to_threads_table(int hv, GC_thread me)
759 /* Remove all entries from the GC_threads table, except the */
760 /* one for the current thread. We need to do this in the child */
761 /* process after a fork(), since only the current thread */
762 /* survives in the child. */
763 STATIC void GC_remove_all_threads_but_me(void)
765 pthread_t self = pthread_self();
768 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
772 for (p = GC_threads[hv]; 0 != p; p = next) {
774 if (THREAD_EQUAL(p -> id, self)
775 && me == NULL) { /* ignore dead threads with the same id */
778 # ifdef GC_DARWIN_THREADS
779 /* Update thread Id after fork (it is OK to call */
780 /* GC_destroy_thread_local and GC_free_internal */
781 /* before update). */
782 me -> stop_info.mach_thread = mach_thread_self();
784 # ifdef USE_TKILL_ON_ANDROID
785 me -> kernel_id = gettid();
787 # if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
791 /* Some TLS implementations might be not fork-friendly, so */
792 /* we re-assign thread-local pointer to 'tlfs' for safety */
793 /* instead of the assertion check (again, it is OK to call */
794 /* GC_destroy_thread_local and GC_free_internal before). */
795 res = GC_setspecific(GC_thread_key, &me->tlfs);
796 if (COVERT_DATAFLOW(res) != 0)
797 ABORT("GC_setspecific failed (in child)");
801 # ifdef THREAD_LOCAL_ALLOC
802 if (!(p -> flags & FINISHED)) {
803 /* Cannot call GC_destroy_thread_local here. The free */
804 /* lists may be in an inconsistent state (as thread p may */
805 /* be updating one of the lists by GC_generic_malloc_many */
806 /* or GC_FAST_MALLOC_GRANS when fork is invoked). */
807 /* This should not be a problem because the lost elements */
808 /* of the free lists will be collected during GC. */
809 GC_remove_specific_after_fork(GC_thread_key, p -> id);
812 /* TODO: To avoid TSan hang (when updating GC_bytes_freed), */
813 /* we just skip explicit freeing of GC_threads entries. */
814 # if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
815 if (p != &first_thread) GC_INTERNAL_FREE(p);
819 store_to_threads_table(hv, me);
822 #endif /* CAN_HANDLE_FORK */
824 #ifdef USE_PROC_FOR_LIBRARIES
825 GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
830 GC_ASSERT(I_HOLD_LOCK());
831 # ifdef PARALLEL_MARK
832 for (i = 0; i < GC_markers_m1; ++i) {
833 if ((word)marker_sp[i] > (word)lo && (word)marker_sp[i] < (word)hi)
836 if ((word)marker_bsp[i] > (word)lo
837 && (word)marker_bsp[i] < (word)hi)
842 for (i = 0; i < THREAD_TABLE_SZ; i++) {
843 for (p = GC_threads[i]; p != 0; p = p -> next) {
844 if (0 != p -> stack_end) {
845 # ifdef STACK_GROWS_UP
846 if ((word)p->stack_end >= (word)lo
847 && (word)p->stack_end < (word)hi)
849 # else /* STACK_GROWS_DOWN */
850 if ((word)p->stack_end > (word)lo
851 && (word)p->stack_end <= (word)hi)
859 #endif /* USE_PROC_FOR_LIBRARIES */
862 /* Find the largest stack_base smaller than bound. May be used */
863 /* to find the boundary between a register stack and adjacent */
864 /* immediately preceding memory stack. */
865 GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound)
871 GC_ASSERT(I_HOLD_LOCK());
872 # ifdef PARALLEL_MARK
873 for (i = 0; i < GC_markers_m1; ++i) {
874 if ((word)marker_sp[i] > (word)result
875 && (word)marker_sp[i] < (word)bound)
876 result = marker_sp[i];
879 for (i = 0; i < THREAD_TABLE_SZ; i++) {
880 for (p = GC_threads[i]; p != 0; p = p -> next) {
881 if ((word)p->stack_end > (word)result
882 && (word)p->stack_end < (word)bound) {
883 result = p -> stack_end;
892 /* Also defined in os_dep.c. */
893 # define STAT_BUF_SIZE 4096
894 # define STAT_READ read
895 /* If read is wrapped, this may need to be redefined to call */
899 #ifdef GC_HPUX_THREADS
900 # define GC_get_nprocs() pthread_num_processors_np()
902 #elif defined(GC_OSF1_THREADS) || defined(GC_AIX_THREADS) \
903 || defined(GC_HAIKU_THREADS) || defined(GC_SOLARIS_THREADS) \
904 || defined(HURD) || defined(HOST_ANDROID) || defined(NACL)
905 GC_INLINE int GC_get_nprocs(void)
907 int nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
908 return nprocs > 0 ? nprocs : 1; /* ignore error silently */
911 #elif defined(GC_IRIX_THREADS)
912 GC_INLINE int GC_get_nprocs(void)
914 int nprocs = (int)sysconf(_SC_NPROC_ONLN);
915 return nprocs > 0 ? nprocs : 1; /* ignore error silently */
918 #elif defined(GC_LINUX_THREADS) /* && !HOST_ANDROID && !NACL */
919 /* Return the number of processors. */
920 STATIC int GC_get_nprocs(void)
922 /* Should be "return sysconf(_SC_NPROCESSORS_ONLN);" but that */
923 /* appears to be buggy in many cases. */
924 /* We look for lines "cpu<n>" in /proc/stat. */
925 char stat_buf[STAT_BUF_SIZE];
929 f = open("/proc/stat", O_RDONLY);
931 WARN("Couldn't read /proc/stat\n", 0);
932 return 1; /* assume an uniprocessor */
934 len = STAT_READ(f, stat_buf, STAT_BUF_SIZE);
938 /* Some old kernels only have a single "cpu nnnn ..." */
939 /* entry in /proc/stat. We identify those as */
942 for (i = 0; i < len - 100; ++i) {
943 if (stat_buf[i] == '\n' && stat_buf[i+1] == 'c'
944 && stat_buf[i+2] == 'p' && stat_buf[i+3] == 'u') {
945 int cpu_no = atoi(&stat_buf[i + 4]);
946 if (cpu_no >= result)
953 #elif defined(GC_DGUX386_THREADS)
954 /* Return the number of processors, or i <= 0 if it can't be determined. */
955 STATIC int GC_get_nprocs(void)
958 struct dg_sys_info_pm_info pm_sysinfo;
961 status = dg_sys_info((long int *) &pm_sysinfo,
962 DG_SYS_INFO_PM_INFO_TYPE, DG_SYS_INFO_PM_CURRENT_VERSION);
964 /* set -1 for error */
968 numCpus = pm_sysinfo.idle_vp_count;
972 #elif defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS) \
973 || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
974 STATIC int GC_get_nprocs(void)
976 int mib[] = {CTL_HW,HW_NCPU};
978 size_t len = sizeof(res);
980 sysctl(mib, sizeof(mib)/sizeof(int), &res, &len, NULL, 0);
985 /* E.g., GC_RTEMS_PTHREADS */
986 # define GC_get_nprocs() 1 /* not implemented */
987 #endif /* !GC_LINUX_THREADS && !GC_DARWIN_THREADS && ... */
989 #if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
990 /* Some buggy Linux/arm kernels show only non-sleeping CPUs in */
991 /* /proc/stat (and /proc/cpuinfo), so another data system source is */
992 /* tried first. Result <= 0 on error. */
993 STATIC int GC_get_nprocs_present(void)
999 f = open("/sys/devices/system/cpu/present", O_RDONLY);
1001 return -1; /* cannot open the file */
1003 len = STAT_READ(f, stat_buf, sizeof(stat_buf));
1006 /* Recognized file format: "0\n" or "0-<max_cpu_id>\n" */
1007 /* The file might probably contain a comma-separated list */
1008 /* but we do not need to handle it (just silently ignore). */
1009 if (len < 2 || stat_buf[0] != '0' || stat_buf[len - 1] != '\n') {
1010 return 0; /* read error or unrecognized content */
1011 } else if (len == 2) {
1012 return 1; /* an uniprocessor */
1013 } else if (stat_buf[1] != '-') {
1014 return 0; /* unrecognized content */
1017 stat_buf[len - 1] = '\0'; /* terminate the string */
1018 return atoi(&stat_buf[2]) + 1; /* skip "0-" and parse max_cpu_num */
1020 #endif /* ARM32 && GC_LINUX_THREADS && !NACL */
1022 /* We hold the GC lock. Wait until an in-progress GC has finished. */
1023 /* Repeatedly RELEASES GC LOCK in order to wait. */
1024 /* If wait_for_all is true, then we exit with the GC lock held and no */
1025 /* collection in progress; otherwise we just wait for the current GC */
1027 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
1030 # if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
1031 /* GC_lock_holder is accessed with the lock held, so there is no */
1032 /* data race actually (unlike what is reported by TSan). */
1033 GC_ASSERT(I_HOLD_LOCK());
1035 ASSERT_CANCEL_DISABLED();
1036 if (GC_incremental && GC_collection_in_progress()) {
1037 word old_gc_no = GC_gc_no;
1039 /* Make sure that no part of our stack is still on the mark stack, */
1040 /* since it's about to be unmapped. */
1041 while (GC_incremental && GC_collection_in_progress()
1042 && (wait_for_all || old_gc_no == GC_gc_no)) {
1044 GC_in_thread_creation = TRUE;
1045 GC_collect_a_little_inner(1);
1046 GC_in_thread_creation = FALSE;
1055 #ifdef CAN_HANDLE_FORK
1056 /* Procedures called before and after a fork. The goal here is to make */
1057 /* it safe to call GC_malloc() in a forked child. It's unclear that is */
1058 /* attainable, since the single UNIX spec seems to imply that one */
1059 /* should only call async-signal-safe functions, and we probably can't */
1060 /* quite guarantee that. But we give it our best shot. (That same */
1061 /* spec also implies that it's not safe to call the system malloc */
1062 /* between fork() and exec(). Thus we're doing no worse than it.) */
1064 IF_CANCEL(static int fork_cancel_state;)
1065 /* protected by allocation lock. */
1067 /* Called before a fork() */
1068 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1069 /* GC_lock_holder is updated safely (no data race actually). */
1070 GC_ATTR_NO_SANITIZE_THREAD
1072 static void fork_prepare_proc(void)
1074 /* Acquire all relevant locks, so that after releasing the locks */
1075 /* the child will see a consistent state in which monitor */
1076 /* invariants hold. Unfortunately, we can't acquire libc locks */
1077 /* we might need, and there seems to be no guarantee that libc */
1078 /* must install a suitable fork handler. */
1079 /* Wait for an ongoing GC to finish, since we can't finish it in */
1080 /* the (one remaining thread in) the child. */
1082 DISABLE_CANCEL(fork_cancel_state);
1083 /* Following waits may include cancellation points. */
1084 # if defined(PARALLEL_MARK)
1086 GC_wait_for_reclaim();
1088 GC_wait_for_gc_completion(TRUE);
1089 # if defined(PARALLEL_MARK)
1091 GC_acquire_mark_lock();
1093 GC_acquire_dirty_lock();
1096 /* Called in parent after a fork() (even if the latter failed). */
1097 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1098 GC_ATTR_NO_SANITIZE_THREAD
1100 static void fork_parent_proc(void)
1102 GC_release_dirty_lock();
1103 # if defined(PARALLEL_MARK)
1105 GC_release_mark_lock();
1107 RESTORE_CANCEL(fork_cancel_state);
1111 /* Called in child after a fork() */
1112 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1113 GC_ATTR_NO_SANITIZE_THREAD
1115 static void fork_child_proc(void)
1117 GC_release_dirty_lock();
1118 # if defined(PARALLEL_MARK)
1120 GC_release_mark_lock();
1122 /* Clean up the thread table, so that just our thread is left. */
1123 GC_remove_all_threads_but_me();
1124 # ifdef PARALLEL_MARK
1125 /* Turn off parallel marking in the child, since we are probably */
1126 /* just going to exec, and we would have to restart mark threads. */
1127 GC_parallel = FALSE;
1128 # endif /* PARALLEL_MARK */
1129 RESTORE_CANCEL(fork_cancel_state);
1131 /* Even though after a fork the child only inherits the single */
1132 /* thread that called the fork(), if another thread in the parent */
1133 /* was attempting to lock the mutex while being held in */
1134 /* fork_child_prepare(), the mutex will be left in an inconsistent */
1135 /* state in the child after the UNLOCK. This is the case, at */
1136 /* least, in Mac OS X and leads to an unusable GC in the child */
1137 /* which will block when attempting to perform any GC operation */
1138 /* that acquires the allocation mutex. */
1139 # ifdef USE_PTHREAD_LOCKS
1140 GC_ASSERT(I_DONT_HOLD_LOCK());
1141 /* Reinitialize the mutex. It should be safe since we are */
1142 /* running this in the child which only inherits a single thread. */
1143 /* mutex_destroy() may return EBUSY, which makes no sense, but */
1144 /* that is the reason for the need of the reinitialization. */
1145 (void)pthread_mutex_destroy(&GC_allocate_ml);
1146 /* TODO: Probably some targets might need the default mutex */
1147 /* attribute to be passed instead of NULL. */
1148 if (0 != pthread_mutex_init(&GC_allocate_ml, NULL))
1149 ABORT("pthread_mutex_init failed (in child)");
1153 /* Routines for fork handling by client (no-op if pthread_atfork works). */
1154 GC_API void GC_CALL GC_atfork_prepare(void)
1156 if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
1157 # if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB)
1158 if (GC_auto_incremental) {
1159 GC_ASSERT(0 == GC_handle_fork);
1160 ABORT("Unable to fork while mprotect_thread is running");
1163 if (GC_handle_fork <= 0)
1164 fork_prepare_proc();
1167 GC_API void GC_CALL GC_atfork_parent(void)
1169 if (GC_handle_fork <= 0)
1173 GC_API void GC_CALL GC_atfork_child(void)
1175 if (GC_handle_fork <= 0)
1178 #endif /* CAN_HANDLE_FORK */
1180 #ifdef INCLUDE_LINUX_THREAD_DESCR
1181 __thread int GC_dummy_thread_local;
1184 #ifdef PARALLEL_MARK
1185 static void setup_mark_lock(void);
1188 GC_INNER void GC_thr_init(void)
1190 GC_ASSERT(I_HOLD_LOCK());
1191 if (GC_thr_initialized) return;
1192 GC_thr_initialized = TRUE;
1194 GC_ASSERT((word)&GC_threads % sizeof(word) == 0);
1195 # ifdef CAN_HANDLE_FORK
1196 /* Prepare for forks if requested. */
1197 if (GC_handle_fork) {
1198 # ifdef CAN_CALL_ATFORK
1199 if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
1200 fork_child_proc) == 0) {
1201 /* Handlers successfully registered. */
1205 /* else */ if (GC_handle_fork != -1)
1206 ABORT("pthread_atfork failed");
1209 # ifdef INCLUDE_LINUX_THREAD_DESCR
1210 /* Explicitly register the region including the address */
1211 /* of a thread local variable. This should include thread */
1212 /* locals for the main thread, except for those allocated */
1213 /* in response to dlopen calls. */
1215 ptr_t thread_local_addr = (ptr_t)(&GC_dummy_thread_local);
1216 ptr_t main_thread_start, main_thread_end;
1217 if (!GC_enclosing_mapping(thread_local_addr, &main_thread_start,
1218 &main_thread_end)) {
1219 ABORT("Failed to find mapping for main thread thread locals");
1221 /* main_thread_start and main_thread_end are initialized. */
1222 GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
1226 /* Add the initial thread, so we can stop it. */
1228 pthread_t self = pthread_self();
1229 GC_thread t = GC_new_thread(self);
1232 ABORT("Failed to allocate memory for the initial thread");
1233 # ifdef GC_DARWIN_THREADS
1234 t -> stop_info.mach_thread = mach_thread_self();
1236 t -> stop_info.stack_ptr = GC_approx_sp();
1238 t -> flags = DETACHED | MAIN_THREAD;
1239 if (THREAD_EQUAL(self, main_pthread_id)) {
1240 t -> stack = (ptr_t)main_stack;
1241 t -> stack_size = main_stack_size;
1242 t -> altstack = (ptr_t)main_altstack;
1243 t -> altstack_size = main_altstack_size;
1247 # ifndef GC_DARWIN_THREADS
1251 /* Set GC_nprocs. */
1253 char * nprocs_string = GETENV("GC_NPROCS");
1255 if (nprocs_string != NULL) GC_nprocs = atoi(nprocs_string);
1258 # if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
1259 && (GC_nprocs = GC_get_nprocs_present()) <= 1
1260 /* Workaround for some Linux/arm kernels */
1264 GC_nprocs = GC_get_nprocs();
1266 if (GC_nprocs <= 0) {
1267 WARN("GC_get_nprocs() returned %" WARN_PRIdPTR "\n", GC_nprocs);
1268 GC_nprocs = 2; /* assume dual-core */
1269 # ifdef PARALLEL_MARK
1270 available_markers_m1 = 0; /* but use only one marker */
1273 # ifdef PARALLEL_MARK
1275 char * markers_string = GETENV("GC_MARKERS");
1278 if (markers_string != NULL) {
1279 markers = atoi(markers_string);
1280 if (markers <= 0 || markers > MAX_MARKERS) {
1281 WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
1282 "; using maximum threads\n", (signed_word)markers);
1283 markers = MAX_MARKERS;
1286 markers = GC_nprocs;
1287 # if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
1288 /* This is primarily for targets without getenv(). */
1289 if (markers < GC_MIN_MARKERS)
1290 markers = GC_MIN_MARKERS;
1292 if (markers > MAX_MARKERS)
1293 markers = MAX_MARKERS; /* silently limit the value */
1295 available_markers_m1 = markers - 1;
1299 GC_COND_LOG_PRINTF("Number of processors = %d\n", GC_nprocs);
1300 # ifdef PARALLEL_MARK
1301 if (available_markers_m1 <= 0) {
1302 /* Disable parallel marking. */
1303 GC_parallel = FALSE;
1305 "Single marker thread, turning off parallel marking\n");
1312 /* Perform all initializations, including those that */
1313 /* may require allocation. */
1314 /* Called without allocation lock. */
1315 /* Must be called before a second thread is created. */
1316 /* Did we say it's called without the allocation lock? */
1317 GC_INNER void GC_init_parallel(void)
1319 # if defined(THREAD_LOCAL_ALLOC)
1322 if (parallel_initialized) return;
1323 parallel_initialized = TRUE;
1325 /* GC_init() calls us back, so set flag first. */
1326 if (!GC_is_initialized) GC_init();
1327 /* Initialize thread local free lists if used. */
1328 # if defined(THREAD_LOCAL_ALLOC)
1330 GC_init_thread_local(&(GC_lookup_thread(pthread_self())->tlfs));
1335 #ifndef GC_NO_PTHREAD_SIGMASK
1336 GC_API int WRAP_FUNC(pthread_sigmask)(int how, const sigset_t *set,
1339 sigset_t fudged_set;
1342 if (set != NULL && (how == SIG_BLOCK || how == SIG_SETMASK)) {
1343 int sig_suspend = GC_get_suspend_signal();
1346 GC_ASSERT(sig_suspend >= 0);
1347 if (sigdelset(&fudged_set, sig_suspend) != 0)
1348 ABORT("sigdelset failed");
1351 return(REAL_FUNC(pthread_sigmask)(how, set, oset));
1353 #endif /* !GC_NO_PTHREAD_SIGMASK */
1355 /* Wrapper for functions that are likely to block for an appreciable */
1356 /* length of time. */
1358 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
1360 struct blocking_data * d = (struct blocking_data *) data;
1361 pthread_t self = pthread_self();
1363 # if defined(SPARC) || defined(IA64)
1364 ptr_t stack_ptr = GC_save_regs_in_stack();
1366 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1367 GC_bool topOfStackUnset = FALSE;
1372 me = GC_lookup_thread(self);
1373 GC_ASSERT(!(me -> thread_blocked));
1375 me -> stop_info.stack_ptr = stack_ptr;
1377 me -> stop_info.stack_ptr = GC_approx_sp();
1379 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1380 if (me -> topOfStack == NULL) {
1381 /* GC_do_blocking_inner is not called recursively, */
1382 /* so topOfStack should be computed now. */
1383 topOfStackUnset = TRUE;
1384 me -> topOfStack = GC_FindTopOfStack(0);
1388 me -> backing_store_ptr = stack_ptr;
1390 me -> thread_blocked = (unsigned char)TRUE;
1391 /* Save context here if we want to support precise stack marking */
1393 d -> client_data = (d -> fn)(d -> client_data);
1394 LOCK(); /* This will block if the world is stopped. */
1395 # if defined(CPPCHECK)
1396 GC_noop1((unsigned)me->thread_blocked);
1398 me -> thread_blocked = FALSE;
1399 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1400 if (topOfStackUnset)
1401 me -> topOfStack = NULL; /* make topOfStack unset again */
1406 GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
1407 const struct GC_stack_base *sb)
1409 GC_thread t = (GC_thread)gc_thread_handle;
1411 GC_ASSERT(sb -> mem_base != NULL);
1412 if (!EXPECT(GC_is_initialized, TRUE)) {
1413 GC_ASSERT(NULL == t);
1415 GC_ASSERT(I_HOLD_LOCK());
1416 if (NULL == t) /* current thread? */
1417 t = GC_lookup_thread(pthread_self());
1418 GC_ASSERT((t -> flags & FINISHED) == 0);
1419 GC_ASSERT(!(t -> thread_blocked)
1420 && NULL == t -> traced_stack_sect); /* for now */
1422 if ((t -> flags & MAIN_THREAD) == 0) {
1423 t -> stack_end = (ptr_t)sb->mem_base;
1425 t -> backing_store_end = (ptr_t)sb->reg_base;
1429 /* Otherwise alter the stack bottom of the primordial thread. */
1432 GC_stackbottom = (char*)sb->mem_base;
1434 GC_register_stackbottom = (ptr_t)sb->reg_base;
1438 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
1440 pthread_t self = pthread_self();
1445 me = GC_lookup_thread(self);
1446 /* The thread is assumed to be registered. */
1447 if ((me -> flags & MAIN_THREAD) == 0) {
1448 sb -> mem_base = me -> stack_end;
1450 sb -> reg_base = me -> backing_store_end;
1453 sb -> mem_base = GC_stackbottom;
1455 sb -> reg_base = GC_register_stackbottom;
1459 return (void *)me; /* gc_thread_handle */
1462 /* GC_call_with_gc_active() has the opposite to GC_do_blocking() */
1463 /* functionality. It might be called from a user function invoked by */
1464 /* GC_do_blocking() to temporarily back allow calling any GC function */
1465 /* and/or manipulating pointers to the garbage collected heap. */
1466 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
1469 struct GC_traced_stack_sect_s stacksect;
1470 pthread_t self = pthread_self();
1474 LOCK(); /* This will block if the world is stopped. */
1475 me = GC_lookup_thread(self);
1477 /* Adjust our stack bottom value (this could happen unless */
1478 /* GC_get_stack_base() was used which returned GC_SUCCESS). */
1479 if ((me -> flags & MAIN_THREAD) == 0) {
1480 GC_ASSERT(me -> stack_end != NULL);
1481 if ((word)me->stack_end HOTTER_THAN (word)(&stacksect))
1482 me -> stack_end = (ptr_t)(&stacksect);
1484 /* The original stack. */
1485 if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect))
1486 GC_stackbottom = (ptr_t)COVERT_DATAFLOW(&stacksect);
1489 if (!me->thread_blocked) {
1490 /* We are not inside GC_do_blocking() - do nothing more. */
1492 client_data = fn(client_data);
1493 /* Prevent treating the above as a tail call. */
1494 GC_noop1(COVERT_DATAFLOW(&stacksect));
1495 return client_data; /* result */
1498 /* Setup new "stack section". */
1499 stacksect.saved_stack_ptr = me -> stop_info.stack_ptr;
1501 /* This is the same as in GC_call_with_stack_base(). */
1502 stacksect.backing_store_end = GC_save_regs_in_stack();
1503 /* Unnecessarily flushes register stack, */
1504 /* but that probably doesn't hurt. */
1505 stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
1507 stacksect.prev = me -> traced_stack_sect;
1508 me -> thread_blocked = FALSE;
1509 me -> traced_stack_sect = &stacksect;
1512 client_data = fn(client_data);
1513 GC_ASSERT(me -> thread_blocked == FALSE);
1514 GC_ASSERT(me -> traced_stack_sect == &stacksect);
1516 /* Restore original "stack section". */
1517 # if defined(CPPCHECK)
1518 GC_noop1((word)me->traced_stack_sect);
1521 me -> traced_stack_sect = stacksect.prev;
1523 me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
1525 me -> thread_blocked = (unsigned char)TRUE;
1526 me -> stop_info.stack_ptr = stacksect.saved_stack_ptr;
1529 return client_data; /* result */
1532 STATIC void GC_unregister_my_thread_inner(GC_thread me)
1534 # ifdef DEBUG_THREADS
1536 "Unregistering thread %p, gc_thread = %p, n_threads = %d\n",
1537 (void *)me->id, (void *)me, GC_count_threads());
1539 GC_ASSERT(!(me -> flags & FINISHED));
1540 # if defined(THREAD_LOCAL_ALLOC)
1541 GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
1542 GC_destroy_thread_local(&(me->tlfs));
1544 # if defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)
1545 /* Handle DISABLED_GC flag which is set by the */
1546 /* intercepted pthread_cancel or pthread_exit. */
1547 if ((me -> flags & DISABLED_GC) != 0) {
1551 if (me -> flags & DETACHED) {
1552 GC_delete_thread(pthread_self());
1554 me -> flags |= FINISHED;
1556 # if defined(THREAD_LOCAL_ALLOC)
1557 /* It is required to call remove_specific defined in specific.c. */
1558 GC_remove_specific(GC_thread_key);
1562 GC_API int GC_CALL GC_unregister_my_thread(void)
1564 pthread_t self = pthread_self();
1566 IF_CANCEL(int cancel_state;)
1570 DISABLE_CANCEL(cancel_state);
1571 /* Wait for any GC that may be marking from our stack to */
1572 /* complete before we remove this thread. */
1573 GC_wait_for_gc_completion(FALSE);
1574 me = GC_lookup_thread(self);
1575 # ifdef DEBUG_THREADS
1577 "Called GC_unregister_my_thread on %p, gc_thread = %p\n",
1578 (void *)self, (void *)me);
1580 GC_ASSERT(THREAD_EQUAL(me->id, self));
1581 GC_unregister_my_thread_inner(me);
1582 RESTORE_CANCEL(cancel_state);
1587 /* Called at thread exit. */
1588 /* Never called for main thread. That's OK, since it */
1589 /* results in at most a tiny one-time leak. And */
1590 /* linuxthreads doesn't reclaim the main threads */
1591 /* resources or id anyway. */
1592 GC_INNER_PTHRSTART void GC_thread_exit_proc(void *arg)
1594 IF_CANCEL(int cancel_state;)
1597 # ifdef DEBUG_THREADS
1598 GC_log_printf("Called GC_thread_exit_proc on %p, gc_thread = %p\n",
1599 (void *)((GC_thread)arg)->id, arg);
1602 DISABLE_CANCEL(cancel_state);
1603 GC_wait_for_gc_completion(FALSE);
1604 GC_unregister_my_thread_inner((GC_thread)arg);
1605 RESTORE_CANCEL(cancel_state);
1609 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
1610 GC_API int WRAP_FUNC(pthread_join)(pthread_t thread, void **retval)
1616 ASSERT_SYMS_INITIALIZED();
1618 t = GC_lookup_thread(thread);
1619 /* This is guaranteed to be the intended one, since the thread id */
1620 /* can't have been recycled by pthreads. */
1622 result = REAL_FUNC(pthread_join)(thread, retval);
1623 # if defined(GC_FREEBSD_THREADS)
1624 /* On FreeBSD, the wrapped pthread_join() sometimes returns (what
1625 appears to be) a spurious EINTR which caused the test and real code
1626 to gratuitously fail. Having looked at system pthread library source
1627 code, I see how this return code may be generated. In one path of
1628 code, pthread_join() just returns the errno setting of the thread
1629 being joined. This does not match the POSIX specification or the
1630 local man pages thus I have taken the liberty to catch this one
1631 spurious return value properly conditionalized on GC_FREEBSD_THREADS. */
1632 if (result == EINTR) result = 0;
1636 /* Here the pthread thread id may have been recycled. */
1637 /* Delete the thread from GC_threads (unless it has been */
1638 /* registered again from the client thread key destructor). */
1639 if ((t -> flags & FINISHED) != 0)
1640 GC_delete_gc_thread(t);
1646 GC_API int WRAP_FUNC(pthread_detach)(pthread_t thread)
1652 ASSERT_SYMS_INITIALIZED();
1654 t = GC_lookup_thread(thread);
1656 result = REAL_FUNC(pthread_detach)(thread);
1659 t -> flags |= DETACHED;
1660 /* Here the pthread thread id may have been recycled. */
1661 if ((t -> flags & FINISHED) != 0) {
1662 GC_delete_gc_thread(t);
1668 #endif /* !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */
1670 #ifndef GC_NO_PTHREAD_CANCEL
1671 /* We should deal with the fact that apparently on Solaris and, */
1672 /* probably, on some Linux we can't collect while a thread is */
1673 /* exiting, since signals aren't handled properly. This currently */
1674 /* gives rise to deadlocks. The only workaround seen is to intercept */
1675 /* pthread_cancel() and pthread_exit(), and disable the collections */
1676 /* until the thread exit handler is called. That's ugly, because we */
1677 /* risk growing the heap unnecessarily. But it seems that we don't */
1678 /* really have an option in that the process is not in a fully */
1679 /* functional state while a thread is exiting. */
1680 GC_API int WRAP_FUNC(pthread_cancel)(pthread_t thread)
1690 t = GC_lookup_thread(thread);
1691 /* We test DISABLED_GC because pthread_exit could be called at */
1692 /* the same time. (If t is NULL then pthread_cancel should */
1693 /* return ESRCH.) */
1694 if (t != NULL && (t -> flags & DISABLED_GC) == 0) {
1695 t -> flags |= DISABLED_GC;
1700 return REAL_FUNC(pthread_cancel)(thread);
1702 #endif /* !GC_NO_PTHREAD_CANCEL */
1704 #ifdef GC_HAVE_PTHREAD_EXIT
1705 GC_API GC_PTHREAD_EXIT_ATTRIBUTE void WRAP_FUNC(pthread_exit)(void *retval)
1707 pthread_t self = pthread_self();
1713 me = GC_lookup_thread(self);
1714 /* We test DISABLED_GC because someone else could call */
1715 /* pthread_cancel at the same time. */
1716 if (me != 0 && (me -> flags & DISABLED_GC) == 0) {
1717 me -> flags |= DISABLED_GC;
1722 REAL_FUNC(pthread_exit)(retval);
1724 #endif /* GC_HAVE_PTHREAD_EXIT */
1726 GC_INNER GC_bool GC_in_thread_creation = FALSE;
1727 /* Protected by allocation lock. */
1729 GC_INLINE void GC_record_stack_base(GC_thread me,
1730 const struct GC_stack_base *sb)
1732 # ifndef GC_DARWIN_THREADS
1733 me -> stop_info.stack_ptr = (ptr_t)sb->mem_base;
1735 me -> stack_end = (ptr_t)sb->mem_base;
1736 if (me -> stack_end == NULL)
1737 ABORT("Bad stack base in GC_register_my_thread");
1739 me -> backing_store_end = (ptr_t)sb->reg_base;
1743 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
1744 pthread_t my_pthread)
1748 GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */
1749 me = GC_new_thread(my_pthread);
1750 GC_in_thread_creation = FALSE;
1752 ABORT("Failed to allocate memory for thread registering");
1753 # ifdef GC_DARWIN_THREADS
1754 me -> stop_info.mach_thread = mach_thread_self();
1756 GC_record_stack_base(me, sb);
1757 # ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
1758 /* Since this could be executed from a detached thread */
1759 /* destructor, our signals might already be blocked. */
1760 GC_unblock_gc_signals();
1765 GC_API void GC_CALL GC_allow_register_threads(void)
1767 /* Check GC is initialized and the current thread is registered. */
1768 GC_ASSERT(GC_lookup_thread(pthread_self()) != 0);
1772 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
1774 pthread_t self = pthread_self();
1778 if (GC_need_to_lock == FALSE)
1779 ABORT("Threads explicit registering is not previously enabled");
1782 me = GC_lookup_thread(self);
1784 me = GC_register_my_thread_inner(sb, self);
1785 # if defined(CPPCHECK)
1786 GC_noop1(me->flags);
1788 me -> flags |= DETACHED;
1789 /* Treat as detached, since we do not need to worry about */
1790 /* pointer results. */
1791 # if defined(THREAD_LOCAL_ALLOC)
1792 GC_init_thread_local(&(me->tlfs));
1796 } else if ((me -> flags & FINISHED) != 0) {
1797 /* This code is executed when a thread is registered from the */
1798 /* client thread key destructor. */
1799 # ifdef GC_DARWIN_THREADS
1800 /* Reinitialize mach_thread to avoid thread_suspend fail */
1801 /* with MACH_SEND_INVALID_DEST error. */
1802 me -> stop_info.mach_thread = mach_thread_self();
1804 GC_record_stack_base(me, sb);
1805 me -> flags &= ~FINISHED; /* but not DETACHED */
1806 # ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
1807 /* Since this could be executed from a thread destructor, */
1808 /* our signals might be blocked. */
1809 GC_unblock_gc_signals();
1811 # if defined(THREAD_LOCAL_ALLOC)
1812 GC_init_thread_local(&(me->tlfs));
1818 return GC_DUPLICATE;
1823 void *(*start_routine)(void *);
1826 sem_t registered; /* 1 ==> in our thread table, but */
1827 /* parent hasn't yet noticed. */
1830 /* Called from GC_inner_start_routine(). Defined in this file to */
1831 /* minimize the number of include files in pthread_start.c (because */
1832 /* sem_t and sem_post() are not used that file directly). */
1833 GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
1834 void *(**pstart)(void *),
1836 struct GC_stack_base *sb, void *arg)
1838 struct start_info * si = (struct start_info *)arg;
1839 pthread_t self = pthread_self();
1843 # ifdef DEBUG_THREADS
1844 GC_log_printf("Starting thread %p, pid = %ld, sp = %p\n",
1845 (void *)self, (long)getpid(), (void *)&arg);
1848 me = GC_register_my_thread_inner(sb, self);
1849 me -> flags = si -> flags;
1850 # if defined(THREAD_LOCAL_ALLOC)
1851 GC_init_thread_local(&(me->tlfs));
1854 *pstart = si -> start_routine;
1855 # ifdef DEBUG_THREADS
1856 GC_log_printf("start_routine = %p\n", (void *)(signed_word)(*pstart));
1858 *pstart_arg = si -> arg;
1859 sem_post(&(si -> registered)); /* Last action on si. */
1860 /* OK to deallocate. */
1864 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
1865 STATIC void * GC_start_routine(void * arg)
1867 # ifdef INCLUDE_LINUX_THREAD_DESCR
1868 struct GC_stack_base sb;
1870 # ifdef REDIRECT_MALLOC
1871 /* GC_get_stack_base may call pthread_getattr_np, which can */
1872 /* unfortunately call realloc, which may allocate from an */
1873 /* unregistered thread. This is unpleasant, since it might */
1874 /* force heap growth (or, even, heap overflow). */
1877 if (GC_get_stack_base(&sb) != GC_SUCCESS)
1878 ABORT("Failed to get thread stack base");
1879 # ifdef REDIRECT_MALLOC
1882 return GC_inner_start_routine(&sb, arg);
1884 return GC_call_with_stack_base(GC_inner_start_routine, arg);
1888 GC_API int WRAP_FUNC(pthread_create)(pthread_t *new_thread,
1889 GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
1890 void *(*start_routine)(void *), void *arg)
1895 struct start_info * si;
1897 /* This is otherwise saved only in an area mmapped by the thread */
1898 /* library, which isn't visible to the collector. */
1900 /* We resist the temptation to muck with the stack size here, */
1901 /* even if the default is unreasonably small. That's the client's */
1902 /* responsibility. */
1906 si = (struct start_info *)GC_INTERNAL_MALLOC(sizeof(struct start_info),
1909 if (!EXPECT(parallel_initialized, TRUE))
1911 if (EXPECT(0 == si, FALSE) &&
1912 (si = (struct start_info *)
1913 (*GC_get_oom_fn())(sizeof(struct start_info))) == 0)
1915 if (sem_init(&(si -> registered), GC_SEM_INIT_PSHARED, 0) != 0)
1916 ABORT("sem_init failed");
1918 si -> start_routine = start_routine;
1921 if (!EXPECT(GC_thr_initialized, TRUE))
1923 # ifdef GC_ASSERTIONS
1925 size_t stack_size = 0;
1927 if (pthread_attr_getstacksize(attr, &stack_size) != 0)
1928 ABORT("pthread_attr_getstacksize failed");
1930 if (0 == stack_size) {
1931 pthread_attr_t my_attr;
1933 if (pthread_attr_init(&my_attr) != 0)
1934 ABORT("pthread_attr_init failed");
1935 if (pthread_attr_getstacksize(&my_attr, &stack_size) != 0)
1936 ABORT("pthread_attr_getstacksize failed");
1937 (void)pthread_attr_destroy(&my_attr);
1939 /* On Solaris 10, with default attr initialization, */
1940 /* stack_size remains 0. Fudge it. */
1941 if (0 == stack_size) {
1943 WARN("Failed to get stack size for assertion checking\n", 0);
1945 stack_size = 1000000;
1947 GC_ASSERT(stack_size >= 65536);
1948 /* Our threads may need to do some work for the GC. */
1949 /* Ridiculously small threads won't work, and they */
1950 /* probably wouldn't work anyway. */
1954 detachstate = PTHREAD_CREATE_JOINABLE;
1956 pthread_attr_getdetachstate(attr, &detachstate);
1958 if (PTHREAD_CREATE_DETACHED == detachstate) my_flags |= DETACHED;
1959 si -> flags = my_flags;
1961 # ifdef DEBUG_THREADS
1962 GC_log_printf("About to start new thread from thread %p\n",
1963 (void *)pthread_self());
1966 result = REAL_FUNC(pthread_create)(new_thread, attr, GC_start_routine, si);
1968 /* Wait until child has been added to the thread table. */
1969 /* This also ensures that we hold onto si until the child is done */
1970 /* with it. Thus it doesn't matter whether it is otherwise */
1971 /* visible to the collector. */
1973 IF_CANCEL(int cancel_state;)
1975 # ifdef DEBUG_THREADS
1976 /* new_thread is non-NULL because pthread_create requires it. */
1977 GC_log_printf("Started thread %p\n", (void *)(*new_thread));
1979 DISABLE_CANCEL(cancel_state);
1980 /* pthread_create is not a cancellation point. */
1981 while (0 != sem_wait(&(si -> registered))) {
1982 # if defined(GC_HAIKU_THREADS)
1983 /* To workaround some bug in Haiku semaphores. */
1984 if (EACCES == errno) continue;
1986 if (EINTR != errno) ABORT("sem_wait failed");
1988 RESTORE_CANCEL(cancel_state);
1990 sem_destroy(&(si -> registered));
1992 GC_INTERNAL_FREE(si);
1997 #endif /* !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */
1999 #if defined(USE_SPIN_LOCK) || !defined(NO_PTHREAD_TRYLOCK)
2000 /* Spend a few cycles in a way that can't introduce contention with */
2001 /* other threads. */
2002 #define GC_PAUSE_SPIN_CYCLES 10
2003 STATIC void GC_pause(void)
2007 for (i = 0; i < GC_PAUSE_SPIN_CYCLES; ++i) {
2008 /* Something that's unlikely to be optimized away. */
2009 # if defined(AO_HAVE_compiler_barrier) \
2010 && !defined(BASE_ATOMIC_OPS_EMULATED)
2011 AO_compiler_barrier();
2020 # define SPIN_MAX 128 /* Maximum number of calls to GC_pause before */
2024 GC_INNER volatile GC_bool GC_collecting = FALSE;
2025 /* A hint that we're in the collector and */
2026 /* holding the allocation lock for an */
2027 /* extended period. */
2029 #if (!defined(USE_SPIN_LOCK) && !defined(NO_PTHREAD_TRYLOCK)) \
2030 || defined(PARALLEL_MARK)
2031 /* If we don't want to use the below spinlock implementation, either */
2032 /* because we don't have a GC_test_and_set implementation, or because */
2033 /* we don't want to risk sleeping, we can still try spinning on */
2034 /* pthread_mutex_trylock for a while. This appears to be very */
2035 /* beneficial in many cases. */
2036 /* I suspect that under high contention this is nearly always better */
2037 /* than the spin lock. But it's a bit slower on a uniprocessor. */
2038 /* Hence we still default to the spin lock. */
2039 /* This is also used to acquire the mark lock for the parallel */
2042 /* Here we use a strict exponential backoff scheme. I don't know */
2043 /* whether that's better or worse than the above. We eventually */
2044 /* yield by calling pthread_mutex_lock(); it never makes sense to */
2045 /* explicitly sleep. */
2047 /* #define LOCK_STATS */
2048 /* Note that LOCK_STATS requires AO_HAVE_test_and_set. */
2050 volatile AO_t GC_spin_count = 0;
2051 volatile AO_t GC_block_count = 0;
2052 volatile AO_t GC_unlocked_count = 0;
2055 STATIC void GC_generic_lock(pthread_mutex_t * lock)
2057 #ifndef NO_PTHREAD_TRYLOCK
2058 unsigned pause_length = 1;
2061 if (0 == pthread_mutex_trylock(lock)) {
2063 (void)AO_fetch_and_add1(&GC_unlocked_count);
2067 for (; pause_length <= SPIN_MAX; pause_length <<= 1) {
2068 for (i = 0; i < pause_length; ++i) {
2071 switch(pthread_mutex_trylock(lock)) {
2074 (void)AO_fetch_and_add1(&GC_spin_count);
2080 ABORT("Unexpected error from pthread_mutex_trylock");
2083 #endif /* !NO_PTHREAD_TRYLOCK */
2085 (void)AO_fetch_and_add1(&GC_block_count);
2087 pthread_mutex_lock(lock);
2090 #endif /* !USE_SPIN_LOCK || ... */
2092 #if defined(AO_HAVE_char_load) && !defined(BASE_ATOMIC_OPS_EMULATED)
2093 # define is_collecting() \
2094 ((GC_bool)AO_char_load((unsigned char *)&GC_collecting))
2096 /* GC_collecting is a hint, a potential data race between */
2097 /* GC_lock() and ENTER/EXIT_GC() is OK to ignore. */
2098 # define is_collecting() GC_collecting
2101 #if defined(USE_SPIN_LOCK)
2103 /* Reasonably fast spin locks. Basically the same implementation */
2104 /* as STL alloc.h. This isn't really the right way to do this. */
2105 /* but until the POSIX scheduling mess gets straightened out ... */
2107 GC_INNER volatile AO_TS_t GC_allocate_lock = AO_TS_INITIALIZER;
2109 # define low_spin_max 30 /* spin cycles if we suspect uniprocessor */
2110 # define high_spin_max SPIN_MAX /* spin cycles for multiprocessor */
2112 static volatile AO_t spin_max = low_spin_max;
2113 static volatile AO_t last_spins = 0;
2114 /* A potential data race between */
2115 /* threads invoking GC_lock which reads */
2116 /* and updates spin_max and last_spins */
2117 /* could be ignored because these */
2118 /* variables are hints only. */
2120 GC_INNER void GC_lock(void)
2122 unsigned my_spin_max;
2123 unsigned my_last_spins;
2126 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2129 my_spin_max = (unsigned)AO_load(&spin_max);
2130 my_last_spins = (unsigned)AO_load(&last_spins);
2131 for (i = 0; i < my_spin_max; i++) {
2132 if (is_collecting() || GC_nprocs == 1)
2134 if (i < my_last_spins/2) {
2138 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2141 * Spinning worked. Thus we're probably not being scheduled
2142 * against the other process with which we were contending.
2143 * Thus it makes sense to spin longer the next time.
2145 AO_store(&last_spins, (AO_t)i);
2146 AO_store(&spin_max, (AO_t)high_spin_max);
2150 /* We are probably being scheduled against the other process. Sleep. */
2151 AO_store(&spin_max, (AO_t)low_spin_max);
2154 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2157 # define SLEEP_THRESHOLD 12
2158 /* Under Linux very short sleeps tend to wait until */
2159 /* the current time quantum expires. On old Linux */
2160 /* kernels nanosleep (<= 2 ms) just spins. */
2161 /* (Under 2.4, this happens only for real-time */
2162 /* processes.) We want to minimize both behaviors */
2164 if (i < SLEEP_THRESHOLD) {
2170 /* Don't wait for more than about 15 ms, */
2171 /* even under extreme contention. */
2173 ts.tv_nsec = 1 << i;
2179 #elif defined(USE_PTHREAD_LOCKS)
2181 # ifndef NO_PTHREAD_TRYLOCK
2182 GC_INNER void GC_lock(void)
2184 if (1 == GC_nprocs || is_collecting()) {
2185 pthread_mutex_lock(&GC_allocate_ml);
2187 GC_generic_lock(&GC_allocate_ml);
2190 # elif defined(GC_ASSERTIONS)
2191 GC_INNER void GC_lock(void)
2193 pthread_mutex_lock(&GC_allocate_ml);
2197 #endif /* !USE_SPIN_LOCK && USE_PTHREAD_LOCKS */
2199 #ifdef PARALLEL_MARK
2201 # ifdef GC_ASSERTIONS
2202 STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
2203 # define SET_MARK_LOCK_HOLDER \
2204 (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
2205 # define UNSET_MARK_LOCK_HOLDER \
2207 GC_ASSERT(GC_mark_lock_holder \
2208 == NUMERIC_THREAD_ID(pthread_self())); \
2209 GC_mark_lock_holder = NO_THREAD; \
2212 # define SET_MARK_LOCK_HOLDER (void)0
2213 # define UNSET_MARK_LOCK_HOLDER (void)0
2214 # endif /* !GC_ASSERTIONS */
2216 #ifdef GLIBC_2_1_MUTEX_HACK
2217 /* Ugly workaround for a linux threads bug in the final versions */
2218 /* of glibc2.1. Pthread_mutex_trylock sets the mutex owner */
2219 /* field even when it fails to acquire the mutex. This causes */
2220 /* pthread_cond_wait to die. Remove for glibc2.2. */
2221 /* According to the man page, we should use */
2222 /* PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, but that isn't actually */
2224 static pthread_mutex_t mark_mutex =
2225 {0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, {0, 0}};
2227 static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
2230 static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
2232 #ifdef GLIBC_2_19_TSX_BUG
2233 /* Parse string like <major>[.<minor>[<tail>]] and return major value. */
2234 static int parse_version(int *pminor, const char *pverstr) {
2236 unsigned long value = strtoul(pverstr, &endp, 10);
2237 int major = (int)value;
2239 if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
2244 /* No minor part. */
2247 value = strtoul(endp + 1, &endp, 10);
2248 *pminor = (int)value;
2249 if (*pminor < 0 || (unsigned)(*pminor) != value) {
2255 #endif /* GLIBC_2_19_TSX_BUG */
2257 static void setup_mark_lock(void)
2259 # ifdef GLIBC_2_19_TSX_BUG
2260 pthread_mutexattr_t mattr;
2261 int glibc_minor = -1;
2262 int glibc_major = parse_version(&glibc_minor, gnu_get_libc_version());
2264 if (glibc_major > 2 || (glibc_major == 2 && glibc_minor >= 19)) {
2265 /* TODO: disable this workaround for glibc with fixed TSX */
2266 /* This disables lock elision to workaround a bug in glibc 2.19+ */
2267 if (0 != pthread_mutexattr_init(&mattr)) {
2268 ABORT("pthread_mutexattr_init failed");
2270 if (0 != pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL)) {
2271 ABORT("pthread_mutexattr_settype failed");
2273 if (0 != pthread_mutex_init(&mark_mutex, &mattr)) {
2274 ABORT("pthread_mutex_init failed");
2276 (void)pthread_mutexattr_destroy(&mattr);
2281 GC_INNER void GC_acquire_mark_lock(void)
2283 # if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
2284 GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
2286 GC_generic_lock(&mark_mutex);
2287 SET_MARK_LOCK_HOLDER;
2290 GC_INNER void GC_release_mark_lock(void)
2292 UNSET_MARK_LOCK_HOLDER;
2293 if (pthread_mutex_unlock(&mark_mutex) != 0) {
2294 ABORT("pthread_mutex_unlock failed");
2298 /* Collector must wait for a freelist builders for 2 reasons: */
2299 /* 1) Mark bits may still be getting examined without lock. */
2300 /* 2) Partial free lists referenced only by locals may not be scanned */
2301 /* correctly, e.g. if they contain "pointer-free" objects, since the */
2302 /* free-list link may be ignored. */
2303 STATIC void GC_wait_builder(void)
2305 ASSERT_CANCEL_DISABLED();
2306 UNSET_MARK_LOCK_HOLDER;
2307 if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
2308 ABORT("pthread_cond_wait failed");
2310 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2311 SET_MARK_LOCK_HOLDER;
2314 GC_INNER void GC_wait_for_reclaim(void)
2316 GC_acquire_mark_lock();
2317 while (GC_fl_builder_count > 0) {
2320 GC_release_mark_lock();
2323 GC_INNER void GC_notify_all_builder(void)
2325 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
2326 if (pthread_cond_broadcast(&builder_cv) != 0) {
2327 ABORT("pthread_cond_broadcast failed");
2331 GC_INNER void GC_wait_marker(void)
2333 ASSERT_CANCEL_DISABLED();
2334 GC_ASSERT(GC_parallel);
2335 UNSET_MARK_LOCK_HOLDER;
2336 if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
2337 ABORT("pthread_cond_wait failed");
2339 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2340 SET_MARK_LOCK_HOLDER;
2343 GC_INNER void GC_notify_all_marker(void)
2345 GC_ASSERT(GC_parallel);
2346 if (pthread_cond_broadcast(&mark_cv) != 0) {
2347 ABORT("pthread_cond_broadcast failed");
2351 #endif /* PARALLEL_MARK */
2353 #ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS
2354 /* Workaround "undefined reference" linkage errors on some targets. */
2356 extern void __pthread_register_cancel(void) __attribute__((__weak__));
2357 extern void __pthread_unregister_cancel(void) __attribute__((__weak__));
2360 void __pthread_register_cancel(void) {}
2361 void __pthread_unregister_cancel(void) {}
2364 #endif /* GC_PTHREADS */