2 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
5 * Copyright (c) 2000-2008 by Hewlett-Packard Development Company.
8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
11 * Permission is hereby granted to use or copy this program
12 * for any purpose, provided the above notices are retained on all copies.
13 * Permission to modify the code and to distribute modified code is granted,
14 * provided the above notices are retained, and a notice that the code was
15 * modified is included with the above copyright notice.
18 #include "private/gc_priv.h"
20 #if defined(GC_WIN32_THREADS)
22 #ifdef THREAD_LOCAL_ALLOC
23 # include "private/thread_local_alloc.h"
24 #endif /* THREAD_LOCAL_ALLOC */
26 /* Allocation lock declarations. */
27 #if !defined(USE_PTHREAD_LOCKS)
28 GC_INNER CRITICAL_SECTION GC_allocate_ml;
30 GC_INNER DWORD GC_lock_holder = NO_THREAD;
31 /* Thread id for current holder of allocation lock */
34 GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
36 GC_INNER unsigned long GC_lock_holder = NO_THREAD;
46 # include <errno.h> /* for EAGAIN */
48 /* Cygwin-specific forward decls */
49 # undef pthread_create
51 # undef pthread_detach
53 # ifndef GC_NO_PTHREAD_SIGMASK
54 # undef pthread_sigmask
57 STATIC void * GC_pthread_start(void * arg);
58 STATIC void GC_thread_exit_proc(void *arg);
61 # ifdef CAN_CALL_ATFORK
65 #elif !defined(MSWINCE)
66 # include <process.h> /* For _beginthreadex, _endthreadex */
67 # include <errno.h> /* for errno, EAGAIN */
69 #endif /* !GC_PTHREADS && !MSWINCE */
71 /* PUSHED_REGS_COUNT is the number of copied registers in copy_ptr_regs. */
72 static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext);
74 # ifdef WOW64_THREAD_CONTEXT_WORKAROUND
75 # define PUSHED_REGS_COUNT 9
77 # define PUSHED_REGS_COUNT 7
79 #elif defined(X86_64) || defined(SHx)
80 # define PUSHED_REGS_COUNT 15
82 # define PUSHED_REGS_COUNT 13
83 #elif defined(AARCH64)
84 # define PUSHED_REGS_COUNT 30
85 #elif defined(MIPS) || defined(ALPHA)
86 # define PUSHED_REGS_COUNT 28
88 # define PUSHED_REGS_COUNT 29
91 /* DllMain-based thread registration is currently incompatible */
92 /* with thread-local allocation, pthreads and WinCE. */
93 #if (defined(GC_DLL) || defined(GC_INSIDE_DLL)) && !defined(NO_CRT) \
94 && !defined(GC_NO_THREADS_DISCOVERY) && !defined(MSWINCE) \
95 && !defined(THREAD_LOCAL_ALLOC) && !defined(GC_PTHREADS)
97 /* This code operates in two distinct modes, depending on */
98 /* the setting of GC_win32_dll_threads. */
99 /* If GC_win32_dll_threads is set, all threads in the process */
100 /* are implicitly registered with the GC by DllMain. */
101 /* No explicit registration is required, and attempts at */
102 /* explicit registration are ignored. This mode is */
103 /* very different from the Posix operation of the collector. */
104 /* In this mode access to the thread table is lock-free. */
105 /* Hence there is a static limit on the number of threads. */
107 # ifdef GC_DISCOVER_TASK_THREADS
108 /* GC_DISCOVER_TASK_THREADS should be used if DllMain-based */
109 /* thread registration is required but it is impossible to */
110 /* call GC_use_threads_discovery before other GC routines. */
111 # define GC_win32_dll_threads TRUE
113 STATIC GC_bool GC_win32_dll_threads = FALSE;
114 /* GC_win32_dll_threads must be set (if needed) at the */
115 /* application initialization time, i.e. before any */
116 /* collector or thread calls. We make it a "dynamic" */
117 /* option only to avoid multiple library versions. */
121 /* If GC_win32_dll_threads is FALSE (or the collector is */
122 /* built without GC_DLL defined), things operate in a way */
123 /* that is very similar to Posix platforms, and new threads */
124 /* must be registered with the collector, e.g. by using */
125 /* preprocessor-based interception of the thread primitives. */
126 /* In this case, we use a real data structure for the thread */
127 /* table. Note that there is no equivalent of linker-based */
128 /* call interception, since we don't have ELF-like */
129 /* facilities. The Windows analog appears to be "API */
130 /* hooking", which really seems to be a standard way to */
131 /* do minor binary rewriting (?). I'd prefer not to have */
132 /* the basic collector rely on such facilities, but an */
133 /* optional package that intercepts thread calls this way */
134 /* would probably be nice. */
135 # ifndef GC_NO_THREADS_DISCOVERY
136 # define GC_NO_THREADS_DISCOVERY
138 # define GC_win32_dll_threads FALSE
140 # define MAX_THREADS 1 /* dll_thread_table[] is always empty. */
141 #endif /* GC_NO_THREADS_DISCOVERY */
143 /* We have two versions of the thread table. Which one */
144 /* we us depends on whether or not GC_win32_dll_threads */
145 /* is set. Note that before initialization, we don't */
146 /* add any entries to either table, even if DllMain is */
147 /* called. The main thread will be added on */
148 /* initialization. */
150 /* The type of the first argument to InterlockedExchange. */
151 /* Documented to be LONG volatile *, but at least gcc likes */
155 STATIC GC_bool GC_thr_initialized = FALSE;
157 #ifndef GC_ALWAYS_MULTITHREADED
158 GC_INNER GC_bool GC_need_to_lock = FALSE;
161 static GC_bool parallel_initialized = FALSE;
163 /* GC_use_threads_discovery() is currently incompatible with pthreads */
164 /* and WinCE. It might be possible to get DllMain-based thread */
165 /* registration to work with Cygwin, but if you try it then you are on */
167 GC_API void GC_CALL GC_use_threads_discovery(void)
169 # ifdef GC_NO_THREADS_DISCOVERY
170 ABORT("GC DllMain-based thread registration unsupported");
172 /* Turn on GC_win32_dll_threads. */
173 GC_ASSERT(!parallel_initialized);
174 /* Note that GC_use_threads_discovery is expected to be called by */
175 /* the client application (not from DllMain) at start-up. */
176 # ifndef GC_DISCOVER_TASK_THREADS
177 GC_win32_dll_threads = TRUE;
183 #define ADDR_LIMIT ((ptr_t)GC_WORD_MAX)
185 struct GC_Thread_Rep {
187 # ifndef GC_NO_THREADS_DISCOVERY
188 volatile AO_t in_use;
189 /* Updated without lock. */
190 /* We assert that unused */
191 /* entries have invalid ids of */
192 /* zero and zero stack fields. */
193 /* Used only with GC_win32_dll_threads. */
194 LONG long_in_use; /* The same but of the type that */
195 /* matches the first argument of */
196 /* InterlockedExchange(); volatile is */
197 /* omitted because the ancient version */
198 /* of the prototype lacks the qualifier.*/
200 struct GC_Thread_Rep * next;
201 /* Hash table link without */
202 /* GC_win32_dll_threads. */
203 /* More recently allocated threads */
204 /* with a given pthread id come */
205 /* first. (All but the first are */
206 /* guaranteed to be dead, but we may */
207 /* not yet have registered the join.) */
208 } tm; /* table_management */
212 /* According to MSDN specs for WinCE targets: */
213 /* - DuplicateHandle() is not applicable to thread handles; and */
214 /* - the value returned by GetCurrentThreadId() could be used as */
215 /* a "real" thread handle (for SuspendThread(), ResumeThread() and */
216 /* GetThreadContext()). */
217 # define THREAD_HANDLE(t) (HANDLE)(word)(t)->id
220 # define THREAD_HANDLE(t) (t)->handle
223 ptr_t stack_base; /* The cold end of the stack. */
224 /* 0 ==> entry not valid. */
225 /* !in_use ==> stack_base == 0 */
226 ptr_t last_stack_min; /* Last known minimum (hottest) address */
227 /* in stack or ADDR_LIMIT if unset */
229 ptr_t backing_store_end;
230 ptr_t backing_store_ptr;
232 ptr_t initial_stack_base;
233 /* The cold end of the stack saved by */
234 /* GC_record_stack_base (never modified */
235 /* by GC_set_stackbottom). */
238 ptr_t thread_blocked_sp; /* Protected by GC lock. */
239 /* NULL value means thread unblocked. */
240 /* If set to non-NULL, thread will */
241 /* acquire GC lock before doing any */
242 /* pointer manipulations. Thus it does */
243 /* not need to stop this thread. */
245 struct GC_traced_stack_sect_s *traced_stack_sect;
246 /* Points to the "stack section" data */
247 /* held in stack by the innermost */
248 /* GC_call_with_gc_active() of this */
249 /* thread. May be NULL. */
251 unsigned short finalizer_skipped;
252 unsigned char finalizer_nested;
253 /* Used by GC_check_finalizer_nested() */
254 /* to minimize the level of recursion */
255 /* when a client finalizer allocates */
256 /* memory (initially both are 0). */
258 unsigned char suspended; /* really of GC_bool type */
261 unsigned char flags; /* Protected by GC lock. */
262 # define FINISHED 1 /* Thread has exited. */
263 # define DETACHED 2 /* Thread is intended to be detached. */
264 # define KNOWN_FINISHED(t) (((t) -> flags) & FINISHED)
265 pthread_t pthread_id;
266 void *status; /* hold exit value until join in case it's a pointer */
268 # define KNOWN_FINISHED(t) 0
271 # ifdef THREAD_LOCAL_ALLOC
272 struct thread_local_freelists tlfs;
275 # ifdef RETRY_GET_THREAD_CONTEXT
277 word context_regs[PUSHED_REGS_COUNT];
278 /* Populated as part of GC_suspend() as */
279 /* resume/suspend loop may be needed for the */
280 /* call to GetThreadContext() to succeed. */
284 typedef struct GC_Thread_Rep * GC_thread;
285 typedef volatile struct GC_Thread_Rep * GC_vthread;
287 #ifndef GC_NO_THREADS_DISCOVERY
288 STATIC DWORD GC_main_thread = 0;
290 /* We track thread attachments while the world is supposed to be */
291 /* stopped. Unfortunately, we cannot stop them from starting, since */
292 /* blocking in DllMain seems to cause the world to deadlock. Thus, */
293 /* we have to recover if we notice this in the middle of marking. */
294 STATIC volatile AO_t GC_attached_thread = FALSE;
296 /* We assumed that volatile ==> memory ordering, at least among */
297 /* volatiles. This code should consistently use atomic_ops. */
298 STATIC volatile GC_bool GC_please_stop = FALSE;
299 #elif defined(GC_ASSERTIONS)
300 STATIC GC_bool GC_please_stop = FALSE;
301 #endif /* GC_NO_THREADS_DISCOVERY && GC_ASSERTIONS */
303 #if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
304 /* Return TRUE if an thread was attached since we last asked or */
305 /* since GC_attached_thread was explicitly reset. */
306 GC_INNER GC_bool GC_started_thread_while_stopped(void)
308 # ifndef GC_NO_THREADS_DISCOVERY
309 if (GC_win32_dll_threads) {
310 # ifdef AO_HAVE_compare_and_swap_release
311 if (AO_compare_and_swap_release(&GC_attached_thread, TRUE,
315 AO_nop_full(); /* Prior heap reads need to complete earlier. */
316 if (AO_load(&GC_attached_thread)) {
317 AO_store(&GC_attached_thread, FALSE);
325 #endif /* WRAP_MARK_SOME */
327 /* Thread table used if GC_win32_dll_threads is set. */
328 /* This is a fixed size array. */
329 /* Since we use runtime conditionals, both versions */
330 /* are always defined. */
332 # define MAX_THREADS 512
335 /* Things may get quite slow for large numbers of threads, */
336 /* since we look them up with sequential search. */
337 volatile struct GC_Thread_Rep dll_thread_table[MAX_THREADS];
339 STATIC volatile LONG GC_max_thread_index = 0;
340 /* Largest index in dll_thread_table */
341 /* that was ever used. */
343 /* And now the version used if GC_win32_dll_threads is not set. */
344 /* This is a chained hash table, with much of the code borrowed */
345 /* from the Posix implementation. */
346 #ifndef THREAD_TABLE_SZ
347 # define THREAD_TABLE_SZ 256 /* Power of 2 (for speed). */
349 #define THREAD_TABLE_INDEX(id) /* id is of DWORD type */ \
350 (int)((((id) >> 8) ^ (id)) % THREAD_TABLE_SZ)
351 STATIC GC_thread GC_threads[THREAD_TABLE_SZ];
353 /* It may not be safe to allocate when we register the first thread. */
354 /* Thus we allocated one statically. It does not contain any pointer */
355 /* field we need to push ("next" and "status" fields are unused). */
356 static struct GC_Thread_Rep first_thread;
357 static GC_bool first_thread_used = FALSE;
359 /* Add a thread to GC_threads. We assume it wasn't already there. */
360 /* Caller holds allocation lock. */
361 /* Unlike the pthreads version, the id field is set by the caller. */
362 STATIC GC_thread GC_new_thread(DWORD id)
364 int hv = THREAD_TABLE_INDEX(id);
367 # ifdef DEBUG_THREADS
368 GC_log_printf("Creating thread 0x%lx\n", (long)id);
369 if (GC_threads[hv] != NULL)
370 GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
372 GC_ASSERT(I_HOLD_LOCK());
373 if (!EXPECT(first_thread_used, TRUE)) {
374 result = &first_thread;
375 first_thread_used = TRUE;
376 GC_ASSERT(NULL == GC_threads[hv]);
378 GC_ASSERT(!GC_win32_dll_threads);
379 result = (struct GC_Thread_Rep *)
380 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
381 if (result == 0) return(0);
383 /* result -> id = id; Done by caller. */
384 result -> tm.next = GC_threads[hv];
385 GC_threads[hv] = result;
387 GC_ASSERT(result -> flags == 0);
389 GC_ASSERT(result -> thread_blocked_sp == NULL);
390 if (EXPECT(result != &first_thread, TRUE))
395 GC_INNER GC_bool GC_in_thread_creation = FALSE;
396 /* Protected by allocation lock. */
398 GC_INLINE void GC_record_stack_base(GC_vthread me,
399 const struct GC_stack_base *sb)
401 me -> stack_base = (ptr_t)sb->mem_base;
403 me -> backing_store_end = (ptr_t)sb->reg_base;
405 me -> initial_stack_base = (ptr_t)sb->mem_base;
407 if (me -> stack_base == NULL)
408 ABORT("Bad stack base in GC_register_my_thread");
411 /* This may be called from DllMain, and hence operates under unusual */
412 /* constraints. In particular, it must be lock-free if */
413 /* GC_win32_dll_threads is set. Always called from the thread being */
414 /* added. If GC_win32_dll_threads is not set, we already hold the */
415 /* allocation lock except possibly during single-threaded startup code. */
416 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
421 /* The following should be a no-op according to the win32 */
422 /* documentation. There is empirical evidence that it */
424 # if defined(MPROTECT_VDB) && !defined(CYGWIN32)
425 if (GC_auto_incremental
427 && !GC_gww_dirty_init()
430 GC_set_write_fault_handler();
433 # ifndef GC_NO_THREADS_DISCOVERY
434 if (GC_win32_dll_threads) {
436 /* It appears to be unsafe to acquire a lock here, since this */
437 /* code is apparently not preemptible on some systems. */
438 /* (This is based on complaints, not on Microsoft's official */
439 /* documentation, which says this should perform "only simple */
440 /* initialization tasks".) */
441 /* Hence we make do with nonblocking synchronization. */
442 /* It has been claimed that DllMain is really only executed with */
443 /* a particular system lock held, and thus careful use of locking */
444 /* around code that doesn't call back into the system libraries */
445 /* might be OK. But this hasn't been tested across all win32 */
448 InterlockedExchange(&dll_thread_table[i].tm.long_in_use, 1) != 0;
450 /* Compare-and-swap would make this cleaner, but that's not */
451 /* supported before Windows 98 and NT 4.0. In Windows 2000, */
452 /* InterlockedExchange is supposed to be replaced by */
453 /* InterlockedExchangePointer, but that's not really what I */
455 /* FIXME: We should eventually declare Win95 dead and use AO_ */
456 /* primitives here. */
457 if (i == MAX_THREADS - 1)
458 ABORT("Too many threads");
460 /* Update GC_max_thread_index if necessary. The following is */
461 /* safe, and unlike CompareExchange-based solutions seems to work */
462 /* on all Windows95 and later platforms. */
463 /* Unfortunately, GC_max_thread_index may be temporarily out of */
464 /* bounds, so readers have to compensate. */
465 while (i > GC_max_thread_index) {
466 InterlockedIncrement((IE_t)&GC_max_thread_index);
468 if (GC_max_thread_index >= MAX_THREADS) {
469 /* We overshot due to simultaneous increments. */
470 /* Setting it to MAX_THREADS-1 is always safe. */
471 GC_max_thread_index = MAX_THREADS - 1;
473 me = dll_thread_table + i;
476 /* else */ /* Not using DllMain */ {
477 GC_ASSERT(I_HOLD_LOCK());
478 GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */
479 me = GC_new_thread(thread_id);
480 GC_in_thread_creation = FALSE;
482 ABORT("Failed to allocate memory for thread registering");
485 /* me can be NULL -> segfault */
486 me -> pthread_id = pthread_self();
489 /* GetCurrentThread() returns a pseudohandle (a const value). */
490 if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
492 (HANDLE*)&(me -> handle),
493 0 /* dwDesiredAccess */, FALSE /* bInheritHandle */,
494 DUPLICATE_SAME_ACCESS)) {
495 ABORT_ARG1("DuplicateHandle failed",
496 ": errcode= 0x%X", (unsigned)GetLastError());
499 me -> last_stack_min = ADDR_LIMIT;
500 GC_record_stack_base(me, sb);
501 /* Up until this point, GC_push_all_stacks considers this thread */
503 /* Up until this point, this entry is viewed as reserved but invalid */
504 /* by GC_delete_thread. */
505 me -> id = thread_id;
506 # if defined(THREAD_LOCAL_ALLOC)
507 GC_init_thread_local((GC_tlfs)(&(me->tlfs)));
509 # ifndef GC_NO_THREADS_DISCOVERY
510 if (GC_win32_dll_threads) {
511 if (GC_please_stop) {
512 AO_store(&GC_attached_thread, TRUE);
513 AO_nop_full(); /* Later updates must become visible after this. */
515 /* We'd like to wait here, but can't, since waiting in DllMain */
516 /* provokes deadlocks. */
517 /* Thus we force marking to be restarted instead. */
521 GC_ASSERT(!GC_please_stop);
522 /* Otherwise both we and the thread stopping code would be */
523 /* holding the allocation lock. */
525 return (GC_thread)(me);
529 * GC_max_thread_index may temporarily be larger than MAX_THREADS.
530 * To avoid subscript errors, we check on access.
532 GC_INLINE LONG GC_get_max_thread_index(void)
534 LONG my_max = GC_max_thread_index;
535 if (my_max >= MAX_THREADS) return MAX_THREADS - 1;
539 /* Return the GC_thread corresponding to a thread id. May be called */
540 /* without a lock, but should be called in contexts in which the */
541 /* requested thread cannot be asynchronously deleted, e.g. from the */
543 /* This version assumes that either GC_win32_dll_threads is set, or */
544 /* we hold the allocator lock. */
545 /* Also used (for assertion checking only) from thread_local_alloc.c. */
546 STATIC GC_thread GC_lookup_thread_inner(DWORD thread_id)
548 # ifndef GC_NO_THREADS_DISCOVERY
549 if (GC_win32_dll_threads) {
551 LONG my_max = GC_get_max_thread_index();
552 for (i = 0; i <= my_max &&
553 (!AO_load_acquire(&dll_thread_table[i].tm.in_use)
554 || dll_thread_table[i].id != thread_id);
555 /* Must still be in_use, since nobody else can store our */
560 return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
564 GC_thread p = GC_threads[THREAD_TABLE_INDEX(thread_id)];
566 GC_ASSERT(I_HOLD_LOCK());
567 while (p != 0 && p -> id != thread_id) p = p -> tm.next;
573 # define CHECK_LOOKUP_MY_THREAD(me) \
574 if (!(me)) ABORT("GC_lookup_thread_inner(GetCurrentThreadId) failed")
576 # define CHECK_LOOKUP_MY_THREAD(me) /* empty */
579 /* Called by GC_finalize() (in case of an allocation failure observed). */
580 /* GC_reset_finalizer_nested() is the same as in pthread_support.c. */
581 GC_INNER void GC_reset_finalizer_nested(void)
583 GC_thread me = GC_lookup_thread_inner(GetCurrentThreadId());
584 CHECK_LOOKUP_MY_THREAD(me);
585 me->finalizer_nested = 0;
588 /* Checks and updates the thread-local level of finalizers recursion. */
589 /* Returns NULL if GC_invoke_finalizers() should not be called by the */
590 /* collector (to minimize the risk of a deep finalizers recursion), */
591 /* otherwise returns a pointer to the thread-local finalizer_nested. */
592 /* Called by GC_notify_or_invoke_finalizers() only (the lock is held). */
593 /* GC_check_finalizer_nested() is the same as in pthread_support.c. */
594 GC_INNER unsigned char *GC_check_finalizer_nested(void)
596 GC_thread me = GC_lookup_thread_inner(GetCurrentThreadId());
597 unsigned nesting_level;
598 CHECK_LOOKUP_MY_THREAD(me);
599 nesting_level = me->finalizer_nested;
601 /* We are inside another GC_invoke_finalizers(). */
602 /* Skip some implicitly-called GC_invoke_finalizers() */
603 /* depending on the nesting (recursion) level. */
604 if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
605 me->finalizer_skipped = 0;
607 me->finalizer_nested = (unsigned char)(nesting_level + 1);
608 return &me->finalizer_nested;
611 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
612 /* This is called from thread-local GC_malloc(). */
613 GC_bool GC_is_thread_tsd_valid(void *tsd)
619 me = GC_lookup_thread_inner(GetCurrentThreadId());
621 return (word)tsd >= (word)(&me->tlfs)
622 && (word)tsd < (word)(&me->tlfs) + sizeof(me->tlfs);
624 #endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
626 GC_API int GC_CALL GC_thread_is_registered(void)
628 DWORD thread_id = GetCurrentThreadId();
633 me = GC_lookup_thread_inner(thread_id);
638 GC_API void GC_CALL GC_register_altstack(void *stack GC_ATTR_UNUSED,
639 GC_word stack_size GC_ATTR_UNUSED,
640 void *altstack GC_ATTR_UNUSED,
641 GC_word altstack_size GC_ATTR_UNUSED)
643 /* TODO: Implement */
646 /* Make sure thread descriptor t is not protected by the VDB */
647 /* implementation. */
648 /* Used to prevent write faults when the world is (partially) stopped, */
649 /* since it may have been stopped with a system lock held, and that */
650 /* lock may be required for fault handling. */
651 #if defined(MPROTECT_VDB)
652 # define UNPROTECT_THREAD(t) \
653 if (!GC_win32_dll_threads && GC_auto_incremental \
654 && t != &first_thread) { \
655 GC_ASSERT(SMALL_OBJ(GC_size(t))); \
656 GC_remove_protection(HBLKPTR(t), 1, FALSE); \
659 # define UNPROTECT_THREAD(t) (void)0
663 # define GC_PTHREAD_PTRVAL(pthread_id) pthread_id
664 #elif defined(GC_WIN32_PTHREADS) || defined(GC_PTHREADS_PARAMARK)
665 # include <pthread.h> /* to check for winpthreads */
666 # if defined(__WINPTHREADS_VERSION_MAJOR)
667 # define GC_PTHREAD_PTRVAL(pthread_id) pthread_id
669 # define GC_PTHREAD_PTRVAL(pthread_id) pthread_id.p
673 /* If a thread has been joined, but we have not yet */
674 /* been notified, then there may be more than one thread */
675 /* in the table with the same win32 id. */
676 /* This is OK, but we need a way to delete a specific one. */
677 /* Assumes we hold the allocation lock unless */
678 /* GC_win32_dll_threads is set. Does not actually free */
679 /* GC_thread entry (only unlinks it). */
680 /* If GC_win32_dll_threads is set it should be called from the */
681 /* thread being deleted. */
682 STATIC void GC_delete_gc_thread_no_free(GC_vthread t)
685 CloseHandle(t->handle);
687 # ifndef GC_NO_THREADS_DISCOVERY
688 if (GC_win32_dll_threads) {
689 /* This is intended to be lock-free. */
690 /* It is either called synchronously from the thread being */
691 /* deleted, or by the joining thread. */
692 /* In this branch asynchronous changes to (*t) are possible. */
693 /* It's not allowed to call GC_printf (and the friends) here, */
694 /* see GC_stop_world() for the information. */
697 t -> suspended = FALSE;
698 # ifdef RETRY_GET_THREAD_CONTEXT
699 t -> context_sp = NULL;
701 AO_store_release(&t->tm.in_use, FALSE);
705 DWORD id = ((GC_thread)t) -> id;
706 /* Cast away volatile qualifier, since we have lock. */
707 int hv = THREAD_TABLE_INDEX(id);
708 GC_thread p = GC_threads[hv];
709 GC_thread prev = NULL;
711 GC_ASSERT(I_HOLD_LOCK());
712 while (p != (GC_thread)t) {
717 GC_threads[hv] = p -> tm.next;
719 GC_ASSERT(prev != &first_thread);
720 prev -> tm.next = p -> tm.next;
726 /* Delete a thread from GC_threads. We assume it is there. */
727 /* (The code intentionally traps if it wasn't.) Assumes we */
728 /* hold the allocation lock unless GC_win32_dll_threads is set. */
729 /* If GC_win32_dll_threads is set then it should be called from */
730 /* the thread being deleted. It is also safe to delete the */
731 /* main thread (unless GC_win32_dll_threads). */
732 STATIC void GC_delete_thread(DWORD id)
734 if (GC_win32_dll_threads) {
735 GC_vthread t = GC_lookup_thread_inner(id);
738 WARN("Removing nonexistent thread, id = %" WARN_PRIdPTR "\n", id);
740 GC_delete_gc_thread_no_free(t);
743 int hv = THREAD_TABLE_INDEX(id);
744 GC_thread p = GC_threads[hv];
745 GC_thread prev = NULL;
747 GC_ASSERT(I_HOLD_LOCK());
748 while (p -> id != id) {
753 CloseHandle(p->handle);
756 GC_threads[hv] = p -> tm.next;
758 GC_ASSERT(prev != &first_thread);
759 prev -> tm.next = p -> tm.next;
762 if (EXPECT(p != &first_thread, TRUE)) {
768 GC_API void GC_CALL GC_allow_register_threads(void)
770 /* Check GC is initialized and the current thread is registered. */
771 GC_ASSERT(GC_lookup_thread_inner(GetCurrentThreadId()) != 0);
772 # if !defined(GC_ALWAYS_MULTITHREADED) && !defined(PARALLEL_MARK) \
773 && !defined(GC_NO_THREADS_DISCOVERY)
774 /* GC_init() does not call GC_init_parallel() in this case. */
775 parallel_initialized = TRUE;
780 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
783 DWORD thread_id = GetCurrentThreadId();
786 if (GC_need_to_lock == FALSE)
787 ABORT("Threads explicit registering is not previously enabled");
789 /* We lock here, since we want to wait for an ongoing GC. */
791 me = GC_lookup_thread_inner(thread_id);
794 me = GC_register_my_thread_inner(sb, thread_id);
795 # if defined(CPPCHECK)
798 me -> flags |= DETACHED;
799 /* Treat as detached, since we do not need to worry about */
800 /* pointer results. */
802 GC_register_my_thread_inner(sb, thread_id);
808 /* else */ if ((me -> flags & FINISHED) != 0) {
809 GC_record_stack_base(me, sb);
810 me -> flags &= ~FINISHED; /* but not DETACHED */
811 # ifdef THREAD_LOCAL_ALLOC
812 GC_init_thread_local((GC_tlfs)(&me->tlfs));
824 /* Similar to that in pthread_support.c. */
825 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
827 GC_ASSERT(I_HOLD_LOCK());
828 if (GC_incremental && GC_collection_in_progress()) {
829 word old_gc_no = GC_gc_no;
831 /* Make sure that no part of our stack is still on the mark stack, */
832 /* since it's about to be unmapped. */
835 GC_in_thread_creation = TRUE;
836 GC_collect_a_little_inner(1);
837 GC_in_thread_creation = FALSE;
841 Sleep(0); /* yield */
843 } while (GC_incremental && GC_collection_in_progress()
844 && (wait_for_all || old_gc_no == GC_gc_no));
848 GC_API int GC_CALL GC_unregister_my_thread(void)
852 # ifdef DEBUG_THREADS
853 GC_log_printf("Unregistering thread 0x%lx\n", (long)GetCurrentThreadId());
856 if (GC_win32_dll_threads) {
857 # if defined(THREAD_LOCAL_ALLOC)
858 /* Can't happen: see GC_use_threads_discovery(). */
861 /* FIXME: Should we just ignore this? */
862 GC_delete_thread(GetCurrentThreadId());
865 # if defined(THREAD_LOCAL_ALLOC) || defined(GC_PTHREADS)
868 DWORD thread_id = GetCurrentThreadId();
871 GC_wait_for_gc_completion(FALSE);
872 # if defined(THREAD_LOCAL_ALLOC) || defined(GC_PTHREADS)
873 me = GC_lookup_thread_inner(thread_id);
874 CHECK_LOOKUP_MY_THREAD(me);
875 GC_ASSERT(!KNOWN_FINISHED(me));
877 # if defined(THREAD_LOCAL_ALLOC)
878 GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
879 GC_destroy_thread_local(&(me->tlfs));
882 if ((me -> flags & DETACHED) == 0) {
883 me -> flags |= FINISHED;
887 GC_delete_thread(thread_id);
889 # if defined(THREAD_LOCAL_ALLOC)
890 /* It is required to call remove_specific defined in specific.c. */
891 GC_remove_specific(GC_thread_key);
898 /* Wrapper for functions that are likely to block for an appreciable */
899 /* length of time. */
901 /* GC_do_blocking_inner() is nearly the same as in pthread_support.c */
902 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
904 struct blocking_data * d = (struct blocking_data *) data;
905 DWORD thread_id = GetCurrentThreadId();
908 ptr_t stack_ptr = GC_save_regs_in_stack();
913 me = GC_lookup_thread_inner(thread_id);
914 CHECK_LOOKUP_MY_THREAD(me);
915 GC_ASSERT(me -> thread_blocked_sp == NULL);
917 me -> backing_store_ptr = stack_ptr;
919 me -> thread_blocked_sp = (ptr_t) &d; /* save approx. sp */
920 /* Save context here if we want to support precise stack marking */
922 d -> client_data = (d -> fn)(d -> client_data);
923 LOCK(); /* This will block if the world is stopped. */
924 # if defined(CPPCHECK)
925 GC_noop1((word)me->thread_blocked_sp);
927 me -> thread_blocked_sp = NULL;
931 /* GC_call_with_gc_active() has the opposite to GC_do_blocking() */
932 /* functionality. It might be called from a user function invoked by */
933 /* GC_do_blocking() to temporarily back allow calling any GC function */
934 /* and/or manipulating pointers to the garbage collected heap. */
935 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
938 struct GC_traced_stack_sect_s stacksect;
939 DWORD thread_id = GetCurrentThreadId();
943 LOCK(); /* This will block if the world is stopped. */
944 me = GC_lookup_thread_inner(thread_id);
945 CHECK_LOOKUP_MY_THREAD(me);
946 /* Adjust our stack bottom pointer (this could happen unless */
947 /* GC_get_stack_base() was used which returned GC_SUCCESS). */
948 GC_ASSERT(me -> stack_base != NULL);
949 if ((word)me->stack_base < (word)(&stacksect)) {
950 me -> stack_base = (ptr_t)(&stacksect);
952 me -> initial_stack_base = me -> stack_base;
956 if (me -> thread_blocked_sp == NULL) {
957 /* We are not inside GC_do_blocking() - do nothing more. */
959 client_data = fn(client_data);
960 /* Prevent treating the above as a tail call. */
961 GC_noop1(COVERT_DATAFLOW(&stacksect));
962 return client_data; /* result */
965 /* Setup new "stack section". */
966 stacksect.saved_stack_ptr = me -> thread_blocked_sp;
968 /* This is the same as in GC_call_with_stack_base(). */
969 stacksect.backing_store_end = GC_save_regs_in_stack();
970 /* Unnecessarily flushes register stack, */
971 /* but that probably doesn't hurt. */
972 stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
974 stacksect.prev = me -> traced_stack_sect;
975 me -> thread_blocked_sp = NULL;
976 me -> traced_stack_sect = &stacksect;
979 client_data = fn(client_data);
980 GC_ASSERT(me -> thread_blocked_sp == NULL);
981 GC_ASSERT(me -> traced_stack_sect == &stacksect);
983 /* Restore original "stack section". */
985 # if defined(CPPCHECK)
986 GC_noop1((word)me->traced_stack_sect);
988 me -> traced_stack_sect = stacksect.prev;
990 me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
992 me -> thread_blocked_sp = stacksect.saved_stack_ptr;
995 return client_data; /* result */
998 GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
999 const struct GC_stack_base *sb)
1001 GC_thread t = (GC_thread)gc_thread_handle;
1003 GC_ASSERT(sb -> mem_base != NULL);
1004 if (!EXPECT(GC_is_initialized, TRUE)) {
1005 GC_ASSERT(NULL == t);
1006 GC_stackbottom = (char *)sb->mem_base;
1008 GC_register_stackbottom = (ptr_t)sb->reg_base;
1013 GC_ASSERT(I_HOLD_LOCK());
1014 if (NULL == t) { /* current thread? */
1015 t = GC_lookup_thread_inner(GetCurrentThreadId());
1016 CHECK_LOOKUP_MY_THREAD(t);
1018 GC_ASSERT(!KNOWN_FINISHED(t));
1019 GC_ASSERT(NULL == t -> thread_blocked_sp
1020 && NULL == t -> traced_stack_sect); /* for now */
1021 t -> stack_base = (ptr_t)sb->mem_base;
1022 t -> last_stack_min = ADDR_LIMIT; /* reset the known minimum */
1024 t -> backing_store_end = (ptr_t)sb->reg_base;
1028 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
1030 DWORD thread_id = GetCurrentThreadId();
1035 me = GC_lookup_thread_inner(thread_id);
1036 CHECK_LOOKUP_MY_THREAD(me); /* the thread is assumed to be registered */
1037 sb -> mem_base = me -> stack_base;
1039 sb -> reg_base = me -> backing_store_end;
1042 return (void *)me; /* gc_thread_handle */
1047 /* A quick-and-dirty cache of the mapping between pthread_t */
1048 /* and win32 thread id. */
1049 # define PTHREAD_MAP_SIZE 512
1050 DWORD GC_pthread_map_cache[PTHREAD_MAP_SIZE] = {0};
1051 # define PTHREAD_MAP_INDEX(pthread_id) \
1052 ((NUMERIC_THREAD_ID(pthread_id) >> 5) % PTHREAD_MAP_SIZE)
1053 /* It appears pthread_t is really a pointer type ... */
1054 # define SET_PTHREAD_MAP_CACHE(pthread_id, win32_id) \
1055 (void)(GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)] = (win32_id))
1056 # define GET_PTHREAD_MAP_CACHE(pthread_id) \
1057 GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)]
1059 /* Return a GC_thread corresponding to a given pthread_t. */
1060 /* Returns 0 if it's not there. */
1061 /* We assume that this is only called for pthread ids that */
1062 /* have not yet terminated or are still joinable, and */
1063 /* cannot be concurrently terminated. */
1064 /* Assumes we do NOT hold the allocation lock. */
1065 STATIC GC_thread GC_lookup_pthread(pthread_t id)
1067 # ifndef GC_NO_THREADS_DISCOVERY
1068 if (GC_win32_dll_threads) {
1070 LONG my_max = GC_get_max_thread_index();
1072 for (i = 0; i <= my_max &&
1073 (!AO_load_acquire(&dll_thread_table[i].tm.in_use)
1074 || THREAD_EQUAL(dll_thread_table[i].pthread_id, id));
1075 /* Must still be in_use, since nobody else can */
1076 /* store our thread_id. */
1080 return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
1084 /* We first try the cache. If that fails, we use a very slow */
1086 DWORD win32_id = GET_PTHREAD_MAP_CACHE(id);
1087 int hv_guess = THREAD_TABLE_INDEX(win32_id);
1093 for (p = GC_threads[hv_guess]; 0 != p; p = p -> tm.next) {
1094 if (THREAD_EQUAL(p -> pthread_id, id))
1097 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
1098 for (p = GC_threads[hv]; 0 != p; p = p -> tm.next) {
1099 if (THREAD_EQUAL(p -> pthread_id, id))
1110 #endif /* GC_PTHREADS */
1112 #ifdef CAN_HANDLE_FORK
1113 /* Similar to that in pthread_support.c but also rehashes the table */
1114 /* since hash map key (thread_id) differs from that in the parent. */
1115 STATIC void GC_remove_all_threads_but_me(void)
1118 GC_thread me = NULL;
1120 pthread_t pthread_id = pthread_self(); /* same as in parent */
1122 GC_ASSERT(!GC_win32_dll_threads);
1123 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
1126 for (p = GC_threads[hv]; 0 != p; p = next) {
1127 next = p -> tm.next;
1128 if (THREAD_EQUAL(p -> pthread_id, pthread_id)
1129 && me == NULL) { /* ignore dead threads with the same id */
1133 # ifdef THREAD_LOCAL_ALLOC
1134 if ((p -> flags & FINISHED) == 0) {
1135 /* Cannot call GC_destroy_thread_local here (see the */
1136 /* corresponding comment in pthread_support.c). */
1137 GC_remove_specific_after_fork(GC_thread_key, p -> pthread_id);
1140 if (&first_thread != p)
1141 GC_INTERNAL_FREE(p);
1144 GC_threads[hv] = NULL;
1147 /* Put "me" back to GC_threads. */
1148 GC_ASSERT(me != NULL);
1149 thread_id = GetCurrentThreadId(); /* differs from that in parent */
1150 GC_threads[THREAD_TABLE_INDEX(thread_id)] = me;
1152 /* Update Win32 thread Id and handle. */
1153 me -> id = thread_id;
1155 if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1156 GetCurrentProcess(), (HANDLE *)&me->handle,
1157 0 /* dwDesiredAccess */, FALSE /* bInheritHandle */,
1158 DUPLICATE_SAME_ACCESS))
1159 ABORT("DuplicateHandle failed");
1162 # if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
1163 /* For Cygwin, we need to re-assign thread-local pointer to */
1164 /* 'tlfs' (it is OK to call GC_destroy_thread_local and */
1165 /* GC_free_internal before this action). */
1166 if (GC_setspecific(GC_thread_key, &me->tlfs) != 0)
1167 ABORT("GC_setspecific failed (in child)");
1171 static void fork_prepare_proc(void)
1174 # ifdef PARALLEL_MARK
1176 GC_wait_for_reclaim();
1178 GC_wait_for_gc_completion(TRUE);
1179 # ifdef PARALLEL_MARK
1181 GC_acquire_mark_lock();
1185 static void fork_parent_proc(void)
1187 # ifdef PARALLEL_MARK
1189 GC_release_mark_lock();
1194 static void fork_child_proc(void)
1196 # ifdef PARALLEL_MARK
1198 GC_release_mark_lock();
1199 GC_parallel = FALSE; /* or GC_markers_m1 = 0 */
1200 /* Turn off parallel marking in the child, since we are */
1201 /* probably just going to exec, and we would have to */
1202 /* restart mark threads. */
1205 GC_remove_all_threads_but_me();
1209 /* Routines for fork handling by client (no-op if pthread_atfork works). */
1210 GC_API void GC_CALL GC_atfork_prepare(void)
1212 if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
1213 if (GC_handle_fork <= 0)
1214 fork_prepare_proc();
1217 GC_API void GC_CALL GC_atfork_parent(void)
1219 if (GC_handle_fork <= 0)
1223 GC_API void GC_CALL GC_atfork_child(void)
1225 if (GC_handle_fork <= 0)
1228 #endif /* CAN_HANDLE_FORK */
1230 void GC_push_thread_structures(void)
1232 GC_ASSERT(I_HOLD_LOCK());
1233 # ifndef GC_NO_THREADS_DISCOVERY
1234 if (GC_win32_dll_threads) {
1235 /* Unlike the other threads implementations, the thread table */
1236 /* here contains no pointers to the collectible heap (note also */
1237 /* that GC_PTHREADS is incompatible with DllMain-based thread */
1238 /* registration). Thus we have no private structures we need */
1243 GC_PUSH_ALL_SYM(GC_threads);
1245 # if defined(THREAD_LOCAL_ALLOC)
1246 GC_PUSH_ALL_SYM(GC_thread_key);
1247 /* Just in case we ever use our own TLS implementation. */
1251 #ifdef WOW64_THREAD_CONTEXT_WORKAROUND
1252 # ifndef CONTEXT_EXCEPTION_ACTIVE
1253 # define CONTEXT_EXCEPTION_ACTIVE 0x08000000
1254 # define CONTEXT_EXCEPTION_REQUEST 0x40000000
1255 # define CONTEXT_EXCEPTION_REPORTING 0x80000000
1257 static BOOL isWow64; /* Is running 32-bit code on Win64? */
1258 # define GET_THREAD_CONTEXT_FLAGS (isWow64 \
1259 ? CONTEXT_INTEGER | CONTEXT_CONTROL \
1260 | CONTEXT_EXCEPTION_REQUEST | CONTEXT_SEGMENTS \
1261 : CONTEXT_INTEGER | CONTEXT_CONTROL)
1263 # define GET_THREAD_CONTEXT_FLAGS (CONTEXT_INTEGER | CONTEXT_CONTROL)
1264 #endif /* !WOW64_THREAD_CONTEXT_WORKAROUND */
1266 /* Suspend the given thread, if it's still active. */
1267 STATIC void GC_suspend(GC_thread t)
1272 # ifdef RETRY_GET_THREAD_CONTEXT
1274 # define MAX_SUSPEND_THREAD_RETRIES (1000 * 1000)
1277 # ifdef DEBUG_THREADS
1278 GC_log_printf("Suspending 0x%x\n", (int)t->id);
1280 UNPROTECT_THREAD(t);
1281 GC_acquire_dirty_lock();
1284 /* SuspendThread() will fail if thread is running kernel code. */
1285 while (SuspendThread(THREAD_HANDLE(t)) == (DWORD)-1) {
1286 GC_release_dirty_lock();
1287 Sleep(10); /* in millis */
1288 GC_acquire_dirty_lock();
1290 # elif defined(RETRY_GET_THREAD_CONTEXT)
1292 /* Apparently the Windows 95 GetOpenFileName call creates */
1293 /* a thread that does not properly get cleaned up, and */
1294 /* SuspendThread on its descriptor may provoke a crash. */
1295 /* This reduces the probability of that event, though it still */
1296 /* appears there is a race here. */
1297 if (GetExitCodeThread(t -> handle, &exitCode)
1298 && exitCode != STILL_ACTIVE) {
1299 GC_release_dirty_lock();
1301 t -> stack_base = 0; /* prevent stack from being pushed */
1303 /* This breaks pthread_join on Cygwin, which is guaranteed to */
1304 /* only see user threads. */
1305 GC_ASSERT(GC_win32_dll_threads);
1306 GC_delete_gc_thread_no_free(t);
1311 if (SuspendThread(t->handle) != (DWORD)-1) {
1314 context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
1315 if (GetThreadContext(t->handle, &context)) {
1316 /* TODO: WoW64 extra workaround: if CONTEXT_EXCEPTION_ACTIVE */
1317 /* then Sleep(1) and retry. */
1318 t->context_sp = copy_ptr_regs(t->context_regs, &context);
1319 break; /* success; the context pointer registers are saved */
1322 /* Resume the thread, try to suspend it in a better location. */
1323 if (ResumeThread(t->handle) == (DWORD)-1)
1324 ABORT("ResumeThread failed in suspend loop");
1326 if (retry_cnt > 1) {
1327 GC_release_dirty_lock();
1328 Sleep(0); /* yield */
1329 GC_acquire_dirty_lock();
1331 if (++retry_cnt >= MAX_SUSPEND_THREAD_RETRIES)
1332 ABORT("SuspendThread loop failed"); /* something must be wrong */
1335 if (GetExitCodeThread(t -> handle, &exitCode)
1336 && exitCode != STILL_ACTIVE) {
1337 GC_release_dirty_lock();
1339 t -> stack_base = 0; /* prevent stack from being pushed */
1341 GC_ASSERT(GC_win32_dll_threads);
1342 GC_delete_gc_thread_no_free(t);
1346 if (SuspendThread(t -> handle) == (DWORD)-1)
1347 ABORT("SuspendThread failed");
1349 t -> suspended = (unsigned char)TRUE;
1350 GC_release_dirty_lock();
1351 if (GC_on_thread_event)
1352 GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, THREAD_HANDLE(t));
1355 #if defined(GC_ASSERTIONS) \
1356 && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
1357 GC_INNER GC_bool GC_write_disabled = FALSE;
1358 /* TRUE only if GC_stop_world() acquired GC_write_cs. */
1361 GC_INNER void GC_stop_world(void)
1363 DWORD thread_id = GetCurrentThreadId();
1365 if (!GC_thr_initialized)
1366 ABORT("GC_stop_world() called before GC_thr_init()");
1367 GC_ASSERT(I_HOLD_LOCK());
1369 /* This code is the same as in pthread_stop_world.c */
1370 # ifdef PARALLEL_MARK
1372 GC_acquire_mark_lock();
1373 GC_ASSERT(GC_fl_builder_count == 0);
1374 /* We should have previously waited for it to become zero. */
1376 # endif /* PARALLEL_MARK */
1378 # if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
1379 GC_please_stop = TRUE;
1381 # if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
1382 GC_ASSERT(!GC_write_disabled);
1383 EnterCriticalSection(&GC_write_cs);
1384 /* It's not allowed to call GC_printf() (and friends) here down to */
1385 /* LeaveCriticalSection (same applies recursively to GC_suspend, */
1386 /* GC_delete_gc_thread_no_free, GC_get_max_thread_index, GC_size */
1387 /* and GC_remove_protection). */
1388 # ifdef GC_ASSERTIONS
1389 GC_write_disabled = TRUE;
1392 # ifndef GC_NO_THREADS_DISCOVERY
1393 if (GC_win32_dll_threads) {
1396 /* Any threads being created during this loop will end up setting */
1397 /* GC_attached_thread when they start. This will force marking */
1398 /* to restart. This is not ideal, but hopefully correct. */
1399 AO_store(&GC_attached_thread, FALSE);
1400 my_max = (int)GC_get_max_thread_index();
1401 for (i = 0; i <= my_max; i++) {
1402 GC_vthread t = dll_thread_table + i;
1403 if (t -> stack_base != 0 && t -> thread_blocked_sp == NULL
1404 && t -> id != thread_id) {
1405 GC_suspend((GC_thread)t);
1414 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1415 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1416 if (t -> stack_base != 0 && t -> thread_blocked_sp == NULL
1417 && !KNOWN_FINISHED(t) && t -> id != thread_id) {
1423 # if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
1424 # ifdef GC_ASSERTIONS
1425 GC_write_disabled = FALSE;
1427 LeaveCriticalSection(&GC_write_cs);
1429 # ifdef PARALLEL_MARK
1431 GC_release_mark_lock();
1435 GC_INNER void GC_start_world(void)
1437 # ifdef GC_ASSERTIONS
1438 DWORD thread_id = GetCurrentThreadId();
1441 GC_ASSERT(I_HOLD_LOCK());
1442 if (GC_win32_dll_threads) {
1443 LONG my_max = GC_get_max_thread_index();
1446 for (i = 0; i <= my_max; i++) {
1447 GC_thread t = (GC_thread)(dll_thread_table + i);
1448 if (t -> suspended) {
1449 # ifdef DEBUG_THREADS
1450 GC_log_printf("Resuming 0x%x\n", (int)t->id);
1452 GC_ASSERT(t -> stack_base != 0 && t -> id != thread_id);
1453 if (ResumeThread(THREAD_HANDLE(t)) == (DWORD)-1)
1454 ABORT("ResumeThread failed");
1455 t -> suspended = FALSE;
1456 if (GC_on_thread_event)
1457 GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(t));
1459 /* Else thread is unregistered or not suspended. */
1465 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1466 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1467 if (t -> suspended) {
1468 # ifdef DEBUG_THREADS
1469 GC_log_printf("Resuming 0x%x\n", (int)t->id);
1471 GC_ASSERT(t -> stack_base != 0 && t -> id != thread_id);
1472 if (ResumeThread(THREAD_HANDLE(t)) == (DWORD)-1)
1473 ABORT("ResumeThread failed");
1474 UNPROTECT_THREAD(t);
1475 t -> suspended = FALSE;
1476 if (GC_on_thread_event)
1477 GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(t));
1479 # ifdef DEBUG_THREADS
1480 GC_log_printf("Not resuming thread 0x%x as it is not suspended\n",
1487 # if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
1488 GC_please_stop = FALSE;
1493 /* The VirtualQuery calls below won't work properly on some old WinCE */
1494 /* versions, but since each stack is restricted to an aligned 64 KiB */
1495 /* region of virtual memory we can just take the next lowest multiple */
1496 /* of 64 KiB. The result of this macro must not be used as its */
1497 /* argument later and must not be used as the lower bound for sp */
1498 /* check (since the stack may be bigger than 64 KiB). */
1499 # define GC_wince_evaluate_stack_min(s) \
1500 (ptr_t)(((word)(s) - 1) & ~(word)0xFFFF)
1501 #elif defined(GC_ASSERTIONS)
1502 # define GC_dont_query_stack_min FALSE
1505 /* A cache holding the results of the recent VirtualQuery call. */
1506 /* Protected by the allocation lock. */
1507 static ptr_t last_address = 0;
1508 static MEMORY_BASIC_INFORMATION last_info;
1510 /* Probe stack memory region (starting at "s") to find out its */
1511 /* lowest address (i.e. stack top). */
1512 /* S must be a mapped address inside the region, NOT the first */
1513 /* unmapped address. */
1514 STATIC ptr_t GC_get_stack_min(ptr_t s)
1518 GC_ASSERT(I_HOLD_LOCK());
1519 if (s != last_address) {
1520 VirtualQuery(s, &last_info, sizeof(last_info));
1524 bottom = (ptr_t)last_info.BaseAddress;
1525 VirtualQuery(bottom - 1, &last_info, sizeof(last_info));
1526 last_address = bottom - 1;
1527 } while ((last_info.Protect & PAGE_READWRITE)
1528 && !(last_info.Protect & PAGE_GUARD));
1532 /* Return true if the page at s has protections appropriate */
1533 /* for a stack page. */
1534 static GC_bool may_be_in_stack(ptr_t s)
1536 GC_ASSERT(I_HOLD_LOCK());
1537 if (s != last_address) {
1538 VirtualQuery(s, &last_info, sizeof(last_info));
1541 return (last_info.Protect & PAGE_READWRITE)
1542 && !(last_info.Protect & PAGE_GUARD);
1545 /* Copy all registers that might point into the heap. Frame */
1546 /* pointer registers are included in case client code was */
1547 /* compiled with the 'omit frame pointer' optimization. */
1548 /* The context register values are stored to regs argument */
1549 /* which is expected to be of PUSHED_REGS_COUNT length exactly. */
1550 /* The functions returns the context stack pointer value. */
1551 static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext) {
1554 # define context (*pcontext)
1555 # define PUSH1(reg) (regs[cnt++] = (word)pcontext->reg)
1556 # define PUSH2(r1,r2) (PUSH1(r1), PUSH1(r2))
1557 # define PUSH4(r1,r2,r3,r4) (PUSH2(r1,r2), PUSH2(r3,r4))
1559 # ifdef WOW64_THREAD_CONTEXT_WORKAROUND
1560 PUSH2(ContextFlags, SegFs); /* cannot contain pointers */
1562 PUSH4(Edi,Esi,Ebx,Edx), PUSH2(Ecx,Eax), PUSH1(Ebp);
1563 sp = (ptr_t)context.Esp;
1564 # elif defined(X86_64)
1565 PUSH4(Rax,Rcx,Rdx,Rbx); PUSH2(Rbp, Rsi); PUSH1(Rdi);
1566 PUSH4(R8, R9, R10, R11); PUSH4(R12, R13, R14, R15);
1567 sp = (ptr_t)context.Rsp;
1568 # elif defined(ARM32)
1569 PUSH4(R0,R1,R2,R3),PUSH4(R4,R5,R6,R7),PUSH4(R8,R9,R10,R11);
1571 sp = (ptr_t)context.Sp;
1572 # elif defined(AARCH64)
1573 PUSH4(X0,X1,X2,X3),PUSH4(X4,X5,X6,X7),PUSH4(X8,X9,X10,X11);
1574 PUSH4(X12,X13,X14,X15),PUSH4(X16,X17,X18,X19),PUSH4(X20,X21,X22,X23);
1575 PUSH4(X24,X25,X26,X27),PUSH1(X28);
1577 sp = (ptr_t)context.Sp;
1579 PUSH4(R0,R1,R2,R3), PUSH4(R4,R5,R6,R7), PUSH4(R8,R9,R10,R11);
1580 PUSH2(R12,R13), PUSH1(R14);
1581 sp = (ptr_t)context.R15;
1582 # elif defined(MIPS)
1583 PUSH4(IntAt,IntV0,IntV1,IntA0), PUSH4(IntA1,IntA2,IntA3,IntT0);
1584 PUSH4(IntT1,IntT2,IntT3,IntT4), PUSH4(IntT5,IntT6,IntT7,IntS0);
1585 PUSH4(IntS1,IntS2,IntS3,IntS4), PUSH4(IntS5,IntS6,IntS7,IntT8);
1586 PUSH4(IntT9,IntK0,IntK1,IntS8);
1587 sp = (ptr_t)context.IntSp;
1589 PUSH4(Gpr0, Gpr3, Gpr4, Gpr5), PUSH4(Gpr6, Gpr7, Gpr8, Gpr9);
1590 PUSH4(Gpr10,Gpr11,Gpr12,Gpr14), PUSH4(Gpr15,Gpr16,Gpr17,Gpr18);
1591 PUSH4(Gpr19,Gpr20,Gpr21,Gpr22), PUSH4(Gpr23,Gpr24,Gpr25,Gpr26);
1592 PUSH4(Gpr27,Gpr28,Gpr29,Gpr30), PUSH1(Gpr31);
1593 sp = (ptr_t)context.Gpr1;
1594 # elif defined(ALPHA)
1595 PUSH4(IntV0,IntT0,IntT1,IntT2), PUSH4(IntT3,IntT4,IntT5,IntT6);
1596 PUSH4(IntT7,IntS0,IntS1,IntS2), PUSH4(IntS3,IntS4,IntS5,IntFp);
1597 PUSH4(IntA0,IntA1,IntA2,IntA3), PUSH4(IntA4,IntA5,IntT8,IntT9);
1598 PUSH4(IntT10,IntT11,IntT12,IntAt);
1599 sp = (ptr_t)context.IntSp;
1600 # elif defined(CPPCHECK)
1601 sp = (ptr_t)(word)cnt; /* to workaround "cnt not used" false positive */
1603 # error Architecture is not supported
1606 GC_ASSERT(cnt == PUSHED_REGS_COUNT);
1610 STATIC word GC_push_stack_for(GC_thread thread, DWORD me)
1612 ptr_t sp, stack_min;
1614 struct GC_traced_stack_sect_s *traced_stack_sect =
1615 thread -> traced_stack_sect;
1616 if (thread -> id == me) {
1617 GC_ASSERT(thread -> thread_blocked_sp == NULL);
1618 sp = GC_approx_sp();
1619 } else if ((sp = thread -> thread_blocked_sp) == NULL) {
1620 /* Use saved sp value for blocked threads. */
1621 # ifdef RETRY_GET_THREAD_CONTEXT
1622 /* We cache context when suspending the thread since it may */
1623 /* require looping. */
1624 word *regs = thread->context_regs;
1626 if (thread->suspended) {
1627 sp = thread->context_sp;
1630 word regs[PUSHED_REGS_COUNT];
1636 /* For unblocked threads call GetThreadContext(). */
1637 context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
1638 if (GetThreadContext(THREAD_HANDLE(thread), &context)) {
1639 sp = copy_ptr_regs(regs, &context);
1641 # ifdef RETRY_GET_THREAD_CONTEXT
1642 /* At least, try to use the stale context if saved. */
1643 sp = thread->context_sp;
1645 /* Skip the current thread, anyway its stack will */
1646 /* be pushed when the world is stopped. */
1650 ABORT("GetThreadContext failed");
1654 # ifdef THREAD_LOCAL_ALLOC
1655 GC_ASSERT(thread->suspended || !GC_world_stopped);
1658 # ifndef WOW64_THREAD_CONTEXT_WORKAROUND
1659 GC_push_many_regs(regs, PUSHED_REGS_COUNT);
1661 GC_push_many_regs(regs + 2, PUSHED_REGS_COUNT - 2);
1662 /* skip ContextFlags and SegFs */
1664 /* WoW64 workaround. */
1666 DWORD ContextFlags = (DWORD)regs[0];
1667 WORD SegFs = (WORD)regs[1];
1669 if ((ContextFlags & CONTEXT_EXCEPTION_REPORTING) != 0
1670 && (ContextFlags & (CONTEXT_EXCEPTION_ACTIVE
1671 /* | CONTEXT_SERVICE_ACTIVE */)) != 0) {
1675 if (!GetThreadSelectorEntry(THREAD_HANDLE(thread), SegFs, &selector))
1676 ABORT("GetThreadSelectorEntry failed");
1677 tib = (PNT_TIB)(selector.BaseLow
1678 | (selector.HighWord.Bits.BaseMid << 16)
1679 | (selector.HighWord.Bits.BaseHi << 24));
1680 # ifdef DEBUG_THREADS
1681 GC_log_printf("TIB stack limit/base: %p .. %p\n",
1682 (void *)tib->StackLimit, (void *)tib->StackBase);
1684 GC_ASSERT(!((word)thread->stack_base
1685 COOLER_THAN (word)tib->StackBase));
1686 if (thread->stack_base != thread->initial_stack_base
1687 /* We are in a coroutine. */
1688 && ((word)thread->stack_base <= (word)tib->StackLimit
1689 || (word)tib->StackBase < (word)thread->stack_base)) {
1690 /* The coroutine stack is not within TIB stack. */
1691 WARN("GetThreadContext might return stale register values"
1692 " including ESP=%p\n", sp);
1693 /* TODO: Because of WoW64 bug, there is no guarantee that */
1694 /* sp really points to the stack top but, for now, we do */
1695 /* our best as the TIB stack limit/base cannot be used */
1696 /* while we are inside a coroutine. */
1698 /* GetThreadContext() might return stale register values, */
1699 /* so we scan the entire stack region (down to the stack */
1700 /* limit). There is no 100% guarantee that all the */
1701 /* registers are pushed but we do our best (the proper */
1702 /* solution would be to fix it inside Windows OS). */
1703 sp = (ptr_t)tib->StackLimit;
1706 # ifdef DEBUG_THREADS
1708 static GC_bool logged;
1710 && (ContextFlags & CONTEXT_EXCEPTION_REPORTING) == 0) {
1711 GC_log_printf("CONTEXT_EXCEPTION_REQUEST not supported\n");
1717 # endif /* WOW64_THREAD_CONTEXT_WORKAROUND */
1718 } /* ! current thread */
1720 /* Set stack_min to the lowest address in the thread stack, */
1721 /* or to an address in the thread stack no larger than sp, */
1722 /* taking advantage of the old value to avoid slow traversals */
1723 /* of large stacks. */
1724 if (thread -> last_stack_min == ADDR_LIMIT) {
1726 if (GC_dont_query_stack_min) {
1727 stack_min = GC_wince_evaluate_stack_min(traced_stack_sect != NULL ?
1728 (ptr_t)traced_stack_sect : thread -> stack_base);
1729 /* Keep last_stack_min value unmodified. */
1733 stack_min = GC_get_stack_min(traced_stack_sect != NULL ?
1734 (ptr_t)traced_stack_sect : thread -> stack_base);
1735 UNPROTECT_THREAD(thread);
1736 thread -> last_stack_min = stack_min;
1739 /* First, adjust the latest known minimum stack address if we */
1740 /* are inside GC_call_with_gc_active(). */
1741 if (traced_stack_sect != NULL &&
1742 (word)thread->last_stack_min > (word)traced_stack_sect) {
1743 UNPROTECT_THREAD(thread);
1744 thread -> last_stack_min = (ptr_t)traced_stack_sect;
1747 if ((word)sp < (word)thread->stack_base
1748 && (word)sp >= (word)thread->last_stack_min) {
1751 /* In the current thread it is always safe to use sp value. */
1752 if (may_be_in_stack(thread -> id == me &&
1753 (word)sp < (word)thread->last_stack_min ?
1754 sp : thread -> last_stack_min)) {
1755 stack_min = (ptr_t)last_info.BaseAddress;
1756 /* Do not probe rest of the stack if sp is correct. */
1757 if ((word)sp < (word)stack_min
1758 || (word)sp >= (word)thread->stack_base)
1759 stack_min = GC_get_stack_min(thread -> last_stack_min);
1761 /* Stack shrunk? Is this possible? */
1762 stack_min = GC_get_stack_min(thread -> stack_base);
1764 UNPROTECT_THREAD(thread);
1765 thread -> last_stack_min = stack_min;
1769 GC_ASSERT(GC_dont_query_stack_min
1770 || stack_min == GC_get_stack_min(thread -> stack_base)
1771 || ((word)sp >= (word)stack_min
1772 && (word)stack_min < (word)thread->stack_base
1774 > (word)GC_get_stack_min(thread -> stack_base)));
1776 if ((word)sp >= (word)stack_min && (word)sp < (word)thread->stack_base) {
1777 # ifdef DEBUG_THREADS
1778 GC_log_printf("Pushing stack for 0x%x from sp %p to %p from 0x%x\n",
1779 (int)thread->id, (void *)sp, (void *)thread->stack_base,
1782 GC_push_all_stack_sections(sp, thread->stack_base, traced_stack_sect);
1784 /* If not current thread then it is possible for sp to point to */
1785 /* the guarded (untouched yet) page just below the current */
1786 /* stack_min of the thread. */
1787 if (thread -> id == me || (word)sp >= (word)thread->stack_base
1788 || (word)(sp + GC_page_size) < (word)stack_min)
1789 WARN("Thread stack pointer %p out of range, pushing everything\n",
1791 # ifdef DEBUG_THREADS
1792 GC_log_printf("Pushing stack for 0x%x from (min) %p to %p from 0x%x\n",
1793 (int)thread->id, (void *)stack_min,
1794 (void *)thread->stack_base, (int)me);
1796 /* Push everything - ignore "traced stack section" data. */
1797 GC_push_all_stack(stack_min, thread->stack_base);
1799 return thread->stack_base - sp; /* stack grows down */
1802 /* We hold allocation lock. Should do exactly the right thing if the */
1803 /* world is stopped. Should not fail if it isn't. */
1804 GC_INNER void GC_push_all_stacks(void)
1806 DWORD thread_id = GetCurrentThreadId();
1807 GC_bool found_me = FALSE;
1808 # ifndef SMALL_CONFIG
1809 unsigned nthreads = 0;
1811 word total_size = 0;
1812 # ifndef GC_NO_THREADS_DISCOVERY
1813 if (GC_win32_dll_threads) {
1815 LONG my_max = GC_get_max_thread_index();
1817 for (i = 0; i <= my_max; i++) {
1818 GC_thread t = (GC_thread)(dll_thread_table + i);
1819 if (t -> tm.in_use && t -> stack_base) {
1820 # ifndef SMALL_CONFIG
1823 total_size += GC_push_stack_for(t, thread_id);
1824 if (t -> id == thread_id) found_me = TRUE;
1831 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1833 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1834 if (!KNOWN_FINISHED(t) && t -> stack_base) {
1835 # ifndef SMALL_CONFIG
1838 total_size += GC_push_stack_for(t, thread_id);
1839 if (t -> id == thread_id) found_me = TRUE;
1844 # ifndef SMALL_CONFIG
1845 GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks%s\n", nthreads,
1846 GC_win32_dll_threads ?
1847 " based on DllMain thread tracking" : "");
1849 if (!found_me && !GC_in_thread_creation)
1850 ABORT("Collecting from unknown thread");
1851 GC_total_stacksize = total_size;
1854 #ifdef PARALLEL_MARK
1856 # ifndef MAX_MARKERS
1857 # define MAX_MARKERS 16
1860 static ptr_t marker_sp[MAX_MARKERS - 1]; /* The cold end of the stack */
1863 static ptr_t marker_bsp[MAX_MARKERS - 1];
1866 static ptr_t marker_last_stack_min[MAX_MARKERS - 1];
1867 /* Last known minimum (hottest) address */
1868 /* in stack (or ADDR_LIMIT if unset) */
1871 #endif /* PARALLEL_MARK */
1873 /* Find stack with the lowest address which overlaps the */
1874 /* interval [start, limit). */
1875 /* Return stack bounds in *lo and *hi. If no such stack */
1876 /* is found, both *hi and *lo will be set to an address */
1877 /* higher than limit. */
1878 GC_INNER void GC_get_next_stack(char *start, char *limit,
1879 char **lo, char **hi)
1882 char * current_min = ADDR_LIMIT; /* Least in-range stack base */
1883 ptr_t *plast_stack_min = NULL; /* Address of last_stack_min */
1884 /* field for thread corresponding */
1885 /* to current_min. */
1886 GC_thread thread = NULL; /* Either NULL or points to the */
1887 /* thread's hash table entry */
1888 /* containing *plast_stack_min. */
1890 /* First set current_min, ignoring limit. */
1891 if (GC_win32_dll_threads) {
1892 LONG my_max = GC_get_max_thread_index();
1894 for (i = 0; i <= my_max; i++) {
1895 ptr_t s = (ptr_t)(dll_thread_table[i].stack_base);
1897 if ((word)s > (word)start && (word)s < (word)current_min) {
1898 /* Update address of last_stack_min. */
1899 plast_stack_min = (ptr_t * /* no volatile */)
1900 &dll_thread_table[i].last_stack_min;
1902 # if defined(CPPCHECK)
1903 /* To avoid a warning that thread is always null. */
1904 thread = (GC_thread)&dll_thread_table[i];
1909 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1912 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1913 ptr_t s = t -> stack_base;
1915 if ((word)s > (word)start && (word)s < (word)current_min) {
1916 /* Update address of last_stack_min. */
1917 plast_stack_min = &t -> last_stack_min;
1918 thread = t; /* Remember current thread to unprotect. */
1923 # ifdef PARALLEL_MARK
1924 for (i = 0; i < GC_markers_m1; ++i) {
1925 ptr_t s = marker_sp[i];
1927 /* FIXME: not implemented */
1929 if ((word)s > (word)start && (word)s < (word)current_min) {
1930 GC_ASSERT(marker_last_stack_min[i] != NULL);
1931 plast_stack_min = &marker_last_stack_min[i];
1933 thread = NULL; /* Not a thread's hash table entry. */
1940 if (current_min == ADDR_LIMIT) {
1945 GC_ASSERT((word)current_min > (word)start && plast_stack_min != NULL);
1947 if (GC_dont_query_stack_min) {
1948 *lo = GC_wince_evaluate_stack_min(current_min);
1949 /* Keep last_stack_min value unmodified. */
1954 if ((word)current_min > (word)limit && !may_be_in_stack(limit)) {
1955 /* Skip the rest since the memory region at limit address is */
1956 /* not a stack (so the lowest address of the found stack would */
1957 /* be above the limit value anyway). */
1962 /* Get the minimum address of the found stack by probing its memory */
1963 /* region starting from the recent known minimum (if set). */
1964 if (*plast_stack_min == ADDR_LIMIT
1965 || !may_be_in_stack(*plast_stack_min)) {
1966 /* Unsafe to start from last_stack_min value. */
1967 *lo = GC_get_stack_min(current_min);
1969 /* Use the recent value to optimize search for min address. */
1970 *lo = GC_get_stack_min(*plast_stack_min);
1973 /* Remember current stack_min value. */
1974 if (thread != NULL) {
1975 UNPROTECT_THREAD(thread);
1977 *plast_stack_min = *lo;
1980 #ifdef PARALLEL_MARK
1982 # if defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
1983 /* Use pthread-based parallel mark implementation. */
1985 /* Workaround a deadlock in winpthreads-3.0b internals (observed */
1986 /* with MinGW 32/64). */
1987 # if !defined(__MINGW32__)
1988 # define GC_PTHREADS_PARAMARK
1992 # if !defined(GC_PTHREADS_PARAMARK)
1993 STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1] = {0};
1994 /* Events with manual reset (one for each */
1997 STATIC DWORD GC_marker_Id[MAX_MARKERS - 1] = {0};
1998 /* This table is used for mapping helper */
1999 /* threads ID to mark helper index (linear */
2000 /* search is used since the mapping contains */
2001 /* only a few entries). */
2004 /* GC_mark_thread() is the same as in pthread_support.c */
2005 # ifdef GC_PTHREADS_PARAMARK
2006 STATIC void * GC_mark_thread(void * id)
2007 # elif defined(MSWINCE)
2008 STATIC DWORD WINAPI GC_mark_thread(LPVOID id)
2010 STATIC unsigned __stdcall GC_mark_thread(void * id)
2013 word my_mark_no = 0;
2015 if ((word)id == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
2016 marker_sp[(word)id] = GC_approx_sp();
2018 marker_bsp[(word)id] = GC_save_regs_in_stack();
2020 # if !defined(GC_PTHREADS_PARAMARK)
2021 GC_marker_Id[(word)id] = GetCurrentThreadId();
2024 /* Inform GC_start_mark_threads about completion of marker data init. */
2025 GC_acquire_mark_lock();
2026 if (0 == --GC_fl_builder_count) /* count may have a negative value */
2027 GC_notify_all_builder();
2029 for (;; ++my_mark_no) {
2030 if (my_mark_no - GC_mark_no > (word)2) {
2031 /* resynchronize if we get far off, e.g. because GC_mark_no */
2033 my_mark_no = GC_mark_no;
2035 # ifdef DEBUG_THREADS
2036 GC_log_printf("Starting mark helper for mark number %lu\n",
2037 (unsigned long)my_mark_no);
2039 GC_help_marker(my_mark_no);
2043 # ifndef GC_ASSERTIONS
2044 # define SET_MARK_LOCK_HOLDER (void)0
2045 # define UNSET_MARK_LOCK_HOLDER (void)0
2048 /* GC_mark_threads[] is unused here unlike that in pthread_support.c */
2050 # ifdef CAN_HANDLE_FORK
2051 static int available_markers_m1 = 0;
2053 # define available_markers_m1 GC_markers_m1
2056 # ifdef GC_PTHREADS_PARAMARK
2057 # include <pthread.h>
2059 # if defined(GC_ASSERTIONS) && !defined(NUMERIC_THREAD_ID)
2060 # define NUMERIC_THREAD_ID(id) (unsigned long)(word)GC_PTHREAD_PTRVAL(id)
2061 /* Id not guaranteed to be unique. */
2064 # ifdef CAN_HANDLE_FORK
2065 static pthread_cond_t mark_cv;
2066 /* initialized by GC_start_mark_threads_inner */
2068 static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
2071 /* GC_start_mark_threads is the same as in pthread_support.c except */
2072 /* for thread stack that is assumed to be large enough. */
2074 GC_INNER void GC_start_mark_threads_inner(void)
2077 pthread_attr_t attr;
2078 pthread_t new_thread;
2079 # ifndef NO_MARKER_SPECIAL_SIGMASK
2080 sigset_t set, oldset;
2083 GC_ASSERT(I_DONT_HOLD_LOCK());
2084 if (available_markers_m1 <= 0) return;
2085 /* Skip if parallel markers disabled or already started. */
2086 # ifdef CAN_HANDLE_FORK
2087 if (GC_parallel) return;
2089 /* Reset mark_cv state after forking (as in pthread_support.c). */
2091 pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
2092 BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
2096 GC_ASSERT(GC_fl_builder_count == 0);
2097 if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
2098 if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
2099 ABORT("pthread_attr_setdetachstate failed");
2101 # ifndef NO_MARKER_SPECIAL_SIGMASK
2102 /* Apply special signal mask to GC marker threads, and don't drop */
2103 /* user defined signals by GC marker threads. */
2104 if (sigfillset(&set) != 0)
2105 ABORT("sigfillset failed");
2106 if (pthread_sigmask(SIG_BLOCK, &set, &oldset) < 0) {
2107 WARN("pthread_sigmask set failed, no markers started,"
2108 " errno = %" WARN_PRIdPTR "\n", errno);
2110 (void)pthread_attr_destroy(&attr);
2113 # endif /* !NO_MARKER_SPECIAL_SIGMASK */
2115 # ifdef CAN_HANDLE_FORK
2116 /* To have proper GC_parallel value in GC_help_marker. */
2117 GC_markers_m1 = available_markers_m1;
2119 for (i = 0; i < available_markers_m1; ++i) {
2120 marker_last_stack_min[i] = ADDR_LIMIT;
2121 if (0 != pthread_create(&new_thread, &attr,
2122 GC_mark_thread, (void *)(word)i)) {
2123 WARN("Marker thread creation failed\n", 0);
2124 /* Don't try to create other marker threads. */
2130 # ifndef NO_MARKER_SPECIAL_SIGMASK
2131 /* Restore previous signal mask. */
2132 if (pthread_sigmask(SIG_SETMASK, &oldset, NULL) < 0) {
2133 WARN("pthread_sigmask restore failed, errno = %" WARN_PRIdPTR "\n",
2138 (void)pthread_attr_destroy(&attr);
2139 GC_wait_for_markers_init();
2140 GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
2143 # ifdef GC_ASSERTIONS
2144 STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
2145 # define SET_MARK_LOCK_HOLDER \
2146 (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
2147 # define UNSET_MARK_LOCK_HOLDER \
2149 GC_ASSERT(GC_mark_lock_holder \
2150 == NUMERIC_THREAD_ID(pthread_self())); \
2151 GC_mark_lock_holder = NO_THREAD; \
2153 # endif /* GC_ASSERTIONS */
2155 static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
2157 static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
2159 /* GC_acquire/release_mark_lock(), GC_wait_builder/marker(), */
2160 /* GC_wait_for_reclaim(), GC_notify_all_builder/marker() are the same */
2161 /* as in pthread_support.c except that GC_generic_lock() is not used. */
2164 volatile AO_t GC_block_count = 0;
2167 GC_INNER void GC_acquire_mark_lock(void)
2169 # if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
2170 GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
2172 if (pthread_mutex_lock(&mark_mutex) != 0) {
2173 ABORT("pthread_mutex_lock failed");
2176 (void)AO_fetch_and_add1(&GC_block_count);
2178 /* GC_generic_lock(&mark_mutex); */
2179 SET_MARK_LOCK_HOLDER;
2182 GC_INNER void GC_release_mark_lock(void)
2184 UNSET_MARK_LOCK_HOLDER;
2185 if (pthread_mutex_unlock(&mark_mutex) != 0) {
2186 ABORT("pthread_mutex_unlock failed");
2190 /* Collector must wait for a freelist builders for 2 reasons: */
2191 /* 1) Mark bits may still be getting examined without lock. */
2192 /* 2) Partial free lists referenced only by locals may not be */
2193 /* scanned correctly, e.g. if they contain "pointer-free" objects, */
2194 /* since the free-list link may be ignored. */
2195 STATIC void GC_wait_builder(void)
2197 UNSET_MARK_LOCK_HOLDER;
2198 if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
2199 ABORT("pthread_cond_wait failed");
2201 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2202 SET_MARK_LOCK_HOLDER;
2205 GC_INNER void GC_wait_for_reclaim(void)
2207 GC_acquire_mark_lock();
2208 while (GC_fl_builder_count > 0) {
2211 GC_release_mark_lock();
2214 GC_INNER void GC_notify_all_builder(void)
2216 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
2217 if (pthread_cond_broadcast(&builder_cv) != 0) {
2218 ABORT("pthread_cond_broadcast failed");
2222 GC_INNER void GC_wait_marker(void)
2224 GC_ASSERT(GC_parallel);
2225 UNSET_MARK_LOCK_HOLDER;
2226 if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
2227 ABORT("pthread_cond_wait failed");
2229 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2230 SET_MARK_LOCK_HOLDER;
2233 GC_INNER void GC_notify_all_marker(void)
2235 GC_ASSERT(GC_parallel);
2236 if (pthread_cond_broadcast(&mark_cv) != 0) {
2237 ABORT("pthread_cond_broadcast failed");
2241 # else /* ! GC_PTHREADS_PARAMARK */
2243 # ifndef MARK_THREAD_STACK_SIZE
2244 # define MARK_THREAD_STACK_SIZE 0 /* default value */
2247 /* mark_mutex_event, builder_cv, mark_cv are initialized in GC_thr_init */
2248 static HANDLE mark_mutex_event = (HANDLE)0; /* Event with auto-reset. */
2249 static HANDLE builder_cv = (HANDLE)0; /* Event with manual reset. */
2250 static HANDLE mark_cv = (HANDLE)0; /* Event with manual reset. */
2252 GC_INNER void GC_start_mark_threads_inner(void)
2256 GC_ASSERT(I_DONT_HOLD_LOCK());
2257 if (available_markers_m1 <= 0) return;
2259 GC_ASSERT(GC_fl_builder_count == 0);
2260 /* Initialize GC_marker_cv[] fully before starting the */
2261 /* first helper thread. */
2262 for (i = 0; i < GC_markers_m1; ++i) {
2263 if ((GC_marker_cv[i] = CreateEvent(NULL /* attrs */,
2264 TRUE /* isManualReset */,
2265 FALSE /* initialState */,
2266 NULL /* name (A/W) */)) == (HANDLE)0)
2267 ABORT("CreateEvent failed");
2270 for (i = 0; i < GC_markers_m1; ++i) {
2271 # if defined(MSWINCE) || defined(MSWIN_XBOX1)
2275 marker_last_stack_min[i] = ADDR_LIMIT;
2276 /* There is no _beginthreadex() in WinCE. */
2277 handle = CreateThread(NULL /* lpsa */,
2278 MARK_THREAD_STACK_SIZE /* ignored */,
2279 GC_mark_thread, (LPVOID)(word)i,
2280 0 /* fdwCreate */, &thread_id);
2281 if (handle == NULL) {
2282 WARN("Marker thread creation failed\n", 0);
2283 /* The most probable failure reason is "not enough memory". */
2284 /* Don't try to create other marker threads. */
2287 /* It's safe to detach the thread. */
2288 CloseHandle(handle);
2291 GC_uintptr_t handle;
2294 marker_last_stack_min[i] = ADDR_LIMIT;
2295 handle = _beginthreadex(NULL /* security_attr */,
2296 MARK_THREAD_STACK_SIZE, GC_mark_thread,
2297 (void *)(word)i, 0 /* flags */, &thread_id);
2298 if (!handle || handle == (GC_uintptr_t)-1L) {
2299 WARN("Marker thread creation failed\n", 0);
2300 /* Don't try to create other marker threads. */
2302 } else {/* We may detach the thread (if handle is of HANDLE type) */
2303 /* CloseHandle((HANDLE)handle); */
2308 /* Adjust GC_markers_m1 (and free unused resources) if failed. */
2309 while (GC_markers_m1 > i) {
2311 CloseHandle(GC_marker_cv[GC_markers_m1]);
2313 GC_wait_for_markers_init();
2314 GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
2316 CloseHandle(mark_cv);
2317 CloseHandle(builder_cv);
2318 CloseHandle(mark_mutex_event);
2322 # ifdef GC_ASSERTIONS
2323 STATIC DWORD GC_mark_lock_holder = NO_THREAD;
2324 # define SET_MARK_LOCK_HOLDER \
2325 (void)(GC_mark_lock_holder = GetCurrentThreadId())
2326 # define UNSET_MARK_LOCK_HOLDER \
2328 GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId()); \
2329 GC_mark_lock_holder = NO_THREAD; \
2331 # endif /* GC_ASSERTIONS */
2333 STATIC /* volatile */ LONG GC_mark_mutex_state = 0;
2334 /* Mutex state: 0 - unlocked, */
2335 /* 1 - locked and no other waiters, */
2336 /* -1 - locked and waiters may exist. */
2337 /* Accessed by InterlockedExchange(). */
2339 /* #define LOCK_STATS */
2341 volatile AO_t GC_block_count = 0;
2342 volatile AO_t GC_unlocked_count = 0;
2345 GC_INNER void GC_acquire_mark_lock(void)
2347 # ifndef THREAD_SANITIZER
2348 GC_ASSERT(GC_mark_lock_holder != GetCurrentThreadId());
2350 if (InterlockedExchange(&GC_mark_mutex_state, 1 /* locked */) != 0) {
2352 (void)AO_fetch_and_add1(&GC_block_count);
2354 /* Repeatedly reset the state and wait until acquire the lock. */
2355 while (InterlockedExchange(&GC_mark_mutex_state,
2356 -1 /* locked_and_has_waiters */) != 0) {
2357 if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
2358 ABORT("WaitForSingleObject failed");
2363 (void)AO_fetch_and_add1(&GC_unlocked_count);
2367 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2368 SET_MARK_LOCK_HOLDER;
2371 GC_INNER void GC_release_mark_lock(void)
2373 UNSET_MARK_LOCK_HOLDER;
2374 if (InterlockedExchange(&GC_mark_mutex_state, 0 /* unlocked */) < 0) {
2376 if (SetEvent(mark_mutex_event) == FALSE)
2377 ABORT("SetEvent failed");
2381 /* In GC_wait_for_reclaim/GC_notify_all_builder() we emulate POSIX */
2382 /* cond_wait/cond_broadcast() primitives with WinAPI Event object */
2383 /* (working in "manual reset" mode). This works here because */
2384 /* GC_notify_all_builder() is always called holding lock on */
2385 /* mark_mutex and the checked condition (GC_fl_builder_count == 0) */
2386 /* is the only one for which broadcasting on builder_cv is performed. */
2388 GC_INNER void GC_wait_for_reclaim(void)
2390 GC_ASSERT(builder_cv != 0);
2392 GC_acquire_mark_lock();
2393 if (GC_fl_builder_count == 0)
2395 if (ResetEvent(builder_cv) == FALSE)
2396 ABORT("ResetEvent failed");
2397 GC_release_mark_lock();
2398 if (WaitForSingleObject(builder_cv, INFINITE) == WAIT_FAILED)
2399 ABORT("WaitForSingleObject failed");
2401 GC_release_mark_lock();
2404 GC_INNER void GC_notify_all_builder(void)
2406 GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId());
2407 GC_ASSERT(builder_cv != 0);
2408 GC_ASSERT(GC_fl_builder_count == 0);
2409 if (SetEvent(builder_cv) == FALSE)
2410 ABORT("SetEvent failed");
2413 /* mark_cv is used (for waiting) by a non-helper thread. */
2415 GC_INNER void GC_wait_marker(void)
2417 HANDLE event = mark_cv;
2418 DWORD thread_id = GetCurrentThreadId();
2419 int i = GC_markers_m1;
2422 if (GC_marker_Id[i] == thread_id) {
2423 event = GC_marker_cv[i];
2428 if (ResetEvent(event) == FALSE)
2429 ABORT("ResetEvent failed");
2430 GC_release_mark_lock();
2431 if (WaitForSingleObject(event, INFINITE) == WAIT_FAILED)
2432 ABORT("WaitForSingleObject failed");
2433 GC_acquire_mark_lock();
2436 GC_INNER void GC_notify_all_marker(void)
2438 DWORD thread_id = GetCurrentThreadId();
2439 int i = GC_markers_m1;
2442 /* Notify every marker ignoring self (for efficiency). */
2443 if (SetEvent(GC_marker_Id[i] != thread_id ? GC_marker_cv[i] :
2445 ABORT("SetEvent failed");
2449 # endif /* ! GC_PTHREADS_PARAMARK */
2451 #endif /* PARALLEL_MARK */
2453 /* We have no DllMain to take care of new threads. Thus we */
2454 /* must properly intercept thread creation. */
2457 LPTHREAD_START_ROUTINE start;
2461 STATIC void * GC_CALLBACK GC_win32_start_inner(struct GC_stack_base *sb,
2465 LPTHREAD_START_ROUTINE start = ((thread_args *)arg)->start;
2466 LPVOID param = ((thread_args *)arg)->param;
2468 GC_register_my_thread(sb); /* This waits for an in-progress GC. */
2470 # ifdef DEBUG_THREADS
2471 GC_log_printf("thread 0x%lx starting...\n", (long)GetCurrentThreadId());
2476 /* Clear the thread entry even if we exit with an exception. */
2477 /* This is probably pointless, since an uncaught exception is */
2478 /* supposed to result in the process being killed. */
2479 # if !defined(__GNUC__) && !defined(NO_CRT)
2480 ret = NULL; /* to suppress "might be uninitialized" compiler warning */
2484 ret = (void *)(word)(*start)(param);
2486 # if !defined(__GNUC__) && !defined(NO_CRT)
2490 GC_unregister_my_thread();
2493 # ifdef DEBUG_THREADS
2494 GC_log_printf("thread 0x%lx returned from start routine\n",
2495 (long)GetCurrentThreadId());
2500 STATIC DWORD WINAPI GC_win32_start(LPVOID arg)
2502 return (DWORD)(word)GC_call_with_stack_base(GC_win32_start_inner, arg);
2505 GC_API HANDLE WINAPI GC_CreateThread(
2506 LPSECURITY_ATTRIBUTES lpThreadAttributes,
2507 GC_WIN32_SIZE_T dwStackSize,
2508 LPTHREAD_START_ROUTINE lpStartAddress,
2509 LPVOID lpParameter, DWORD dwCreationFlags,
2512 if (!EXPECT(parallel_initialized, TRUE))
2514 /* make sure GC is initialized (i.e. main thread is */
2515 /* attached, tls initialized). */
2517 # ifdef DEBUG_THREADS
2518 GC_log_printf("About to create a thread from 0x%lx\n",
2519 (long)GetCurrentThreadId());
2521 if (GC_win32_dll_threads) {
2522 return CreateThread(lpThreadAttributes, dwStackSize, lpStartAddress,
2523 lpParameter, dwCreationFlags, lpThreadId);
2526 (thread_args *)GC_malloc_uncollectable(sizeof(thread_args));
2527 /* Handed off to and deallocated by child thread. */
2531 SetLastError(ERROR_NOT_ENOUGH_MEMORY);
2535 /* set up thread arguments */
2536 args -> start = lpStartAddress;
2537 args -> param = lpParameter;
2539 REACHABLE_AFTER_DIRTY(lpParameter);
2542 thread_h = CreateThread(lpThreadAttributes, dwStackSize, GC_win32_start,
2543 args, dwCreationFlags, lpThreadId);
2544 if (thread_h == 0) GC_free(args);
2549 GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread(DWORD dwExitCode)
2551 GC_unregister_my_thread();
2552 ExitThread(dwExitCode);
2555 # if !defined(CYGWIN32) && !defined(MSWINCE) && !defined(MSWIN_XBOX1) \
2557 GC_API GC_uintptr_t GC_CALL GC_beginthreadex(
2558 void *security, unsigned stack_size,
2559 unsigned (__stdcall *start_address)(void *),
2560 void *arglist, unsigned initflag,
2563 if (!EXPECT(parallel_initialized, TRUE))
2565 /* make sure GC is initialized (i.e. main thread is */
2566 /* attached, tls initialized). */
2567 # ifdef DEBUG_THREADS
2568 GC_log_printf("About to create a thread from 0x%lx\n",
2569 (long)GetCurrentThreadId());
2572 if (GC_win32_dll_threads) {
2573 return _beginthreadex(security, stack_size, start_address,
2574 arglist, initflag, thrdaddr);
2576 GC_uintptr_t thread_h;
2578 (thread_args *)GC_malloc_uncollectable(sizeof(thread_args));
2579 /* Handed off to and deallocated by child thread. */
2582 /* MSDN docs say _beginthreadex() returns 0 on error and sets */
2583 /* errno to either EAGAIN (too many threads) or EINVAL (the */
2584 /* argument is invalid or the stack size is incorrect), so we */
2585 /* set errno to EAGAIN on "not enough memory". */
2590 /* set up thread arguments */
2591 args -> start = (LPTHREAD_START_ROUTINE)start_address;
2592 args -> param = arglist;
2594 REACHABLE_AFTER_DIRTY(arglist);
2597 thread_h = _beginthreadex(security, stack_size,
2598 (unsigned (__stdcall *)(void *))GC_win32_start,
2599 args, initflag, thrdaddr);
2600 if (thread_h == 0) GC_free(args);
2605 GC_API void GC_CALL GC_endthreadex(unsigned retval)
2607 GC_unregister_my_thread();
2608 _endthreadex(retval);
2610 # endif /* !CYGWIN32 && !MSWINCE && !MSWIN_XBOX1 && !NO_CRT */
2612 #ifdef GC_WINMAIN_REDIRECT
2613 /* This might be useful on WinCE. Shouldn't be used with GC_DLL. */
2615 # if defined(MSWINCE) && defined(UNDER_CE)
2616 # define WINMAIN_LPTSTR LPWSTR
2618 # define WINMAIN_LPTSTR LPSTR
2621 /* This is defined in gc.h. */
2624 /* Defined outside GC by an application. */
2625 int WINAPI GC_WinMain(HINSTANCE, HINSTANCE, WINMAIN_LPTSTR, int);
2628 HINSTANCE hInstance;
2629 HINSTANCE hPrevInstance;
2630 WINMAIN_LPTSTR lpCmdLine;
2634 static DWORD WINAPI main_thread_start(LPVOID arg)
2636 main_thread_args * args = (main_thread_args *) arg;
2637 return (DWORD)GC_WinMain(args->hInstance, args->hPrevInstance,
2638 args->lpCmdLine, args->nShowCmd);
2641 STATIC void * GC_waitForSingleObjectInfinite(void * handle)
2643 return (void *)(word)WaitForSingleObject((HANDLE)handle, INFINITE);
2646 # ifndef WINMAIN_THREAD_STACK_SIZE
2647 # define WINMAIN_THREAD_STACK_SIZE 0 /* default value */
2650 int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
2651 WINMAIN_LPTSTR lpCmdLine, int nShowCmd)
2653 DWORD exit_code = 1;
2655 main_thread_args args = {
2656 hInstance, hPrevInstance, lpCmdLine, nShowCmd
2661 /* initialize everything */
2664 /* start the main thread */
2665 thread_h = GC_CreateThread(NULL /* lpsa */,
2666 WINMAIN_THREAD_STACK_SIZE /* ignored on WinCE */,
2667 main_thread_start, &args, 0 /* fdwCreate */,
2670 if (thread_h != NULL) {
2671 if ((DWORD)(word)GC_do_blocking(GC_waitForSingleObjectInfinite,
2672 (void *)thread_h) == WAIT_FAILED)
2673 ABORT("WaitForSingleObject(main_thread) failed");
2674 GetExitCodeThread (thread_h, &exit_code);
2675 CloseHandle (thread_h);
2677 ABORT("GC_CreateThread(main_thread) failed");
2683 return (int) exit_code;
2686 #endif /* GC_WINMAIN_REDIRECT */
2688 GC_INNER void GC_thr_init(void)
2690 struct GC_stack_base sb;
2692 GC_ASSERT(I_HOLD_LOCK());
2693 if (GC_thr_initialized) return;
2695 GC_ASSERT((word)&GC_threads % sizeof(word) == 0);
2696 # ifdef GC_NO_THREADS_DISCOVERY
2697 # define GC_main_thread GetCurrentThreadId()
2699 GC_main_thread = GetCurrentThreadId();
2701 GC_thr_initialized = TRUE;
2703 # ifdef CAN_HANDLE_FORK
2704 /* Prepare for forks if requested. */
2705 if (GC_handle_fork) {
2706 # ifdef CAN_CALL_ATFORK
2707 if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
2708 fork_child_proc) == 0) {
2709 /* Handlers successfully registered. */
2713 /* else */ if (GC_handle_fork != -1)
2714 ABORT("pthread_atfork failed");
2718 # ifdef WOW64_THREAD_CONTEXT_WORKAROUND
2719 /* Set isWow64 flag. */
2721 HMODULE hK32 = GetModuleHandle(TEXT("kernel32.dll"));
2723 FARPROC pfn = GetProcAddress(hK32, "IsWow64Process");
2725 && !(*(BOOL (WINAPI*)(HANDLE, BOOL*))pfn)(GetCurrentProcess(),
2727 isWow64 = FALSE; /* IsWow64Process failed */
2732 /* Add the initial thread, so we can stop it. */
2733 sb.mem_base = GC_stackbottom;
2734 GC_ASSERT(sb.mem_base != NULL);
2736 sb.reg_base = GC_register_stackbottom;
2739 # if defined(PARALLEL_MARK)
2741 char * markers_string = GETENV("GC_MARKERS");
2744 if (markers_string != NULL) {
2745 markers = atoi(markers_string);
2746 if (markers <= 0 || markers > MAX_MARKERS) {
2747 WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
2748 "; using maximum threads\n", (signed_word)markers);
2749 markers = MAX_MARKERS;
2753 /* There is no GetProcessAffinityMask() in WinCE. */
2754 /* GC_sysinfo is already initialized. */
2755 markers = (int)GC_sysinfo.dwNumberOfProcessors;
2758 DWORD_PTR procMask = 0;
2767 GetProcessAffinityMask(GetCurrentProcess(), &procMask, &sysMask)
2769 /* Cast args to void* for compatibility with some old SDKs. */
2770 GetProcessAffinityMask(GetCurrentProcess(),
2771 (void *)&procMask, (void *)&sysMask)
2776 } while ((procMask &= procMask - 1) != 0);
2780 # if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
2781 /* This is primarily for testing on systems without getenv(). */
2782 if (markers < GC_MIN_MARKERS)
2783 markers = GC_MIN_MARKERS;
2785 if (markers > MAX_MARKERS)
2786 markers = MAX_MARKERS; /* silently limit the value */
2788 available_markers_m1 = markers - 1;
2791 /* Check whether parallel mode could be enabled. */
2792 if (GC_win32_dll_threads || available_markers_m1 <= 0) {
2793 /* Disable parallel marking. */
2794 GC_parallel = FALSE;
2796 "Single marker thread, turning off parallel marking\n");
2798 # ifndef GC_PTHREADS_PARAMARK
2799 /* Initialize Win32 event objects for parallel marking. */
2800 mark_mutex_event = CreateEvent(NULL /* attrs */,
2801 FALSE /* isManualReset */,
2802 FALSE /* initialState */, NULL /* name */);
2803 builder_cv = CreateEvent(NULL /* attrs */,
2804 TRUE /* isManualReset */,
2805 FALSE /* initialState */, NULL /* name */);
2806 mark_cv = CreateEvent(NULL /* attrs */, TRUE /* isManualReset */,
2807 FALSE /* initialState */, NULL /* name */);
2808 if (mark_mutex_event == (HANDLE)0 || builder_cv == (HANDLE)0
2809 || mark_cv == (HANDLE)0)
2810 ABORT("CreateEvent failed");
2813 # endif /* PARALLEL_MARK */
2815 GC_ASSERT(0 == GC_lookup_thread_inner(GC_main_thread));
2816 GC_register_my_thread_inner(&sb, GC_main_thread);
2817 # undef GC_main_thread
2823 void *(*start_routine)(void *);
2828 GC_API int GC_pthread_join(pthread_t pthread_id, void **retval)
2831 # ifndef GC_WIN32_PTHREADS
2836 GC_ASSERT(!GC_win32_dll_threads);
2837 # ifdef DEBUG_THREADS
2838 GC_log_printf("thread %p(0x%lx) is joining thread %p\n",
2839 (void *)GC_PTHREAD_PTRVAL(pthread_self()),
2840 (long)GetCurrentThreadId(),
2841 (void *)GC_PTHREAD_PTRVAL(pthread_id));
2844 /* Thread being joined might not have registered itself yet. */
2845 /* After the join, thread id may have been recycled. */
2846 /* FIXME: It would be better if this worked more like */
2847 /* pthread_support.c. */
2848 # ifndef GC_WIN32_PTHREADS
2849 while ((t = GC_lookup_pthread(pthread_id)) == 0)
2852 result = pthread_join(pthread_id, retval);
2854 # ifdef GC_WIN32_PTHREADS
2855 /* pthreads-win32 and winpthreads id are unique (not recycled). */
2856 GC_thread t = GC_lookup_pthread(pthread_id);
2857 if (NULL == t) ABORT("Thread not registered");
2861 if ((t -> flags & FINISHED) != 0) {
2862 GC_delete_gc_thread_no_free(t);
2863 GC_INTERNAL_FREE(t);
2868 # ifdef DEBUG_THREADS
2869 GC_log_printf("thread %p(0x%lx) join with thread %p %s\n",
2870 (void *)GC_PTHREAD_PTRVAL(pthread_self()),
2871 (long)GetCurrentThreadId(),
2872 (void *)GC_PTHREAD_PTRVAL(pthread_id),
2873 result != 0 ? "failed" : "succeeded");
2878 /* Cygwin-pthreads calls CreateThread internally, but it's not easily */
2879 /* interceptable by us..., so intercept pthread_create instead. */
2880 GC_API int GC_pthread_create(pthread_t *new_thread,
2881 GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
2882 void *(*start_routine)(void *), void *arg)
2885 struct start_info * si;
2887 if (!EXPECT(parallel_initialized, TRUE))
2889 /* make sure GC is initialized (i.e. main thread is attached) */
2890 GC_ASSERT(!GC_win32_dll_threads);
2892 /* This is otherwise saved only in an area mmapped by the thread */
2893 /* library, which isn't visible to the collector. */
2894 si = (struct start_info *)GC_malloc_uncollectable(
2895 sizeof(struct start_info));
2899 si -> start_routine = start_routine;
2902 REACHABLE_AFTER_DIRTY(arg);
2904 pthread_attr_getdetachstate(attr, &si->detached)
2905 == PTHREAD_CREATE_DETACHED) {
2906 si->detached = TRUE;
2909 # ifdef DEBUG_THREADS
2910 GC_log_printf("About to create a thread from %p(0x%lx)\n",
2911 (void *)GC_PTHREAD_PTRVAL(pthread_self()),
2912 (long)GetCurrentThreadId());
2915 result = pthread_create(new_thread, attr, GC_pthread_start, si);
2917 if (result) { /* failure */
2923 STATIC void * GC_CALLBACK GC_pthread_start_inner(struct GC_stack_base *sb,
2926 struct start_info * si = (struct start_info *)arg;
2928 void *(*start)(void *);
2930 DWORD thread_id = GetCurrentThreadId();
2931 pthread_t pthread_id = pthread_self();
2935 # ifdef DEBUG_THREADS
2936 GC_log_printf("thread %p(0x%x) starting...\n",
2937 (void *)GC_PTHREAD_PTRVAL(pthread_id), (int)thread_id);
2940 GC_ASSERT(!GC_win32_dll_threads);
2941 /* If a GC occurs before the thread is registered, that GC will */
2942 /* ignore this thread. That's fine, since it will block trying to */
2943 /* acquire the allocation lock, and won't yet hold interesting */
2946 /* We register the thread here instead of in the parent, so that */
2947 /* we don't need to hold the allocation lock during pthread_create. */
2948 me = GC_register_my_thread_inner(sb, thread_id);
2949 SET_PTHREAD_MAP_CACHE(pthread_id, thread_id);
2950 GC_ASSERT(me != &first_thread);
2951 me -> pthread_id = pthread_id;
2952 if (si->detached) me -> flags |= DETACHED;
2955 start = si -> start_routine;
2956 start_arg = si -> arg;
2958 GC_free(si); /* was allocated uncollectible */
2960 pthread_cleanup_push(GC_thread_exit_proc, (void *)me);
2961 result = (*start)(start_arg);
2962 me -> status = result;
2964 pthread_cleanup_pop(1);
2966 # ifdef DEBUG_THREADS
2967 GC_log_printf("thread %p(0x%x) returned from start routine\n",
2968 (void *)GC_PTHREAD_PTRVAL(pthread_id), (int)thread_id);
2973 STATIC void * GC_pthread_start(void * arg)
2975 return GC_call_with_stack_base(GC_pthread_start_inner, arg);
2978 STATIC void GC_thread_exit_proc(void *arg)
2980 GC_thread me = (GC_thread)arg;
2983 GC_ASSERT(!GC_win32_dll_threads);
2984 # ifdef DEBUG_THREADS
2985 GC_log_printf("thread %p(0x%lx) called pthread_exit()\n",
2986 (void *)GC_PTHREAD_PTRVAL(pthread_self()),
2987 (long)GetCurrentThreadId());
2991 GC_wait_for_gc_completion(FALSE);
2992 # if defined(THREAD_LOCAL_ALLOC)
2993 GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
2994 GC_destroy_thread_local(&(me->tlfs));
2996 if (me -> flags & DETACHED) {
2997 GC_delete_thread(GetCurrentThreadId());
2999 /* deallocate it as part of join */
3000 me -> flags |= FINISHED;
3002 # if defined(THREAD_LOCAL_ALLOC)
3003 /* It is required to call remove_specific defined in specific.c. */
3004 GC_remove_specific(GC_thread_key);
3009 # ifndef GC_NO_PTHREAD_SIGMASK
3010 /* Win32 pthread does not support sigmask. */
3011 /* So, nothing required here... */
3012 GC_API int GC_pthread_sigmask(int how, const sigset_t *set,
3015 return pthread_sigmask(how, set, oset);
3017 # endif /* !GC_NO_PTHREAD_SIGMASK */
3019 GC_API int GC_pthread_detach(pthread_t thread)
3025 GC_ASSERT(!GC_win32_dll_threads);
3026 /* The thread might not have registered itself yet. */
3027 /* TODO: Wait for registration of the created thread in pthread_create. */
3028 while ((t = GC_lookup_pthread(thread)) == NULL)
3030 result = pthread_detach(thread);
3033 t -> flags |= DETACHED;
3034 /* Here the pthread thread id may have been recycled. */
3035 if ((t -> flags & FINISHED) != 0) {
3036 GC_delete_gc_thread_no_free(t);
3037 GC_INTERNAL_FREE(t);
3044 #elif !defined(GC_NO_THREADS_DISCOVERY)
3045 /* We avoid acquiring locks here, since this doesn't seem to be */
3046 /* preemptible. This may run with an uninitialized collector, in */
3047 /* which case we don't do much. This implies that no threads other */
3048 /* than the main one should be created with an uninitialized */
3049 /* collector. (The alternative of initializing the collector here */
3050 /* seems dangerous, since DllMain is limited in what it can do.) */
3052 # ifdef GC_INSIDE_DLL
3053 /* Export only if needed by client. */
3056 # define GC_DllMain DllMain
3058 BOOL WINAPI GC_DllMain(HINSTANCE inst GC_ATTR_UNUSED, ULONG reason,
3059 LPVOID reserved GC_ATTR_UNUSED)
3063 /* Note that GC_use_threads_discovery should be called by the */
3064 /* client application at start-up to activate automatic thread */
3065 /* registration (it is the default GC behavior); */
3066 /* to always have automatic thread registration turned on, the GC */
3067 /* should be compiled with -D GC_DISCOVER_TASK_THREADS. */
3068 if (!GC_win32_dll_threads && parallel_initialized) return TRUE;
3071 case DLL_THREAD_ATTACH:
3072 # ifdef PARALLEL_MARK
3073 /* Don't register marker threads. */
3075 /* We could reach here only if parallel_initialized == FALSE. */
3080 case DLL_PROCESS_ATTACH:
3081 /* This may run with the collector uninitialized. */
3082 thread_id = GetCurrentThreadId();
3083 if (parallel_initialized && GC_main_thread != thread_id) {
3084 # ifdef PARALLEL_MARK
3085 ABORT("Cannot initialize parallel marker from DllMain");
3087 struct GC_stack_base sb;
3088 /* Don't lock here. */
3089 # ifdef GC_ASSERTIONS
3092 GC_get_stack_base(&sb);
3093 GC_ASSERT(sb_result == GC_SUCCESS);
3094 GC_register_my_thread_inner(&sb, thread_id);
3096 } /* o.w. we already did it during GC_thr_init, called by GC_init */
3099 case DLL_THREAD_DETACH:
3100 /* We are hopefully running in the context of the exiting thread. */
3101 GC_ASSERT(parallel_initialized);
3102 if (GC_win32_dll_threads) {
3103 GC_delete_thread(GetCurrentThreadId());
3107 case DLL_PROCESS_DETACH:
3108 if (GC_win32_dll_threads) {
3110 int my_max = (int)GC_get_max_thread_index();
3112 for (i = 0; i <= my_max; ++i) {
3113 if (AO_load(&(dll_thread_table[i].tm.in_use)))
3114 GC_delete_gc_thread_no_free(&dll_thread_table[i]);
3122 #endif /* !GC_NO_THREADS_DISCOVERY && !GC_PTHREADS */
3124 /* Perform all initializations, including those that */
3125 /* may require allocation. */
3126 /* Called without allocation lock. */
3127 /* Must be called before a second thread is created. */
3128 GC_INNER void GC_init_parallel(void)
3130 # if defined(THREAD_LOCAL_ALLOC)
3135 if (parallel_initialized) return;
3136 parallel_initialized = TRUE;
3137 /* GC_init() calls us back, so set flag first. */
3139 if (!GC_is_initialized) GC_init();
3140 # if defined(CPPCHECK) && !defined(GC_NO_THREADS_DISCOVERY)
3141 GC_noop1((word)&GC_DllMain);
3143 if (GC_win32_dll_threads) {
3145 /* Cannot intercept thread creation. Hence we don't know if */
3146 /* other threads exist. However, client is not allowed to */
3147 /* create other threads before collector initialization. */
3148 /* Thus it's OK not to lock before this. */
3150 /* Initialize thread local free lists if used. */
3151 # if defined(THREAD_LOCAL_ALLOC)
3153 me = GC_lookup_thread_inner(GetCurrentThreadId());
3154 CHECK_LOOKUP_MY_THREAD(me);
3155 GC_init_thread_local(&me->tlfs);
3160 #if defined(USE_PTHREAD_LOCKS)
3161 /* Support for pthread locking code. */
3162 /* pthread_mutex_trylock may not win here, */
3163 /* due to builtin support for spinning first? */
3165 GC_INNER void GC_lock(void)
3167 pthread_mutex_lock(&GC_allocate_ml);
3169 #endif /* USE_PTHREAD_LOCKS */
3171 #if defined(THREAD_LOCAL_ALLOC)
3173 /* Add thread-local allocation support. VC++ uses __declspec(thread). */
3175 /* We must explicitly mark ptrfree and gcj free lists, since the free */
3176 /* list links wouldn't otherwise be found. We also set them in the */
3177 /* normal free lists, since that involves touching less memory than if */
3178 /* we scanned them normally. */
3179 GC_INNER void GC_mark_thread_local_free_lists(void)
3184 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
3185 for (p = GC_threads[i]; 0 != p; p = p -> tm.next) {
3186 if (!KNOWN_FINISHED(p)) {
3187 # ifdef DEBUG_THREADS
3188 GC_log_printf("Marking thread locals for 0x%x\n", (int)p -> id);
3190 GC_mark_thread_local_fls_for(&(p->tlfs));
3196 # if defined(GC_ASSERTIONS)
3197 /* Check that all thread-local free-lists are completely marked. */
3198 /* also check that thread-specific-data structures are marked. */
3199 void GC_check_tls(void)
3204 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
3205 for (p = GC_threads[i]; 0 != p; p = p -> tm.next) {
3206 if (!KNOWN_FINISHED(p))
3207 GC_check_tls_for(&(p->tlfs));
3210 # if defined(USE_CUSTOM_SPECIFIC)
3211 if (GC_thread_key != 0)
3212 GC_check_tsd_marks(GC_thread_key);
3215 # endif /* GC_ASSERTIONS */
3217 #endif /* THREAD_LOCAL_ALLOC ... */
3219 # ifndef GC_NO_THREAD_REDIRECTS
3220 /* Restore thread calls redirection. */
3221 # define CreateThread GC_CreateThread
3222 # define ExitThread GC_ExitThread
3223 # undef _beginthreadex
3224 # define _beginthreadex GC_beginthreadex
3225 # undef _endthreadex
3226 # define _endthreadex GC_endthreadex
3227 # endif /* !GC_NO_THREAD_REDIRECTS */
3229 #endif /* GC_WIN32_THREADS */