-/*
+/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
* modified is included with the above copyright notice.
*/
-#include <stdio.h>
#include "private/gc_priv.h"
-signed_word GC_mem_found = 0;
- /* Number of words of memory reclaimed */
+#ifdef ENABLE_DISCLAIM
+# include "gc_disclaim.h"
+#endif
-#if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
- word GC_fl_builder_count = 0;
- /* Number of threads currently building free lists without */
- /* holding GC lock. It is not safe to collect if this is */
- /* nonzero. */
+#include <stdio.h>
+
+GC_INNER signed_word GC_bytes_found = 0;
+ /* Number of bytes of memory reclaimed */
+ /* minus the number of bytes originally */
+ /* on free lists which we had to drop. */
+
+#if defined(PARALLEL_MARK)
+ GC_INNER signed_word GC_fl_builder_count = 0;
+ /* Number of threads currently building free lists without */
+ /* holding GC lock. It is not safe to collect if this is */
+ /* nonzero. Also, together with the mark lock, it is used as */
+ /* a semaphore during marker threads startup. */
#endif /* PARALLEL_MARK */
-static void report_leak(p, sz)
-ptr_t p;
-word sz;
-{
- if (HDR(p) -> hb_obj_kind == PTRFREE) {
- GC_err_printf0("Leaked atomic object at ");
- } else {
- GC_err_printf0("Leaked composite object at ");
- }
- GC_print_heap_obj(p);
- GC_err_printf0("\n");
-}
+/* We defer printing of leaked objects until we're done with the GC */
+/* cycle, since the routine for printing objects needs to run outside */
+/* the collector, e.g. without the allocation lock. */
+#ifndef MAX_LEAKED
+# define MAX_LEAKED 40
+#endif
+STATIC ptr_t GC_leaked[MAX_LEAKED] = { NULL };
+STATIC unsigned GC_n_leaked = 0;
-# define FOUND_FREE(hblk, word_no) \
- { \
- report_leak((ptr_t)hblk + WORDS_TO_BYTES(word_no), \
- HDR(hblk) -> hb_sz); \
- }
-
-/*
- * reclaim phase
- *
- */
+GC_INNER GC_bool GC_have_errors = FALSE;
+#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
+ STATIC void GC_reclaim_unconditionally_marked(void);
+#endif
-/*
- * Test whether a block is completely empty, i.e. contains no marked
- * objects. This does not require the block to be in physical
- * memory.
- */
-
-GC_bool GC_block_empty(hhdr)
-register hdr * hhdr;
+GC_INLINE void GC_add_leaked(ptr_t leaked)
{
- /* We treat hb_marks as an array of words here, even if it is */
- /* actually an array of bytes. Since we only check for zero, there */
- /* are no endian-ness issues. */
- register word *p = (word *)(&(hhdr -> hb_marks[0]));
- register word * plim =
- (word *)(&(hhdr -> hb_marks[MARK_BITS_SZ]));
- while (p < plim) {
- if (*p++) return(FALSE);
+# ifndef SHORT_DBG_HDRS
+ if (GC_findleak_delay_free && !GC_check_leaked(leaked))
+ return;
+# endif
+
+ GC_have_errors = TRUE;
+ if (GC_n_leaked < MAX_LEAKED) {
+ GC_leaked[GC_n_leaked++] = leaked;
+ /* Make sure it's not reclaimed this cycle */
+ GC_set_mark_bit(leaked);
}
- return(TRUE);
}
-/* The following functions sometimes return a DONT_KNOW value. */
-#define DONT_KNOW 2
-
-#ifdef SMALL_CONFIG
-# define GC_block_nearly_full1(hhdr, pat1) DONT_KNOW
-# define GC_block_nearly_full3(hhdr, pat1, pat2) DONT_KNOW
-# define GC_block_nearly_full(hhdr) DONT_KNOW
-#endif
-
-#if !defined(SMALL_CONFIG) && defined(USE_MARK_BYTES)
+/* Print all objects on the list after printing any smashed objects. */
+/* Clear both lists. Called without the allocation lock held. */
+GC_INNER void GC_print_all_errors(void)
+{
+ static GC_bool printing_errors = FALSE;
+ GC_bool have_errors;
+ unsigned i, n_leaked;
+ ptr_t leaked[MAX_LEAKED];
+ DCL_LOCK_STATE;
+
+ LOCK();
+ if (printing_errors) {
+ UNLOCK();
+ return;
+ }
+ have_errors = GC_have_errors;
+ printing_errors = TRUE;
+ n_leaked = GC_n_leaked;
+ if (n_leaked > 0) {
+ GC_ASSERT(n_leaked <= MAX_LEAKED);
+ BCOPY(GC_leaked, leaked, n_leaked * sizeof(ptr_t));
+ GC_n_leaked = 0;
+ BZERO(GC_leaked, n_leaked * sizeof(ptr_t));
+ }
+ UNLOCK();
-# define GC_block_nearly_full1(hhdr, pat1) GC_block_nearly_full(hhdr)
-# define GC_block_nearly_full3(hhdr, pat1, pat2) GC_block_nearly_full(hhdr)
+ if (GC_debugging_started) {
+ GC_print_all_smashed();
+ } else {
+ have_errors = FALSE;
+ }
-
-GC_bool GC_block_nearly_full(hhdr)
-register hdr * hhdr;
-{
- /* We again treat hb_marks as an array of words, even though it */
- /* isn't. We first sum up all the words, resulting in a word */
- /* containing 4 or 8 separate partial sums. */
- /* We then sum the bytes in the word of partial sums. */
- /* This is still endian independant. This fails if the partial */
- /* sums can overflow. */
-# if (BYTES_TO_WORDS(MARK_BITS_SZ)) >= 256
- --> potential overflow; fix the code
-# endif
- register word *p = (word *)(&(hhdr -> hb_marks[0]));
- register word * plim =
- (word *)(&(hhdr -> hb_marks[MARK_BITS_SZ]));
- word sum_vector = 0;
- unsigned sum;
- while (p < plim) {
- sum_vector += *p;
- ++p;
+ if (n_leaked > 0) {
+ GC_err_printf("Found %u leaked objects:\n", n_leaked);
+ have_errors = TRUE;
+ }
+ for (i = 0; i < n_leaked; i++) {
+ ptr_t p = leaked[i];
+# ifndef SKIP_LEAKED_OBJECTS_PRINTING
+ GC_print_heap_obj(p);
+# endif
+ GC_free(p);
}
- sum = 0;
- while (sum_vector > 0) {
- sum += sum_vector & 0xff;
- sum_vector >>= 8;
+
+ if (have_errors
+# ifndef GC_ABORT_ON_LEAK
+ && GETENV("GC_ABORT_ON_LEAK") != NULL
+# endif
+ ) {
+ ABORT("Leaked or smashed objects encountered");
}
- return (sum > BYTES_TO_WORDS(7*HBLKSIZE/8)/(hhdr -> hb_sz));
+
+ LOCK();
+ printing_errors = FALSE;
+ UNLOCK();
}
-#endif /* USE_MARK_BYTES */
-#if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES)
/*
- * Test whether nearly all of the mark words consist of the same
- * repeating pattern.
+ * reclaim phase
+ *
*/
-#define FULL_THRESHOLD (MARK_BITS_SZ/16)
-GC_bool GC_block_nearly_full1(hhdr, pat1)
-hdr *hhdr;
-word pat1;
+/* Test whether a block is completely empty, i.e. contains no marked */
+/* objects. This does not require the block to be in physical memory. */
+GC_INNER GC_bool GC_block_empty(hdr *hhdr)
{
- unsigned i;
- unsigned misses = 0;
- GC_ASSERT((MARK_BITS_SZ & 1) == 0);
- for (i = 0; i < MARK_BITS_SZ; ++i) {
- if ((hhdr -> hb_marks[i] | ~pat1) != ONES) {
- if (++misses > FULL_THRESHOLD) return FALSE;
- }
- }
- return TRUE;
+ return (hhdr -> hb_n_marks == 0);
}
-/*
- * Test whether the same repeating 3 word pattern occurs in nearly
- * all the mark bit slots.
- * This is used as a heuristic, so we're a bit sloppy and ignore
- * the last one or two words.
- */
-GC_bool GC_block_nearly_full3(hhdr, pat1, pat2, pat3)
-hdr *hhdr;
-word pat1, pat2, pat3;
+STATIC GC_bool GC_block_nearly_full(hdr *hhdr, word sz)
{
- unsigned i;
- unsigned misses = 0;
-
- if (MARK_BITS_SZ < 4) {
- return DONT_KNOW;
- }
- for (i = 0; i < MARK_BITS_SZ - 2; i += 3) {
- if ((hhdr -> hb_marks[i] | ~pat1) != ONES) {
- if (++misses > FULL_THRESHOLD) return FALSE;
- }
- if ((hhdr -> hb_marks[i+1] | ~pat2) != ONES) {
- if (++misses > FULL_THRESHOLD) return FALSE;
- }
- if ((hhdr -> hb_marks[i+2] | ~pat3) != ONES) {
- if (++misses > FULL_THRESHOLD) return FALSE;
- }
- }
- return TRUE;
+ return hhdr -> hb_n_marks > HBLK_OBJS(sz) * 7 / 8;
}
-/* Check whether a small object block is nearly full by looking at only */
-/* the mark bits. */
-/* We manually precomputed the mark bit patterns that need to be */
-/* checked for, and we give up on the ones that are unlikely to occur, */
-/* or have period > 3. */
-/* This would be a lot easier with a mark bit per object instead of per */
-/* word, but that would rewuire computing object numbers in the mark */
-/* loop, which would require different data structures ... */
-GC_bool GC_block_nearly_full(hhdr)
-hdr *hhdr;
-{
- int sz = hhdr -> hb_sz;
+/* TODO: This should perhaps again be specialized for USE_MARK_BYTES */
+/* and USE_MARK_BITS cases. */
-# if CPP_WORDSZ != 32 && CPP_WORDSZ != 64
- return DONT_KNOW; /* Shouldn't be used in any standard config. */
-# endif
-# if CPP_WORDSZ == 32
- switch(sz) {
- case 1:
- return GC_block_nearly_full1(hhdr, 0xffffffffl);
- case 2:
- return GC_block_nearly_full1(hhdr, 0x55555555l);
- case 4:
- return GC_block_nearly_full1(hhdr, 0x11111111l);
- case 6:
- return GC_block_nearly_full3(hhdr, 0x41041041l,
- 0x10410410l,
- 0x04104104l);
- case 8:
- return GC_block_nearly_full1(hhdr, 0x01010101l);
- case 12:
- return GC_block_nearly_full3(hhdr, 0x01001001l,
- 0x10010010l,
- 0x00100100l);
- case 16:
- return GC_block_nearly_full1(hhdr, 0x00010001l);
- case 32:
- return GC_block_nearly_full1(hhdr, 0x00000001l);
- default:
- return DONT_KNOW;
- }
-# endif
-# if CPP_WORDSZ == 64
- switch(sz) {
- case 1:
- return GC_block_nearly_full1(hhdr, 0xffffffffffffffffl);
- case 2:
- return GC_block_nearly_full1(hhdr, 0x5555555555555555l);
- case 4:
- return GC_block_nearly_full1(hhdr, 0x1111111111111111l);
- case 6:
- return GC_block_nearly_full3(hhdr, 0x1041041041041041l,
- 0x4104104104104104l,
- 0x0410410410410410l);
- case 8:
- return GC_block_nearly_full1(hhdr, 0x0101010101010101l);
- case 12:
- return GC_block_nearly_full3(hhdr, 0x1001001001001001l,
- 0x0100100100100100l,
- 0x0010010010010010l);
- case 16:
- return GC_block_nearly_full1(hhdr, 0x0001000100010001l);
- case 32:
- return GC_block_nearly_full1(hhdr, 0x0000000100000001l);
- default:
- return DONT_KNOW;
- }
-# endif
-}
-#endif /* !SMALL_CONFIG && !USE_MARK_BYTES */
-
-/* We keep track of reclaimed memory if we are either asked to, or */
-/* we are using the parallel marker. In the latter case, we assume */
-/* that most allocation goes through GC_malloc_many for scalability. */
-/* GC_malloc_many needs the count anyway. */
-# if defined(GATHERSTATS) || defined(PARALLEL_MARK)
-# define INCR_WORDS(sz) n_words_found += (sz)
-# define COUNT_PARAM , count
-# define COUNT_ARG , count
-# define COUNT_DECL signed_word * count;
-# define NWORDS_DECL signed_word n_words_found = 0;
-# define COUNT_UPDATE *count += n_words_found;
-# define MEM_FOUND_ADDR , &GC_mem_found
-# else
-# define INCR_WORDS(sz)
-# define COUNT_PARAM
-# define COUNT_ARG
-# define COUNT_DECL
-# define NWORDS_DECL
-# define COUNT_UPDATE
-# define MEM_FOUND_ADDR
-# endif
/*
* Restore unmarked small objects in h of size sz to the object
* free list. Returns the new list.
- * Clears unmarked objects.
+ * Clears unmarked objects. Sz is in bytes.
*/
-/*ARGSUSED*/
-ptr_t GC_reclaim_clear(hbp, hhdr, sz, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-register hdr * hhdr;
-register ptr_t list;
-register word sz;
-COUNT_DECL
+STATIC ptr_t GC_reclaim_clear(struct hblk *hbp, hdr *hhdr, word sz,
+ ptr_t list, signed_word *count)
{
- register int word_no;
- register word *p, *q, *plim;
- NWORDS_DECL
-
+ word bit_no = 0;
+ word *p, *q, *plim;
+ signed_word n_bytes_found = 0;
+
GC_ASSERT(hhdr == GC_find_header((ptr_t)hbp));
+# ifndef THREADS
+ GC_ASSERT(sz == hhdr -> hb_sz);
+# else
+ /* Skip the assertion because of a potential race with GC_realloc. */
+# endif
+ GC_ASSERT((sz & (BYTES_PER_WORD-1)) == 0);
p = (word *)(hbp->hb_body);
- word_no = 0;
- plim = (word *)((((word)hbp) + HBLKSIZE)
- - WORDS_TO_BYTES(sz));
+ plim = (word *)(hbp->hb_body + HBLKSIZE - sz);
/* go through all words in block */
- while( p <= plim ) {
- if( mark_bit_from_hdr(hhdr, word_no) ) {
- p += sz;
- } else {
- INCR_WORDS(sz);
- /* object is available - put on list */
- obj_link(p) = list;
- list = ((ptr_t)p);
- /* Clear object, advance p to next object in the process */
- q = p + sz;
-# ifdef USE_MARK_BYTES
- GC_ASSERT(!(sz & 1)
- && !((word)p & (2 * sizeof(word) - 1)));
- p[1] = 0;
+ while ((word)p <= (word)plim) {
+ if (mark_bit_from_hdr(hhdr, bit_no)) {
+ p = (word *)((ptr_t)p + sz);
+ } else {
+ n_bytes_found += sz;
+ /* object is available - put on list */
+ obj_link(p) = list;
+ list = ((ptr_t)p);
+ /* Clear object, advance p to next object in the process */
+ q = (word *)((ptr_t)p + sz);
+# ifdef USE_MARK_BYTES
+ GC_ASSERT(!(sz & 1)
+ && !((word)p & (2 * sizeof(word) - 1)));
+ p[1] = 0;
p += 2;
- while (p < q) {
- CLEAR_DOUBLE(p);
- p += 2;
- }
-# else
+ while ((word)p < (word)q) {
+ CLEAR_DOUBLE(p);
+ p += 2;
+ }
+# else
p++; /* Skip link field */
- while (p < q) {
- *p++ = 0;
- }
-# endif
- }
- word_no += sz;
- }
- COUNT_UPDATE
- return(list);
-}
-
-#if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES)
-
-/*
- * A special case for 2 word composite objects (e.g. cons cells):
- */
-/*ARGSUSED*/
-ptr_t GC_reclaim_clear2(hbp, hhdr, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-hdr * hhdr;
-register ptr_t list;
-COUNT_DECL
-{
- register word * mark_word_addr = &(hhdr->hb_marks[0]);
- register word *p, *plim;
- register word mark_word;
- register int i;
- NWORDS_DECL
-# define DO_OBJ(start_displ) \
- if (!(mark_word & ((word)1 << start_displ))) { \
- p[start_displ] = (word)list; \
- list = (ptr_t)(p+start_displ); \
- p[start_displ+1] = 0; \
- INCR_WORDS(2); \
- }
-
- p = (word *)(hbp->hb_body);
- plim = (word *)(((word)hbp) + HBLKSIZE);
-
- /* go through all words in block */
- while( p < plim ) {
- mark_word = *mark_word_addr++;
- for (i = 0; i < WORDSZ; i += 8) {
- DO_OBJ(0);
- DO_OBJ(2);
- DO_OBJ(4);
- DO_OBJ(6);
- p += 8;
- mark_word >>= 8;
- }
- }
- COUNT_UPDATE
- return(list);
-# undef DO_OBJ
-}
-
-/*
- * Another special case for 4 word composite objects:
- */
-/*ARGSUSED*/
-ptr_t GC_reclaim_clear4(hbp, hhdr, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-hdr * hhdr;
-register ptr_t list;
-COUNT_DECL
-{
- register word * mark_word_addr = &(hhdr->hb_marks[0]);
- register word *p, *plim;
- register word mark_word;
- NWORDS_DECL
-# define DO_OBJ(start_displ) \
- if (!(mark_word & ((word)1 << start_displ))) { \
- p[start_displ] = (word)list; \
- list = (ptr_t)(p+start_displ); \
- p[start_displ+1] = 0; \
- CLEAR_DOUBLE(p + start_displ + 2); \
- INCR_WORDS(4); \
- }
-
- p = (word *)(hbp->hb_body);
- plim = (word *)(((word)hbp) + HBLKSIZE);
-
- /* go through all words in block */
- while( p < plim ) {
- mark_word = *mark_word_addr++;
- DO_OBJ(0);
- DO_OBJ(4);
- DO_OBJ(8);
- DO_OBJ(12);
- DO_OBJ(16);
- DO_OBJ(20);
- DO_OBJ(24);
- DO_OBJ(28);
-# if CPP_WORDSZ == 64
- DO_OBJ(32);
- DO_OBJ(36);
- DO_OBJ(40);
- DO_OBJ(44);
- DO_OBJ(48);
- DO_OBJ(52);
- DO_OBJ(56);
- DO_OBJ(60);
-# endif
- p += WORDSZ;
- }
- COUNT_UPDATE
+ while ((word)p < (word)q) {
+ *p++ = 0;
+ }
+# endif
+ }
+ bit_no += MARK_BIT_OFFSET(sz);
+ }
+ *count += n_bytes_found;
return(list);
-# undef DO_OBJ
}
-#endif /* !SMALL_CONFIG && !USE_MARK_BYTES */
-
/* The same thing, but don't clear objects: */
-/*ARGSUSED*/
-ptr_t GC_reclaim_uninit(hbp, hhdr, sz, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-register hdr * hhdr;
-register ptr_t list;
-register word sz;
-COUNT_DECL
+STATIC ptr_t GC_reclaim_uninit(struct hblk *hbp, hdr *hhdr, word sz,
+ ptr_t list, signed_word *count)
{
- register int word_no = 0;
- register word *p, *plim;
- NWORDS_DECL
-
- p = (word *)(hbp->hb_body);
- plim = (word *)((((word)hbp) + HBLKSIZE)
- - WORDS_TO_BYTES(sz));
+ word bit_no = 0;
+ word *p, *plim;
+ signed_word n_bytes_found = 0;
- /* go through all words in block */
- while( p <= plim ) {
- if( !mark_bit_from_hdr(hhdr, word_no) ) {
- INCR_WORDS(sz);
- /* object is available - put on list */
- obj_link(p) = list;
- list = ((ptr_t)p);
- }
- p += sz;
- word_no += sz;
- }
- COUNT_UPDATE
- return(list);
-}
-
-/* Don't really reclaim objects, just check for unmarked ones: */
-/*ARGSUSED*/
-void GC_reclaim_check(hbp, hhdr, sz)
-register struct hblk *hbp; /* ptr to current heap block */
-register hdr * hhdr;
-register word sz;
-{
- register int word_no = 0;
- register word *p, *plim;
-# ifdef GATHERSTATS
- register int n_words_found = 0;
+# ifndef THREADS
+ GC_ASSERT(sz == hhdr -> hb_sz);
# endif
-
- p = (word *)(hbp->hb_body);
- plim = (word *)((((word)hbp) + HBLKSIZE)
- - WORDS_TO_BYTES(sz));
-
- /* go through all words in block */
- while( p <= plim ) {
- if( !mark_bit_from_hdr(hhdr, word_no) ) {
- FOUND_FREE(hbp, word_no);
- }
- p += sz;
- word_no += sz;
- }
-}
-
-#if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES)
-/*
- * Another special case for 2 word atomic objects:
- */
-/*ARGSUSED*/
-ptr_t GC_reclaim_uninit2(hbp, hhdr, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-hdr * hhdr;
-register ptr_t list;
-COUNT_DECL
-{
- register word * mark_word_addr = &(hhdr->hb_marks[0]);
- register word *p, *plim;
- register word mark_word;
- register int i;
- NWORDS_DECL
-# define DO_OBJ(start_displ) \
- if (!(mark_word & ((word)1 << start_displ))) { \
- p[start_displ] = (word)list; \
- list = (ptr_t)(p+start_displ); \
- INCR_WORDS(2); \
- }
-
p = (word *)(hbp->hb_body);
- plim = (word *)(((word)hbp) + HBLKSIZE);
+ plim = (word *)((ptr_t)hbp + HBLKSIZE - sz);
/* go through all words in block */
- while( p < plim ) {
- mark_word = *mark_word_addr++;
- for (i = 0; i < WORDSZ; i += 8) {
- DO_OBJ(0);
- DO_OBJ(2);
- DO_OBJ(4);
- DO_OBJ(6);
- p += 8;
- mark_word >>= 8;
- }
- }
- COUNT_UPDATE
+ while ((word)p <= (word)plim) {
+ if (!mark_bit_from_hdr(hhdr, bit_no)) {
+ n_bytes_found += sz;
+ /* object is available - put on list */
+ obj_link(p) = list;
+ list = ((ptr_t)p);
+ }
+ p = (word *)((ptr_t)p + sz);
+ bit_no += MARK_BIT_OFFSET(sz);
+ }
+ *count += n_bytes_found;
return(list);
-# undef DO_OBJ
}
-/*
- * Another special case for 4 word atomic objects:
- */
-/*ARGSUSED*/
-ptr_t GC_reclaim_uninit4(hbp, hhdr, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-hdr * hhdr;
-register ptr_t list;
-COUNT_DECL
-{
- register word * mark_word_addr = &(hhdr->hb_marks[0]);
- register word *p, *plim;
- register word mark_word;
- NWORDS_DECL
-# define DO_OBJ(start_displ) \
- if (!(mark_word & ((word)1 << start_displ))) { \
- p[start_displ] = (word)list; \
- list = (ptr_t)(p+start_displ); \
- INCR_WORDS(4); \
- }
-
- p = (word *)(hbp->hb_body);
- plim = (word *)(((word)hbp) + HBLKSIZE);
-
- /* go through all words in block */
- while( p < plim ) {
- mark_word = *mark_word_addr++;
- DO_OBJ(0);
- DO_OBJ(4);
- DO_OBJ(8);
- DO_OBJ(12);
- DO_OBJ(16);
- DO_OBJ(20);
- DO_OBJ(24);
- DO_OBJ(28);
-# if CPP_WORDSZ == 64
- DO_OBJ(32);
- DO_OBJ(36);
- DO_OBJ(40);
- DO_OBJ(44);
- DO_OBJ(48);
- DO_OBJ(52);
- DO_OBJ(56);
- DO_OBJ(60);
-# endif
- p += WORDSZ;
- }
- COUNT_UPDATE
- return(list);
-# undef DO_OBJ
-}
+#ifdef ENABLE_DISCLAIM
+ /* Call reclaim notifier for block's kind on each unmarked object in */
+ /* block, all within a pair of corresponding enter/leave callbacks. */
+ STATIC ptr_t GC_disclaim_and_reclaim(struct hblk *hbp, hdr *hhdr, word sz,
+ ptr_t list, signed_word *count)
+ {
+ word bit_no = 0;
+ word *p, *q, *plim;
+ signed_word n_bytes_found = 0;
+ struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind];
+ int (GC_CALLBACK *disclaim)(void *) = ok->ok_disclaim_proc;
+
+# ifndef THREADS
+ GC_ASSERT(sz == hhdr -> hb_sz);
+# endif
+ p = (word *)(hbp -> hb_body);
+ plim = (word *)((ptr_t)p + HBLKSIZE - sz);
+
+ while ((word)p <= (word)plim) {
+ int marked = mark_bit_from_hdr(hhdr, bit_no);
+ if (!marked && (*disclaim)(p)) {
+ set_mark_bit_from_hdr(hhdr, bit_no);
+ hhdr -> hb_n_marks++;
+ marked = 1;
+ }
+ if (marked)
+ p = (word *)((ptr_t)p + sz);
+ else {
+ n_bytes_found += sz;
+ /* object is available - put on list */
+ obj_link(p) = list;
+ list = ((ptr_t)p);
+ /* Clear object, advance p to next object in the process */
+ q = (word *)((ptr_t)p + sz);
+# ifdef USE_MARK_BYTES
+ GC_ASSERT((sz & 1) == 0);
+ GC_ASSERT(((word)p & (2 * sizeof(word) - 1)) == 0);
+ p[1] = 0;
+ p += 2;
+ while ((word)p < (word)q) {
+ CLEAR_DOUBLE(p);
+ p += 2;
+ }
+# else
+ p++; /* Skip link field */
+ while ((word)p < (word)q) {
+ *p++ = 0;
+ }
+# endif
+ }
+ bit_no += MARK_BIT_OFFSET(sz);
+ }
+ *count += n_bytes_found;
+ return list;
+ }
+#endif /* ENABLE_DISCLAIM */
-/* Finally the one word case, which never requires any clearing: */
-/*ARGSUSED*/
-ptr_t GC_reclaim1(hbp, hhdr, list COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-hdr * hhdr;
-register ptr_t list;
-COUNT_DECL
+/* Don't really reclaim objects, just check for unmarked ones: */
+STATIC void GC_reclaim_check(struct hblk *hbp, hdr *hhdr, word sz)
{
- register word * mark_word_addr = &(hhdr->hb_marks[0]);
- register word *p, *plim;
- register word mark_word;
- register int i;
- NWORDS_DECL
-# define DO_OBJ(start_displ) \
- if (!(mark_word & ((word)1 << start_displ))) { \
- p[start_displ] = (word)list; \
- list = (ptr_t)(p+start_displ); \
- INCR_WORDS(1); \
- }
-
- p = (word *)(hbp->hb_body);
- plim = (word *)(((word)hbp) + HBLKSIZE);
+ word bit_no;
+ ptr_t p, plim;
+# ifndef THREADS
+ GC_ASSERT(sz == hhdr -> hb_sz);
+# endif
/* go through all words in block */
- while( p < plim ) {
- mark_word = *mark_word_addr++;
- for (i = 0; i < WORDSZ; i += 4) {
- DO_OBJ(0);
- DO_OBJ(1);
- DO_OBJ(2);
- DO_OBJ(3);
- p += 4;
- mark_word >>= 4;
- }
- }
- COUNT_UPDATE
- return(list);
-# undef DO_OBJ
+ p = hbp->hb_body;
+ plim = p + HBLKSIZE - sz;
+ for (bit_no = 0; (word)p <= (word)plim;
+ p += sz, bit_no += MARK_BIT_OFFSET(sz)) {
+ if (!mark_bit_from_hdr(hhdr, bit_no)) {
+ GC_add_leaked(p);
+ }
+ }
}
-#endif /* !SMALL_CONFIG && !USE_MARK_BYTES */
+/* Is a pointer-free block? Same as IS_PTRFREE macro (in os_dep.c) but */
+/* uses unordered atomic access to avoid racing with GC_realloc. */
+#ifdef AO_HAVE_load
+# define IS_PTRFREE_SAFE(hhdr) \
+ (AO_load((volatile AO_t *)&(hhdr)->hb_descr) == 0)
+#else
+ /* No race as GC_realloc holds the lock while updating hb_descr. */
+# define IS_PTRFREE_SAFE(hhdr) ((hhdr)->hb_descr == 0)
+#endif
/*
* Generic procedure to rebuild a free list in hbp.
* Also called directly from GC_malloc_many.
+ * Sz is now in bytes.
*/
-ptr_t GC_reclaim_generic(hbp, hhdr, sz, init, list COUNT_PARAM)
-struct hblk *hbp; /* ptr to current heap block */
-hdr * hhdr;
-GC_bool init;
-ptr_t list;
-word sz;
-COUNT_DECL
+GC_INNER ptr_t GC_reclaim_generic(struct hblk * hbp, hdr *hhdr, size_t sz,
+ GC_bool init, ptr_t list,
+ signed_word *count)
{
- ptr_t result = list;
+ ptr_t result;
GC_ASSERT(GC_find_header((ptr_t)hbp) == hhdr);
- if (init) {
- switch(sz) {
-# if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES)
- case 1:
- /* We now issue the hint even if GC_nearly_full returned */
- /* DONT_KNOW. */
- GC_write_hint(hbp);
- result = GC_reclaim1(hbp, hhdr, list COUNT_ARG);
- break;
- case 2:
- GC_write_hint(hbp);
- result = GC_reclaim_clear2(hbp, hhdr, list COUNT_ARG);
- break;
- case 4:
- GC_write_hint(hbp);
- result = GC_reclaim_clear4(hbp, hhdr, list COUNT_ARG);
- break;
-# endif /* !SMALL_CONFIG && !USE_MARK_BYTES */
- default:
- GC_write_hint(hbp);
- result = GC_reclaim_clear(hbp, hhdr, sz, list COUNT_ARG);
- break;
- }
+# ifndef GC_DISABLE_INCREMENTAL
+ GC_remove_protection(hbp, 1, IS_PTRFREE_SAFE(hhdr));
+# endif
+# ifdef ENABLE_DISCLAIM
+ if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
+ result = GC_disclaim_and_reclaim(hbp, hhdr, sz, list, count);
+ } else
+# endif
+ /* else */ if (init || GC_debugging_started) {
+ result = GC_reclaim_clear(hbp, hhdr, sz, list, count);
} else {
- switch(sz) {
-# if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES)
- case 1:
- GC_write_hint(hbp);
- result = GC_reclaim1(hbp, hhdr, list COUNT_ARG);
- break;
- case 2:
- GC_write_hint(hbp);
- result = GC_reclaim_uninit2(hbp, hhdr, list COUNT_ARG);
- break;
- case 4:
- GC_write_hint(hbp);
- result = GC_reclaim_uninit4(hbp, hhdr, list COUNT_ARG);
- break;
-# endif /* !SMALL_CONFIG && !USE_MARK_BYTES */
- default:
- GC_write_hint(hbp);
- result = GC_reclaim_uninit(hbp, hhdr, sz, list COUNT_ARG);
- break;
- }
- }
+ GC_ASSERT(IS_PTRFREE_SAFE(hhdr));
+ result = GC_reclaim_uninit(hbp, hhdr, sz, list, count);
+ }
if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) GC_set_hdr_marks(hhdr);
return result;
}
* If entirely empty blocks are to be completely deallocated, then
* caller should perform that check.
*/
-void GC_reclaim_small_nonempty_block(hbp, report_if_found COUNT_PARAM)
-register struct hblk *hbp; /* ptr to current heap block */
-int report_if_found; /* Abort if a reclaimable object is found */
-COUNT_DECL
+STATIC void GC_reclaim_small_nonempty_block(struct hblk *hbp, word sz,
+ GC_bool report_if_found)
{
hdr *hhdr = HDR(hbp);
- word sz = hhdr -> hb_sz;
- int kind = hhdr -> hb_obj_kind;
- struct obj_kind * ok = &GC_obj_kinds[kind];
- ptr_t * flh = &(ok -> ok_freelist[sz]);
-
+ struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
+ void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);
+
hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
if (report_if_found) {
- GC_reclaim_check(hbp, hhdr, sz);
+ GC_reclaim_check(hbp, hhdr, sz);
} else {
- *flh = GC_reclaim_generic(hbp, hhdr, sz,
- (ok -> ok_init || GC_debugging_started),
- *flh MEM_FOUND_ADDR);
+ *flh = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init,
+ (ptr_t)(*flh), &GC_bytes_found);
}
}
+#ifdef ENABLE_DISCLAIM
+ STATIC void GC_disclaim_and_reclaim_or_free_small_block(struct hblk *hbp)
+ {
+ hdr *hhdr = HDR(hbp);
+ word sz = hhdr -> hb_sz;
+ struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
+ void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);
+ void *flh_next;
+
+ hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
+ flh_next = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init,
+ (ptr_t)(*flh), &GC_bytes_found);
+ if (hhdr -> hb_n_marks)
+ *flh = flh_next;
+ else {
+ GC_bytes_found += HBLKSIZE;
+ GC_freehblk(hbp);
+ }
+ }
+#endif /* ENABLE_DISCLAIM */
+
/*
* Restore an unmarked large object or an entirely empty blocks of small objects
* to the heap block free list.
* If report_if_found is TRUE, then process any block immediately, and
* simply report free objects; do not actually reclaim them.
*/
-# if defined(__STDC__) || defined(__cplusplus)
- void GC_reclaim_block(register struct hblk *hbp, word report_if_found)
-# else
- void GC_reclaim_block(hbp, report_if_found)
- register struct hblk *hbp; /* ptr to current heap block */
- word report_if_found; /* Abort if a reclaimable object is found */
-# endif
+STATIC void GC_reclaim_block(struct hblk *hbp, word report_if_found)
{
- register hdr * hhdr;
- register word sz; /* size of objects in current block */
- register struct obj_kind * ok;
- struct hblk ** rlh;
-
- hhdr = HDR(hbp);
- sz = hhdr -> hb_sz;
- ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
-
- if( sz > MAXOBJSZ ) { /* 1 big object */
+ hdr * hhdr = HDR(hbp);
+ word sz; /* size of objects in current block */
+ struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
+
+# ifdef AO_HAVE_load
+ /* Atomic access is used to avoid racing with GC_realloc. */
+ sz = (word)AO_load((volatile AO_t *)&hhdr->hb_sz);
+# else
+ /* No race as GC_realloc holds the lock while updating hb_sz. */
+ sz = hhdr -> hb_sz;
+# endif
+ if( sz > MAXOBJBYTES ) { /* 1 big object */
if( !mark_bit_from_hdr(hhdr, 0) ) {
- if (report_if_found) {
- FOUND_FREE(hbp, 0);
- } else {
- word blocks = OBJ_SZ_TO_BLOCKS(sz);
- if (blocks > 1) {
- GC_large_allocd_bytes -= blocks * HBLKSIZE;
- }
-# ifdef GATHERSTATS
- GC_mem_found += sz;
-# endif
- GC_freehblk(hbp);
- }
- }
+ if (report_if_found) {
+ GC_add_leaked((ptr_t)hbp);
+ } else {
+ word blocks;
+
+# ifdef ENABLE_DISCLAIM
+ if (EXPECT(hhdr->hb_flags & HAS_DISCLAIM, 0)) {
+ if ((*ok->ok_disclaim_proc)(hbp)) {
+ /* Not disclaimed => resurrect the object. */
+ set_mark_bit_from_hdr(hhdr, 0);
+ goto in_use;
+ }
+ }
+# endif
+ blocks = OBJ_SZ_TO_BLOCKS(sz);
+ if (blocks > 1) {
+ GC_large_allocd_bytes -= blocks * HBLKSIZE;
+ }
+ GC_bytes_found += sz;
+ GC_freehblk(hbp);
+ }
+ } else {
+# ifdef ENABLE_DISCLAIM
+ in_use:
+# endif
+ if (IS_PTRFREE_SAFE(hhdr)) {
+ GC_atomic_in_use += sz;
+ } else {
+ GC_composite_in_use += sz;
+ }
+ }
} else {
GC_bool empty = GC_block_empty(hhdr);
+# ifdef PARALLEL_MARK
+ /* Count can be low or one too high because we sometimes */
+ /* have to ignore decrements. Objects can also potentially */
+ /* be repeatedly marked by each marker. */
+ /* Here we assume two markers, but this is extremely */
+ /* unlikely to fail spuriously with more. And if it does, it */
+ /* should be looked at. */
+ GC_ASSERT(hhdr -> hb_n_marks <= 2 * (HBLKSIZE/sz + 1) + 16);
+# else
+ GC_ASSERT(sz * hhdr -> hb_n_marks <= HBLKSIZE);
+# endif
if (report_if_found) {
- GC_reclaim_small_nonempty_block(hbp, (int)report_if_found
- MEM_FOUND_ADDR);
+ GC_reclaim_small_nonempty_block(hbp, sz,
+ TRUE /* report_if_found */);
} else if (empty) {
-# ifdef GATHERSTATS
- GC_mem_found += BYTES_TO_WORDS(HBLKSIZE);
-# endif
- GC_freehblk(hbp);
- } else if (TRUE != GC_block_nearly_full(hhdr)){
+# ifdef ENABLE_DISCLAIM
+ if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
+ GC_disclaim_and_reclaim_or_free_small_block(hbp);
+ } else
+# endif
+ /* else */ {
+ GC_bytes_found += HBLKSIZE;
+ GC_freehblk(hbp);
+ }
+ } else if (GC_find_leak || !GC_block_nearly_full(hhdr, sz)) {
/* group of smaller objects, enqueue the real work */
- rlh = &(ok -> ok_reclaim_list[sz]);
- hhdr -> hb_next = *rlh;
- *rlh = hbp;
+ struct hblk **rlh = ok -> ok_reclaim_list;
+
+ if (rlh != NULL) {
+ rlh += BYTES_TO_GRANULES(sz);
+ hhdr -> hb_next = *rlh;
+ *rlh = hbp;
+ }
} /* else not worth salvaging. */
- /* We used to do the nearly_full check later, but we */
- /* already have the right cache context here. Also */
- /* doing it here avoids some silly lock contention in */
- /* GC_malloc_many. */
+ /* We used to do the nearly_full check later, but we */
+ /* already have the right cache context here. Also */
+ /* doing it here avoids some silly lock contention in */
+ /* GC_malloc_many. */
+ if (IS_PTRFREE_SAFE(hhdr)) {
+ GC_atomic_in_use += sz * hhdr -> hb_n_marks;
+ } else {
+ GC_composite_in_use += sz * hhdr -> hb_n_marks;
+ }
}
}
#if !defined(NO_DEBUGGING)
-/* Routines to gather and print heap block info */
-/* intended for debugging. Otherwise should be called */
-/* with lock. */
+/* Routines to gather and print heap block info */
+/* intended for debugging. Otherwise should be called */
+/* with lock. */
struct Print_stats
{
- size_t number_of_blocks;
- size_t total_bytes;
+ size_t number_of_blocks;
+ size_t total_bytes;
};
#ifdef USE_MARK_BYTES
-/* Return the number of set mark bits in the given header */
-int GC_n_set_marks(hhdr)
-hdr * hhdr;
+/* Return the number of set mark bits in the given header. */
+/* Remains externally visible as used by GNU GCJ currently. */
+unsigned GC_n_set_marks(hdr *hhdr)
{
- register int result = 0;
- register int i;
-
- for (i = 0; i < MARK_BITS_SZ; i++) {
+ unsigned result = 0;
+ word i;
+ word sz = hhdr -> hb_sz;
+ word offset = MARK_BIT_OFFSET(sz);
+ word limit = FINAL_MARK_BIT(sz);
+
+ for (i = 0; i < limit; i += offset) {
result += hhdr -> hb_marks[i];
}
+ GC_ASSERT(hhdr -> hb_marks[limit]);
return(result);
}
#else
-/* Number of set bits in a word. Not performance critical. */
-static int set_bits(n)
-word n;
+/* Number of set bits in a word. Not performance critical. */
+static unsigned set_bits(word n)
{
- register word m = n;
- register int result = 0;
-
+ word m = n;
+ unsigned result = 0;
+
while (m > 0) {
- if (m & 1) result++;
- m >>= 1;
+ if (m & 1) result++;
+ m >>= 1;
}
return(result);
}
-/* Return the number of set mark bits in the given header */
-int GC_n_set_marks(hhdr)
-hdr * hhdr;
+unsigned GC_n_set_marks(hdr *hhdr)
{
- register int result = 0;
- register int i;
-
- for (i = 0; i < MARK_BITS_SZ; i++) {
+ unsigned result = 0;
+ word i;
+ word n_mark_words;
+# ifdef MARK_BIT_PER_OBJ
+ word n_objs = HBLK_OBJS(hhdr -> hb_sz);
+
+ if (0 == n_objs) n_objs = 1;
+ n_mark_words = divWORDSZ(n_objs + WORDSZ - 1);
+# else /* MARK_BIT_PER_GRANULE */
+ n_mark_words = MARK_BITS_SZ;
+# endif
+ for (i = 0; i < n_mark_words - 1; i++) {
result += set_bits(hhdr -> hb_marks[i]);
}
- return(result);
+# ifdef MARK_BIT_PER_OBJ
+ result += set_bits((hhdr -> hb_marks[n_mark_words - 1])
+ << (n_mark_words * WORDSZ - n_objs));
+# else
+ result += set_bits(hhdr -> hb_marks[n_mark_words - 1]);
+# endif
+ return result; /* the number of set bits excluding the one past the end */
}
#endif /* !USE_MARK_BYTES */
-/*ARGSUSED*/
-# if defined(__STDC__) || defined(__cplusplus)
- void GC_print_block_descr(struct hblk *h, word dummy)
-# else
- void GC_print_block_descr(h, dummy)
- struct hblk *h;
- word dummy;
-# endif
+STATIC void GC_print_block_descr(struct hblk *h,
+ word /* struct PrintStats */ raw_ps)
{
- register hdr * hhdr = HDR(h);
- register size_t bytes = WORDS_TO_BYTES(hhdr -> hb_sz);
+ hdr * hhdr = HDR(h);
+ size_t bytes = hhdr -> hb_sz;
struct Print_stats *ps;
-
- GC_printf3("(%lu:%lu,%lu)", (unsigned long)(hhdr -> hb_obj_kind),
- (unsigned long)bytes,
- (unsigned long)(GC_n_set_marks(hhdr)));
- bytes += HBLKSIZE-1;
- bytes &= ~(HBLKSIZE-1);
-
- ps = (struct Print_stats *)dummy;
- ps->total_bytes += bytes;
+ unsigned n_marks = GC_n_set_marks(hhdr);
+ unsigned n_objs = (unsigned)HBLK_OBJS(bytes);
+
+ if (0 == n_objs) n_objs = 1;
+ if (hhdr -> hb_n_marks != n_marks) {
+ GC_printf("%u,%u,%u!=%u,%u\n", hhdr->hb_obj_kind, (unsigned)bytes,
+ (unsigned)hhdr->hb_n_marks, n_marks, n_objs);
+ } else {
+ GC_printf("%u,%u,%u,%u\n", hhdr->hb_obj_kind, (unsigned)bytes,
+ n_marks, n_objs);
+ }
+
+ ps = (struct Print_stats *)raw_ps;
+ ps->total_bytes += (bytes + (HBLKSIZE-1)) & ~(HBLKSIZE-1); /* round up */
ps->number_of_blocks++;
}
-void GC_print_block_list()
+void GC_print_block_list(void)
{
struct Print_stats pstats;
- GC_printf0("(kind(0=ptrfree,1=normal,2=unc.,3=stubborn):size_in_bytes, #_marks_set)\n");
+ GC_printf("kind(0=ptrfree,1=normal,2=unc.),"
+ "size_in_bytes,#_marks_set,#objs\n");
pstats.number_of_blocks = 0;
pstats.total_bytes = 0;
GC_apply_to_all_blocks(GC_print_block_descr, (word)&pstats);
- GC_printf2("\nblocks = %lu, bytes = %lu\n",
- (unsigned long)pstats.number_of_blocks,
- (unsigned long)pstats.total_bytes);
+ GC_printf("blocks= %lu, bytes= %lu\n",
+ (unsigned long)pstats.number_of_blocks,
+ (unsigned long)pstats.total_bytes);
}
-#endif /* NO_DEBUGGING */
+#include "gc_inline.h" /* for GC_print_free_list prototype */
+
+/* Currently for debugger use only: */
+GC_API void GC_CALL GC_print_free_list(int kind, size_t sz_in_granules)
+{
+ void *flh_next;
+ int n;
+
+ GC_ASSERT(kind < MAXOBJKINDS);
+ GC_ASSERT(sz_in_granules <= MAXOBJGRANULES);
+ flh_next = GC_obj_kinds[kind].ok_freelist[sz_in_granules];
+ for (n = 0; flh_next; n++) {
+ GC_printf("Free object in heap block %p [%d]: %p\n",
+ (void *)HBLKPTR(flh_next), n, flh_next);
+ flh_next = obj_link(flh_next);
+ }
+}
+
+#endif /* !NO_DEBUGGING */
+
+/*
+ * Clear all obj_link pointers in the list of free objects *flp.
+ * Clear *flp.
+ * This must be done before dropping a list of free gcj-style objects,
+ * since may otherwise end up with dangling "descriptor" pointers.
+ * It may help for other pointer-containing objects.
+ */
+STATIC void GC_clear_fl_links(void **flp)
+{
+ void *next = *flp;
+
+ while (0 != next) {
+ *flp = 0;
+ flp = &(obj_link(next));
+ next = *flp;
+ }
+}
/*
* Perform GC_reclaim_block on the entire heap, after first clearing
* small object free lists (if we are not just looking for leaks).
*/
-void GC_start_reclaim(report_if_found)
-int report_if_found; /* Abort if a GC_reclaimable object is found */
+GC_INNER void GC_start_reclaim(GC_bool report_if_found)
{
- int kind;
-
-# if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
+ unsigned kind;
+
+# if defined(PARALLEL_MARK)
GC_ASSERT(0 == GC_fl_builder_count);
# endif
+ /* Reset in use counters. GC_reclaim_block recomputes them. */
+ GC_composite_in_use = 0;
+ GC_atomic_in_use = 0;
/* Clear reclaim- and free-lists */
for (kind = 0; kind < GC_n_kinds; kind++) {
- register ptr_t *fop;
- register ptr_t *lim;
- register struct hblk ** rlp;
- register struct hblk ** rlim;
- register struct hblk ** rlist = GC_obj_kinds[kind].ok_reclaim_list;
-
- if (rlist == 0) continue; /* This kind not used. */
+ struct hblk ** rlist = GC_obj_kinds[kind].ok_reclaim_list;
+ GC_bool should_clobber = (GC_obj_kinds[kind].ok_descriptor != 0);
+
+ if (rlist == 0) continue; /* This kind not used. */
if (!report_if_found) {
- lim = &(GC_obj_kinds[kind].ok_freelist[MAXOBJSZ+1]);
- for( fop = GC_obj_kinds[kind].ok_freelist; fop < lim; fop++ ) {
- *fop = 0;
- }
- } /* otherwise free list objects are marked, */
- /* and its safe to leave them */
- rlim = rlist + MAXOBJSZ+1;
- for( rlp = rlist; rlp < rlim; rlp++ ) {
- *rlp = 0;
- }
+ void **fop;
+ void **lim = &(GC_obj_kinds[kind].ok_freelist[MAXOBJGRANULES+1]);
+
+ for (fop = GC_obj_kinds[kind].ok_freelist;
+ (word)fop < (word)lim; (*(word **)&fop)++) {
+ if (*fop != 0) {
+ if (should_clobber) {
+ GC_clear_fl_links(fop);
+ } else {
+ *fop = 0;
+ }
+ }
+ }
+ } /* otherwise free list objects are marked, */
+ /* and its safe to leave them */
+ BZERO(rlist, (MAXOBJGRANULES + 1) * sizeof(void *));
}
-
-# ifdef PRINTBLOCKS
- GC_printf0("GC_reclaim: current block sizes:\n");
- GC_print_block_list();
-# endif
+
/* Go through all heap blocks (in hblklist) and reclaim unmarked objects */
- /* or enqueue the block for later processing. */
+ /* or enqueue the block for later processing. */
GC_apply_to_all_blocks(GC_reclaim_block, (word)report_if_found);
# ifdef EAGER_SWEEP
- /* This is a very stupid thing to do. We make it possible anyway, */
- /* so that you can convince yourself that it really is very stupid. */
+ /* This is a very stupid thing to do. We make it possible anyway, */
+ /* so that you can convince yourself that it really is very stupid. */
GC_reclaim_all((GC_stop_func)0, FALSE);
+# elif defined(ENABLE_DISCLAIM)
+ /* However, make sure to clear reclaimable objects of kinds with */
+ /* unconditional marking enabled before we do any significant */
+ /* marking work. */
+ GC_reclaim_unconditionally_marked();
# endif
-# if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
+# if defined(PARALLEL_MARK)
GC_ASSERT(0 == GC_fl_builder_count);
# endif
-
+
}
/*
* appropriate free list is nonempty, or there are no more blocks to
* sweep.
*/
-void GC_continue_reclaim(sz, kind)
-word sz; /* words */
-int kind;
+GC_INNER void GC_continue_reclaim(word sz /* granules */, int kind)
{
- register hdr * hhdr;
- register struct hblk * hbp;
- register struct obj_kind * ok = &(GC_obj_kinds[kind]);
+ hdr * hhdr;
+ struct hblk * hbp;
+ struct obj_kind * ok = &(GC_obj_kinds[kind]);
struct hblk ** rlh = ok -> ok_reclaim_list;
- ptr_t *flh = &(ok -> ok_freelist[sz]);
-
- if (rlh == 0) return; /* No blocks of this kind. */
- rlh += sz;
- while ((hbp = *rlh) != 0) {
+ void **flh = &(ok -> ok_freelist[sz]);
+
+ if (NULL == rlh)
+ return; /* No blocks of this kind. */
+
+ for (rlh += sz; (hbp = *rlh) != NULL; ) {
hhdr = HDR(hbp);
*rlh = hhdr -> hb_next;
- GC_reclaim_small_nonempty_block(hbp, FALSE MEM_FOUND_ADDR);
- if (*flh != 0) break;
+ GC_reclaim_small_nonempty_block(hbp, hhdr -> hb_sz, FALSE);
+ if (*flh != 0)
+ break;
}
}
* Abort and return FALSE when/if (*stop_func)() returns TRUE.
* If this returns TRUE, then it's safe to restart the world
* with incorrectly cleared mark bits.
- * If ignore_old is TRUE, then reclaim only blocks that have been
+ * If ignore_old is TRUE, then reclaim only blocks that have been
* recently reclaimed, and discard the rest.
* Stop_func may be 0.
*/
-GC_bool GC_reclaim_all(stop_func, ignore_old)
-GC_stop_func stop_func;
-GC_bool ignore_old;
+GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old)
{
- register word sz;
- register int kind;
- register hdr * hhdr;
- register struct hblk * hbp;
- register struct obj_kind * ok;
+ word sz;
+ unsigned kind;
+ hdr * hhdr;
+ struct hblk * hbp;
+ struct obj_kind * ok;
struct hblk ** rlp;
struct hblk ** rlh;
-# ifdef PRINTTIMES
- CLOCK_TYPE start_time;
- CLOCK_TYPE done_time;
-
- GET_TIME(start_time);
+# ifndef NO_CLOCK
+ CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
+
+ if (GC_print_stats == VERBOSE)
+ GET_TIME(start_time);
# endif
-
+
for (kind = 0; kind < GC_n_kinds; kind++) {
- ok = &(GC_obj_kinds[kind]);
- rlp = ok -> ok_reclaim_list;
- if (rlp == 0) continue;
- for (sz = 1; sz <= MAXOBJSZ; sz++) {
- rlh = rlp + sz;
- while ((hbp = *rlh) != 0) {
- if (stop_func != (GC_stop_func)0 && (*stop_func)()) {
- return(FALSE);
- }
- hhdr = HDR(hbp);
- *rlh = hhdr -> hb_next;
- if (!ignore_old || hhdr -> hb_last_reclaimed == GC_gc_no - 1) {
- /* It's likely we'll need it this time, too */
- /* It's been touched recently, so this */
- /* shouldn't trigger paging. */
- GC_reclaim_small_nonempty_block(hbp, FALSE MEM_FOUND_ADDR);
- }
+ ok = &(GC_obj_kinds[kind]);
+ rlp = ok -> ok_reclaim_list;
+ if (rlp == 0) continue;
+ for (sz = 1; sz <= MAXOBJGRANULES; sz++) {
+ for (rlh = rlp + sz; (hbp = *rlh) != NULL; ) {
+ if (stop_func != (GC_stop_func)0 && (*stop_func)()) {
+ return(FALSE);
+ }
+ hhdr = HDR(hbp);
+ *rlh = hhdr -> hb_next;
+ if (!ignore_old
+ || (word)hhdr->hb_last_reclaimed == GC_gc_no - 1) {
+ /* It's likely we'll need it this time, too */
+ /* It's been touched recently, so this */
+ /* shouldn't trigger paging. */
+ GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
+ }
}
}
}
-# ifdef PRINTTIMES
- GET_TIME(done_time);
- GC_printf1("Disposing of reclaim lists took %lu msecs\n",
- MS_TIME_DIFF(done_time,start_time));
+# ifndef NO_CLOCK
+ if (GC_print_stats == VERBOSE) {
+ CLOCK_TYPE done_time;
+
+ GET_TIME(done_time);
+ GC_verbose_log_printf(
+ "Disposing of reclaim lists took %lu ms %lu ns\n",
+ MS_TIME_DIFF(done_time, start_time),
+ NS_FRAC_TIME_DIFF(done_time, start_time));
+ }
# endif
return(TRUE);
}
+
+#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
+/* We do an eager sweep on heap blocks where unconditional marking has */
+/* been enabled, so that any reclaimable objects have been reclaimed */
+/* before we start marking. This is a simplified GC_reclaim_all */
+/* restricted to kinds where ok_mark_unconditionally is true. */
+ STATIC void GC_reclaim_unconditionally_marked(void)
+ {
+ word sz;
+ unsigned kind;
+ hdr * hhdr;
+ struct hblk * hbp;
+ struct obj_kind * ok;
+ struct hblk ** rlp;
+ struct hblk ** rlh;
+
+ for (kind = 0; kind < GC_n_kinds; kind++) {
+ ok = &(GC_obj_kinds[kind]);
+ if (!ok->ok_mark_unconditionally)
+ continue;
+ rlp = ok->ok_reclaim_list;
+ if (rlp == 0)
+ continue;
+ for (sz = 1; sz <= MAXOBJGRANULES; sz++) {
+ rlh = rlp + sz;
+ while ((hbp = *rlh) != 0) {
+ hhdr = HDR(hbp);
+ *rlh = hhdr->hb_next;
+ GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
+ }
+ }
+ }
+ }
+#endif /* !EAGER_SWEEP && ENABLE_DISCLAIM */
+
+struct enumerate_reachable_s {
+ GC_reachable_object_proc proc;
+ void *client_data;
+};
+
+STATIC void GC_do_enumerate_reachable_objects(struct hblk *hbp, word ped)
+{
+ struct hblkhdr *hhdr = HDR(hbp);
+ size_t sz = (size_t)hhdr->hb_sz;
+ size_t bit_no;
+ char *p, *plim;
+
+ if (GC_block_empty(hhdr)) {
+ return;
+ }
+
+ p = hbp->hb_body;
+ if (sz > MAXOBJBYTES) { /* one big object */
+ plim = p;
+ } else {
+ plim = hbp->hb_body + HBLKSIZE - sz;
+ }
+ /* Go through all words in block. */
+ for (bit_no = 0; p <= plim; bit_no += MARK_BIT_OFFSET(sz), p += sz) {
+ if (mark_bit_from_hdr(hhdr, bit_no)) {
+ ((struct enumerate_reachable_s *)ped)->proc(p, sz,
+ ((struct enumerate_reachable_s *)ped)->client_data);
+ }
+ }
+}
+
+GC_API void GC_CALL GC_enumerate_reachable_objects_inner(
+ GC_reachable_object_proc proc,
+ void *client_data)
+{
+ struct enumerate_reachable_s ed;
+
+ GC_ASSERT(I_HOLD_LOCK());
+ ed.proc = proc;
+ ed.client_data = client_data;
+ GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects, (word)&ed);
+}
projects / gc / blobdiff