From 4c24e89123648e6080622bde687c004f60532f73 Mon Sep 17 00:00:00 2001 From: Ivan Maidanski Date: Mon, 19 Dec 2016 10:56:19 +0300 Subject: [PATCH] Skip grungy_pages update when mark state invalid to speedup read_dirty * include/private/gc_priv.h [!GC_DISABLE_INCREMENTAL] (GC_read_dirty): Add output_unneeded argument; update comment. * os_dep.c [MANUAL_VDB || PROC_VDB] (GC_read_dirty): Likewise. * mark.c [!GC_DISABLE_INCREMENTAL] (GC_initiate_gc): Pass output_unneeded argument. * misc.c [!GC_DISABLE_INCREMENTAL && !KEEP_BACK_PTRS] (GC_enable_incremental): Likewise. * os_dep.c [MPROTECT_VDB] (GC_read_dirty): Likewise. * os_dep.c [GWW_VDB] (GC_gww_read_dirty): Add output_unneeded argument. * os_dep.c [DEFAULT_VDB || PCR_VDB] (GC_read_dirty): Likewise. * os_dep.c [GWW_VDB] (GC_gww_read_dirty): Do not call BZERO(GC_grungy_pages) and set_pht_entry_from_index() if output_unneeded. * os_dep.c [GWW_VDB && CHECKSUMS] (GC_gww_read_dirty): Add assertion that output_unneeded is false. * os_dep.c [MANUAL_VDB || MPROTECT_VDB] (GC_read_dirty): Do not call BCOPY(GC_dirty_pages, GC_grungy_pages) if output_unneeded. * os_dep.c [PROC_VDB] (GC_read_dirty): Do not call memset(GC_grungy_pages) if output_unneeded. --- include/private/gc_priv.h | 6 +++-- mark.c | 8 +++++- misc.c | 2 +- os_dep.c | 54 ++++++++++++++++++++++----------------- 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/include/private/gc_priv.h b/include/private/gc_priv.h index a92fba0f..6be33a2e 100644 --- a/include/private/gc_priv.h +++ b/include/private/gc_priv.h @@ -2107,8 +2107,10 @@ GC_EXTERN GC_bool GC_print_back_height; /* Virtual dirty bit implementation: */ /* Each implementation exports the following: */ - GC_INNER void GC_read_dirty(void); - /* Retrieve dirty bits. */ + GC_INNER void GC_read_dirty(GC_bool output_unneeded); + /* Retrieve dirty bits. Set output_unneeded to */ + /* indicate that reading of the retrieved dirty */ + /* bits is not planned till the next retrieval. */ GC_INNER GC_bool GC_page_was_dirty(struct hblk *h); /* Read retrieved dirty bits. */ diff --git a/mark.c b/mark.c index 36b39e5b..e93b433e 100644 --- a/mark.c +++ b/mark.c @@ -242,7 +242,13 @@ GC_INNER void GC_clear_marks(void) GC_INNER void GC_initiate_gc(void) { # ifndef GC_DISABLE_INCREMENTAL - if (GC_dirty_maintained) GC_read_dirty(); + if (GC_dirty_maintained) { +# ifdef CHECKSUMS + GC_read_dirty(FALSE); +# else + GC_read_dirty(GC_mark_state == MS_INVALID); +# endif + } # endif # ifdef STUBBORN_ALLOC GC_read_changed(); diff --git a/misc.c b/misc.c index b228dbfb..2b6494db 100644 --- a/misc.c +++ b/misc.c @@ -1346,7 +1346,7 @@ GC_API void GC_CALL GC_enable_incremental(void) /* else we're OK in assuming everything's */ /* clean since nothing can point to an */ /* unmarked object. */ - GC_read_dirty(); + GC_read_dirty(FALSE); } } UNLOCK(); diff --git a/os_dep.c b/os_dep.c index ca0b989b..da00232b 100644 --- a/os_dep.c +++ b/os_dep.c @@ -2785,14 +2785,15 @@ GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) # endif /* !MPROTECT_VDB */ # ifdef MPROTECT_VDB - STATIC void GC_gww_read_dirty(void) + STATIC void GC_gww_read_dirty(GC_bool output_unneeded) # else - GC_INNER void GC_read_dirty(void) + GC_INNER void GC_read_dirty(GC_bool output_unneeded) # endif { word i; - BZERO(GC_grungy_pages, sizeof(GC_grungy_pages)); + if (!output_unneeded) + BZERO(GC_grungy_pages, sizeof(GC_grungy_pages)); for (i = 0; i != GC_n_heap_sects; ++i) { GC_ULONG_PTR count; @@ -2823,23 +2824,25 @@ GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) &count, &page_size) != 0) { static int warn_count = 0; - unsigned j; struct hblk * start = (struct hblk *)GC_heap_sects[i].hs_start; static struct hblk *last_warned = 0; size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes); - if ( i != 0 && last_warned != start && warn_count++ < 5) { + if (i != 0 && last_warned != start && warn_count++ < 5) { last_warned = start; - WARN( - "GC_gww_read_dirty unexpectedly failed at %p: " - "Falling back to marking all pages dirty\n", start); + WARN("GC_gww_read_dirty unexpectedly failed at %p: " + "Falling back to marking all pages dirty\n", start); } - for (j = 0; j < nblocks; ++j) { + if (!output_unneeded) { + unsigned j; + + for (j = 0; j < nblocks; ++j) { word hash = PHT_HASH(start + j); set_pht_entry_from_index(GC_grungy_pages, hash); + } } count = 1; /* Done with this section. */ - } else /* succeeded */ { + } else /* succeeded */ if (!output_unneeded) { PVOID * pages_end = pages + count; while (pages != pages_end) { @@ -2857,6 +2860,7 @@ GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) } # ifdef CHECKSUMS + GC_ASSERT(!output_unneeded); GC_or_pages(GC_written_pages, GC_grungy_pages); # endif } @@ -2877,7 +2881,7 @@ GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) /* Retrieve system dirty bits for heap to a local buffer. */ /* Restore the systems notion of which pages are dirty. */ - GC_INNER void GC_read_dirty(void) {} + GC_INNER void GC_read_dirty(GC_bool output_unneeded GC_ATTR_UNUSED) {} /* Is the HBLKSIZE sized page at h marked dirty in the local buffer? */ /* If the actual page size is different, this returns TRUE if any */ @@ -2911,12 +2915,13 @@ GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) GC_dirty_maintained = TRUE; } - /* Retrieve system dirty bits for heap to a local buffer. */ - /* Restore the systems notion of which pages are dirty. */ - GC_INNER void GC_read_dirty(void) + /* Retrieve system dirty bits for the heap to a local buffer */ + /* (unless output_unneeded). Restore the systems notion of */ + /* which pages are dirty. */ + GC_INNER void GC_read_dirty(GC_bool output_unneeded) { - BCOPY((word *)GC_dirty_pages, GC_grungy_pages, - (sizeof GC_dirty_pages)); + if (!output_unneeded) + BCOPY((word *)GC_dirty_pages, GC_grungy_pages, sizeof(GC_dirty_pages)); BZERO((word *)GC_dirty_pages, (sizeof GC_dirty_pages)); } @@ -3494,16 +3499,16 @@ STATIC void GC_protect_heap(void) /* We assume that either the world is stopped or its OK to lose dirty */ /* bits while this is happening (as in GC_enable_incremental). */ -GC_INNER void GC_read_dirty(void) +GC_INNER void GC_read_dirty(GC_bool output_unneeded) { # if defined(GWW_VDB) if (GC_GWW_AVAILABLE()) { - GC_gww_read_dirty(); + GC_gww_read_dirty(output_unneeded); return; } # endif - BCOPY((word *)GC_dirty_pages, GC_grungy_pages, - (sizeof GC_dirty_pages)); + if (!output_unneeded) + BCOPY((word *)GC_dirty_pages, GC_grungy_pages, sizeof(GC_dirty_pages)); BZERO((word *)GC_dirty_pages, (sizeof GC_dirty_pages)); GC_protect_heap(); } @@ -3582,7 +3587,7 @@ GC_INNER void GC_dirty_init(void) # define READ read -GC_INNER void GC_read_dirty(void) +GC_INNER void GC_read_dirty(GC_bool output_unneeded) { int nmaps; char * bufp = GC_proc_buf; @@ -3604,7 +3609,8 @@ GC_INNER void GC_read_dirty(void) if (READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) { WARN("Insufficient space for /proc read\n", 0); /* Punt: */ - memset(GC_grungy_pages, 0xff, sizeof (page_hash_table)); + if (!output_unneeded) + memset(GC_grungy_pages, 0xff, sizeof (page_hash_table)); memset(GC_written_pages, 0xff, sizeof(page_hash_table)); return; } @@ -3653,7 +3659,7 @@ GC_INNER void GC_read_dirty(void) GC_log_printf("Proc VDB read done\n"); # endif - /* Update GC_written_pages. */ + /* Update GC_written_pages (even if output_unneeded). */ GC_or_pages(GC_written_pages, GC_grungy_pages); } @@ -3686,7 +3692,7 @@ GC_INNER void GC_dirty_init(void) } } -GC_INNER void GC_read_dirty(void) +GC_INNER void GC_read_dirty(GC_bool output_unneeded GC_ATTR_UNUSED) { /* lazily enable dirty bits on newly added heap sects */ { -- 2.40.0