From: Dmitry Stogov Date: Fri, 2 Mar 2018 14:02:29 +0000 (+0300) Subject: Make distinct between indexes/numbers, GC addresses and pointers to gc_root_buffers. X-Git-Tag: php-7.3.0alpha1~268 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=ae64dd6d566de448d20232436e1aba25c611357c;p=php Make distinct between indexes/numbers, GC addresses and pointers to gc_root_buffers. Perform conversion through macros. --- diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index 7bc4b23560..f536736388 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -119,24 +119,6 @@ GC_TYPE_INFO(ref) |= (GC_COLOR << GC_INFO_SHIFT); \ } while (0) -/* GC buffer size */ -#define GC_INVALID 0 -#define GC_FIRST_REAL_ROOT 1 - -#define GC_DEFAULT_BUF_SIZE (16 * 1024) -#define GC_BUF_GROW_STEP (128 * 1024) - -#define GC_MAX_UNCOMPRESSED (1024 * 1024) -#define GC_MAX_BUF_SIZE 0x40000000 - -#define GC_THRESHOLD_DEFAULT 10000 -#define GC_THRESHOLD_STEP 10000 -#define GC_THRESHOLD_MAX 1000000000 -#define GC_THRESHOLD_TRIGGER 100 - -/* GC flags */ -#define GC_HAS_DESTRUCTORS (1<<0) - /* bit stealing tags for gc_root_buffer.ref */ #define GC_BITS 0x3 @@ -157,9 +139,43 @@ #define GC_MAKE_GARBAGE(ptr) \ ((void*)(((uintptr_t)(ptr)) | GC_GARBAGE)) +/* GC address conversion */ +#define GC_ADDR2NUM(addr) (addr) +#define GC_NUM2ADDR(num) (num) + +#define GC_NEXT_ADDR(addr) ((addr) + 1) +#define GC_PREV_ADDR(addr) ((addr) - 1) + +#define GC_ADDR2PTR(addr) (GC_G(buf) + (addr)) +#define GC_PTR2ADDR(ptr) ((ptr) - GC_G(buf)) + +#define GC_ADDR2LIST(addr) ((void*)(uintptr_t)(((addr) * sizeof(void*)) | GC_UNUSED)) +#define GC_LIST2ADDR(list) (((uint32_t)(uintptr_t)(list)) / sizeof(void*)) + +/* GC buffers */ +#define GC_INVALID_NUM 0 +#define GC_FIRST_ROOT_NUM 1 + +#define GC_INVALID_ADDR GC_NUM2ADDR(GC_INVALID_NUM) +#define GC_FIRST_ROOT_ADDR GC_NUM2ADDR(GC_FIRST_ROOT_NUM) + +#define GC_DEFAULT_BUF_SIZE (16 * 1024) +#define GC_BUF_GROW_STEP (128 * 1024) + +#define GC_MAX_UNCOMPRESSED (1024 * 1024) +#define GC_MAX_BUF_SIZE 0x40000000 + +#define GC_THRESHOLD_DEFAULT 10000 +#define GC_THRESHOLD_STEP 10000 +#define GC_THRESHOLD_MAX 1000000000 +#define GC_THRESHOLD_TRIGGER 100 + +/* GC flags */ +#define GC_HAS_DESTRUCTORS (1<<0) + /* unused buffers */ #define GC_HAS_UNUSED() \ - (GC_G(unused) != GC_INVALID) + (GC_G(unused) != GC_INVALID_ADDR) #define GC_FETCH_UNUSED() \ gc_fetch_unused() #define GC_LINK_UNUSED(root) \ @@ -185,10 +201,10 @@ typedef struct _zend_gc_globals { zend_bool gc_full; gc_root_buffer *buf; /* preallocated arrays of buffers */ - uint32_t buf_size; /* size of the GC buffer */ - uint32_t gc_threshold; /* GC collection threshold */ uint32_t unused; /* linked list of unused buffers */ uint32_t first_unused; /* first unused buffer */ + uint32_t gc_threshold; /* GC collection threshold [addr] */ + uint32_t buf_size; /* size of the GC buffer [addr] */ uint32_t num_roots; /* number of roots in GC buffer */ uint32_t gc_runs; @@ -248,7 +264,7 @@ static zend_always_inline uint32_t gc_compress(uint32_t addr) static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, uint32_t addr) { - gc_root_buffer *root = GC_G(buf) + addr; + gc_root_buffer *root = GC_ADDR2PTR(addr); if (EXPECTED(GC_GET_PTR(root->ref) == ref)) { return root; @@ -257,7 +273,7 @@ static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, ui while (1) { addr += GC_MAX_UNCOMPRESSED; ZEND_ASSERT(addr < GC_G(first_unused)); - root = GC_G(buf) + addr; + root = GC_ADDR2PTR(addr); if (GC_GET_PTR(root->ref) == ref) { return root; } @@ -271,17 +287,16 @@ static zend_always_inline uint32_t gc_fetch_unused(void) ZEND_ASSERT(GC_HAS_UNUSED()); addr = GC_G(unused); - root = GC_G(buf) + addr; + root = GC_ADDR2PTR(addr); ZEND_ASSERT(GC_IS_UNUSED(root->ref)); - /* optimization: GC_GET_PTR(root->ref) is not necessary because it shifted anyway */ - GC_G(unused) = (uint32_t)(uintptr_t)root->ref / sizeof(void*); + GC_G(unused) = GC_LIST2ADDR(root->ref); return addr; } static zend_always_inline void gc_link_unused(gc_root_buffer *root) { - root->ref = (void*)(uintptr_t)((GC_G(unused) * sizeof(void*)) | GC_UNUSED); - GC_G(unused) = root - GC_G(buf); + root->ref = GC_ADDR2LIST(GC_G(unused)); + GC_G(unused) = GC_PTR2ADDR(root); } static zend_always_inline uint32_t gc_fetch_next_unused(void) @@ -290,7 +305,7 @@ static zend_always_inline uint32_t gc_fetch_next_unused(void) ZEND_ASSERT(GC_HAS_NEXT_UNUSED()); addr = GC_G(first_unused); - GC_G(first_unused)++; + GC_G(first_unused) = GC_NEXT_ADDR(GC_G(first_unused)); return addr; } @@ -349,10 +364,10 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) gc_globals->gc_full = 0; gc_globals->buf = NULL; - gc_globals->buf_size = 0; - gc_globals->gc_threshold = 0; - gc_globals->unused = GC_INVALID; - gc_globals->first_unused = 0; + gc_globals->unused = GC_INVALID_ADDR; + gc_globals->first_unused = GC_INVALID_ADDR; + gc_globals->gc_threshold = GC_INVALID_ADDR; + gc_globals->buf_size = GC_INVALID_ADDR; gc_globals->num_roots = 0; gc_globals->gc_runs = 0; @@ -390,8 +405,8 @@ void gc_reset(void) GC_G(gc_active) = 0; GC_G(gc_protected) = 0; GC_G(gc_full) = 0; - GC_G(unused) = GC_INVALID; - GC_G(first_unused) = GC_FIRST_REAL_ROOT; + GC_G(unused) = GC_INVALID_ADDR; + GC_G(first_unused) = GC_FIRST_ROOT_ADDR; GC_G(num_roots) = 0; GC_G(gc_runs) = 0; @@ -414,8 +429,8 @@ ZEND_API zend_bool gc_enable(zend_bool enable) GC_G(gc_enabled) = enable; if (enable && !old_enabled && GC_G(buf) == NULL) { GC_G(buf) = (gc_root_buffer*) malloc(sizeof(gc_root_buffer) * GC_DEFAULT_BUF_SIZE); - GC_G(buf_size) = GC_DEFAULT_BUF_SIZE; - GC_G(gc_threshold) = GC_THRESHOLD_DEFAULT + GC_FIRST_REAL_ROOT; + GC_G(buf_size) = GC_NUM2ADDR(GC_DEFAULT_BUF_SIZE); + GC_G(gc_threshold) = GC_NUM2ADDR(GC_THRESHOLD_DEFAULT + GC_FIRST_ROOT_NUM); gc_reset(); } return old_enabled; @@ -442,7 +457,7 @@ static void gc_grow_root_buffer(void) { size_t new_size; - if (GC_G(buf_size) >= GC_MAX_BUF_SIZE) { + if (GC_ADDR2NUM(GC_G(buf_size)) >= GC_MAX_BUF_SIZE) { if (!GC_G(gc_full)) { zend_error(E_WARNING, "GC buffer overflow (GC disabled)\n"); GC_G(gc_active) = 1; @@ -451,16 +466,16 @@ static void gc_grow_root_buffer(void) return; } } - if (GC_G(buf_size) < GC_BUF_GROW_STEP) { - new_size = GC_G(buf_size) * 2; + if (GC_G(buf_size) < GC_NUM2ADDR(GC_BUF_GROW_STEP)) { + new_size = GC_ADDR2NUM(GC_G(buf_size)) * 2; } else { - new_size = GC_G(buf_size) + GC_BUF_GROW_STEP; + new_size = GC_ADDR2NUM(GC_G(buf_size)) + GC_BUF_GROW_STEP; } if (new_size > GC_MAX_BUF_SIZE) { new_size = GC_MAX_BUF_SIZE; } GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1); - GC_G(buf_size) = new_size; + GC_G(buf_size) = GC_NUM2ADDR(new_size); } static void gc_adjust_threshold(int count) @@ -472,24 +487,24 @@ static void gc_adjust_threshold(int count) * by a fixed step */ if (count < GC_THRESHOLD_TRIGGER) { /* increase */ - if (GC_G(gc_threshold) < GC_THRESHOLD_MAX) { - new_threshold = GC_G(gc_threshold) + GC_THRESHOLD_STEP; + if (GC_G(gc_threshold) < GC_NUM2ADDR(GC_THRESHOLD_MAX)) { + new_threshold = GC_ADDR2NUM(GC_G(gc_threshold)) + GC_THRESHOLD_STEP; if (new_threshold > GC_THRESHOLD_MAX) { new_threshold = GC_THRESHOLD_MAX; } - if (new_threshold > GC_G(buf_size)) { + if (new_threshold > GC_ADDR2NUM(GC_G(buf_size))) { gc_grow_root_buffer(); } - if (new_threshold <= GC_G(buf_size)) { - GC_G(gc_threshold) = new_threshold; + if (new_threshold <= GC_ADDR2NUM(GC_G(buf_size))) { + GC_G(gc_threshold) = GC_NUM2ADDR(new_threshold); } } - } else if (GC_G(gc_threshold) > GC_THRESHOLD_DEFAULT) { - new_threshold = GC_G(gc_threshold) - GC_THRESHOLD_STEP; + } else if (GC_G(gc_threshold) > GC_NUM2ADDR(GC_THRESHOLD_DEFAULT)) { + new_threshold = GC_ADDR2NUM(GC_G(gc_threshold)) - GC_THRESHOLD_STEP; if (new_threshold < GC_THRESHOLD_DEFAULT) { new_threshold = GC_THRESHOLD_DEFAULT; } - GC_G(gc_threshold) = new_threshold; + GC_G(gc_threshold) = GC_NUM2ADDR(new_threshold); } } @@ -518,11 +533,13 @@ static zend_never_inline void ZEND_FASTCALL gc_possible_root_when_full(zend_refc addr = GC_FETCH_NEXT_UNUSED(); } else { gc_grow_root_buffer(); - ZEND_ASSERT(GC_HAS_NEXT_UNUSED()); + if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) { + return; + } addr = GC_FETCH_NEXT_UNUSED(); } - newRoot = GC_G(buf) + addr; + newRoot = GC_ADDR2PTR(addr); newRoot->ref = ref; /* GC_ROOT tag is 0 */ GC_TRACE_SET_COLOR(ref, GC_PURPLE); @@ -558,7 +575,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT); ZEND_ASSERT(GC_INFO(ref) == 0); - newRoot = GC_G(buf) + addr; + newRoot = GC_ADDR2PTR(addr); newRoot->ref = ref; /* GC_ROOT tag is 0 */ GC_TRACE_SET_COLOR(ref, GC_PURPLE); @@ -592,12 +609,12 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) GC_REF_SET_INFO(ref, 0); /* Perform decopression only in case of large buffers */ - if (UNEXPECTED(GC_G(first_unused) >= GC_MAX_UNCOMPRESSED)) { + if (UNEXPECTED(GC_G(first_unused) >= GC_NUM2ADDR(GC_MAX_UNCOMPRESSED))) { gc_remove_compressed(ref, addr); return; } - root = GC_G(buf) + addr; + root = GC_ADDR2PTR(addr); gc_remove_from_roots(root); } @@ -813,30 +830,30 @@ tail_call: /* Two-Finger compaction algorithm */ static void gc_compact(void) { - if (GC_G(num_roots) + GC_FIRST_REAL_ROOT != GC_G(first_unused)) { + if (GC_NUM2ADDR(GC_G(num_roots) + GC_FIRST_ROOT_NUM) != GC_G(first_unused)) { if (GC_G(num_roots)) { - gc_root_buffer *buf = GC_G(buf); - uint32_t free = GC_FIRST_REAL_ROOT; - uint32_t scan = GC_G(first_unused) - 1; + gc_root_buffer *free = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); + gc_root_buffer *scan = GC_ADDR2PTR(GC_PREV_ADDR(GC_G(first_unused))); + gc_root_buffer *end = GC_ADDR2PTR(GC_NUM2ADDR(GC_G(num_roots))); uint32_t addr; zend_refcounted *p; while (free < scan) { - while (!GC_IS_UNUSED(buf[free].ref)) { + while (!GC_IS_UNUSED(free->ref)) { free++; } - while (GC_IS_UNUSED(buf[scan].ref)) { + while (GC_IS_UNUSED(scan->ref)) { scan--; } if (scan > free) { - p = buf[scan].ref; - buf[free].ref = p; + p = scan->ref; + free->ref = p; p = GC_GET_PTR(p); - addr = gc_compress(free); + addr = gc_compress(GC_PTR2ADDR(free)); GC_REF_SET_INFO(p, addr | GC_REF_COLOR(p)); free++; scan--; - if (scan <= GC_G(num_roots)) { + if (scan <= end) { break; } } else { @@ -844,8 +861,8 @@ static void gc_compact(void) } } } - GC_G(unused) = GC_INVALID; - GC_G(first_unused) = GC_G(num_roots) + GC_FIRST_REAL_ROOT; + GC_G(unused) = GC_INVALID_ADDR; + GC_G(first_unused) = GC_NUM2ADDR(GC_G(num_roots) + GC_FIRST_ROOT_NUM); } } @@ -855,8 +872,8 @@ static void gc_mark_roots(void) gc_compact(); - current = GC_G(buf) + GC_FIRST_REAL_ROOT; - last = GC_G(buf) + GC_G(first_unused); + current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);; + last = GC_ADDR2PTR(GC_G(first_unused)); while (current != last) { if (GC_IS_ROOT(current->ref)) { if (GC_REF_CHECK_COLOR(current->ref, GC_PURPLE)) { @@ -966,8 +983,8 @@ tail_call: static void gc_scan_roots(void) { - gc_root_buffer *current = GC_G(buf) + GC_FIRST_REAL_ROOT; - gc_root_buffer *last = GC_G(buf) + GC_G(first_unused); + gc_root_buffer *current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); + gc_root_buffer *last = GC_ADDR2PTR(GC_G(first_unused)); while (current != last) { if (GC_IS_ROOT(current->ref)) { @@ -988,11 +1005,13 @@ static void gc_add_garbage(zend_refcounted *ref) addr = GC_FETCH_NEXT_UNUSED(); } else { gc_grow_root_buffer(); - ZEND_ASSERT(GC_HAS_NEXT_UNUSED()); + if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) { + return; + } addr = GC_FETCH_NEXT_UNUSED(); } - buf = GC_G(buf) + addr; + buf = GC_ADDR2PTR(addr); buf->ref = GC_MAKE_GARBAGE(ref); addr = gc_compress(addr); @@ -1131,11 +1150,11 @@ tail_call: static int gc_collect_roots(uint32_t *flags) { - uint32_t n, end; + uint32_t addr, end; zend_refcounted *ref; int count = 0; - gc_root_buffer *current = GC_G(buf) + GC_FIRST_REAL_ROOT; - gc_root_buffer *last = GC_G(buf) + GC_G(first_unused); + gc_root_buffer *current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); + gc_root_buffer *last = GC_ADDR2PTR(GC_G(first_unused)); /* remove non-garbage from the list */ while (current != last) { @@ -1152,10 +1171,10 @@ static int gc_collect_roots(uint32_t *flags) /* Root buffer might be reallocated during gc_collect_white, * make sure to reload pointers. */ - n = GC_FIRST_REAL_ROOT; + addr = GC_FIRST_ROOT_ADDR; end = GC_G(first_unused); - while (n != end) { - current = GC_G(buf) + n; + while (addr != end) { + current = GC_ADDR2PTR(addr); ref = current->ref; if (GC_IS_ROOT(ref)) { if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { @@ -1163,7 +1182,7 @@ static int gc_collect_roots(uint32_t *flags) count += gc_collect_white(ref, flags); } } - n++; + addr = GC_NEXT_ADDR(addr); } return count; @@ -1275,7 +1294,7 @@ ZEND_API int zend_gc_collect_cycles(void) gc_root_buffer *current, *last; zend_refcounted *p; uint32_t gc_flags = 0; - uint32_t n, end; + uint32_t addr, end; if (GC_G(gc_active)) { return 0; @@ -1303,17 +1322,18 @@ ZEND_API int zend_gc_collect_cycles(void) end = GC_G(first_unused); if (gc_flags & GC_HAS_DESTRUCTORS) { - uint32_t *refcounts; + uint32_t *refcounts, count, n; GC_TRACE("Calling destructors"); // TODO: may be use emalloc() ??? - refcounts = pemalloc(sizeof(uint32_t) * GC_G(first_unused), 1); + count = GC_ADDR2NUM(GC_G(first_unused)); + refcounts = pemalloc(sizeof(uint32_t) * count, 1); /* Remember reference counters before calling destructors */ - n = GC_FIRST_REAL_ROOT; - current = GC_G(buf) + GC_FIRST_REAL_ROOT; - while (n != end) { + n = GC_FIRST_ROOT_NUM; + current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); + while (n != count) { if (GC_IS_GARBAGE(current->ref)) { p = GC_GET_PTR(current->ref); refcounts[n] = GC_REFCOUNT(p); @@ -1326,9 +1346,9 @@ ZEND_API int zend_gc_collect_cycles(void) * * The root buffer might be reallocated during destructors calls, * make sure to reload pointers as necessary. */ - n = GC_FIRST_REAL_ROOT; - while (n != end) { - current = GC_G(buf) + n; + addr = GC_FIRST_ROOT_ADDR; + while (addr != end) { + current = GC_ADDR2PTR(addr); if (GC_IS_GARBAGE(current->ref)) { p = GC_GET_PTR(current->ref); if (GC_TYPE(p) == IS_OBJECT @@ -1346,13 +1366,13 @@ ZEND_API int zend_gc_collect_cycles(void) } } } - n++; + addr = GC_NEXT_ADDR(addr); } /* Remove values captured in destructors */ - n = GC_FIRST_REAL_ROOT; - current = GC_G(buf) + GC_FIRST_REAL_ROOT; - while (n != end) { + n = GC_FIRST_ROOT_NUM; + current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); + while (n != count) { if (GC_IS_GARBAGE(current->ref)) { p = GC_GET_PTR(current->ref); if (GC_REFCOUNT(p) > refcounts[n]) { @@ -1374,8 +1394,8 @@ ZEND_API int zend_gc_collect_cycles(void) /* Destroy zvals */ GC_TRACE("Destroying zvals"); GC_G(gc_protected) = 1; - current = GC_G(buf) + GC_FIRST_REAL_ROOT; - last = GC_G(buf) + GC_G(first_unused); + current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); + last = GC_ADDR2PTR(GC_G(first_unused)); while (current != last) { if (GC_IS_GARBAGE(current->ref)) { p = GC_GET_PTR(current->ref); @@ -1413,7 +1433,7 @@ ZEND_API int zend_gc_collect_cycles(void) } /* Free objects */ - current = GC_G(buf) + GC_FIRST_REAL_ROOT; + current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR); while (current != last) { if (GC_IS_GARBAGE(current->ref)) { p = GC_GET_PTR(current->ref);