From: Dmitry Stogov <dmitry@zend.com>
Date: Thu, 9 Oct 2014 16:29:02 +0000 (+0400)
Subject: Improved VM stack primitives for fast paths. Slow paths are not inlined anymore.
X-Git-Tag: POST_NATIVE_TLS_MERGE^2~76^2~16
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=0390cde428f850b86948e755c7f9e075920fe6c4;p=php

Improved VM stack primitives for fast paths. Slow paths are not inlined anymore.
---

diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c
index 45a0dadcbb..fa5de685c9 100644
--- a/Zend/zend_execute.c
+++ b/Zend/zend_execute.c
@@ -123,6 +123,61 @@ static const zend_internal_function zend_pass_function = {
 #define DECODE_CTOR(ce) \
 	((zend_class_entry*)(((zend_uintptr_t)(ce)) & ~(CTOR_CALL_BIT|CTOR_USED_BIT)))
 
+#define ZEND_VM_STACK_PAGE_SLOTS (16 * 1024) /* should be a power of 2 */
+
+#define ZEND_VM_STACK_PAGE_SIZE  (ZEND_VM_STACK_PAGE_SLOTS * sizeof(zval))
+
+#define ZEND_VM_STACK_FREE_PAGE_SIZE \
+	((ZEND_VM_STACK_PAGE_SLOTS - ZEND_VM_STACK_HEADER_SLOTS) * sizeof(zval))
+
+#define ZEND_VM_STACK_PAGE_ALIGNED_SIZE(size) \
+	(((size) + (ZEND_VM_STACK_FREE_PAGE_SIZE - 1)) & ~ZEND_VM_STACK_PAGE_SIZE)
+
+static zend_always_inline zend_vm_stack zend_vm_stack_new_page(size_t size, zend_vm_stack prev) {
+	zend_vm_stack page = (zend_vm_stack)emalloc(size);
+
+	page->top = ZEND_VM_STACK_ELEMETS(page);
+	page->end = (zval*)((char*)page + size);
+	page->prev = prev;
+	return page;
+}
+
+ZEND_API void zend_vm_stack_init(TSRMLS_D)
+{
+	EG(vm_stack) = zend_vm_stack_new_page(ZEND_VM_STACK_PAGE_SIZE, NULL);
+	EG(vm_stack)->top++;
+	EG(vm_stack_top) = EG(vm_stack)->top;
+	EG(vm_stack_end) = EG(vm_stack)->end;
+}
+
+ZEND_API void zend_vm_stack_destroy(TSRMLS_D)
+{
+	zend_vm_stack stack = EG(vm_stack);
+
+	while (stack != NULL) {
+		zend_vm_stack p = stack->prev;
+		efree(stack);
+		stack = p;
+	}
+}
+
+ZEND_API void* zend_vm_stack_extend(size_t size TSRMLS_DC)
+{
+    zend_vm_stack stack;
+    void *ptr;
+
+    stack = EG(vm_stack);
+    stack->top = EG(vm_stack_top);
+	EG(vm_stack) = stack = zend_vm_stack_new_page(
+		EXPECTED(size < ZEND_VM_STACK_FREE_PAGE_SIZE) ?
+			ZEND_VM_STACK_PAGE_SIZE : ZEND_VM_STACK_PAGE_ALIGNED_SIZE(size), 
+		stack);
+	ptr = stack->top;
+	EG(vm_stack_top) = (void*)(((char*)ptr) + size);
+	EG(vm_stack_end) = stack->end;
+	return ptr;
+}
+
 ZEND_API zval* zend_get_compiled_variable_value(const zend_execute_data *execute_data, uint32_t var)
 {
 	return EX_VAR(var);
@@ -1610,11 +1665,13 @@ ZEND_API zend_execute_data *zend_create_generator_execute_data(zend_execute_data
 	uint32_t num_args = call->num_args;
 	size_t stack_size = (ZEND_CALL_FRAME_SLOT + MAX(op_array->last_var + op_array->T, num_args)) * sizeof(zval);
 
-	EG(argument_stack) = zend_vm_stack_new_page(
+	EG(vm_stack) = zend_vm_stack_new_page(
 		EXPECTED(stack_size < ZEND_VM_STACK_FREE_PAGE_SIZE) ?
 			ZEND_VM_STACK_PAGE_SIZE :
 			ZEND_VM_STACK_PAGE_ALIGNED_SIZE(stack_size),
 		NULL);
+	EG(vm_stack_top) = EG(vm_stack)->top;
+	EG(vm_stack_end) = EG(vm_stack)->end;
 
 	execute_data = zend_vm_stack_push_call_frame(
 		VM_FRAME_TOP_FUNCTION,
@@ -1662,12 +1719,10 @@ static zend_always_inline zend_bool zend_is_by_ref_func_arg_fetch(const zend_op
 static zend_execute_data *zend_vm_stack_copy_call_frame(zend_execute_data *call, uint32_t passed_args, uint32_t additional_args TSRMLS_DC) /* {{{ */
 {
 	zend_execute_data *new_call;
-	int used_stack = (EG(argument_stack)->top - (zval*)call) + additional_args;
+	int used_stack = (EG(vm_stack_top) - (zval*)call) + additional_args;
 		
 	/* copy call frame into new stack segment */
-	zend_vm_stack_extend(used_stack * sizeof(zval) TSRMLS_CC);
-	new_call = (zend_execute_data*)EG(argument_stack)->top;
-	EG(argument_stack)->top += used_stack;		
+	new_call = zend_vm_stack_extend(used_stack * sizeof(zval) TSRMLS_CC);
 	*new_call = *call;
 	if (passed_args) {
 		zval *src = ZEND_CALL_ARG(call, 1);
@@ -1681,13 +1736,13 @@ static zend_execute_data *zend_vm_stack_copy_call_frame(zend_execute_data *call,
 	}
 
 	/* delete old call_frame from previous stack segment */
-	EG(argument_stack)->prev->top = (zval*)call;
+	EG(vm_stack)->prev->top = (zval*)call;
 
 	/* delete previous stack segment if it becames empty */
-	if (UNEXPECTED(EG(argument_stack)->prev->top == ZEND_VM_STACK_ELEMETS(EG(argument_stack)->prev))) {
-		zend_vm_stack r = EG(argument_stack)->prev;
+	if (UNEXPECTED(EG(vm_stack)->prev->top == ZEND_VM_STACK_ELEMETS(EG(vm_stack)->prev))) {
+		zend_vm_stack r = EG(vm_stack)->prev;
 
-		EG(argument_stack)->prev = r->prev;
+		EG(vm_stack)->prev = r->prev;
 		efree(r);
 	}
 
@@ -1697,8 +1752,8 @@ static zend_execute_data *zend_vm_stack_copy_call_frame(zend_execute_data *call,
 
 static zend_always_inline void zend_vm_stack_extend_call_frame(zend_execute_data **call, uint32_t passed_args, uint32_t additional_args TSRMLS_DC) /* {{{ */
 {
-	if (EXPECTED(EG(argument_stack)->end - EG(argument_stack)->top > additional_args)) {
-		EG(argument_stack)->top += additional_args;
+	if (EXPECTED(EG(vm_stack_end) - EG(vm_stack_top) > additional_args)) {
+		EG(vm_stack_top) += additional_args;
 	} else {
 		*call = zend_vm_stack_copy_call_frame(*call, passed_args, additional_args TSRMLS_CC);
 	}
diff --git a/Zend/zend_execute.h b/Zend/zend_execute.h
index 370c71de74..bc913f926d 100644
--- a/Zend/zend_execute.h
+++ b/Zend/zend_execute.h
@@ -135,10 +135,6 @@ ZEND_API int zval_update_constant_no_inline_change(zval *pp, zend_class_entry *s
 ZEND_API int zval_update_constant_ex(zval *pp, zend_bool inline_change, zend_class_entry *scope TSRMLS_DC);
 
 /* dedicated Zend executor functions - do not use! */
-#define ZEND_VM_STACK_PAGE_SLOTS (16 * 1024) /* should be a power of 2 */
-
-#define ZEND_VM_STACK_PAGE_SIZE  (ZEND_VM_STACK_PAGE_SLOTS * sizeof(zval))
-
 struct _zend_vm_stack {
 	zval *top;
 	zval *end;
@@ -148,58 +144,21 @@ struct _zend_vm_stack {
 #define ZEND_VM_STACK_HEADER_SLOTS \
 	((ZEND_MM_ALIGNED_SIZE(sizeof(struct _zend_vm_stack)) + ZEND_MM_ALIGNED_SIZE(sizeof(zval)) - 1) / ZEND_MM_ALIGNED_SIZE(sizeof(zval)))
 
-#define ZEND_VM_STACK_FREE_PAGE_SIZE \
-	((ZEND_VM_STACK_PAGE_SLOTS - ZEND_VM_STACK_HEADER_SLOTS) * sizeof(zval))
-
-#define ZEND_VM_STACK_PAGE_ALIGNED_SIZE(size) \
-	(((size) + (ZEND_VM_STACK_FREE_PAGE_SIZE - 1)) & ~ZEND_VM_STACK_PAGE_SIZE)
-
 #define ZEND_VM_STACK_ELEMETS(stack) \
 	(((zval*)(stack)) + ZEND_VM_STACK_HEADER_SLOTS)
 
-static zend_always_inline zend_vm_stack zend_vm_stack_new_page(size_t size, zend_vm_stack prev) {
-	zend_vm_stack page = (zend_vm_stack)emalloc(size);
-
-	page->top = ZEND_VM_STACK_ELEMETS(page);
-	page->end = (zval*)((char*)page + size);
-	page->prev = prev;
-	return page;
-}
-
-static zend_always_inline void zend_vm_stack_init(TSRMLS_D)
-{
-	EG(argument_stack) = zend_vm_stack_new_page(ZEND_VM_STACK_PAGE_SIZE, NULL);
-	EG(argument_stack)->top++;
-}
-
-static zend_always_inline void zend_vm_stack_destroy(TSRMLS_D)
-{
-	zend_vm_stack stack = EG(argument_stack);
-
-	while (stack != NULL) {
-		zend_vm_stack p = stack->prev;
-		efree(stack);
-		stack = p;
-	}
-}
-
-static zend_always_inline void zend_vm_stack_extend(size_t size TSRMLS_DC)
-{
-	EG(argument_stack) = zend_vm_stack_new_page(
-		EXPECTED(size < ZEND_VM_STACK_FREE_PAGE_SIZE) ?
-			ZEND_VM_STACK_PAGE_SIZE : ZEND_VM_STACK_PAGE_ALIGNED_SIZE(size), 
-		EG(argument_stack));
-}
+ZEND_API void zend_vm_stack_init(TSRMLS_D);
+ZEND_API void zend_vm_stack_destroy(TSRMLS_D);
+ZEND_API void* zend_vm_stack_extend(size_t size TSRMLS_DC);
 
 static zend_always_inline zval* zend_vm_stack_alloc(size_t size TSRMLS_DC)
 {
-	char *top = (char*)EG(argument_stack)->top;
+	char *top = (char*)EG(vm_stack_top);
 
-	if (UNEXPECTED(size > (size_t)(((char*)EG(argument_stack)->end) - top))) {
-		zend_vm_stack_extend(size TSRMLS_CC);
-		top = (char*)EG(argument_stack)->top;
+	if (UNEXPECTED(size > (size_t)(((char*)EG(vm_stack_end)) - top))) {
+		return (zval*)zend_vm_stack_extend(size TSRMLS_CC);
 	}
-	EG(argument_stack)->top = (zval*)(top + size);
+	EG(vm_stack_top) = (zval*)(top + size);
 	return (zval*)top;
 }
 
@@ -252,12 +211,16 @@ static zend_always_inline void zend_vm_stack_free_args(zend_execute_data *call T
 
 static zend_always_inline void zend_vm_stack_free_call_frame(zend_execute_data *call TSRMLS_DC)
 {
-	zend_vm_stack p = EG(argument_stack);
+	zend_vm_stack p = EG(vm_stack);
 	if (UNEXPECTED(ZEND_VM_STACK_ELEMETS(p) == (zval*)call)) {
-		EG(argument_stack) = p->prev;
+		zend_vm_stack prev = p->prev;
+
+		EG(vm_stack_top) = prev->top;
+		EG(vm_stack_end) = prev->end;
+		EG(vm_stack) = prev;
 		efree(p);
 	} else {
-		p->top = (zval*)call;
+		EG(vm_stack_top) = (zval*)call;
 	}
 }
 
diff --git a/Zend/zend_generators.c b/Zend/zend_generators.c
index 6ffd264a1d..c0d087fd99 100644
--- a/Zend/zend_generators.c
+++ b/Zend/zend_generators.c
@@ -217,8 +217,9 @@ ZEND_API void zend_generator_create_zval(zend_execute_data *call, zend_op_array
 	zend_generator *generator;
 	zend_execute_data *current_execute_data;
 	zend_execute_data *execute_data;
-	zend_vm_stack current_stack = EG(argument_stack);
+	zend_vm_stack current_stack = EG(vm_stack);
 
+	current_stack->top = EG(vm_stack_top);
 	/* Create a clone of closure, because it may be destroyed */
 	if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
 		zend_op_array *op_array_copy = (zend_op_array*)emalloc(sizeof(zend_op_array));
@@ -259,8 +260,11 @@ ZEND_API void zend_generator_create_zval(zend_execute_data *call, zend_op_array
 	generator = (zend_generator *) Z_OBJ_P(return_value);
 	execute_data->prev_execute_data = NULL;
 	generator->execute_data = execute_data;
-	generator->stack = EG(argument_stack);
-	EG(argument_stack) = current_stack;
+	generator->stack = EG(vm_stack);
+	generator->stack->top = EG(vm_stack_top);
+	EG(vm_stack_top) = current_stack->top;
+	EG(vm_stack_end) = current_stack->end;
+	EG(vm_stack) = current_stack;
 
 	/* EX(return_value) keeps pointer to zend_object (not a real zval) */
 	execute_data->return_value = (zval*)generator;
@@ -293,12 +297,15 @@ ZEND_API void zend_generator_resume(zend_generator *generator TSRMLS_DC) /* {{{
 		/* Backup executor globals */
 		zend_execute_data *original_execute_data = EG(current_execute_data);
 		zend_class_entry *original_scope = EG(scope);
-		zend_vm_stack original_stack = EG(argument_stack);
+		zend_vm_stack original_stack = EG(vm_stack);
 
+		original_stack->top = EG(vm_stack_top);
 		/* Set executor globals */
 		EG(current_execute_data) = generator->execute_data;
 		EG(scope) = generator->execute_data->scope;
-		EG(argument_stack) = generator->stack;
+		EG(vm_stack_top) = generator->stack->top;
+		EG(vm_stack_end) = generator->stack->end;
+		EG(vm_stack) = generator->stack;
 
 		/* We want the backtrace to look as if the generator function was
 		 * called from whatever method we are current running (e.g. next()).
@@ -319,7 +326,9 @@ ZEND_API void zend_generator_resume(zend_generator *generator TSRMLS_DC) /* {{{
 		/* Restore executor globals */
 		EG(current_execute_data) = original_execute_data;
 		EG(scope) = original_scope;
-		EG(argument_stack) = original_stack;
+		EG(vm_stack_top) = original_stack->top;
+		EG(vm_stack_end) = original_stack->end;
+		EG(vm_stack) = original_stack;
 
 		/* If an exception was thrown in the generator we have to internally
 		 * rethrow it in the parent scope. */
diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h
index 8a42fbe1f2..3e9fba56f1 100644
--- a/Zend/zend_globals.h
+++ b/Zend/zend_globals.h
@@ -171,6 +171,11 @@ struct _zend_executor_globals {
 	HashTable *class_table;		/* class table */
 	HashTable *zend_constants;	/* constants table */
 
+	zval          *vm_stack_top;
+	zval          *vm_stack_end;
+	zend_vm_stack  vm_stack;
+
+	struct _zend_execute_data *current_execute_data;
 	zend_class_entry *scope;
 
 	zend_long precision;
@@ -192,8 +197,6 @@ struct _zend_executor_globals {
 	HashTable regular_list;
 	HashTable persistent_list;
 
-	zend_vm_stack argument_stack;
-
 	int user_error_handler_error_reporting;
 	zval user_error_handler;
 	zval user_exception_handler;
@@ -218,8 +221,6 @@ struct _zend_executor_globals {
 	const zend_op *opline_before_exception;
 	zend_op exception_op[3];
 
-	struct _zend_execute_data *current_execute_data;
-
 	struct _zend_module_entry *current_module;
 
 	zend_property_info std_property_info;
diff --git a/ext/soap/soap.c b/ext/soap/soap.c
index 7b3cee80cd..2b37622a7a 100644
--- a/ext/soap/soap.c
+++ b/ext/soap/soap.c
@@ -93,7 +93,7 @@ static void soap_error_handler(int error_num, const char *error_filename, const
 	int _old_soap_version = SOAP_GLOBAL(soap_version);\
 	zend_bool _old_in_compilation = CG(in_compilation); \
 	zend_execute_data *_old_current_execute_data = EG(current_execute_data); \
-	zval *_old_stack_top = EG(argument_stack)->top; \
+	zval *_old_stack_top = EG(vm_stack_top); \
 	int _bailout = 0;\
 	SOAP_GLOBAL(use_soap_error_handler) = 1;\
 	SOAP_GLOBAL(error_code) = "Client";\
@@ -108,15 +108,16 @@ static void soap_error_handler(int error_num, const char *error_filename, const
 		    !instanceof_function(EG(exception)->ce, soap_fault_class_entry TSRMLS_CC)) {\
 			_bailout = 1;\
 		}\
-		if (_old_stack_top != EG(argument_stack)->top) { \
-			while (EG(argument_stack)->prev != NULL && \
-			       ((char*)_old_stack_top < (char*)EG(argument_stack) || \
-			        (char*) _old_stack_top > (char*)EG(argument_stack)->end)) { \
-				zend_vm_stack tmp = EG(argument_stack)->prev; \
-				efree(EG(argument_stack)); \
-				EG(argument_stack) = tmp; \
+		if (_old_stack_top != EG(vm_stack_top)) { \
+			while (EG(vm_stack)->prev != NULL && \
+			       ((char*)_old_stack_top < (char*)EG(vm_stack) || \
+			        (char*) _old_stack_top > (char*)EG(vm_stack)->end)) { \
+				zend_vm_stack tmp = EG(vm_stack)->prev; \
+				efree(EG(vm_stack)); \
+				EG(vm_stack) = tmp; \
+				EG(vm_stack_end) = tmp->end; \
 			} \
-			EG(argument_stack)->top = _old_stack_top; \
+			EG(vm_stack)->top = _old_stack_top; \
 		} \
 	} zend_end_try();\
 	SOAP_GLOBAL(use_soap_error_handler) = _old_handler;\