Preallocate stack space for JIT in execute_ex() to eliminate JIT prologue/epilogue.

diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c
index c58e1fa228e17c981705af17cd22164a0a1c0e5c..fb35b8a1162ce0241a010f03eb184bebf0e62f9b 100644 (file)
--- a/Zend/zend_execute.c
+++ b/Zend/zend_execute.c
@@ -4757,6 +4757,11 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
                } \
        } while (0)
 
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+/* This callback disables optimization of "vm_stack_data" variable in VM */
+void (*zend_touch_vm_stack_data)(void *vm_stack_data) = NULL;
+#endif
+
 #include "zend_vm_execute.h"
 
 ZEND_API zend_result zend_set_user_opcode_handler(zend_uchar opcode, user_opcode_handler_t handler)

diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h
index 0d14ff3bab26d75281e0133fe7cf8a29f68f0e39..60d725b3654be0537ef8ea49db12ea072a5cecab 100644 (file)
--- a/Zend/zend_vm_execute.h
+++ b/Zend/zend_vm_execute.h
@@ -50826,15 +50826,34 @@ ZEND_API void execute_ex(zend_execute_data *ex)
 {
        DCL_OPLINE
 
+#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)
+       struct {
 #ifdef ZEND_VM_IP_GLOBAL_REG
-       const zend_op *orig_opline = opline;
+               const zend_op *orig_opline;
 #endif
 #ifdef ZEND_VM_FP_GLOBAL_REG
-       zend_execute_data *orig_execute_data = execute_data;
+               zend_execute_data *orig_execute_data;
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+               char hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];
+#endif
+#endif
+       } vm_stack_data;
+#endif
+#ifdef ZEND_VM_IP_GLOBAL_REG
+       vm_stack_data.orig_opline = opline;
+#endif
+#ifdef ZEND_VM_FP_GLOBAL_REG
+       vm_stack_data.orig_execute_data = execute_data;
        execute_data = ex;
 #else
        zend_execute_data *execute_data = ex;
 #endif
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+       memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);
+       if (zend_touch_vm_stack_data) {
+               zend_touch_vm_stack_data(&vm_stack_data);
+       }
+#endif
 
 #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)
        if (UNEXPECTED(execute_data == NULL)) {
@@ -58788,10 +58807,10 @@ zend_leave_helper_SPEC_LABEL:
                                HYBRID_BREAK();
                        HYBRID_CASE(HYBRID_HALT):
 #ifdef ZEND_VM_FP_GLOBAL_REG
-                               execute_data = orig_execute_data;
+                               execute_data = vm_stack_data.orig_execute_data;
 #endif
 #ifdef ZEND_VM_IP_GLOBAL_REG
-                               opline = orig_opline;
+                               opline = vm_stack_data.orig_opline;
 #endif
                                return;
                        HYBRID_DEFAULT:
@@ -58800,9 +58819,9 @@ zend_leave_helper_SPEC_LABEL:
                                HYBRID_BREAK(); /* Never reached */
 #else
 #ifdef ZEND_VM_FP_GLOBAL_REG
-                       execute_data = orig_execute_data;
+                       execute_data = vm_stack_data.orig_execute_data;
 # ifdef ZEND_VM_IP_GLOBAL_REG
-                       opline = orig_opline;
+                       opline = vm_stack_data.orig_opline;
 # endif
                        return;
 #else
@@ -58811,7 +58830,7 @@ zend_leave_helper_SPEC_LABEL:
                                ZEND_VM_LOOP_INTERRUPT_CHECK();
                        } else {
 # ifdef ZEND_VM_IP_GLOBAL_REG
-                               opline = orig_opline;
+                               opline = vm_stack_data.orig_opline;
 # endif
                                return;
                        }

diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php
index 4958189ce4cf42ee0bd5798c17a45962644ed8b9..46030cad0983fc65482741ae727b85ae9ba2b28f 100755 (executable)
--- a/Zend/zend_vm_gen.php
+++ b/Zend/zend_vm_gen.php
@@ -1780,10 +1780,10 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
         case ZEND_VM_KIND_HYBRID:
             out($f,"\t\t\tHYBRID_CASE(HYBRID_HALT):\n");
             out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
-            out($f,"\t\t\t\texecute_data = orig_execute_data;\n");
+            out($f,"\t\t\t\texecute_data = vm_stack_data.orig_execute_data;\n");
             out($f,"#endif\n");
             out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
-            out($f,"\t\t\t\topline = orig_opline;\n");
+            out($f,"\t\t\t\topline = vm_stack_data.orig_opline;\n");
             out($f,"#endif\n");
             out($f,"\t\t\t\treturn;\n");
             out($f,"\t\t\tHYBRID_DEFAULT:\n");
@@ -2065,15 +2065,34 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
                         out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
                         out($f,"#endif\n");
                     } else {
+                        out($f,"#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)\n");
+                        out($f,$m[1]."struct {\n");
                         out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
-                        out($f,$m[1]."const zend_op *orig_opline = opline;\n");
+                        out($f,$m[1]."\tconst zend_op *orig_opline;\n");
                         out($f,"#endif\n");
                         out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
-                        out($f,$m[1]."zend_execute_data *orig_execute_data = execute_data;\n");
+                        out($f,$m[1]."\tzend_execute_data *orig_execute_data;\n");
+                        out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
+                        out($f,$m[1]."\tchar hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];\n");
+                        out($f,"#endif\n");
+                        out($f,"#endif\n");
+                        out($f,$m[1]."} vm_stack_data;\n");
+                        out($f,"#endif\n");
+                        out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
+                        out($f,$m[1]."vm_stack_data.orig_opline = opline;\n");
+                        out($f,"#endif\n");
+                        out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
+                        out($f,$m[1]."vm_stack_data.orig_execute_data = execute_data;\n");
                         out($f,$m[1]."execute_data = ex;\n");
                         out($f,"#else\n");
                         out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
                         out($f,"#endif\n");
+                        out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
+                        out($f,$m[1]."memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n");
+                        out($f,$m[1]."if (zend_touch_vm_stack_data) {\n");
+                        out($f,$m[1]."\tzend_touch_vm_stack_data(&vm_stack_data);\n");
+                        out($f,$m[1]."}\n");
+                        out($f,"#endif\n");
                     }
                     break;
                 case "INTERNAL_LABELS":
@@ -2159,9 +2178,9 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
                         }
                         out($f,
                                 "#ifdef ZEND_VM_FP_GLOBAL_REG\n" .
-                                $m[1]."execute_data = orig_execute_data;\n" .
+                                $m[1]."execute_data = vm_stack_data.orig_execute_data;\n" .
                                 "# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
-                                $m[1]."opline = orig_opline;\n" .
+                                $m[1]."opline = vm_stack_data.orig_opline;\n" .
                                 "# endif\n" .
                                 $m[1]."return;\n" .
                                 "#else\n" .
@@ -2170,7 +2189,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
                                 $m[1]."\tZEND_VM_LOOP_INTERRUPT_CHECK();\n".
                                 $m[1]."} else {\n" .
                                 "# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
-                                $m[1]."\topline = orig_opline;\n" .
+                                $m[1]."\topline = vm_stack_data.orig_opline;\n" .
                                 "# endif\n".
                                 $m[1]."\treturn;\n".
                                 $m[1]."}\n".
@@ -2578,6 +2597,12 @@ function gen_vm($def, $skel) {
         fputs($f, "#define ZEND_VM_KIND\t\t" . $GLOBALS["vm_kind_name"][ZEND_VM_KIND] . "\n");
     }
     fputs($f, "\n");
+    fputs($f, "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n");
+    fputs($f, "# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))\n");
+    fputs($f, "#  define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n");
+    fputs($f, "# endif\n");
+    fputs($f, "#endif\n");
+    fputs($f, "\n");
     foreach($vm_op_flags as $name => $val) {
         fprintf($f, "#define %-24s 0x%08x\n", $name, $val);
     }

diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h
index df5a14799953a36a0a2f5d943a4b239a38762299..584a44f5ae1a59bbe40737be3bf24c579d7623a6 100644 (file)
--- a/Zend/zend_vm_opcodes.h
+++ b/Zend/zend_vm_opcodes.h
@@ -34,6 +34,12 @@
 # define ZEND_VM_KIND          ZEND_VM_KIND_CALL
 #endif
 
+#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)
+# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))
+#  define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16
+# endif
+#endif
+
 #define ZEND_VM_OP_SPEC          0x00000001
 #define ZEND_VM_OP_CONST         0x00000002
 #define ZEND_VM_OP_TMPVAR        0x00000004

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index ca06f7bdea0e84e1aa20ed1c894296a260ad0ad5..9d518ae9e5b8550371912b0aed00584f7f8c97be 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -171,8 +171,21 @@ static void* dasm_labels[zend_lb_MAX];
 
 #define BP_JIT_IS 6
 
+
 #define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX)
 
+|.macro ADD_HYBRID_SPAD
+||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+|              add r4, HYBRID_SPAD
+||#endif
+|.endmacro
+
+|.macro SUB_HYBRID_SPAD
+||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+|              sub r4, HYBRID_SPAD
+||#endif
+|.endmacro
+
 |.macro LOAD_ADDR, reg, addr
 |      .if X64
 ||             if (IS_32BIT(addr)) {
@@ -1752,7 +1765,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
        }
        |       //ZEND_VM_CONTINUE()
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       JMP_IP
        } else if (GCC_GLOBAL_REGS) {
                |       add r4, SPAD // stack alignment
@@ -1774,7 +1787,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst)
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
                const void *handler = zend_get_opcode_handler_func(EG(exception_op));
 
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       EXT_CALL handler, r0
                |       JMP_IP
        } else {
@@ -1827,11 +1840,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst)
                |       test FCARG1d, ZEND_CALL_TOP
                |       jnz >1
                |       EXT_CALL zend_jit_leave_nested_func_helper, r0
-               |       add r4, HYBRID_SPAD // stack alignment
+               |       ADD_HYBRID_SPAD
                |       JMP_IP
                |1:
                |       EXT_CALL zend_jit_leave_top_func_helper, r0
-               |       add r4, HYBRID_SPAD // stack alignment
+               |       ADD_HYBRID_SPAD
                |       JMP_IP
        } else {
                if (GCC_GLOBAL_REGS) {
@@ -2413,7 +2426,7 @@ static int zend_jit_trace_halt_stub(dasm_State **Dst)
 {
        |->trace_halt:
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       EXT_JMP zend_jit_halt_op->handler, r0
        } else if (GCC_GLOBAL_REGS) {
                |       add r4, SPAD // stack alignment
@@ -2505,7 +2518,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst)
        |       LOAD_IP
 
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       JMP_IP
        } else if (GCC_GLOBAL_REGS) {
                |       add r4, SPAD // stack alignment
@@ -2531,7 +2544,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst)
        |       jne ->interrupt_handler
 
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       mov r0, EX->func
                |       mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
                |       mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
@@ -2566,7 +2579,7 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst)
        |->trace_escape:
        |
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       JMP_IP
        } else if (GCC_GLOBAL_REGS) {
                |       add r4, SPAD // stack alignment
@@ -2608,7 +2621,7 @@ static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
        |->context_threaded_call:
        |       pop r0
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                |       jmp aword [IP]
        } else if (GCC_GLOBAL_REGS) {
                |       add r4, SPAD // stack alignment
@@ -2990,7 +3003,7 @@ static int zend_jit_align_func(dasm_State **Dst)
 static int zend_jit_prologue(dasm_State **Dst)
 {
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       sub r4, HYBRID_SPAD
+               |       SUB_HYBRID_SPAD
        } else if (GCC_GLOBAL_REGS) {
                |       sub r4, SPAD // stack alignment
        } else {
@@ -3344,10 +3357,13 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t,
        /* Skip prologue. */
        // TODO: don't hardcode this ???
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+               prologue_size = 0;
+#elif defined(__x86_64__) || defined(_M_X64)
                // sub r4, HYBRID_SPAD
-#if defined(__x86_64__) || defined(_M_X64)
                prologue_size = 4;
 #else
+               // sub r4, HYBRID_SPAD
                prologue_size = 3;
 #endif
        } else if (GCC_GLOBAL_REGS) {
@@ -3387,7 +3403,7 @@ static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler)
        |       jmp ->trace_escape
 #else
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
                if (!original_handler) {
                        |       JMP_IP
                } else {
@@ -3652,13 +3668,13 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
                        /* Use inlined HYBRID VM handler */
                        const void *handler = opline->handler;
 
-                       |       add r4, HYBRID_SPAD
+                       |       ADD_HYBRID_SPAD
                        |       EXT_JMP handler, r0
                } else {
                        const void *handler = zend_get_opcode_handler_func(opline);
 
                        |       EXT_CALL handler, r0
-                       |       add r4, HYBRID_SPAD
+                       |       ADD_HYBRID_SPAD
                        |       JMP_IP
                }
        } else {
@@ -10044,7 +10060,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend
                        }
 #else
                        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-                               |       add r4, HYBRID_SPAD
+                               |       ADD_HYBRID_SPAD
                                |       JMP_IP
                        } else if (GCC_GLOBAL_REGS) {
                                |       add r4, SPAD // stack alignment
@@ -11283,7 +11299,11 @@ static int zend_jit_leave_func(dasm_State          **Dst,
                                trace_info->flags |= ZEND_JIT_TRACE_LOOP;
                                |       CMP_IP next_opline
                                |       je =>0 // LOOP
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+                               |       JMP_IP
+#else
                                |       jmp ->trace_escape
+#endif
                        } else {
                                |       CMP_IP next_opline
                                |       jne ->trace_escape
@@ -11313,7 +11333,7 @@ static int zend_jit_leave_func(dasm_State          **Dst,
        }
 
        if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
-               |       add r4, HYBRID_SPAD
+               |       ADD_HYBRID_SPAD
 #ifdef CONTEXT_THREADED_JIT
                |       push aword [IP]
                |       ret