Properly patch jmp tables

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index e2e85f3431904aaee8c46c632d8fb0692b5fb660..90bed235d26dcb1c9fb450ca1b18a8bd1a0acdb1 100644 (file)
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -2925,7 +2925,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
                                        case ZEND_SWITCH_LONG:
                                        case ZEND_SWITCH_STRING:
                                        case ZEND_MATCH:
-                                               if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL)) {
+                                               if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
                                                        goto jit_failure;
                                                }
                                                goto done;

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index 7dd8786fde2e1873979befe6ff3d23acd4e8208e..b40553b03dbde9ba662e517fc3581c83ea28f35a 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -350,6 +350,7 @@ typedef struct _zend_jit_trace_info {
        uint32_t                  stack_map_size;
        uint32_t                  flags;         /* See ZEND_JIT_TRACE_... defines above */
        uint32_t                  polymorphism;  /* Counter of polymorphic calls */
+       uint32_t                  jmp_table_size;/* number of jmp_table slots */
        const zend_op            *opline;        /* first opline */
        const void               *code_start;    /* address of native code */
        zend_jit_trace_exit_info *exit_info;     /* info about side exits */

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index 96fa37abb79a56f2082bdacf0fa380ae5522cdd2..24b4e7cd81905dfe18dc8fc6eede3bf639707f86 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -4163,7 +4163,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                        case ZEND_SWITCH_LONG:
                                        case ZEND_SWITCH_STRING:
                                        case ZEND_MATCH:
-                                               if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1)) {
+                                               if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
                                                        goto jit_failure;
                                                }
                                                goto done;
@@ -4827,6 +4827,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace
                t->stack_map_size = 0;
                t->flags = 0;
                t->polymorphism = 0;
+               t->jmp_table_size = 0;
                t->opline = trace_buffer[1].opline;
                t->exit_info = exit_info;
                t->stack_map = NULL;
@@ -5353,6 +5354,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
                        zend_jit_link_side_trace(
                                zend_jit_traces[trace_num].code_start,
                                zend_jit_traces[trace_num].code_size,
+                               zend_jit_traces[trace_num].jmp_table_size,
                                exit_num,
                                handler);
                }
@@ -5421,6 +5423,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
                t->stack_map_size = 0;
                t->flags = 0;
                t->polymorphism = polymorphism;
+               t->jmp_table_size = 0;
                t->opline = NULL;
                t->exit_info = exit_info;
                t->stack_map = NULL;
@@ -5464,6 +5467,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
                        zend_jit_link_side_trace(
                                zend_jit_traces[parent_num].code_start,
                                zend_jit_traces[parent_num].code_size,
+                               zend_jit_traces[parent_num].jmp_table_size,
                                exit_num,
                                handler);
 

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 32bd3b46774e27dd41719f71e0b533355c233063..520a3facfa633776a6d5b9206ceb1339430f0fd1 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -153,7 +153,7 @@ static size_t tsrm_tls_offset;
 |.globals zend_lb
 static void* dasm_labels[zend_lb_MAX];
 
-|.section code, cold_code
+|.section code, cold_code, jmp_table
 
 #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff)
 
@@ -3093,12 +3093,26 @@ mrm:
 typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t);
 typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t);
 
-static int zend_jit_patch(const void *code, size_t size, const void *from_addr, const void *to_addr)
+static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
 {
        int ret = 0;
-       uint8_t *p = (uint8_t*)code;
-       uint8_t *end = p + size - 5;
+       uint8_t *p, *end;
 
+       if (jmp_table_size) {
+               const void **jmp_slot = (const void **)((char*)code + size);
+
+               size -= jmp_table_size * sizeof(void*);
+               do {
+                       jmp_slot--;
+                       if (*jmp_slot == from_addr) {
+                               *jmp_slot = to_addr;
+                               ret++;
+                       }
+               } while (--jmp_table_size);
+       }
+
+       p = (uint8_t*)code;
+       end = p + size - 5;
        while (p < end) {
                if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) {
                        *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6));
@@ -3115,9 +3129,9 @@ static int zend_jit_patch(const void *code, size_t size, const void *from_addr,
        return ret;
 }
 
-static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t exit_num, const void *addr)
+static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
 {
-       return zend_jit_patch(code, size, zend_jit_trace_get_exit_addr(exit_num), addr);
+       return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
 }
 
 static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t)
@@ -11782,7 +11796,7 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze
        return 1;
 }
 
-static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace)
+static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
 {
        HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
        const zend_op *next_opline = NULL;
@@ -11920,9 +11934,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |.else
                                        |       jmp aword [FCARG2a * 4 + >4]
                                        |.endif
-                                       |.cold_code
+                                       |.jmp_table
                                        |.align aword
                                        |4:
+                                       if (trace_info) {
+                                               trace_info->jmp_table_size += count;
+                                       }
                                        p = jumptable->arData;
                                        do {
                                                if (Z_TYPE(p->val) == IS_UNDEF) {
@@ -11981,9 +11998,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |.else
                                        |       jmp aword [r0 + >4]
                                        |.endif
-                                       |.cold_code
+                                       |.jmp_table
                                        |.align aword
                                        |4:
+                                       if (trace_info) {
+                                               trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
+                                       }
                                        ZEND_HASH_FOREACH_VAL(jumptable, val) {
                                                target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
                                                if (!next_opline) {
@@ -12063,9 +12083,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                |.else
                                |       jmp aword [r0 + >4]
                                |.endif
-                               |.cold_code
+                               |.jmp_table
                                |.align aword
                                |4:
+                               if (trace_info) {
+                                       trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
+                               }
                                ZEND_HASH_FOREACH_VAL(jumptable, val) {
                                        target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
                                        if (!next_opline) {