Tracing JIT for SWITCH instructions

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index e8c82f3886e696f83eaeb8169cf098796dfdce7f..7c352f080240e33cecb9d7f85167f13a72d5165d 100644 (file)
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -2861,7 +2861,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
                                        case ZEND_SWITCH_LONG:
                                        case ZEND_SWITCH_STRING:
                                        case ZEND_MATCH:
-                                               if (!zend_jit_switch(&dasm_state, opline, op_array, ssa)) {
+                                               if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL)) {
                                                        goto jit_failure;
                                                }
                                                goto done;

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index 7109c690324021bbf458a843ab7bbe63dad1f01e..655bcbc2b286b125ded06855ff539271a35db81d 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -3950,15 +3950,13 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                                        goto jit_failure;
                                                }
                                                goto done;
-#if 0
                                        case ZEND_SWITCH_LONG:
                                        case ZEND_SWITCH_STRING:
                                        case ZEND_MATCH:
-                                               if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa)) {
+                                               if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1)) {
                                                        goto jit_failure;
                                                }
                                                goto done;
-#endif
                                        case ZEND_INIT_METHOD_CALL:
                                        case ZEND_INIT_DYNAMIC_CALL:
                                                if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index bedd4275a7ddf3a3d10d87d6ee9c22562b0ab5b8..2bc4da07a4a5b38310d1d5772624a625cc9e186f 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -11401,9 +11401,16 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze
        return 1;
 }
 
-static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace)
 {
        HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
+       const zend_op *next_opline = NULL;
+
+       if (trace) {
+               ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
+               ZEND_ASSERT(trace->opline != NULL);
+               next_opline = trace->opline;
+       }
 
        // TODO: Implement for match instructions
        if (opline->opcode == ZEND_MATCH) {
@@ -11420,22 +11427,34 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                if (opline->opcode == ZEND_SWITCH_LONG) {
                        if (Z_TYPE_P(zv) == IS_LONG) {
                                jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
-                               if (jump_zv != NULL) {
-                                       b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
+                               if (next_opline) {
+                                       const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
+
+                                       ZEND_ASSERT(target == next_opline);
                                } else {
-                                       b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+                                       if (jump_zv != NULL) {
+                                               b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
+                                       } else {
+                                               b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+                                       }
+                                       |       jmp =>b
                                }
-                               |       jmp =>b
                        }
                } else if (opline->opcode == ZEND_SWITCH_STRING) {
                        if (Z_TYPE_P(zv) == IS_STRING) {
                                jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1);
-                               if (jump_zv != NULL) {
-                                       b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
+                               if (next_opline) {
+                                       const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
+
+                                       ZEND_ASSERT(target == next_opline);
                                } else {
-                                       b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+                                       if (jump_zv != NULL) {
+                                               b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
+                                       } else {
+                                               b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+                                       }
+                                       |       jmp =>b
                                }
-                               |       jmp =>b
                        }
                } else {
                        ZEND_UNREACHABLE();
@@ -11444,8 +11463,26 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
                uint32_t op1_info = OP1_INFO();
                zend_jit_addr op1_addr = OP1_ADDR();
-               int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+               const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
+               const zend_op *target;
+               int default_b = ssa->cfg.map[default_opline - op_array->opcodes];
+               int b;
                zval *val;
+               int32_t exit_point;
+               const void *fallback_label = NULL;
+               const void *default_label = NULL;
+               const void *exit_addr;
+
+               if (next_opline) {
+                       if (next_opline != opline + 1) {
+                               exit_point = zend_jit_trace_get_exit_point(opline, opline + 1, NULL, 0);
+                               fallback_label = zend_jit_trace_get_exit_addr(exit_point);
+                       }
+                       if (next_opline != default_opline) {
+                               exit_point = zend_jit_trace_get_exit_point(opline, default_opline, NULL, 0);
+                               default_label = zend_jit_trace_get_exit_addr(exit_point);
+                       }
+               }
 
                if (opline->opcode == ZEND_SWITCH_LONG) {
                        if (op1_info & MAY_BE_LONG) {
@@ -11455,16 +11492,28 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |.cold_code
                                        |1:
                                        |       // ZVAL_DEREF(op)
-                                       |       IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
+                                       if (fallback_label) {
+                                               |       IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label
+                                       } else {
+                                               |       IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
+                                       }
                                        |       GET_ZVAL_PTR FCARG2a, op1_addr
-                                       |       IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3
+                                       if (fallback_label) {
+                                               |       IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label
+                                       } else {
+                                               |       IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3
+                                       }
                                        |       mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)]
                                        |       jmp >2
                                        |.code
                                        |2:
                                } else {
                                        if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
-                                               |       IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
+                                               if (fallback_label) {
+                                                       |       IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label
+                                               } else {
+                                                       |       IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
+                                               }
                                        }
                                        |       GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr
                                }
@@ -11473,7 +11522,13 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        Bucket *p = jumptable->arData;
 
                                        |       cmp FCARG2a, jumptable->nNumUsed
-                                       |       jae >3
+                                       if (default_label) {
+                                               |       jae &default_label
+                                       } else if (next_opline) {
+                                               |       jae >3
+                                       } else {
+                                               |       jae =>default_b
+                                       }
                                        |.if X64
                                                if (!IS_32BIT(dasm_end)) {
                                                        |       lea r0, aword [>4]
@@ -11484,27 +11539,48 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |.else
                                        |       jmp aword [FCARG2a * 4 + >4]
                                        |.endif
-                                       |3:
                                        |.cold_code
                                        |.align aword
                                        |4:
                                        p = jumptable->arData;
                                        do {
                                                if (Z_TYPE(p->val) == IS_UNDEF) {
-                                                       |       .aword =>b
+                                                       if (default_label) {
+                                                               |       .aword &default_label
+                                                       } else if (next_opline) {
+                                                               |       .aword >3
+                                                       } else {
+                                                               |       .aword =>default_b
+                                                       }
                                                } else {
-                                                       int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes];
-                                                       |       .aword =>b
+                                                       target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
+                                                       if (!next_opline) {
+                                                               b = ssa->cfg.map[target - op_array->opcodes];
+                                                               |       .aword =>b
+                                                       } else if (next_opline == target) {
+                                                               |       .aword >3
+                                                       } else {
+                                                               exit_point = zend_jit_trace_get_exit_point(opline, target, NULL, 0);
+                                                               exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+                                                               |       .aword &exit_addr
+                                                       }
                                                }
                                                p++;
                                                count--;
                                        } while (count);
                                        |.code
+                                       |3:
                                } else {
                                        |       LOAD_ADDR FCARG1a, jumptable
                                        |       EXT_CALL zend_hash_index_find, r0
                                        |       test r0, r0
-                                       |       jz =>b
+                                       if (default_label) {
+                                               |       jz &default_label
+                                       } else if (next_opline) {
+                                               |       jz >3
+                                       } else {
+                                               |       jz =>default_b
+                                       }
                                        |       LOAD_ADDR FCARG1a, jumptable
                                        |       sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
                                        |       mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
@@ -11524,15 +11600,24 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |.else
                                        |       jmp aword [r0 + >4]
                                        |.endif
-                                       |3:
                                        |.cold_code
                                        |.align aword
                                        |4:
                                        ZEND_HASH_FOREACH_VAL(jumptable, val) {
-                                               b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
-                                               |       .aword =>b
+                                               target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
+                                               if (!next_opline) {
+                                                       b = ssa->cfg.map[target - op_array->opcodes];
+                                                       |       .aword =>b
+                                               } else if (next_opline == target) {
+                                                       |       .aword >3
+                                               } else {
+                                                       exit_point = zend_jit_trace_get_exit_point(opline, target, NULL, 0);
+                                                       exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+                                                       |       .aword &exit_addr
+                                               }
                                        } ZEND_HASH_FOREACH_END();
                                        |.code
+                                       |3:
                                }
                        }
                } else if (opline->opcode == ZEND_SWITCH_STRING) {
@@ -11543,23 +11628,41 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |.cold_code
                                        |1:
                                        |       // ZVAL_DEREF(op)
-                                       |       IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
+                                       if (fallback_label) {
+                                               |       IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label
+                                       } else {
+                                               |       IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
+                                       }
                                        |       GET_ZVAL_PTR FCARG2a, op1_addr
-                                       |       IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3
+                                       if (fallback_label) {
+                                               |       IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label
+                                       } else {
+                                               |       IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3
+                                       }
                                        |       mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)]
                                        |       jmp >2
                                        |.code
                                        |2:
                                } else {
                                        if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
-                                               |       IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
+                                               if (fallback_label) {
+                                                       |       IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label
+                                               } else {
+                                                       |       IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
+                                               }
                                        }
                                        |       GET_ZVAL_PTR FCARG2a, op1_addr
                                }
                                |       LOAD_ADDR FCARG1a, jumptable
                                |       EXT_CALL zend_hash_find, r0
                                |       test r0, r0
-                               |       jz =>b
+                               if (default_label) {
+                                       |       jz &default_label
+                               } else if (next_opline) {
+                                       |       jz >3
+                               } else {
+                                       |       jz =>default_b
+                               }
                                |       LOAD_ADDR FCARG1a, jumptable
                                |       sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
                                |       mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
@@ -11579,15 +11682,24 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                |.else
                                |       jmp aword [r0 + >4]
                                |.endif
-                               |3:
                                |.cold_code
                                |.align aword
                                |4:
                                ZEND_HASH_FOREACH_VAL(jumptable, val) {
-                                       b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
-                                       |       .aword =>b
+                                       target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
+                                       if (!next_opline) {
+                                               b = ssa->cfg.map[target - op_array->opcodes];
+                                               |       .aword =>b
+                                       } else if (next_opline == target) {
+                                               |       .aword >3
+                                       } else {
+                                               exit_point = zend_jit_trace_get_exit_point(opline, target, NULL, 0);
+                                               exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+                                               |       .aword &exit_addr
+                                       }
                                } ZEND_HASH_FOREACH_END();
                                |.code
+                               |3:
                        }
                } else {
                        ZEND_UNREACHABLE();