From: Dmitry Stogov Date: Tue, 14 Jul 2020 12:15:08 +0000 (+0300) Subject: Tracing JIT for SWITCH instructions X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=7c16d11e3cef9280fcfc35484ad7fcd8240ee4ec;p=php Tracing JIT for SWITCH instructions --- diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e8c82f3886..7c352f0802 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2861,7 +2861,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op case ZEND_SWITCH_LONG: case ZEND_SWITCH_STRING: case ZEND_MATCH: - if (!zend_jit_switch(&dasm_state, opline, op_array, ssa)) { + if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 7109c69032..655bcbc2b2 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -3950,15 +3950,13 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } goto done; -#if 0 case ZEND_SWITCH_LONG: case ZEND_SWITCH_STRING: case ZEND_MATCH: - if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa)) { + if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1)) { goto jit_failure; } goto done; -#endif case ZEND_INIT_METHOD_CALL: case ZEND_INIT_DYNAMIC_CALL: if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index bedd4275a7..2bc4da07a4 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -11401,9 +11401,16 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze return 1; } -static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace) { HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); + const zend_op *next_opline = NULL; + + if (trace) { + ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); + ZEND_ASSERT(trace->opline != NULL); + next_opline = trace->opline; + } // TODO: Implement for match instructions if (opline->opcode == ZEND_MATCH) { @@ -11420,22 +11427,34 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (opline->opcode == ZEND_SWITCH_LONG) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); - if (jump_zv != NULL) { - b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + if (next_opline) { + const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); + + ZEND_ASSERT(target == next_opline); } else { - b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + if (jump_zv != NULL) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + } else { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + } + | jmp =>b } - | jmp =>b } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); - if (jump_zv != NULL) { - b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + if (next_opline) { + const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); + + ZEND_ASSERT(target == next_opline); } else { - b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + if (jump_zv != NULL) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + } else { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + } + | jmp =>b } - | jmp =>b } } else { ZEND_UNREACHABLE(); @@ -11444,8 +11463,26 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); - int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); + const zend_op *target; + int default_b = ssa->cfg.map[default_opline - op_array->opcodes]; + int b; zval *val; + int32_t exit_point; + const void *fallback_label = NULL; + const void *default_label = NULL; + const void *exit_addr; + + if (next_opline) { + if (next_opline != opline + 1) { + exit_point = zend_jit_trace_get_exit_point(opline, opline + 1, NULL, 0); + fallback_label = zend_jit_trace_get_exit_addr(exit_point); + } + if (next_opline != default_opline) { + exit_point = zend_jit_trace_get_exit_point(opline, default_opline, NULL, 0); + default_label = zend_jit_trace_get_exit_addr(exit_point); + } + } if (opline->opcode == ZEND_SWITCH_LONG) { if (op1_info & MAY_BE_LONG) { @@ -11455,16 +11492,28 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |.cold_code |1: | // ZVAL_DEREF(op) - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 + } | GET_ZVAL_PTR FCARG2a, op1_addr - | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 + if (fallback_label) { + | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label + } else { + | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 + } | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] | jmp >2 |.code |2: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 + } } | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr } @@ -11473,7 +11522,13 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o Bucket *p = jumptable->arData; | cmp FCARG2a, jumptable->nNumUsed - | jae >3 + if (default_label) { + | jae &default_label + } else if (next_opline) { + | jae >3 + } else { + | jae =>default_b + } |.if X64 if (!IS_32BIT(dasm_end)) { | lea r0, aword [>4] @@ -11484,27 +11539,48 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |.else | jmp aword [FCARG2a * 4 + >4] |.endif - |3: |.cold_code |.align aword |4: p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { - | .aword =>b + if (default_label) { + | .aword &default_label + } else if (next_opline) { + | .aword >3 + } else { + | .aword =>default_b + } } else { - int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes]; - | .aword =>b + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .aword =>b + } else if (next_opline == target) { + | .aword >3 + } else { + exit_point = zend_jit_trace_get_exit_point(opline, target, NULL, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .aword &exit_addr + } } p++; count--; } while (count); |.code + |3: } else { | LOAD_ADDR FCARG1a, jumptable | EXT_CALL zend_hash_index_find, r0 | test r0, r0 - | jz =>b + if (default_label) { + | jz &default_label + } else if (next_opline) { + | jz >3 + } else { + | jz =>default_b + } | LOAD_ADDR FCARG1a, jumptable | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) @@ -11524,15 +11600,24 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |.else | jmp aword [r0 + >4] |.endif - |3: |.cold_code |.align aword |4: ZEND_HASH_FOREACH_VAL(jumptable, val) { - b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes]; - | .aword =>b + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .aword =>b + } else if (next_opline == target) { + | .aword >3 + } else { + exit_point = zend_jit_trace_get_exit_point(opline, target, NULL, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .aword &exit_addr + } } ZEND_HASH_FOREACH_END(); |.code + |3: } } } else if (opline->opcode == ZEND_SWITCH_STRING) { @@ -11543,23 +11628,41 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |.cold_code |1: | // ZVAL_DEREF(op) - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 + } | GET_ZVAL_PTR FCARG2a, op1_addr - | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 + if (fallback_label) { + | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label + } else { + | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 + } | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] | jmp >2 |.code |2: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 + } } | GET_ZVAL_PTR FCARG2a, op1_addr } | LOAD_ADDR FCARG1a, jumptable | EXT_CALL zend_hash_find, r0 | test r0, r0 - | jz =>b + if (default_label) { + | jz &default_label + } else if (next_opline) { + | jz >3 + } else { + | jz =>default_b + } | LOAD_ADDR FCARG1a, jumptable | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) @@ -11579,15 +11682,24 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |.else | jmp aword [r0 + >4] |.endif - |3: |.cold_code |.align aword |4: ZEND_HASH_FOREACH_VAL(jumptable, val) { - b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes]; - | .aword =>b + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .aword =>b + } else if (next_opline == target) { + | .aword >3 + } else { + exit_point = zend_jit_trace_get_exit_point(opline, target, NULL, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .aword &exit_addr + } } ZEND_HASH_FOREACH_END(); |.code + |3: } } else { ZEND_UNREACHABLE();