From 4bf2d09edeb14467ba79551a08d84efdff314899 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 13 Mar 2020 22:11:07 +0300 Subject: [PATCH] Tracing JIT (it doesn't support register allocation yet) Use opcache.jit=1255 to swith it on (the third digit 5 really matters) Use opcache.jit_debug=0xff001 to see how it works and what code it generates --- ext/opcache/Optimizer/zend_dfg.c | 384 ++- ext/opcache/Optimizer/zend_dfg.h | 1 + ext/opcache/Optimizer/zend_dump.c | 59 +- ext/opcache/Optimizer/zend_dump.h | 3 +- ext/opcache/Optimizer/zend_inference.c | 517 +-- ext/opcache/Optimizer/zend_inference.h | 109 +- ext/opcache/Optimizer/zend_ssa.c | 451 +-- ext/opcache/Optimizer/zend_ssa.h | 1 + ext/opcache/jit/Makefile.frag | 1 + ext/opcache/jit/Makefile.frag.w32 | 1 + ext/opcache/jit/zend_jit.c | 284 +- ext/opcache/jit/zend_jit.h | 10 + ext/opcache/jit/zend_jit_disasm_x86.c | 6 + ext/opcache/jit/zend_jit_internal.h | 292 +- ext/opcache/jit/zend_jit_trace.c | 3972 ++++++++++++++++++++++++ ext/opcache/jit/zend_jit_vm_helpers.c | 624 +++- ext/opcache/jit/zend_jit_x86.dasc | 2170 ++++++++++--- 17 files changed, 7712 insertions(+), 1173 deletions(-) create mode 100644 ext/opcache/jit/zend_jit_trace.c diff --git a/ext/opcache/Optimizer/zend_dfg.c b/ext/opcache/Optimizer/zend_dfg.c index 3bb76fb05c..fda6b4f9a8 100644 --- a/ext/opcache/Optimizer/zend_dfg.c +++ b/ext/opcache/Optimizer/zend_dfg.c @@ -20,6 +20,236 @@ #include "zend_compile.h" #include "zend_dfg.h" +static zend_always_inline void _zend_dfg_add_use_def_op(const zend_op_array *op_array, const zend_op *opline, uint32_t build_flags, zend_bitset use, zend_bitset def) /* {{{ */ +{ + uint32_t var_num; + const zend_op *next; + + if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(opline->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + } + if (((opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0 + && opline->opcode != ZEND_FE_FETCH_R + && opline->opcode != ZEND_FE_FETCH_RW) + || (opline->op2_type == IS_CV)) { + var_num = EX_VAR_TO_NUM(opline->op2.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + } + if ((build_flags & ZEND_SSA_USE_CV_RESULTS) + && opline->result_type == IS_CV + && opline->opcode != ZEND_RECV) { + var_num = EX_VAR_TO_NUM(opline->result.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + } + + switch (opline->opcode) { + case ZEND_ASSIGN: + if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op2_type == IS_CV) { + zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var)); + } + if (opline->op1_type == IS_CV) { +add_op1_def: + zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op1.var)); + } + break; + case ZEND_ASSIGN_REF: + if (opline->op2_type == IS_CV) { + zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var)); + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(next->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + if (build_flags & ZEND_SSA_RC_INFERENCE && next->op1_type == IS_CV) { + zend_bitset_incl(def, var_num); + } + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_OBJ_REF: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(next->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + if (next->op1_type == IS_CV) { + zend_bitset_incl(def, var_num); + } + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_STATIC_PROP: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(next->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } +#if 0 + if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) { + zend_bitset_incl(def, var_num); + } +#endif + } + break; + case ZEND_ASSIGN_STATIC_PROP_REF: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(next->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + if (next->op1_type == IS_CV) { + zend_bitset_incl(def, var_num); + } + } + break; + case ZEND_ASSIGN_STATIC_PROP_OP: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(next->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + } + break; + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_OBJ_OP: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + var_num = EX_VAR_TO_NUM(next->op1.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_OP: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + case ZEND_BIND_GLOBAL: + case ZEND_BIND_STATIC: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_UNPACK: + case ZEND_FE_RESET_RW: + case ZEND_MAKE_REF: + case ZEND_PRE_INC_OBJ: + case ZEND_PRE_DEC_OBJ: + case ZEND_POST_INC_OBJ: + case ZEND_POST_DEC_OBJ: + case ZEND_UNSET_DIM: + case ZEND_UNSET_OBJ: + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_DIM_RW: + case ZEND_FETCH_DIM_FUNC_ARG: + case ZEND_FETCH_DIM_UNSET: + case ZEND_FETCH_LIST_W: + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_SEND_VAR: + case ZEND_CAST: + case ZEND_QM_ASSIGN: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_FE_RESET_R: + if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ADD_ARRAY_UNPACK: + var_num = EX_VAR_TO_NUM(opline->result.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + break; + case ZEND_ADD_ARRAY_ELEMENT: + var_num = EX_VAR_TO_NUM(opline->result.var); + if (!zend_bitset_in(def, var_num)) { + zend_bitset_incl(use, var_num); + } + /* break missing intentionally */ + case ZEND_INIT_ARRAY: + if (((build_flags & ZEND_SSA_RC_INFERENCE) + || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF)) + && opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_YIELD: + if (opline->op1_type == IS_CV + && ((op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) + || (build_flags & ZEND_SSA_RC_INFERENCE))) { + goto add_op1_def; + } + break; + case ZEND_UNSET_CV: + goto add_op1_def; + case ZEND_VERIFY_RETURN_TYPE: + if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { + goto add_op1_def; + } + break; + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: +#if 0 + /* This special case was handled above the switch */ + if (opline->op2_type != IS_CV) { + op2_use = -1; /* not used */ + } +#endif + zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var)); + break; + case ZEND_BIND_LEXICAL: + if ((opline->extended_value & ZEND_BIND_REF) || (build_flags & ZEND_SSA_RC_INFERENCE)) { + zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var)); + } + break; + default: + break; + } + + if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + zend_bitset_incl(def, EX_VAR_TO_NUM(opline->result.var)); + } +} +/* }}} */ + +void zend_dfg_add_use_def_op(const zend_op_array *op_array, const zend_op *opline, uint32_t build_flags, zend_bitset use, zend_bitset def) /* {{{ */ +{ + _zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); +} +/* }}} */ + int zend_build_dfg(const zend_op_array *op_array, const zend_cfg *cfg, zend_dfg *dfg, uint32_t build_flags) /* {{{ */ { int set_size; @@ -27,7 +257,6 @@ int zend_build_dfg(const zend_op_array *op_array, const zend_cfg *cfg, zend_dfg int blocks_count = cfg->blocks_count; zend_bitset tmp, def, use, in, out; int k; - uint32_t var_num; int j; set_size = dfg->size; @@ -40,162 +269,19 @@ int zend_build_dfg(const zend_op_array *op_array, const zend_cfg *cfg, zend_dfg /* Collect "def" and "use" sets */ for (j = 0; j < blocks_count; j++) { zend_op *opline, *end; + zend_bitset b_use, b_def; + if ((blocks[j].flags & ZEND_BB_REACHABLE) == 0) { continue; } opline = op_array->opcodes + blocks[j].start; end = opline + blocks[j].len; + b_use = DFG_BITSET(use, set_size, j); + b_def = DFG_BITSET(def, set_size, j); for (; opline < end; opline++) { if (opline->opcode != ZEND_OP_DATA) { - zend_op *next = opline + 1; - if (next < end && next->opcode == ZEND_OP_DATA) { - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - var_num = EX_VAR_TO_NUM(next->op1.var); - if (next->op1_type == IS_CV && (opline->opcode == ZEND_ASSIGN_OBJ_REF - || opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF)) { - DFG_SET(use, set_size, j, var_num); - DFG_SET(def, set_size, j, var_num); - } else { - if (!DFG_ISSET(def, set_size, j, var_num)) { - DFG_SET(use, set_size, j, var_num); - } - } - } - if (next->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - var_num = EX_VAR_TO_NUM(next->op2.var); - if (!DFG_ISSET(def, set_size, j, var_num)) { - DFG_SET(use, set_size, j, var_num); - } - } - } - if (opline->op1_type == IS_CV) { - var_num = EX_VAR_TO_NUM(opline->op1.var); - switch (opline->opcode) { - case ZEND_ADD_ARRAY_ELEMENT: - case ZEND_INIT_ARRAY: - if ((build_flags & ZEND_SSA_RC_INFERENCE) - || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF)) { - goto op1_def; - } - goto op1_use; - case ZEND_FE_RESET_R: - case ZEND_SEND_VAR: - case ZEND_CAST: - case ZEND_QM_ASSIGN: - case ZEND_JMP_SET: - case ZEND_COALESCE: - if (build_flags & ZEND_SSA_RC_INFERENCE) { - goto op1_def; - } - goto op1_use; - case ZEND_YIELD: - if ((build_flags & ZEND_SSA_RC_INFERENCE) - || (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE)) { - goto op1_def; - } - goto op1_use; - case ZEND_UNSET_CV: - case ZEND_ASSIGN: - case ZEND_ASSIGN_REF: - case ZEND_ASSIGN_OBJ_REF: - case ZEND_BIND_GLOBAL: - case ZEND_BIND_STATIC: - case ZEND_SEND_VAR_EX: - case ZEND_SEND_FUNC_ARG: - case ZEND_SEND_REF: - case ZEND_SEND_VAR_NO_REF: - case ZEND_SEND_VAR_NO_REF_EX: - case ZEND_FE_RESET_RW: - case ZEND_ASSIGN_OP: - case ZEND_ASSIGN_DIM_OP: - case ZEND_ASSIGN_OBJ_OP: - case ZEND_ASSIGN_STATIC_PROP_OP: - case ZEND_PRE_INC: - case ZEND_PRE_DEC: - case ZEND_POST_INC: - case ZEND_POST_DEC: - case ZEND_ASSIGN_DIM: - case ZEND_ASSIGN_OBJ: - case ZEND_UNSET_DIM: - case ZEND_UNSET_OBJ: - case ZEND_FETCH_DIM_W: - case ZEND_FETCH_DIM_RW: - case ZEND_FETCH_DIM_FUNC_ARG: - case ZEND_FETCH_DIM_UNSET: - case ZEND_FETCH_LIST_W: - case ZEND_VERIFY_RETURN_TYPE: - case ZEND_PRE_INC_OBJ: - case ZEND_PRE_DEC_OBJ: - case ZEND_POST_INC_OBJ: - case ZEND_POST_DEC_OBJ: -op1_def: - /* `def` always come along with dtor or separation, - * thus the origin var info might be also `use`d in the feature(CG) */ - DFG_SET(use, set_size, j, var_num); - DFG_SET(def, set_size, j, var_num); - break; - default: -op1_use: - if (!DFG_ISSET(def, set_size, j, var_num)) { - DFG_SET(use, set_size, j, var_num); - } - } - } else if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { - var_num = EX_VAR_TO_NUM(opline->op1.var); - if (!DFG_ISSET(def, set_size, j, var_num)) { - DFG_SET(use, set_size, j, var_num); - } - if (opline->opcode == ZEND_VERIFY_RETURN_TYPE) { - DFG_SET(def, set_size, j, var_num); - } - } - if (opline->op2_type == IS_CV) { - var_num = EX_VAR_TO_NUM(opline->op2.var); - switch (opline->opcode) { - case ZEND_ASSIGN: - if (build_flags & ZEND_SSA_RC_INFERENCE) { - goto op2_def; - } - goto op2_use; - case ZEND_BIND_LEXICAL: - if ((build_flags & ZEND_SSA_RC_INFERENCE) || (opline->extended_value & ZEND_BIND_REF)) { - goto op2_def; - } - goto op2_use; - case ZEND_ASSIGN_REF: - case ZEND_FE_FETCH_R: - case ZEND_FE_FETCH_RW: -op2_def: - // FIXME: include into "use" too ...? - DFG_SET(use, set_size, j, var_num); - DFG_SET(def, set_size, j, var_num); - break; - default: -op2_use: - if (!DFG_ISSET(def, set_size, j, var_num)) { - DFG_SET(use, set_size, j, var_num); - } - break; - } - } else if (opline->op2_type & (IS_VAR|IS_TMP_VAR)) { - var_num = EX_VAR_TO_NUM(opline->op2.var); - if (opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW) { - DFG_SET(def, set_size, j, var_num); - } else { - if (!DFG_ISSET(def, set_size, j, var_num)) { - DFG_SET(use, set_size, j, var_num); - } - } - } - if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - var_num = EX_VAR_TO_NUM(opline->result.var); - if ((build_flags & ZEND_SSA_USE_CV_RESULTS) - && opline->result_type == IS_CV) { - DFG_SET(use, set_size, j, var_num); - } - DFG_SET(def, set_size, j, var_num); - } + _zend_dfg_add_use_def_op(op_array, opline, build_flags, b_use, b_def); } } } diff --git a/ext/opcache/Optimizer/zend_dfg.h b/ext/opcache/Optimizer/zend_dfg.h index 9720f79cc1..a675187794 100644 --- a/ext/opcache/Optimizer/zend_dfg.h +++ b/ext/opcache/Optimizer/zend_dfg.h @@ -44,6 +44,7 @@ typedef struct _zend_dfg { BEGIN_EXTERN_C() int zend_build_dfg(const zend_op_array *op_array, const zend_cfg *cfg, zend_dfg *dfg, uint32_t build_flags); +void zend_dfg_add_use_def_op(const zend_op_array *op_array, const zend_op *opline, uint32_t build_flags, zend_bitset use, zend_bitset def); END_EXTERN_C() diff --git a/ext/opcache/Optimizer/zend_dump.c b/ext/opcache/Optimizer/zend_dump.c index 59288fabdd..1a18f8727a 100644 --- a/ext/opcache/Optimizer/zend_dump.c +++ b/ext/opcache/Optimizer/zend_dump.c @@ -169,6 +169,9 @@ static void zend_dump_type_info(uint32_t info, zend_class_entry *ce, int is_inst int first = 1; fprintf(stderr, " ["); + if (info & MAY_BE_GUARD) { + fprintf(stderr, "!"); + } if (info & MAY_BE_UNDEF) { if (first) first = 0; else fprintf(stderr, ", "); fprintf(stderr, "undef"); @@ -329,7 +332,7 @@ static void zend_dump_ssa_var_info(const zend_ssa *ssa, int ssa_var_num, uint32_ dump_flags); } -static void zend_dump_ssa_var(const zend_op_array *op_array, const zend_ssa *ssa, int ssa_var_num, zend_uchar var_type, int var_num, uint32_t dump_flags) +void zend_dump_ssa_var(const zend_op_array *op_array, const zend_ssa *ssa, int ssa_var_num, zend_uchar var_type, int var_num, uint32_t dump_flags) { if (ssa_var_num >= 0) { fprintf(stderr, "#%d.", ssa_var_num); @@ -402,21 +405,16 @@ static void zend_dump_range_constraint(const zend_op_array *op_array, const zend } } -void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const void *data) +void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const zend_ssa *ssa, const zend_ssa_op *ssa_op) { const char *name = zend_get_opcode_name(opline->opcode); uint32_t flags = zend_get_opcode_flags(opline->opcode); uint32_t n = 0; - const zend_ssa *ssa = NULL; - if (dump_flags & ZEND_DUMP_SSA) { - ssa = (const zend_ssa*)data; - } - - if (!ssa || !ssa->ops || ssa->ops[opline - op_array->opcodes].result_use < 0) { + if (!ssa_op || ssa_op->result_use < 0) { if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - if (ssa && ssa->ops && ssa->ops[opline - op_array->opcodes].result_def >= 0) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].result_def; + if (ssa_op && ssa_op->result_def >= 0) { + int ssa_var_num = ssa_op->result_def; zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->result_type, EX_VAR_TO_NUM(opline->result.var), dump_flags); } else { zend_dump_var(op_array, opline->result_type, EX_VAR_TO_NUM(opline->result.var)); @@ -582,12 +580,12 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons if (opline->op1_type == IS_CONST) { zend_dump_const(CRT_CONSTANT(opline->op1)); } else if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - if (ssa && ssa->ops) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].op1_use; + if (ssa_op) { + int ssa_var_num = ssa_op->op1_use; if (ssa_var_num >= 0) { fprintf(stderr, " "); zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var), dump_flags); - } else if (ssa->ops[opline - op_array->opcodes].op1_def < 0) { + } else if (ssa_op->op1_def < 0) { fprintf(stderr, " "); zend_dump_var(op_array, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var)); } @@ -595,8 +593,8 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons fprintf(stderr, " "); zend_dump_var(op_array, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var)); } - if (ssa && ssa->ops) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].op1_def; + if (ssa_op) { + int ssa_var_num = ssa_op->op1_def; if (ssa_var_num >= 0) { fprintf(stderr, " -> "); zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var), dump_flags); @@ -643,12 +641,12 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons zend_dump_const(op); } } else if (opline->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - if (ssa && ssa->ops) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].op2_use; + if (ssa_op) { + int ssa_var_num = ssa_op->op2_use; if (ssa_var_num >= 0) { fprintf(stderr, " "); zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var), dump_flags); - } else if (ssa->ops[opline - op_array->opcodes].op2_def < 0) { + } else if (ssa_op->op2_def < 0) { fprintf(stderr, " "); zend_dump_var(op_array, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var)); } @@ -656,8 +654,8 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons fprintf(stderr, " "); zend_dump_var(op_array, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var)); } - if (ssa && ssa->ops) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].op2_def; + if (ssa_op) { + int ssa_var_num = ssa_op->op2_def; if (ssa_var_num >= 0) { fprintf(stderr, " -> "); zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var), dump_flags); @@ -697,10 +695,10 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons } else if (opline->result_type & IS_SMART_BRANCH_JMPNZ) { fprintf(stderr, " jmpnz"); #endif - } else if (ssa && ssa->ops && ssa->ops[opline - op_array->opcodes].result_use >= 0) { + } else if (ssa_op && ssa_op->result_use >= 0) { if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - if (ssa && ssa->ops) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].result_use; + if (ssa_op) { + int ssa_var_num = ssa_op->result_use; if (ssa_var_num >= 0) { fprintf(stderr, " "); zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->result_type, EX_VAR_TO_NUM(opline->result.var), dump_flags); @@ -709,8 +707,8 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons fprintf(stderr, " "); zend_dump_var(op_array, opline->result_type, EX_VAR_TO_NUM(opline->result.var)); } - if (ssa && ssa->ops) { - int ssa_var_num = ssa->ops[opline - op_array->opcodes].result_def; + if (ssa_op) { + int ssa_var_num = ssa_op->result_def; if (ssa_var_num >= 0) { fprintf(stderr, " -> "); zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->result_type, EX_VAR_TO_NUM(opline->result.var), dump_flags); @@ -723,6 +721,8 @@ void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, cons static void zend_dump_op_line(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const void *data) { int len = 0; + const zend_ssa *ssa = NULL; + zend_ssa_op *ssa_op = NULL; if (dump_flags & ZEND_DUMP_NUMERIC_OPLINES) { len = fprintf(stderr, "%04u:", (uint32_t)(opline - op_array->opcodes)); @@ -731,7 +731,14 @@ static void zend_dump_op_line(const zend_op_array *op_array, const zend_basic_bl } fprintf(stderr, "%*c", 12-len, ' '); - zend_dump_op(op_array, b, opline, dump_flags, data); + if (dump_flags & ZEND_DUMP_SSA) { + ssa = (const zend_ssa*)data; + if (ssa && ssa->ops) { + ssa_op = &ssa->ops[opline - op_array->opcodes]; + } + } + + zend_dump_op(op_array, b, opline, dump_flags, ssa, ssa_op); fprintf(stderr, "\n"); } diff --git a/ext/opcache/Optimizer/zend_dump.h b/ext/opcache/Optimizer/zend_dump.h index 5f1968fd23..669ac69980 100644 --- a/ext/opcache/Optimizer/zend_dump.h +++ b/ext/opcache/Optimizer/zend_dump.h @@ -32,12 +32,13 @@ BEGIN_EXTERN_C() void zend_dump_op_array(const zend_op_array *op_array, uint32_t dump_flags, const char *msg, const void *data); -void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const void *data); +void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const zend_ssa *ssa, const zend_ssa_op *ssa_op); void zend_dump_dominators(const zend_op_array *op_array, const zend_cfg *cfg); void zend_dump_dfg(const zend_op_array *op_array, const zend_cfg *cfg, const zend_dfg *dfg); void zend_dump_phi_placement(const zend_op_array *op_array, const zend_ssa *ssa); void zend_dump_variables(const zend_op_array *op_array); void zend_dump_ssa_variables(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t dump_flags); +void zend_dump_ssa_var(const zend_op_array *op_array, const zend_ssa *ssa, int ssa_var_num, zend_uchar var_type, int var_num, uint32_t dump_flags); void zend_dump_var(const zend_op_array *op_array, zend_uchar var_type, int var_num); void zend_dump_op_array_name(const zend_op_array *op_array); void zend_dump_const(const zval *zv); diff --git a/ext/opcache/Optimizer/zend_inference.c b/ext/opcache/Optimizer/zend_inference.c index e11c3cd9c7..2be19fcf8a 100644 --- a/ext/opcache/Optimizer/zend_inference.c +++ b/ext/opcache/Optimizer/zend_inference.c @@ -1828,7 +1828,7 @@ static uint32_t get_ssa_alias_types(zend_ssa_alias_kind alias) { #define UPDATE_SSA_TYPE(_type, _var) \ do { \ - uint32_t __type = (_type); \ + uint32_t __type = (_type) & ~MAY_BE_GUARD; \ int __var = (_var); \ if (__type & MAY_BE_REF) { \ __type |= MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; \ @@ -1848,12 +1848,19 @@ static uint32_t get_ssa_alias_types(zend_ssa_alias_kind alias) { } \ } \ if (ssa_var_info[__var].type != __type) { \ - if (ssa_var_info[__var].type & ~__type) { \ - emit_type_narrowing_warning(op_array, ssa, __var); \ - return FAILURE; \ + if (update_worklist) { \ + if (ssa_var_info[__var].type & ~__type) { \ + emit_type_narrowing_warning(op_array, ssa, __var);\ + return FAILURE; \ + } \ + ssa_var_info[__var].type = __type; \ + add_usages(op_array, ssa, worklist, __var); \ + } else { \ + if (ssa_var_info[__var].type & ~__type) { \ + return FAILURE; \ + } \ + ssa_var_info[__var].type = __type; \ } \ - ssa_var_info[__var].type = __type; \ - add_usages(op_array, ssa, worklist, __var); \ } \ /*zend_bitset_excl(worklist, var);*/ \ } \ @@ -1866,7 +1873,9 @@ static uint32_t get_ssa_alias_types(zend_ssa_alias_kind alias) { ssa_var_info[var].is_instanceof != (_is_instanceof)) { \ ssa_var_info[var].ce = (_ce); \ ssa_var_info[var].is_instanceof = (_is_instanceof); \ - add_usages(op_array, ssa, worklist, var); \ + if (update_worklist) { \ + add_usages(op_array, ssa, worklist, var); \ + } \ } \ /*zend_bitset_excl(worklist, var);*/ \ } \ @@ -2207,7 +2216,7 @@ static zend_property_info *lookup_prop_info(zend_class_entry *ce, zend_string *n return NULL; } -static zend_property_info *zend_fetch_prop_info(const zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int i) +static zend_property_info *zend_fetch_prop_info(const zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, zend_ssa_op *ssa_op) { zend_property_info *prop_info = NULL; if (opline->op2_type == IS_CONST) { @@ -2215,8 +2224,8 @@ static zend_property_info *zend_fetch_prop_info(const zend_op_array *op_array, z if (opline->op1_type == IS_UNUSED) { ce = op_array->scope; - } else if (ssa->ops[i].op1_use >= 0) { - ce = ssa->var_info[ssa->ops[i].op1_use].ce; + } else if (ssa_op->op1_use >= 0) { + ce = ssa->var_info[ssa_op->op1_use].ce; } if (ce) { prop_info = lookup_prop_info(ce, @@ -2294,17 +2303,19 @@ static uint32_t zend_fetch_prop_type(const zend_script *script, zend_property_in return MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_RC1 | MAY_BE_RCN; } -static int zend_update_type_info(const zend_op_array *op_array, - zend_ssa *ssa, - const zend_script *script, - zend_bitset worklist, - int i, - zend_long optimization_level) +static zend_always_inline int _zend_update_type_info( + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_script *script, + zend_bitset worklist, + zend_op *opline, + zend_ssa_op *ssa_op, + const zend_op **ssa_opcodes, + zend_long optimization_level, + zend_bool update_worklist) { uint32_t t1, t2; uint32_t tmp, orig; - zend_op *opline = op_array->opcodes + i; - zend_ssa_op *ssa_ops = ssa->ops; zend_ssa_var *ssa_vars = ssa->vars; zend_ssa_var_info *ssa_var_info = ssa->var_info; zend_class_entry *ce; @@ -2312,11 +2323,11 @@ static int zend_update_type_info(const zend_op_array *op_array, if (opline->opcode == ZEND_OP_DATA) { opline--; - i--; + ssa_op--; } - t1 = OP1_INFO(); - t2 = OP2_INFO(); + t1 = OP1_INFO_EX(); + t2 = OP2_INFO_EX(); /* If one of the operands cannot have any type, this means the operand derives from * unreachable code. Propagate the empty result early, so that that the following @@ -2324,14 +2335,14 @@ static int zend_update_type_info(const zend_op_array *op_array, if (!(t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_CLASS)) || !(t2 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_CLASS))) { tmp = 0; - if (ssa_ops[i].result_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + if (ssa_op->result_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } - if (ssa_ops[i].op1_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + if (ssa_op->op1_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } - if (ssa_ops[i].op2_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def); + if (ssa_op->op2_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->op2_def); } return 1; } @@ -2349,8 +2360,8 @@ static int zend_update_type_info(const zend_op_array *op_array, case ZEND_SL: case ZEND_SR: case ZEND_CONCAT: - tmp = binary_op_result_type(ssa, opline->opcode, t1, t2, ssa_ops[i].result_def, optimization_level); - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + tmp = binary_op_result_type(ssa, opline->opcode, t1, t2, ssa_op->result_def, optimization_level); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_BW_NOT: tmp = 0; @@ -2366,10 +2377,10 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_OBJECT | MAY_BE_RC1; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_BEGIN_SILENCE: - UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_op->result_def); break; case ZEND_BOOL_NOT: case ZEND_BOOL_XOR: @@ -2392,10 +2403,10 @@ static int zend_update_type_info(const zend_op_array *op_array, case ZEND_ASSERT_CHECK: case ZEND_IN_ARRAY: case ZEND_ARRAY_KEY_EXISTS: - UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_op->result_def); break; case ZEND_CAST: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = t1; if ((t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) && (opline->op1_type == IS_CV) && @@ -2407,8 +2418,8 @@ static int zend_update_type_info(const zend_op_array *op_array, opline->extended_value == IS_STRING) { tmp |= MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } tmp = 1 << opline->extended_value; if (tmp & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { @@ -2435,19 +2446,19 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= ((t1 & MAY_BE_ANY) << MAY_BE_ARRAY_SHIFT) | ((t1 & MAY_BE_ANY)? MAY_BE_ARRAY_KEY_LONG : 0); } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_QM_ASSIGN: case ZEND_JMP_SET: case ZEND_COALESCE: case ZEND_COPY_TMP: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = t1; if ((t1 & (MAY_BE_RC1|MAY_BE_REF)) && (opline->op1_type == IS_CV)) { tmp |= MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } tmp = t1 & ~(MAY_BE_UNDEF|MAY_BE_REF); if (t1 & MAY_BE_UNDEF) { @@ -2467,8 +2478,8 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp &= ~MAY_BE_FALSE; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->result_def); break; case ZEND_ASSIGN_OP: case ZEND_ASSIGN_DIM_OP: @@ -2479,21 +2490,21 @@ static int zend_update_type_info(const zend_op_array *op_array, orig = 0; tmp = 0; if (opline->opcode == ZEND_ASSIGN_OBJ_OP) { - prop_info = zend_fetch_prop_info(op_array, ssa, opline, i); + prop_info = zend_fetch_prop_info(op_array, ssa, opline, ssa_op); orig = t1; t1 = zend_fetch_prop_type(script, prop_info, &ce); - t2 = OP1_DATA_INFO(); + t2 = OP1_DATA_INFO_EX(); } else if (opline->opcode == ZEND_ASSIGN_DIM_OP) { if (t1 & MAY_BE_ARRAY_OF_REF) { tmp |= MAY_BE_REF; } orig = t1; t1 = zend_array_element_type(t1, 1, 0); - t2 = OP1_DATA_INFO(); + t2 = OP1_DATA_INFO_EX(); } else if (opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP) { prop_info = zend_fetch_static_prop_info(script, op_array, ssa, opline); t1 = zend_fetch_prop_type(script, prop_info, &ce); - t2 = OP1_DATA_INFO(); + t2 = OP1_DATA_INFO_EX(); } else { if (t1 & MAY_BE_REF) { tmp |= MAY_BE_REF; @@ -2501,7 +2512,7 @@ static int zend_update_type_info(const zend_op_array *op_array, } tmp |= binary_op_result_type( - ssa, opline->extended_value, t1, t2, ssa_ops[i].op1_def, optimization_level); + ssa, opline->extended_value, t1, t2, ssa_op->op1_def, optimization_level); if (tmp & (MAY_BE_STRING|MAY_BE_ARRAY)) { tmp |= MAY_BE_RC1; } @@ -2511,9 +2522,9 @@ static int zend_update_type_info(const zend_op_array *op_array, if (opline->opcode == ZEND_ASSIGN_DIM_OP) { if (opline->op1_type == IS_CV) { - orig = assign_dim_result_type(orig, OP2_INFO(), tmp, opline->op2_type); - UPDATE_SSA_TYPE(orig, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + orig = assign_dim_result_type(orig, OP2_INFO_EX(), tmp, opline->op2_type); + UPDATE_SSA_TYPE(orig, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } } else if (opline->opcode == ZEND_ASSIGN_OBJ_OP) { if (opline->op1_type == IS_CV) { @@ -2526,15 +2537,15 @@ static int zend_update_type_info(const zend_op_array *op_array, orig |= (MAY_BE_RC1|MAY_BE_RCN); } } - UPDATE_SSA_TYPE(orig, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(orig, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } } else if (opline->opcode == ZEND_ASSIGN_STATIC_PROP) { /* Nothing to do */ } else { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { ce = NULL; if (opline->opcode == ZEND_ASSIGN_DIM_OP) { if (opline->op2_type == IS_UNUSED) { @@ -2569,9 +2580,9 @@ static int zend_update_type_info(const zend_op_array *op_array, } } tmp &= ~MAY_BE_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def); } } break; @@ -2584,18 +2595,18 @@ static int zend_update_type_info(const zend_op_array *op_array, } if (t1 & (MAY_BE_RC1|MAY_BE_RCN)) { tmp |= MAY_BE_RC1; - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp |= MAY_BE_RCN; } } if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { - if (!ssa_var_info[ssa_ops[i].op1_use].has_range || + if (!ssa_var_info[ssa_op->op1_use].has_range || (opline->opcode == ZEND_PRE_DEC && - (ssa_var_info[ssa_ops[i].op1_use].range.underflow || - ssa_var_info[ssa_ops[i].op1_use].range.min == ZEND_LONG_MIN)) || + (ssa_var_info[ssa_op->op1_use].range.underflow || + ssa_var_info[ssa_op->op1_use].range.min == ZEND_LONG_MIN)) || (opline->opcode == ZEND_PRE_INC && - (ssa_var_info[ssa_ops[i].op1_use].range.overflow || - ssa_var_info[ssa_ops[i].op1_use].range.max == ZEND_LONG_MAX))) { + (ssa_var_info[ssa_op->op1_use].range.overflow || + ssa_var_info[ssa_op->op1_use].range.max == ZEND_LONG_MAX))) { /* may overflow */ tmp |= MAY_BE_LONG | MAY_BE_DOUBLE; } else { @@ -2620,16 +2631,16 @@ static int zend_update_type_info(const zend_op_array *op_array, } tmp |= t1 & (MAY_BE_FALSE | MAY_BE_TRUE | MAY_BE_RESOURCE | MAY_BE_ARRAY | MAY_BE_OBJECT | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_ARRAY_KEY_ANY); } - if (ssa_ops[i].op1_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + if (ssa_op->op1_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + if (ssa_op->result_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; case ZEND_POST_INC: case ZEND_POST_DEC: - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp = 0; if (t1 & (MAY_BE_RC1|MAY_BE_RCN)) { tmp |= MAY_BE_RC1|MAY_BE_RCN; @@ -2638,7 +2649,7 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t1 & MAY_BE_UNDEF) { tmp |= MAY_BE_NULL; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } tmp = 0; if (t1 & MAY_BE_REF) { @@ -2648,13 +2659,13 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_RC1; } if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { - if (!ssa_var_info[ssa_ops[i].op1_use].has_range || + if (!ssa_var_info[ssa_op->op1_use].has_range || (opline->opcode == ZEND_POST_DEC && - (ssa_var_info[ssa_ops[i].op1_use].range.underflow || - ssa_var_info[ssa_ops[i].op1_use].range.min == ZEND_LONG_MIN)) || + (ssa_var_info[ssa_op->op1_use].range.underflow || + ssa_var_info[ssa_op->op1_use].range.min == ZEND_LONG_MIN)) || (opline->opcode == ZEND_POST_INC && - (ssa_var_info[ssa_ops[i].op1_use].range.overflow || - ssa_var_info[ssa_ops[i].op1_use].range.max == ZEND_LONG_MAX))) { + (ssa_var_info[ssa_op->op1_use].range.overflow || + ssa_var_info[ssa_op->op1_use].range.max == ZEND_LONG_MAX))) { /* may overflow */ tmp |= MAY_BE_LONG | MAY_BE_DOUBLE; } else { @@ -2679,23 +2690,23 @@ static int zend_update_type_info(const zend_op_array *op_array, } tmp |= t1 & (MAY_BE_FALSE | MAY_BE_TRUE | MAY_BE_RESOURCE | MAY_BE_ARRAY | MAY_BE_OBJECT | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_ARRAY_KEY_ANY); } - if (ssa_ops[i].op1_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + if (ssa_op->op1_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_ASSIGN_DIM: if (opline->op1_type == IS_CV) { - tmp = assign_dim_result_type(t1, t2, OP1_DATA_INFO(), opline->op2_type); - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + tmp = assign_dim_result_type(t1, t2, OP1_DATA_INFO_EX(), opline->op2_type); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp = 0; if (t1 & MAY_BE_STRING) { tmp |= MAY_BE_STRING; } if (t1 & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_STRING)) { - tmp |= (OP1_DATA_INFO() & (MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF)); + tmp |= (OP1_DATA_INFO_EX() & (MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF)); if (opline->op2_type == IS_UNUSED) { /* When appending to an array and the LONG_MAX key is already used @@ -2716,18 +2727,18 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t1 & MAY_BE_OBJECT) { tmp |= MAY_BE_REF; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } - if ((opline+1)->op1_type == IS_CV && ssa_ops[i+1].op1_def >= 0) { + if ((opline+1)->op1_type == IS_CV && (ssa_op+1)->op1_def >= 0) { opline++; - i++; - tmp = OP1_INFO(); + ssa_op++; + tmp = OP1_INFO_EX(); if (tmp & (MAY_BE_ANY | MAY_BE_REF)) { if (tmp & MAY_BE_RC1) { tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_ASSIGN_OBJ: @@ -2740,28 +2751,28 @@ static int zend_update_type_info(const zend_op_array *op_array, if (tmp & MAY_BE_OBJECT) { tmp |= MAY_BE_RC1 | MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { // TODO: If there is no __set we might do better tmp = zend_fetch_prop_type(script, - zend_fetch_prop_info(op_array, ssa, opline, i), &ce); - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + zend_fetch_prop_info(op_array, ssa, opline, ssa_op), &ce); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def); } } if ((opline+1)->op1_type == IS_CV) { opline++; - i++; - tmp = OP1_INFO(); + ssa_op++; + tmp = OP1_INFO_EX(); if (tmp & (MAY_BE_ANY | MAY_BE_REF)) { if (tmp & MAY_BE_RC1) { tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_PRE_INC_OBJ: @@ -2777,24 +2788,24 @@ static int zend_update_type_info(const zend_op_array *op_array, if (tmp & MAY_BE_OBJECT) { tmp |= MAY_BE_RC1 | MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { // TODO: ??? tmp = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; case ZEND_ASSIGN: - if (opline->op2_type == IS_CV && ssa_ops[i].op2_def >= 0) { + if (opline->op2_type == IS_CV && ssa_op->op2_def >= 0) { tmp = t2; if (tmp & (MAY_BE_ANY | MAY_BE_REF)) { if (tmp & MAY_BE_RC1) { tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op2_def); } tmp = t2 & ~(MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); if (t2 & MAY_BE_UNDEF) { @@ -2813,17 +2824,17 @@ static int zend_update_type_info(const zend_op_array *op_array, if (RETURN_VALUE_USED(opline) && (tmp & MAY_BE_RC1)) { tmp |= MAY_BE_RCN; } - if (ssa_ops[i].op1_def >= 0) { - if (ssa_var_info[ssa_ops[i].op1_def].use_as_double) { + if (ssa_op->op1_def >= 0) { + if (ssa_var_info[ssa_op->op1_def].use_as_double) { tmp &= ~MAY_BE_LONG; tmp |= MAY_BE_DOUBLE; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { - UPDATE_SSA_TYPE(tmp & ~MAY_BE_REF, ssa_ops[i].result_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].result_def); + if (ssa_op->result_def >= 0) { + UPDATE_SSA_TYPE(tmp & ~MAY_BE_REF, ssa_op->result_def); + COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->result_def); } break; case ZEND_ASSIGN_REF: @@ -2833,7 +2844,7 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t2 & MAY_BE_UNDEF) { tmp |= MAY_BE_NULL; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op2_def); } if (opline->op2_type == IS_VAR && opline->extended_value == ZEND_RETURNS_FUNCTION) { tmp = (MAY_BE_REF | MAY_BE_RCN | MAY_BE_RC1 | t2) & ~MAY_BE_UNDEF; @@ -2843,9 +2854,9 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t2 & MAY_BE_UNDEF) { tmp |= MAY_BE_NULL; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - if (ssa_ops[i].result_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + if (ssa_op->result_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; case ZEND_ASSIGN_OBJ_REF: @@ -2858,11 +2869,11 @@ static int zend_update_type_info(const zend_op_array *op_array, if (tmp & MAY_BE_OBJECT) { tmp |= MAY_BE_RC1 | MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } - t2 = OP1_DATA_INFO(); + t2 = OP1_DATA_INFO_EX(); if ((opline+1)->op1_type == IS_VAR && (opline->extended_value & ZEND_RETURNS_FUNCTION)) { tmp = (MAY_BE_REF | MAY_BE_RCN | MAY_BE_RC1 | t2) & ~MAY_BE_UNDEF; } else { @@ -2871,30 +2882,30 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t2 & MAY_BE_UNDEF) { tmp |= MAY_BE_NULL; } - if (ssa_ops[i].result_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + if (ssa_op->result_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } if ((opline+1)->op1_type == IS_CV) { opline++; - i++; + ssa_op++; tmp = (MAY_BE_REF | t2) & ~(MAY_BE_UNDEF|MAY_BE_RC1|MAY_BE_RCN); if (t2 & MAY_BE_UNDEF) { tmp |= MAY_BE_NULL; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_ASSIGN_STATIC_PROP_REF: if ((opline+1)->op1_type == IS_CV) { opline++; - i++; - UPDATE_SSA_TYPE(MAY_BE_REF, ssa_ops[i].op1_def); + ssa_op++; + UPDATE_SSA_TYPE(MAY_BE_REF, ssa_op->op1_def); } break; case ZEND_BIND_GLOBAL: tmp = MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); break; case ZEND_BIND_STATIC: tmp = MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF @@ -2902,20 +2913,20 @@ static int zend_update_type_info(const zend_op_array *op_array, if (opline->extended_value & ZEND_BIND_IMPLICIT) { tmp |= MAY_BE_UNDEF; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); break; case ZEND_SEND_VAR: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = t1; if ((t1 & (MAY_BE_RC1|MAY_BE_REF)) && (opline->op1_type == IS_CV)) { tmp |= MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } break; case ZEND_BIND_LEXICAL: - if (ssa_ops[i].op2_def >= 0) { + if (ssa_op->op2_def >= 0) { if (opline->extended_value & ZEND_BIND_REF) { tmp = t2 | MAY_BE_REF; } else { @@ -2924,12 +2935,12 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].op2_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op2_def); + COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->op2_def); } break; case ZEND_YIELD: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) { tmp = t1 | MAY_BE_REF; } else { @@ -2938,30 +2949,30 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp = MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_RC1 | MAY_BE_RCN; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; case ZEND_SEND_VAR_EX: case ZEND_SEND_FUNC_ARG: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = (t1 & MAY_BE_UNDEF)|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_SEND_REF: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_SEND_UNPACK: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = t1; if (t1 & MAY_BE_ARRAY) { tmp |= MAY_BE_RC1 | MAY_BE_RCN; @@ -2973,14 +2984,14 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t1 & MAY_BE_OBJECT) { tmp |= MAY_BE_RC1 | MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_FAST_CONCAT: case ZEND_ROPE_INIT: case ZEND_ROPE_ADD: case ZEND_ROPE_END: - UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_op->result_def); break; case ZEND_RECV: case ZEND_RECV_INIT: @@ -3008,8 +3019,8 @@ static int zend_update_type_info(const zend_op_array *op_array, } #if 0 /* We won't receive unused arguments */ - if (ssa_vars[ssa_ops[i].result_def].use_chain < 0 && - ssa_vars[ssa_ops[i].result_def].phi_use_chain == NULL && + if (ssa_vars[ssa_op->result_def].use_chain < 0 && + ssa_vars[ssa_op->result_def].phi_use_chain == NULL && op_array->arg_info && opline->op1.num <= op_array->num_args && op_array->arg_info[opline->op1.num-1].class_name == NULL && @@ -3017,81 +3028,81 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp = MAY_BE_UNDEF|MAY_BE_RCN; } #endif - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (func_info && (int)opline->op1.num-1 < func_info->num_args && func_info->arg_info[opline->op1.num-1].info.ce) { UPDATE_SSA_OBJ_TYPE( func_info->arg_info[opline->op1.num-1].info.ce, func_info->arg_info[opline->op1.num-1].info.is_instanceof, - ssa_ops[i].result_def); + ssa_op->result_def); } else if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); } break; } case ZEND_DECLARE_ANON_CLASS: - UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_op->result_def); if (script && (ce = zend_hash_find_ptr(&script->class_table, Z_STR_P(CRT_CONSTANT(opline->op1)))) != NULL) { - UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_op->result_def); } break; case ZEND_FETCH_CLASS: - UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_op->result_def); if (opline->op2_type == IS_UNUSED) { switch (opline->op1.num & ZEND_FETCH_CLASS_MASK) { case ZEND_FETCH_CLASS_SELF: if (op_array->scope) { - UPDATE_SSA_OBJ_TYPE(op_array->scope, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(op_array->scope, 0, ssa_op->result_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); } break; case ZEND_FETCH_CLASS_PARENT: if (op_array->scope && op_array->scope->parent && (op_array->scope->ce_flags & ZEND_ACC_LINKED)) { - UPDATE_SSA_OBJ_TYPE(op_array->scope->parent, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(op_array->scope->parent, 0, ssa_op->result_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); } break; case ZEND_FETCH_CLASS_STATIC: default: - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); break; } } else if (opline->op2_type == IS_CONST) { zval *zv = CRT_CONSTANT(opline->op2); if (Z_TYPE_P(zv) == IS_STRING) { ce = get_class_entry(script, Z_STR_P(zv+1)); - UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_op->result_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); } } else { - COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].result_def); + COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->result_def); } break; case ZEND_NEW: tmp = MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT; if (opline->op1_type == IS_CONST && (ce = get_class_entry(script, Z_STR_P(CRT_CONSTANT(opline->op1)+1))) != NULL) { - UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_ops[i].result_def); - } else if ((t1 & MAY_BE_CLASS) && ssa_ops[i].op1_use >= 0 && ssa_var_info[ssa_ops[i].op1_use].ce) { - UPDATE_SSA_OBJ_TYPE(ssa_var_info[ssa_ops[i].op1_use].ce, ssa_var_info[ssa_ops[i].op1_use].is_instanceof, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_op->result_def); + } else if ((t1 & MAY_BE_CLASS) && ssa_op->op1_use >= 0 && ssa_var_info[ssa_op->op1_use].ce) { + UPDATE_SSA_OBJ_TYPE(ssa_var_info[ssa_op->op1_use].ce, ssa_var_info[ssa_op->op1_use].is_instanceof, ssa_op->result_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_CLONE: - UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_ops[i].result_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_op->result_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->result_def); break; case ZEND_INIT_ARRAY: case ZEND_ADD_ARRAY_ELEMENT: - if (opline->op1_type == IS_CV && ssa_ops[i].op1_def >= 0) { + if (opline->op1_type == IS_CV && ssa_op->op1_def >= 0) { if (opline->extended_value & ZEND_ARRAY_ELEMENT_REF) { tmp = (MAY_BE_REF | t1) & ~(MAY_BE_UNDEF|MAY_BE_RC1|MAY_BE_RCN); if (t1 & MAY_BE_UNDEF) { @@ -3110,12 +3121,12 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp = MAY_BE_RC1|MAY_BE_ARRAY; - if (ssa_ops[i].result_use >= 0) { - tmp |= ssa_var_info[ssa_ops[i].result_use].type; + if (ssa_op->result_use >= 0) { + tmp |= ssa_var_info[ssa_op->result_use].type; } if (opline->op1_type != IS_UNUSED) { tmp |= (t1 & MAY_BE_ANY) << MAY_BE_ARRAY_SHIFT; @@ -3143,11 +3154,11 @@ static int zend_update_type_info(const zend_op_array *op_array, } } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; case ZEND_ADD_ARRAY_UNPACK: - tmp = ssa_var_info[ssa_ops[i].result_use].type; + tmp = ssa_var_info[ssa_op->result_use].type; ZEND_ASSERT(tmp & MAY_BE_ARRAY); /* Ignore string keys as they will throw. */ if (t1 & MAY_BE_ARRAY_KEY_LONG) { @@ -3156,7 +3167,7 @@ static int zend_update_type_info(const zend_op_array *op_array, if (t1 & MAY_BE_OBJECT) { tmp |= MAY_BE_ARRAY_KEY_LONG | MAY_BE_ARRAY_OF_ANY; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_UNSET_CV: tmp = MAY_BE_UNDEF; @@ -3164,18 +3175,18 @@ static int zend_update_type_info(const zend_op_array *op_array, /* In global scope, we know nothing */ tmp |= MAY_BE_REF; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); break; case ZEND_UNSET_DIM: case ZEND_UNSET_OBJ: - if (ssa_ops[i].op1_def >= 0) { - UPDATE_SSA_TYPE(t1, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + if (ssa_op->op1_def >= 0) { + UPDATE_SSA_TYPE(t1, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } break; case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = t1; if (opline->opcode == ZEND_FE_RESET_RW) { tmp |= MAY_BE_REF; @@ -3184,8 +3195,8 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } if (opline->opcode == ZEND_FE_RESET_RW) { //??? @@ -3193,8 +3204,8 @@ static int zend_update_type_info(const zend_op_array *op_array, } else { tmp = MAY_BE_RC1 | MAY_BE_RCN | (t1 & (MAY_BE_ARRAY | MAY_BE_OBJECT | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF)); } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->result_def); break; case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: @@ -3221,9 +3232,9 @@ static int zend_update_type_info(const zend_op_array *op_array, } } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def); - if (ssa_ops[i].result_def >= 0) { - tmp = (ssa_ops[i].result_use >= 0) ? RES_USE_INFO() : 0; + UPDATE_SSA_TYPE(tmp, ssa_op->op2_def); + if (ssa_op->result_def >= 0) { + tmp = (ssa_op->result_use >= 0) ? RES_USE_INFO_EX() : 0; if (t1 & MAY_BE_OBJECT) { tmp |= MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -3235,7 +3246,7 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp |= MAY_BE_STRING | MAY_BE_RCN; } } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; case ZEND_FETCH_DIM_R: @@ -3246,7 +3257,7 @@ static int zend_update_type_info(const zend_op_array *op_array, case ZEND_FETCH_DIM_FUNC_ARG: case ZEND_FETCH_LIST_R: case ZEND_FETCH_LIST_W: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { uint32_t key_type = 0; tmp = t1 & ~(MAY_BE_RC1|MAY_BE_RCN); if (opline->opcode == ZEND_FETCH_DIM_W || @@ -3297,10 +3308,18 @@ static int zend_update_type_info(const zend_op_array *op_array, || opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_DIM_FUNC_ARG || opline->opcode == ZEND_FETCH_LIST_W) { - j = ssa_vars[ssa_ops[i].result_def].use_chain; + j = ssa_vars[ssa_op->result_def].use_chain; while (j >= 0) { - ZEND_ASSERT(j == i + 1 && "Use must be in next opline"); - switch (op_array->opcodes[j].opcode) { + zend_uchar opcode; + + if (!ssa_opcodes) { + ZEND_ASSERT(j == (opline - op_array->opcodes) + 1 && "Use must be in next opline"); + opcode = op_array->opcodes[j].opcode; + } else { + ZEND_ASSERT(ssa_opcodes[j] == opline + 1 && "Use must be in next opline"); + opcode = ssa_opcodes[j]->opcode; + } + switch (opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_DIM_RW: case ZEND_FETCH_DIM_FUNC_ARG: @@ -3354,18 +3373,18 @@ static int zend_update_type_info(const zend_op_array *op_array, break; EMPTY_SWITCH_DEFAULT_CASE() } - j = zend_ssa_next_use(ssa_ops, ssa_ops[i].result_def, j); + j = zend_ssa_next_use(ssa->ops, ssa_op->result_def, j); ZEND_ASSERT(j < 0 && "There should only be one use"); } } if ((tmp & MAY_BE_ARRAY) && (tmp & MAY_BE_ARRAY_KEY_ANY)) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } else { /* invalid key type */ tmp = (tmp & (MAY_BE_RC1|MAY_BE_RCN)) | (t1 & ~(MAY_BE_RC1|MAY_BE_RCN)); - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } - COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def); + COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def); } /* FETCH_LIST on a string behaves like FETCH_R on null */ tmp = zend_array_element_type( @@ -3375,11 +3394,11 @@ static int zend_update_type_info(const zend_op_array *op_array, if (opline->opcode == ZEND_FETCH_DIM_IS && (t1 & MAY_BE_STRING)) { tmp |= MAY_BE_NULL; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_FETCH_THIS: - UPDATE_SSA_OBJ_TYPE(op_array->scope, 1, ssa_ops[i].result_def); - UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(op_array->scope, 1, ssa_op->result_def); + UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_op->result_def); break; case ZEND_FETCH_OBJ_R: case ZEND_FETCH_OBJ_IS: @@ -3387,15 +3406,15 @@ static int zend_update_type_info(const zend_op_array *op_array, case ZEND_FETCH_OBJ_W: case ZEND_FETCH_OBJ_UNSET: case ZEND_FETCH_OBJ_FUNC_ARG: - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp = zend_fetch_prop_type(script, - zend_fetch_prop_info(op_array, ssa, opline, i), &ce); + zend_fetch_prop_info(op_array, ssa, opline, ssa_op), &ce); if (opline->result_type != IS_TMP_VAR) { tmp |= MAY_BE_REF; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def); } } break; @@ -3410,16 +3429,16 @@ static int zend_update_type_info(const zend_op_array *op_array, if (opline->result_type != IS_TMP_VAR) { tmp |= MAY_BE_REF; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def); } break; case ZEND_DO_FCALL: case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { zend_func_info *func_info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info; @@ -3431,46 +3450,46 @@ static int zend_update_type_info(const zend_op_array *op_array, goto unknown_opcode; } tmp = zend_get_func_info(call_info, ssa); - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (call_info->callee_func->type == ZEND_USER_FUNCTION) { func_info = ZEND_FUNC_INFO(&call_info->callee_func->op_array); if (func_info) { UPDATE_SSA_OBJ_TYPE( func_info->return_info.ce, func_info->return_info.is_instanceof, - ssa_ops[i].result_def); + ssa_op->result_def); } } } break; case ZEND_FETCH_CONSTANT: case ZEND_FETCH_CLASS_CONSTANT: - UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY, ssa_op->result_def); break; case ZEND_STRLEN: tmp = MAY_BE_LONG; if (t1 & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING))) { tmp |= MAY_BE_NULL; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); break; case ZEND_COUNT: case ZEND_FUNC_NUM_ARGS: - UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_op->result_def); break; case ZEND_FUNC_GET_ARGS: - UPDATE_SSA_TYPE(MAY_BE_RC1| MAY_BE_ARRAY | MAY_BE_ARRAY_KEY_LONG | MAY_BE_ARRAY_OF_ANY, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_RC1| MAY_BE_ARRAY | MAY_BE_ARRAY_KEY_LONG | MAY_BE_ARRAY_OF_ANY, ssa_op->result_def); break; case ZEND_GET_CLASS: case ZEND_GET_CALLED_CLASS: - UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_STRING|MAY_BE_RCN, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_STRING|MAY_BE_RCN, ssa_op->result_def); break; case ZEND_GET_TYPE: - UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_op->result_def); break; case ZEND_TYPE_CHECK: case ZEND_DEFINED: - UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_op->result_def); break; case ZEND_VERIFY_RETURN_TYPE: if (t1 & MAY_BE_REF) { @@ -3481,26 +3500,26 @@ static int zend_update_type_info(const zend_op_array *op_array, tmp = zend_fetch_arg_info_type(script, ret_info, &ce); } if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].op1_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->op1_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].op1_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->op1_def); } } else { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); if (ce) { - UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def); } else { - UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def); + UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def); } } break; case ZEND_MAKE_REF: tmp = MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); - if (ssa_ops[i].op1_def >= 0) { - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); + if (ssa_op->op1_def >= 0) { + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } break; case ZEND_CATCH: @@ -3510,18 +3529,18 @@ static int zend_update_type_info(const zend_op_array *op_array, break; default: unknown_opcode: - if (ssa_ops[i].op1_def >= 0) { + if (ssa_op->op1_def >= 0) { tmp = MAY_BE_ANY | MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; - UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def); + UPDATE_SSA_TYPE(tmp, ssa_op->op1_def); } - if (ssa_ops[i].result_def >= 0) { + if (ssa_op->result_def >= 0) { tmp = MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; if (opline->result_type == IS_TMP_VAR) { tmp |= MAY_BE_RC1 | MAY_BE_RCN; } else { tmp |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN; } - UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def); + UPDATE_SSA_TYPE(tmp, ssa_op->result_def); } break; } @@ -3529,6 +3548,18 @@ unknown_opcode: return SUCCESS; } +int zend_update_type_info( + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_script *script, + zend_op *opline, + zend_ssa_op *ssa_op, + const zend_op **ssa_opcodes, + zend_long optimization_level) +{ + return _zend_update_type_info(op_array, ssa, script, NULL, opline, ssa_op, ssa_opcodes, optimization_level, 0); +} + static uint32_t get_class_entry_rank(zend_class_entry *ce) { uint32_t rank = 0; if (ce->ce_flags & ZEND_ACC_LINKED) { @@ -3583,6 +3614,7 @@ int zend_infer_types_ex(const zend_op_array *op_array, const zend_script *script int ssa_vars_count = ssa->vars_count; int i, j; uint32_t tmp, worklist_len = zend_bitset_len(ssa_vars_count); + zend_bool update_worklist = 1; while (!zend_bitset_empty(worklist, worklist_len)) { j = zend_bitset_first(worklist, worklist_len); @@ -3645,7 +3677,7 @@ int zend_infer_types_ex(const zend_op_array *op_array, const zend_script *script } } else if (ssa_vars[j].definition >= 0) { i = ssa_vars[j].definition; - if (zend_update_type_info(op_array, ssa, script, worklist, i, optimization_level) == FAILURE) { + if (_zend_update_type_info(op_array, ssa, script, worklist, op_array->opcodes + i, ssa->ops + i, NULL, optimization_level, 1) == FAILURE) { return FAILURE; } } @@ -4220,10 +4252,10 @@ void zend_inference_check_recursive_dependencies(zend_op_array *op_array) free_alloca(worklist, use_heap); } -int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +int zend_may_throw_ex(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa) { - uint32_t t1 = OP1_INFO(); - uint32_t t2 = OP2_INFO(); + uint32_t t1 = OP1_INFO_EX(); + uint32_t t2 = OP2_INFO_EX(); if (opline->op1_type == IS_CV) { if (t1 & MAY_BE_UNDEF) { @@ -4331,7 +4363,7 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss return 0; case ZEND_BIND_GLOBAL: if ((opline+1)->opcode == ZEND_BIND_GLOBAL) { - return zend_may_throw(opline + 1, op_array, ssa); + return zend_may_throw_ex(opline + 1, ssa_op + 1, op_array, ssa); } return 0; case ZEND_ADD: @@ -4343,8 +4375,8 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); case ZEND_DIV: case ZEND_MOD: - if (!OP2_HAS_RANGE() || - (OP2_MIN_RANGE() <= 0 && OP2_MAX_RANGE() >= 0)) { + if (!OP2_HAS_RANGE_EX() || + (OP2_MIN_RANGE_EX() <= 0 && OP2_MAX_RANGE_EX() >= 0)) { /* Division by zero */ return 1; } @@ -4358,8 +4390,8 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss case ZEND_SR: return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || - !OP2_HAS_RANGE() || - OP2_MIN_RANGE() < 0; + !OP2_HAS_RANGE_EX() || + OP2_MIN_RANGE_EX() < 0; case ZEND_CONCAT: case ZEND_FAST_CONCAT: return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) || @@ -4411,8 +4443,8 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)); } else if (opline->extended_value == ZEND_DIV || opline->extended_value == ZEND_MOD) { - if (!OP2_HAS_RANGE() || - (OP2_MIN_RANGE() <= 0 && OP2_MAX_RANGE() >= 0)) { + if (!OP2_HAS_RANGE_EX() || + (OP2_MIN_RANGE_EX() <= 0 && OP2_MAX_RANGE_EX() >= 0)) { /* Division by zero */ return 1; } @@ -4427,8 +4459,8 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss opline->extended_value == ZEND_SR) { return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || (t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) || - !OP2_HAS_RANGE() || - OP2_MIN_RANGE() < 0; + !OP2_HAS_RANGE_EX() || + OP2_MIN_RANGE_EX() < 0; } else if (opline->extended_value == ZEND_CONCAT) { return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) || (t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT)); @@ -4452,7 +4484,7 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss return (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)); case ZEND_ASSIGN_DIM: if ((opline+1)->op1_type == IS_CV) { - if (_ssa_op1_info(op_array, ssa, opline+1) & MAY_BE_UNDEF) { + if (_ssa_op1_info_ex(op_array, ssa, opline+1, ssa_op+1) & MAY_BE_UNDEF) { return 1; } } @@ -4462,8 +4494,8 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss if (t1 & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_OBJECT))) { return 1; } - if (ssa->ops[opline - op_array->opcodes].op1_use) { - zend_ssa_var_info *var_info = ssa->var_info + ssa->ops[opline - op_array->opcodes].op1_use; + if (ssa_op->op1_use) { + zend_ssa_var_info *var_info = ssa->var_info + ssa_op->op1_use; zend_class_entry *ce = var_info->ce; if (var_info->is_instanceof || @@ -4555,3 +4587,8 @@ int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ss return 1; } } + +int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +{ + return zend_may_throw_ex(opline, &ssa->ops[opline - op_array->opcodes], op_array, ssa); +} diff --git a/ext/opcache/Optimizer/zend_inference.h b/ext/opcache/Optimizer/zend_inference.h index f01f6cc786..c5db1feede 100644 --- a/ext/opcache/Optimizer/zend_inference.h +++ b/ext/opcache/Optimizer/zend_inference.h @@ -26,6 +26,7 @@ /* Bitmask for type inference (zend_ssa_var_info.type) */ #include "zend_type_info.h" +#define MAY_BE_GUARD (1<<28) /* needs type guard */ #define MAY_BE_IN_REG (1<<29) /* value allocated in CPU register */ //TODO: remome MAY_BE_RC1, MAY_BE_RCN??? @@ -37,23 +38,26 @@ |MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE) #define DEFINE_SSA_OP_HAS_RANGE(opN) \ - static zend_always_inline zend_bool _ssa_##opN##_has_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + static zend_always_inline zend_bool _ssa_##opN##_has_range_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ { \ if (opline->opN##_type == IS_CONST) { \ zval *zv = CRT_CONSTANT(opline->opN); \ return (Z_TYPE_P(zv) == IS_LONG || Z_TYPE_P(zv) == IS_TRUE || Z_TYPE_P(zv) == IS_FALSE || Z_TYPE_P(zv) == IS_NULL); \ } else { \ return (opline->opN##_type != IS_UNUSED && \ - ssa->ops && \ ssa->var_info && \ - ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \ - ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range); \ + ssa_op->opN##_use >= 0 && \ + ssa->var_info[ssa_op->opN##_use].has_range); \ } \ return 0; \ + } \ + static zend_always_inline zend_bool _ssa_##opN##_has_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + { \ + return _ssa_##opN##_has_range_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } #define DEFINE_SSA_OP_MIN_RANGE(opN) \ - static zend_always_inline zend_long _ssa_##opN##_min_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + static zend_always_inline zend_long _ssa_##opN##_min_range_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ { \ if (opline->opN##_type == IS_CONST) { \ zval *zv = CRT_CONSTANT(opline->opN); \ @@ -67,17 +71,20 @@ return 0; \ } \ } else if (opline->opN##_type != IS_UNUSED && \ - ssa->ops && \ ssa->var_info && \ - ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \ - ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \ - return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.min; \ + ssa_op->opN##_use >= 0 && \ + ssa->var_info[ssa_op->opN##_use].has_range) { \ + return ssa->var_info[ssa_op->opN##_use].range.min; \ } \ return ZEND_LONG_MIN; \ + } \ + static zend_always_inline zend_long _ssa_##opN##_min_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + { \ + return _ssa_##opN##_min_range_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } #define DEFINE_SSA_OP_MAX_RANGE(opN) \ - static zend_always_inline zend_long _ssa_##opN##_max_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + static zend_always_inline zend_long _ssa_##opN##_max_range_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ { \ if (opline->opN##_type == IS_CONST) { \ zval *zv = CRT_CONSTANT(opline->opN); \ @@ -91,17 +98,20 @@ return 0; \ } \ } else if (opline->opN##_type != IS_UNUSED && \ - ssa->ops && \ ssa->var_info && \ - ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \ - ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \ - return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.max; \ + ssa_op->opN##_use >= 0 && \ + ssa->var_info[ssa_op->opN##_use].has_range) { \ + return ssa->var_info[ssa_op->opN##_use].range.max; \ } \ return ZEND_LONG_MAX; \ + } \ + static zend_always_inline zend_long _ssa_##opN##_max_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + { \ + return _ssa_##opN##_max_range_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } #define DEFINE_SSA_OP_RANGE_UNDERFLOW(opN) \ - static zend_always_inline char _ssa_##opN##_range_underflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + static zend_always_inline char _ssa_##opN##_range_underflow_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ { \ if (opline->opN##_type == IS_CONST) { \ zval *zv = CRT_CONSTANT(opline->opN); \ @@ -109,17 +119,20 @@ return 0; \ } \ } else if (opline->opN##_type != IS_UNUSED && \ - ssa->ops && \ ssa->var_info && \ - ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \ - ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \ - return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.underflow; \ + ssa_op->opN##_use >= 0 && \ + ssa->var_info[ssa_op->opN##_use].has_range) { \ + return ssa->var_info[ssa_op->opN##_use].range.underflow; \ } \ return 1; \ + } \ + static zend_always_inline char _ssa_##opN##_range_underflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + { \ + return _ssa_##opN##_range_underflow_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } #define DEFINE_SSA_OP_RANGE_OVERFLOW(opN) \ - static zend_always_inline char _ssa_##opN##_range_overflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + static zend_always_inline char _ssa_##opN##_range_overflow_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ { \ if (opline->opN##_type == IS_CONST) { \ zval *zv = CRT_CONSTANT(opline->opN); \ @@ -127,13 +140,16 @@ return 0; \ } \ } else if (opline->opN##_type != IS_UNUSED && \ - ssa->ops && \ ssa->var_info && \ - ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \ - ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \ - return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.overflow; \ + ssa_op->opN##_use >= 0 && \ + ssa->var_info[ssa_op->opN##_use].has_range) { \ + return ssa->var_info[ssa_op->opN##_use].range.overflow; \ } \ return 1; \ + } \ + static zend_always_inline char _ssa_##opN##_range_overflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + { \ + return _ssa_##opN##_range_overflow_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } DEFINE_SSA_OP_HAS_RANGE(op1) @@ -158,6 +174,17 @@ DEFINE_SSA_OP_RANGE_OVERFLOW(op2) #define OP2_RANGE_UNDERFLOW() (_ssa_op2_range_underflow (op_array, ssa, opline)) #define OP2_RANGE_OVERFLOW() (_ssa_op2_range_overflow (op_array, ssa, opline)) +#define OP1_HAS_RANGE_EX() (_ssa_op1_has_range_ex (op_array, ssa, opline, ssa_op)) +#define OP1_MIN_RANGE_EX() (_ssa_op1_min_range_ex (op_array, ssa, opline, ssa_op)) +#define OP1_MAX_RANGE_EX() (_ssa_op1_max_range_ex (op_array, ssa, opline, ssa_op)) +#define OP1_RANGE_UNDERFLOW_EX() (_ssa_op1_range_underflow_ex (op_array, ssa, opline, ssa_op)) +#define OP1_RANGE_OVERFLOW_EX() (_ssa_op1_range_overflow_ex (op_array, ssa, opline, ssa_op)) +#define OP2_HAS_RANGE_EX() (_ssa_op2_has_range_ex (op_array, ssa, opline, ssa_op)) +#define OP2_MIN_RANGE_EX() (_ssa_op2_min_range_ex (op_array, ssa, opline, ssa_op)) +#define OP2_MAX_RANGE_EX() (_ssa_op2_max_range_ex (op_array, ssa, opline, ssa_op)) +#define OP2_RANGE_UNDERFLOW_EX() (_ssa_op2_range_underflow_ex (op_array, ssa, opline, ssa_op)) +#define OP2_RANGE_OVERFLOW_EX() (_ssa_op2_range_overflow_ex (op_array, ssa, opline, ssa_op)) + static zend_always_inline uint32_t _const_op_type(const zval *zv) { if (Z_TYPE_P(zv) == IS_CONSTANT_AST) { return MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY; @@ -204,19 +231,27 @@ static zend_always_inline uint32_t get_ssa_var_info(const zend_ssa *ssa, int ssa } #define DEFINE_SSA_OP_INFO(opN) \ - static zend_always_inline uint32_t _ssa_##opN##_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + static zend_always_inline uint32_t _ssa_##opN##_info_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ { \ if (opline->opN##_type == IS_CONST) { \ return _const_op_type(CRT_CONSTANT(opline->opN)); \ } else { \ - return get_ssa_var_info(ssa, ssa->ops ? ssa->ops[opline - op_array->opcodes].opN##_use : -1); \ + return get_ssa_var_info(ssa, ssa->var_info ? ssa_op->opN##_use : -1); \ } \ + } \ + static zend_always_inline uint32_t _ssa_##opN##_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ + { \ + return _ssa_##opN##_info_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } #define DEFINE_SSA_OP_DEF_INFO(opN) \ + static zend_always_inline uint32_t _ssa_##opN##_def_info_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \ + { \ + return get_ssa_var_info(ssa, ssa->var_info ? ssa_op->opN##_def : -1); \ + } \ static zend_always_inline uint32_t _ssa_##opN##_def_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \ { \ - return get_ssa_var_info(ssa, ssa->ops ? ssa->ops[opline - op_array->opcodes].opN##_def : -1); \ + return _ssa_##opN##_def_info_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \ } @@ -238,6 +273,17 @@ DEFINE_SSA_OP_DEF_INFO(result) #define OP2_DATA_DEF_INFO() (_ssa_op2_def_info(op_array, ssa, (opline+1))) #define RES_INFO() (_ssa_result_def_info(op_array, ssa, opline)) +#define OP1_INFO_EX() (_ssa_op1_info_ex(op_array, ssa, opline, ssa_op)) +#define OP2_INFO_EX() (_ssa_op2_info_ex(op_array, ssa, opline, ssa_op)) +#define OP1_DATA_INFO_EX() (_ssa_op1_info_ex(op_array, ssa, (opline+1), (ssa_op+1))) +#define OP2_DATA_INFO_EX() (_ssa_op2_info_ex(op_array, ssa, (opline+1), (ssa_op+1))) +#define RES_USE_INFO_EX() (_ssa_result_info_ex(op_array, ssa, opline, ssa_op)) +#define OP1_DEF_INFO_EX() (_ssa_op1_def_info_ex(op_array, ssa, opline, ssa_op)) +#define OP2_DEF_INFO_EX() (_ssa_op2_def_info_ex(op_array, ssa, opline, ssa_op)) +#define OP1_DATA_DEF_INFO_EX() (_ssa_op1_def_info_ex(op_array, ssa, (opline+1), (ssa_op+1))) +#define OP2_DATA_DEF_INFO_EX() (_ssa_op2_def_info_ex(op_array, ssa, (opline+1), (ssa_op+1))) +#define RES_INFO_EX() (_ssa_result_def_info_ex(op_array, ssa, opline, ssa_op)) + static zend_always_inline zend_bool zend_add_will_overflow(zend_long a, zend_long b) { return (b > 0 && a > ZEND_LONG_MAX - b) || (b < 0 && a < ZEND_LONG_MIN - b); @@ -274,8 +320,17 @@ void zend_func_return_info(const zend_op_array *op_array, int widening, zend_ssa_var_info *ret); +int zend_may_throw_ex(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa); int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa); +int zend_update_type_info(const zend_op_array *op_array, + zend_ssa *ssa, + const zend_script *script, + zend_op *opline, + zend_ssa_op *ssa_op, + const zend_op **ssa_opcodes, + zend_long optimization_level); + END_EXTERN_C() #endif /* ZEND_INFERENCE_H */ diff --git a/ext/opcache/Optimizer/zend_ssa.c b/ext/opcache/Optimizer/zend_ssa.c index 4cd9619f20..10e772d4ba 100644 --- a/ext/opcache/Optimizer/zend_ssa.c +++ b/ext/opcache/Optimizer/zend_ssa.c @@ -537,6 +537,237 @@ static void place_essa_pis( } /* }}} */ +static zend_always_inline int _zend_ssa_rename_op(const zend_op_array *op_array, const zend_op *opline, uint32_t k, uint32_t build_flags, int ssa_vars_count, zend_ssa_op *ssa_ops, int *var) /* {{{ */ +{ + const zend_op *next; + + if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k].op1_use = var[EX_VAR_TO_NUM(opline->op1.var)]; + //USE_SSA_VAR(op_array->last_var + opline->op1.var) + } + if (opline->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k].op2_use = var[EX_VAR_TO_NUM(opline->op2.var)]; + //USE_SSA_VAR(op_array->last_var + opline->op2.var) + } + if ((build_flags & ZEND_SSA_USE_CV_RESULTS) + && opline->result_type == IS_CV + && opline->opcode != ZEND_RECV) { + ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)]; + //USE_SSA_VAR(op_array->last_var + opline->result.var) + } + + switch (opline->opcode) { + case ZEND_ASSIGN: + if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op2_type == IS_CV) { + ssa_ops[k].op2_def = ssa_vars_count; + var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(opline->op2.var) + } + if (opline->op1_type == IS_CV) { +add_op1_def: + ssa_ops[k].op1_def = ssa_vars_count; + var[EX_VAR_TO_NUM(opline->op1.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(opline->op1.var) + } + break; + case ZEND_ASSIGN_REF: + if (opline->op2_type == IS_CV) { + ssa_ops[k].op2_def = ssa_vars_count; + var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(opline->op2.var) + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; + //USE_SSA_VAR(op_array->last_var + next->op1.var); + if (build_flags & ZEND_SSA_RC_INFERENCE && next->op1_type == IS_CV) { + ssa_ops[k + 1].op1_def = ssa_vars_count; + var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(next->op1.var) + } + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_OBJ_REF: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; + //USE_SSA_VAR(op_array->last_var + next->op1.var); + if (next->op1_type == IS_CV) { + ssa_ops[k + 1].op1_def = ssa_vars_count; + var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(next->op1.var) + } + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_STATIC_PROP: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; + //USE_SSA_VAR(op_array->last_var + next->op1.var); +#if 0 + if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) { + ssa_ops[k + 1].op1_def = ssa_vars_count; + var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(next->op1.var) + } +#endif + } + break; + case ZEND_ASSIGN_STATIC_PROP_REF: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; + //USE_SSA_VAR(op_array->last_var + next->op1.var); + if (next->op1_type == IS_CV) { + ssa_ops[k + 1].op1_def = ssa_vars_count; + var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(next->op1.var) + } + } + break; + case ZEND_ASSIGN_STATIC_PROP_OP: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; + //USE_SSA_VAR(op_array->last_var + next->op1.var); + } + break; + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_OBJ_OP: + next = opline + 1; + if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; + //USE_SSA_VAR(op_array->last_var + next->op1.var); + } + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ASSIGN_OP: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + case ZEND_BIND_GLOBAL: + case ZEND_BIND_STATIC: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_UNPACK: + case ZEND_FE_RESET_RW: + case ZEND_MAKE_REF: + case ZEND_PRE_INC_OBJ: + case ZEND_PRE_DEC_OBJ: + case ZEND_POST_INC_OBJ: + case ZEND_POST_DEC_OBJ: + case ZEND_UNSET_DIM: + case ZEND_UNSET_OBJ: + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_DIM_RW: + case ZEND_FETCH_DIM_FUNC_ARG: + case ZEND_FETCH_DIM_UNSET: + case ZEND_FETCH_LIST_W: + if (opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_SEND_VAR: + case ZEND_CAST: + case ZEND_QM_ASSIGN: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_FE_RESET_R: + if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_ADD_ARRAY_UNPACK: + ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)]; + break; + case ZEND_ADD_ARRAY_ELEMENT: + ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)]; + /* break missing intentionally */ + case ZEND_INIT_ARRAY: + if (((build_flags & ZEND_SSA_RC_INFERENCE) + || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF)) + && opline->op1_type == IS_CV) { + goto add_op1_def; + } + break; + case ZEND_YIELD: + if (opline->op1_type == IS_CV + && ((op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) + || (build_flags & ZEND_SSA_RC_INFERENCE))) { + goto add_op1_def; + } + break; + case ZEND_UNSET_CV: + goto add_op1_def; + case ZEND_VERIFY_RETURN_TYPE: + if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { + goto add_op1_def; + } + break; + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + if (opline->op2_type != IS_CV) { + ssa_ops[k].op2_use = -1; /* not used */ + } + ssa_ops[k].op2_def = ssa_vars_count; + var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(opline->op2.var) + break; + case ZEND_BIND_LEXICAL: + if ((opline->extended_value & ZEND_BIND_REF) || (build_flags & ZEND_SSA_RC_INFERENCE)) { + ssa_ops[k].op2_def = ssa_vars_count; + var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(opline->op2.var) + } + break; + default: + break; + } + + if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { + ssa_ops[k].result_def = ssa_vars_count; + var[EX_VAR_TO_NUM(opline->result.var)] = ssa_vars_count; + ssa_vars_count++; + //NEW_SSA_VAR(op_array->last_var + opline->result.var) + } + + return ssa_vars_count; +} +/* }}} */ + +int zend_ssa_rename_op(const zend_op_array *op_array, const zend_op *opline, uint32_t k, uint32_t build_flags, int ssa_vars_count, zend_ssa_op *ssa_ops, int *var) /* {{{ */ +{ + return _zend_ssa_rename_op(op_array, opline, k, build_flags, ssa_vars_count, ssa_ops, var); +} +/* }}} */ + static int zend_ssa_rename(const zend_op_array *op_array, uint32_t build_flags, zend_ssa *ssa, int *var, int n) /* {{{ */ { zend_basic_block *blocks = ssa->cfg.blocks; @@ -544,7 +775,7 @@ static int zend_ssa_rename(const zend_op_array *op_array, uint32_t build_flags, zend_ssa_op *ssa_ops = ssa->ops; int ssa_vars_count = ssa->vars_count; int i, j; - zend_op *opline, *end, *next; + zend_op *opline, *end; int *tmp = NULL; ALLOCA_FLAG(use_heap = 0); @@ -574,223 +805,7 @@ static int zend_ssa_rename(const zend_op_array *op_array, uint32_t build_flags, for (; opline < end; opline++) { uint32_t k = opline - op_array->opcodes; if (opline->opcode != ZEND_OP_DATA) { - - if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k].op1_use = var[EX_VAR_TO_NUM(opline->op1.var)]; - //USE_SSA_VAR(op_array->last_var + opline->op1.var) - } - if (opline->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k].op2_use = var[EX_VAR_TO_NUM(opline->op2.var)]; - //USE_SSA_VAR(op_array->last_var + opline->op2.var) - } - if ((build_flags & ZEND_SSA_USE_CV_RESULTS) - && opline->result_type == IS_CV - && opline->opcode != ZEND_RECV) { - ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)]; - //USE_SSA_VAR(op_array->last_var + opline->result.var) - } - - switch (opline->opcode) { - case ZEND_ASSIGN: - if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op2_type == IS_CV) { - ssa_ops[k].op2_def = ssa_vars_count; - var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(opline->op2.var) - } - if (opline->op1_type == IS_CV) { -add_op1_def: - ssa_ops[k].op1_def = ssa_vars_count; - var[EX_VAR_TO_NUM(opline->op1.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(opline->op1.var) - } - break; - case ZEND_ASSIGN_REF: - if (opline->op2_type == IS_CV) { - ssa_ops[k].op2_def = ssa_vars_count; - var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(opline->op2.var) - } - if (opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_ASSIGN_DIM: - case ZEND_ASSIGN_OBJ: - next = opline + 1; - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; - //USE_SSA_VAR(op_array->last_var + next->op1.var); - if (build_flags & ZEND_SSA_RC_INFERENCE && next->op1_type == IS_CV) { - ssa_ops[k + 1].op1_def = ssa_vars_count; - var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(next->op1.var) - } - } - if (opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_ASSIGN_OBJ_REF: - next = opline + 1; - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; - //USE_SSA_VAR(op_array->last_var + next->op1.var); - if (next->op1_type == IS_CV) { - ssa_ops[k + 1].op1_def = ssa_vars_count; - var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(next->op1.var) - } - } - if (opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_ASSIGN_STATIC_PROP: - next = opline + 1; - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; - //USE_SSA_VAR(op_array->last_var + next->op1.var); -#if 0 - if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) { - ssa_ops[k + 1].op1_def = ssa_vars_count; - var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(next->op1.var) - } -#endif - } - break; - case ZEND_ASSIGN_STATIC_PROP_REF: - next = opline + 1; - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; - //USE_SSA_VAR(op_array->last_var + next->op1.var); - if (next->op1_type == IS_CV) { - ssa_ops[k + 1].op1_def = ssa_vars_count; - var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(next->op1.var) - } - } - break; - case ZEND_ASSIGN_STATIC_PROP_OP: - next = opline + 1; - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; - //USE_SSA_VAR(op_array->last_var + next->op1.var); - } - break; - case ZEND_ASSIGN_DIM_OP: - case ZEND_ASSIGN_OBJ_OP: - next = opline + 1; - if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)]; - //USE_SSA_VAR(op_array->last_var + next->op1.var); - } - if (opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_ASSIGN_OP: - case ZEND_PRE_INC: - case ZEND_PRE_DEC: - case ZEND_POST_INC: - case ZEND_POST_DEC: - case ZEND_BIND_GLOBAL: - case ZEND_BIND_STATIC: - case ZEND_SEND_VAR_NO_REF: - case ZEND_SEND_VAR_NO_REF_EX: - case ZEND_SEND_VAR_EX: - case ZEND_SEND_FUNC_ARG: - case ZEND_SEND_REF: - case ZEND_SEND_UNPACK: - case ZEND_FE_RESET_RW: - case ZEND_MAKE_REF: - case ZEND_PRE_INC_OBJ: - case ZEND_PRE_DEC_OBJ: - case ZEND_POST_INC_OBJ: - case ZEND_POST_DEC_OBJ: - case ZEND_UNSET_DIM: - case ZEND_UNSET_OBJ: - case ZEND_FETCH_DIM_W: - case ZEND_FETCH_DIM_RW: - case ZEND_FETCH_DIM_FUNC_ARG: - case ZEND_FETCH_DIM_UNSET: - case ZEND_FETCH_LIST_W: - if (opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_SEND_VAR: - case ZEND_CAST: - case ZEND_QM_ASSIGN: - case ZEND_JMP_SET: - case ZEND_COALESCE: - case ZEND_FE_RESET_R: - if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_ADD_ARRAY_UNPACK: - ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)]; - break; - case ZEND_ADD_ARRAY_ELEMENT: - ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)]; - /* break missing intentionally */ - case ZEND_INIT_ARRAY: - if (((build_flags & ZEND_SSA_RC_INFERENCE) - || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF)) - && opline->op1_type == IS_CV) { - goto add_op1_def; - } - break; - case ZEND_YIELD: - if (opline->op1_type == IS_CV - && ((op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) - || (build_flags & ZEND_SSA_RC_INFERENCE))) { - goto add_op1_def; - } - break; - case ZEND_UNSET_CV: - goto add_op1_def; - case ZEND_VERIFY_RETURN_TYPE: - if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { - goto add_op1_def; - } - break; - case ZEND_FE_FETCH_R: - case ZEND_FE_FETCH_RW: - if (opline->op2_type != IS_CV) { - ssa_ops[k].op2_use = -1; /* not used */ - } - ssa_ops[k].op2_def = ssa_vars_count; - var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(opline->op2.var) - break; - case ZEND_BIND_LEXICAL: - if ((opline->extended_value & ZEND_BIND_REF) || (build_flags & ZEND_SSA_RC_INFERENCE)) { - ssa_ops[k].op2_def = ssa_vars_count; - var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(opline->op2.var) - } - break; - default: - break; - } - - if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) { - ssa_ops[k].result_def = ssa_vars_count; - var[EX_VAR_TO_NUM(opline->result.var)] = ssa_vars_count; - ssa_vars_count++; - //NEW_SSA_VAR(op_array->last_var + opline->result.var) - } + ssa_vars_count = _zend_ssa_rename_op(op_array, opline, k, build_flags, ssa_vars_count, ssa_ops, var); } } diff --git a/ext/opcache/Optimizer/zend_ssa.h b/ext/opcache/Optimizer/zend_ssa.h index 5abe44c93d..02544506fe 100644 --- a/ext/opcache/Optimizer/zend_ssa.h +++ b/ext/opcache/Optimizer/zend_ssa.h @@ -142,6 +142,7 @@ typedef struct _zend_ssa { BEGIN_EXTERN_C() int zend_build_ssa(zend_arena **arena, const zend_script *script, const zend_op_array *op_array, uint32_t build_flags, zend_ssa *ssa); +int zend_ssa_rename_op(const zend_op_array *op_array, const zend_op *opline, uint32_t k, uint32_t build_flags, int ssa_vars_count, zend_ssa_op *ssa_ops, int *var); int zend_ssa_compute_use_def_chains(zend_arena **arena, const zend_op_array *op_array, zend_ssa *ssa); int zend_ssa_unlink_use_chain(zend_ssa *ssa, int op, int var); diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index 02e44c2654..d4f97de76d 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -13,5 +13,6 @@ $(builddir)/jit/zend_jit.lo: \ $(srcdir)/jit/zend_jit_perf_dump.c \ $(srcdir)/jit/zend_jit_oprofile.c \ $(srcdir)/jit/zend_jit_vtune.c \ + $(srcdir)/jit/zend_jit_trace.c \ $(srcdir)/jit/zend_elf.c diff --git a/ext/opcache/jit/Makefile.frag.w32 b/ext/opcache/jit/Makefile.frag.w32 index 282f81bb74..306c89f029 100644 --- a/ext/opcache/jit/Makefile.frag.w32 +++ b/ext/opcache/jit/Makefile.frag.w32 @@ -13,4 +13,5 @@ $(BUILD_DIR)\opcache_jit\zend_jit.obj: \ ext/opcache/jit/zend_jit_gdb.c \ ext/opcache/jit/zend_jit_perf_dump.c \ ext/opcache/jit/zend_jit_oprofile.c \ + ext/opcache/jit/zend_jit_trace.c \ ext/opcache/jit/zend_jit_vtune.c diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1b3fbcee2f..7b13d3ce54 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -24,7 +24,6 @@ #include "Zend/zend_constants.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" -#include "jit/zend_jit_internal.h" #ifdef HAVE_JIT @@ -34,6 +33,14 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" +#include "jit/zend_jit_internal.h" + +#ifdef ZTS +int zend_jit_globals_id; +#else +zend_jit_globals jit_globals; +#endif + //#define CONTEXT_THREADED_JIT #define ZEND_JIT_USE_RC_INFERENCE @@ -49,6 +56,7 @@ #define JIT_PREFIX "JIT$" #define JIT_STUB_PREFIX "JIT$$" +#define TRACE_PREFIX "TRACE-" #define DASM_M_GROW(ctx, t, p, sz, need) \ do { \ @@ -97,11 +105,17 @@ static zend_long jit_bisect_pos = 0; static const void *zend_jit_runtime_jit_handler = NULL; static const void *zend_jit_profile_jit_handler = NULL; static const void *zend_jit_func_counter_handler = NULL; +static const void *zend_jit_ret_counter_handler = NULL; static const void *zend_jit_loop_counter_handler = NULL; static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa); static void ZEND_FASTCALL zend_runtime_jit(void); +static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline, zend_jit_trace_rec *trace); +static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace); +static const void *zend_jit_trace_get_exit_addr(uint32_t n); +static void zend_jit_trace_add_code(const void *start, uint32_t size); + static zend_bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use) { if (ssa->vars[var].phi_use_chain) { @@ -133,22 +147,26 @@ static zend_bool zend_long_is_power_of_two(zend_long x) return (x > 0) && !(x & (x - 1)); } -#define OP_RANGE(line, opN) \ +#define OP_RANGE_EX(ssa_op, opN) \ (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \ - ssa->ops && \ - ssa->var_info && \ - ssa->ops[line].opN##_use >= 0 && \ - ssa->var_info[ssa->ops[line].opN##_use].has_range) ? \ - &ssa->var_info[ssa->ops[line].opN##_use].range : NULL) + (ssa_op)->opN##_use >= 0 && \ + ssa->var_info[(ssa_op)->opN##_use].has_range) ? \ + &ssa->var_info[(ssa_op)->opN##_use].range : NULL) + +#define OP_RANGE(line, opN) \ + (ssa->var_info ? OP_RANGE_EX(ssa->ops + (line), opN) : NULL) #define OP1_RANGE() OP_RANGE(opline - op_array->opcodes, op1) #define OP2_RANGE() OP_RANGE(opline - op_array->opcodes, op2) #define OP1_DATA_RANGE() OP_RANGE(opline - op_array->opcodes + 1, op1) +#define OP1_RANGE_EX() OP_RANGE_EX(ssa_op, op1) +#define OP2_RANGE_EX() OP_RANGE_EX(ssa_op, op2) +#define OP1_DATA_RANGE_EX() OP_RANGE_EX(ssa_op + 1, op1) + #include "dynasm/dasm_x86.h" #include "jit/zend_jit_x86.h" #include "jit/zend_jit_helpers.c" -#include "jit/zend_jit_x86.c" #include "jit/zend_jit_disasm_x86.c" #ifndef _WIN32 #include "jit/zend_jit_gdb.c" @@ -159,6 +177,8 @@ static zend_bool zend_long_is_power_of_two(zend_long x) #endif #include "jit/zend_jit_vtune.c" +#include "jit/zend_jit_x86.c" + #if _WIN32 # include #else @@ -168,8 +188,6 @@ static zend_bool zend_long_is_power_of_two(zend_long x) # endif #endif -#define DASM_ALIGNMENT 16 - ZEND_EXT_API void zend_jit_status(zval *ret) { zval stats; @@ -220,7 +238,8 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, zend_ssa *ssa, const zend_op *rt_opline, zend_lifetime_interval **ra, - const char *name) + const char *name, + zend_bool is_trace) { size_t size; int ret; @@ -289,6 +308,10 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); + if (is_trace) { + zend_jit_trace_add_code(entry, size); + } + if (op_array && ssa) { int b; @@ -339,7 +362,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } else { if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) { zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) { + if (is_trace || (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) != 0) { zend_jit_disasm( name, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, @@ -411,9 +434,8 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return entry; } -static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +static int zend_may_overflow_ex(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa) { - uint32_t num; int res; if (!ssa->ops || !ssa->var_info) { @@ -422,21 +444,18 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_POST_INC: - num = opline - op_array->opcodes; - res = ssa->ops[num].op1_def; + res = ssa_op->op1_def; return (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow); case ZEND_PRE_DEC: case ZEND_POST_DEC: - num = opline - op_array->opcodes; - res = ssa->ops[num].op1_def; + res = ssa_op->op1_def; return (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow); case ZEND_ADD: - num = opline - op_array->opcodes; - res = ssa->ops[num].result_def; + res = ssa_op->result_def; if (res < 0 || !ssa->var_info[res].has_range) { return 1; @@ -444,11 +463,11 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.underflow) { zend_long op1_min, op2_min; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); + op1_min = OP1_MIN_RANGE_EX(); + op2_min = OP2_MIN_RANGE_EX(); if (zend_add_will_overflow(op1_min, op2_min)) { return 1; } @@ -456,19 +475,18 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.overflow) { zend_long op1_max, op2_max; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); + op1_max = OP1_MAX_RANGE_EX(); + op2_max = OP2_MAX_RANGE_EX(); if (zend_add_will_overflow(op1_max, op2_max)) { return 1; } } return 0; case ZEND_SUB: - num = opline - op_array->opcodes; - res = ssa->ops[num].result_def; + res = ssa_op->result_def; if (res < 0 || !ssa->var_info[res].has_range) { return 1; @@ -476,11 +494,11 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.underflow) { zend_long op1_min, op2_max; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + op1_min = OP1_MIN_RANGE_EX(); + op2_max = OP2_MAX_RANGE_EX(); if (zend_sub_will_overflow(op1_min, op2_max)) { return 1; } @@ -488,27 +506,25 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.overflow) { zend_long op1_max, op2_min; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); + op1_max = OP1_MAX_RANGE_EX(); + op2_min = OP2_MIN_RANGE_EX(); if (zend_sub_will_overflow(op1_max, op2_min)) { return 1; } } return 0; case ZEND_MUL: - num = opline - op_array->opcodes; - res = ssa->ops[num].result_def; + res = ssa_op->result_def; return (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow || ssa->var_info[res].range.overflow); case ZEND_ASSIGN_OP: if (opline->extended_value == ZEND_ADD) { - num = opline - op_array->opcodes; - res = ssa->ops[num].op1_def; + res = ssa_op->op1_def; if (res < 0 || !ssa->var_info[res].has_range) { return 1; @@ -516,11 +532,11 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.underflow) { zend_long op1_min, op2_min; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); + op1_min = OP1_MIN_RANGE_EX(); + op2_min = OP2_MIN_RANGE_EX(); if (zend_add_will_overflow(op1_min, op2_min)) { return 1; } @@ -528,19 +544,18 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.overflow) { zend_long op1_max, op2_max; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); + op1_max = OP1_MAX_RANGE_EX(); + op2_max = OP2_MAX_RANGE_EX(); if (zend_add_will_overflow(op1_max, op2_max)) { return 1; } } return 0; } else if (opline->extended_value == ZEND_SUB) { - num = opline - op_array->opcodes; - res = ssa->ops[num].op1_def; + res = ssa_op->op1_def; if (res < 0 || !ssa->var_info[res].has_range) { return 1; @@ -548,11 +563,11 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.underflow) { zend_long op1_min, op2_max; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + op1_min = OP1_MIN_RANGE_EX(); + op2_max = OP2_MAX_RANGE_EX(); if (zend_sub_will_overflow(op1_min, op2_max)) { return 1; } @@ -560,19 +575,18 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra if (ssa->var_info[res].range.overflow) { zend_long op1_max, op2_min; - if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { + if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) { return 1; } - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); + op1_max = OP1_MAX_RANGE_EX(); + op2_min = OP2_MIN_RANGE_EX(); if (zend_sub_will_overflow(op1_max, op2_min)) { return 1; } } return 0; } else if (opline->extended_value == ZEND_MUL) { - num = opline - op_array->opcodes; - res = ssa->ops[num].op1_def; + res = ssa_op->op1_def; return (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow || @@ -583,6 +597,11 @@ static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_arra } } +static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +{ + return zend_may_overflow_ex(opline, &ssa->ops[opline - op_array->opcodes], op_array, ssa); +} + static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg) { uint32_t flags; @@ -2417,7 +2436,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op goto done; case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: - if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level)) { + if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level, NULL)) { goto jit_failure; } goto done; @@ -2466,7 +2485,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op case ZEND_DO_ICALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_DO_FCALL: - if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1)) { + if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) { goto jit_failure; } goto done; @@ -2503,7 +2522,8 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op OP2_INFO(), OP2_REG_ADDR(), res_addr, zend_may_throw(opline, op_array, ssa), - smart_branch_opcode, target_label, target_label2)) { + smart_branch_opcode, target_label, target_label2, + NULL)) { goto jit_failure; } goto done; @@ -2530,7 +2550,8 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op OP2_INFO(), OP2_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, op_array, ssa), - smart_branch_opcode, target_label, target_label2)) { + smart_branch_opcode, target_label, target_label2, + NULL)) { goto jit_failure; } goto done; @@ -2550,7 +2571,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op smart_branch_opcode = 0; target_label = target_label2 = (uint32_t)-1; } - if (!zend_jit_defined(&dasm_state, opline, op_array, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_defined(&dasm_state, opline, op_array, smart_branch_opcode, target_label, target_label2, NULL)) { goto jit_failure; } goto done; @@ -2574,7 +2595,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op smart_branch_opcode = 0; target_label = target_label2 = (uint32_t)-1; } - if (!zend_jit_type_check(&dasm_state, opline, op_array, OP1_INFO(), smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_type_check(&dasm_state, opline, op_array, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) { goto jit_failure; } goto done; @@ -2589,9 +2610,38 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_tail_handler(&dasm_state, opline)) { goto jit_failure; } - } else if (!zend_jit_return(&dasm_state, opline, op_array, ssa, - op1_info, OP1_REG_ADDR())) { - goto jit_failure; + } else { + int j; + + if (!zend_jit_return(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR())) { + goto jit_failure; + } + if (jit_return_label >= 0) { + if (!zend_jit_jmp(&dasm_state, jit_return_label)) { + goto jit_failure; + } + goto done; + } + jit_return_label = ssa->cfg.blocks_count * 2; + if (!zend_jit_label(&dasm_state, jit_return_label)) { + goto jit_failure; + } + if (!zend_jit_leave_frame(&dasm_state)) { + goto jit_failure; + } + for (j = 0 ; j < op_array->last_var; j++) { + uint32_t info = zend_ssa_cv_info(opline, op_array, ssa, j); + + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (!zend_jit_free_cv(&dasm_state, opline, op_array, info, j)) { + goto jit_failure; + } + } + } + if (!zend_jit_leave_func(&dasm_state, opline, op_array, NULL)) { + goto jit_failure; + } } goto done; case ZEND_BOOL: @@ -2599,7 +2649,8 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array, OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(), -1, -1, - zend_may_throw(opline, op_array, ssa))) { + zend_may_throw(opline, op_array, ssa), + opline->opcode, NULL)) { goto jit_failure; } goto done; @@ -2625,7 +2676,8 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array, OP1_INFO(), OP1_REG_ADDR(), res_addr, ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1], - zend_may_throw(opline, op_array, ssa))) { + zend_may_throw(opline, op_array, ssa), + opline->opcode, NULL)) { goto jit_failure; } goto done; @@ -2666,7 +2718,8 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_isset_isempty_dim(&dasm_state, opline, op_array, OP1_INFO(), OP2_INFO(), zend_may_throw(opline, op_array, ssa), - smart_branch_opcode, target_label, target_label2)) { + smart_branch_opcode, target_label, target_label2, + NULL)) { goto jit_failure; } goto done; @@ -2895,7 +2948,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_call(&dasm_state, next_opline, b + 1); is_terminated = 1; } else { - zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1); + zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL); } } } @@ -2916,7 +2969,7 @@ done: } } - handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL); + handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0); if (!handler) { goto jit_failure; } @@ -3072,17 +3125,16 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) { zend_op_array *op_array = &EX(func)->op_array; - zend_jit_op_array_extension *jit_extension; + zend_jit_op_array_hot_extension *jit_extension; uint32_t i; zend_shared_alloc_lock(); - jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); if (jit_extension) { SHM_UNPROTECT(); zend_jit_unprotect(); - *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; for (i = 0; i < op_array->last; i++) { op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } @@ -3103,7 +3155,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend static int zend_jit_setup_hot_counters(zend_op_array *op_array) { zend_op *opline = op_array->opcodes; - zend_jit_op_array_extension *jit_extension; + zend_jit_op_array_hot_extension *jit_extension; zend_cfg cfg; uint32_t i; @@ -3114,7 +3166,7 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) return FAILURE; } - jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension) + (op_array->last - 1) * sizeof(void*)); + jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*)); jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)]; for (i = 0; i < op_array->last; i++) { jit_extension->orig_handlers[i] = op_array->opcodes[i].handler; @@ -3158,6 +3210,8 @@ static int zend_needs_manual_jit(const zend_op_array *op_array) return 0; } +#include "jit/zend_jit_trace.c" + ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { if (dasm_ptr == NULL) { @@ -3195,6 +3249,8 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) return SUCCESS; } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { return zend_jit_setup_hot_counters(op_array); + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + return zend_jit_setup_hot_trace_counters(op_array); } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { return zend_real_jit_func(op_array, script, NULL); } else if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { @@ -3230,7 +3286,8 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC || zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST || - zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { + zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS || + zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { for (i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { @@ -3381,7 +3438,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_stubs[i].stub(&dasm_state)) { return 0; } - if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name)) { + if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0)) { return 0; } } @@ -3392,7 +3449,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) { return 0; } - zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit"); + zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit", 0); if (!zend_jit_runtime_jit_handler) { return 0; } @@ -3401,7 +3458,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) { return 0; } - zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit"); + zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit", 0); if (!zend_jit_profile_jit_handler) { return 0; } @@ -3410,7 +3467,7 @@ static int zend_jit_make_stubs(void) if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) { return 0; } - zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter"); + zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0); if (!zend_jit_func_counter_handler) { return 0; } @@ -3419,26 +3476,73 @@ static int zend_jit_make_stubs(void) if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) { return 0; } - zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter"); + zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0); + if (!zend_jit_loop_counter_handler) { + return 0; + } + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_func_trace_counter_stub(&dasm_state)) { + return 0; + } + zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0); + if (!zend_jit_func_counter_handler) { + return 0; + } + + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_ret_trace_counter_stub(&dasm_state)) { + return 0; + } + zend_jit_ret_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_ret_counter", 0); + if (!zend_jit_ret_counter_handler) { + return 0; + } + + dasm_setup(&dasm_state, dasm_actions); + if (!zend_jit_hybrid_loop_trace_counter_stub(&dasm_state)) { + return 0; + } + zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0); if (!zend_jit_loop_counter_handler) { return 0; } } } else { - zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; - zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; - zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper; - zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper; + if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { + zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; + } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { + zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { + zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper; + zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper; + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + zend_jit_func_counter_handler = (const void*)zend_jit_func_trace_helper; + zend_jit_ret_counter_handler = (const void*)zend_jit_ret_trace_helper; + zend_jit_loop_counter_handler = (const void*)zend_jit_loop_trace_helper; + } } dasm_free(&dasm_state); return 1; } +static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) +{ + memset(jit_globals, 0, sizeof(zend_jit_globals)); + zend_jit_trace_init_caches(); +} + ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bool reattached) { int ret; +#ifdef ZTS + zend_jit_globals_id = ts_allocate_id(&zend_jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL); +#else + zend_jit_globals_ctor(&jit_globals); +#endif + zend_jit_level = ZEND_JIT_LEVEL(jit); zend_jit_trigger = ZEND_JIT_TRIGGER(jit); zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit); @@ -3549,6 +3653,12 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo #endif } + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (zend_jit_trace_startup() != SUCCESS) { + return FAILURE; + } + } + return SUCCESS; } @@ -3587,6 +3697,14 @@ ZEND_EXT_API void zend_jit_activate(void) for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { zend_jit_hot_counters[i] = ZEND_JIT_HOT_COUNTER_INIT; } + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + int i; + + for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { + zend_jit_hot_counters[i] = ZEND_JIT_TRACE_COUNTER_INIT; + } + + zend_jit_trace_reset_caches(); } } diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 5234660398..aea15ea1b7 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -33,6 +33,7 @@ #define ZEND_JIT_ON_PROF_REQUEST 2 /* compile the most frequently caled on first request functions */ #define ZEND_JIT_ON_HOT_COUNTERS 3 /* compile functions after N calls or loop iterations */ #define ZEND_JIT_ON_DOC_COMMENT 4 /* compile functions with "@jit" tag in doc-comments */ +#define ZEND_JIT_ON_HOT_TRACE 5 /* trace functions after N calls or loop iterations */ #define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) @@ -75,6 +76,15 @@ #define ZEND_JIT_DEBUG_GDB (1<<8) +#define ZEND_JIT_DEBUG_TRACE_START (1<<12) +#define ZEND_JIT_DEBUG_TRACE_STOP (1<<13) +#define ZEND_JIT_DEBUG_TRACE_COMPILED (1<<14) +#define ZEND_JIT_DEBUG_TRACE_EXIT (1<<15) +#define ZEND_JIT_DEBUG_TRACE_ABORT (1<<16) +#define ZEND_JIT_DEBUG_TRACE_BLACKLIST (1<<17) +#define ZEND_JIT_DEBUG_TRACE_BYTECODE (1<<18) +#define ZEND_JIT_DEBUG_TRACE_TSSA (1<<19) + ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_EXT_API int zend_jit_script(zend_script *script); ZEND_EXT_API void zend_jit_unprotect(void); diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 05ece4d276..d784170d7c 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -137,6 +137,10 @@ static void zend_jit_disasm_add_symbol(const char *name, } } else { ZEND_ASSERT(sym->addr == node->addr); + if (strcmp(name, node->name) == 0 && sym->end < node->end) { + /* reduce size of the existing symbol */ + node->end = sym->end; + } free(sym); return; } @@ -397,6 +401,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_int_extend_stack_helper); REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); + REGISTER_HELPER(zend_jit_leave_func_helper); REGISTER_HELPER(zend_jit_symtable_find); REGISTER_HELPER(zend_jit_hash_index_lookup_rw); REGISTER_HELPER(zend_jit_hash_index_lookup_w); @@ -448,6 +453,7 @@ static int zend_jit_disasm_init(void) REGISTER_HELPER(zend_jit_pre_dec); REGISTER_HELPER(zend_runtime_jit); REGISTER_HELPER(zend_jit_hot_func); + REGISTER_HELPER(zend_jit_check_constant); #undef REGISTER_HELPER #ifndef _WIN32 diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index f327fd5128..504cb67065 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -33,18 +33,11 @@ extern int zend_jit_profile_counter_rid; extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; -void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); - -typedef struct _zend_jit_op_array_extension { - int16_t *counter; - const void *orig_handlers[1]; -} zend_jit_op_array_extension; - -static zend_always_inline zend_long zend_jit_op_array_hash(const zend_op_array *op_array) +static zend_always_inline zend_long zend_jit_hash(const void *ptr) { uintptr_t x; - x = (uintptr_t)op_array->opcodes >> 3; + x = (uintptr_t)ptr >> 3; #if SIZEOF_SIZE_T == 4 x = ((x >> 16) ^ x) * 0x45d9f3b; x = ((x >> 16) ^ x) * 0x45d9f3b; @@ -57,6 +50,16 @@ static zend_always_inline zend_long zend_jit_op_array_hash(const zend_op_array * return x; } +void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); + +typedef struct _zend_jit_op_array_hot_extension { + int16_t *counter; + const void *orig_handlers[1]; +} zend_jit_op_array_hot_extension; + +#define zend_jit_op_array_hash(op_array) \ + zend_jit_hash((op_array)->opcodes) + extern const zend_op *zend_jit_halt_op; #ifdef HAVE_GCC_GLOBAL_REGS @@ -78,6 +81,10 @@ extern const zend_op *zend_jit_halt_op; handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \ return; \ } while(0) +# define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \ + handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \ + return; \ + } while(0) #else # define EXECUTE_DATA_D zend_execute_data* execute_data # define EXECUTE_DATA_C execute_data @@ -96,6 +103,9 @@ extern const zend_op *zend_jit_halt_op; # define ZEND_OPCODE_TAIL_CALL(handler) do { \ return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \ } while(0) +# define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \ + return handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \ + } while(0) #endif /* VM handlers */ @@ -104,6 +114,7 @@ typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_O /* VM helpers */ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC); +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS); @@ -116,4 +127,267 @@ void ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D); void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags); int ZEND_FASTCALL zend_jit_check_constant(const zval *key); +/* Tracer */ +#define zend_jit_opline_hash(opline) \ + zend_jit_hash(opline) + +#define ZEND_JIT_TRACE_FUNC_COST (1*250) +#define ZEND_JIT_TRACE_RET_COST (1*250-1) +#define ZEND_JIT_TRACE_LOOP_COST (2*250) +#define ZEND_JIT_TRACE_COUNTER_INIT (127*250) + +#define ZEND_JIT_TRACE_MAX_TRACES 1024 /* max number of traces */ +#define ZEND_JIT_TRACE_MAX_LENGTH 1024 /* max length of single trace */ +#define ZEND_JIT_TRACE_MAX_EXITS 512 /* max number of side exits per trace */ +#define ZEND_JIT_TRACE_MAX_SIDE_TRACES 128 /* max number of side traces of a root trace */ +#define ZEND_JIT_TRACE_MAX_EXIT_COUNTERS 8192 /* max number of side exits for all trace */ + +#define ZEND_JIT_TRACE_MAX_FUNCS 30 /* max number of different functions in a single trace */ +#define ZEND_JIT_TRACE_MAX_CALL_DEPTH 10 /* max depth of inlined calls */ +#define ZEND_JIT_TRACE_MAX_RET_DEPTH 4 /* max depth of inlined returns */ +#define ZEND_JIT_TRACE_MAX_RECURSION 2 /* max number of recursive inlined calls */ +#define ZEND_JIT_TRACE_MAX_UNROLL_LOOPS 8 /* max number of unrolled loops */ + +#define ZEND_JIT_TRACE_HOT_SIDE_COUNT 8 /* number of exits before taking side trace */ +#define ZEND_JIT_TRACE_HOT_RETURN_COUNT 8 /* number of returns before taking continuation trace */ + +#define ZEND_JIT_TRACE_MAX_ROOT_FAILURES 16 /* number of attemts to record/compile a root trace */ +#define ZEND_JIT_TRACE_MAX_SIDE_FAILURES 4 /* number of attemts to record/compile a side trace */ + +#define ZEND_JIT_TRACE_BAD_ROOT_SLOTS 64 /* number of slots in bad root trace cache */ + +#define ZEND_JIT_TRACE_STOP(_) \ + _(LOOP, "loop") \ + _(RECURSIVE_CALL, "recursive call") \ + _(RECURSIVE_RET, "recursive return") \ + _(RETURN, "return") \ + _(LINK, "link to another trace") \ + /* compilation and linking successful */ \ + _(COMPILED, "compiled") \ + _(ALREADY_DONE, "already prcessed") \ + /* failures */ \ + _(ERROR, "error") /* not used */ \ + _(NOT_SUPPORTED, "not supported instructions") \ + _(EXCEPTION, "exception") \ + _(TOO_LONG, "trace too long") \ + _(TOO_DEEP, "trace too deep") \ + _(TOO_DEEP_RET, "trace too deep return") \ + _(DEEP_RECURSION, "deep recursion") \ + _(LOOP_UNROLL, "loop unroll limit reached") \ + _(LOOP_EXIT, "exit from loop") \ + _(BLACK_LIST, "trace blacklisted") \ + _(INNER_LOOP, "inner loop") /* trace it */ \ + _(COMPILED_LOOP, "compiled loop") \ + _(TOPLEVEL, "toplevel") \ + _(TRAMPOLINE, "trampoline call") \ + _(BAD_FUNC, "bad function call") \ + _(HALT, "exit from interpreter") \ + _(COMPILER_ERROR, "JIT compilation error") \ + /* no recoverable error (blacklist immediately) */ \ + _(NO_SHM, "insufficient shared memory") \ + _(TOO_MANY_TRACES, "too many traces") \ + _(TOO_MANY_CHILDREN, "too many side traces") \ + _(TOO_MANY_EXITS, "too many side exits") \ + +#define ZEND_JIT_TRACE_STOP_NAME(name, description) \ + ZEND_JIT_TRACE_STOP_ ## name, + +typedef enum _zend_jit_trace_stop { + ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_NAME) +} zend_jit_trace_stop; + +#define ZEND_JIT_TRACE_STOP_OK(ret) \ + (ret < ZEND_JIT_TRACE_STOP_COMPILED) + +#define ZEND_JIT_TRACE_STOP_DONE(ret) \ + (ret < ZEND_JIT_TRACE_STOP_ERROR) + +#define ZEND_JIT_TRACE_STOP_REPEAT(ret) \ + (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP) + +#define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \ + (ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR) + +#define ZEND_JIT_TRACE_START_MASK 0xf + +#define ZEND_JIT_TRACE_START_LOOP (1<<0) +#define ZEND_JIT_TRACE_START_ENTER (1<<1) +#define ZEND_JIT_TRACE_START_RETURN (1<<2) +#define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */ + +#define ZEND_JIT_TRACE_JITED (1<<4) +#define ZEND_JIT_TRACE_BLACKLISTED (1<<5) +#define ZEND_JIT_TRACE_UNSUPPORTED (1<<6) + +#define ZEND_JIT_TRACE_SUPPORTED 0 + +#define ZEND_JIT_EXIT_JITED (1<<0) +#define ZEND_JIT_EXIT_BLACKLISTED (1<<1) + +typedef union _zend_op_trace_info { + zend_op dummy; /* the size of this structure must be the same as zend_op */ + struct { + const void *orig_handler; + const void *call_handler; + int16_t *counter; + uint8_t trace_flags; + }; +} zend_op_trace_info; + +typedef struct _zend_jit_op_array_trace_extension { + zend_func_info func_info; + size_t offset; /* offset from "zend_op" to corresponding "op_info" */ + zend_op_trace_info trace_info[1]; +} zend_jit_op_array_trace_extension; + +#define ZEND_OP_TRACE_INFO(opline, offset) \ + ((zend_op_trace_info*)(((char*)opline) + offset)) + +/* Recorder */ +typedef enum _zend_jit_trace_op { + ZEND_JIT_TRACE_VM, + ZEND_JIT_TRACE_OP1_TYPE, + ZEND_JIT_TRACE_OP2_TYPE, + ZEND_JIT_TRACE_INIT_CALL, + ZEND_JIT_TRACE_DO_ICALL, + ZEND_JIT_TRACE_ENTER, + ZEND_JIT_TRACE_BACK, + ZEND_JIT_TRACE_START, + ZEND_JIT_TRACE_END, +} zend_jit_trace_op; + +#define IS_UNKNOWN 255 /* may be used for zend_jit_trace_rec.op?_type */ +#define IS_TRACE_REFERENCE (1<<5) + +typedef struct _zend_jit_trace_rec { + uint8_t op; /* zend_jit_trace_op */ + union { + struct { + uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */ + uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */ + uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */ + }; + struct { + union { + uint8_t recursive; /* part of recursive return sequence for ZEND_JIT_TRACE_BACK */ + int8_t return_value_used; /* for ZEND_JIT_TRACE_ENTER */ + uint8_t fake; /* for ZEND_JIT_TRACE_INIT_FCALL */ + }; + uint8_t first_ssa_var; /* may be used for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK */ + }; + struct { + uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */ + uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */ + uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */ + }; + }; + union { + const void *ptr; + const zend_function *func; + const zend_op_array *op_array; + const zend_op *opline; + const zend_class_entry *ce; + }; +} zend_jit_trace_rec; + +typedef struct _zend_jit_trace_start_rec { + uint8_t op; /* zend_jit_trace_op */ + uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */ + uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */ + uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */ + const zend_op_array *op_array; + const zend_op *opline; +} zend_jit_trace_start_rec; + +#define ZEND_JIT_TRACE_START_REC_SIZE 2 + +typedef struct _zend_jit_trace_exit_info { + const zend_op *opline; /* opline where VM should continue execution */ + uint32_t stack_size; + uint32_t stack_offset; +} zend_jit_trace_exit_info; + +typedef int32_t zend_jit_trace_stack; + +typedef struct _zend_jit_trace_info { + uint32_t id; /* trace id */ + uint32_t root; /* root trace id or self id for root traces */ + uint32_t parent; /* parent trace id or 0 for root traces */ + uint32_t link; /* link trace id or self id for loop) */ + uint32_t exit_count; /* number of side exits */ + uint32_t child_count; /* number of side traces for root traces */ + uint32_t code_size; /* size of native code */ + uint32_t exit_counters; /* offset in exit counters array */ + uint32_t stack_map_size; + const void *code_start; /* address of native code */ + zend_jit_trace_exit_info *exit_info; /* info about side exits */ + zend_jit_trace_stack *stack_map; + //uint32_t loop_offset; +} zend_jit_trace_info; + +typedef struct _zend_jit_trace_stack_frame zend_jit_trace_stack_frame; + +struct _zend_jit_trace_stack_frame { + zend_jit_trace_stack_frame *call; + zend_jit_trace_stack_frame *prev; + const zend_function *func; + union { + struct { + int8_t return_value_used; + int8_t nested; + int8_t num_args; + }; + int return_ssa_var; + }; + zend_jit_trace_stack stack[1]; +}; + +typedef struct _zend_jit_globals { + zend_jit_trace_stack_frame *current_frame; + + const zend_op *bad_root_cache_opline[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; + uint8_t bad_root_cache_count[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; + uint8_t bad_root_cache_stop[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; + uint32_t bad_root_slot; + + uint8_t exit_counters[ZEND_JIT_TRACE_MAX_EXIT_COUNTERS]; +} zend_jit_globals; + +#ifdef ZTS +# define JIT_G(v) ZEND_TSRMG(zend_jit_globals_id, zend_jit_globals *, v) +extern int zend_jit_globals_id; +#else +# define JIT_G(v) (jit_globals.v) +extern zend_jit_globals jit_globals; +#endif + +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS); +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS); +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS); + +int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline); +int ZEND_FASTCALL zend_jit_trace_exit(uint32_t trace_num, uint32_t exit_num); +zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start); + +static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true) +{ + if (trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END) { + if (trace->opline == opline + 1) { + /* not taken branch */ + *exit_if_true = opline->opcode == ZEND_JMPNZ; + return OP_JMP_ADDR(opline, opline->op2); + } else if (trace->opline == OP_JMP_ADDR(opline, opline->op2)) { + /* taken branch */ + *exit_if_true = opline->opcode == ZEND_JMPZ; + return opline + 1; + } else { + ZEND_ASSERT(0); + } + } else { + ZEND_ASSERT(0); + } + *exit_if_true = 0; + return NULL; +} + #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c new file mode 100644 index 0000000000..abc2a0146d --- /dev/null +++ b/ext/opcache/jit/zend_jit_trace.c @@ -0,0 +1,3972 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + +----------------------------------------------------------------------+ +*/ + +static zend_jit_trace_info *zend_jit_traces = NULL; +static const void **zend_jit_exit_groups = NULL; + +#define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root +#define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id +#define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count +#define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters + +static int zend_jit_trace_startup(void) +{ + zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * ZEND_JIT_TRACE_MAX_TRACES); + if (!zend_jit_traces) { + return FAILURE; + } + zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP)); + if (!zend_jit_exit_groups) { + return FAILURE; + } + ZEND_JIT_TRACE_NUM = 1; + ZEND_JIT_COUNTER_NUM = 0; + ZEND_JIT_EXIT_NUM = 0; + ZEND_JIT_EXIT_COUNTERS = 0; + return SUCCESS; +} + +static const void *zend_jit_trace_allocate_exit_group(uint32_t n) +{ + dasm_State* dasm_state = NULL; + const void *entry; + char name[32]; + + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + zend_jit_trace_exit_group_stub(&dasm_state, n); + + sprintf(name, "jit$$trace_exit_%d", n); + entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0); + dasm_free(&dasm_state); + +#ifdef HAVE_DISASM + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + uint32_t i; + + for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + sprintf(name, "jit$$trace_exit_%d", n + i); + zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING); + } + } +#endif + + return entry; +} + +static const void *zend_jit_trace_allocate_exit_point(uint32_t n) +{ + const void *group = NULL; + + if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) { + return NULL; + } + do { + group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM); + if (!group) { + return NULL; + } + zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] = + group; + ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP; + } while (n >= ZEND_JIT_EXIT_NUM); + return (const void*) + ((const char*)group + + ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING)); +} + +static const void *zend_jit_trace_get_exit_addr(uint32_t n) +{ + if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) { + return zend_jit_trace_allocate_exit_point(n); + } + return (const void*) + ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] + + ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING)); +} + +static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace) +{ + zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + uint32_t exit_point; + const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array; + uint32_t stack_offset = (uint32_t)-1; + uint32_t stack_size = op_array->last_var + op_array->T; + zend_jit_trace_stack *stack = NULL; + + if (stack_size) { + stack = JIT_G(current_frame)->stack; + do { + if (stack[stack_size-1] != IS_UNKNOWN) { + break; + } + stack_size--; + } while (stack_size); + } + + /* Try to reuse exit points */ + if (to_opline != NULL && t->exit_count > 0) { + uint32_t i = t->exit_count; + + do { + i--; + if (stack_size == 0 + || (t->exit_info[i].stack_size >= stack_size + && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) { + stack_offset = t->exit_info[i].stack_offset; + if (t->exit_info[i].opline == to_opline) { + return i; + } + } + } while (i > 0); + } + + exit_point = t->exit_count; + if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) { + if (stack_size != 0 && stack_offset == (uint32_t)-1) { + stack_offset = t->stack_map_size; + t->stack_map_size += stack_size; + // TODO: reduce number of reallocations ??? + t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); + memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)); + } + t->exit_count++; + t->exit_info[exit_point].opline = to_opline; + t->exit_info[exit_point].stack_size = stack_size; + t->exit_info[exit_point].stack_offset = stack_offset; + } + + return exit_point; +} + +static void zend_jit_trace_add_code(const void *start, uint32_t size) +{ + zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + t->code_start = start; + t->code_size = size; +} + +static uint32_t zend_jit_find_trace(const void *addr) +{ + uint32_t i; + + for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) { + if (zend_jit_traces[i].code_start == addr) { + return i; + } + } + ZEND_ASSERT(0); + return 0; +} + +static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno) +{ + smart_str buf = {0}; + + smart_str_appends(&buf, TRACE_PREFIX); + smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM); + smart_str_appendc(&buf, '$'); + if (op_array->function_name) { + if (op_array->scope) { + smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name)); + smart_str_appends(&buf, "::"); + smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); + } else { + smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); + } + } else if (op_array->filename) { + smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename)); + } + smart_str_appendc(&buf, '$'); + smart_str_append_long(&buf, (zend_long)lineno); + smart_str_0(&buf); + return buf.s; +} + +static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline, zend_jit_trace_rec *trace) +{ + switch (opline->opcode) { + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_CASE: + case ZEND_ISSET_ISEMPTY_CV: + case ZEND_ISSET_ISEMPTY_VAR: + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + case ZEND_ISSET_ISEMPTY_PROP_OBJ: + case ZEND_ISSET_ISEMPTY_STATIC_PROP: + case ZEND_INSTANCEOF: + case ZEND_TYPE_CHECK: + case ZEND_DEFINED: + case ZEND_IN_ARRAY: + case ZEND_ARRAY_KEY_EXISTS: + if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) { + /* smart branch */ + return 1; + } + break; + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_ASSERT_CHECK: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + case ZEND_SWITCH_LONG: + case ZEND_SWITCH_STRING: + /* branch opcdoes */ + return 1; + case ZEND_NEW: + if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { + /* NEW may skip constructor without arguments */ + return 1; + } + break; + case ZEND_CATCH: + case ZEND_FAST_CALL: + case ZEND_FAST_RET: + case ZEND_GENERATOR_CREATE: + case ZEND_GENERATOR_RETURN: + case ZEND_EXIT: + case ZEND_YIELD: + case ZEND_YIELD_FROM: + case ZEND_INCLUDE_OR_EVAL: + /* unsupported */ + return 1; + case ZEND_DO_FCALL: + /* potentially polymorphic call */ + return 1; +#if 0 + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + /* monomorphic call */ + // TODO: recompilation may change traget ??? + return 0; +#endif + case ZEND_RETURN_BY_REF: + case ZEND_RETURN: + /* return */ + return trace->op == ZEND_JIT_TRACE_BACK && trace->recursive; + default: + break; + } + return 0; +} + +static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info) +{ + if (type == IS_UNKNOWN) { + return info; + } + ZEND_ASSERT(info & (1 << type)); + if (type < IS_STRING) { + return (1 << type); + } else if (type != IS_ARRAY) { + return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN)); + } else { + return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); + } +} + +static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type) +{ + return zend_jit_trace_type_to_info_ex(type, -1); +} + +#define STACK_VAR_TYPE(_var) \ + ((zend_uchar)stack[EX_VAR_TO_NUM(_var)]) + +#define SET_STACK_VAR_TYPE(_var, _type) do { \ + stack[EX_VAR_TO_NUM(_var)] = _type; \ + } while (0) + +#define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \ + if (op_type != IS_UNKNOWN) { \ + if ((op_info & MAY_BE_GUARD) != 0 \ + && op_type != STACK_VAR_TYPE(_var)) { \ + if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \ + goto jit_failure; \ + } \ + op_info &= ~MAY_BE_GUARD; \ + ssa->var_info[_ssa_var].type &= op_info; \ + } \ + SET_STACK_VAR_TYPE(_var, op_type); \ + } \ + } while (0) + +#define USE_OP_TRACE_TYPE(_type, _var, op_info) do { \ + if (_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { \ + op_info = zend_jit_trace_type_to_info_ex(STACK_VAR_TYPE(_var), op_info); \ + } \ + } while (0) + +#define CHECK_OP1_TRACE_TYPE() \ + CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type) +#define CHECK_OP2_TRACE_TYPE() \ + CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type) +#define CHECK_OP1_DATA_TRACE_TYPE() \ + CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type) +#define USE_OP1_TRACE_TYPE() \ + USE_OP_TRACE_TYPE(opline->op1_type, opline->op1.var, op1_info) +#define USE_OP2_TRACE_TYPE() \ + USE_OP_TRACE_TYPE(opline->op2_type, opline->op2.var, op2_info) +#define USE_OP1_DATA_TRACE_TYPE() \ + USE_OP_TRACE_TYPE((opline+1)->op1_type, (opline+1)->op1.var, op1_data_info) +#define USE_RES_TRACE_TYPE() \ + USE_OP_TRACE_TYPE(opline->result_type, opline->result.var, res_use_info) +#define SET_OP1_STACK_VAR_TYPE(_type) \ + SET_STACK_VAR_TYPE(opline->op1.var, _type) +#define SET_OP2_STACK_VAR_TYPE( _type) \ + SET_STACK_VAR_TYPE(opline->op2.var, _type) +#define SET_OP1_DATA_STACK_VAR_TYPE(_type) \ + SET_STACK_VAR_TYPE((opline+1)->op1.var, _type) +#define SET_RES_STACK_VAR_TYPE(_type) \ + SET_STACK_VAR_TYPE(opline->result.var, _type) + +static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array) +{ + if (op_array->type == ZEND_USER_FUNCTION) { + return offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)); + } else { + return offsetof(zend_jit_trace_stack_frame, stack); + } +} + +static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +{ + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array)); +} + +static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +{ + return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array)); +} + +static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type) +{ + zend_jit_trace_stack *stack = call->stack; + const zend_op_array *op_array = &call->func->op_array; + uint32_t arg_num = opline->op2.num; + + if (arg_num > op_array->num_args) { + return; + } + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + zend_arg_info *arg_info; + + ZEND_ASSERT(arg_num <= op_array->num_args); + arg_info = &op_array->arg_info[arg_num-1]; + + if (ZEND_TYPE_IS_SET(arg_info->type)) { + if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) { + return; + } + } + } + SET_STACK_VAR_TYPE(opline->result.var, type); +} + +static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script) +{ + zend_jit_op_array_trace_extension *jit_extension; + zend_ssa *ssa; + + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + memset(&jit_extension->func_info, 0, sizeof(jit_extension->func_info)); + jit_extension->func_info.num_args = -1; + jit_extension->func_info.return_value_used = -1; + ssa = &jit_extension->func_info.ssa; + + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { + do { + if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) { + break; + } + + if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { + if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) { + break; + } + jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, (zend_op_array*)op_array); + if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { + zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info); + } + } + + if (zend_jit_op_array_analyze2(op_array, script, ssa) != SUCCESS) { + break; + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa); + } + return ssa; + } while (0); + } + + memset(ssa, 0, sizeof(zend_ssa)); + ssa->cfg.blocks_count = 1; + return ssa; +} + +static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa); + +static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline) +{ + int len; + + switch (opline->opcode) { + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_OBJ_OP: + case ZEND_ASSIGN_STATIC_PROP_OP: + case ZEND_ASSIGN_OBJ_REF: + case ZEND_ASSIGN_STATIC_PROP_REF: + return 2; /* OP_DATA */ + case ZEND_RECV_INIT: + len = 1; + opline++; + while (opline->opcode == ZEND_RECV_INIT) { + len++; + opline++; + } + return len; + case ZEND_BIND_GLOBAL: + len = 1; + opline++; + while (opline->opcode == ZEND_BIND_GLOBAL) { + len++; + opline++; + } + return len; +// case ZEND_IS_IDENTICAL: +// case ZEND_IS_NOT_IDENTICAL: +// case ZEND_IS_EQUAL: +// case ZEND_IS_NOT_EQUAL: +// case ZEND_IS_SMALLER: +// case ZEND_IS_SMALLER_OR_EQUAL: +// case ZEND_CASE: +// case ZEND_ISSET_ISEMPTY_CV: +// case ZEND_ISSET_ISEMPTY_VAR: +// case ZEND_ISSET_ISEMPTY_DIM_OBJ: +// case ZEND_ISSET_ISEMPTY_PROP_OBJ: +// case ZEND_ISSET_ISEMPTY_STATIC_PROP: +// case ZEND_INSTANCEOF: +// case ZEND_TYPE_CHECK: +// case ZEND_DEFINED: +// case ZEND_IN_ARRAY: +// case ZEND_ARRAY_KEY_EXISTS: + default: + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + return 2; /* JMPZ/JMPNZ */ + } + return 1; + } +} + +static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, int *var) +{ + const zend_op_array *op_array; + zend_jit_trace_rec *p; + int k, vars_count; + zend_bitset use, def; + uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; + uint32_t set_size; + zend_ssa_phi *prev = NULL; + int level = 0; + ALLOCA_FLAG(use_heap); + + op_array = trace_buffer->op_array; + set_size = zend_bitset_len(op_array->last_var + op_array->T); + use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap); + memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE); + def = use + set_size; + p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; + for (;;p++) { + if (p->op == ZEND_JIT_TRACE_VM && level == 0) { + const zend_op *opline = p->opline; + int len; + + zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); + len = zend_jit_trace_op_len(opline); + while (len > 1) { + opline++; + if (opline->opcode != ZEND_OP_DATA) { + zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); + } + len--; + } + } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + level++; + } else if (p->op == ZEND_JIT_TRACE_BACK) { + if (level == 0) { + // Phi for recursive calls and returns are not supporte yet ??? + assert(0); + } else { + level--; + } + } else if (p->op == ZEND_JIT_TRACE_END) { + break; + } + } + + zend_bitset_intersection(use, def, set_size); + + if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { + vars_count = op_array->last_var; + } else { + vars_count = op_array->last_var + op_array->T; + } + for (k = 0; k < vars_count; k++) { + if (zend_bitset_in(use, k)) { + zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + + sizeof(void*) * 2); + phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi))); + phi->sources[0] = var[k]; + phi->sources[1] = -1; + phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2)); + phi->pi = -1; + phi->var = k; + phi->ssa_var = ssa_vars_count; + var[k] = ssa_vars_count; + ssa_vars_count++; + phi->block = 1; + if (prev) { + prev->next = phi; + } else { + tssa->blocks[1].phis = phi; + } + prev = phi; + } + } + + free_alloca(use, use_heap); + + return ssa_vars_count; +} + +static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +{ + int var, use; + zend_ssa_op *op; + zend_ssa_var_info *info; + unsigned int no_val; + + if (tssa->vars[ssa_var].phi_use_chain) { + // TODO: this may be incorrect ??? + var = tssa->vars[ssa_var].phi_use_chain->ssa_var; + } else { + var = ssa_var; + } + use = tssa->vars[var].use_chain; + if (use >= 0) { + ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last); + op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes); + if (tssa->ops[use].op1_use == var) { + no_val = ssa->vars[op->op1_use].no_val; + info = ssa->var_info + op->op1_use; + } else if (tssa->ops[use].op2_use == var) { + no_val = ssa->vars[op->op2_use].no_val; + info = ssa->var_info + op->op2_use; + } else if (tssa->ops[use].result_use == var) { + no_val = ssa->vars[op->result_use].no_val; + info = ssa->var_info + op->result_use; + } else { + assert(0); + } + tssa->vars[ssa_var].no_val = no_val; + memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info)); + return 1; + } + return 0; +} + +static int zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +{ + int def; + zend_ssa_op *op; + zend_ssa_var_info *info; + unsigned int no_val; + + def = tssa->vars[ssa_var].definition; + if (def >= 0) { + ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); + op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + if (tssa->ops[def].op1_def == ssa_var) { + no_val = ssa->vars[op->op1_def].no_val; + info = ssa->var_info + op->op1_def; + } else if (tssa->ops[def].op2_def == ssa_var) { + no_val = ssa->vars[op->op2_def].no_val; + info = ssa->var_info + op->op2_def; + } else if (tssa->ops[def].result_def == ssa_var) { + no_val = ssa->vars[op->result_def].no_val; + info = ssa->var_info + op->result_def; + } else { + assert(0); + } + + tssa->vars[ssa_var].no_val = no_val; + + if (info->has_range) { + if (tssa->var_info[ssa_var].has_range) { + tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min); + tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max); + tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; + tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; + } else { + tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].range = info->range; + } + } + return 1; + } + return 0; +} + +static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +{ + int def; + zend_ssa_op *op; + zend_ssa_var_info *info; + + def = tssa->vars[ssa_var].definition; + if (def >= 0) { + ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); + op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + if (tssa->ops[def].op1_def == ssa_var) { + info = ssa->var_info + op->op1_def; + } else if (tssa->ops[def].op2_def == ssa_var) { + info = ssa->var_info + op->op2_def; + } else if (tssa->ops[def].result_def == ssa_var) { + info = ssa->var_info + op->result_def; + } else { + assert(0); + } + tssa->var_info[ssa_var].type &= info->type; + if (info->ce) { + if (tssa->var_info[ssa_var].ce) { + ZEND_ASSERT(tssa->var_info[ssa_var].ce == info->ce); + tssa->var_info[ssa_var].is_instanceof = + tssa->var_info[ssa_var].is_instanceof && info->is_instanceof; + } else { + tssa->var_info[ssa_var].ce = info->ce; + tssa->var_info[ssa_var].is_instanceof = info->is_instanceof; + } + } + if (info->has_range) { + if (tssa->var_info[ssa_var].has_range) { + tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min); + tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max); + tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; + tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; + } else { + tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].range = info->range; + } + } + return 1; + } + return 0; +} + +static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op) +{ + while (1) { + if (p->op == ZEND_JIT_TRACE_VM) { + if (p->opline->opcode == ZEND_DO_UCALL + || p->opline->opcode == ZEND_DO_FCALL_BY_NAME + || p->opline->opcode == ZEND_DO_FCALL) { + if (p->opline->result_type != IS_UNUSED) { + return ssa_op->result_def; + } + } + return -1; + } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) { + /*skip */ + } else { + return -1; + } + p--; + } +} + +static int find_call_num_args(zend_jit_trace_rec *p) +{ + while (1) { + if (p->op == ZEND_JIT_TRACE_VM) { + if (p->opline->opcode == ZEND_INIT_FCALL + || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME + || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME + || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL + || p->opline->opcode == ZEND_INIT_USER_CALL + || p->opline->opcode == ZEND_NEW + || p->opline->opcode == ZEND_INIT_METHOD_CALL + || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) { + if (p->opline->extended_value <= 127) { + return p->opline->extended_value; + } else { + return -1; + } + } + return -1; + } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) { + /*skip */ + } else { + return -1; + } + p--; + } +} + +static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var) +{ + if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG) { + uint32_t idx = tssa->vars[var].definition; + + if (idx >= 0) { + if (tssa->ops[idx].op1_def == var) { + const zend_op *opline = ssa_opcodes[idx]; + if (opline->opcode == ZEND_PRE_DEC + || opline->opcode == ZEND_PRE_INC + || opline->opcode == ZEND_POST_DEC + || opline->opcode == ZEND_POST_INC) { + return 1; + } + } + if (tssa->ops[idx].result_def == var) { + const zend_op *opline = ssa_opcodes[idx]; + if (opline->opcode == ZEND_ADD + || opline->opcode == ZEND_SUB + || opline->opcode == ZEND_MUL + || opline->opcode == ZEND_PRE_DEC + || opline->opcode == ZEND_PRE_INC + || opline->opcode == ZEND_POST_DEC + || opline->opcode == ZEND_POST_INC) { + return 1; + } + } + } + } + return 0; +} + +static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr) +{ + zend_ssa *tssa; + zend_ssa_op *ssa_ops, *op; + zend_ssa_var *ssa_vars; + zend_ssa_var_info *ssa_var_info; + const zend_op_array *op_array; + const zend_op *opline; + const zend_op **ssa_opcodes; + zend_jit_trace_rec *p; + int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count; + int *var; + uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; + uint32_t optimization_level = ZCG(accel_directives).optimization_level; + int call_level, level, num_op_arrays; + size_t frame_size, stack_top, stack_size, stack_bottom; + zend_jit_op_array_trace_extension *jit_extension; + zend_ssa *ssa; + zend_jit_trace_stack_frame *frame, *top, *call; + zend_ssa_var_info return_value_info; + + /* 1. Count number of TSSA opcodes; + * Count number of activation frames; + * Calculate size of abstract stack; + * Construct regular SSA for involved op_array */ + op_array = trace_buffer->op_array; + stack_top = stack_size = zend_jit_trace_frame_size(op_array); + stack_bottom = 0; + p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; + ssa_ops_count = 0; + call_level = 0; + level = 0; + num_op_arrays = 0; + /* Remember op_array to cleanup */ + op_arrays[num_op_arrays++] = op_array; + /* Build SSA */ + ssa = zend_jit_trace_build_ssa(op_array, script); + for (;;p++) { + if (p->op == ZEND_JIT_TRACE_VM) { + ssa_ops_count += zend_jit_trace_op_len(p->opline); + } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + call_level++; + stack_top += zend_jit_trace_frame_size(p->op_array); + if (stack_top > stack_size) { + stack_size = stack_top; + } + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + frame_size = zend_jit_trace_frame_size(p->op_array); + if (call_level == 0) { + if (stack_top + frame_size > stack_size) { + stack_size = stack_top + frame_size; + } + } else { + call_level--; + stack_top -= frame_size; + } + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + op_array = p->op_array; + if (call_level == 0) { + stack_top += zend_jit_trace_frame_size(op_array); + if (stack_top > stack_size) { + stack_size = stack_top; + } + } else { + call_level--; + } + level++; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ssa = &jit_extension->func_info.ssa; + if (ssa->cfg.blocks_count) { + /* pass */ + } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { + /* Too many functions in single trace */ + return NULL; + } else { + /* Remember op_array to cleanup */ + op_arrays[num_op_arrays++] = op_array; + /* Build SSA */ + ssa = zend_jit_trace_build_ssa(op_array, script); + } + } else if (p->op == ZEND_JIT_TRACE_BACK) { + if (level == 0) { + stack_bottom += zend_jit_trace_frame_size(op_array);; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ssa = &jit_extension->func_info.ssa; + if (ssa->cfg.blocks_count) { + /* pass */ + } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { + /* Too many functions in single trace */ + return NULL; + } else { + /* Remember op_array to cleanup */ + op_arrays[num_op_arrays++] = op_array; + /* Build SSA */ + ssa = zend_jit_trace_build_ssa(op_array, script); + } + } else { + stack_top -= zend_jit_trace_frame_size(op_array); + level--; + } + op_array = p->op_array; + } else if (p->op == ZEND_JIT_TRACE_END) { + break; + } + } + *num_op_arrays_ptr = num_op_arrays; + + /* 2. Construct TSSA */ + tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_ssa)); + tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block)); + tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block)); + tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int)); + tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op)); + memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op)); + ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count, sizeof(zend_op*)); + JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom); + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + tssa->cfg.blocks_count = 2; + tssa->cfg.edges_count = 2; + + tssa->cfg.predecessors[0] = 0; + tssa->cfg.predecessors[1] = 1; + + tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE; + tssa->cfg.blocks[0].successors_count = 1; + tssa->cfg.blocks[0].predecessors_count = 0; + tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage; + tssa->cfg.blocks[0].successors[0] = 1; + + tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE; + tssa->cfg.blocks[1].successors_count = 1; + tssa->cfg.blocks[1].predecessors_count = 2; + tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage; + tssa->cfg.blocks[1].successors[1] = 1; + } else { + tssa->cfg.blocks_count = 1; + tssa->cfg.edges_count = 0; + + tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE; + tssa->cfg.blocks[0].successors_count = 0; + tssa->cfg.blocks[0].predecessors_count = 0; + } + + op_array = trace_buffer->op_array; + if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { + ssa_vars_count = op_array->last_var; + } else { + ssa_vars_count = op_array->last_var + op_array->T; + } + var = frame->stack; + for (i = 0; i < ssa_vars_count; i++) { + var[i] = i; + } + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + // TODO: For tracing, it's possible, to create pseudo Phi functions + // at the end of loop, without this additional pass (like LuaJIT) ??? + ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, var); + } + + p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; + idx = 0; + level = 0; + for (;;p++) { + if (p->op == ZEND_JIT_TRACE_VM) { + opline = p->opline; + ssa_opcodes[idx] = opline; + ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, var); + idx++; + len = zend_jit_trace_op_len(p->opline); + while (len > 1) { + opline++; + ssa_opcodes[idx] = opline; + if (opline->opcode != ZEND_OP_DATA) { + ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, var); + } + idx++; + len--; + } + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + frame = zend_jit_trace_call_frame(frame, op_array); + var = frame->stack; + op_array = p->op_array; + level++; + ZEND_ASSERT(ssa_vars_count < 0xff); + p->first_ssa_var = ssa_vars_count; + for (i = 0; i < op_array->last_var; i++) { + var[i] = ssa_vars_count++; + } + } else if (p->op == ZEND_JIT_TRACE_BACK) { + op_array = p->op_array; + frame = zend_jit_trace_ret_frame(frame, op_array); + var = frame->stack; + if (level == 0) { + ZEND_ASSERT(ssa_vars_count <= 0xff); + p->first_ssa_var = ssa_vars_count; + for (i = 0; i < op_array->last_var + op_array->T; i++) { + var[i] = ssa_vars_count++; + } + } else { + level--; + } + } else if (p->op == ZEND_JIT_TRACE_END) { + break; + } + } + + op_array = trace_buffer->op_array; + tssa->vars_count = ssa_vars_count; + tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var)); + if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { + vars_count = op_array->last_var; + } else { + vars_count = op_array->last_var + op_array->T; + } + i = 0; + while (i < vars_count) { + ssa_vars[i].var = i; + ssa_vars[i].scc = -1; + ssa_vars[i].definition = -1; + ssa_vars[i].use_chain = -1; + i++; + } + while (i < tssa->vars_count) { + ssa_vars[i].var = -1; + ssa_vars[i].scc = -1; + ssa_vars[i].definition = -1; + ssa_vars[i].use_chain = -1; + i++; + } + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + /* Update Phi sources */ + zend_ssa_phi *phi = tssa->blocks[1].phis; + + while (phi) { + phi->sources[1] = var[phi->var]; + ssa_vars[phi->ssa_var].var = phi->var; + ssa_vars[phi->ssa_var].definition_phi = phi; + ssa_vars[phi->sources[0]].phi_use_chain = phi; + ssa_vars[phi->sources[1]].phi_use_chain = phi; + phi = phi->next; + } + } + + /* 3. Compute use-def chains */ + idx = (ssa_ops_count - 1); + op = ssa_ops + idx; + while (idx >= 0) { + opline = ssa_opcodes[idx]; + if (op->op1_use >= 0) { + op->op1_use_chain = ssa_vars[op->op1_use].use_chain; + ssa_vars[op->op1_use].use_chain = idx; + } + if (op->op2_use >= 0 && op->op2_use != op->op1_use) { + op->op2_use_chain = ssa_vars[op->op2_use].use_chain; + ssa_vars[op->op2_use].use_chain = idx; + } + if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) { + op->res_use_chain = ssa_vars[op->result_use].use_chain; + ssa_vars[op->result_use].use_chain = idx; + } + if (op->op1_def >= 0) { + ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var); + ssa_vars[op->op1_def].definition = idx; + } + if (op->op2_def >= 0) { + ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var); + ssa_vars[op->op2_def].definition = idx; + } + if (op->result_def >= 0) { + ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var); + ssa_vars[op->result_def].definition = idx; + } + op--; + idx--; + } + + /* 4. Type inference */ + op_array = trace_buffer->op_array; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ssa = &jit_extension->func_info.ssa; + + tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); + + if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { + i = 0; + while (i < op_array->last_var) { + if (!ssa->var_info + || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) { + if (i < op_array->num_args) { + if (op_array->arg_info) { + zend_arg_info *arg_info = &op_array->arg_info[i]; + zend_class_entry *ce; + uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); + + if (ZEND_ARG_SEND_MODE(arg_info)) { + tmp |= MAY_BE_REF; + } + ssa_var_info[i].type = tmp; + } else { + ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + } else { + ssa_var_info[i].type = MAY_BE_UNDEF; + } + } + i++; + } + } else { + int parent_vars_count = 0; + zend_jit_trace_stack *parent_stack = NULL; + + i = 0; + if (parent_trace) { + parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, + op_array->last_var + op_array->T); + if (parent_vars_count) { + parent_stack = + zend_jit_traces[parent_trace].stack_map + + zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset; + } + } + while (i < op_array->last_var + op_array->T) { + if (!ssa->var_info + || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) { + if (i < op_array->last_var) { + ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } else { + ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + } + if (i < parent_vars_count) { + /* Initialize TSSA variable from parent trace */ + zend_uchar op_type = parent_stack[i]; + + if (op_type != IS_UNKNOWN) { + ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type); + } + } + i++; + } + } + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + /* Propagae initial value through Phi functions */ + zend_ssa_phi *phi = tssa->blocks[1].phis; + + while (phi) { + if (!ssa->var_info + || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) { + ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type; + } + phi = phi->next; + } + } + + frame = JIT_G(current_frame); + top = zend_jit_trace_call_frame(frame, op_array); + frame->call = NULL; + frame->prev = NULL; + frame->func = (const zend_function*)op_array; + frame->return_ssa_var = -1; + for (i = 0; i < op_array->last_var + op_array->T; i++) { + frame->stack[i] = -1; + } + memset(&return_value_info, 0, sizeof(return_value_info)); + + p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; + idx = 0; + level = 0; + for (;;p++) { + if (p->op == ZEND_JIT_TRACE_VM) { + uint8_t op1_type, op2_type, op3_type; + + // TODO: use types recorded in trace (add guards) ??? + // TODO: range inference ??? + opline = p->opline; + + + op1_type = p->op1_type; + op2_type = p->op2_type; + op3_type = p->op3_type; + if (op1_type & IS_TRACE_REFERENCE) { + op1_type = IS_UNKNOWN; + } + if (op2_type & IS_TRACE_REFERENCE) { + op2_type = IS_UNKNOWN; + } + if (op3_type & IS_TRACE_REFERENCE) { + op3_type = IS_UNKNOWN; + } + + if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { + // TODO: support for recorded classes ??? + p++; + } + if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { + // TODO: support for recorded classes ??? + p++; + } + + switch (opline->opcode) { + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_DIM: + if (tssa->ops[idx+1].op1_use >= 0 && op3_type != IS_UNKNOWN) { + zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx+1].op1_use]; + if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op3_type)) { + info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op3_type, info->type); + } + } + /* break missing intentionally */ + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: +// case ZEND_DIV: // TODO: check for division by zero ??? + case ZEND_CONCAT: + case ZEND_FAST_CONCAT: + case ZEND_ASSIGN_OP: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_CASE: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_IS: + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if (tssa->ops[idx].op2_use >= 0 && op2_type != IS_UNKNOWN) { + zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op2_use]; + if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op2_type)) { + info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op2_type, info->type); + } + } + /* break missing intentionally */ + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + case ZEND_QM_ASSIGN: + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + if (tssa->ops[idx].op1_use >= 0 && op1_type != IS_UNKNOWN) { + zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_use]; + if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op1_type)) { + info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op1_type, info->type); + } + } + break; + case ZEND_ASSIGN: + if (tssa->ops[idx].op2_use >= 0 && op2_type != IS_UNKNOWN) { + zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op2_use]; + if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op2_type)) { + info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op2_type, info->type); + } + } + break; + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + if (tssa->ops[idx].op1_use >= 0 && op1_type != IS_UNKNOWN) { + zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_use]; + if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op1_type)) { + info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op1_type, info->type); + } + } + /* Propagate argument type */ + if (frame->call + && frame->call->func->type == ZEND_USER_FUNCTION + && opline->op2.num <= frame->call->func->op_array.num_args) { + uint32_t info; + + if (opline->op1_type == IS_CONST) { + info = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op1))); + } else { + ZEND_ASSERT(ssa_ops[idx].op1_use >= 0); + info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD; + } + if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + zend_arg_info *arg_info; + + ZEND_ASSERT(frame->call->func->op_array.arg_info); + arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1]; + if (ZEND_TYPE_IS_SET(arg_info->type)) { + info &= ZEND_TYPE_FULL_MASK(arg_info->type); + if (!info) { + break; + } + } + } + if (info & MAY_BE_UNDEF) { + info |= MAY_BE_NULL; + info &= ~MAY_BE_UNDEF; + } + if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { + info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY; + } + frame->call->stack[opline->op2.num - 1] = info; + } + break; + case ZEND_RETURN: + if (tssa->ops[idx].op1_use >= 0 && op1_type != IS_UNKNOWN) { + zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_use]; + if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op1_type)) { + info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op1_type, info->type); + } + } + /* Propagate return value types */ + if (opline->op1_type == IS_UNUSED) { + return_value_info.type = MAY_BE_NULL; + } else if (opline->op1_type == IS_CONST) { + return_value_info.type = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op1))); + } else { + ZEND_ASSERT(ssa_ops[idx].op1_use >= 0); + return_value_info = ssa_var_info[ssa_ops[idx].op1_use]; + return_value_info.type &= ~MAY_BE_GUARD; + } + break; + default: + break; + } + len = zend_jit_trace_op_len(opline); + if (ssa->var_info) { + /* Add statically inferred ranges */ + if (ssa_ops[idx].op1_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); + } + if (ssa_ops[idx].op2_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); + } + if (ssa_ops[idx].result_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); + } + if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) { + if (ssa_ops[idx+1].op1_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def); + } + if (ssa_ops[idx+1].op2_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def); + } + if (ssa_ops[idx+1].result_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def); + } + } + } + if (opline->opcode == ZEND_RECV_INIT + && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + /* RECV_INIT always copy the constant */ + ssa_var_info[ssa_ops[idx].result_def].type = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op2))); + } else { + if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) { + // TODO: + assert(0); + } + } + if (ssa->var_info) { + /* Add statically inferred restrictions */ + if (ssa_ops[idx].op1_def >= 0) { + if ((opline->opcode == ZEND_SEND_VAR_EX + || opline->opcode == ZEND_FETCH_DIM_FUNC_ARG + || opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) + && frame + && frame->call + && frame->call->func + && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { + ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use]; + if (opline->opcode == ZEND_SEND_VAR_EX) { + ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD; + } + if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) { + ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN; + } + } else { + zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); + } + } + if (ssa_ops[idx].op2_def >= 0) { + zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); + } + if (ssa_ops[idx].result_def >= 0) { + zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); + } + } + idx++; + while (len > 1) { + opline++; + if (opline->opcode != ZEND_OP_DATA) { + if (ssa->var_info) { + /* Add statically inferred ranges */ + if (ssa_ops[idx].op1_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); + } + if (ssa_ops[idx].op2_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); + } + if (ssa_ops[idx].result_def >= 0) { + zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); + } + } + if (opline->opcode == ZEND_RECV_INIT + && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + /* RECV_INIT always copy the constant */ + ssa_var_info[ssa_ops[idx].result_def].type = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op2))); + } else { + if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) { + // TODO: + assert(0); + } + } + } + if (ssa->var_info) { + /* Add statically inferred restrictions */ + if (ssa_ops[idx].op1_def >= 0) { + zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); + } + if (ssa_ops[idx].op2_def >= 0) { + zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); + } + if (ssa_ops[idx].result_def >= 0) { + zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); + } + } + idx++; + len--; + } + + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + op_array = p->op_array; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ssa = &jit_extension->func_info.ssa; + + call = frame->call; + if (!call) { + /* Trace missed INIT_FCALL opcode */ + call = top; + call->call = NULL; + call->prev = NULL; + call->func = (const zend_function*)op_array; + top = zend_jit_trace_call_frame(top, op_array); + for (i = 0; i < op_array->last_var + op_array->T; i++) { + call->stack[i] = -1; + } + } else { + ZEND_ASSERT(&call->func->op_array == op_array); + } + frame->call = call->prev; + call->prev = frame; + call->return_ssa_var = find_return_ssa_var(p - 1, ssa_ops + (idx -1)); + frame = call; + + level++; + i = 0; + v = p->first_ssa_var; + while (i < op_array->last_var) { + if (!ssa->var_info + || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) { + if (i < op_array->num_args) { + if (op_array->arg_info) { + zend_arg_info *arg_info = &op_array->arg_info[i]; + zend_class_entry *ce; + uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); + + if (ZEND_ARG_SEND_MODE(arg_info)) { + tmp |= MAY_BE_REF; + } + ssa_var_info[v].type = tmp; + } else { + ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + } else { + ssa_var_info[v].type = MAY_BE_UNDEF; + } + } + if (i < op_array->num_args) { + /* Propagate argument type */ + ssa_var_info[v].type &= frame->stack[i]; + } + i++; + v++; + } + + } else if (p->op == ZEND_JIT_TRACE_BACK) { + op_array = p->op_array; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ssa = &jit_extension->func_info.ssa; + if (level == 0) { + i = 0; + v = p->first_ssa_var; + while (i < op_array->last_var) { + if (!ssa->var_info + || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) { + ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + i++; + v++; + } + while (i < op_array->last_var + op_array->T) { + if (!ssa->var_info + || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) { + ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + i++; + v++; + } + if (return_value_info.type != 0) { + if ((p+1)->op == ZEND_JIT_TRACE_VM) { + const zend_op *opline = (p+1)->opline - 1; + if (opline->result_type != IS_UNUSED) { + ssa_var_info[ + p->first_ssa_var + + EX_VAR_TO_NUM(opline->result.var)] = return_value_info; + } + } + memset(&return_value_info, 0, sizeof(return_value_info)); + } + } else { + level--; + if (return_value_info.type != 0) { + if ((p+1)->op == ZEND_JIT_TRACE_VM) { + const zend_op *opline = (p+1)->opline - 1; + if (opline->result_type != IS_UNUSED) { + if (frame->return_ssa_var >= 0) { + ssa_var_info[frame->return_ssa_var] = return_value_info; + } + } + } + memset(&return_value_info, 0, sizeof(return_value_info)); + } + } + + top = frame; + if (frame->prev) { + frame = frame->prev; + ZEND_ASSERT(&frame->func->op_array == op_array); + } else { + frame = zend_jit_trace_ret_frame(frame, op_array); + frame->call = NULL; + frame->prev = NULL; + frame->func = (const zend_function*)op_array; + frame->return_ssa_var = -1; + for (i = 0; i < op_array->last_var + op_array->T; i++) { + frame->stack[i] = -1; + } + } + + } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + call = top; + call->call = NULL; + call->prev = frame->call; + call->func = p->func; + frame->call = call; + top = zend_jit_trace_call_frame(top, p->op_array); + if (p->func->type == ZEND_USER_FUNCTION) { + for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { + call->stack[i] = -1; + } + } + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + call = frame->call; + if (call) { + top = call; + frame->call = call->prev; + } + } else if (p->op == ZEND_JIT_TRACE_END) { + break; + } + } + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + /* Propagate guards through Phi sources */ + zend_ssa_phi *phi = tssa->blocks[1].phis; + + while (phi) { + uint32_t t = ssa_var_info[phi->ssa_var].type; + uint32_t t0 = ssa_var_info[phi->sources[0]].type; + uint32_t t1 = ssa_var_info[phi->sources[1]].type; + + if (t & MAY_BE_GUARD) { + if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { + if (!((t0 | t1) & MAY_BE_GUARD)) { + ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD; + } + } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { + if (!(t1 & MAY_BE_GUARD)) { + ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD; + ssa_var_info[phi->sources[0]].type = t | MAY_BE_GUARD; + } + } else { + if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { + ssa_var_info[phi->sources[0]].type = MAY_BE_GUARD | (t & t0); + } + if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { + if (is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) { + ssa_var_info[phi->sources[1]].type = MAY_BE_GUARD | (t & t1); + ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD; + } + } + } + } + phi = phi->next; + } + } + + if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_TSSA)) { + zend_jit_dump_trace(trace_buffer, tssa); + fprintf(stderr, "---- TRACE analysed\n"); + } + + return tssa; +} + +static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) +{ + const void *handler = NULL; + dasm_State* dasm_state = NULL; + zend_script *script = NULL; + zend_lifetime_interval **ra = NULL; + zend_string *name = NULL; + void *checkpoint; + const zend_op_array *op_array; + zend_ssa *ssa, *op_array_ssa; + zend_jit_trace_rec *p; + int call_level = -1; // TODO: proper support for inlined functions ??? + zend_jit_op_array_trace_extension *jit_extension; + int num_op_arrays = 0; + zend_jit_trace_info *t; + const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS]; + zend_uchar smart_branch_opcode; + const void *exit_addr; + uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; + zend_bool send_result = 0; + zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; + zend_class_entry *ce; + uint32_t i; + zend_jit_trace_stack_frame *frame, *top, *call; + zend_jit_trace_stack *stack; + zend_uchar res_type = IS_UNKNOWN; + const zend_op *opline, *orig_opline; + const zend_ssa_op *ssa_op, *orig_ssa_op; + + checkpoint = zend_arena_checkpoint(CG(arena)); + + ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays); + + p = trace_buffer; + ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); + op_array = p->op_array; + frame = JIT_G(current_frame); + top = zend_jit_trace_call_frame(frame, op_array); + frame->call = NULL; + frame->prev = NULL; + frame->func = (const zend_function*)op_array; + frame->return_value_used = -1; + frame->nested = 0; + frame->num_args = -1; + stack = frame->stack; + + if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { + i = 0; + while (i < op_array->last_var) { + if (!(ssa->var_info[i].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[i].type)) { + stack[i] = concrete_type(ssa->var_info[i].type); + } else if (i < op_array->num_args) { + stack[i] = IS_UNKNOWN; + } else { + stack[i] = IS_UNDEF; + } + i++; + } + } else { + int parent_vars_count = 0; + zend_jit_trace_stack *parent_stack = NULL; + + i = 0; + if (parent_trace) { + parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, + op_array->last_var + op_array->T); + if (parent_vars_count) { + parent_stack = + zend_jit_traces[parent_trace].stack_map + + zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset; + } + } + while (i < op_array->last_var + op_array->T) { + if (!(ssa->var_info[i].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[i].type)) { + stack[i] = concrete_type(ssa->var_info[i].type); + } else if (i < parent_vars_count + && (zend_uchar)parent_stack[i] != IS_UNKNOWN) { + stack[i] = parent_stack[i]; + } else { + stack[i] = IS_UNKNOWN; + } + i++; + } + } + + opline = ((zend_jit_trace_start_rec*)p)->opline; + name = zend_jit_trace_name(op_array, opline->lineno); + p += ZEND_JIT_TRACE_START_REC_SIZE; + + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + op_array_ssa = &jit_extension->func_info.ssa; + + // TODO: register allocation ??? + + dasm_growpc(&dasm_state, 1); /* trace needs just one global lable for loop */ + + zend_jit_align_func(&dasm_state); + if (!parent_trace) { + zend_jit_prologue(&dasm_state); + } + zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM); + + if (!parent_trace) { + zend_jit_set_opline(&dasm_state, opline); + } else { + if (!((uintptr_t)zend_jit_traces[parent_trace].exit_info[exit_num].opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) { + zend_jit_trace_opline_guard(&dasm_state, opline); + zend_jit_set_opline(&dasm_state, opline); + } else { + zend_jit_reset_opline(&dasm_state, opline); + } + } + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP + || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL + || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { + + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + /* Check loop-invariant varaible types */ + for (i = 0; i < op_array->last_var + op_array->T; i++) { + uint32_t info = ssa->var_info[i].type; + + ZEND_ASSERT(ssa->vars[i].definition == -1); + ZEND_ASSERT(ssa->vars[i].definition_phi == NULL); + ZEND_ASSERT(ssa->vars[i].var == i); + + if (info & MAY_BE_GUARD) { + if (ssa->vars[i].use_chain != -1 + || (ssa->vars[i].phi_use_chain + && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD))) { + if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) { + goto jit_failure; + } + ssa->var_info[i].type = info & ~MAY_BE_GUARD; + stack[i] = concrete_type(info); + } + } + } + } + + zend_jit_label(&dasm_state, 0); /* start of of trace loop */ + + if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { + // TODO: interupt exit may require deoptimization through side exit ??? + zend_jit_check_timeout(&dasm_state, opline); + } + } + + ssa_op = ssa->ops; + for (;;p++) { + if (p->op == ZEND_JIT_TRACE_VM) { + uint8_t op1_type = p->op1_type; + uint8_t op2_type = p->op2_type; + uint8_t op3_type = p->op3_type; + + opline = p->opline; + if (op1_type & IS_TRACE_REFERENCE) { + op1_type = IS_UNKNOWN; + } + if (op2_type & IS_TRACE_REFERENCE) { + op2_type = IS_UNKNOWN; + } + if (op3_type & IS_TRACE_REFERENCE) { + op3_type = IS_UNKNOWN; + } + + if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { + // TODO: support for recorded classes ??? + p++; + } + if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { + // TODO: support for recorded classes ??? + p++; + } + +#if 0 + // TODO: call level calculation doesn't work for traces ??? + switch (opline->opcode) { + case ZEND_INIT_FCALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_USER_CALL: + case ZEND_NEW: + call_level++; + } +#endif + + if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { + switch (opline->opcode) { + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (opline->op1_type != IS_CV) { + break; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + if (!(op1_info & MAY_BE_LONG)) { + break; + } + if (opline->result_type != IS_UNUSED) { + res_use_info = RES_USE_INFO_EX(); + USE_RES_TRACE_TYPE(); + res_info = RES_INFO_EX(); + res_addr = RES_REG_ADDR(); + } else { + res_use_info = -1; + res_info = -1; + res_addr = 0; + } + op1_def_info = OP1_DEF_INFO_EX(); + if (!zend_jit_inc_dec(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), + op1_def_info, OP1_DEF_REG_ADDR(), + res_use_info, res_info, + res_addr, + (op1_def_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) { + ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; + if (opline->result_type != IS_UNUSED) { + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + } + } + goto done; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + break; + } + if (!(op1_info & MAY_BE_LONG) + || !(op2_info & MAY_BE_LONG)) { + break; + } + if (opline->result_type == IS_TMP_VAR + && (p+1)->op == ZEND_JIT_TRACE_VM + && (p+1)->opline == opline + 1 + && (opline+1)->opcode == ZEND_SEND_VAL + && (opline+1)->op1_type == IS_TMP_VAR + && (opline+1)->op1.var == opline->result.var) { + p++; + if (frame->call) { + uint8_t res_type = p->op1_type; + if (res_type & IS_TRACE_REFERENCE) { + res_type = IS_UNKNOWN;; + } + if (res_type != IS_UNKNOWN) { + zend_jit_trace_send_type(opline+1, frame->call, res_type); + } + } + while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE || + (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { + p++; + } + send_result = 1; + res_use_info = -1; + res_addr = 0; /* set inside backend */ + } else { + send_result = 0; + res_use_info = RES_USE_INFO_EX(); + USE_RES_TRACE_TYPE(); + res_addr = RES_REG_ADDR(); + } + res_info = RES_INFO_EX(); + if (!zend_jit_long_math(&dasm_state, opline, op_array, + op1_info, OP1_RANGE_EX(), OP1_REG_ADDR(), + op2_info, OP2_RANGE_EX(), OP2_REG_ADDR(), + res_use_info, res_info, res_addr, + send_result, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: +// case ZEND_DIV: // TODO: check for division by zero ??? + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + break; + } + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + break; + } + if (opline->result_type == IS_TMP_VAR + && (p+1)->op == ZEND_JIT_TRACE_VM + && (p+1)->opline == opline + 1 + && (opline+1)->opcode == ZEND_SEND_VAL + && (opline+1)->op1_type == IS_TMP_VAR + && (opline+1)->op1.var == opline->result.var) { + p++; + if (frame->call + && frame->call->func->type == ZEND_USER_FUNCTION) { + uint8_t res_type = p->op1_type; + if (res_type & IS_TRACE_REFERENCE) { + res_type = IS_UNKNOWN;; + } + if (res_type != IS_UNKNOWN) { + zend_jit_trace_send_type(opline+1, frame->call, res_type); + } + } + while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE || + (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { + p++; + } + send_result = 1; + res_use_info = -1; + res_addr = 0; /* set inside backend */ + } else { + send_result = 0; + res_use_info = RES_USE_INFO_EX(); + USE_RES_TRACE_TYPE(); + res_addr = RES_REG_ADDR(); + } + res_info = RES_INFO_EX(); + if (!zend_jit_math(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), + op2_info, OP2_REG_ADDR(), + res_use_info, res_info, res_addr, + send_result, + (res_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow_ex(opline, ssa_op, op_array, ssa), + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + if ((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) { + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + } + goto done; + case ZEND_CONCAT: + case ZEND_FAST_CONCAT: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + break; + } + if (!(op1_info & MAY_BE_STRING) || + !(op2_info & MAY_BE_STRING)) { + break; + } + if (opline->result_type == IS_TMP_VAR + && (p+1)->op == ZEND_JIT_TRACE_VM + && (p+1)->opline == opline + 1 + && (opline+1)->opcode == ZEND_SEND_VAL + && (opline+1)->op1_type == IS_TMP_VAR + && (opline+1)->op1.var == opline->result.var) { + p++; + if (frame->call + && frame->call->func->type == ZEND_USER_FUNCTION) { + uint8_t res_type = p->op1_type; + if (res_type & IS_TRACE_REFERENCE) { + res_type = IS_UNKNOWN;; + } + if (res_type != IS_UNKNOWN) { + zend_jit_trace_send_type(opline+1, frame->call, res_type); + } + } + while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE || + (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { + p++; + } + send_result = 1; + } else { + send_result = 0; + } + res_info = RES_INFO_EX(); + if (!zend_jit_concat(&dasm_state, opline, op_array, + op1_info, op2_info, res_info, send_result, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_ASSIGN_OP: + if (opline->extended_value == ZEND_POW + || opline->extended_value == ZEND_DIV) { + // TODO: check for division by zero ??? + break; + } + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + break; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + break; + } + if (opline->extended_value == ZEND_ADD + || opline->extended_value == ZEND_SUB + || opline->extended_value == ZEND_MUL + || opline->extended_value == ZEND_DIV) { + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) + || !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + break; + } + } else if (opline->extended_value == ZEND_BW_OR + || opline->extended_value == ZEND_BW_AND + || opline->extended_value == ZEND_BW_XOR + || opline->extended_value == ZEND_SL + || opline->extended_value == ZEND_SR + || opline->extended_value == ZEND_MOD) { + if (!(op1_info & MAY_BE_LONG) + || !(op2_info & MAY_BE_LONG)) { + break; + } + } else if (opline->extended_value == ZEND_CONCAT) { + if (!(op1_info & MAY_BE_STRING) + || !(op2_info & MAY_BE_STRING)) { + break; + } + } + op1_def_info = OP1_DEF_INFO_EX(); + if (!zend_jit_assign_op(&dasm_state, opline, op_array, + op1_info, op1_def_info, OP1_RANGE_EX(), + op2_info, OP2_RANGE_EX(), + (op1_def_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow_ex(opline, ssa_op, op_array, ssa), + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) { + ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; + if (opline->result_type != IS_UNUSED) { + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + } + } + goto done; + case ZEND_ASSIGN_DIM_OP: + if (opline->extended_value == ZEND_POW + || opline->extended_value == ZEND_DIV) { + // TODO: check for division by zero ??? + break; + } + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + break; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + op1_data_info = OP1_DATA_INFO_EX(); + CHECK_OP1_DATA_TRACE_TYPE(); + op1_def_info = OP1_DEF_INFO_EX(); + if (!zend_jit_assign_dim_op(&dasm_state, opline, op_array, + op1_info, op1_def_info, op2_info, + op1_data_info, OP1_DATA_RANGE(), + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_ASSIGN_DIM: + if (opline->op1_type != IS_CV) { + break; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + op1_data_info = OP1_DATA_INFO_EX(); + CHECK_OP1_DATA_TRACE_TYPE(); + if (!zend_jit_assign_dim(&dasm_state, opline, op_array, + op1_info, op2_info, op1_data_info, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_ASSIGN: + if (opline->op1_type != IS_CV) { + break; + } + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + res_info = -1; + } else { + res_addr = RES_REG_ADDR(); + res_info = RES_INFO_EX(); + } + op2_addr = OP2_REG_ADDR(); + if (ra + && ssa_op->op2_def >= 0 + && !ssa->vars[ssa_op->op2_def].no_val) { + op2_def_addr = OP2_DEF_REG_ADDR(); + } else { + op2_def_addr = op2_addr; + } + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + op1_info = OP1_INFO_EX(); + op1_def_info = OP1_DEF_INFO_EX(); + USE_OP1_TRACE_TYPE(); + if (!zend_jit_assign(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), + op1_def_info, OP1_DEF_REG_ADDR(), + op2_info, op2_addr, op2_def_addr, + res_info, res_addr, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_QM_ASSIGN: + op1_addr = OP1_REG_ADDR(); + if (ra + && ssa_op->op1_def >= 0 + && !ssa->vars[ssa_op->op1_def].no_val) { + op1_def_addr = OP1_DEF_REG_ADDR(); + } else { + op1_def_addr = op1_addr; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE();//???USE_OP1_TRACE_TYPE(); + res_info = RES_INFO_EX(); + if (!zend_jit_qm_assign(&dasm_state, opline, op_array, + op1_info, op1_addr, op1_def_addr, + res_info, RES_REG_ADDR())) { + goto jit_failure; + } + goto done; + case ZEND_INIT_FCALL: + case ZEND_INIT_FCALL_BY_NAME: + if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, op_array_ssa, call_level, p + 1)) { + goto jit_failure; + } + goto done; + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + if (opline->opcode == ZEND_SEND_VAL_EX + && opline->op2.num > MAX_ARG_FLAG_NUM) { + break; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); //???USE_OP1_TRACE_TYPE(); + if (!zend_jit_send_val(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR())) { + goto jit_failure; + } + if (frame->call + && frame->call->func->type == ZEND_USER_FUNCTION) { + if (opline->op1_type == IS_CONST) { + zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1))); + } else if (op1_type != IS_UNKNOWN) { + zend_jit_trace_send_type(opline, frame->call, op1_type); + } + } + goto done; + case ZEND_SEND_REF: + op1_info = OP1_INFO_EX(); + USE_OP1_TRACE_TYPE(); + if (!zend_jit_send_ref(&dasm_state, opline, op_array, + op1_info, 0)) { + goto jit_failure; + } + goto done; + case ZEND_SEND_VAR: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + if ((opline->opcode == ZEND_SEND_VAR_EX + || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) + && opline->op2.num > MAX_ARG_FLAG_NUM) { + break; + } + op1_addr = OP1_REG_ADDR(); + if (ra + && ssa_op->op1_def >= 0 + && !ssa->vars[ssa_op->op1_def].no_val) { + op1_def_addr = OP1_DEF_REG_ADDR(); + } else { + op1_def_addr = op1_addr; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); //???USE_OP1_TRACE_TYPE(); + if (!zend_jit_send_var(&dasm_state, opline, op_array, + op1_info, op1_addr, op1_def_addr)) { + goto jit_failure; + } + if (frame->call + && frame->call->func->type == ZEND_USER_FUNCTION) { + if (opline->opcode == ZEND_SEND_VAR_EX + && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { + // TODO: this may require invalidation, if caller is changed ??? + goto done; + } + if (op1_type != IS_UNKNOWN) { + zend_jit_trace_send_type(opline, frame->call, op1_type); + } + } + goto done; + case ZEND_DO_UCALL: + case ZEND_DO_ICALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, call_level, -1, p + 1)) { + goto jit_failure; + } + goto done; + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_CASE: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + zend_bool exit_if_true = 0; + const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + goto jit_failure; + } + smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + } else { + smart_branch_opcode = 0; + exit_addr = NULL; + } + if (!zend_jit_cmp(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), + op2_info, OP2_REG_ADDR(), + RES_REG_ADDR(), + zend_may_throw_ex(opline, ssa_op, op_array, ssa), + smart_branch_opcode, -1, -1, exit_addr)) { + goto jit_failure; + } + goto done; + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + zend_bool exit_if_true = 0; + const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + goto jit_failure; + } + if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { + exit_if_true = !exit_if_true; + } + smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + } else { + smart_branch_opcode = 0; + exit_addr = NULL; + } + if (!zend_jit_identical(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), + op2_info, OP2_REG_ADDR(), + RES_REG_ADDR(), + zend_may_throw_ex(opline, ssa_op, op_array, ssa), + smart_branch_opcode, -1, -1, exit_addr)) { + goto jit_failure; + } + goto done; + case ZEND_DEFINED: + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + zend_bool exit_if_true = 0; + const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + goto jit_failure; + } + smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + } else { + smart_branch_opcode = 0; + exit_addr = NULL; + } + if (!zend_jit_defined(&dasm_state, opline, op_array, smart_branch_opcode, -1, -1, exit_addr)) { + goto jit_failure; + } + goto done; + case ZEND_TYPE_CHECK: + if (opline->extended_value == MAY_BE_RESOURCE) { + // TODO: support for is_resource() ??? + break; + } + op1_info = OP1_INFO_EX(); + USE_OP1_TRACE_TYPE(); + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + zend_bool exit_if_true = 0; + const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + goto jit_failure; + } + smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + } else { + smart_branch_opcode = 0; + exit_addr = NULL; + } + if (!zend_jit_type_check(&dasm_state, opline, op_array, op1_info, smart_branch_opcode, -1, -1, exit_addr)) { + goto jit_failure; + } + goto done; + case ZEND_RETURN: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + if (opline->op1_type == IS_CONST) { + res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1)); + } else if (op1_type != IS_UNKNOWN) { + res_type = op1_type; + } + if (op_array->type == ZEND_EVAL_CODE + // TODO: support for top-level code + || !op_array->function_name + // TODO: support for IS_UNDEF ??? + || (op1_info & MAY_BE_UNDEF)) { + if (!zend_jit_tail_handler(&dasm_state, opline)) { + goto jit_failure; + } + } else { + int j; + + if (!zend_jit_return(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR())) { + goto jit_failure; + } + if (!zend_jit_leave_frame(&dasm_state)) { + goto jit_failure; + } + for (j = 0 ; j < op_array->last_var; j++) { + // TODO: get info from trace ??? + uint32_t info = zend_ssa_cv_info(opline, op_array, op_array_ssa, j); + zend_uchar type = stack[j]; + + info = zend_jit_trace_type_to_info_ex(type, info); + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (!zend_jit_free_cv(&dasm_state, opline, op_array, info, j)) { + goto jit_failure; + } + } + } + if (!zend_jit_leave_func(&dasm_state, opline, op_array, p + 1)) { + goto jit_failure; + } + } + goto done; + case ZEND_BOOL: + case ZEND_BOOL_NOT: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), + -1, -1, + zend_may_throw_ex(opline, ssa_op, op_array, ssa), + opline->opcode, NULL)) { + goto jit_failure; + } + goto done; + case ZEND_JMPZ: + case ZEND_JMPNZ: + if (/*opline > op_array->opcodes + ssa->cfg.blocks[b].start && ??? */ + opline->op1_type == IS_TMP_VAR && + ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + /* smart branch */ + break; + } + /* break missing intentionally */ + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + if (opline->result_type == IS_UNDEF) { + res_addr = 0; + } else { + res_addr = RES_REG_ADDR(); + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) { + const zend_op *exit_opline = NULL; + uint32_t exit_point; + + if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) { + /* taken branch */ + if (opline->opcode == ZEND_JMPNZ_EX) { + smart_branch_opcode = ZEND_JMPZ_EX; + } else if (opline->opcode == ZEND_JMPZ_EX) { + smart_branch_opcode = ZEND_JMPNZ_EX; + } else if (opline->opcode == ZEND_JMPNZ) { + smart_branch_opcode = ZEND_JMPZ; + } else { + smart_branch_opcode = ZEND_JMPNZ; + } + exit_opline = (opline->opcode == ZEND_JMPZNZ) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + opline + 1; + } else if (opline->opcode == ZEND_JMPZNZ) { + ZEND_ASSERT((p+1)->opline == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)); + smart_branch_opcode = ZEND_JMPZ; + exit_opline = OP_JMP_ADDR(opline, opline->op2); + } else if ((p+1)->opline == opline + 1) { + /* not taken branch */ + smart_branch_opcode = opline->opcode; + exit_opline = OP_JMP_ADDR(opline, opline->op2); + } else { + ZEND_ASSERT(0); + } + exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p+1); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + goto jit_failure; + } + } else { + ZEND_ASSERT(0); + } + if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array, + op1_info, OP1_REG_ADDR(), res_addr, + -1, -1, + zend_may_throw_ex(opline, ssa_op, op_array, ssa), + smart_branch_opcode, exit_addr)) { + goto jit_failure; + } + goto done; + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_IS: + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + res_info = RES_INFO_EX(); + if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, + op1_info, op2_info, res_info, + ( + (op1_info & MAY_BE_ANY) != MAY_BE_ARRAY || + (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) != 0 || + ((op1_info & MAY_BE_UNDEF) != 0 && + opline->opcode == ZEND_FETCH_DIM_R) || + ((opline->op1_type & (IS_TMP_VAR|IS_VAR)) != 0 && + (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) != 0) || + (op2_info & MAY_BE_UNDEF) != 0 || + ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) != 0 && + (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) != 0)))) { + goto jit_failure; + } + goto done; + goto done; + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if ((opline->extended_value & ZEND_ISEMPTY)) { + // TODO: support for empty() ??? + break; + } + op1_info = OP1_INFO_EX(); + CHECK_OP1_TRACE_TYPE(); + op2_info = OP2_INFO_EX(); + CHECK_OP2_TRACE_TYPE(); + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { + zend_bool exit_if_true = 0; + const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + goto jit_failure; + } + smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + } else { + smart_branch_opcode = 0; + exit_addr = NULL; + } + if (!zend_jit_isset_isempty_dim(&dasm_state, opline, op_array, + op1_info, op2_info, + zend_may_throw_ex(opline, ssa_op, op_array, ssa), + smart_branch_opcode, -1, -1, + exit_addr)) { + goto jit_failure; + } + goto done; + case ZEND_FETCH_OBJ_R: + case ZEND_FETCH_OBJ_IS: + if (opline->op2_type != IS_CONST + || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING + || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { + break; + } + ce = NULL; + if (opline->op1_type == IS_UNUSED) { + op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; + ce = op_array->scope; + } else { + op1_info = OP1_INFO_EX(); + if (ssa->var_info && ssa->ops) { + if (ssa_op->op1_use >= 0) { + zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use; + if (op1_ssa->ce && !op1_ssa->is_instanceof && !op1_ssa->ce->create_object) { + ce = op1_ssa->ce; + } + } + } + } + if (!(op1_info & MAY_BE_OBJECT)) { + break; + } + if (!zend_jit_fetch_obj_read(&dasm_state, opline, op_array, + op1_info, ce, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_BIND_GLOBAL: + orig_opline = opline; + orig_ssa_op = ssa_op; + while (1) { + if (!ssa->ops || !ssa->var_info) { + op1_info = MAY_BE_ANY|MAY_BE_REF; + } else { + op1_info = OP1_INFO_EX(); + } + if (!zend_jit_bind_global(&dasm_state, opline, op_array, op1_info)) { + goto jit_failure; + } + if ((opline+1)->opcode == ZEND_BIND_GLOBAL) { + opline++; + ssa_op++; + } else { + break; + } + } + opline = orig_opline; + ssa_op = orig_ssa_op; + goto done; + case ZEND_RECV: + if (!zend_jit_recv(&dasm_state, opline, op_array)) { + goto jit_failure; + } + goto done; + case ZEND_RECV_INIT: + orig_opline = opline; + orig_ssa_op = ssa_op; + while (1) { + if (!zend_jit_recv_init(&dasm_state, opline, op_array, + (opline + 1)->opcode != ZEND_RECV_INIT, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + if ((opline+1)->opcode == ZEND_RECV_INIT) { + opline++; + ssa_op++; + } else { + break; + } + } + opline = orig_opline; + ssa_op = orig_ssa_op; + goto done; + case ZEND_FREE: + case ZEND_FE_FREE: + op1_info = OP1_INFO_EX(); + USE_OP1_TRACE_TYPE(); + if (!zend_jit_free(&dasm_state, opline, op_array, op1_info, + zend_may_throw_ex(opline, ssa_op, op_array, ssa))) { + goto jit_failure; + } + goto done; + case ZEND_ECHO: + if (opline->op1_type != IS_CONST + || Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) != IS_STRING) { + break; + } + if (!zend_jit_echo(&dasm_state, opline, op_array)) { + goto jit_failure; + } + goto done; +#if 0 + case ZEND_SWITCH_LONG: + case ZEND_SWITCH_STRING: + if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa)) { + goto jit_failure; + } + goto done; +#endif +// case ZEND_INIT_NS_FCALL_BY_NAME: + // TODO: we may need a guard after INIT_NS_FCALL??? + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_DYNAMIC_CALL: + if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) { + goto jit_failure; + } + if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL) { + if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) { + goto jit_failure; + } + } + goto done; + case ZEND_INIT_STATIC_METHOD_CALL: + if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) { + goto jit_failure; + } + if ((opline->op1_type != IS_CONST + || opline->op2_type != IS_CONST) + && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL) { + if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) { + goto jit_failure; + } + } + goto done; + case ZEND_INIT_USER_CALL: + if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) { + goto jit_failure; + } + if (opline->op2_type != IS_CONST + && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL) { + if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) { + goto jit_failure; + } + } + goto done; + case ZEND_NEW: + if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) { + goto jit_failure; + } + if (opline->op1_type != IS_CONST + && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL) { + if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) { + goto jit_failure; + } + } + goto done; + default: + break; + } + } + + if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) { + if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) { + goto jit_failure; + } + } + +done: +#if 0 + // TODO: call level calculation doesn't work for traces ??? + switch (opline->opcode) { + case ZEND_DO_FCALL: + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + call_level--; + } +#endif + + /* Keep information about known types on abstract stack */ + if (ssa_op->result_def >= 0) { + zend_uchar type = IS_UNKNOWN; + + if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0 + || send_result) { + /* we didn't set result variable */ + type = IS_UNKNOWN; + } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) { + type = concrete_type(ssa->var_info[ssa_op->result_def].type); + } else if (opline->opcode == ZEND_QM_ASSIGN) { + if (opline->op1_type != IS_CONST) { + /* copy */ + type = STACK_VAR_TYPE(opline->op1.var); + } + } else if (opline->opcode == ZEND_ASSIGN) { + if (opline->op2_type != IS_CONST) { + /* copy */ + type = STACK_VAR_TYPE(opline->op2.var); + } + } else if (opline->opcode == ZEND_POST_INC + || opline->opcode == ZEND_POST_DEC) { + /* copy */ + type = STACK_VAR_TYPE(opline->op1.var); + } + SET_RES_STACK_VAR_TYPE(type); + if (type != IS_UNKNOWN) { + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + } + } + if (ssa_op->op1_def >= 0) { + zend_uchar type = IS_UNKNOWN; + + if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) { + type = concrete_type(ssa->var_info[ssa_op->op1_def].type); + } else if (opline->opcode == ZEND_ASSIGN) { + if (!(OP1_INFO_EX() & MAY_BE_REF) + || STACK_VAR_TYPE(opline->op1.var) != IS_UNKNOWN) { + if (opline->op2_type != IS_CONST) { + /* copy */ + type = STACK_VAR_TYPE(opline->op2.var); + } + } + } else if (opline->opcode == ZEND_SEND_VAR + || opline->opcode == ZEND_CAST + || opline->opcode == ZEND_QM_ASSIGN + || opline->opcode == ZEND_JMP_SET + || opline->opcode == ZEND_COALESCE + || opline->opcode == ZEND_FE_RESET_R) { + /* keep old value */ + type = STACK_VAR_TYPE(opline->op1.var); + } + SET_OP1_STACK_VAR_TYPE(type); + if (type != IS_UNKNOWN) { + ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; + } + } + if (ssa_op->op2_def >= 0) { + zend_uchar type = IS_UNKNOWN; + + if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) { + type = concrete_type(ssa->var_info[ssa_op->op2_def].type); + } else if (opline->opcode == ZEND_ASSIGN) { + /* keep old value */ + type = STACK_VAR_TYPE(opline->op2.var); + } + SET_OP2_STACK_VAR_TYPE(type); + if (type != IS_UNKNOWN) { + ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD; + } + } + + switch (opline->opcode) { + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_OBJ_OP: + case ZEND_ASSIGN_STATIC_PROP_OP: + case ZEND_ASSIGN_OBJ_REF: + case ZEND_ASSIGN_STATIC_PROP_REF: + /* OP_DATA */ + ssa_op++; + opline++; + if (ssa_op->op1_def >= 0) { + zend_uchar type = IS_UNKNOWN; + + if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) { + type = concrete_type(ssa->var_info[ssa_op->op1_def].type); + } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM + || (opline-1)->opcode == ZEND_ASSIGN_OBJ + || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) { + /* keep old value */ + type = STACK_VAR_TYPE(opline->op1.var); + } + SET_OP1_STACK_VAR_TYPE(type); + if (type != IS_UNKNOWN) { + ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; + } + } + ssa_op++; + break; + case ZEND_RECV_INIT: + ssa_op++; + opline++; + while (opline->opcode == ZEND_RECV_INIT) { + if (ssa_op->result_def >= 0) { + zend_uchar type = IS_UNKNOWN; + + if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) { + type = concrete_type(ssa->var_info[ssa_op->result_def].type); + } + SET_RES_STACK_VAR_TYPE(type); + } + ssa_op++; + opline++; + } + break; + case ZEND_BIND_GLOBAL: + ssa_op++; + opline++; + while (opline->opcode == ZEND_BIND_GLOBAL) { + if (ssa_op->op1_def >= 0) { + zend_uchar type = IS_UNKNOWN; + + if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) { + type = concrete_type(ssa->var_info[ssa_op->op1_def].type); + } + SET_OP1_STACK_VAR_TYPE(type); + } + ssa_op++; + opline++; + } + break; + default: + ssa_op += zend_jit_trace_op_len(opline); + break; + } + + if (send_result) { + ssa_op++; + send_result = 0; + } + + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + op_array = (zend_op_array*)p->op_array; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + op_array_ssa = &jit_extension->func_info.ssa; + call = frame->call; + if (!call) { + /* Trace missed INIT_FCALL opcode */ + call = top; + call->call = NULL; + call->prev = NULL; + call->func = (const zend_function*)op_array; + call->nested = 0; + call->num_args = -1; // TODO: should be possible to get the real number ??? + top = zend_jit_trace_call_frame(top, op_array); + i = 0; + while (i < p->op_array->num_args) { + /* Initialize abstract stack using SSA */ + if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) { + call->stack[i] = concrete_type(ssa->var_info[p->first_ssa_var + i].type); + } else { + call->stack[i] = IS_UNKNOWN; + } + i++; + } + while (i < p->op_array->last_var) { + call->stack[i] = IS_UNDEF; + i++; + } + while (i < p->op_array->last_var + p->op_array->T) { + call->stack[i] = IS_UNKNOWN; + i++; + } + } else { + ZEND_ASSERT(&call->func->op_array == op_array); + } + frame->call = call->prev; + call->prev = frame; + call->return_value_used = p->return_value_used; + JIT_G(current_frame) = frame = call; + stack = frame->stack; + zend_jit_set_opline(&dasm_state, (p+1)->opline); + } else if (p->op == ZEND_JIT_TRACE_BACK) { + op_array = (zend_op_array*)p->op_array; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + op_array_ssa = &jit_extension->func_info.ssa; + top = frame; + if (frame->prev) { + frame = frame->prev; + stack = frame->stack; + ZEND_ASSERT(&frame->func->op_array == op_array); + } else { + frame = zend_jit_trace_ret_frame(frame, op_array); + frame->call = NULL; + frame->prev = NULL; + frame->func = (const zend_function*)op_array; + frame->return_value_used = -1; + frame->nested = 0; + frame->num_args = -1; + stack = frame->stack; + for (i = 0; i < op_array->last_var + op_array->T; i++) { + /* Initialize abstract stack using SSA */ + if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD) + && has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) { + stack[i] = concrete_type(ssa->var_info[p->first_ssa_var + i].type); + } else { + stack[i] = IS_UNKNOWN; + } + } + } + JIT_G(current_frame) = frame; + if (res_type != IS_UNKNOWN + && (p+1)->op == ZEND_JIT_TRACE_VM) { + const zend_op *opline = (p+1)->opline - 1; + if (opline->result_type != IS_UNUSED) { + SET_RES_STACK_VAR_TYPE(res_type); + } + } + res_type = IS_UNKNOWN; + } else if (p->op == ZEND_JIT_TRACE_END) { + break; + } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + call = top; + call->call = NULL; + call->prev = frame->call; + call->func = p->func; + call->nested = 1; + call->num_args = find_call_num_args(p-1); + frame->call = call; + top = zend_jit_trace_call_frame(top, p->op_array); + if (p->func->type == ZEND_USER_FUNCTION) { + i = 0; + while (i < p->op_array->num_args) { + /* Types of arguments are going to be stored in abstract stack when proseccin SEV onstruction */ + call->stack[i] = IS_UNKNOWN; + i++; + } + while (i < p->op_array->last_var) { + call->stack[i] = IS_UNDEF; + i++; + } + while (i < p->op_array->last_var + p->op_array->T) { + call->stack[i] = IS_UNKNOWN; + i++; + } + } + if (p->fake) { + if (!zend_jit_init_fcall_guard(&dasm_state, NULL, p->func)) { + goto jit_failure; + } + } + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + call = frame->call; + if (call) { + top = call; + frame->call = call->prev; + } + } else { + ZEND_ASSERT(0); + } + } + + ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END); + + t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + if (p->stop == ZEND_JIT_TRACE_STOP_LOOP + || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL + || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { + if (p->stop == ZEND_JIT_TRACE_STOP_LOOP) { + if (!zend_jit_set_valid_ip(&dasm_state, p->opline)) { + goto jit_failure; + } + } + t->link = ZEND_JIT_TRACE_NUM; + zend_jit_jmp(&dasm_state, 0); /* jump back to start of the trace loop */ + } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK) { + if (!zend_jit_set_valid_ip(&dasm_state, p->opline)) { + goto jit_failure; + } + t->link = zend_jit_find_trace(p->opline->handler); + zend_jit_trace_link_to_root(&dasm_state, p->opline->handler); + } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) { + zend_jit_trace_return(&dasm_state); + } else { + // TODO: not implemented ??? + ZEND_ASSERT(0 && p->stop); + } + + if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= ZEND_JIT_TRACE_MAX_EXIT_COUNTERS) { + goto jit_failure; + } + + handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), 1); + +jit_failure: + dasm_free(&dasm_state); + + if (name) { + zend_string_release(name); + } + + /* Clenup used op_arrays */ + while (num_op_arrays > 0) { + op_array = op_arrays[--num_op_arrays]; + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + + memset(&jit_extension->func_info, 0, sizeof(jit_extension->func_info)); + jit_extension->func_info.num_args = -1; + jit_extension->func_info.return_value_used = -1; + } + + zend_arena_release(&CG(arena), checkpoint); + + JIT_G(current_frame) = NULL; + + return handler; +} + +static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) +{ + const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + + opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)); + if (opline) { + return 1; + } + + return 0; +} + +static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num) +{ + const void *handler = NULL; + dasm_State* dasm_state = NULL; + void *checkpoint; + char name[32]; + const zend_op *opline; + + if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) { + return dasm_labels[zend_lbtrace_escape]; + } + + checkpoint = zend_arena_checkpoint(CG(arena));; + + sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num); + + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + dasm_growpc(&dasm_state, 0); + + zend_jit_align_func(&dasm_state); + + // TODO: Generate deoptimization code ??? + + opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)); + if (opline) { + zend_jit_set_ip(&dasm_state, opline); + } + + zend_jit_trace_return(&dasm_state); + + handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 1); + + dasm_free(&dasm_state); + + zend_arena_release(&CG(arena), checkpoint); + + return handler; +} + +static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset) +{ + zend_jit_trace_stop ret; + const void *handler; + zend_jit_trace_info *t; + zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; + + zend_shared_alloc_lock(); + + /* Checks under lock */ + if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) { + ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; + } else if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) { + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; + } else { + SHM_UNPROTECT(); + zend_jit_unprotect(); + + t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + t->id = ZEND_JIT_TRACE_NUM; + t->root = ZEND_JIT_TRACE_NUM; + t->parent = 0; + t->link = 0; + t->exit_count = 0; + t->child_count = 0; + t->stack_map_size = 0; + t->exit_info = exit_info; + t->stack_map = NULL; + + handler = zend_jit_trace(trace_buffer, 0, 0); + + if (handler) { + zend_jit_trace_exit_info *shared_exit_info = NULL; + + t->exit_info = NULL; + if (t->exit_count) { + /* reallocate exit_info into shared memory */ + shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( + sizeof(zend_jit_trace_exit_info) * t->exit_count); + + if (!shared_exit_info) { + if (t->stack_map) { + efree(t->stack_map); + t->stack_map = NULL; + } + ret = ZEND_JIT_TRACE_STOP_NO_SHM; + goto exit; + } + memcpy(shared_exit_info, exit_info, + sizeof(zend_jit_trace_exit_info) * t->exit_count); + t->exit_info = shared_exit_info; + } + + if (t->stack_map_size) { + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); + if (!shared_stack_map) { + ret = ZEND_JIT_TRACE_STOP_NO_SHM; + goto exit; + } + memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; + } + + t->exit_counters = ZEND_JIT_EXIT_COUNTERS; + ZEND_JIT_EXIT_COUNTERS += t->exit_count; + + ((zend_op*)opline)->handler = handler; + + ZEND_JIT_TRACE_NUM++; + ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED; + + ret = ZEND_JIT_TRACE_STOP_COMPILED; + } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || + ZEND_JIT_EXIT_COUNTERS + t->exit_count >= ZEND_JIT_TRACE_MAX_EXIT_COUNTERS) { + if (t->stack_map) { + efree(t->stack_map); + t->stack_map = NULL; + } + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; + } else { + if (t->stack_map) { + efree(t->stack_map); + t->stack_map = NULL; + } + ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; + } + +exit: + zend_jit_protect(); + SHM_PROTECT(); + } + + zend_shared_alloc_unlock(); + + return ret; +} + +static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset) +{ + zend_shared_alloc_lock(); + + if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) { + SHM_UNPROTECT(); + zend_jit_unprotect(); + + ((zend_op*)opline)->handler = + ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; + + ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED; + + zend_jit_protect(); + SHM_PROTECT(); + } + + zend_shared_alloc_unlock(); +} + +static zend_bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset) +{ + const zend_op **cache_opline = JIT_G(bad_root_cache_opline); + uint8_t *cache_count = JIT_G(bad_root_cache_count); + uint8_t *cache_stop = JIT_G(bad_root_cache_stop); + uint32_t cache_slot = JIT_G(bad_root_slot); + uint32_t i; + + for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + if (cache_opline[i] == opline) { + if (cache_count[i] >= ZEND_JIT_TRACE_MAX_ROOT_FAILURES - 1) { + cache_opline[i] = NULL; + return 1; + } else { +#if 0 + if (ZEND_OP_TRACE_INFO(opline, offset)->counter) { + *ZEND_OP_TRACE_INFO(opline, offset)->counter = + random() % ZEND_JIT_TRACE_COUNTER_MAX; + } +#endif + cache_count[i]++; + cache_stop[i] = stop; + return 0; + } + } + } + i = cache_slot; + cache_opline[i] = opline; + cache_count[i] = 1; + cache_stop[i] = stop; + cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS; + JIT_G(bad_root_slot) = cache_slot; + return 0; +} + +#define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \ + description, + +static const char * zend_jit_trace_stop_description[] = { + ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION) +}; + +static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa) +{ + zend_jit_trace_rec *p = trace_buffer; + const zend_op_array *op_array; + const zend_op *opline; + uint32_t level = 1 + trace_buffer[0].level; + int idx, len, i, v, vars_count, call_level; + + ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); + op_array = p->op_array; + p += ZEND_JIT_TRACE_START_REC_SIZE; + idx = 0; + call_level = 0; + + if (tssa && tssa->var_info) { + if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { + vars_count = op_array->last_var; + } else { + vars_count = op_array->last_var + op_array->T; + } + for (i = 0; i < vars_count; i++) { + if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) { + fprintf(stderr, " %*c;", level, ' '); + zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE); + fprintf(stderr, "\n"); + } + } + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP + || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL + || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { + fprintf(stderr, "LOOP:\n"); + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + zend_ssa_phi *p = tssa->blocks[1].phis; + + while (p) { + fprintf(stderr, " ;"); + zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE); + fprintf(stderr, " = Phi("); + zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE); + fprintf(stderr, ", "); + zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE); + fprintf(stderr, ")\n"); + p = p->next; + } + } + } + } + + while (1) { + if (p->op == ZEND_JIT_TRACE_VM) { + uint8_t op1_type, op2_type, op3_type; + + opline = p->opline; + fprintf(stderr, "%04d%*c", + (int)(opline - op_array->opcodes), + level, ' '); + zend_dump_op(op_array, NULL, opline, ZEND_DUMP_NUMERIC_OPLINES|ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL); + + op1_type = p->op1_type; + op2_type = p->op2_type; + op3_type = p->op3_type; + if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) { + fprintf(stderr, " ;"); + if (op1_type != IS_UNKNOWN) { + const char *ref = (op1_type & IS_TRACE_REFERENCE) ? "&" : ""; + if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { + p++; + fprintf(stderr, " op1(%sobject of class %s)", ref, + ZSTR_VAL(p->ce->name)); + } else { + const char *type = (op1_type == 0) ? "undef" : zend_get_type_by_const(op1_type & ~IS_TRACE_REFERENCE); + fprintf(stderr, " op1(%s%s)", ref, type); + } + } + if (op2_type != IS_UNKNOWN) { + const char *ref = (op2_type & IS_TRACE_REFERENCE) ? "&" : ""; + if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { + p++; + fprintf(stderr, " op2(%sobject of class %s)", ref, + ZSTR_VAL(p->ce->name)); + } else { + const char *type = (op2_type == 0) ? "undef" : zend_get_type_by_const(op2_type & ~IS_TRACE_REFERENCE); + fprintf(stderr, " op2(%s%s)", ref, type); + } + } + if (op3_type != IS_UNKNOWN) { + const char *ref = (op3_type & IS_TRACE_REFERENCE) ? "&" : ""; + const char *type = (op3_type == 0) ? "undef" : zend_get_type_by_const(op3_type & ~IS_TRACE_REFERENCE); + fprintf(stderr, " op3(%s%s)", ref, type); + } + } + fprintf(stderr, "\n"); + idx++; + + len = zend_jit_trace_op_len(opline); + while (len > 1) { + opline++; + fprintf(stderr, "%04d%*c;", + (int)(opline - op_array->opcodes), + level, ' '); + zend_dump_op(op_array, NULL, opline, ZEND_DUMP_NUMERIC_OPLINES|ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL); + idx++; + len--; + fprintf(stderr, "\n"); + } + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + op_array = p->op_array; + fprintf(stderr, " %*c>enter %s%s%s\n", + level, ' ', + op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", + op_array->scope ? "::" : "", + op_array->function_name ? + ZSTR_VAL(op_array->function_name) : + ZSTR_VAL(op_array->filename)); + level++; + if (tssa && tssa->var_info) { + call_level++; + v = p->first_ssa_var; + vars_count = op_array->last_var; + for (i = 0; i < vars_count; i++, v++) { + if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { + fprintf(stderr, " %*c;", level, ' '); + zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); + fprintf(stderr, "\n"); + } + } + } + } else if (p->op == ZEND_JIT_TRACE_BACK) { + op_array = p->op_array; + level--; + fprintf(stderr, " %*cscope ? ZSTR_VAL(op_array->scope->name) : "", + op_array->scope ? "::" : "", + op_array->function_name ? + ZSTR_VAL(op_array->function_name) : + ZSTR_VAL(op_array->filename)); + if (tssa && tssa->var_info) { + if (call_level == 0) { + v = p->first_ssa_var; + vars_count = op_array->last_var + op_array->T; + for (i = 0; i < vars_count; i++, v++) { + if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { + fprintf(stderr, " %*c;", level, ' '); + zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); + fprintf(stderr, "\n"); + } + } + } else { + call_level--; + } + } + } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + if (p->func != (zend_function*)&zend_pass_function) { + fprintf(stderr, p->fake ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n", + level, ' ', + p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "", + p->func->common.scope ? "::" : "", + ZSTR_VAL(p->func->common.function_name)); + } else { + fprintf(stderr, " %*c>skip\n", + level, ' '); + } + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + if (p->func != (zend_function*)&zend_pass_function) { + fprintf(stderr, " %*c>call %s%s%s\n", + level, ' ', + p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "", + p->func->common.scope ? "::" : "", + ZSTR_VAL(p->func->common.function_name)); + } else { + fprintf(stderr, " %*c>skip\n", + level, ' '); + } + } else if (p->op == ZEND_JIT_TRACE_END) { + break; + } + p++; + } +} + +static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags) +{ + if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { + return "loop"; + } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) { + return "enter"; + } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) { + return "return"; + } else { + ZEND_ASSERT(0); + return "???"; + } +} + +int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline) +{ + const zend_op *orig_opline; + zend_jit_trace_stop stop; + zend_op_array *op_array; + zend_jit_op_array_trace_extension *jit_extension; + size_t offset; + uint32_t trace_num; + zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; + + ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); + ZEND_ASSERT(opline >= EX(func)->op_array.opcodes && + opline < EX(func)->op_array.opcodes + EX(func)->op_array.last); + +repeat: + trace_num = ZEND_JIT_TRACE_NUM; + orig_opline = opline; + op_array = &EX(func)->op_array; + jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + offset = jit_extension->offset; + + EX(opline) = opline; + + /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */ + if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { + return 0; + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) { + fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n", + trace_num, + zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), + EX(func)->op_array.function_name ? + ZSTR_VAL(EX(func)->op_array.function_name) : "$main", + ZSTR_VAL(EX(func)->op_array.filename), + opline->lineno); + } + + if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) { + stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; + goto abort; + } + + stop = zend_jit_trace_execute(execute_data, opline, trace_buffer, + ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK); + + if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) { + /* op_array may be already deallocated */ + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + fprintf(stderr, "---- TRACE %d abort (%s)\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } + goto blacklist; + } + + if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { + zend_jit_dump_trace(trace_buffer, NULL); + } + + if (ZEND_JIT_TRACE_STOP_OK(stop)) { + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) { + if (stop == ZEND_JIT_TRACE_STOP_LINK) { + uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);; + fprintf(stderr, "---- TRACE %d stop (link to %d)\n", + trace_num, + link_to); + } else { + fprintf(stderr, "---- TRACE %d stop (%s)\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } + } + stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset); + if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) { + fprintf(stderr, "---- TRACE %d %s\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } + } else { + goto abort; + } + } else { +abort: + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + fprintf(stderr, "---- TRACE %d abort (%s)\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } +blacklist: + if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) + || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) { + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { + fprintf(stderr, "---- TRACE %d blacklisted\n", + trace_num); + } + zend_jit_blacklist_root_trace(orig_opline, offset); + } + if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) { + execute_data = EG(current_execute_data); + opline = EX(opline); + goto repeat; + } + } + + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { + fprintf(stderr, "\n"); + } + + return (stop == ZEND_JIT_TRACE_STOP_HALT) ? -1 : 0; +} + +static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) +{ + const void *handler; + + zend_shared_alloc_lock(); + + if (!((uintptr_t)zend_jit_traces[trace_num].exit_info[exit_num].opline & ZEND_JIT_EXIT_BLACKLISTED)) { + SHM_UNPROTECT(); + zend_jit_unprotect(); + + handler = zend_jit_trace_exit_to_vm(trace_num, exit_num); + + if (handler) { + zend_jit_link_side_trace( + zend_jit_traces[trace_num].code_start, + zend_jit_traces[trace_num].code_size, + exit_num, + handler); + } + + zend_jit_traces[trace_num].exit_info[exit_num].opline = (const zend_op*) + ((uintptr_t)zend_jit_traces[trace_num].exit_info[exit_num].opline | ZEND_JIT_EXIT_BLACKLISTED); + + zend_jit_protect(); + SHM_PROTECT(); + } + + zend_shared_alloc_unlock(); +} + +static zend_bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num) +{ + uint8_t *counter = JIT_G(exit_counters) + + zend_jit_traces[trace_num].exit_counters + exit_num; + + if (*counter + 1 >= ZEND_JIT_TRACE_HOT_SIDE_COUNT + ZEND_JIT_TRACE_MAX_SIDE_FAILURES) { + return 1; + } + (*counter)++; + return 0; +} + +static zend_bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) +{ + uint8_t *counter = JIT_G(exit_counters) + + zend_jit_traces[trace_num].exit_counters + exit_num; + + if (*counter + 1 >= ZEND_JIT_TRACE_HOT_SIDE_COUNT) { + return 1; + } + (*counter)++; + return 0; +} + +static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num) +{ + zend_jit_trace_stop ret; + const void *handler; + zend_jit_trace_info *t; + zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; + + zend_shared_alloc_lock(); + + /* Checks under lock */ + if (((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline & ZEND_JIT_EXIT_JITED)) { + ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; + } else if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) { + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; + } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= ZEND_JIT_TRACE_MAX_SIDE_TRACES) { + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN; + } else { + SHM_UNPROTECT(); + zend_jit_unprotect(); + + t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + t->id = ZEND_JIT_TRACE_NUM; + t->root = zend_jit_traces[parent_num].root; + t->parent = parent_num; + t->link = 0; + t->exit_count = 0; + t->child_count = 0; + t->stack_map_size = 0; + t->exit_info = exit_info; + t->stack_map = NULL; + + handler = zend_jit_trace(trace_buffer, parent_num, exit_num); + + if (handler) { + zend_jit_trace_exit_info *shared_exit_info = NULL; + + t->exit_info = NULL; + if (t->exit_count) { + /* reallocate exit_info into shared memory */ + shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( + sizeof(zend_jit_trace_exit_info) * t->exit_count); + + if (!shared_exit_info) { + if (t->stack_map) { + efree(t->stack_map); + t->stack_map = NULL; + } + ret = ZEND_JIT_TRACE_STOP_NO_SHM; + goto exit; + } + memcpy(shared_exit_info, exit_info, + sizeof(zend_jit_trace_exit_info) * t->exit_count); + t->exit_info = shared_exit_info; + } + + if (t->stack_map_size) { + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); + if (!shared_stack_map) { + efree(t->stack_map); + ret = ZEND_JIT_TRACE_STOP_NO_SHM; + goto exit; + } + memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; + } + + zend_jit_link_side_trace( + zend_jit_traces[parent_num].code_start, + zend_jit_traces[parent_num].code_size, + exit_num, + handler); + + t->exit_counters = ZEND_JIT_EXIT_COUNTERS; + ZEND_JIT_EXIT_COUNTERS += t->exit_count; + + zend_jit_traces[zend_jit_traces[parent_num].root].child_count++; + ZEND_JIT_TRACE_NUM++; + zend_jit_traces[parent_num].exit_info[exit_num].opline = (const zend_op*) + ((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline | ZEND_JIT_EXIT_JITED); + + ret = ZEND_JIT_TRACE_STOP_COMPILED; + } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || + ZEND_JIT_EXIT_COUNTERS + t->exit_count >= ZEND_JIT_TRACE_MAX_EXIT_COUNTERS) { + if (t->stack_map) { + efree(t->stack_map); + t->stack_map = NULL; + } + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; + } else { + if (t->stack_map) { + efree(t->stack_map); + t->stack_map = NULL; + } + ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; + } + +exit: + zend_jit_protect(); + SHM_PROTECT(); + } + + zend_shared_alloc_unlock(); + + return ret; +} + +int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num) +{ + zend_jit_trace_stop stop; + uint32_t trace_num; + zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; + + trace_num = ZEND_JIT_TRACE_NUM; + + /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ + if ((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { + return 0; + } + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) { + fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s() %s:%d\n", + trace_num, parent_num, exit_num, + EX(func)->op_array.function_name ? + ZSTR_VAL(EX(func)->op_array.function_name) : "$main", + ZSTR_VAL(EX(func)->op_array.filename), + EX(opline)->lineno); + } + + if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) { + stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; + goto abort; + } + + if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= ZEND_JIT_TRACE_MAX_SIDE_TRACES) { + stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN; + goto abort; + } + + stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE); + + if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) { + /* op_array may be already deallocated */ + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + fprintf(stderr, "---- TRACE %d abort (%s)\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } + goto blacklist; + } + + if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { + zend_jit_dump_trace(trace_buffer, NULL); + } + + if (ZEND_JIT_TRACE_STOP_OK(stop)) { + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) { + if (stop == ZEND_JIT_TRACE_STOP_LINK) { + uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);; + fprintf(stderr, "---- TRACE %d stop (link to %d)\n", + trace_num, + link_to); + } else { + fprintf(stderr, "---- TRACE %d stop (%s)\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } + } + if (EXPECTED(stop != ZEND_JIT_TRACE_STOP_LOOP)) { + stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num); + } else { + const zend_op_array *op_array = trace_buffer[0].op_array; + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + const zend_op *opline = trace_buffer[1].opline; + + stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset); + } + if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) { + fprintf(stderr, "---- TRACE %d %s\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } + } else { + goto abort; + } + } else { +abort: + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + fprintf(stderr, "---- TRACE %d abort (%s)\n", + trace_num, + zend_jit_trace_stop_description[stop]); + } +blacklist: + if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) + || zend_jit_trace_exit_is_bad(parent_num, exit_num)) { + zend_jit_blacklist_trace_exit(parent_num, exit_num); + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { + fprintf(stderr, "---- EXIT %d/%d blacklisted\n", + parent_num, exit_num); + } + } + } + + if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { + fprintf(stderr, "\n"); + } + + return (stop == ZEND_JIT_TRACE_STOP_HALT) ? -1 : 0; +} + +int ZEND_FASTCALL zend_jit_trace_exit(uint32_t trace_num, uint32_t exit_num) +{ + zend_execute_data *execute_data = EG(current_execute_data); + const zend_op *opline; + zend_jit_trace_info *t = &zend_jit_traces[trace_num]; + + // TODO: Deoptimizatoion of VM stack state ??? + + /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ + // TODO: We may remoive this, becaus of the same check in zend_jit_trace_hot_side() ??? + opline = t->exit_info[exit_num].opline; + if ((uintptr_t)opline & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { + opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)); + if (opline) { + /* Set VM opline to continue interpretation */ + EX(opline) = opline; + } + return 0; + } + + if (opline) { + /* Set VM opline to continue interpretation */ + EX(opline) = opline; + } + + ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); + ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes && + EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last); + + if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT) { + fprintf(stderr, " TRACE %d exit %d %s() %s:%d\n", + trace_num, + exit_num, + EX(func)->op_array.function_name ? + ZSTR_VAL(EX(func)->op_array.function_name) : "$main", + ZSTR_VAL(EX(func)->op_array.filename), + EX(opline)->lineno); + } + + if (zend_jit_trace_exit_is_hot(trace_num, exit_num)) { + return zend_jit_trace_hot_side(execute_data, trace_num, exit_num); + } + + return 0; +} + +static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline) +{ + switch (opline->opcode) { + case ZEND_CATCH: + case ZEND_FAST_CALL: + case ZEND_FAST_RET: + case ZEND_GENERATOR_CREATE: + case ZEND_GENERATOR_RETURN: + case ZEND_EXIT: + case ZEND_YIELD: + case ZEND_YIELD_FROM: + case ZEND_INCLUDE_OR_EVAL: + return ZEND_JIT_TRACE_UNSUPPORTED; + default: + return ZEND_JIT_TRACE_SUPPORTED; + } +} + +static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) +{ + zend_op *opline; + zend_jit_op_array_trace_extension *jit_extension; + zend_cfg cfg; + uint32_t i; + + ZEND_ASSERT(zend_jit_func_counter_handler != NULL); + ZEND_ASSERT(zend_jit_ret_counter_handler != NULL); + ZEND_ASSERT(zend_jit_loop_counter_handler != NULL); + ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op)); + + if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { + return FAILURE; + } + + jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info)); + memset(&jit_extension->func_info, 0, sizeof(jit_extension->func_info)); + jit_extension->func_info.num_args = -1; + jit_extension->func_info.return_value_used = -1; + jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes; + for (i = 0; i < op_array->last; i++) { + jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler; + jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]); + jit_extension->trace_info[i].counter = NULL; + jit_extension->trace_info[i].trace_flags = + zend_jit_trace_supported(&op_array->opcodes[i]); + } + ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); + + opline = op_array->opcodes; + if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + } + } + + if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { + /* function entry */ + opline->handler = (const void*)zend_jit_func_counter_handler; + ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = + &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; + ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + ZEND_JIT_TRACE_START_ENTER; + } + + for (i = 0; i < cfg.blocks_count; i++) { + if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) { + if (cfg.blocks[i].flags & ZEND_BB_ENTRY) { + /* continuation after return from function call */ + opline = op_array->opcodes + cfg.blocks[i].start; + if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { + opline->handler = (const void*)zend_jit_ret_counter_handler; + if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { + ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = + &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; + ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + } + ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + ZEND_JIT_TRACE_START_RETURN; + } + } + if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) { + /* loop header */ + opline = op_array->opcodes + cfg.blocks[i].start; + if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { + opline->handler = (const void*)zend_jit_loop_counter_handler; + if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { + ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = + &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; + ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + } + ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + ZEND_JIT_TRACE_START_LOOP; + } + } + } + } + + return SUCCESS; +} + +static void zend_jit_trace_init_caches(void) +{ + memset(JIT_G(bad_root_cache_opline), 0, sizeof(JIT_G(bad_root_cache_opline))); + memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count))); + memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count))); + JIT_G(bad_root_slot) = 0; + + memset(JIT_G(exit_counters), 0, sizeof(JIT_G(exit_counters))); +} + +static void zend_jit_trace_reset_caches(void) +{ +} diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 9862fd1b42..9f23beda17 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -26,6 +26,7 @@ #include #include "Optimizer/zend_func_info.h" +#include "Optimizer/zend_call_graph.h" #include "zend_jit.h" #include "zend_jit_internal.h" @@ -98,6 +99,15 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t ca #endif } +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC) +{ + if (call_info & ZEND_CALL_TOP) { + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info); + } else { + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info); + } +} + void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D) { zend_op_array *op_array = &EX(func)->op_array; @@ -182,8 +192,8 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLE ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS) { - zend_jit_op_array_extension *jit_extension = - (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); + zend_jit_op_array_hot_extension *jit_extension = + (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif @@ -191,6 +201,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_H *(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST; if (UNEXPECTED(*(jit_extension->counter) <= 0)) { + *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; zend_jit_hot_func(execute_data, opline); ZEND_OPCODE_RETURN(); } else { @@ -201,8 +212,8 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_H ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS) { - zend_jit_op_array_extension *jit_extension = - (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); + zend_jit_op_array_hot_extension *jit_extension = + (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif @@ -210,6 +221,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H *(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST; if (UNEXPECTED(*(jit_extension->counter) <= 0)) { + *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; zend_jit_hot_func(execute_data, opline); ZEND_OPCODE_RETURN(); } else { @@ -266,3 +278,607 @@ int ZEND_FASTCALL zend_jit_check_constant(const zval *key) { return _zend_quick_get_constant(key, 0, 1); } + +static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC) +{ + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); + size_t offset = jit_extension->offset; +#ifndef HAVE_GCC_GLOBAL_REGS + const zend_op *opline = EX(opline); +#endif + + *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= ZEND_JIT_TRACE_FUNC_COST; + + if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) { + *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_TRACE_COUNTER_INIT; + if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) { +#ifndef HAVE_GCC_GLOBAL_REGS + return -1; +#endif + } +#ifdef HAVE_GCC_GLOBAL_REGS + execute_data = EG(current_execute_data); + opline = EX(opline); + return; +#else + return 1; +#endif + } else { + zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; + ZEND_OPCODE_TAIL_CALL(handler); + } +} + +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS) +{ + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_FUNC_COST); +} + +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS) +{ + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_RET_COST); +} + +ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS) +{ + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_LOOP_COST); +} + +#define TRACE_RECORD(_op, _ptr) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].ptr = _ptr; \ + idx++; \ + if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \ + stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \ + break; \ + } + +#define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].op1_type = _op1_type; \ + trace_buffer[idx].op2_type = _op2_type; \ + trace_buffer[idx].op3_type = _op3_type; \ + trace_buffer[idx].ptr = _ptr; \ + idx++; \ + if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \ + stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \ + break; \ + } + +#define TRACE_RECORD_ENTER(_op, _return_value_used, _ptr) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].return_value_used = _return_value_used; \ + trace_buffer[idx].ptr = _ptr; \ + idx++; \ + if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \ + stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \ + break; \ + } + +#define TRACE_RECORD_INIT(_op, _fake, _ptr) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].fake = _fake; \ + trace_buffer[idx].ptr = _ptr; \ + idx++; \ + if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \ + stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \ + break; \ + } + +#define TRACE_RECORD_BACK(_op, _recursive, _ptr) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].recursive = _recursive; \ + trace_buffer[idx].ptr = _ptr; \ + idx++; \ + if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \ + stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \ + break; \ + } + +#define TRACE_RECORD_TYPE(_op, _ptr) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].ptr = _ptr; \ + idx++; \ + if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \ + stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \ + break; \ + } + +#define TRACE_START(_op, _start, _ptr) \ + trace_buffer[0].op = _op; \ + trace_buffer[0].start = _start; \ + trace_buffer[0].level = 0; \ + trace_buffer[0].ptr = _ptr; \ + idx = ZEND_JIT_TRACE_START_REC_SIZE; + +#define TRACE_END(_op, _stop, _ptr) \ + trace_buffer[idx].op = _op; \ + trace_buffer[idx].start = trace_buffer[idx].start; \ + trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \ + trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \ + trace_buffer[idx].ptr = _ptr; + +#ifndef ZEND_JIT_RECORD_RECURSIVE_RETURN +# define ZEND_JIT_RECORD_RECURSIVE_RETURN 1 +#endif + +static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level) +{ + int i; + int count = 0; + + for (i = ret_level; i < level; i++) { + count += (unrolled_calls[i] == op_array); + } + return count; +} + +static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level) +{ + int i; + int count = 0; + + for (i = 0; i < ret_level; i++) { + count += (unrolled_calls[i] == op_array); + } + return count; +} + +static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level) +{ + while (ex != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) { + if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) { + return 1; + } + ex = ex->prev_execute_data; + ret_level++; + } + return 0; +} + +static int zend_jit_trace_bad_inner_loop(const zend_op *opline) +{ + const zend_op **cache_opline = JIT_G(bad_root_cache_opline); + uint8_t *cache_count = JIT_G(bad_root_cache_count); + uint8_t *cache_stop = JIT_G(bad_root_cache_stop); + uint32_t i; + + for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + if (cache_opline[i] == opline) { + if ((cache_stop[i] == ZEND_JIT_TRACE_STOP_INNER_LOOP + || cache_stop[i] == ZEND_JIT_TRACE_STOP_LOOP_EXIT) + && cache_count[i] > ZEND_JIT_TRACE_MAX_ROOT_FAILURES / 2) { + return 1; + } + break; + } + } + return 0; +} + +static int zend_jit_trace_bad_compiled_loop(const zend_op *opline) +{ + const zend_op **cache_opline = JIT_G(bad_root_cache_opline); + uint8_t *cache_count = JIT_G(bad_root_cache_count); + uint8_t *cache_stop = JIT_G(bad_root_cache_stop); + uint32_t i; + + for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + if (cache_opline[i] == opline) { + if (cache_stop[i] == ZEND_JIT_TRACE_STOP_COMPILED_LOOP + && cache_count[i] >= ZEND_JIT_TRACE_MAX_ROOT_FAILURES - 1) { + return 1; + } + break; + } + } + return 0; +} + +static int zend_jit_trace_bad_loop_exit(const zend_op *opline) +{ + const zend_op **cache_opline = JIT_G(bad_root_cache_opline); + uint8_t *cache_count = JIT_G(bad_root_cache_count); + uint8_t *cache_stop = JIT_G(bad_root_cache_stop); + uint32_t i; + + for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + if (cache_opline[i] == opline) { + if (cache_stop[i] == ZEND_JIT_TRACE_STOP_LOOP_EXIT + && cache_count[i] >= ZEND_JIT_TRACE_MAX_ROOT_FAILURES - 1) { + return 1; + } + break; + } + } + return 0; +} + +static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx) +{ + zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR; + + do { + if (call->prev_execute_data) { + idx = zend_jit_trace_record_fake_init_call(call->prev_execute_data, trace_buffer, idx); + } + TRACE_RECORD_INIT(ZEND_JIT_TRACE_INIT_CALL, 1, call->func); + } while (0); + return idx; +} + +/* + * Trace Linking Rules + * =================== + * + * flags + * +----------+----------+----------++----------+----------+----------+ + * | || JIT | + * +----------+----------+----------++----------+----------+----------+ + * start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN | + * +========+==========+==========+==========++==========+==========+==========+ + * | LOOP | loop | | loop-ret || COMPILED | LINK | LINK | + * +--------+----------+----------+----------++----------+----------+----------+ + * | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK | + * +--------+----------+----------+----------++----------+----------+----------+ + * | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK | + * +--------+----------+----------+----------++----------+----------+----------+ + * | SIDE | unroll | | return || LINK | LINK | LINK | + * +--------+----------+----------+----------++----------+----------+----------+ + * + * loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP + * INNER_LOOP: abort recording and start new one (wit for loop) + * COMPILED: abort recording (wait while side exit creates outer loop) + * unroll: continue recording while unroll limit reached + * rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue + * loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop) + * return: RETURN if level == 0 + * rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue + * + */ + +zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start) +{ +#ifdef HAVE_GCC_GLOBAL_REGS + zend_execute_data *save_execute_data = execute_data; + const zend_op *save_opline = opline; +#endif + const zend_op *orig_opline; + zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR; + int level = 0; + int ret_level = 0; + zend_vm_opcode_handler_t handler; + zend_jit_op_array_trace_extension *jit_extension; + size_t offset; + int idx, count; + uint8_t trace_flags, op1_type, op2_type, op3_type; + int backtrack_recursion = -1; + int backtrack_ret_recursion = -1; + int backtrack_ret_recursion_level = 0; + int loop_unroll_limit = 0; + const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH]; +#if ZEND_JIT_DETECT_UNROLLED_LOOPS + uint32_t unrolled_loops[ZEND_JIT_TRACE_MAX_UNROLL_LOOPS]; +#endif + zend_bool is_toplevel; +#ifdef HAVE_GCC_GLOBAL_REGS + zend_execute_data *prev_execute_data = ex; + + execute_data = ex; + opline = EX(opline) = op; +#else + int rc; + zend_execute_data *execute_data = ex; + const zend_op *opline = EX(opline); +#endif + zend_execute_data *prev_call = EX(call); + + orig_opline = opline; + + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); + offset = jit_extension->offset; + + TRACE_START(ZEND_JIT_TRACE_START, start, &EX(func)->op_array); + ((zend_jit_trace_start_rec*)trace_buffer)->opline = opline; + is_toplevel = EX(func)->op_array.function_name == NULL; + + + if (prev_call) { + idx = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx); + } + + while (1) { + if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) { + /* Abort trace because of exception */ + stop = ZEND_JIT_TRACE_STOP_EXCEPTION; + break; + } + + op1_type = op2_type = op3_type = IS_UNKNOWN; + if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) + && (opline->opcode != ZEND_ROPE_ADD && opline->opcode != ZEND_ROPE_END)) { + zval *zv = EX_VAR(opline->op1.var); + op1_type = Z_TYPE_P(zv); + if (op1_type == IS_INDIRECT) { + zv = Z_INDIRECT_P(zv); + op1_type = Z_TYPE_P(zv); + } + if (op1_type == IS_REFERENCE) { + op1_type = Z_TYPE_P(Z_REFVAL_P(zv)) | IS_TRACE_REFERENCE; + } + } + if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { + zval *zv = EX_VAR(opline->op2.var); + op2_type = Z_TYPE_P(zv); + if (op2_type == IS_INDIRECT) { + zv = Z_INDIRECT_P(zv); + op2_type = Z_TYPE_P(zv); + } + if (op2_type == IS_REFERENCE) { + op2_type = Z_TYPE_P(Z_REFVAL_P(zv)) | IS_TRACE_REFERENCE; + } + } + if (opline->opcode == ZEND_ASSIGN_DIM || + opline->opcode == ZEND_ASSIGN_OBJ || + opline->opcode == ZEND_ASSIGN_STATIC_PROP || + opline->opcode == ZEND_ASSIGN_DIM_OP || + opline->opcode == ZEND_ASSIGN_OBJ_OP || + opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP || + opline->opcode == ZEND_ASSIGN_OBJ_REF || + opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) { + if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { + zval *zv = EX_VAR((opline+1)->op1.var); + op3_type = Z_TYPE_P(zv); + if (op3_type == IS_INDIRECT) { + zv = Z_INDIRECT_P(zv); + op3_type = Z_TYPE_P(zv); + } + if (op3_type == IS_REFERENCE) { + op3_type = Z_TYPE_P(Z_REFVAL_P(zv)) | IS_TRACE_REFERENCE; + } + } + } + + TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type); + + if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { + zval *var = EX_VAR(opline->op1.var); + uint8_t type = Z_TYPE_P(var); + + if (type == IS_OBJECT) { + TRACE_RECORD_TYPE(ZEND_JIT_TRACE_OP1_TYPE, Z_OBJCE_P(var)); + } + } + + if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { + zval *var = EX_VAR(opline->op2.var); + uint8_t type = Z_TYPE_P(var); + + if (type == IS_OBJECT) { + TRACE_RECORD_TYPE(ZEND_JIT_TRACE_OP2_TYPE, Z_OBJCE_P(var)); + } + } + + switch (opline->opcode) { + case ZEND_DO_FCALL: + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { + TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, EX(call)->func); + } + break; + default: + break; + } + + handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; +#ifdef HAVE_GCC_GLOBAL_REGS + handler(); + if (UNEXPECTED(opline == zend_jit_halt_op)) { + stop = ZEND_JIT_TRACE_STOP_HALT; + break; + } + if (UNEXPECTED(execute_data != prev_execute_data)) { + if (execute_data->prev_execute_data == prev_execute_data) { +#else + rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); + if (rc != 0) { + if (rc < 0) { + stop = ZEND_JIT_TRACE_STOP_HALT; + break; + } + execute_data = EG(current_execute_data); + opline = EX(opline); + if (rc == 1) { +#endif + /* Enter into function */ + if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) { + stop = ZEND_JIT_TRACE_STOP_TOO_DEEP; + break; + } + + if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + /* TODO: Can we continue recording ??? */ + stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; + break; + } + + TRACE_RECORD_ENTER(ZEND_JIT_TRACE_ENTER, EX(return_value) != NULL, &EX(func)->op_array); + + count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level); + + if (opline == orig_opline) { + if (count + 1 >= ZEND_JIT_TRACE_MAX_RECURSION) { + stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL; + break; + } + backtrack_recursion = idx; + } else if (count >= ZEND_JIT_TRACE_MAX_RECURSION) { + stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION; + break; + } + + unrolled_calls[ret_level + level] = &EX(func)->op_array; + level++; + } else { + /* Return from function */ + if (level == 0) { + if (is_toplevel) { + stop = ZEND_JIT_TRACE_STOP_TOPLEVEL; + break; +#if ZEND_JIT_RECORD_RECURSIVE_RETURN + } else if (start == ZEND_JIT_TRACE_START_RETURN + && execute_data->prev_execute_data + && execute_data->prev_execute_data->func + && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION + && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) { + if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) { + stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET; + break; + } + TRACE_RECORD_BACK(ZEND_JIT_TRACE_BACK, 1, &EX(func)->op_array); + count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level); + if (opline == orig_opline) { + if (count + 1 >= ZEND_JIT_TRACE_MAX_RECURSION) { + stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET; + break; + } + backtrack_ret_recursion = idx; + backtrack_ret_recursion_level = ret_level; + } else if (count >= ZEND_JIT_TRACE_MAX_RECURSION) { + stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION; + break; + } + + unrolled_calls[ret_level] = &EX(func)->op_array; + ret_level++; + is_toplevel = EX(func)->op_array.function_name == NULL; +#endif + } else if (start & ZEND_JIT_TRACE_START_LOOP + && !zend_jit_trace_bad_loop_exit(orig_opline)) { + /* Fail to try close the loop. + If this doesn't work terminate it. */ + stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT; + break; + } else { + stop = ZEND_JIT_TRACE_STOP_RETURN; + break; + } + } else { + level--; + TRACE_RECORD_BACK(ZEND_JIT_TRACE_BACK, 0, &EX(func)->op_array); + } + } +#ifdef HAVE_GCC_GLOBAL_REGS + prev_execute_data = execute_data; +#endif + jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); + if (UNEXPECTED(!jit_extension)) { + stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; + break; + } + offset = jit_extension->offset; + } + if (EX(call) != prev_call) { + if (trace_buffer[idx-1].op != ZEND_JIT_TRACE_BACK + && EX(call) + && EX(call)->prev_execute_data == prev_call) { + if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { + /* TODO: Can we continue recording ??? */ + stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; + break; + } + TRACE_RECORD_INIT(ZEND_JIT_TRACE_INIT_CALL, 0, EX(call)->func); + } + prev_call = EX(call); + } + +#ifndef HAVE_GCC_GLOBAL_REGS + opline = EX(opline); +#endif + + trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags; + if (trace_flags) { + if (trace_flags & ZEND_JIT_TRACE_JITED) { + if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { + if ((start & ZEND_JIT_TRACE_START_LOOP) != 0 + && level + ret_level == 0 + && !zend_jit_trace_bad_compiled_loop(orig_opline)) { + /* Fail to try close outer loop throgh side exit. + If this doesn't work just link. */ + stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP; + break; + } else { + stop = ZEND_JIT_TRACE_STOP_LINK; + break; + } + } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) { + if (start != ZEND_JIT_TRACE_START_RETURN) { + // TODO: We may try to inline function ??? + stop = ZEND_JIT_TRACE_STOP_LINK; + break; + } + } else { + stop = ZEND_JIT_TRACE_STOP_LINK; + break; + } + } else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) { + stop = ZEND_JIT_TRACE_STOP_BLACK_LIST; + break; + } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { + if (start != ZEND_JIT_TRACE_START_SIDE) { + if (opline == orig_opline && level + ret_level == 0) { + stop = ZEND_JIT_TRACE_STOP_LOOP; + break; + } + /* Fail to try creating a trace for inner loop first. + If this doesn't work try unroling loop. */ + if (!zend_jit_trace_bad_inner_loop(opline)) { + stop = ZEND_JIT_TRACE_STOP_INNER_LOOP; + break; + } + } + if (loop_unroll_limit < ZEND_JIT_TRACE_MAX_UNROLL_LOOPS) { + loop_unroll_limit++; + } else { + stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL; + break; + } + } else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) { + TRACE_RECORD(ZEND_JIT_TRACE_VM, opline); + stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED; + break; + } + } + } + + if (!ZEND_JIT_TRACE_STOP_OK(stop)) { + if (backtrack_recursion > 0) { + idx = backtrack_recursion; + stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL; + } else if (backtrack_ret_recursion > 0) { + idx = backtrack_ret_recursion; + ret_level = backtrack_ret_recursion_level; + stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET; + } + } + + TRACE_END(ZEND_JIT_TRACE_END, stop, opline); + +#ifdef HAVE_GCC_GLOBAL_REGS + if (stop != ZEND_JIT_TRACE_STOP_HALT) { + EX(opline) = opline; + } +#endif + +#ifdef HAVE_GCC_GLOBAL_REGS + execute_data = save_execute_data; + opline = save_opline; +#endif + + return stop; +} diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 1ba4a61b6e..de5138d00a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -91,6 +91,8 @@ |.define HYBRID_SPAD, 16 // padding for stack alignment +#define DASM_ALIGNMENT 16 + /* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to * guarantee proper alignment of 128-bit SSE data allocated on stack. * With broken alignment any execution of SSE code, including calls to @@ -100,6 +102,10 @@ #include "Zend/zend_cpuinfo.h" #include "jit/zend_jit_x86.h" +#ifdef HAVE_VALGRIND +# include +#endif + /* The generated code may contain tautological comparisons, ignore them. */ #if defined(__clang__) # pragma clang diagnostic push @@ -903,10 +909,10 @@ static void* dasm_labels[zend_lb_MAX]; || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { -|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != MAY_BE_DOUBLE) { +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE || } -|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1<func + | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] + | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] + | sub word [r2], cost + | jle >1 + | jmp aword [IP + r1] + |1: + | mov word [r2], ZEND_JIT_TRACE_COUNTER_INIT + | mov FCARG1a, FP + | GET_IP FCARG2a + | EXT_CALL zend_jit_trace_hot_root, r0 + | test eax, eax // TODO : remove this check at least for HYBRID VM ??? + | jl >1 + | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 + | LOAD_OPLINE + | JMP_IP + |1: + | EXT_JMP zend_jit_halt_op->handler, r0 + return 1; +} + +static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) +{ + |->hybrid_func_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_FUNC_COST); +} + +static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) +{ + |->hybrid_ret_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_RET_COST); +} + +static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) +{ + |->hybrid_loop_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_LOOP_COST); +} + +static int zend_jit_trace_halt_stub(dasm_State **Dst) +{ + |->trace_halt: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | EXT_JMP zend_jit_halt_op->handler, r0 + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | ret // PC must be zero + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, -1 // ZEND_VM_RETURN + | ret + } + return 1; +} + +static int zend_jit_trace_exit_stub(dasm_State **Dst) +{ + |->trace_exit: + | + | // TODO: Save CPU registers ??? + | + | // trace_num = EG(reserved)[zend_func_info_rid] + | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, reserved[zend_func_info_rid], r0 + | // exit_num = POP + | pop FCARG2a + | // EX(opline) = opline + | SAVE_OPLINE + | // zend_jit_trace_exit(trace_num, exit_num) + | EXT_CALL zend_jit_trace_exit, r0 + | // execute_data = EG(current_excute_data) + | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 + | test eax, eax + | jl ->trace_halt + | // opline = EX(opline) + | LOAD_OPLINE + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#if 1 + //TODO: this doesn't work for exit from first instruction ??? + | add r4, HYBRID_SPAD + | JMP_IP +#else + | mov r0, EX->func + | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] + | jmp aword [IP + r1] +#endif + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +static int zend_jit_trace_escape_stub(dasm_State **Dst) +{ + |->trace_escape: + | + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | JMP_IP + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +/* Keep 32 exit points in a single code block */ +#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes +#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points + +static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) +{ + uint32_t i; + + for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { + | push byte i + | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 + } + | push byte i + |// 1: + | .byte 0x81, 0x04, 0x24; .dword n // add aword [r4], n + | jmp ->trace_exit + + return 1; +} + #ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call_stub(dasm_State **Dst) { @@ -2280,6 +2440,9 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(undefined_function), JIT_STUB(negative_shift), JIT_STUB(mod_by_zero), + JIT_STUB(trace_halt), + JIT_STUB(trace_exit), + JIT_STUB(trace_escape), JIT_STUB(double_one), #ifdef CONTEXT_THREADED_JIT JIT_STUB(context_threaded_call), @@ -2415,6 +2578,12 @@ static int zend_jit_setup(void) return SUCCESS; } +static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) +{ + | int3 + return 1; +} + static int zend_jit_align_func(dasm_State **Dst) { reuse_ip = 0; @@ -2533,6 +2702,379 @@ static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op return zend_jit_check_exception(Dst); } +static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num) +{ + current_trace_num = trace_num; + + | //EG(reserved)[zend_func_info_rid] = trace_num; + | MEM_OP2_1_ZTS mov, aword, executor_globals, reserved[zend_func_info_rid], trace_num, r0 + + return 1; +} + +static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) +{ + uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, NULL, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | CMP_IP opline + | jne &exit_addr + + return 1; +} + +/* This taken from LuaJIT. Thanks to Mike Pall. */ +static uint32_t _asm_x86_inslen(const uint8_t* p) +{ + static const uint8_t map_op1[256] = { + 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, + 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, + 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, + 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, +#if defined(__x86_64__) || defined(_M_X64) + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, +#else + 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, +#endif + 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, + 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, + 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, + 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, + 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, +#if defined(__x86_64__) || defined(_M_X64) + 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, +#else + 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, +#endif + 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, + 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, + 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, + 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, + 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 + }; + static const uint8_t map_op2[256] = { + 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, + 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, + 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 + }; + uint32_t result = 0; + uint32_t prefixes = 0; + uint32_t x = map_op1[*p]; + + for (;;) { + switch (x >> 4) { + case 0: + return result + x + (prefixes & 4); + case 1: + prefixes |= x; + x = map_op1[*++p]; + result++; + break; + case 2: + x = map_op2[*++p]; + break; + case 3: + p++; + goto mrm; + case 4: + result -= (prefixes & 2); + /* fallthrough */ + case 5: + return result + (x & 15); + case 6: /* Group 3. */ + if (p[1] & 0x38) { + x = 2; + } else if ((prefixes & 2) && (x == 0x66)) { + x = 4; + } + goto mrm; + case 7: /* VEX c4/c5. */ +#if !defined(__x86_64__) && !defined(_M_X64) + if (p[1] < 0xc0) { + x = 2; + goto mrm; + } +#endif + if (x == 0x70) { + x = *++p & 0x1f; + result++; + if (x >= 2) { + p += 2; + result += 2; + goto mrm; + } + } + p++; + result++; + x = map_op2[*++p]; + break; + case 8: + result -= (prefixes & 2); + /* fallthrough */ + case 9: +mrm: + /* ModR/M and possibly SIB. */ + result += (x & 15); + x = *++p; + switch (x >> 6) { + case 0: + if ((x & 7) == 5) { + return result + 4; + } + break; + case 1: + result++; + break; + case 2: + result += 4; + break; + case 3: + return result; + } + if ((x & 7) == 4) { + result++; + if (x < 0x40 && (p[1] & 7) == 5) { + result += 4; + } + } + return result; + } + } +} + +static int zend_jit_patch(const void *code, size_t size, const void *from_addr, const void *to_addr) +{ + int ret = 0; + uint8_t *p = (uint8_t*)code; + uint8_t *end = p + size - 5; + + while (p < end) { + if ((*(uint16_t*)p & 0xf0ff) == 0x800f && p + *(int32_t*)(p+2) == (uint8_t*)from_addr - 6) { + *(int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); + ret++; + } else if (*p == 0xe9 && p + *(int32_t*)(p+1) == (uint8_t*)from_addr - 5) { + *(int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); + ret++; + } + p += _asm_x86_inslen(p); + } +#ifdef HAVE_VALGRIND + VALGRIND_DISCARD_TRANSLATIONS(code, size); +#endif + return ret; +} + +static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t exit_num, const void *addr) +{ + return zend_jit_patch(code, size, zend_jit_trace_get_exit_addr(exit_num), addr); +} + +static int zend_jit_trace_link_to_root(dasm_State **Dst, const void *code) +{ + const void *exit_addr; + size_t prologue_size; + + /* Skip prologue. */ + // TODO: don't hardcode this ??? + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + // sub r4, HYBRID_SPAD +#if defined(__x86_64__) || defined(_M_X64) + prologue_size = 4; +#else + prologue_size = 3; +#endif + } else if (GCC_GLOBAL_REGS) { + // sub r4, SPAD // stack alignment +#if defined(__x86_64__) || defined(_M_X64) + prologue_size = 4; +#else + prologue_size = 3; +#endif + } else { + // sub r4, NR_SPAD // stack alignment + // mov aword T2, FP // save FP + // mov aword T3, RX // save IP + // mov FP, FCARG1a +#if defined(__x86_64__) || defined(_M_X64) + prologue_size = 17; +#else + prologue_size = 12; +#endif + } + exit_addr = (const void*)((const char*)code + prologue_size); + + | jmp &exit_addr + return 1; +} + +static int zend_jit_trace_return(dasm_State **Dst) +{ +#if 0 + | jmp ->trace_escape +#else + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | JMP_IP + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 2 // ZEND_VM_LEAVE + | ret + } +#endif + return 1; +} + +static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_Z_TYPE FP + var, type, &exit_addr + + return 1; +} + +static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) +{ + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + size_t offset = jit_extension->offset; + const void *handler = + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, FP + } + | EXT_CALL handler, r0 + if (may_throw) { + zend_jit_check_exception(Dst); + } + + if (!GCC_GLOBAL_REGS) { + if (opline->opcode == ZEND_RETURN || + opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME || + opline->opcode == ZEND_DO_FCALL) { + | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 + } + } + + if (zend_jit_trace_may_exit(op_array, opline, trace)) { + // TODO: try to avoid this check ??? + if (opline->opcode == ZEND_RETURN) { + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | cmp IP, zend_jit_halt_op + | je ->trace_halt + } else if (GCC_GLOBAL_REGS) { + | test IP, IP + | je ->trace_halt + } else { + | test eax, eax + | jl ->trace_halt + } + } + while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { + trace++; + } + if (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN) { + const zend_op *next_opline = trace->opline; + const zend_op *exit_opline = NULL; + uint32_t exit_point; + const void *exit_addr; + + if (zend_is_smart_branch(opline)) { + zend_bool exit_if_true = 0; + exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); + } else { + switch (opline->opcode) { + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + exit_opline = (trace->opline == opline + 1) ? + OP_JMP_ADDR(opline, opline->op2) : + opline + 1; + break; + case ZEND_JMPZNZ: + exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + OP_JMP_ADDR(opline, opline->op2); + break; + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + exit_opline = (trace->opline == opline + 1) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + opline + 1; + break; + + } + } + exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, trace); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | CMP_IP next_opline + | jne &exit_addr + } + } else { + while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { + trace++; + } + // TODO: remove this ??? + if (opline->opcode == ZEND_RETURN + && trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_RETURN) { + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | cmp IP, zend_jit_halt_op + | je ->trace_halt + } else if (GCC_GLOBAL_REGS) { + | test IP, IP + | je ->trace_halt + } else { + | test eax, eax + | jl ->trace_halt + } + } + } + + last_valid_opline = trace->opline; + + return 1; +} + static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { const void *handler; @@ -2810,7 +3352,11 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_ | LONG_OP_WITH_CONST sub, op1_def_addr, Z_L(1) } - if (may_overflow) { + if (may_overflow && (op1_def_info & MAY_BE_GUARD)) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | jo &exit_addr + } else if (may_overflow) { | jo >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { @@ -3024,19 +3570,25 @@ static int zend_jit_math_long_long(dasm_State **Dst, } } if (may_overflow) { - | jo >1 + if (res_info & MAY_BE_GUARD) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | jo &exit_addr + } else { + | jo >1 + } } if (Z_MODE(res_addr) == IS_MEM_ZVAL) { | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } } } - if (may_overflow) { + if (may_overflow && !(res_info & MAY_BE_GUARD)) { zend_reg tmp_reg1 = ZREG_XMM0; zend_reg tmp_reg2 = ZREG_XMM1; @@ -3077,7 +3629,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 } while (0); - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } | jmp >2 @@ -3107,7 +3659,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } @@ -3170,7 +3722,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } @@ -3241,7 +3793,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } @@ -3686,7 +4238,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { - if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } } @@ -3908,6 +4460,15 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o { zend_jit_addr op2_addr = OP2_ADDR(); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + const void *exit_addr = NULL; + + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } if (op2_info & MAY_BE_LONG) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { @@ -3936,6 +4497,8 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif if (type == BP_JIT_IS) { | jbe >9 // NOT_FOUND + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { + | jbe &exit_addr } else { | jbe >2 // NOT_FOUND } @@ -3963,6 +4526,8 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif if (type == BP_JIT_IS) { | jbe >9 // NOT_FOUND + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { + | jbe &exit_addr } else { | jbe >2 // NOT_FOUND } @@ -4000,8 +4565,14 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { - | jmp >2 // NOT_FOUND + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | jmp &exit_addr + } else { + | jmp >2 // NOT_FOUND + } } + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | jmp &exit_addr } else { | jmp >2 // NOT_FOUND } @@ -4009,15 +4580,21 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | EXT_CALL zend_hash_index_find, r0 | test r0, r0 - | jz >2 // NOT_FOUND + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | jz &exit_addr + } else { + | jz >2 // NOT_FOUND + } |.cold_code |2: switch (type) { case BP_VAR_R: - | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); - | // retval = &EG(uninitialized_zval); - | UNDEFINED_OFFSET opline - | jmp >9 + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + | // retval = &EG(uninitialized_zval); + | UNDEFINED_OFFSET opline + | jmp >9 + } break; case BP_VAR_IS: case BP_VAR_UNSET: @@ -4032,12 +4609,16 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o break; case BP_VAR_RW: |2: - | SAVE_VALID_OPLINE opline - | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); - | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); - | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + | SAVE_VALID_OPLINE opline + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); + | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 + } if (op1_info & MAY_BE_ARRAY_KEY_LONG) { - | jmp >8 + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + | jmp >8 + } |4: | SAVE_VALID_OPLINE opline | EXT_CALL zend_jit_hash_index_lookup_rw, r0 @@ -4119,7 +4700,11 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL _zend_hash_find_known_hash, r0 } | test r0, r0 - | jz >2 // NOT_FOUND + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | jz &exit_addr + } else { + | jz >2 // NOT_FOUND + } | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW |.cold_code @@ -4130,9 +4715,11 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |2: switch (type) { case BP_VAR_R: - // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); - | UNDEFINED_INDEX opline - | jmp >9 + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + | UNDEFINED_INDEX opline + | jmp >9 + } break; case BP_VAR_IS: case BP_VAR_UNSET: @@ -4978,7 +5565,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, const zen return result; } -static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_bool swap = 0; @@ -5050,24 +5637,54 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: - | jne => target_label + if (exit_addr) { + | jne &exit_addr + } else { + | jne => target_label + } break; case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | je &exit_addr + } else { + | je => target_label + } + break; case ZEND_IS_NOT_IDENTICAL: - | je => target_label + if (exit_addr) { + | jne &exit_addr + } else { + | je => target_label + } break; case ZEND_IS_SMALLER: if (swap) { - | jle => target_label + if (exit_addr) { + | jle &exit_addr + } else { + | jle => target_label + } } else { - | jge => target_label + if (exit_addr) { + | jge &exit_addr + } else { + | jge => target_label + } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | jl => target_label + if (exit_addr) { + | jl &exit_addr + } else { + | jl => target_label + } } else { - | jg => target_label + if (exit_addr) { + | jg &exit_addr + } else { + | jg => target_label + } } break; default: @@ -5079,24 +5696,54 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_ case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: - | je => target_label + if (exit_addr) { + | je &exit_addr + } else { + | je => target_label + } break; case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | jne &exit_addr + } else { + | jne => target_label + } + break; case ZEND_IS_NOT_IDENTICAL: - | jne => target_label + if (exit_addr) { + | je &exit_addr + } else { + | jne => target_label + } break; case ZEND_IS_SMALLER: if (swap) { - | jg => target_label + if (exit_addr) { + | jg &exit_addr + } else { + | jg => target_label + } } else { - | jl => target_label + if (exit_addr) { + | jl &exit_addr + } else { + | jl => target_label + } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | jge => target_label + if (exit_addr) { + | jge &exit_addr + } else { + | jge => target_label + } } else { - | jle => target_label + if (exit_addr) { + | jle &exit_addr + } else { + | jle => target_label + } } break; default: @@ -5170,7 +5817,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_ return 1; } -static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { @@ -5178,29 +5825,65 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: - | jne => target_label - | jp => target_label + if (exit_addr) { + | jne &exit_addr + | jp &exit_addr + } else { + | jne => target_label + | jp => target_label + } break; case ZEND_IS_NOT_EQUAL: - case ZEND_IS_NOT_IDENTICAL: | jp >1 - | je => target_label + if (exit_addr) { + | je &exit_addr + } else { + | je => target_label + } |1: break; + case ZEND_IS_NOT_IDENTICAL: + if (exit_addr) { + | jne &exit_addr + | jp &exit_addr + } else { + | jp >1 + | je => target_label + |1: + } + break; case ZEND_IS_SMALLER: if (swap) { - | jbe => target_label + if (exit_addr) { + | jbe &exit_addr + } else { + | jbe => target_label + } } else { - | jae => target_label - | jp => target_label + if (exit_addr) { + | jae &exit_addr + | jp &exit_addr + } else { + | jae => target_label + | jp => target_label + } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | jb => target_label + if (exit_addr) { + | jb &exit_addr + } else { + | jb => target_label + } } else { - | ja => target_label - | jp => target_label + if (exit_addr) { + | ja &exit_addr + | jp &exit_addr + } else { + | ja => target_label + | jp => target_label + } } break; default: @@ -5212,29 +5895,63 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_IDENTICAL: case ZEND_CASE: | jp >1 - | je => target_label + if (exit_addr) { + | je &exit_addr + } else { + | je => target_label + } |1: break; case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | jne &exit_addr + | jp &exit_addr + } else { + | jne => target_label + | jp => target_label + } + break; case ZEND_IS_NOT_IDENTICAL: - | jne => target_label - | jp => target_label + if (exit_addr) { + |1: + | je &exit_addr + |1: + } else { + | jne => target_label + | jp => target_label + } break; case ZEND_IS_SMALLER: if (swap) { - | ja => target_label + if (exit_addr) { + | ja &exit_addr + } else { + | ja => target_label + } } else { | jp >1 - | jb => target_label + if (exit_addr) { + | jb &exit_addr + } else { + | jb => target_label + } |1: } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { - | jae => target_label + if (exit_addr) { + | jae &exit_addr + } else { + | jae => target_label + } } else { | jp >1 - | jbe => target_label + if (exit_addr) { + | jbe &exit_addr + } else { + | jbe => target_label + } |1: } break; @@ -5431,27 +6148,27 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z return 1; } -static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr - return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2); + return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); } -static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr - return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2); + return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); } -static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_bool swap = 0; @@ -5467,10 +6184,10 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr } - return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2); + return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); } -static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0) if (smart_branch_opcode) { @@ -5502,16 +6219,32 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: - | jne => target_label + if (exit_addr) { + | jne &exit_addr + } else { + | jne => target_label + } break; case ZEND_IS_NOT_EQUAL: - | je => target_label + if (exit_addr) { + | je &exit_addr + } else { + | je => target_label + } break; case ZEND_IS_SMALLER: - | jge => target_label + if (exit_addr) { + | jge &exit_addr + } else { + | jge => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | jg => target_label + if (exit_addr) { + | jg &exit_addr + } else { + | jg => target_label + } break; default: ZEND_ASSERT(0); @@ -5521,16 +6254,32 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: - | je => target_label + if (exit_addr) { + | je &exit_addr + } else { + | je => target_label + } break; case ZEND_IS_NOT_EQUAL: - | jne => target_label + if (exit_addr) { + | jne &exit_addr + } else { + | jne => target_label + } break; case ZEND_IS_SMALLER: - | jl => target_label + if (exit_addr) { + | jl &exit_addr + } else { + | jl => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | jle => target_label + if (exit_addr) { + | jle &exit_addr + } else { + | jle => target_label + } break; default: ZEND_ASSERT(0); @@ -5583,7 +6332,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a return 1; } -static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); zend_bool has_slow; @@ -5610,7 +6359,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } - if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | jmp >6 @@ -5619,7 +6368,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } } - if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -5636,7 +6385,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } - if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | jmp >6 @@ -5646,7 +6395,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } - if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | jmp >6 @@ -5667,7 +6416,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } - if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } } @@ -5679,7 +6428,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } - if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { @@ -5701,7 +6450,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } } - if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } } @@ -5713,7 +6462,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } - if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { @@ -5794,7 +6543,7 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } - if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2)) { + if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (has_slow) { @@ -5808,12 +6557,12 @@ static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_a return 1; } -static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; - if (smart_branch_opcode) { + if (smart_branch_opcode && !exit_addr) { if (opline->opcode == ZEND_IS_IDENTICAL) { if (smart_branch_opcode == ZEND_JMPZ) { not_identical_label = target_label; @@ -5841,6 +6590,20 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen } } + if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && + (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { + if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + return 1; + } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && + (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + return 1; + } + if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { op1_info |= MAY_BE_NULL; op2_info |= MAY_BE_NULL; @@ -5956,7 +6719,11 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen } if (smart_branch_opcode) { zend_jit_check_exception_undef_result(Dst, opline); - if (not_identical_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else { @@ -5968,7 +6735,11 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen concrete_type(op1_info) == concrete_type(op2_info) && concrete_type(op1_info) <= IS_TRUE) { if (smart_branch_opcode) { - if (identical_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } + } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } } else { @@ -5977,7 +6748,11 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { if (smart_branch_opcode) { - if (identical_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } + } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } } else { @@ -5985,7 +6760,11 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen } } else { if (smart_branch_opcode) { - if (not_identical_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else { @@ -6002,12 +6781,16 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen | SAVE_VALID_OPLINE opline | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline zend_jit_check_exception_undef_result(Dst, opline); - if (identical_label != (uint32_t)-1) { + if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } else { | jmp >9 } |8: + } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | je &exit_addr } else if (identical_label != (uint32_t)-1) { | je =>identical_label } else { @@ -6029,7 +6812,11 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline zend_jit_check_exception_undef_result(Dst, opline); } - if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } + } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { @@ -6042,12 +6829,16 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen | SAVE_VALID_OPLINE opline | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline zend_jit_check_exception_undef_result(Dst, opline); - if (identical_label != (uint32_t)-1) { + if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } else { | jmp >9 } |8: + } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | je &exit_addr } else if (identical_label != (uint32_t)-1) { | je =>identical_label } else { @@ -6069,18 +6860,14 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline zend_jit_check_exception_undef_result(Dst, opline); } - if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { - | jmp =>not_identical_label - } - } else if ((op1_info & MAY_BE_ANY) == MAY_BE_LONG && - (op2_info & MAY_BE_ANY) == MAY_BE_LONG) { - if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { - return 0; - } - } else if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE && - (op2_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) { - return 0; + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } } } else { if (opline->op1_type == IS_CONST) { @@ -6103,7 +6890,13 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen } if (smart_branch_opcode) { | test al, al - if (not_identical_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jnz &exit_addr + } else { + | jz &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { | jz =>not_identical_label if (identical_label != (uint32_t)-1) { | jmp =>identical_label @@ -6127,11 +6920,10 @@ static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zen if (may_throw) { zend_jit_check_exception(Dst); } - return 1; } -static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw) +static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) { uint32_t true_label = -1; uint32_t false_label = -1; @@ -6139,22 +6931,22 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z zend_bool set_bool_not = 0; zend_bool jmp_done = 0; - if (opline->opcode == ZEND_JMPZ) { + if (branch_opcode == ZEND_BOOL) { + set_bool = 1; + } else if (branch_opcode == ZEND_BOOL_NOT) { + set_bool = 1; + set_bool_not = 1; + } else if (branch_opcode == ZEND_JMPZ) { false_label = target_label; - } else if (opline->opcode == ZEND_JMPNZ) { + } else if (branch_opcode == ZEND_JMPNZ) { true_label = target_label; - } else if (opline->opcode == ZEND_JMPZNZ) { + } else if (branch_opcode == ZEND_JMPZNZ) { true_label = target_label2; false_label = target_label; - } else if (opline->opcode == ZEND_BOOL) { - set_bool = 1; - } else if (opline->opcode == ZEND_BOOL_NOT) { - set_bool = 1; - set_bool_not = 1; - } else if (opline->opcode == ZEND_JMPZ_EX) { + } else if (branch_opcode == ZEND_JMPZ_EX) { set_bool = 1; false_label = target_label; - } else if (opline->opcode == ZEND_JMPNZ_EX) { + } else if (branch_opcode == ZEND_JMPNZ_EX) { set_bool = 1; true_label = target_label; } else { @@ -6225,7 +7017,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z if ((op1_info & MAY_BE_LONG) && !(op1_info & MAY_BE_UNDEF) && !set_bool) { - if (false_label != (uint32_t)-1) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ) { + | jl >9 + } else { + | jl &exit_addr + } + } else if (false_label != (uint32_t)-1) { | jl =>false_label } else { | jl >9 @@ -6245,7 +7043,32 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } } } else { - if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (exit_addr) { + if (set_bool) { + | jne >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jmp &exit_addr + } else { + | jmp >9 + } + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { + if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | jne &exit_addr + } + } + } else { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | je &exit_addr + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | jne &exit_addr + } else { + | je >9 + } + } + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (set_bool) { | jne >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE @@ -6297,11 +7120,19 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } } - if (false_label != (uint32_t)-1) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { + | jmp &exit_addr + } + } else if (false_label != (uint32_t)-1) { | jmp =>false_label } if (op1_info & MAY_BE_ANY) { - if (false_label == (uint32_t)-1) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jmp >9 + } + } else if (false_label == (uint32_t)-1) { | jmp >9 } |.code @@ -6309,7 +7140,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } if (!jmp_done) { - if (false_label != (uint32_t)-1) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jmp >9 + } else if (op1_info & MAY_BE_LONG) { + | jmp &exit_addr + } + } else if (false_label != (uint32_t)-1) { | jmp =>false_label } else if (op1_info & MAY_BE_LONG) { | jmp >9 @@ -6339,7 +7176,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } | SET_ZVAL_TYPE_INFO res_addr, eax } - if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jne &exit_addr + } else { + | je &exit_addr + } + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { @@ -6360,7 +7203,20 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr if (set_bool) { - if (false_label != (uint32_t)-1) { // JMPZ_EX + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jne &exit_addr + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp &exit_addr + | je &exit_addr + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + } else if (false_label != (uint32_t)-1) { // JMPZ_EX | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jp >1 | je => false_label @@ -6391,20 +7247,31 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } } else { ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); - if (false_label != (uint32_t)-1) { - | jp =>false_label + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jp >1 + | jne &exit_addr + |1: + } else { + | jp &exit_addr + | je &exit_addr + } } else { - | jp >1 - } - if (true_label != (uint32_t)-1) { - | jne =>true_label if (false_label != (uint32_t)-1) { - | jmp =>false_label + | jp =>false_label + } else { + | jp >1 } - } else { - | je =>false_label + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } + } else { + | je =>false_label + } + |1: } - |1: } } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { @@ -6443,7 +7310,14 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z | add eax, 2 } | SET_ZVAL_TYPE_INFO res_addr, eax - if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (exit_addr) { + | CMP_ZVAL_TYPE res_addr, IS_FALSE + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jne &exit_addr + } else { + | je &exit_addr + } + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { | CMP_ZVAL_TYPE res_addr, IS_FALSE if (true_label != (uint32_t)-1) { | jne =>true_label @@ -6462,7 +7336,19 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } } else { | test r0, r0 - if (true_label != (uint32_t)-1) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | jne &exit_addr + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } else { + | je &exit_addr + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } + } else if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { | jmp =>false_label @@ -6533,7 +7419,7 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, const zend_o return 1; } -static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func) +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, const void *exit_addr) { uint32_t used_stack; @@ -6572,27 +7458,32 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con } else { | cmp r2, FCARG1a } - | jb >1 - | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); - |.cold_code - |1: - | SAVE_VALID_OPLINE opline - if (func) { - | mov FCARG1d, used_stack - } + + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + | jb &exit_addr + } else { + | jb >1 + | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + if (func) { + | mov FCARG1d, used_stack + } #ifdef _WIN32 - if (0) { + if (0) { #else - if (func && func->type == ZEND_INTERNAL_FUNCTION) { + if (func && func->type == ZEND_INTERNAL_FUNCTION) { #endif - | EXT_CALL zend_jit_int_extend_stack_helper, r0 - } else { - | mov FCARG2a, r0 - | EXT_CALL zend_jit_extend_stack_helper, r0 - } - | mov RX, r0 - | jmp >1 - |.code + | EXT_CALL zend_jit_int_extend_stack_helper, r0 + } else { + | mov FCARG2a, r0 + | EXT_CALL zend_jit_extend_stack_helper, r0 + } + | mov RX, r0 + | jmp >1 + |.code + } if (func) { | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 @@ -6730,11 +7621,83 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons } } -static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, int call_level) +static int zend_jit_init_fcall_guard(dasm_State **Dst, const zend_op *opline, const zend_function *func) +{ + int32_t exit_point; + const void *exit_addr; + + if (func->type == ZEND_INTERNAL_FUNCTION) { +#ifdef ZEND_WIN32 + // TODO: ASLR may cause different addresses in different workers ??? + return 0; +#endif + } else if (func->type == ZEND_USER_FUNCTION) { + if (!zend_accel_in_shm(func->op_array.opcodes)) { + /* op_array and op_array->opcodes are not persistent. We can't link. */ + return 0; + } + } else { + ZEND_ASSERT(0); + return 0; + } + + exit_point = zend_jit_trace_get_exit_point(opline, opline ? (opline+1) : NULL, NULL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + if (func->type == ZEND_USER_FUNCTION + && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { + const zend_op *opcodes = func->op_array.opcodes; + + | // call = EX(call); + | mov r1, EX->call + | mov r1, aword EX:r1->func + | .if X64 + || if (!IS_SIGNED_32BIT(opcodes)) { + | mov64 r0, ((ptrdiff_t)opcodes) + | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r0 + || } else { + | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes + || } + | .else + | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes + | .endif + | jne &exit_addr + } else { + | // call = EX(call); + | mov r1, EX->call + | .if X64 + || if (!IS_SIGNED_32BIT(func)) { + | mov64 r0, ((ptrdiff_t)func) + | cmp aword EX:r1->func, r0 + || } else { + | cmp aword EX:r1->func, func + || } + | .else + | cmp aword EX:r1->func, func + | .endif + | jne &exit_addr + } + + return 1; +} + +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, int call_level, zend_jit_trace_rec *trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; + const void *exit_addr = NULL; + + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { @@ -6752,6 +7715,14 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } } + if (!func + && trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL + && (opline->opcode == ZEND_INIT_FCALL || opline->opcode == ZEND_INIT_FCALL_BY_NAME)) { + /* TODO: add guard ??? */ + func = (zend_function*)trace->func; + } + #ifdef _WIN32 if (0) { #else @@ -6791,19 +7762,23 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | mov aword [r1 + opline->result.num], r0 | test r0, r0 | jnz >3 - | // SAVE_OPLINE(); - | SAVE_VALID_OPLINE opline - | jmp ->undefined_function + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + | jmp &exit_addr + } else { + | // SAVE_OPLINE(); + | SAVE_VALID_OPLINE opline + | jmp ->undefined_function + } } |.code |3: } - if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) { + if (!zend_jit_push_call_frame(Dst, opline, op_array, func, exit_addr)) { return 0; } - if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) { + if (trace || zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } @@ -6815,7 +7790,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t return 1; } -static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, zend_call_info *call_info) +static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) { uint32_t num_args = 0; zend_function *func = call_info->callee_func; @@ -6839,13 +7814,15 @@ static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ss return num_args; } -static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block) +static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; - zend_function *func = NULL; + const zend_function *func = NULL; uint32_t i; zend_jit_addr res_addr; + uint32_t call_num_args = 0; + zend_bool unknown_num_args = 0; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); @@ -6859,6 +7836,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend #endif } + if ((opline-1)->opcode == ZEND_SEND_UNPACK|| (opline-1)->opcode == ZEND_SEND_ARRAY) { + unknown_num_args = 1; + } + if (info) { call_info = info->callee_info; while (call_info && call_info->caller_call_opline != opline) { @@ -6872,25 +7853,51 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend /* resolve function ar run time */ } else if (func->type == ZEND_USER_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); - if (call_info->num_args > func->op_array.num_args || - (opline-1)->opcode == ZEND_SEND_UNPACK || - (opline-1)->opcode == ZEND_SEND_ARRAY) { - goto fallback; - } + call_num_args = call_info->num_args; } else if (func->type == ZEND_INTERNAL_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); + call_num_args = call_info->num_args; #if ZEND_DEBUG if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { - goto fallback; + // TODO: Mow most internal functions have type hints ??? + if (!trace) { + goto fallback; + } } #endif - if ((opline-1)->opcode == ZEND_SEND_UNPACK || (opline-1)->opcode == ZEND_SEND_ARRAY) { - goto fallback; - } } else { ZEND_ASSERT(0); } + if (trace && !func) { + if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { + ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); +#ifndef ZEND_WIN32 + // TODO: ASLR may cause different addresses in different workers ??? + func = trace->func; + if (JIT_G(current_frame) && + JIT_G(current_frame)->call && + JIT_G(current_frame)->call->num_args >= 0) { + call_num_args = JIT_G(current_frame)->call->num_args; + } else { + unknown_num_args = 1; + } +#endif + } else if (trace->op == ZEND_JIT_TRACE_ENTER) { + ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); + if (zend_accel_in_shm(trace->func->op_array.opcodes)) { + func = trace->func; + if (JIT_G(current_frame) && + JIT_G(current_frame)->call && + JIT_G(current_frame)->call->num_args >= 0) { + call_num_args = JIT_G(current_frame)->call->num_args; + } else { + unknown_num_args = 1; + } + } + } + } + if (!reuse_ip) { zend_jit_start_reuse_ip(); | // call = EX(call); @@ -6923,20 +7930,31 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (opline->opcode == ZEND_DO_FCALL) { if (!func) { - | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED - | jnz >1 - |.cold_code - |1: - if (!GCC_GLOBAL_REGS) { - | mov FCARG1a, RX + if (trace) { + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED + | jnz &exit_addr + } else { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED + | jnz >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_helper, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + | mov r0, EX:RX->func // reload + | jmp >1 + |.code + |1: } - | EXT_CALL zend_jit_deprecated_helper, r0 - | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 - | jne ->exception_handler - | mov r0, EX:RX->func // reload - | jmp >1 - |.code - |1: } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { if (!GCC_GLOBAL_REGS) { | mov FCARG1a, RX @@ -6944,7 +7962,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | EXT_CALL zend_jit_deprecated_helper, r0 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler - | mov r0, EX:RX->func // reload } } @@ -6969,8 +7986,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | mov aword EX:RX->return_value, 0 } - if (func) { - for (i = call_info->num_args; i < func->op_array.last_var; i++) { + if (func + && !unknown_num_args + && call_num_args <= func->op_array.num_args) { + for (i = call_num_args; i < func->op_array.last_var; i++) { uint32_t n = EX_NUM_TO_VAR(i); | SET_Z_TYPE_INFO RX + n, IS_UNDEF } @@ -6980,7 +7999,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func || func->op_array.cache_size) { if (func && op_array == &func->op_array) { /* recursive call */ - if (func->op_array.cache_size > sizeof(void*)) { + if (trace || func->op_array.cache_size > sizeof(void*)) { | mov r2, EX->run_time_cache | mov EX:RX->run_time_cache, r2 } @@ -7009,13 +8028,16 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | mov FP, RX | // opline = op_array->opcodes; - if (func) { + if (func + && !unknown_num_args + && call_num_args <= func->op_array.num_args) { uint32_t num_args; - if (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0 + && call_info) { num_args = skip_valid_arguments(op_array, ssa, call_info); } else { - num_args = call_info->num_args; + num_args = call_num_args; } if (func && zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes + num_args) @@ -7037,7 +8059,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } } - if (op_array == &func->op_array) { + if (!trace && op_array == &func->op_array) { /* recursive call */ #ifdef CONTEXT_THREADED_JIT | call >1 @@ -7051,45 +8073,83 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend #endif return 1; } + } else if (func + && !unknown_num_args + && call_num_args > func->op_array.num_args) { + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes) + } else if (GCC_GLOBAL_REGS) { + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + } else { + | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] + | mov aword EX->opline, FCARG1a + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, FP + } + | EXT_CALL zend_jit_copy_extra_args_helper, r0 + for (i = call_num_args; i < func->op_array.last_var; i++) { + uint32_t n = EX_NUM_TO_VAR(i); + | SET_Z_TYPE_INFO FP + n, IS_UNDEF + } } else { | // opline = op_array->opcodes - if (GCC_GLOBAL_REGS) { + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes) + } else if (GCC_GLOBAL_REGS) { | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] } else { | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] | mov aword EX->opline, FCARG1a } - | // first_extra_arg = op_array->num_args; - | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] - | // num_args = EX_NUM_ARGS(); - | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] - | // if (UNEXPECTED(num_args > first_extra_arg)) - | cmp edx, ecx - | jl >1 + if (func) { + | // num_args = EX_NUM_ARGS(); + | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp ecx, (func->op_array.num_args) + } else { + | // first_extra_arg = op_array->num_args; + | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] + | // num_args = EX_NUM_ARGS(); + | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp ecx, edx + } + | jg >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1a, FP } | EXT_CALL zend_jit_copy_extra_args_helper, r0 - | mov r0, EX->func // reload + if (!func) { + | mov r0, EX->func // reload + } | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload | jmp >1 |.code - | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) - | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS - | jnz >1 - | // opline += num_args; - |.if X64 - | movsxd r2, ecx - | imul r2, r2, sizeof(zend_op) - |.else - | imul r2, ecx, sizeof(zend_op) - |.endif - | ADD_IP r2 + if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { + if (!func) { + | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS + | jnz >1 + } + | // opline += num_args; + |.if X64 + | movsxd r2, ecx + | imul r2, r2, sizeof(zend_op) + |.else + | imul r2, ecx, sizeof(zend_op) + |.endif + | ADD_IP r2 + } |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { - | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] + if (func) { + | mov edx, (func->op_array.last_var) + } else { + | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] + } | sub edx, ecx | jle >3 //??? | // zval *var = EX_VAR_NUM(num_args); @@ -7106,24 +8166,34 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |3: } + if (trace) { + if (!func && (opline->opcode != ZEND_DO_UCALL)) { + | jmp >9 + } + } else { #ifdef CONTEXT_THREADED_JIT - | call ->context_threaded_call - if (!func && (opline->opcode != ZEND_DO_UCALL)) { - | jmp >9 - } + | call ->context_threaded_call + if (!func && (opline->opcode != ZEND_DO_UCALL)) { + | jmp >9 + } + | call ->context_threaded_call + if (!func) { + | jmp >9 + } #else - if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD - | JMP_IP - } else if (GCC_GLOBAL_REGS) { - | add r4, SPAD // stack alignment - | JMP_IP - } else { - | mov FP, aword T2 // restore FP - | mov RX, aword T3 // restore IP - | add r4, NR_SPAD // stack alignment - | mov r0, 1 // ZEND_VM_ENTER - | ret + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | JMP_IP + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret + } } #endif } @@ -7135,20 +8205,31 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { - | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED - | jnz >1 - |.cold_code - |1: - if (!GCC_GLOBAL_REGS) { - | mov FCARG1a, RX + if (trace) { + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED + | jnz &exit_addr + } else { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED + | jnz >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_helper, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + | mov r0, EX:RX->func // reload + | jmp >1 + |.code + |1: } - | EXT_CALL zend_jit_deprecated_helper, r0 - | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 - | jne ->exception_handler - | mov r0, EX:RX->func // reload - | jmp >1 - |.code - |1: } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { if (!GCC_GLOBAL_REGS) { | mov FCARG1a, RX @@ -7183,12 +8264,23 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | call aword [r0 + offsetof(zend_internal_function, handler)] } + if (trace) { + // TODO: This is a qucik dirty fix ?????? + // + // Internal function may call another trace that, + // replaces EG(trace_id) and the following side exit + // from this trace is going to be mad !!!!!! + // + // Lets set EG(trace_id) once again... + zend_jit_trace_begin(Dst, current_trace_num); + } + | // EG(current_execute_data) = execute_data; | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 | // zend_vm_stack_free_args(call); - if (func) { - for (i = 0; i < call_info->num_args; i++ ) { + if (func && call_info && !unknown_num_args) { + for (i = 0; i < call_num_args; i++ ) { uint32_t offset = EX_NUM_TO_VAR(i); | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 0, opline } @@ -7213,15 +8305,21 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |2: } - | // zend_vm_stack_free_call_frame(call); - | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) - | jnz >1 - |.cold_code - |1: - | mov FCARG1a, RX - | EXT_CALL zend_jit_free_call_frame, r0 - | jmp >1 - |.code + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !JIT_G(current_frame)->call || + !JIT_G(current_frame)->call->nested) { + + | // zend_vm_stack_free_call_frame(call); + | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) + | jnz >1 + |.cold_code + |1: + | mov FCARG1a, RX + | EXT_CALL zend_jit_free_call_frame, r0 + | jmp >1 + |.code + } | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0 |1: @@ -7261,9 +8359,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } return 1; - +#if ZEND_DEBUG fallback: /* fallback to subroutine threading */ + if (trace) { + return zend_jit_trace_handler(Dst, op_array, opline, zend_may_throw(opline, op_array, ssa), trace); + } if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME ){ @@ -7271,6 +8372,7 @@ fallback: } else { return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); } +#endif } static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) @@ -7289,18 +8391,36 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); - | mov r0, EX:RX->func - if (arg_num <= MAX_ARG_FLAG_NUM) { + ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); + + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + /* Don't generate code that always throws exception */ + return 0; + } + } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask - | jnz >1 + | jnz &exit_addr } else { - ZEND_ASSERT(0); + | mov r0, EX:RX->func + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask + | jnz >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | jmp ->throw_cannot_pass_by_ref + |.code + } - |.cold_code - |1: - | SAVE_VALID_OPLINE opline - | jmp ->throw_cannot_pass_by_ref - |.code } arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); @@ -7308,12 +8428,12 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); - | ZVAL_COPY_CONST arg_addr, -1, -1, zv, r0 + | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } } else { - | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 } return 1; @@ -7382,11 +8502,11 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); | mov r1, aword T1 // restore - | ZVAL_COPY_VALUE ref_addr, -1, val_addr, op1_info, ZREG_R2, ZREG_R2 + | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 | SET_ZVAL_PTR val_addr, r0 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX } else { - | ZVAL_COPY_VALUE ref_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 | SET_ZVAL_PTR op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX } @@ -7418,46 +8538,93 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | mov RX, EX->call } - if (opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { - uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + if (opline->opcode == ZEND_SEND_VAR_EX) { + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { + return 0; + } + return 1; + } + } else { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); - | mov r0, EX:RX->func - if (arg_num <= MAX_ARG_FLAG_NUM) { + | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 - } else { - ZEND_ASSERT(0); + |.cold_code + |1: + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { + return 0; + } + | jmp >7 + |.code } + } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { - |.cold_code - |1: + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 - if (opline->opcode == ZEND_SEND_VAR_EX) { - if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { - return 0; + if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!(op1_info & MAY_BE_REF)) { + /* Don't generate code that always throws exception */ + return 0; + } else { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | cmp cl, IS_REFERENCE + | jne &exit_addr + } + } + return 1; } - } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + } else { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + | mov r0, EX:RX->func + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask + | jnz >1 + |.cold_code + |1: + mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); - | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 } | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >7 - | SAVE_VALID_OPLINE opline - | LOAD_ZVAL_ADDR FCARG1a, arg_addr - | EXT_CALL zend_jit_only_vars_by_reference, r0 - if (!zend_jit_check_exception(Dst)) { - return 0; + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | jmp &exit_addr + } else { + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG1a, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, r0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + | jmp >7 } - } else { - ZEND_ASSERT(0); - } - | jmp >7 - |.code + |.code + } } if (op1_info & MAY_BE_UNDEF) { @@ -7479,16 +8646,25 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { - | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 } - | SAVE_VALID_OPLINE opline - | LOAD_ZVAL_ADDR FCARG1a, arg_addr - | EXT_CALL zend_jit_only_vars_by_reference, r0 - if (!zend_jit_check_exception(Dst)) { - return 0; + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | jmp &exit_addr + } else { + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG1a, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, r0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } } } else if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { if (op1_info & MAY_BE_REF) { @@ -7497,7 +8673,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info - | ZVAL_COPY_VALUE arg_addr, -1, val_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 | TRY_ADDREF op1_info, ah, r2 } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); @@ -7508,7 +8684,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR FCARG1a, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE arg_addr, -1, ref_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 | GC_DELREF FCARG1a | je >1 | IF_NOT_REFCOUNTED ah, >2 @@ -7518,7 +8694,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | EFREE_REG_24 op_array, opline | jmp >2 |.code - | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 |2: } } else { @@ -7530,7 +8706,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend op1_addr= op1_def_addr; } } - | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 } @@ -7593,14 +8769,14 @@ static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp return 1; } -static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t defined_label = (uint32_t)-1; uint32_t undefined_label = (uint32_t)-1; zval *zv = RT_CONSTANT(opline, opline->op1); - zend_jit_addr res_addr; + zend_jit_addr res_addr = 0; - if (smart_branch_opcode) { + if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { undefined_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { @@ -7625,8 +8801,15 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_ | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a | shr r0, 1 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax + if (smart_branch_opcode) { - if (undefined_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jz &exit_addr + } else { + | jz >3 + } + } else if (undefined_label != (uint32_t)-1) { | jz =>undefined_label } else { | jz >3 @@ -7639,7 +8822,14 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_ | LOAD_ADDR FCARG1a, zv | EXT_CALL zend_jit_check_constant, r0 | test r0, r0 - if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jnz >3 + } else { + | jz >3 + } + | jmp &exit_addr + } else if (smart_branch_opcode) { if (undefined_label != (uint32_t)-1) { | jnz =>undefined_label } else { @@ -7659,7 +8849,11 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_ } |.code if (smart_branch_opcode) { - if (defined_label != (uint32_t)-1) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } + } else if (defined_label != (uint32_t)-1) { | jmp =>defined_label } } else { @@ -7671,7 +8865,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_ return 1; } -static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t mask; zend_uchar type; @@ -7691,11 +8885,23 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const ze | EXT_CALL zend_jit_undefined_op_helper, r0 zend_jit_check_exception_undef_result(Dst, opline); if (opline->extended_value & MAY_BE_NULL) { - if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { + | jmp >7 + } + } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { return 0; } } else { - if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { + | jmp >7 + } + } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { return 0; } } @@ -7719,14 +8925,22 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const ze type = 0; } - if (!(op1_info & (MAY_BE_ANY - mask))) { + if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline - if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } + } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { return 0; } - } else if (!(op1_info & mask)) { + } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline - if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } + } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { return 0; } } else { @@ -7785,7 +8999,13 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const ze | mov eax, 1 | shl eax, cl | test eax, mask - if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jne &exit_addr + } else { + | je &exit_addr + } + } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | je =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { @@ -7854,7 +9074,13 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const ze | cmp byte [FP + opline->op1.var + 8], type } } - if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | je &exit_addr + } else { + | jne &exit_addr + } + } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jne =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { @@ -7883,75 +9109,99 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const ze return 1; } -static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +static uint32_t zend_ssa_cv_info(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) { - uint32_t i, j, info; + uint32_t j, info; - // Use type inference to avoid useless zval_ptr_dtor() - for (i = 0 ; i < op_array->last_var; i++) { - if (ssa->vars && ssa->var_info) { - info = ssa->var_info[i].type; - for (j = op_array->last_var; j < ssa->vars_count; j++) { - if (ssa->vars[j].var == i) { - info |= ssa->var_info[j].type; - } + if (ssa->vars && ssa->var_info) { + info = ssa->var_info[var].type; + for (j = op_array->last_var; j < ssa->vars_count; j++) { + if (ssa->vars[j].var == var) { + info |= ssa->var_info[j].type; } - } else { - info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF; } + } else { + info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF; + } #ifdef ZEND_JIT_USE_RC_INFERENCE - /* Refcount may be increased by RETURN opcode */ - if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { - for (j = 0; j < ssa->cfg.blocks_count; j++) { - if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && - ssa->cfg.blocks[j].len > 0) { - const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; - - if (opline->opcode == ZEND_RETURN) { - if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(i)) { - info |= MAY_BE_RCN; - break; - } + /* Refcount may be increased by RETURN opcode */ + if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { + for (j = 0; j < ssa->cfg.blocks_count; j++) { + if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && + ssa->cfg.blocks[j].len > 0) { + const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; + + if (opline->opcode == ZEND_RETURN) { + if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) { + info |= MAY_BE_RCN; + break; } } } } + } #endif - if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - uint32_t offset = EX_NUM_TO_VAR(i); - | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 0, opline - } - } - return 1; + return info; } -static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa) +static int zend_jit_leave_frame(dasm_State **Dst) { - // Avoid multiple leave sequences - if (jit_return_label >= 0) { - | jmp =>jit_return_label - return 1; - } - - jit_return_label = ssa->cfg.blocks_count * 2; - - |=>jit_return_label: - | // EG(current_execute_data) = EX(prev_execute_data); | mov r0, EX->prev_execute_data | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2 + return 1; +} - // i_free_compiled_variables(execute_data); - if (!zend_jit_free_compiled_variables(Dst, opline, op_array, ssa)) { - return 0; +static int zend_jit_free_cv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t info, uint32_t var) +{ + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + uint32_t offset = EX_NUM_TO_VAR(var); + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 0, opline } + return 1; +} +static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_jit_trace_rec *trace) +{ /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_FAKE_CLOSURE) - | jnz ->leave_function_handler + if (trace && trace->op != ZEND_JIT_TRACE_END) { + | jnz >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG2a, FP + } + | EXT_CALL zend_jit_leave_func_helper, r0 + + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !JIT_G(current_frame)->nested) { + // TODO: try to avoid this check ??? + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | cmp IP, zend_jit_halt_op + | je ->trace_halt + } else if (GCC_GLOBAL_REGS) { + | test IP, IP + | je ->trace_halt + } else { + | test eax, eax + | jl ->trace_halt + } + } + + if (!GCC_GLOBAL_REGS) { + | // execute_data = EG(current_execute_data) + | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 + } + | jmp >8 + |.code + } else { + | jnz ->leave_function_handler + } if ((op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { @@ -7975,12 +9225,53 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const ze | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 | // execute_data = EX(prev_execute_data); | mov FP, EX->prev_execute_data - | // if (EG(exception)) - | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 - | LOAD_OPLINE - | jne ->leave_throw_handler - | // opline = EX(opline) + 1 - | ADD_IP sizeof(zend_op) + + |9: + if (trace) { + if (trace->op != ZEND_JIT_TRACE_END + && (trace->op != ZEND_JIT_TRACE_BACK || !trace->recursive)) { + zend_jit_reset_opline(Dst, NULL); + } else { + // TODO: exception handling for tracing JIT ??? + | LOAD_OPLINE + | ADD_IP sizeof(zend_op) + } + + |8: + + if (opline->opcode == ZEND_RETURN + && trace->op == ZEND_JIT_TRACE_BACK + && trace->recursive) { + const zend_op *next_opline = trace->opline; + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, NULL, trace); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + trace++; + ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); + next_opline = trace->opline; + exit_point = zend_jit_trace_get_exit_point(opline, NULL, trace); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | // TODO: exception handling ??? + | CMP_IP next_opline + | jne &exit_addr + + last_valid_opline = trace->opline; + + return 1; + } + return 1; + } else { + | // if (EG(exception)) + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | LOAD_OPLINE + | jne ->leave_throw_handler + | // opline = EX(opline) + 1 + | ADD_IP sizeof(zend_op) + } + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | add r4, HYBRID_SPAD #ifdef CONTEXT_THREADED_JIT @@ -8021,56 +9312,77 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const ze return 1; } -static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, uint32_t op1_info, zend_jit_addr op1_addr) +static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) { zend_jit_addr ret_addr; + int8_t return_value_used; ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { + return_value_used = JIT_G(current_frame)->return_value_used; + } else { + return_value_used = -1; + } + // if (!EX(return_value)) if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { - | mov r2, EX->return_value - | test r2, r2 + if (return_value_used != 0) { + | mov r2, EX->return_value + } + if (return_value_used == -1) { + | test r2, r2 + } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); } else { - | mov r1, EX->return_value - | test r1, r1 + if (return_value_used != 0) { + | mov r1, EX->return_value + } + if (return_value_used == -1) { + | test r1, r1 + } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | jz >1 - |.cold_code - |1: - if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - if (jit_return_label >= 0) { - | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label - } else { - | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 - } + if (return_value_used == -1) { + | jz >1 + |.cold_code + |1: } - | GET_ZVAL_PTR FCARG1a, op1_addr - | GC_DELREF FCARG1a - if (RC_MAY_BE_1(op1_info)) { - if (RC_MAY_BE_N(op1_info)) { + if (return_value_used != 1) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (jit_return_label >= 0) { - | jnz =>jit_return_label + | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label } else { - | jnz >9 + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 } } - | //SAVE_OPLINE() - | ZVAL_DTOR_FUNC op1_info, opline - | //????mov r1, EX->return_value // reload ??? - } - if (jit_return_label >= 0) { - | jmp =>jit_return_label - } else { - | jmp >9 + | GET_ZVAL_PTR FCARG1a, op1_addr + | GC_DELREF FCARG1a + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + if (jit_return_label >= 0) { + | jnz =>jit_return_label + } else { + | jnz >9 + } + } + | //SAVE_OPLINE() + | ZVAL_DTOR_FUNC op1_info, opline + | //????mov r1, EX->return_value // reload ??? + } + if (return_value_used == -1) { + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } + |.code + } } - |.code - } else { + } else if (return_value_used == -1) { if (jit_return_label >= 0) { | jz =>jit_return_label } else { @@ -8078,21 +9390,25 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } } + if (return_value_used == 0) { + return 1; + } + if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); - | ZVAL_COPY_CONST ret_addr, -1, -1, zv, r0 + | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } } else if (opline->op1_type == IS_TMP_VAR) { - | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } - | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? | TRY_ADDREF op1_info, ah, r2 } else { @@ -8105,7 +9421,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR r0, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); - | ZVAL_COPY_VALUE ret_addr, -1, ref_addr, op1_info, ZREG_R2, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 | GC_DELREF r0 | je >2 | // if (IS_REFCOUNTED()) @@ -8131,12 +9447,11 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } |.code } - | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 } |9: - //JIT: ZEND_VM_DISPATCH_TO_HELPER(zend_leave_helper); - return zend_jit_leave_func(Dst, opline, op_array, ssa); + return 1; } static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, uint32_t op2_info, uint32_t res_info, int may_throw) @@ -8316,7 +9631,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, cons return 1; } -static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, uint32_t op2_info, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, uint32_t op2_info, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr op1_addr, op2_addr, res_addr; @@ -8385,7 +9700,13 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, c } } if (!(opline->extended_value & ZEND_ISEMPTY)) { - if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | jmp &exit_addr + } else { + | jmp >8 + } + } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp =>target_label2 } else if (smart_branch_opcode == ZEND_JMPNZ) { @@ -8413,7 +9734,11 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, c } } if (!(opline->extended_value & ZEND_ISEMPTY)) { - if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | jmp &exit_addr + } + } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { @@ -8695,8 +10020,11 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | cmp dword EX->This.u2.num_args, arg_num - | jae >5 + if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || + (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | cmp dword EX->This.u2.num_args, arg_num + | jae >5 + } | ZVAL_COPY_CONST res_addr, -1, -1, zv, r0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 @@ -8890,7 +10218,7 @@ static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string return 0; } -static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_class_entry *ce, int may_throw) +static int zend_jit_fetch_obj_read(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_class_entry *ce, int may_throw) { zval *member; uint32_t offset; @@ -8917,7 +10245,17 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, const zend op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 + if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 + } } | GET_ZVAL_PTR FCARG1a, op1_addr } @@ -8995,7 +10333,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, const zend | jmp >9 } - if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) { + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { |7: if (opline->opcode == ZEND_FETCH_OBJ_R) { | SAVE_VALID_OPLINE opline -- 2.49.0