#include "zend_compile.h"
#include "zend_dfg.h"
+static zend_always_inline void _zend_dfg_add_use_def_op(const zend_op_array *op_array, const zend_op *opline, uint32_t build_flags, zend_bitset use, zend_bitset def) /* {{{ */
+{
+ uint32_t var_num;
+ const zend_op *next;
+
+ if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(opline->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ }
+ if (((opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
+ && opline->opcode != ZEND_FE_FETCH_R
+ && opline->opcode != ZEND_FE_FETCH_RW)
+ || (opline->op2_type == IS_CV)) {
+ var_num = EX_VAR_TO_NUM(opline->op2.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ }
+ if ((build_flags & ZEND_SSA_USE_CV_RESULTS)
+ && opline->result_type == IS_CV
+ && opline->opcode != ZEND_RECV) {
+ var_num = EX_VAR_TO_NUM(opline->result.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ }
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN:
+ if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op2_type == IS_CV) {
+ zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var));
+ }
+ if (opline->op1_type == IS_CV) {
+add_op1_def:
+ zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op1.var));
+ }
+ break;
+ case ZEND_ASSIGN_REF:
+ if (opline->op2_type == IS_CV) {
+ zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var));
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_DIM:
+ case ZEND_ASSIGN_OBJ:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(next->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ if (build_flags & ZEND_SSA_RC_INFERENCE && next->op1_type == IS_CV) {
+ zend_bitset_incl(def, var_num);
+ }
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_OBJ_REF:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(next->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ if (next->op1_type == IS_CV) {
+ zend_bitset_incl(def, var_num);
+ }
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_STATIC_PROP:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(next->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+#if 0
+ if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) {
+ zend_bitset_incl(def, var_num);
+ }
+#endif
+ }
+ break;
+ case ZEND_ASSIGN_STATIC_PROP_REF:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(next->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ if (next->op1_type == IS_CV) {
+ zend_bitset_incl(def, var_num);
+ }
+ }
+ break;
+ case ZEND_ASSIGN_STATIC_PROP_OP:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(next->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ }
+ break;
+ case ZEND_ASSIGN_DIM_OP:
+ case ZEND_ASSIGN_OBJ_OP:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ var_num = EX_VAR_TO_NUM(next->op1.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_OP:
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ case ZEND_BIND_GLOBAL:
+ case ZEND_BIND_STATIC:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_FUNC_ARG:
+ case ZEND_SEND_REF:
+ case ZEND_SEND_UNPACK:
+ case ZEND_FE_RESET_RW:
+ case ZEND_MAKE_REF:
+ case ZEND_PRE_INC_OBJ:
+ case ZEND_PRE_DEC_OBJ:
+ case ZEND_POST_INC_OBJ:
+ case ZEND_POST_DEC_OBJ:
+ case ZEND_UNSET_DIM:
+ case ZEND_UNSET_OBJ:
+ case ZEND_FETCH_DIM_W:
+ case ZEND_FETCH_DIM_RW:
+ case ZEND_FETCH_DIM_FUNC_ARG:
+ case ZEND_FETCH_DIM_UNSET:
+ case ZEND_FETCH_LIST_W:
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_SEND_VAR:
+ case ZEND_CAST:
+ case ZEND_QM_ASSIGN:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_FE_RESET_R:
+ if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ADD_ARRAY_UNPACK:
+ var_num = EX_VAR_TO_NUM(opline->result.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ break;
+ case ZEND_ADD_ARRAY_ELEMENT:
+ var_num = EX_VAR_TO_NUM(opline->result.var);
+ if (!zend_bitset_in(def, var_num)) {
+ zend_bitset_incl(use, var_num);
+ }
+ /* break missing intentionally */
+ case ZEND_INIT_ARRAY:
+ if (((build_flags & ZEND_SSA_RC_INFERENCE)
+ || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF))
+ && opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_YIELD:
+ if (opline->op1_type == IS_CV
+ && ((op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE)
+ || (build_flags & ZEND_SSA_RC_INFERENCE))) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_UNSET_CV:
+ goto add_op1_def;
+ case ZEND_VERIFY_RETURN_TYPE:
+ if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+#if 0
+ /* This special case was handled above the switch */
+ if (opline->op2_type != IS_CV) {
+ op2_use = -1; /* not used */
+ }
+#endif
+ zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var));
+ break;
+ case ZEND_BIND_LEXICAL:
+ if ((opline->extended_value & ZEND_BIND_REF) || (build_flags & ZEND_SSA_RC_INFERENCE)) {
+ zend_bitset_incl(def, EX_VAR_TO_NUM(opline->op2.var));
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ zend_bitset_incl(def, EX_VAR_TO_NUM(opline->result.var));
+ }
+}
+/* }}} */
+
+void zend_dfg_add_use_def_op(const zend_op_array *op_array, const zend_op *opline, uint32_t build_flags, zend_bitset use, zend_bitset def) /* {{{ */
+{
+ _zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
+}
+/* }}} */
+
int zend_build_dfg(const zend_op_array *op_array, const zend_cfg *cfg, zend_dfg *dfg, uint32_t build_flags) /* {{{ */
{
int set_size;
int blocks_count = cfg->blocks_count;
zend_bitset tmp, def, use, in, out;
int k;
- uint32_t var_num;
int j;
set_size = dfg->size;
/* Collect "def" and "use" sets */
for (j = 0; j < blocks_count; j++) {
zend_op *opline, *end;
+ zend_bitset b_use, b_def;
+
if ((blocks[j].flags & ZEND_BB_REACHABLE) == 0) {
continue;
}
opline = op_array->opcodes + blocks[j].start;
end = opline + blocks[j].len;
+ b_use = DFG_BITSET(use, set_size, j);
+ b_def = DFG_BITSET(def, set_size, j);
for (; opline < end; opline++) {
if (opline->opcode != ZEND_OP_DATA) {
- zend_op *next = opline + 1;
- if (next < end && next->opcode == ZEND_OP_DATA) {
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- var_num = EX_VAR_TO_NUM(next->op1.var);
- if (next->op1_type == IS_CV && (opline->opcode == ZEND_ASSIGN_OBJ_REF
- || opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF)) {
- DFG_SET(use, set_size, j, var_num);
- DFG_SET(def, set_size, j, var_num);
- } else {
- if (!DFG_ISSET(def, set_size, j, var_num)) {
- DFG_SET(use, set_size, j, var_num);
- }
- }
- }
- if (next->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- var_num = EX_VAR_TO_NUM(next->op2.var);
- if (!DFG_ISSET(def, set_size, j, var_num)) {
- DFG_SET(use, set_size, j, var_num);
- }
- }
- }
- if (opline->op1_type == IS_CV) {
- var_num = EX_VAR_TO_NUM(opline->op1.var);
- switch (opline->opcode) {
- case ZEND_ADD_ARRAY_ELEMENT:
- case ZEND_INIT_ARRAY:
- if ((build_flags & ZEND_SSA_RC_INFERENCE)
- || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF)) {
- goto op1_def;
- }
- goto op1_use;
- case ZEND_FE_RESET_R:
- case ZEND_SEND_VAR:
- case ZEND_CAST:
- case ZEND_QM_ASSIGN:
- case ZEND_JMP_SET:
- case ZEND_COALESCE:
- if (build_flags & ZEND_SSA_RC_INFERENCE) {
- goto op1_def;
- }
- goto op1_use;
- case ZEND_YIELD:
- if ((build_flags & ZEND_SSA_RC_INFERENCE)
- || (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
- goto op1_def;
- }
- goto op1_use;
- case ZEND_UNSET_CV:
- case ZEND_ASSIGN:
- case ZEND_ASSIGN_REF:
- case ZEND_ASSIGN_OBJ_REF:
- case ZEND_BIND_GLOBAL:
- case ZEND_BIND_STATIC:
- case ZEND_SEND_VAR_EX:
- case ZEND_SEND_FUNC_ARG:
- case ZEND_SEND_REF:
- case ZEND_SEND_VAR_NO_REF:
- case ZEND_SEND_VAR_NO_REF_EX:
- case ZEND_FE_RESET_RW:
- case ZEND_ASSIGN_OP:
- case ZEND_ASSIGN_DIM_OP:
- case ZEND_ASSIGN_OBJ_OP:
- case ZEND_ASSIGN_STATIC_PROP_OP:
- case ZEND_PRE_INC:
- case ZEND_PRE_DEC:
- case ZEND_POST_INC:
- case ZEND_POST_DEC:
- case ZEND_ASSIGN_DIM:
- case ZEND_ASSIGN_OBJ:
- case ZEND_UNSET_DIM:
- case ZEND_UNSET_OBJ:
- case ZEND_FETCH_DIM_W:
- case ZEND_FETCH_DIM_RW:
- case ZEND_FETCH_DIM_FUNC_ARG:
- case ZEND_FETCH_DIM_UNSET:
- case ZEND_FETCH_LIST_W:
- case ZEND_VERIFY_RETURN_TYPE:
- case ZEND_PRE_INC_OBJ:
- case ZEND_PRE_DEC_OBJ:
- case ZEND_POST_INC_OBJ:
- case ZEND_POST_DEC_OBJ:
-op1_def:
- /* `def` always come along with dtor or separation,
- * thus the origin var info might be also `use`d in the feature(CG) */
- DFG_SET(use, set_size, j, var_num);
- DFG_SET(def, set_size, j, var_num);
- break;
- default:
-op1_use:
- if (!DFG_ISSET(def, set_size, j, var_num)) {
- DFG_SET(use, set_size, j, var_num);
- }
- }
- } else if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
- var_num = EX_VAR_TO_NUM(opline->op1.var);
- if (!DFG_ISSET(def, set_size, j, var_num)) {
- DFG_SET(use, set_size, j, var_num);
- }
- if (opline->opcode == ZEND_VERIFY_RETURN_TYPE) {
- DFG_SET(def, set_size, j, var_num);
- }
- }
- if (opline->op2_type == IS_CV) {
- var_num = EX_VAR_TO_NUM(opline->op2.var);
- switch (opline->opcode) {
- case ZEND_ASSIGN:
- if (build_flags & ZEND_SSA_RC_INFERENCE) {
- goto op2_def;
- }
- goto op2_use;
- case ZEND_BIND_LEXICAL:
- if ((build_flags & ZEND_SSA_RC_INFERENCE) || (opline->extended_value & ZEND_BIND_REF)) {
- goto op2_def;
- }
- goto op2_use;
- case ZEND_ASSIGN_REF:
- case ZEND_FE_FETCH_R:
- case ZEND_FE_FETCH_RW:
-op2_def:
- // FIXME: include into "use" too ...?
- DFG_SET(use, set_size, j, var_num);
- DFG_SET(def, set_size, j, var_num);
- break;
- default:
-op2_use:
- if (!DFG_ISSET(def, set_size, j, var_num)) {
- DFG_SET(use, set_size, j, var_num);
- }
- break;
- }
- } else if (opline->op2_type & (IS_VAR|IS_TMP_VAR)) {
- var_num = EX_VAR_TO_NUM(opline->op2.var);
- if (opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW) {
- DFG_SET(def, set_size, j, var_num);
- } else {
- if (!DFG_ISSET(def, set_size, j, var_num)) {
- DFG_SET(use, set_size, j, var_num);
- }
- }
- }
- if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- var_num = EX_VAR_TO_NUM(opline->result.var);
- if ((build_flags & ZEND_SSA_USE_CV_RESULTS)
- && opline->result_type == IS_CV) {
- DFG_SET(use, set_size, j, var_num);
- }
- DFG_SET(def, set_size, j, var_num);
- }
+ _zend_dfg_add_use_def_op(op_array, opline, build_flags, b_use, b_def);
}
}
}
BEGIN_EXTERN_C()
int zend_build_dfg(const zend_op_array *op_array, const zend_cfg *cfg, zend_dfg *dfg, uint32_t build_flags);
+void zend_dfg_add_use_def_op(const zend_op_array *op_array, const zend_op *opline, uint32_t build_flags, zend_bitset use, zend_bitset def);
END_EXTERN_C()
int first = 1;
fprintf(stderr, " [");
+ if (info & MAY_BE_GUARD) {
+ fprintf(stderr, "!");
+ }
if (info & MAY_BE_UNDEF) {
if (first) first = 0; else fprintf(stderr, ", ");
fprintf(stderr, "undef");
dump_flags);
}
-static void zend_dump_ssa_var(const zend_op_array *op_array, const zend_ssa *ssa, int ssa_var_num, zend_uchar var_type, int var_num, uint32_t dump_flags)
+void zend_dump_ssa_var(const zend_op_array *op_array, const zend_ssa *ssa, int ssa_var_num, zend_uchar var_type, int var_num, uint32_t dump_flags)
{
if (ssa_var_num >= 0) {
fprintf(stderr, "#%d.", ssa_var_num);
}
}
-void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const void *data)
+void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const zend_ssa *ssa, const zend_ssa_op *ssa_op)
{
const char *name = zend_get_opcode_name(opline->opcode);
uint32_t flags = zend_get_opcode_flags(opline->opcode);
uint32_t n = 0;
- const zend_ssa *ssa = NULL;
- if (dump_flags & ZEND_DUMP_SSA) {
- ssa = (const zend_ssa*)data;
- }
-
- if (!ssa || !ssa->ops || ssa->ops[opline - op_array->opcodes].result_use < 0) {
+ if (!ssa_op || ssa_op->result_use < 0) {
if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- if (ssa && ssa->ops && ssa->ops[opline - op_array->opcodes].result_def >= 0) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].result_def;
+ if (ssa_op && ssa_op->result_def >= 0) {
+ int ssa_var_num = ssa_op->result_def;
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->result_type, EX_VAR_TO_NUM(opline->result.var), dump_flags);
} else {
zend_dump_var(op_array, opline->result_type, EX_VAR_TO_NUM(opline->result.var));
if (opline->op1_type == IS_CONST) {
zend_dump_const(CRT_CONSTANT(opline->op1));
} else if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- if (ssa && ssa->ops) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].op1_use;
+ if (ssa_op) {
+ int ssa_var_num = ssa_op->op1_use;
if (ssa_var_num >= 0) {
fprintf(stderr, " ");
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var), dump_flags);
- } else if (ssa->ops[opline - op_array->opcodes].op1_def < 0) {
+ } else if (ssa_op->op1_def < 0) {
fprintf(stderr, " ");
zend_dump_var(op_array, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var));
}
fprintf(stderr, " ");
zend_dump_var(op_array, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var));
}
- if (ssa && ssa->ops) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].op1_def;
+ if (ssa_op) {
+ int ssa_var_num = ssa_op->op1_def;
if (ssa_var_num >= 0) {
fprintf(stderr, " -> ");
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op1_type, EX_VAR_TO_NUM(opline->op1.var), dump_flags);
zend_dump_const(op);
}
} else if (opline->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- if (ssa && ssa->ops) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].op2_use;
+ if (ssa_op) {
+ int ssa_var_num = ssa_op->op2_use;
if (ssa_var_num >= 0) {
fprintf(stderr, " ");
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var), dump_flags);
- } else if (ssa->ops[opline - op_array->opcodes].op2_def < 0) {
+ } else if (ssa_op->op2_def < 0) {
fprintf(stderr, " ");
zend_dump_var(op_array, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var));
}
fprintf(stderr, " ");
zend_dump_var(op_array, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var));
}
- if (ssa && ssa->ops) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].op2_def;
+ if (ssa_op) {
+ int ssa_var_num = ssa_op->op2_def;
if (ssa_var_num >= 0) {
fprintf(stderr, " -> ");
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->op2_type, EX_VAR_TO_NUM(opline->op2.var), dump_flags);
} else if (opline->result_type & IS_SMART_BRANCH_JMPNZ) {
fprintf(stderr, " jmpnz");
#endif
- } else if (ssa && ssa->ops && ssa->ops[opline - op_array->opcodes].result_use >= 0) {
+ } else if (ssa_op && ssa_op->result_use >= 0) {
if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- if (ssa && ssa->ops) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].result_use;
+ if (ssa_op) {
+ int ssa_var_num = ssa_op->result_use;
if (ssa_var_num >= 0) {
fprintf(stderr, " ");
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->result_type, EX_VAR_TO_NUM(opline->result.var), dump_flags);
fprintf(stderr, " ");
zend_dump_var(op_array, opline->result_type, EX_VAR_TO_NUM(opline->result.var));
}
- if (ssa && ssa->ops) {
- int ssa_var_num = ssa->ops[opline - op_array->opcodes].result_def;
+ if (ssa_op) {
+ int ssa_var_num = ssa_op->result_def;
if (ssa_var_num >= 0) {
fprintf(stderr, " -> ");
zend_dump_ssa_var(op_array, ssa, ssa_var_num, opline->result_type, EX_VAR_TO_NUM(opline->result.var), dump_flags);
static void zend_dump_op_line(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const void *data)
{
int len = 0;
+ const zend_ssa *ssa = NULL;
+ zend_ssa_op *ssa_op = NULL;
if (dump_flags & ZEND_DUMP_NUMERIC_OPLINES) {
len = fprintf(stderr, "%04u:", (uint32_t)(opline - op_array->opcodes));
}
fprintf(stderr, "%*c", 12-len, ' ');
- zend_dump_op(op_array, b, opline, dump_flags, data);
+ if (dump_flags & ZEND_DUMP_SSA) {
+ ssa = (const zend_ssa*)data;
+ if (ssa && ssa->ops) {
+ ssa_op = &ssa->ops[opline - op_array->opcodes];
+ }
+ }
+
+ zend_dump_op(op_array, b, opline, dump_flags, ssa, ssa_op);
fprintf(stderr, "\n");
}
BEGIN_EXTERN_C()
void zend_dump_op_array(const zend_op_array *op_array, uint32_t dump_flags, const char *msg, const void *data);
-void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const void *data);
+void zend_dump_op(const zend_op_array *op_array, const zend_basic_block *b, const zend_op *opline, uint32_t dump_flags, const zend_ssa *ssa, const zend_ssa_op *ssa_op);
void zend_dump_dominators(const zend_op_array *op_array, const zend_cfg *cfg);
void zend_dump_dfg(const zend_op_array *op_array, const zend_cfg *cfg, const zend_dfg *dfg);
void zend_dump_phi_placement(const zend_op_array *op_array, const zend_ssa *ssa);
void zend_dump_variables(const zend_op_array *op_array);
void zend_dump_ssa_variables(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t dump_flags);
+void zend_dump_ssa_var(const zend_op_array *op_array, const zend_ssa *ssa, int ssa_var_num, zend_uchar var_type, int var_num, uint32_t dump_flags);
void zend_dump_var(const zend_op_array *op_array, zend_uchar var_type, int var_num);
void zend_dump_op_array_name(const zend_op_array *op_array);
void zend_dump_const(const zval *zv);
#define UPDATE_SSA_TYPE(_type, _var) \
do { \
- uint32_t __type = (_type); \
+ uint32_t __type = (_type) & ~MAY_BE_GUARD; \
int __var = (_var); \
if (__type & MAY_BE_REF) { \
__type |= MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; \
} \
} \
if (ssa_var_info[__var].type != __type) { \
- if (ssa_var_info[__var].type & ~__type) { \
- emit_type_narrowing_warning(op_array, ssa, __var); \
- return FAILURE; \
+ if (update_worklist) { \
+ if (ssa_var_info[__var].type & ~__type) { \
+ emit_type_narrowing_warning(op_array, ssa, __var);\
+ return FAILURE; \
+ } \
+ ssa_var_info[__var].type = __type; \
+ add_usages(op_array, ssa, worklist, __var); \
+ } else { \
+ if (ssa_var_info[__var].type & ~__type) { \
+ return FAILURE; \
+ } \
+ ssa_var_info[__var].type = __type; \
} \
- ssa_var_info[__var].type = __type; \
- add_usages(op_array, ssa, worklist, __var); \
} \
/*zend_bitset_excl(worklist, var);*/ \
} \
ssa_var_info[var].is_instanceof != (_is_instanceof)) { \
ssa_var_info[var].ce = (_ce); \
ssa_var_info[var].is_instanceof = (_is_instanceof); \
- add_usages(op_array, ssa, worklist, var); \
+ if (update_worklist) { \
+ add_usages(op_array, ssa, worklist, var); \
+ } \
} \
/*zend_bitset_excl(worklist, var);*/ \
} \
return NULL;
}
-static zend_property_info *zend_fetch_prop_info(const zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int i)
+static zend_property_info *zend_fetch_prop_info(const zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, zend_ssa_op *ssa_op)
{
zend_property_info *prop_info = NULL;
if (opline->op2_type == IS_CONST) {
if (opline->op1_type == IS_UNUSED) {
ce = op_array->scope;
- } else if (ssa->ops[i].op1_use >= 0) {
- ce = ssa->var_info[ssa->ops[i].op1_use].ce;
+ } else if (ssa_op->op1_use >= 0) {
+ ce = ssa->var_info[ssa_op->op1_use].ce;
}
if (ce) {
prop_info = lookup_prop_info(ce,
return MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_RC1 | MAY_BE_RCN;
}
-static int zend_update_type_info(const zend_op_array *op_array,
- zend_ssa *ssa,
- const zend_script *script,
- zend_bitset worklist,
- int i,
- zend_long optimization_level)
+static zend_always_inline int _zend_update_type_info(
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_script *script,
+ zend_bitset worklist,
+ zend_op *opline,
+ zend_ssa_op *ssa_op,
+ const zend_op **ssa_opcodes,
+ zend_long optimization_level,
+ zend_bool update_worklist)
{
uint32_t t1, t2;
uint32_t tmp, orig;
- zend_op *opline = op_array->opcodes + i;
- zend_ssa_op *ssa_ops = ssa->ops;
zend_ssa_var *ssa_vars = ssa->vars;
zend_ssa_var_info *ssa_var_info = ssa->var_info;
zend_class_entry *ce;
if (opline->opcode == ZEND_OP_DATA) {
opline--;
- i--;
+ ssa_op--;
}
- t1 = OP1_INFO();
- t2 = OP2_INFO();
+ t1 = OP1_INFO_EX();
+ t2 = OP2_INFO_EX();
/* If one of the operands cannot have any type, this means the operand derives from
* unreachable code. Propagate the empty result early, so that that the following
if (!(t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_CLASS))
|| !(t2 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_CLASS))) {
tmp = 0;
- if (ssa_ops[i].result_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ if (ssa_op->result_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
- if (ssa_ops[i].op1_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ if (ssa_op->op1_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
- if (ssa_ops[i].op2_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def);
+ if (ssa_op->op2_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
}
return 1;
}
case ZEND_SL:
case ZEND_SR:
case ZEND_CONCAT:
- tmp = binary_op_result_type(ssa, opline->opcode, t1, t2, ssa_ops[i].result_def, optimization_level);
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ tmp = binary_op_result_type(ssa, opline->opcode, t1, t2, ssa_op->result_def, optimization_level);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_BW_NOT:
tmp = 0;
tmp |= MAY_BE_OBJECT | MAY_BE_RC1;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_BEGIN_SILENCE:
- UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_op->result_def);
break;
case ZEND_BOOL_NOT:
case ZEND_BOOL_XOR:
case ZEND_ASSERT_CHECK:
case ZEND_IN_ARRAY:
case ZEND_ARRAY_KEY_EXISTS:
- UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_op->result_def);
break;
case ZEND_CAST:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = t1;
if ((t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) &&
(opline->op1_type == IS_CV) &&
opline->extended_value == IS_STRING) {
tmp |= MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
tmp = 1 << opline->extended_value;
if (tmp & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
tmp |= ((t1 & MAY_BE_ANY) << MAY_BE_ARRAY_SHIFT) | ((t1 & MAY_BE_ANY)? MAY_BE_ARRAY_KEY_LONG : 0);
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_QM_ASSIGN:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_COPY_TMP:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = t1;
if ((t1 & (MAY_BE_RC1|MAY_BE_REF)) && (opline->op1_type == IS_CV)) {
tmp |= MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
tmp = t1 & ~(MAY_BE_UNDEF|MAY_BE_REF);
if (t1 & MAY_BE_UNDEF) {
tmp &= ~MAY_BE_FALSE;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->result_def);
break;
case ZEND_ASSIGN_OP:
case ZEND_ASSIGN_DIM_OP:
orig = 0;
tmp = 0;
if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
- prop_info = zend_fetch_prop_info(op_array, ssa, opline, i);
+ prop_info = zend_fetch_prop_info(op_array, ssa, opline, ssa_op);
orig = t1;
t1 = zend_fetch_prop_type(script, prop_info, &ce);
- t2 = OP1_DATA_INFO();
+ t2 = OP1_DATA_INFO_EX();
} else if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
if (t1 & MAY_BE_ARRAY_OF_REF) {
tmp |= MAY_BE_REF;
}
orig = t1;
t1 = zend_array_element_type(t1, 1, 0);
- t2 = OP1_DATA_INFO();
+ t2 = OP1_DATA_INFO_EX();
} else if (opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP) {
prop_info = zend_fetch_static_prop_info(script, op_array, ssa, opline);
t1 = zend_fetch_prop_type(script, prop_info, &ce);
- t2 = OP1_DATA_INFO();
+ t2 = OP1_DATA_INFO_EX();
} else {
if (t1 & MAY_BE_REF) {
tmp |= MAY_BE_REF;
}
tmp |= binary_op_result_type(
- ssa, opline->extended_value, t1, t2, ssa_ops[i].op1_def, optimization_level);
+ ssa, opline->extended_value, t1, t2, ssa_op->op1_def, optimization_level);
if (tmp & (MAY_BE_STRING|MAY_BE_ARRAY)) {
tmp |= MAY_BE_RC1;
}
if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
if (opline->op1_type == IS_CV) {
- orig = assign_dim_result_type(orig, OP2_INFO(), tmp, opline->op2_type);
- UPDATE_SSA_TYPE(orig, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ orig = assign_dim_result_type(orig, OP2_INFO_EX(), tmp, opline->op2_type);
+ UPDATE_SSA_TYPE(orig, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
} else if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
if (opline->op1_type == IS_CV) {
orig |= (MAY_BE_RC1|MAY_BE_RCN);
}
}
- UPDATE_SSA_TYPE(orig, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(orig, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
} else if (opline->opcode == ZEND_ASSIGN_STATIC_PROP) {
/* Nothing to do */
} else {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
ce = NULL;
if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
if (opline->op2_type == IS_UNUSED) {
}
}
tmp &= ~MAY_BE_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def);
}
}
break;
}
if (t1 & (MAY_BE_RC1|MAY_BE_RCN)) {
tmp |= MAY_BE_RC1;
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp |= MAY_BE_RCN;
}
}
if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
- if (!ssa_var_info[ssa_ops[i].op1_use].has_range ||
+ if (!ssa_var_info[ssa_op->op1_use].has_range ||
(opline->opcode == ZEND_PRE_DEC &&
- (ssa_var_info[ssa_ops[i].op1_use].range.underflow ||
- ssa_var_info[ssa_ops[i].op1_use].range.min == ZEND_LONG_MIN)) ||
+ (ssa_var_info[ssa_op->op1_use].range.underflow ||
+ ssa_var_info[ssa_op->op1_use].range.min == ZEND_LONG_MIN)) ||
(opline->opcode == ZEND_PRE_INC &&
- (ssa_var_info[ssa_ops[i].op1_use].range.overflow ||
- ssa_var_info[ssa_ops[i].op1_use].range.max == ZEND_LONG_MAX))) {
+ (ssa_var_info[ssa_op->op1_use].range.overflow ||
+ ssa_var_info[ssa_op->op1_use].range.max == ZEND_LONG_MAX))) {
/* may overflow */
tmp |= MAY_BE_LONG | MAY_BE_DOUBLE;
} else {
}
tmp |= t1 & (MAY_BE_FALSE | MAY_BE_TRUE | MAY_BE_RESOURCE | MAY_BE_ARRAY | MAY_BE_OBJECT | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_ARRAY_KEY_ANY);
}
- if (ssa_ops[i].op1_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ if (ssa_op->op1_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ if (ssa_op->result_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
case ZEND_POST_INC:
case ZEND_POST_DEC:
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp = 0;
if (t1 & (MAY_BE_RC1|MAY_BE_RCN)) {
tmp |= MAY_BE_RC1|MAY_BE_RCN;
if (t1 & MAY_BE_UNDEF) {
tmp |= MAY_BE_NULL;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
tmp = 0;
if (t1 & MAY_BE_REF) {
tmp |= MAY_BE_RC1;
}
if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
- if (!ssa_var_info[ssa_ops[i].op1_use].has_range ||
+ if (!ssa_var_info[ssa_op->op1_use].has_range ||
(opline->opcode == ZEND_POST_DEC &&
- (ssa_var_info[ssa_ops[i].op1_use].range.underflow ||
- ssa_var_info[ssa_ops[i].op1_use].range.min == ZEND_LONG_MIN)) ||
+ (ssa_var_info[ssa_op->op1_use].range.underflow ||
+ ssa_var_info[ssa_op->op1_use].range.min == ZEND_LONG_MIN)) ||
(opline->opcode == ZEND_POST_INC &&
- (ssa_var_info[ssa_ops[i].op1_use].range.overflow ||
- ssa_var_info[ssa_ops[i].op1_use].range.max == ZEND_LONG_MAX))) {
+ (ssa_var_info[ssa_op->op1_use].range.overflow ||
+ ssa_var_info[ssa_op->op1_use].range.max == ZEND_LONG_MAX))) {
/* may overflow */
tmp |= MAY_BE_LONG | MAY_BE_DOUBLE;
} else {
}
tmp |= t1 & (MAY_BE_FALSE | MAY_BE_TRUE | MAY_BE_RESOURCE | MAY_BE_ARRAY | MAY_BE_OBJECT | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF | MAY_BE_ARRAY_KEY_ANY);
}
- if (ssa_ops[i].op1_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ if (ssa_op->op1_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_ASSIGN_DIM:
if (opline->op1_type == IS_CV) {
- tmp = assign_dim_result_type(t1, t2, OP1_DATA_INFO(), opline->op2_type);
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ tmp = assign_dim_result_type(t1, t2, OP1_DATA_INFO_EX(), opline->op2_type);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp = 0;
if (t1 & MAY_BE_STRING) {
tmp |= MAY_BE_STRING;
}
if (t1 & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_STRING)) {
- tmp |= (OP1_DATA_INFO() & (MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF));
+ tmp |= (OP1_DATA_INFO_EX() & (MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF));
if (opline->op2_type == IS_UNUSED) {
/* When appending to an array and the LONG_MAX key is already used
if (t1 & MAY_BE_OBJECT) {
tmp |= MAY_BE_REF;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
- if ((opline+1)->op1_type == IS_CV && ssa_ops[i+1].op1_def >= 0) {
+ if ((opline+1)->op1_type == IS_CV && (ssa_op+1)->op1_def >= 0) {
opline++;
- i++;
- tmp = OP1_INFO();
+ ssa_op++;
+ tmp = OP1_INFO_EX();
if (tmp & (MAY_BE_ANY | MAY_BE_REF)) {
if (tmp & MAY_BE_RC1) {
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_ASSIGN_OBJ:
if (tmp & MAY_BE_OBJECT) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
// TODO: If there is no __set we might do better
tmp = zend_fetch_prop_type(script,
- zend_fetch_prop_info(op_array, ssa, opline, i), &ce);
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ zend_fetch_prop_info(op_array, ssa, opline, ssa_op), &ce);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def);
}
}
if ((opline+1)->op1_type == IS_CV) {
opline++;
- i++;
- tmp = OP1_INFO();
+ ssa_op++;
+ tmp = OP1_INFO_EX();
if (tmp & (MAY_BE_ANY | MAY_BE_REF)) {
if (tmp & MAY_BE_RC1) {
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_PRE_INC_OBJ:
if (tmp & MAY_BE_OBJECT) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
// TODO: ???
tmp = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
case ZEND_ASSIGN:
- if (opline->op2_type == IS_CV && ssa_ops[i].op2_def >= 0) {
+ if (opline->op2_type == IS_CV && ssa_op->op2_def >= 0) {
tmp = t2;
if (tmp & (MAY_BE_ANY | MAY_BE_REF)) {
if (tmp & MAY_BE_RC1) {
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
}
tmp = t2 & ~(MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
if (t2 & MAY_BE_UNDEF) {
if (RETURN_VALUE_USED(opline) && (tmp & MAY_BE_RC1)) {
tmp |= MAY_BE_RCN;
}
- if (ssa_ops[i].op1_def >= 0) {
- if (ssa_var_info[ssa_ops[i].op1_def].use_as_double) {
+ if (ssa_op->op1_def >= 0) {
+ if (ssa_var_info[ssa_op->op1_def].use_as_double) {
tmp &= ~MAY_BE_LONG;
tmp |= MAY_BE_DOUBLE;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
- UPDATE_SSA_TYPE(tmp & ~MAY_BE_REF, ssa_ops[i].result_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].result_def);
+ if (ssa_op->result_def >= 0) {
+ UPDATE_SSA_TYPE(tmp & ~MAY_BE_REF, ssa_op->result_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->result_def);
}
break;
case ZEND_ASSIGN_REF:
if (t2 & MAY_BE_UNDEF) {
tmp |= MAY_BE_NULL;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
}
if (opline->op2_type == IS_VAR && opline->extended_value == ZEND_RETURNS_FUNCTION) {
tmp = (MAY_BE_REF | MAY_BE_RCN | MAY_BE_RC1 | t2) & ~MAY_BE_UNDEF;
if (t2 & MAY_BE_UNDEF) {
tmp |= MAY_BE_NULL;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- if (ssa_ops[i].result_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ if (ssa_op->result_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
case ZEND_ASSIGN_OBJ_REF:
if (tmp & MAY_BE_OBJECT) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
- t2 = OP1_DATA_INFO();
+ t2 = OP1_DATA_INFO_EX();
if ((opline+1)->op1_type == IS_VAR && (opline->extended_value & ZEND_RETURNS_FUNCTION)) {
tmp = (MAY_BE_REF | MAY_BE_RCN | MAY_BE_RC1 | t2) & ~MAY_BE_UNDEF;
} else {
if (t2 & MAY_BE_UNDEF) {
tmp |= MAY_BE_NULL;
}
- if (ssa_ops[i].result_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ if (ssa_op->result_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
if ((opline+1)->op1_type == IS_CV) {
opline++;
- i++;
+ ssa_op++;
tmp = (MAY_BE_REF | t2) & ~(MAY_BE_UNDEF|MAY_BE_RC1|MAY_BE_RCN);
if (t2 & MAY_BE_UNDEF) {
tmp |= MAY_BE_NULL;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_ASSIGN_STATIC_PROP_REF:
if ((opline+1)->op1_type == IS_CV) {
opline++;
- i++;
- UPDATE_SSA_TYPE(MAY_BE_REF, ssa_ops[i].op1_def);
+ ssa_op++;
+ UPDATE_SSA_TYPE(MAY_BE_REF, ssa_op->op1_def);
}
break;
case ZEND_BIND_GLOBAL:
tmp = MAY_BE_REF | MAY_BE_ANY
| MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
break;
case ZEND_BIND_STATIC:
tmp = MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF
if (opline->extended_value & ZEND_BIND_IMPLICIT) {
tmp |= MAY_BE_UNDEF;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
break;
case ZEND_SEND_VAR:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = t1;
if ((t1 & (MAY_BE_RC1|MAY_BE_REF)) && (opline->op1_type == IS_CV)) {
tmp |= MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
break;
case ZEND_BIND_LEXICAL:
- if (ssa_ops[i].op2_def >= 0) {
+ if (ssa_op->op2_def >= 0) {
if (opline->extended_value & ZEND_BIND_REF) {
tmp = t2 | MAY_BE_REF;
} else {
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].op2_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->op2_def);
}
break;
case ZEND_YIELD:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
tmp = t1 | MAY_BE_REF;
} else {
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp = MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF
| MAY_BE_RC1 | MAY_BE_RCN;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
case ZEND_SEND_VAR_EX:
case ZEND_SEND_FUNC_ARG:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = (t1 & MAY_BE_UNDEF)|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_SEND_REF:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_SEND_UNPACK:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = t1;
if (t1 & MAY_BE_ARRAY) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN;
if (t1 & MAY_BE_OBJECT) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_FAST_CONCAT:
case ZEND_ROPE_INIT:
case ZEND_ROPE_ADD:
case ZEND_ROPE_END:
- UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_op->result_def);
break;
case ZEND_RECV:
case ZEND_RECV_INIT:
}
#if 0
/* We won't receive unused arguments */
- if (ssa_vars[ssa_ops[i].result_def].use_chain < 0 &&
- ssa_vars[ssa_ops[i].result_def].phi_use_chain == NULL &&
+ if (ssa_vars[ssa_op->result_def].use_chain < 0 &&
+ ssa_vars[ssa_op->result_def].phi_use_chain == NULL &&
op_array->arg_info &&
opline->op1.num <= op_array->num_args &&
op_array->arg_info[opline->op1.num-1].class_name == NULL &&
tmp = MAY_BE_UNDEF|MAY_BE_RCN;
}
#endif
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (func_info &&
(int)opline->op1.num-1 < func_info->num_args &&
func_info->arg_info[opline->op1.num-1].info.ce) {
UPDATE_SSA_OBJ_TYPE(
func_info->arg_info[opline->op1.num-1].info.ce,
func_info->arg_info[opline->op1.num-1].info.is_instanceof,
- ssa_ops[i].result_def);
+ ssa_op->result_def);
} else if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
}
break;
}
case ZEND_DECLARE_ANON_CLASS:
- UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_op->result_def);
if (script && (ce = zend_hash_find_ptr(&script->class_table, Z_STR_P(CRT_CONSTANT(opline->op1)))) != NULL) {
- UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_op->result_def);
}
break;
case ZEND_FETCH_CLASS:
- UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_CLASS, ssa_op->result_def);
if (opline->op2_type == IS_UNUSED) {
switch (opline->op1.num & ZEND_FETCH_CLASS_MASK) {
case ZEND_FETCH_CLASS_SELF:
if (op_array->scope) {
- UPDATE_SSA_OBJ_TYPE(op_array->scope, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(op_array->scope, 0, ssa_op->result_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
}
break;
case ZEND_FETCH_CLASS_PARENT:
if (op_array->scope && op_array->scope->parent && (op_array->scope->ce_flags & ZEND_ACC_LINKED)) {
- UPDATE_SSA_OBJ_TYPE(op_array->scope->parent, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(op_array->scope->parent, 0, ssa_op->result_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
}
break;
case ZEND_FETCH_CLASS_STATIC:
default:
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
break;
}
} else if (opline->op2_type == IS_CONST) {
zval *zv = CRT_CONSTANT(opline->op2);
if (Z_TYPE_P(zv) == IS_STRING) {
ce = get_class_entry(script, Z_STR_P(zv+1));
- UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_op->result_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
}
} else {
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op2_use, ssa_ops[i].result_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op2_use, ssa_op->result_def);
}
break;
case ZEND_NEW:
tmp = MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT;
if (opline->op1_type == IS_CONST &&
(ce = get_class_entry(script, Z_STR_P(CRT_CONSTANT(opline->op1)+1))) != NULL) {
- UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_ops[i].result_def);
- } else if ((t1 & MAY_BE_CLASS) && ssa_ops[i].op1_use >= 0 && ssa_var_info[ssa_ops[i].op1_use].ce) {
- UPDATE_SSA_OBJ_TYPE(ssa_var_info[ssa_ops[i].op1_use].ce, ssa_var_info[ssa_ops[i].op1_use].is_instanceof, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 0, ssa_op->result_def);
+ } else if ((t1 & MAY_BE_CLASS) && ssa_op->op1_use >= 0 && ssa_var_info[ssa_op->op1_use].ce) {
+ UPDATE_SSA_OBJ_TYPE(ssa_var_info[ssa_op->op1_use].ce, ssa_var_info[ssa_op->op1_use].is_instanceof, ssa_op->result_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_CLONE:
- UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_ops[i].result_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_op->result_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->result_def);
break;
case ZEND_INIT_ARRAY:
case ZEND_ADD_ARRAY_ELEMENT:
- if (opline->op1_type == IS_CV && ssa_ops[i].op1_def >= 0) {
+ if (opline->op1_type == IS_CV && ssa_op->op1_def >= 0) {
if (opline->extended_value & ZEND_ARRAY_ELEMENT_REF) {
tmp = (MAY_BE_REF | t1) & ~(MAY_BE_UNDEF|MAY_BE_RC1|MAY_BE_RCN);
if (t1 & MAY_BE_UNDEF) {
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp = MAY_BE_RC1|MAY_BE_ARRAY;
- if (ssa_ops[i].result_use >= 0) {
- tmp |= ssa_var_info[ssa_ops[i].result_use].type;
+ if (ssa_op->result_use >= 0) {
+ tmp |= ssa_var_info[ssa_op->result_use].type;
}
if (opline->op1_type != IS_UNUSED) {
tmp |= (t1 & MAY_BE_ANY) << MAY_BE_ARRAY_SHIFT;
}
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
case ZEND_ADD_ARRAY_UNPACK:
- tmp = ssa_var_info[ssa_ops[i].result_use].type;
+ tmp = ssa_var_info[ssa_op->result_use].type;
ZEND_ASSERT(tmp & MAY_BE_ARRAY);
/* Ignore string keys as they will throw. */
if (t1 & MAY_BE_ARRAY_KEY_LONG) {
if (t1 & MAY_BE_OBJECT) {
tmp |= MAY_BE_ARRAY_KEY_LONG | MAY_BE_ARRAY_OF_ANY;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_UNSET_CV:
tmp = MAY_BE_UNDEF;
/* In global scope, we know nothing */
tmp |= MAY_BE_REF;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
break;
case ZEND_UNSET_DIM:
case ZEND_UNSET_OBJ:
- if (ssa_ops[i].op1_def >= 0) {
- UPDATE_SSA_TYPE(t1, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ if (ssa_op->op1_def >= 0) {
+ UPDATE_SSA_TYPE(t1, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
break;
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = t1;
if (opline->opcode == ZEND_FE_RESET_RW) {
tmp |= MAY_BE_REF;
tmp |= MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
if (opline->opcode == ZEND_FE_RESET_RW) {
//???
} else {
tmp = MAY_BE_RC1 | MAY_BE_RCN | (t1 & (MAY_BE_ARRAY | MAY_BE_OBJECT | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF));
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->result_def);
break;
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
}
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op2_def);
- if (ssa_ops[i].result_def >= 0) {
- tmp = (ssa_ops[i].result_use >= 0) ? RES_USE_INFO() : 0;
+ UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
+ if (ssa_op->result_def >= 0) {
+ tmp = (ssa_op->result_use >= 0) ? RES_USE_INFO_EX() : 0;
if (t1 & MAY_BE_OBJECT) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
tmp |= MAY_BE_STRING | MAY_BE_RCN;
}
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
case ZEND_FETCH_DIM_R:
case ZEND_FETCH_DIM_FUNC_ARG:
case ZEND_FETCH_LIST_R:
case ZEND_FETCH_LIST_W:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
uint32_t key_type = 0;
tmp = t1 & ~(MAY_BE_RC1|MAY_BE_RCN);
if (opline->opcode == ZEND_FETCH_DIM_W ||
|| opline->opcode == ZEND_FETCH_DIM_W
|| opline->opcode == ZEND_FETCH_DIM_FUNC_ARG
|| opline->opcode == ZEND_FETCH_LIST_W) {
- j = ssa_vars[ssa_ops[i].result_def].use_chain;
+ j = ssa_vars[ssa_op->result_def].use_chain;
while (j >= 0) {
- ZEND_ASSERT(j == i + 1 && "Use must be in next opline");
- switch (op_array->opcodes[j].opcode) {
+ zend_uchar opcode;
+
+ if (!ssa_opcodes) {
+ ZEND_ASSERT(j == (opline - op_array->opcodes) + 1 && "Use must be in next opline");
+ opcode = op_array->opcodes[j].opcode;
+ } else {
+ ZEND_ASSERT(ssa_opcodes[j] == opline + 1 && "Use must be in next opline");
+ opcode = ssa_opcodes[j]->opcode;
+ }
+ switch (opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_DIM_RW:
case ZEND_FETCH_DIM_FUNC_ARG:
break;
EMPTY_SWITCH_DEFAULT_CASE()
}
- j = zend_ssa_next_use(ssa_ops, ssa_ops[i].result_def, j);
+ j = zend_ssa_next_use(ssa->ops, ssa_op->result_def, j);
ZEND_ASSERT(j < 0 && "There should only be one use");
}
}
if ((tmp & MAY_BE_ARRAY) && (tmp & MAY_BE_ARRAY_KEY_ANY)) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
} else {
/* invalid key type */
tmp = (tmp & (MAY_BE_RC1|MAY_BE_RCN)) | (t1 & ~(MAY_BE_RC1|MAY_BE_RCN));
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
- COPY_SSA_OBJ_TYPE(ssa_ops[i].op1_use, ssa_ops[i].op1_def);
+ COPY_SSA_OBJ_TYPE(ssa_op->op1_use, ssa_op->op1_def);
}
/* FETCH_LIST on a string behaves like FETCH_R on null */
tmp = zend_array_element_type(
if (opline->opcode == ZEND_FETCH_DIM_IS && (t1 & MAY_BE_STRING)) {
tmp |= MAY_BE_NULL;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_FETCH_THIS:
- UPDATE_SSA_OBJ_TYPE(op_array->scope, 1, ssa_ops[i].result_def);
- UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(op_array->scope, 1, ssa_op->result_def);
+ UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_OBJECT, ssa_op->result_def);
break;
case ZEND_FETCH_OBJ_R:
case ZEND_FETCH_OBJ_IS:
case ZEND_FETCH_OBJ_W:
case ZEND_FETCH_OBJ_UNSET:
case ZEND_FETCH_OBJ_FUNC_ARG:
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp = zend_fetch_prop_type(script,
- zend_fetch_prop_info(op_array, ssa, opline, i), &ce);
+ zend_fetch_prop_info(op_array, ssa, opline, ssa_op), &ce);
if (opline->result_type != IS_TMP_VAR) {
tmp |= MAY_BE_REF;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def);
}
}
break;
if (opline->result_type != IS_TMP_VAR) {
tmp |= MAY_BE_REF;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def);
}
break;
case ZEND_DO_FCALL:
case ZEND_DO_ICALL:
case ZEND_DO_UCALL:
case ZEND_DO_FCALL_BY_NAME:
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info;
goto unknown_opcode;
}
tmp = zend_get_func_info(call_info, ssa);
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (call_info->callee_func->type == ZEND_USER_FUNCTION) {
func_info = ZEND_FUNC_INFO(&call_info->callee_func->op_array);
if (func_info) {
UPDATE_SSA_OBJ_TYPE(
func_info->return_info.ce,
func_info->return_info.is_instanceof,
- ssa_ops[i].result_def);
+ ssa_op->result_def);
}
}
}
break;
case ZEND_FETCH_CONSTANT:
case ZEND_FETCH_CLASS_CONSTANT:
- UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_RC1|MAY_BE_RCN|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY, ssa_op->result_def);
break;
case ZEND_STRLEN:
tmp = MAY_BE_LONG;
if (t1 & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING))) {
tmp |= MAY_BE_NULL;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
break;
case ZEND_COUNT:
case ZEND_FUNC_NUM_ARGS:
- UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_LONG, ssa_op->result_def);
break;
case ZEND_FUNC_GET_ARGS:
- UPDATE_SSA_TYPE(MAY_BE_RC1| MAY_BE_ARRAY | MAY_BE_ARRAY_KEY_LONG | MAY_BE_ARRAY_OF_ANY, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_RC1| MAY_BE_ARRAY | MAY_BE_ARRAY_KEY_LONG | MAY_BE_ARRAY_OF_ANY, ssa_op->result_def);
break;
case ZEND_GET_CLASS:
case ZEND_GET_CALLED_CLASS:
- UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_STRING|MAY_BE_RCN, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_STRING|MAY_BE_RCN, ssa_op->result_def);
break;
case ZEND_GET_TYPE:
- UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_STRING|MAY_BE_RC1|MAY_BE_RCN, ssa_op->result_def);
break;
case ZEND_TYPE_CHECK:
case ZEND_DEFINED:
- UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(MAY_BE_FALSE|MAY_BE_TRUE, ssa_op->result_def);
break;
case ZEND_VERIFY_RETURN_TYPE:
if (t1 & MAY_BE_REF) {
tmp = zend_fetch_arg_info_type(script, ret_info, &ce);
}
if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].op1_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->op1_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].op1_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->op1_def);
}
} else {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
if (ce) {
- UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(ce, 1, ssa_op->result_def);
} else {
- UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_ops[i].result_def);
+ UPDATE_SSA_OBJ_TYPE(NULL, 0, ssa_op->result_def);
}
}
break;
case ZEND_MAKE_REF:
tmp = MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_KEY_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
- if (ssa_ops[i].op1_def >= 0) {
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
+ if (ssa_op->op1_def >= 0) {
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
break;
case ZEND_CATCH:
break;
default:
unknown_opcode:
- if (ssa_ops[i].op1_def >= 0) {
+ if (ssa_op->op1_def >= 0) {
tmp = MAY_BE_ANY | MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].op1_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->op1_def);
}
- if (ssa_ops[i].result_def >= 0) {
+ if (ssa_op->result_def >= 0) {
tmp = MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
if (opline->result_type == IS_TMP_VAR) {
tmp |= MAY_BE_RC1 | MAY_BE_RCN;
} else {
tmp |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN;
}
- UPDATE_SSA_TYPE(tmp, ssa_ops[i].result_def);
+ UPDATE_SSA_TYPE(tmp, ssa_op->result_def);
}
break;
}
return SUCCESS;
}
+int zend_update_type_info(
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_script *script,
+ zend_op *opline,
+ zend_ssa_op *ssa_op,
+ const zend_op **ssa_opcodes,
+ zend_long optimization_level)
+{
+ return _zend_update_type_info(op_array, ssa, script, NULL, opline, ssa_op, ssa_opcodes, optimization_level, 0);
+}
+
static uint32_t get_class_entry_rank(zend_class_entry *ce) {
uint32_t rank = 0;
if (ce->ce_flags & ZEND_ACC_LINKED) {
int ssa_vars_count = ssa->vars_count;
int i, j;
uint32_t tmp, worklist_len = zend_bitset_len(ssa_vars_count);
+ zend_bool update_worklist = 1;
while (!zend_bitset_empty(worklist, worklist_len)) {
j = zend_bitset_first(worklist, worklist_len);
}
} else if (ssa_vars[j].definition >= 0) {
i = ssa_vars[j].definition;
- if (zend_update_type_info(op_array, ssa, script, worklist, i, optimization_level) == FAILURE) {
+ if (_zend_update_type_info(op_array, ssa, script, worklist, op_array->opcodes + i, ssa->ops + i, NULL, optimization_level, 1) == FAILURE) {
return FAILURE;
}
}
free_alloca(worklist, use_heap);
}
-int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+int zend_may_throw_ex(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
{
- uint32_t t1 = OP1_INFO();
- uint32_t t2 = OP2_INFO();
+ uint32_t t1 = OP1_INFO_EX();
+ uint32_t t2 = OP2_INFO_EX();
if (opline->op1_type == IS_CV) {
if (t1 & MAY_BE_UNDEF) {
return 0;
case ZEND_BIND_GLOBAL:
if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
- return zend_may_throw(opline + 1, op_array, ssa);
+ return zend_may_throw_ex(opline + 1, ssa_op + 1, op_array, ssa);
}
return 0;
case ZEND_ADD:
(t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT));
case ZEND_DIV:
case ZEND_MOD:
- if (!OP2_HAS_RANGE() ||
- (OP2_MIN_RANGE() <= 0 && OP2_MAX_RANGE() >= 0)) {
+ if (!OP2_HAS_RANGE_EX() ||
+ (OP2_MIN_RANGE_EX() <= 0 && OP2_MAX_RANGE_EX() >= 0)) {
/* Division by zero */
return 1;
}
case ZEND_SR:
return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) ||
(t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) ||
- !OP2_HAS_RANGE() ||
- OP2_MIN_RANGE() < 0;
+ !OP2_HAS_RANGE_EX() ||
+ OP2_MIN_RANGE_EX() < 0;
case ZEND_CONCAT:
case ZEND_FAST_CONCAT:
return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) ||
(t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT));
} else if (opline->extended_value == ZEND_DIV ||
opline->extended_value == ZEND_MOD) {
- if (!OP2_HAS_RANGE() ||
- (OP2_MIN_RANGE() <= 0 && OP2_MAX_RANGE() >= 0)) {
+ if (!OP2_HAS_RANGE_EX() ||
+ (OP2_MIN_RANGE_EX() <= 0 && OP2_MAX_RANGE_EX() >= 0)) {
/* Division by zero */
return 1;
}
opline->extended_value == ZEND_SR) {
return (t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) ||
(t2 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT)) ||
- !OP2_HAS_RANGE() ||
- OP2_MIN_RANGE() < 0;
+ !OP2_HAS_RANGE_EX() ||
+ OP2_MIN_RANGE_EX() < 0;
} else if (opline->extended_value == ZEND_CONCAT) {
return (t1 & (MAY_BE_ARRAY|MAY_BE_OBJECT)) ||
(t2 & (MAY_BE_ARRAY|MAY_BE_OBJECT));
return (t1 & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY));
case ZEND_ASSIGN_DIM:
if ((opline+1)->op1_type == IS_CV) {
- if (_ssa_op1_info(op_array, ssa, opline+1) & MAY_BE_UNDEF) {
+ if (_ssa_op1_info_ex(op_array, ssa, opline+1, ssa_op+1) & MAY_BE_UNDEF) {
return 1;
}
}
if (t1 & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_OBJECT))) {
return 1;
}
- if (ssa->ops[opline - op_array->opcodes].op1_use) {
- zend_ssa_var_info *var_info = ssa->var_info + ssa->ops[opline - op_array->opcodes].op1_use;
+ if (ssa_op->op1_use) {
+ zend_ssa_var_info *var_info = ssa->var_info + ssa_op->op1_use;
zend_class_entry *ce = var_info->ce;
if (var_info->is_instanceof ||
return 1;
}
}
+
+int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+{
+ return zend_may_throw_ex(opline, &ssa->ops[opline - op_array->opcodes], op_array, ssa);
+}
/* Bitmask for type inference (zend_ssa_var_info.type) */
#include "zend_type_info.h"
+#define MAY_BE_GUARD (1<<28) /* needs type guard */
#define MAY_BE_IN_REG (1<<29) /* value allocated in CPU register */
//TODO: remome MAY_BE_RC1, MAY_BE_RCN???
|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE)
#define DEFINE_SSA_OP_HAS_RANGE(opN) \
- static zend_always_inline zend_bool _ssa_##opN##_has_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ static zend_always_inline zend_bool _ssa_##opN##_has_range_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
{ \
if (opline->opN##_type == IS_CONST) { \
zval *zv = CRT_CONSTANT(opline->opN); \
return (Z_TYPE_P(zv) == IS_LONG || Z_TYPE_P(zv) == IS_TRUE || Z_TYPE_P(zv) == IS_FALSE || Z_TYPE_P(zv) == IS_NULL); \
} else { \
return (opline->opN##_type != IS_UNUSED && \
- ssa->ops && \
ssa->var_info && \
- ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \
- ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range); \
+ ssa_op->opN##_use >= 0 && \
+ ssa->var_info[ssa_op->opN##_use].has_range); \
} \
return 0; \
+ } \
+ static zend_always_inline zend_bool _ssa_##opN##_has_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ { \
+ return _ssa_##opN##_has_range_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
#define DEFINE_SSA_OP_MIN_RANGE(opN) \
- static zend_always_inline zend_long _ssa_##opN##_min_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ static zend_always_inline zend_long _ssa_##opN##_min_range_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
{ \
if (opline->opN##_type == IS_CONST) { \
zval *zv = CRT_CONSTANT(opline->opN); \
return 0; \
} \
} else if (opline->opN##_type != IS_UNUSED && \
- ssa->ops && \
ssa->var_info && \
- ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \
- ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \
- return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.min; \
+ ssa_op->opN##_use >= 0 && \
+ ssa->var_info[ssa_op->opN##_use].has_range) { \
+ return ssa->var_info[ssa_op->opN##_use].range.min; \
} \
return ZEND_LONG_MIN; \
+ } \
+ static zend_always_inline zend_long _ssa_##opN##_min_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ { \
+ return _ssa_##opN##_min_range_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
#define DEFINE_SSA_OP_MAX_RANGE(opN) \
- static zend_always_inline zend_long _ssa_##opN##_max_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ static zend_always_inline zend_long _ssa_##opN##_max_range_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
{ \
if (opline->opN##_type == IS_CONST) { \
zval *zv = CRT_CONSTANT(opline->opN); \
return 0; \
} \
} else if (opline->opN##_type != IS_UNUSED && \
- ssa->ops && \
ssa->var_info && \
- ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \
- ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \
- return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.max; \
+ ssa_op->opN##_use >= 0 && \
+ ssa->var_info[ssa_op->opN##_use].has_range) { \
+ return ssa->var_info[ssa_op->opN##_use].range.max; \
} \
return ZEND_LONG_MAX; \
+ } \
+ static zend_always_inline zend_long _ssa_##opN##_max_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ { \
+ return _ssa_##opN##_max_range_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
#define DEFINE_SSA_OP_RANGE_UNDERFLOW(opN) \
- static zend_always_inline char _ssa_##opN##_range_underflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ static zend_always_inline char _ssa_##opN##_range_underflow_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
{ \
if (opline->opN##_type == IS_CONST) { \
zval *zv = CRT_CONSTANT(opline->opN); \
return 0; \
} \
} else if (opline->opN##_type != IS_UNUSED && \
- ssa->ops && \
ssa->var_info && \
- ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \
- ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \
- return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.underflow; \
+ ssa_op->opN##_use >= 0 && \
+ ssa->var_info[ssa_op->opN##_use].has_range) { \
+ return ssa->var_info[ssa_op->opN##_use].range.underflow; \
} \
return 1; \
+ } \
+ static zend_always_inline char _ssa_##opN##_range_underflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ { \
+ return _ssa_##opN##_range_underflow_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
#define DEFINE_SSA_OP_RANGE_OVERFLOW(opN) \
- static zend_always_inline char _ssa_##opN##_range_overflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ static zend_always_inline char _ssa_##opN##_range_overflow_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
{ \
if (opline->opN##_type == IS_CONST) { \
zval *zv = CRT_CONSTANT(opline->opN); \
return 0; \
} \
} else if (opline->opN##_type != IS_UNUSED && \
- ssa->ops && \
ssa->var_info && \
- ssa->ops[opline - op_array->opcodes].opN##_use >= 0 && \
- ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].has_range) { \
- return ssa->var_info[ssa->ops[opline - op_array->opcodes].opN##_use].range.overflow; \
+ ssa_op->opN##_use >= 0 && \
+ ssa->var_info[ssa_op->opN##_use].has_range) { \
+ return ssa->var_info[ssa_op->opN##_use].range.overflow; \
} \
return 1; \
+ } \
+ static zend_always_inline char _ssa_##opN##_range_overflow(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ { \
+ return _ssa_##opN##_range_overflow_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
DEFINE_SSA_OP_HAS_RANGE(op1)
#define OP2_RANGE_UNDERFLOW() (_ssa_op2_range_underflow (op_array, ssa, opline))
#define OP2_RANGE_OVERFLOW() (_ssa_op2_range_overflow (op_array, ssa, opline))
+#define OP1_HAS_RANGE_EX() (_ssa_op1_has_range_ex (op_array, ssa, opline, ssa_op))
+#define OP1_MIN_RANGE_EX() (_ssa_op1_min_range_ex (op_array, ssa, opline, ssa_op))
+#define OP1_MAX_RANGE_EX() (_ssa_op1_max_range_ex (op_array, ssa, opline, ssa_op))
+#define OP1_RANGE_UNDERFLOW_EX() (_ssa_op1_range_underflow_ex (op_array, ssa, opline, ssa_op))
+#define OP1_RANGE_OVERFLOW_EX() (_ssa_op1_range_overflow_ex (op_array, ssa, opline, ssa_op))
+#define OP2_HAS_RANGE_EX() (_ssa_op2_has_range_ex (op_array, ssa, opline, ssa_op))
+#define OP2_MIN_RANGE_EX() (_ssa_op2_min_range_ex (op_array, ssa, opline, ssa_op))
+#define OP2_MAX_RANGE_EX() (_ssa_op2_max_range_ex (op_array, ssa, opline, ssa_op))
+#define OP2_RANGE_UNDERFLOW_EX() (_ssa_op2_range_underflow_ex (op_array, ssa, opline, ssa_op))
+#define OP2_RANGE_OVERFLOW_EX() (_ssa_op2_range_overflow_ex (op_array, ssa, opline, ssa_op))
+
static zend_always_inline uint32_t _const_op_type(const zval *zv) {
if (Z_TYPE_P(zv) == IS_CONSTANT_AST) {
return MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY;
}
#define DEFINE_SSA_OP_INFO(opN) \
- static zend_always_inline uint32_t _ssa_##opN##_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ static zend_always_inline uint32_t _ssa_##opN##_info_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
{ \
if (opline->opN##_type == IS_CONST) { \
return _const_op_type(CRT_CONSTANT(opline->opN)); \
} else { \
- return get_ssa_var_info(ssa, ssa->ops ? ssa->ops[opline - op_array->opcodes].opN##_use : -1); \
+ return get_ssa_var_info(ssa, ssa->var_info ? ssa_op->opN##_use : -1); \
} \
+ } \
+ static zend_always_inline uint32_t _ssa_##opN##_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
+ { \
+ return _ssa_##opN##_info_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
#define DEFINE_SSA_OP_DEF_INFO(opN) \
+ static zend_always_inline uint32_t _ssa_##opN##_def_info_ex(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op) \
+ { \
+ return get_ssa_var_info(ssa, ssa->var_info ? ssa_op->opN##_def : -1); \
+ } \
static zend_always_inline uint32_t _ssa_##opN##_def_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *opline) \
{ \
- return get_ssa_var_info(ssa, ssa->ops ? ssa->ops[opline - op_array->opcodes].opN##_def : -1); \
+ return _ssa_##opN##_def_info_ex(op_array, ssa, opline, ssa->ops + (opline - op_array->opcodes)); \
}
#define OP2_DATA_DEF_INFO() (_ssa_op2_def_info(op_array, ssa, (opline+1)))
#define RES_INFO() (_ssa_result_def_info(op_array, ssa, opline))
+#define OP1_INFO_EX() (_ssa_op1_info_ex(op_array, ssa, opline, ssa_op))
+#define OP2_INFO_EX() (_ssa_op2_info_ex(op_array, ssa, opline, ssa_op))
+#define OP1_DATA_INFO_EX() (_ssa_op1_info_ex(op_array, ssa, (opline+1), (ssa_op+1)))
+#define OP2_DATA_INFO_EX() (_ssa_op2_info_ex(op_array, ssa, (opline+1), (ssa_op+1)))
+#define RES_USE_INFO_EX() (_ssa_result_info_ex(op_array, ssa, opline, ssa_op))
+#define OP1_DEF_INFO_EX() (_ssa_op1_def_info_ex(op_array, ssa, opline, ssa_op))
+#define OP2_DEF_INFO_EX() (_ssa_op2_def_info_ex(op_array, ssa, opline, ssa_op))
+#define OP1_DATA_DEF_INFO_EX() (_ssa_op1_def_info_ex(op_array, ssa, (opline+1), (ssa_op+1)))
+#define OP2_DATA_DEF_INFO_EX() (_ssa_op2_def_info_ex(op_array, ssa, (opline+1), (ssa_op+1)))
+#define RES_INFO_EX() (_ssa_result_def_info_ex(op_array, ssa, opline, ssa_op))
+
static zend_always_inline zend_bool zend_add_will_overflow(zend_long a, zend_long b) {
return (b > 0 && a > ZEND_LONG_MAX - b)
|| (b < 0 && a < ZEND_LONG_MIN - b);
int widening,
zend_ssa_var_info *ret);
+int zend_may_throw_ex(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa);
int zend_may_throw(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa);
+int zend_update_type_info(const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_script *script,
+ zend_op *opline,
+ zend_ssa_op *ssa_op,
+ const zend_op **ssa_opcodes,
+ zend_long optimization_level);
+
END_EXTERN_C()
#endif /* ZEND_INFERENCE_H */
}
/* }}} */
+static zend_always_inline int _zend_ssa_rename_op(const zend_op_array *op_array, const zend_op *opline, uint32_t k, uint32_t build_flags, int ssa_vars_count, zend_ssa_op *ssa_ops, int *var) /* {{{ */
+{
+ const zend_op *next;
+
+ if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k].op1_use = var[EX_VAR_TO_NUM(opline->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + opline->op1.var)
+ }
+ if (opline->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k].op2_use = var[EX_VAR_TO_NUM(opline->op2.var)];
+ //USE_SSA_VAR(op_array->last_var + opline->op2.var)
+ }
+ if ((build_flags & ZEND_SSA_USE_CV_RESULTS)
+ && opline->result_type == IS_CV
+ && opline->opcode != ZEND_RECV) {
+ ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)];
+ //USE_SSA_VAR(op_array->last_var + opline->result.var)
+ }
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN:
+ if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op2_type == IS_CV) {
+ ssa_ops[k].op2_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(opline->op2.var)
+ }
+ if (opline->op1_type == IS_CV) {
+add_op1_def:
+ ssa_ops[k].op1_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(opline->op1.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(opline->op1.var)
+ }
+ break;
+ case ZEND_ASSIGN_REF:
+ if (opline->op2_type == IS_CV) {
+ ssa_ops[k].op2_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(opline->op2.var)
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_DIM:
+ case ZEND_ASSIGN_OBJ:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + next->op1.var);
+ if (build_flags & ZEND_SSA_RC_INFERENCE && next->op1_type == IS_CV) {
+ ssa_ops[k + 1].op1_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(next->op1.var)
+ }
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_OBJ_REF:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + next->op1.var);
+ if (next->op1_type == IS_CV) {
+ ssa_ops[k + 1].op1_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(next->op1.var)
+ }
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_STATIC_PROP:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + next->op1.var);
+#if 0
+ if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) {
+ ssa_ops[k + 1].op1_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(next->op1.var)
+ }
+#endif
+ }
+ break;
+ case ZEND_ASSIGN_STATIC_PROP_REF:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + next->op1.var);
+ if (next->op1_type == IS_CV) {
+ ssa_ops[k + 1].op1_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(next->op1.var)
+ }
+ }
+ break;
+ case ZEND_ASSIGN_STATIC_PROP_OP:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + next->op1.var);
+ }
+ break;
+ case ZEND_ASSIGN_DIM_OP:
+ case ZEND_ASSIGN_OBJ_OP:
+ next = opline + 1;
+ if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+ //USE_SSA_VAR(op_array->last_var + next->op1.var);
+ }
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ASSIGN_OP:
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ case ZEND_BIND_GLOBAL:
+ case ZEND_BIND_STATIC:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_FUNC_ARG:
+ case ZEND_SEND_REF:
+ case ZEND_SEND_UNPACK:
+ case ZEND_FE_RESET_RW:
+ case ZEND_MAKE_REF:
+ case ZEND_PRE_INC_OBJ:
+ case ZEND_PRE_DEC_OBJ:
+ case ZEND_POST_INC_OBJ:
+ case ZEND_POST_DEC_OBJ:
+ case ZEND_UNSET_DIM:
+ case ZEND_UNSET_OBJ:
+ case ZEND_FETCH_DIM_W:
+ case ZEND_FETCH_DIM_RW:
+ case ZEND_FETCH_DIM_FUNC_ARG:
+ case ZEND_FETCH_DIM_UNSET:
+ case ZEND_FETCH_LIST_W:
+ if (opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_SEND_VAR:
+ case ZEND_CAST:
+ case ZEND_QM_ASSIGN:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_FE_RESET_R:
+ if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_ADD_ARRAY_UNPACK:
+ ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)];
+ break;
+ case ZEND_ADD_ARRAY_ELEMENT:
+ ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)];
+ /* break missing intentionally */
+ case ZEND_INIT_ARRAY:
+ if (((build_flags & ZEND_SSA_RC_INFERENCE)
+ || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF))
+ && opline->op1_type == IS_CV) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_YIELD:
+ if (opline->op1_type == IS_CV
+ && ((op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE)
+ || (build_flags & ZEND_SSA_RC_INFERENCE))) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_UNSET_CV:
+ goto add_op1_def;
+ case ZEND_VERIFY_RETURN_TYPE:
+ if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
+ goto add_op1_def;
+ }
+ break;
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ if (opline->op2_type != IS_CV) {
+ ssa_ops[k].op2_use = -1; /* not used */
+ }
+ ssa_ops[k].op2_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(opline->op2.var)
+ break;
+ case ZEND_BIND_LEXICAL:
+ if ((opline->extended_value & ZEND_BIND_REF) || (build_flags & ZEND_SSA_RC_INFERENCE)) {
+ ssa_ops[k].op2_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(opline->op2.var)
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
+ ssa_ops[k].result_def = ssa_vars_count;
+ var[EX_VAR_TO_NUM(opline->result.var)] = ssa_vars_count;
+ ssa_vars_count++;
+ //NEW_SSA_VAR(op_array->last_var + opline->result.var)
+ }
+
+ return ssa_vars_count;
+}
+/* }}} */
+
+int zend_ssa_rename_op(const zend_op_array *op_array, const zend_op *opline, uint32_t k, uint32_t build_flags, int ssa_vars_count, zend_ssa_op *ssa_ops, int *var) /* {{{ */
+{
+ return _zend_ssa_rename_op(op_array, opline, k, build_flags, ssa_vars_count, ssa_ops, var);
+}
+/* }}} */
+
static int zend_ssa_rename(const zend_op_array *op_array, uint32_t build_flags, zend_ssa *ssa, int *var, int n) /* {{{ */
{
zend_basic_block *blocks = ssa->cfg.blocks;
zend_ssa_op *ssa_ops = ssa->ops;
int ssa_vars_count = ssa->vars_count;
int i, j;
- zend_op *opline, *end, *next;
+ zend_op *opline, *end;
int *tmp = NULL;
ALLOCA_FLAG(use_heap = 0);
for (; opline < end; opline++) {
uint32_t k = opline - op_array->opcodes;
if (opline->opcode != ZEND_OP_DATA) {
-
- if (opline->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k].op1_use = var[EX_VAR_TO_NUM(opline->op1.var)];
- //USE_SSA_VAR(op_array->last_var + opline->op1.var)
- }
- if (opline->op2_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k].op2_use = var[EX_VAR_TO_NUM(opline->op2.var)];
- //USE_SSA_VAR(op_array->last_var + opline->op2.var)
- }
- if ((build_flags & ZEND_SSA_USE_CV_RESULTS)
- && opline->result_type == IS_CV
- && opline->opcode != ZEND_RECV) {
- ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)];
- //USE_SSA_VAR(op_array->last_var + opline->result.var)
- }
-
- switch (opline->opcode) {
- case ZEND_ASSIGN:
- if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op2_type == IS_CV) {
- ssa_ops[k].op2_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(opline->op2.var)
- }
- if (opline->op1_type == IS_CV) {
-add_op1_def:
- ssa_ops[k].op1_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(opline->op1.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(opline->op1.var)
- }
- break;
- case ZEND_ASSIGN_REF:
- if (opline->op2_type == IS_CV) {
- ssa_ops[k].op2_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(opline->op2.var)
- }
- if (opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_ASSIGN_DIM:
- case ZEND_ASSIGN_OBJ:
- next = opline + 1;
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
- //USE_SSA_VAR(op_array->last_var + next->op1.var);
- if (build_flags & ZEND_SSA_RC_INFERENCE && next->op1_type == IS_CV) {
- ssa_ops[k + 1].op1_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(next->op1.var)
- }
- }
- if (opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_ASSIGN_OBJ_REF:
- next = opline + 1;
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
- //USE_SSA_VAR(op_array->last_var + next->op1.var);
- if (next->op1_type == IS_CV) {
- ssa_ops[k + 1].op1_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(next->op1.var)
- }
- }
- if (opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_ASSIGN_STATIC_PROP:
- next = opline + 1;
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
- //USE_SSA_VAR(op_array->last_var + next->op1.var);
-#if 0
- if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) {
- ssa_ops[k + 1].op1_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(next->op1.var)
- }
-#endif
- }
- break;
- case ZEND_ASSIGN_STATIC_PROP_REF:
- next = opline + 1;
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
- //USE_SSA_VAR(op_array->last_var + next->op1.var);
- if (next->op1_type == IS_CV) {
- ssa_ops[k + 1].op1_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(next->op1.var)
- }
- }
- break;
- case ZEND_ASSIGN_STATIC_PROP_OP:
- next = opline + 1;
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
- //USE_SSA_VAR(op_array->last_var + next->op1.var);
- }
- break;
- case ZEND_ASSIGN_DIM_OP:
- case ZEND_ASSIGN_OBJ_OP:
- next = opline + 1;
- if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
- //USE_SSA_VAR(op_array->last_var + next->op1.var);
- }
- if (opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_ASSIGN_OP:
- case ZEND_PRE_INC:
- case ZEND_PRE_DEC:
- case ZEND_POST_INC:
- case ZEND_POST_DEC:
- case ZEND_BIND_GLOBAL:
- case ZEND_BIND_STATIC:
- case ZEND_SEND_VAR_NO_REF:
- case ZEND_SEND_VAR_NO_REF_EX:
- case ZEND_SEND_VAR_EX:
- case ZEND_SEND_FUNC_ARG:
- case ZEND_SEND_REF:
- case ZEND_SEND_UNPACK:
- case ZEND_FE_RESET_RW:
- case ZEND_MAKE_REF:
- case ZEND_PRE_INC_OBJ:
- case ZEND_PRE_DEC_OBJ:
- case ZEND_POST_INC_OBJ:
- case ZEND_POST_DEC_OBJ:
- case ZEND_UNSET_DIM:
- case ZEND_UNSET_OBJ:
- case ZEND_FETCH_DIM_W:
- case ZEND_FETCH_DIM_RW:
- case ZEND_FETCH_DIM_FUNC_ARG:
- case ZEND_FETCH_DIM_UNSET:
- case ZEND_FETCH_LIST_W:
- if (opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_SEND_VAR:
- case ZEND_CAST:
- case ZEND_QM_ASSIGN:
- case ZEND_JMP_SET:
- case ZEND_COALESCE:
- case ZEND_FE_RESET_R:
- if ((build_flags & ZEND_SSA_RC_INFERENCE) && opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_ADD_ARRAY_UNPACK:
- ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)];
- break;
- case ZEND_ADD_ARRAY_ELEMENT:
- ssa_ops[k].result_use = var[EX_VAR_TO_NUM(opline->result.var)];
- /* break missing intentionally */
- case ZEND_INIT_ARRAY:
- if (((build_flags & ZEND_SSA_RC_INFERENCE)
- || (opline->extended_value & ZEND_ARRAY_ELEMENT_REF))
- && opline->op1_type == IS_CV) {
- goto add_op1_def;
- }
- break;
- case ZEND_YIELD:
- if (opline->op1_type == IS_CV
- && ((op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE)
- || (build_flags & ZEND_SSA_RC_INFERENCE))) {
- goto add_op1_def;
- }
- break;
- case ZEND_UNSET_CV:
- goto add_op1_def;
- case ZEND_VERIFY_RETURN_TYPE:
- if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
- goto add_op1_def;
- }
- break;
- case ZEND_FE_FETCH_R:
- case ZEND_FE_FETCH_RW:
- if (opline->op2_type != IS_CV) {
- ssa_ops[k].op2_use = -1; /* not used */
- }
- ssa_ops[k].op2_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(opline->op2.var)
- break;
- case ZEND_BIND_LEXICAL:
- if ((opline->extended_value & ZEND_BIND_REF) || (build_flags & ZEND_SSA_RC_INFERENCE)) {
- ssa_ops[k].op2_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(opline->op2.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(opline->op2.var)
- }
- break;
- default:
- break;
- }
-
- if (opline->result_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
- ssa_ops[k].result_def = ssa_vars_count;
- var[EX_VAR_TO_NUM(opline->result.var)] = ssa_vars_count;
- ssa_vars_count++;
- //NEW_SSA_VAR(op_array->last_var + opline->result.var)
- }
+ ssa_vars_count = _zend_ssa_rename_op(op_array, opline, k, build_flags, ssa_vars_count, ssa_ops, var);
}
}
BEGIN_EXTERN_C()
int zend_build_ssa(zend_arena **arena, const zend_script *script, const zend_op_array *op_array, uint32_t build_flags, zend_ssa *ssa);
+int zend_ssa_rename_op(const zend_op_array *op_array, const zend_op *opline, uint32_t k, uint32_t build_flags, int ssa_vars_count, zend_ssa_op *ssa_ops, int *var);
int zend_ssa_compute_use_def_chains(zend_arena **arena, const zend_op_array *op_array, zend_ssa *ssa);
int zend_ssa_unlink_use_chain(zend_ssa *ssa, int op, int var);
$(srcdir)/jit/zend_jit_perf_dump.c \
$(srcdir)/jit/zend_jit_oprofile.c \
$(srcdir)/jit/zend_jit_vtune.c \
+ $(srcdir)/jit/zend_jit_trace.c \
$(srcdir)/jit/zend_elf.c
ext/opcache/jit/zend_jit_gdb.c \
ext/opcache/jit/zend_jit_perf_dump.c \
ext/opcache/jit/zend_jit_oprofile.c \
+ ext/opcache/jit/zend_jit_trace.c \
ext/opcache/jit/zend_jit_vtune.c
#include "Zend/zend_constants.h"
#include "zend_smart_str.h"
#include "jit/zend_jit.h"
-#include "jit/zend_jit_internal.h"
#ifdef HAVE_JIT
#include "Optimizer/zend_call_graph.h"
#include "Optimizer/zend_dump.h"
+#include "jit/zend_jit_internal.h"
+
+#ifdef ZTS
+int zend_jit_globals_id;
+#else
+zend_jit_globals jit_globals;
+#endif
+
//#define CONTEXT_THREADED_JIT
#define ZEND_JIT_USE_RC_INFERENCE
#define JIT_PREFIX "JIT$"
#define JIT_STUB_PREFIX "JIT$$"
+#define TRACE_PREFIX "TRACE-"
#define DASM_M_GROW(ctx, t, p, sz, need) \
do { \
static const void *zend_jit_runtime_jit_handler = NULL;
static const void *zend_jit_profile_jit_handler = NULL;
static const void *zend_jit_func_counter_handler = NULL;
+static const void *zend_jit_ret_counter_handler = NULL;
static const void *zend_jit_loop_counter_handler = NULL;
static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa);
static void ZEND_FASTCALL zend_runtime_jit(void);
+static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline, zend_jit_trace_rec *trace);
+static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace);
+static const void *zend_jit_trace_get_exit_addr(uint32_t n);
+static void zend_jit_trace_add_code(const void *start, uint32_t size);
+
static zend_bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
{
if (ssa->vars[var].phi_use_chain) {
return (x > 0) && !(x & (x - 1));
}
-#define OP_RANGE(line, opN) \
+#define OP_RANGE_EX(ssa_op, opN) \
(((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
- ssa->ops && \
- ssa->var_info && \
- ssa->ops[line].opN##_use >= 0 && \
- ssa->var_info[ssa->ops[line].opN##_use].has_range) ? \
- &ssa->var_info[ssa->ops[line].opN##_use].range : NULL)
+ (ssa_op)->opN##_use >= 0 && \
+ ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
+ &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
+
+#define OP_RANGE(line, opN) \
+ (ssa->var_info ? OP_RANGE_EX(ssa->ops + (line), opN) : NULL)
#define OP1_RANGE() OP_RANGE(opline - op_array->opcodes, op1)
#define OP2_RANGE() OP_RANGE(opline - op_array->opcodes, op2)
#define OP1_DATA_RANGE() OP_RANGE(opline - op_array->opcodes + 1, op1)
+#define OP1_RANGE_EX() OP_RANGE_EX(ssa_op, op1)
+#define OP2_RANGE_EX() OP_RANGE_EX(ssa_op, op2)
+#define OP1_DATA_RANGE_EX() OP_RANGE_EX(ssa_op + 1, op1)
+
#include "dynasm/dasm_x86.h"
#include "jit/zend_jit_x86.h"
#include "jit/zend_jit_helpers.c"
-#include "jit/zend_jit_x86.c"
#include "jit/zend_jit_disasm_x86.c"
#ifndef _WIN32
#include "jit/zend_jit_gdb.c"
#endif
#include "jit/zend_jit_vtune.c"
+#include "jit/zend_jit_x86.c"
+
#if _WIN32
# include <Windows.h>
#else
# endif
#endif
-#define DASM_ALIGNMENT 16
-
ZEND_EXT_API void zend_jit_status(zval *ret)
{
zval stats;
zend_ssa *ssa,
const zend_op *rt_opline,
zend_lifetime_interval **ra,
- const char *name)
+ const char *name,
+ zend_bool is_trace)
{
size_t size;
int ret;
entry = *dasm_ptr;
*dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
+ if (is_trace) {
+ zend_jit_trace_add_code(entry, size);
+ }
+
if (op_array && ssa) {
int b;
} else {
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
- if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) {
+ if (is_trace || (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) != 0) {
zend_jit_disasm(
name,
(op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
return entry;
}
-static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+static int zend_may_overflow_ex(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
{
- uint32_t num;
int res;
if (!ssa->ops || !ssa->var_info) {
switch (opline->opcode) {
case ZEND_PRE_INC:
case ZEND_POST_INC:
- num = opline - op_array->opcodes;
- res = ssa->ops[num].op1_def;
+ res = ssa_op->op1_def;
return (res < 0 ||
!ssa->var_info[res].has_range ||
ssa->var_info[res].range.overflow);
case ZEND_PRE_DEC:
case ZEND_POST_DEC:
- num = opline - op_array->opcodes;
- res = ssa->ops[num].op1_def;
+ res = ssa_op->op1_def;
return (res < 0 ||
!ssa->var_info[res].has_range ||
ssa->var_info[res].range.underflow);
case ZEND_ADD:
- num = opline - op_array->opcodes;
- res = ssa->ops[num].result_def;
+ res = ssa_op->result_def;
if (res < 0 ||
!ssa->var_info[res].has_range) {
return 1;
if (ssa->var_info[res].range.underflow) {
zend_long op1_min, op2_min;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_min = OP1_MIN_RANGE();
- op2_min = OP2_MIN_RANGE();
+ op1_min = OP1_MIN_RANGE_EX();
+ op2_min = OP2_MIN_RANGE_EX();
if (zend_add_will_overflow(op1_min, op2_min)) {
return 1;
}
if (ssa->var_info[res].range.overflow) {
zend_long op1_max, op2_max;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_max = OP1_MAX_RANGE();
- op2_max = OP2_MAX_RANGE();
+ op1_max = OP1_MAX_RANGE_EX();
+ op2_max = OP2_MAX_RANGE_EX();
if (zend_add_will_overflow(op1_max, op2_max)) {
return 1;
}
}
return 0;
case ZEND_SUB:
- num = opline - op_array->opcodes;
- res = ssa->ops[num].result_def;
+ res = ssa_op->result_def;
if (res < 0 ||
!ssa->var_info[res].has_range) {
return 1;
if (ssa->var_info[res].range.underflow) {
zend_long op1_min, op2_max;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_min = OP1_MIN_RANGE();
- op2_max = OP2_MAX_RANGE();
+ op1_min = OP1_MIN_RANGE_EX();
+ op2_max = OP2_MAX_RANGE_EX();
if (zend_sub_will_overflow(op1_min, op2_max)) {
return 1;
}
if (ssa->var_info[res].range.overflow) {
zend_long op1_max, op2_min;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_max = OP1_MAX_RANGE();
- op2_min = OP2_MIN_RANGE();
+ op1_max = OP1_MAX_RANGE_EX();
+ op2_min = OP2_MIN_RANGE_EX();
if (zend_sub_will_overflow(op1_max, op2_min)) {
return 1;
}
}
return 0;
case ZEND_MUL:
- num = opline - op_array->opcodes;
- res = ssa->ops[num].result_def;
+ res = ssa_op->result_def;
return (res < 0 ||
!ssa->var_info[res].has_range ||
ssa->var_info[res].range.underflow ||
ssa->var_info[res].range.overflow);
case ZEND_ASSIGN_OP:
if (opline->extended_value == ZEND_ADD) {
- num = opline - op_array->opcodes;
- res = ssa->ops[num].op1_def;
+ res = ssa_op->op1_def;
if (res < 0 ||
!ssa->var_info[res].has_range) {
return 1;
if (ssa->var_info[res].range.underflow) {
zend_long op1_min, op2_min;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_min = OP1_MIN_RANGE();
- op2_min = OP2_MIN_RANGE();
+ op1_min = OP1_MIN_RANGE_EX();
+ op2_min = OP2_MIN_RANGE_EX();
if (zend_add_will_overflow(op1_min, op2_min)) {
return 1;
}
if (ssa->var_info[res].range.overflow) {
zend_long op1_max, op2_max;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_max = OP1_MAX_RANGE();
- op2_max = OP2_MAX_RANGE();
+ op1_max = OP1_MAX_RANGE_EX();
+ op2_max = OP2_MAX_RANGE_EX();
if (zend_add_will_overflow(op1_max, op2_max)) {
return 1;
}
}
return 0;
} else if (opline->extended_value == ZEND_SUB) {
- num = opline - op_array->opcodes;
- res = ssa->ops[num].op1_def;
+ res = ssa_op->op1_def;
if (res < 0 ||
!ssa->var_info[res].has_range) {
return 1;
if (ssa->var_info[res].range.underflow) {
zend_long op1_min, op2_max;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_min = OP1_MIN_RANGE();
- op2_max = OP2_MAX_RANGE();
+ op1_min = OP1_MIN_RANGE_EX();
+ op2_max = OP2_MAX_RANGE_EX();
if (zend_sub_will_overflow(op1_min, op2_max)) {
return 1;
}
if (ssa->var_info[res].range.overflow) {
zend_long op1_max, op2_min;
- if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ if (!OP1_HAS_RANGE_EX() || !OP2_HAS_RANGE_EX()) {
return 1;
}
- op1_max = OP1_MAX_RANGE();
- op2_min = OP2_MIN_RANGE();
+ op1_max = OP1_MAX_RANGE_EX();
+ op2_min = OP2_MIN_RANGE_EX();
if (zend_sub_will_overflow(op1_max, op2_min)) {
return 1;
}
}
return 0;
} else if (opline->extended_value == ZEND_MUL) {
- num = opline - op_array->opcodes;
- res = ssa->ops[num].op1_def;
+ res = ssa_op->op1_def;
return (res < 0 ||
!ssa->var_info[res].has_range ||
ssa->var_info[res].range.underflow ||
}
}
+static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+{
+ return zend_may_overflow_ex(opline, &ssa->ops[opline - op_array->opcodes], op_array, ssa);
+}
+
static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
{
uint32_t flags;
goto done;
case ZEND_INIT_FCALL:
case ZEND_INIT_FCALL_BY_NAME:
- if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level)) {
+ if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level, NULL)) {
goto jit_failure;
}
goto done;
case ZEND_DO_ICALL:
case ZEND_DO_FCALL_BY_NAME:
case ZEND_DO_FCALL:
- if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1)) {
+ if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
goto jit_failure;
}
goto done;
OP2_INFO(), OP2_REG_ADDR(),
res_addr,
zend_may_throw(opline, op_array, ssa),
- smart_branch_opcode, target_label, target_label2)) {
+ smart_branch_opcode, target_label, target_label2,
+ NULL)) {
goto jit_failure;
}
goto done;
OP2_INFO(), OP2_REG_ADDR(),
RES_REG_ADDR(),
zend_may_throw(opline, op_array, ssa),
- smart_branch_opcode, target_label, target_label2)) {
+ smart_branch_opcode, target_label, target_label2,
+ NULL)) {
goto jit_failure;
}
goto done;
smart_branch_opcode = 0;
target_label = target_label2 = (uint32_t)-1;
}
- if (!zend_jit_defined(&dasm_state, opline, op_array, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_defined(&dasm_state, opline, op_array, smart_branch_opcode, target_label, target_label2, NULL)) {
goto jit_failure;
}
goto done;
smart_branch_opcode = 0;
target_label = target_label2 = (uint32_t)-1;
}
- if (!zend_jit_type_check(&dasm_state, opline, op_array, OP1_INFO(), smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_type_check(&dasm_state, opline, op_array, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
goto jit_failure;
}
goto done;
if (!zend_jit_tail_handler(&dasm_state, opline)) {
goto jit_failure;
}
- } else if (!zend_jit_return(&dasm_state, opline, op_array, ssa,
- op1_info, OP1_REG_ADDR())) {
- goto jit_failure;
+ } else {
+ int j;
+
+ if (!zend_jit_return(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR())) {
+ goto jit_failure;
+ }
+ if (jit_return_label >= 0) {
+ if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
+ goto jit_failure;
+ }
+ goto done;
+ }
+ jit_return_label = ssa->cfg.blocks_count * 2;
+ if (!zend_jit_label(&dasm_state, jit_return_label)) {
+ goto jit_failure;
+ }
+ if (!zend_jit_leave_frame(&dasm_state)) {
+ goto jit_failure;
+ }
+ for (j = 0 ; j < op_array->last_var; j++) {
+ uint32_t info = zend_ssa_cv_info(opline, op_array, ssa, j);
+
+ if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ if (!zend_jit_free_cv(&dasm_state, opline, op_array, info, j)) {
+ goto jit_failure;
+ }
+ }
+ }
+ if (!zend_jit_leave_func(&dasm_state, opline, op_array, NULL)) {
+ goto jit_failure;
+ }
}
goto done;
case ZEND_BOOL:
if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array,
OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
-1, -1,
- zend_may_throw(opline, op_array, ssa))) {
+ zend_may_throw(opline, op_array, ssa),
+ opline->opcode, NULL)) {
goto jit_failure;
}
goto done;
if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array,
OP1_INFO(), OP1_REG_ADDR(), res_addr,
ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
- zend_may_throw(opline, op_array, ssa))) {
+ zend_may_throw(opline, op_array, ssa),
+ opline->opcode, NULL)) {
goto jit_failure;
}
goto done;
if (!zend_jit_isset_isempty_dim(&dasm_state, opline, op_array,
OP1_INFO(), OP2_INFO(),
zend_may_throw(opline, op_array, ssa),
- smart_branch_opcode, target_label, target_label2)) {
+ smart_branch_opcode, target_label, target_label2,
+ NULL)) {
goto jit_failure;
}
goto done;
zend_jit_call(&dasm_state, next_opline, b + 1);
is_terminated = 1;
} else {
- zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1);
+ zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
}
}
}
}
}
- handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL);
+ handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0);
if (!handler) {
goto jit_failure;
}
void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
{
zend_op_array *op_array = &EX(func)->op_array;
- zend_jit_op_array_extension *jit_extension;
+ zend_jit_op_array_hot_extension *jit_extension;
uint32_t i;
zend_shared_alloc_lock();
- jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
+ jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
if (jit_extension) {
SHM_UNPROTECT();
zend_jit_unprotect();
- *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT;
for (i = 0; i < op_array->last; i++) {
op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
}
static int zend_jit_setup_hot_counters(zend_op_array *op_array)
{
zend_op *opline = op_array->opcodes;
- zend_jit_op_array_extension *jit_extension;
+ zend_jit_op_array_hot_extension *jit_extension;
zend_cfg cfg;
uint32_t i;
return FAILURE;
}
- jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension) + (op_array->last - 1) * sizeof(void*));
+ jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
for (i = 0; i < op_array->last; i++) {
jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
return 0;
}
+#include "jit/zend_jit_trace.c"
+
ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
{
if (dasm_ptr == NULL) {
return SUCCESS;
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
return zend_jit_setup_hot_counters(op_array);
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ return zend_jit_setup_hot_trace_counters(op_array);
} else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) {
return zend_real_jit_func(op_array, script, NULL);
} else if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) {
if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC ||
zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST ||
- zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
+ zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS ||
+ zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
for (i = 0; i < call_graph.op_arrays_count; i++) {
ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
if (!zend_jit_stubs[i].stub(&dasm_state)) {
return 0;
}
- if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name)) {
+ if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0)) {
return 0;
}
}
if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) {
return 0;
}
- zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit");
+ zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit", 0);
if (!zend_jit_runtime_jit_handler) {
return 0;
}
if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) {
return 0;
}
- zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit");
+ zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit", 0);
if (!zend_jit_profile_jit_handler) {
return 0;
}
if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) {
return 0;
}
- zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter");
+ zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0);
if (!zend_jit_func_counter_handler) {
return 0;
}
if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) {
return 0;
}
- zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter");
+ zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0);
+ if (!zend_jit_loop_counter_handler) {
+ return 0;
+ }
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_func_trace_counter_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0);
+ if (!zend_jit_func_counter_handler) {
+ return 0;
+ }
+
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_ret_trace_counter_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_ret_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_ret_counter", 0);
+ if (!zend_jit_ret_counter_handler) {
+ return 0;
+ }
+
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_loop_trace_counter_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0);
if (!zend_jit_loop_counter_handler) {
return 0;
}
}
} else {
- zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
- zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
- zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper;
- zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper;
+ if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) {
+ zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
+ } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) {
+ zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
+ zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper;
+ zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper;
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ zend_jit_func_counter_handler = (const void*)zend_jit_func_trace_helper;
+ zend_jit_ret_counter_handler = (const void*)zend_jit_ret_trace_helper;
+ zend_jit_loop_counter_handler = (const void*)zend_jit_loop_trace_helper;
+ }
}
dasm_free(&dasm_state);
return 1;
}
+static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
+{
+ memset(jit_globals, 0, sizeof(zend_jit_globals));
+ zend_jit_trace_init_caches();
+}
+
ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bool reattached)
{
int ret;
+#ifdef ZTS
+ zend_jit_globals_id = ts_allocate_id(&zend_jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL);
+#else
+ zend_jit_globals_ctor(&jit_globals);
+#endif
+
zend_jit_level = ZEND_JIT_LEVEL(jit);
zend_jit_trigger = ZEND_JIT_TRIGGER(jit);
zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit);
#endif
}
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ if (zend_jit_trace_startup() != SUCCESS) {
+ return FAILURE;
+ }
+ }
+
return SUCCESS;
}
for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
zend_jit_hot_counters[i] = ZEND_JIT_HOT_COUNTER_INIT;
}
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int i;
+
+ for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
+ zend_jit_hot_counters[i] = ZEND_JIT_TRACE_COUNTER_INIT;
+ }
+
+ zend_jit_trace_reset_caches();
}
}
#define ZEND_JIT_ON_PROF_REQUEST 2 /* compile the most frequently caled on first request functions */
#define ZEND_JIT_ON_HOT_COUNTERS 3 /* compile functions after N calls or loop iterations */
#define ZEND_JIT_ON_DOC_COMMENT 4 /* compile functions with "@jit" tag in doc-comments */
+#define ZEND_JIT_ON_HOT_TRACE 5 /* trace functions after N calls or loop iterations */
#define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10)
#define ZEND_JIT_DEBUG_GDB (1<<8)
+#define ZEND_JIT_DEBUG_TRACE_START (1<<12)
+#define ZEND_JIT_DEBUG_TRACE_STOP (1<<13)
+#define ZEND_JIT_DEBUG_TRACE_COMPILED (1<<14)
+#define ZEND_JIT_DEBUG_TRACE_EXIT (1<<15)
+#define ZEND_JIT_DEBUG_TRACE_ABORT (1<<16)
+#define ZEND_JIT_DEBUG_TRACE_BLACKLIST (1<<17)
+#define ZEND_JIT_DEBUG_TRACE_BYTECODE (1<<18)
+#define ZEND_JIT_DEBUG_TRACE_TSSA (1<<19)
+
ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script);
ZEND_EXT_API int zend_jit_script(zend_script *script);
ZEND_EXT_API void zend_jit_unprotect(void);
}
} else {
ZEND_ASSERT(sym->addr == node->addr);
+ if (strcmp(name, node->name) == 0 && sym->end < node->end) {
+ /* reduce size of the existing symbol */
+ node->end = sym->end;
+ }
free(sym);
return;
}
REGISTER_HELPER(zend_jit_int_extend_stack_helper);
REGISTER_HELPER(zend_jit_leave_nested_func_helper);
REGISTER_HELPER(zend_jit_leave_top_func_helper);
+ REGISTER_HELPER(zend_jit_leave_func_helper);
REGISTER_HELPER(zend_jit_symtable_find);
REGISTER_HELPER(zend_jit_hash_index_lookup_rw);
REGISTER_HELPER(zend_jit_hash_index_lookup_w);
REGISTER_HELPER(zend_jit_pre_dec);
REGISTER_HELPER(zend_runtime_jit);
REGISTER_HELPER(zend_jit_hot_func);
+ REGISTER_HELPER(zend_jit_check_constant);
#undef REGISTER_HELPER
#ifndef _WIN32
extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
-void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
-
-typedef struct _zend_jit_op_array_extension {
- int16_t *counter;
- const void *orig_handlers[1];
-} zend_jit_op_array_extension;
-
-static zend_always_inline zend_long zend_jit_op_array_hash(const zend_op_array *op_array)
+static zend_always_inline zend_long zend_jit_hash(const void *ptr)
{
uintptr_t x;
- x = (uintptr_t)op_array->opcodes >> 3;
+ x = (uintptr_t)ptr >> 3;
#if SIZEOF_SIZE_T == 4
x = ((x >> 16) ^ x) * 0x45d9f3b;
x = ((x >> 16) ^ x) * 0x45d9f3b;
return x;
}
+void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
+
+typedef struct _zend_jit_op_array_hot_extension {
+ int16_t *counter;
+ const void *orig_handlers[1];
+} zend_jit_op_array_hot_extension;
+
+#define zend_jit_op_array_hash(op_array) \
+ zend_jit_hash((op_array)->opcodes)
+
extern const zend_op *zend_jit_halt_op;
#ifdef HAVE_GCC_GLOBAL_REGS
handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
return; \
} while(0)
+# define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
+ handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
+ return; \
+ } while(0)
#else
# define EXECUTE_DATA_D zend_execute_data* execute_data
# define EXECUTE_DATA_C execute_data
# define ZEND_OPCODE_TAIL_CALL(handler) do { \
return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
} while(0)
+# define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
+ return handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
+ } while(0)
#endif
/* VM handlers */
/* VM helpers */
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC);
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);
void ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags);
int ZEND_FASTCALL zend_jit_check_constant(const zval *key);
+/* Tracer */
+#define zend_jit_opline_hash(opline) \
+ zend_jit_hash(opline)
+
+#define ZEND_JIT_TRACE_FUNC_COST (1*250)
+#define ZEND_JIT_TRACE_RET_COST (1*250-1)
+#define ZEND_JIT_TRACE_LOOP_COST (2*250)
+#define ZEND_JIT_TRACE_COUNTER_INIT (127*250)
+
+#define ZEND_JIT_TRACE_MAX_TRACES 1024 /* max number of traces */
+#define ZEND_JIT_TRACE_MAX_LENGTH 1024 /* max length of single trace */
+#define ZEND_JIT_TRACE_MAX_EXITS 512 /* max number of side exits per trace */
+#define ZEND_JIT_TRACE_MAX_SIDE_TRACES 128 /* max number of side traces of a root trace */
+#define ZEND_JIT_TRACE_MAX_EXIT_COUNTERS 8192 /* max number of side exits for all trace */
+
+#define ZEND_JIT_TRACE_MAX_FUNCS 30 /* max number of different functions in a single trace */
+#define ZEND_JIT_TRACE_MAX_CALL_DEPTH 10 /* max depth of inlined calls */
+#define ZEND_JIT_TRACE_MAX_RET_DEPTH 4 /* max depth of inlined returns */
+#define ZEND_JIT_TRACE_MAX_RECURSION 2 /* max number of recursive inlined calls */
+#define ZEND_JIT_TRACE_MAX_UNROLL_LOOPS 8 /* max number of unrolled loops */
+
+#define ZEND_JIT_TRACE_HOT_SIDE_COUNT 8 /* number of exits before taking side trace */
+#define ZEND_JIT_TRACE_HOT_RETURN_COUNT 8 /* number of returns before taking continuation trace */
+
+#define ZEND_JIT_TRACE_MAX_ROOT_FAILURES 16 /* number of attemts to record/compile a root trace */
+#define ZEND_JIT_TRACE_MAX_SIDE_FAILURES 4 /* number of attemts to record/compile a side trace */
+
+#define ZEND_JIT_TRACE_BAD_ROOT_SLOTS 64 /* number of slots in bad root trace cache */
+
+#define ZEND_JIT_TRACE_STOP(_) \
+ _(LOOP, "loop") \
+ _(RECURSIVE_CALL, "recursive call") \
+ _(RECURSIVE_RET, "recursive return") \
+ _(RETURN, "return") \
+ _(LINK, "link to another trace") \
+ /* compilation and linking successful */ \
+ _(COMPILED, "compiled") \
+ _(ALREADY_DONE, "already prcessed") \
+ /* failures */ \
+ _(ERROR, "error") /* not used */ \
+ _(NOT_SUPPORTED, "not supported instructions") \
+ _(EXCEPTION, "exception") \
+ _(TOO_LONG, "trace too long") \
+ _(TOO_DEEP, "trace too deep") \
+ _(TOO_DEEP_RET, "trace too deep return") \
+ _(DEEP_RECURSION, "deep recursion") \
+ _(LOOP_UNROLL, "loop unroll limit reached") \
+ _(LOOP_EXIT, "exit from loop") \
+ _(BLACK_LIST, "trace blacklisted") \
+ _(INNER_LOOP, "inner loop") /* trace it */ \
+ _(COMPILED_LOOP, "compiled loop") \
+ _(TOPLEVEL, "toplevel") \
+ _(TRAMPOLINE, "trampoline call") \
+ _(BAD_FUNC, "bad function call") \
+ _(HALT, "exit from interpreter") \
+ _(COMPILER_ERROR, "JIT compilation error") \
+ /* no recoverable error (blacklist immediately) */ \
+ _(NO_SHM, "insufficient shared memory") \
+ _(TOO_MANY_TRACES, "too many traces") \
+ _(TOO_MANY_CHILDREN, "too many side traces") \
+ _(TOO_MANY_EXITS, "too many side exits") \
+
+#define ZEND_JIT_TRACE_STOP_NAME(name, description) \
+ ZEND_JIT_TRACE_STOP_ ## name,
+
+typedef enum _zend_jit_trace_stop {
+ ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_NAME)
+} zend_jit_trace_stop;
+
+#define ZEND_JIT_TRACE_STOP_OK(ret) \
+ (ret < ZEND_JIT_TRACE_STOP_COMPILED)
+
+#define ZEND_JIT_TRACE_STOP_DONE(ret) \
+ (ret < ZEND_JIT_TRACE_STOP_ERROR)
+
+#define ZEND_JIT_TRACE_STOP_REPEAT(ret) \
+ (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP)
+
+#define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \
+ (ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR)
+
+#define ZEND_JIT_TRACE_START_MASK 0xf
+
+#define ZEND_JIT_TRACE_START_LOOP (1<<0)
+#define ZEND_JIT_TRACE_START_ENTER (1<<1)
+#define ZEND_JIT_TRACE_START_RETURN (1<<2)
+#define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */
+
+#define ZEND_JIT_TRACE_JITED (1<<4)
+#define ZEND_JIT_TRACE_BLACKLISTED (1<<5)
+#define ZEND_JIT_TRACE_UNSUPPORTED (1<<6)
+
+#define ZEND_JIT_TRACE_SUPPORTED 0
+
+#define ZEND_JIT_EXIT_JITED (1<<0)
+#define ZEND_JIT_EXIT_BLACKLISTED (1<<1)
+
+typedef union _zend_op_trace_info {
+ zend_op dummy; /* the size of this structure must be the same as zend_op */
+ struct {
+ const void *orig_handler;
+ const void *call_handler;
+ int16_t *counter;
+ uint8_t trace_flags;
+ };
+} zend_op_trace_info;
+
+typedef struct _zend_jit_op_array_trace_extension {
+ zend_func_info func_info;
+ size_t offset; /* offset from "zend_op" to corresponding "op_info" */
+ zend_op_trace_info trace_info[1];
+} zend_jit_op_array_trace_extension;
+
+#define ZEND_OP_TRACE_INFO(opline, offset) \
+ ((zend_op_trace_info*)(((char*)opline) + offset))
+
+/* Recorder */
+typedef enum _zend_jit_trace_op {
+ ZEND_JIT_TRACE_VM,
+ ZEND_JIT_TRACE_OP1_TYPE,
+ ZEND_JIT_TRACE_OP2_TYPE,
+ ZEND_JIT_TRACE_INIT_CALL,
+ ZEND_JIT_TRACE_DO_ICALL,
+ ZEND_JIT_TRACE_ENTER,
+ ZEND_JIT_TRACE_BACK,
+ ZEND_JIT_TRACE_START,
+ ZEND_JIT_TRACE_END,
+} zend_jit_trace_op;
+
+#define IS_UNKNOWN 255 /* may be used for zend_jit_trace_rec.op?_type */
+#define IS_TRACE_REFERENCE (1<<5)
+
+typedef struct _zend_jit_trace_rec {
+ uint8_t op; /* zend_jit_trace_op */
+ union {
+ struct {
+ uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */
+ uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */
+ uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */
+ };
+ struct {
+ union {
+ uint8_t recursive; /* part of recursive return sequence for ZEND_JIT_TRACE_BACK */
+ int8_t return_value_used; /* for ZEND_JIT_TRACE_ENTER */
+ uint8_t fake; /* for ZEND_JIT_TRACE_INIT_FCALL */
+ };
+ uint8_t first_ssa_var; /* may be used for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK */
+ };
+ struct {
+ uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
+ uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
+ uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
+ };
+ };
+ union {
+ const void *ptr;
+ const zend_function *func;
+ const zend_op_array *op_array;
+ const zend_op *opline;
+ const zend_class_entry *ce;
+ };
+} zend_jit_trace_rec;
+
+typedef struct _zend_jit_trace_start_rec {
+ uint8_t op; /* zend_jit_trace_op */
+ uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
+ uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
+ uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
+ const zend_op_array *op_array;
+ const zend_op *opline;
+} zend_jit_trace_start_rec;
+
+#define ZEND_JIT_TRACE_START_REC_SIZE 2
+
+typedef struct _zend_jit_trace_exit_info {
+ const zend_op *opline; /* opline where VM should continue execution */
+ uint32_t stack_size;
+ uint32_t stack_offset;
+} zend_jit_trace_exit_info;
+
+typedef int32_t zend_jit_trace_stack;
+
+typedef struct _zend_jit_trace_info {
+ uint32_t id; /* trace id */
+ uint32_t root; /* root trace id or self id for root traces */
+ uint32_t parent; /* parent trace id or 0 for root traces */
+ uint32_t link; /* link trace id or self id for loop) */
+ uint32_t exit_count; /* number of side exits */
+ uint32_t child_count; /* number of side traces for root traces */
+ uint32_t code_size; /* size of native code */
+ uint32_t exit_counters; /* offset in exit counters array */
+ uint32_t stack_map_size;
+ const void *code_start; /* address of native code */
+ zend_jit_trace_exit_info *exit_info; /* info about side exits */
+ zend_jit_trace_stack *stack_map;
+ //uint32_t loop_offset;
+} zend_jit_trace_info;
+
+typedef struct _zend_jit_trace_stack_frame zend_jit_trace_stack_frame;
+
+struct _zend_jit_trace_stack_frame {
+ zend_jit_trace_stack_frame *call;
+ zend_jit_trace_stack_frame *prev;
+ const zend_function *func;
+ union {
+ struct {
+ int8_t return_value_used;
+ int8_t nested;
+ int8_t num_args;
+ };
+ int return_ssa_var;
+ };
+ zend_jit_trace_stack stack[1];
+};
+
+typedef struct _zend_jit_globals {
+ zend_jit_trace_stack_frame *current_frame;
+
+ const zend_op *bad_root_cache_opline[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
+ uint8_t bad_root_cache_count[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
+ uint8_t bad_root_cache_stop[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
+ uint32_t bad_root_slot;
+
+ uint8_t exit_counters[ZEND_JIT_TRACE_MAX_EXIT_COUNTERS];
+} zend_jit_globals;
+
+#ifdef ZTS
+# define JIT_G(v) ZEND_TSRMG(zend_jit_globals_id, zend_jit_globals *, v)
+extern int zend_jit_globals_id;
+#else
+# define JIT_G(v) (jit_globals.v)
+extern zend_jit_globals jit_globals;
+#endif
+
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
+
+int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
+int ZEND_FASTCALL zend_jit_trace_exit(uint32_t trace_num, uint32_t exit_num);
+zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start);
+
+static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true)
+{
+ if (trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END) {
+ if (trace->opline == opline + 1) {
+ /* not taken branch */
+ *exit_if_true = opline->opcode == ZEND_JMPNZ;
+ return OP_JMP_ADDR(opline, opline->op2);
+ } else if (trace->opline == OP_JMP_ADDR(opline, opline->op2)) {
+ /* taken branch */
+ *exit_if_true = opline->opcode == ZEND_JMPZ;
+ return opline + 1;
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ *exit_if_true = 0;
+ return NULL;
+}
+
#endif /* ZEND_JIT_INTERNAL_H */
--- /dev/null
+/*
+ +----------------------------------------------------------------------+
+ | Zend JIT |
+ +----------------------------------------------------------------------+
+ | Copyright (c) The PHP Group |
+ +----------------------------------------------------------------------+
+ | This source file is subject to version 3.01 of the PHP license, |
+ | that is bundled with this package in the file LICENSE, and is |
+ | available through the world-wide-web at the following url: |
+ | http://www.php.net/license/3_01.txt |
+ | If you did not receive a copy of the PHP license and are unable to |
+ | obtain it through the world-wide-web, please send a note to |
+ | license@php.net so we can mail you a copy immediately. |
+ +----------------------------------------------------------------------+
+ | Authors: Dmitry Stogov <dmitry@php.net> |
+ +----------------------------------------------------------------------+
+*/
+
+static zend_jit_trace_info *zend_jit_traces = NULL;
+static const void **zend_jit_exit_groups = NULL;
+
+#define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
+#define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
+#define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
+#define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
+
+static int zend_jit_trace_startup(void)
+{
+ zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * ZEND_JIT_TRACE_MAX_TRACES);
+ if (!zend_jit_traces) {
+ return FAILURE;
+ }
+ zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
+ if (!zend_jit_exit_groups) {
+ return FAILURE;
+ }
+ ZEND_JIT_TRACE_NUM = 1;
+ ZEND_JIT_COUNTER_NUM = 0;
+ ZEND_JIT_EXIT_NUM = 0;
+ ZEND_JIT_EXIT_COUNTERS = 0;
+ return SUCCESS;
+}
+
+static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
+{
+ dasm_State* dasm_state = NULL;
+ const void *entry;
+ char name[32];
+
+ dasm_init(&dasm_state, DASM_MAXSECTION);
+ dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
+ dasm_setup(&dasm_state, dasm_actions);
+ zend_jit_trace_exit_group_stub(&dasm_state, n);
+
+ sprintf(name, "jit$$trace_exit_%d", n);
+ entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0);
+ dasm_free(&dasm_state);
+
+#ifdef HAVE_DISASM
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) {
+ uint32_t i;
+
+ for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
+ sprintf(name, "jit$$trace_exit_%d", n + i);
+ zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
+ }
+ }
+#endif
+
+ return entry;
+}
+
+static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
+{
+ const void *group = NULL;
+
+ if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
+ return NULL;
+ }
+ do {
+ group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
+ if (!group) {
+ return NULL;
+ }
+ zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
+ group;
+ ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
+ } while (n >= ZEND_JIT_EXIT_NUM);
+ return (const void*)
+ ((const char*)group +
+ ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
+}
+
+static const void *zend_jit_trace_get_exit_addr(uint32_t n)
+{
+ if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
+ return zend_jit_trace_allocate_exit_point(n);
+ }
+ return (const void*)
+ ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
+ ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
+}
+
+static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace)
+{
+ zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
+ uint32_t exit_point;
+ const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;
+ uint32_t stack_offset = (uint32_t)-1;
+ uint32_t stack_size = op_array->last_var + op_array->T;
+ zend_jit_trace_stack *stack = NULL;
+
+ if (stack_size) {
+ stack = JIT_G(current_frame)->stack;
+ do {
+ if (stack[stack_size-1] != IS_UNKNOWN) {
+ break;
+ }
+ stack_size--;
+ } while (stack_size);
+ }
+
+ /* Try to reuse exit points */
+ if (to_opline != NULL && t->exit_count > 0) {
+ uint32_t i = t->exit_count;
+
+ do {
+ i--;
+ if (stack_size == 0
+ || (t->exit_info[i].stack_size >= stack_size
+ && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
+ stack_offset = t->exit_info[i].stack_offset;
+ if (t->exit_info[i].opline == to_opline) {
+ return i;
+ }
+ }
+ } while (i > 0);
+ }
+
+ exit_point = t->exit_count;
+ if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
+ if (stack_size != 0 && stack_offset == (uint32_t)-1) {
+ stack_offset = t->stack_map_size;
+ t->stack_map_size += stack_size;
+ // TODO: reduce number of reallocations ???
+ t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
+ memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
+ }
+ t->exit_count++;
+ t->exit_info[exit_point].opline = to_opline;
+ t->exit_info[exit_point].stack_size = stack_size;
+ t->exit_info[exit_point].stack_offset = stack_offset;
+ }
+
+ return exit_point;
+}
+
+static void zend_jit_trace_add_code(const void *start, uint32_t size)
+{
+ zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
+
+ t->code_start = start;
+ t->code_size = size;
+}
+
+static uint32_t zend_jit_find_trace(const void *addr)
+{
+ uint32_t i;
+
+ for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
+ if (zend_jit_traces[i].code_start == addr) {
+ return i;
+ }
+ }
+ ZEND_ASSERT(0);
+ return 0;
+}
+
+static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
+{
+ smart_str buf = {0};
+
+ smart_str_appends(&buf, TRACE_PREFIX);
+ smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
+ smart_str_appendc(&buf, '$');
+ if (op_array->function_name) {
+ if (op_array->scope) {
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
+ smart_str_appends(&buf, "::");
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
+ } else {
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
+ }
+ } else if (op_array->filename) {
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
+ }
+ smart_str_appendc(&buf, '$');
+ smart_str_append_long(&buf, (zend_long)lineno);
+ smart_str_0(&buf);
+ return buf.s;
+}
+
+static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline, zend_jit_trace_rec *trace)
+{
+ switch (opline->opcode) {
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_CASE:
+ case ZEND_ISSET_ISEMPTY_CV:
+ case ZEND_ISSET_ISEMPTY_VAR:
+ case ZEND_ISSET_ISEMPTY_DIM_OBJ:
+ case ZEND_ISSET_ISEMPTY_PROP_OBJ:
+ case ZEND_ISSET_ISEMPTY_STATIC_PROP:
+ case ZEND_INSTANCEOF:
+ case ZEND_TYPE_CHECK:
+ case ZEND_DEFINED:
+ case ZEND_IN_ARRAY:
+ case ZEND_ARRAY_KEY_EXISTS:
+ if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
+ /* smart branch */
+ return 1;
+ }
+ break;
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_FE_RESET_R:
+ case ZEND_FE_RESET_RW:
+ case ZEND_ASSERT_CHECK:
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ case ZEND_SWITCH_LONG:
+ case ZEND_SWITCH_STRING:
+ /* branch opcdoes */
+ return 1;
+ case ZEND_NEW:
+ if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
+ /* NEW may skip constructor without arguments */
+ return 1;
+ }
+ break;
+ case ZEND_CATCH:
+ case ZEND_FAST_CALL:
+ case ZEND_FAST_RET:
+ case ZEND_GENERATOR_CREATE:
+ case ZEND_GENERATOR_RETURN:
+ case ZEND_EXIT:
+ case ZEND_YIELD:
+ case ZEND_YIELD_FROM:
+ case ZEND_INCLUDE_OR_EVAL:
+ /* unsupported */
+ return 1;
+ case ZEND_DO_FCALL:
+ /* potentially polymorphic call */
+ return 1;
+#if 0
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ /* monomorphic call */
+ // TODO: recompilation may change traget ???
+ return 0;
+#endif
+ case ZEND_RETURN_BY_REF:
+ case ZEND_RETURN:
+ /* return */
+ return trace->op == ZEND_JIT_TRACE_BACK && trace->recursive;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info)
+{
+ if (type == IS_UNKNOWN) {
+ return info;
+ }
+ ZEND_ASSERT(info & (1 << type));
+ if (type < IS_STRING) {
+ return (1 << type);
+ } else if (type != IS_ARRAY) {
+ return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
+ } else {
+ return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
+ }
+}
+
+static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type)
+{
+ return zend_jit_trace_type_to_info_ex(type, -1);
+}
+
+#define STACK_VAR_TYPE(_var) \
+ ((zend_uchar)stack[EX_VAR_TO_NUM(_var)])
+
+#define SET_STACK_VAR_TYPE(_var, _type) do { \
+ stack[EX_VAR_TO_NUM(_var)] = _type; \
+ } while (0)
+
+#define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
+ if (op_type != IS_UNKNOWN) { \
+ if ((op_info & MAY_BE_GUARD) != 0 \
+ && op_type != STACK_VAR_TYPE(_var)) { \
+ if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
+ goto jit_failure; \
+ } \
+ op_info &= ~MAY_BE_GUARD; \
+ ssa->var_info[_ssa_var].type &= op_info; \
+ } \
+ SET_STACK_VAR_TYPE(_var, op_type); \
+ } \
+ } while (0)
+
+#define USE_OP_TRACE_TYPE(_type, _var, op_info) do { \
+ if (_type & (IS_TMP_VAR|IS_VAR|IS_CV)) { \
+ op_info = zend_jit_trace_type_to_info_ex(STACK_VAR_TYPE(_var), op_info); \
+ } \
+ } while (0)
+
+#define CHECK_OP1_TRACE_TYPE() \
+ CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
+#define CHECK_OP2_TRACE_TYPE() \
+ CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
+#define CHECK_OP1_DATA_TRACE_TYPE() \
+ CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
+#define USE_OP1_TRACE_TYPE() \
+ USE_OP_TRACE_TYPE(opline->op1_type, opline->op1.var, op1_info)
+#define USE_OP2_TRACE_TYPE() \
+ USE_OP_TRACE_TYPE(opline->op2_type, opline->op2.var, op2_info)
+#define USE_OP1_DATA_TRACE_TYPE() \
+ USE_OP_TRACE_TYPE((opline+1)->op1_type, (opline+1)->op1.var, op1_data_info)
+#define USE_RES_TRACE_TYPE() \
+ USE_OP_TRACE_TYPE(opline->result_type, opline->result.var, res_use_info)
+#define SET_OP1_STACK_VAR_TYPE(_type) \
+ SET_STACK_VAR_TYPE(opline->op1.var, _type)
+#define SET_OP2_STACK_VAR_TYPE( _type) \
+ SET_STACK_VAR_TYPE(opline->op2.var, _type)
+#define SET_OP1_DATA_STACK_VAR_TYPE(_type) \
+ SET_STACK_VAR_TYPE((opline+1)->op1.var, _type)
+#define SET_RES_STACK_VAR_TYPE(_type) \
+ SET_STACK_VAR_TYPE(opline->result.var, _type)
+
+static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
+{
+ if (op_array->type == ZEND_USER_FUNCTION) {
+ return offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack));
+ } else {
+ return offsetof(zend_jit_trace_stack_frame, stack);
+ }
+}
+
+static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
+{
+ return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
+}
+
+static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
+{
+ return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
+}
+
+static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type)
+{
+ zend_jit_trace_stack *stack = call->stack;
+ const zend_op_array *op_array = &call->func->op_array;
+ uint32_t arg_num = opline->op2.num;
+
+ if (arg_num > op_array->num_args) {
+ return;
+ }
+ if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ zend_arg_info *arg_info;
+
+ ZEND_ASSERT(arg_num <= op_array->num_args);
+ arg_info = &op_array->arg_info[arg_num-1];
+
+ if (ZEND_TYPE_IS_SET(arg_info->type)) {
+ if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
+ return;
+ }
+ }
+ }
+ SET_STACK_VAR_TYPE(opline->result.var, type);
+}
+
+static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
+{
+ zend_jit_op_array_trace_extension *jit_extension;
+ zend_ssa *ssa;
+
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ memset(&jit_extension->func_info, 0, sizeof(jit_extension->func_info));
+ jit_extension->func_info.num_args = -1;
+ jit_extension->func_info.return_value_used = -1;
+ ssa = &jit_extension->func_info.ssa;
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) {
+ do {
+ if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
+ break;
+ }
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) {
+ if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) {
+ break;
+ }
+ jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, (zend_op_array*)op_array);
+ if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
+ zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
+ }
+ }
+
+ if (zend_jit_op_array_analyze2(op_array, script, ssa) != SUCCESS) {
+ break;
+ }
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) {
+ zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
+ }
+ return ssa;
+ } while (0);
+ }
+
+ memset(ssa, 0, sizeof(zend_ssa));
+ ssa->cfg.blocks_count = 1;
+ return ssa;
+}
+
+static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
+
+static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
+{
+ int len;
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_DIM:
+ case ZEND_ASSIGN_OBJ:
+ case ZEND_ASSIGN_STATIC_PROP:
+ case ZEND_ASSIGN_DIM_OP:
+ case ZEND_ASSIGN_OBJ_OP:
+ case ZEND_ASSIGN_STATIC_PROP_OP:
+ case ZEND_ASSIGN_OBJ_REF:
+ case ZEND_ASSIGN_STATIC_PROP_REF:
+ return 2; /* OP_DATA */
+ case ZEND_RECV_INIT:
+ len = 1;
+ opline++;
+ while (opline->opcode == ZEND_RECV_INIT) {
+ len++;
+ opline++;
+ }
+ return len;
+ case ZEND_BIND_GLOBAL:
+ len = 1;
+ opline++;
+ while (opline->opcode == ZEND_BIND_GLOBAL) {
+ len++;
+ opline++;
+ }
+ return len;
+// case ZEND_IS_IDENTICAL:
+// case ZEND_IS_NOT_IDENTICAL:
+// case ZEND_IS_EQUAL:
+// case ZEND_IS_NOT_EQUAL:
+// case ZEND_IS_SMALLER:
+// case ZEND_IS_SMALLER_OR_EQUAL:
+// case ZEND_CASE:
+// case ZEND_ISSET_ISEMPTY_CV:
+// case ZEND_ISSET_ISEMPTY_VAR:
+// case ZEND_ISSET_ISEMPTY_DIM_OBJ:
+// case ZEND_ISSET_ISEMPTY_PROP_OBJ:
+// case ZEND_ISSET_ISEMPTY_STATIC_PROP:
+// case ZEND_INSTANCEOF:
+// case ZEND_TYPE_CHECK:
+// case ZEND_DEFINED:
+// case ZEND_IN_ARRAY:
+// case ZEND_ARRAY_KEY_EXISTS:
+ default:
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ return 2; /* JMPZ/JMPNZ */
+ }
+ return 1;
+ }
+}
+
+static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, int *var)
+{
+ const zend_op_array *op_array;
+ zend_jit_trace_rec *p;
+ int k, vars_count;
+ zend_bitset use, def;
+ uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
+ uint32_t set_size;
+ zend_ssa_phi *prev = NULL;
+ int level = 0;
+ ALLOCA_FLAG(use_heap);
+
+ op_array = trace_buffer->op_array;
+ set_size = zend_bitset_len(op_array->last_var + op_array->T);
+ use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
+ memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
+ def = use + set_size;
+ p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
+ for (;;p++) {
+ if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
+ const zend_op *opline = p->opline;
+ int len;
+
+ zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
+ len = zend_jit_trace_op_len(opline);
+ while (len > 1) {
+ opline++;
+ if (opline->opcode != ZEND_OP_DATA) {
+ zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
+ }
+ len--;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
+ } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
+ } else if (p->op == ZEND_JIT_TRACE_ENTER) {
+ level++;
+ } else if (p->op == ZEND_JIT_TRACE_BACK) {
+ if (level == 0) {
+ // Phi for recursive calls and returns are not supporte yet ???
+ assert(0);
+ } else {
+ level--;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_END) {
+ break;
+ }
+ }
+
+ zend_bitset_intersection(use, def, set_size);
+
+ if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
+ vars_count = op_array->last_var;
+ } else {
+ vars_count = op_array->last_var + op_array->T;
+ }
+ for (k = 0; k < vars_count; k++) {
+ if (zend_bitset_in(use, k)) {
+ zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
+ ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
+ ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
+ sizeof(void*) * 2);
+ phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
+ phi->sources[0] = var[k];
+ phi->sources[1] = -1;
+ phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
+ phi->pi = -1;
+ phi->var = k;
+ phi->ssa_var = ssa_vars_count;
+ var[k] = ssa_vars_count;
+ ssa_vars_count++;
+ phi->block = 1;
+ if (prev) {
+ prev->next = phi;
+ } else {
+ tssa->blocks[1].phis = phi;
+ }
+ prev = phi;
+ }
+ }
+
+ free_alloca(use, use_heap);
+
+ return ssa_vars_count;
+}
+
+static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
+{
+ int var, use;
+ zend_ssa_op *op;
+ zend_ssa_var_info *info;
+ unsigned int no_val;
+
+ if (tssa->vars[ssa_var].phi_use_chain) {
+ // TODO: this may be incorrect ???
+ var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
+ } else {
+ var = ssa_var;
+ }
+ use = tssa->vars[var].use_chain;
+ if (use >= 0) {
+ ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
+ op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
+ if (tssa->ops[use].op1_use == var) {
+ no_val = ssa->vars[op->op1_use].no_val;
+ info = ssa->var_info + op->op1_use;
+ } else if (tssa->ops[use].op2_use == var) {
+ no_val = ssa->vars[op->op2_use].no_val;
+ info = ssa->var_info + op->op2_use;
+ } else if (tssa->ops[use].result_use == var) {
+ no_val = ssa->vars[op->result_use].no_val;
+ info = ssa->var_info + op->result_use;
+ } else {
+ assert(0);
+ }
+ tssa->vars[ssa_var].no_val = no_val;
+ memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
+ return 1;
+ }
+ return 0;
+}
+
+static int zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
+{
+ int def;
+ zend_ssa_op *op;
+ zend_ssa_var_info *info;
+ unsigned int no_val;
+
+ def = tssa->vars[ssa_var].definition;
+ if (def >= 0) {
+ ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
+ op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
+ if (tssa->ops[def].op1_def == ssa_var) {
+ no_val = ssa->vars[op->op1_def].no_val;
+ info = ssa->var_info + op->op1_def;
+ } else if (tssa->ops[def].op2_def == ssa_var) {
+ no_val = ssa->vars[op->op2_def].no_val;
+ info = ssa->var_info + op->op2_def;
+ } else if (tssa->ops[def].result_def == ssa_var) {
+ no_val = ssa->vars[op->result_def].no_val;
+ info = ssa->var_info + op->result_def;
+ } else {
+ assert(0);
+ }
+
+ tssa->vars[ssa_var].no_val = no_val;
+
+ if (info->has_range) {
+ if (tssa->var_info[ssa_var].has_range) {
+ tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
+ tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
+ tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
+ tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
+ } else {
+ tssa->var_info[ssa_var].has_range = 1;
+ tssa->var_info[ssa_var].range = info->range;
+ }
+ }
+ return 1;
+ }
+ return 0;
+}
+
+static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
+{
+ int def;
+ zend_ssa_op *op;
+ zend_ssa_var_info *info;
+
+ def = tssa->vars[ssa_var].definition;
+ if (def >= 0) {
+ ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
+ op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
+ if (tssa->ops[def].op1_def == ssa_var) {
+ info = ssa->var_info + op->op1_def;
+ } else if (tssa->ops[def].op2_def == ssa_var) {
+ info = ssa->var_info + op->op2_def;
+ } else if (tssa->ops[def].result_def == ssa_var) {
+ info = ssa->var_info + op->result_def;
+ } else {
+ assert(0);
+ }
+ tssa->var_info[ssa_var].type &= info->type;
+ if (info->ce) {
+ if (tssa->var_info[ssa_var].ce) {
+ ZEND_ASSERT(tssa->var_info[ssa_var].ce == info->ce);
+ tssa->var_info[ssa_var].is_instanceof =
+ tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
+ } else {
+ tssa->var_info[ssa_var].ce = info->ce;
+ tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
+ }
+ }
+ if (info->has_range) {
+ if (tssa->var_info[ssa_var].has_range) {
+ tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
+ tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
+ tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
+ tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
+ } else {
+ tssa->var_info[ssa_var].has_range = 1;
+ tssa->var_info[ssa_var].range = info->range;
+ }
+ }
+ return 1;
+ }
+ return 0;
+}
+
+static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
+{
+ while (1) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ if (p->opline->opcode == ZEND_DO_UCALL
+ || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
+ || p->opline->opcode == ZEND_DO_FCALL) {
+ if (p->opline->result_type != IS_UNUSED) {
+ return ssa_op->result_def;
+ }
+ }
+ return -1;
+ } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ /*skip */
+ } else {
+ return -1;
+ }
+ p--;
+ }
+}
+
+static int find_call_num_args(zend_jit_trace_rec *p)
+{
+ while (1) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ if (p->opline->opcode == ZEND_INIT_FCALL
+ || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
+ || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
+ || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
+ || p->opline->opcode == ZEND_INIT_USER_CALL
+ || p->opline->opcode == ZEND_NEW
+ || p->opline->opcode == ZEND_INIT_METHOD_CALL
+ || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
+ if (p->opline->extended_value <= 127) {
+ return p->opline->extended_value;
+ } else {
+ return -1;
+ }
+ }
+ return -1;
+ } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ /*skip */
+ } else {
+ return -1;
+ }
+ p--;
+ }
+}
+
+static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
+{
+ if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG) {
+ uint32_t idx = tssa->vars[var].definition;
+
+ if (idx >= 0) {
+ if (tssa->ops[idx].op1_def == var) {
+ const zend_op *opline = ssa_opcodes[idx];
+ if (opline->opcode == ZEND_PRE_DEC
+ || opline->opcode == ZEND_PRE_INC
+ || opline->opcode == ZEND_POST_DEC
+ || opline->opcode == ZEND_POST_INC) {
+ return 1;
+ }
+ }
+ if (tssa->ops[idx].result_def == var) {
+ const zend_op *opline = ssa_opcodes[idx];
+ if (opline->opcode == ZEND_ADD
+ || opline->opcode == ZEND_SUB
+ || opline->opcode == ZEND_MUL
+ || opline->opcode == ZEND_PRE_DEC
+ || opline->opcode == ZEND_PRE_INC
+ || opline->opcode == ZEND_POST_DEC
+ || opline->opcode == ZEND_POST_INC) {
+ return 1;
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
+{
+ zend_ssa *tssa;
+ zend_ssa_op *ssa_ops, *op;
+ zend_ssa_var *ssa_vars;
+ zend_ssa_var_info *ssa_var_info;
+ const zend_op_array *op_array;
+ const zend_op *opline;
+ const zend_op **ssa_opcodes;
+ zend_jit_trace_rec *p;
+ int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
+ int *var;
+ uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
+ uint32_t optimization_level = ZCG(accel_directives).optimization_level;
+ int call_level, level, num_op_arrays;
+ size_t frame_size, stack_top, stack_size, stack_bottom;
+ zend_jit_op_array_trace_extension *jit_extension;
+ zend_ssa *ssa;
+ zend_jit_trace_stack_frame *frame, *top, *call;
+ zend_ssa_var_info return_value_info;
+
+ /* 1. Count number of TSSA opcodes;
+ * Count number of activation frames;
+ * Calculate size of abstract stack;
+ * Construct regular SSA for involved op_array */
+ op_array = trace_buffer->op_array;
+ stack_top = stack_size = zend_jit_trace_frame_size(op_array);
+ stack_bottom = 0;
+ p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
+ ssa_ops_count = 0;
+ call_level = 0;
+ level = 0;
+ num_op_arrays = 0;
+ /* Remember op_array to cleanup */
+ op_arrays[num_op_arrays++] = op_array;
+ /* Build SSA */
+ ssa = zend_jit_trace_build_ssa(op_array, script);
+ for (;;p++) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ ssa_ops_count += zend_jit_trace_op_len(p->opline);
+ } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
+ call_level++;
+ stack_top += zend_jit_trace_frame_size(p->op_array);
+ if (stack_top > stack_size) {
+ stack_size = stack_top;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
+ frame_size = zend_jit_trace_frame_size(p->op_array);
+ if (call_level == 0) {
+ if (stack_top + frame_size > stack_size) {
+ stack_size = stack_top + frame_size;
+ }
+ } else {
+ call_level--;
+ stack_top -= frame_size;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_ENTER) {
+ op_array = p->op_array;
+ if (call_level == 0) {
+ stack_top += zend_jit_trace_frame_size(op_array);
+ if (stack_top > stack_size) {
+ stack_size = stack_top;
+ }
+ } else {
+ call_level--;
+ }
+ level++;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ ssa = &jit_extension->func_info.ssa;
+ if (ssa->cfg.blocks_count) {
+ /* pass */
+ } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
+ /* Too many functions in single trace */
+ return NULL;
+ } else {
+ /* Remember op_array to cleanup */
+ op_arrays[num_op_arrays++] = op_array;
+ /* Build SSA */
+ ssa = zend_jit_trace_build_ssa(op_array, script);
+ }
+ } else if (p->op == ZEND_JIT_TRACE_BACK) {
+ if (level == 0) {
+ stack_bottom += zend_jit_trace_frame_size(op_array);;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ ssa = &jit_extension->func_info.ssa;
+ if (ssa->cfg.blocks_count) {
+ /* pass */
+ } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
+ /* Too many functions in single trace */
+ return NULL;
+ } else {
+ /* Remember op_array to cleanup */
+ op_arrays[num_op_arrays++] = op_array;
+ /* Build SSA */
+ ssa = zend_jit_trace_build_ssa(op_array, script);
+ }
+ } else {
+ stack_top -= zend_jit_trace_frame_size(op_array);
+ level--;
+ }
+ op_array = p->op_array;
+ } else if (p->op == ZEND_JIT_TRACE_END) {
+ break;
+ }
+ }
+ *num_op_arrays_ptr = num_op_arrays;
+
+ /* 2. Construct TSSA */
+ tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_ssa));
+ tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
+ tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
+ tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
+ tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
+ memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
+ ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count, sizeof(zend_op*));
+ JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ tssa->cfg.blocks_count = 2;
+ tssa->cfg.edges_count = 2;
+
+ tssa->cfg.predecessors[0] = 0;
+ tssa->cfg.predecessors[1] = 1;
+
+ tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
+ tssa->cfg.blocks[0].successors_count = 1;
+ tssa->cfg.blocks[0].predecessors_count = 0;
+ tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
+ tssa->cfg.blocks[0].successors[0] = 1;
+
+ tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
+ tssa->cfg.blocks[1].successors_count = 1;
+ tssa->cfg.blocks[1].predecessors_count = 2;
+ tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
+ tssa->cfg.blocks[1].successors[1] = 1;
+ } else {
+ tssa->cfg.blocks_count = 1;
+ tssa->cfg.edges_count = 0;
+
+ tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
+ tssa->cfg.blocks[0].successors_count = 0;
+ tssa->cfg.blocks[0].predecessors_count = 0;
+ }
+
+ op_array = trace_buffer->op_array;
+ if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
+ ssa_vars_count = op_array->last_var;
+ } else {
+ ssa_vars_count = op_array->last_var + op_array->T;
+ }
+ var = frame->stack;
+ for (i = 0; i < ssa_vars_count; i++) {
+ var[i] = i;
+ }
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ // TODO: For tracing, it's possible, to create pseudo Phi functions
+ // at the end of loop, without this additional pass (like LuaJIT) ???
+ ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, var);
+ }
+
+ p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
+ idx = 0;
+ level = 0;
+ for (;;p++) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ opline = p->opline;
+ ssa_opcodes[idx] = opline;
+ ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, var);
+ idx++;
+ len = zend_jit_trace_op_len(p->opline);
+ while (len > 1) {
+ opline++;
+ ssa_opcodes[idx] = opline;
+ if (opline->opcode != ZEND_OP_DATA) {
+ ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, var);
+ }
+ idx++;
+ len--;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_ENTER) {
+ frame = zend_jit_trace_call_frame(frame, op_array);
+ var = frame->stack;
+ op_array = p->op_array;
+ level++;
+ ZEND_ASSERT(ssa_vars_count < 0xff);
+ p->first_ssa_var = ssa_vars_count;
+ for (i = 0; i < op_array->last_var; i++) {
+ var[i] = ssa_vars_count++;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_BACK) {
+ op_array = p->op_array;
+ frame = zend_jit_trace_ret_frame(frame, op_array);
+ var = frame->stack;
+ if (level == 0) {
+ ZEND_ASSERT(ssa_vars_count <= 0xff);
+ p->first_ssa_var = ssa_vars_count;
+ for (i = 0; i < op_array->last_var + op_array->T; i++) {
+ var[i] = ssa_vars_count++;
+ }
+ } else {
+ level--;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_END) {
+ break;
+ }
+ }
+
+ op_array = trace_buffer->op_array;
+ tssa->vars_count = ssa_vars_count;
+ tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
+ if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
+ vars_count = op_array->last_var;
+ } else {
+ vars_count = op_array->last_var + op_array->T;
+ }
+ i = 0;
+ while (i < vars_count) {
+ ssa_vars[i].var = i;
+ ssa_vars[i].scc = -1;
+ ssa_vars[i].definition = -1;
+ ssa_vars[i].use_chain = -1;
+ i++;
+ }
+ while (i < tssa->vars_count) {
+ ssa_vars[i].var = -1;
+ ssa_vars[i].scc = -1;
+ ssa_vars[i].definition = -1;
+ ssa_vars[i].use_chain = -1;
+ i++;
+ }
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ /* Update Phi sources */
+ zend_ssa_phi *phi = tssa->blocks[1].phis;
+
+ while (phi) {
+ phi->sources[1] = var[phi->var];
+ ssa_vars[phi->ssa_var].var = phi->var;
+ ssa_vars[phi->ssa_var].definition_phi = phi;
+ ssa_vars[phi->sources[0]].phi_use_chain = phi;
+ ssa_vars[phi->sources[1]].phi_use_chain = phi;
+ phi = phi->next;
+ }
+ }
+
+ /* 3. Compute use-def chains */
+ idx = (ssa_ops_count - 1);
+ op = ssa_ops + idx;
+ while (idx >= 0) {
+ opline = ssa_opcodes[idx];
+ if (op->op1_use >= 0) {
+ op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
+ ssa_vars[op->op1_use].use_chain = idx;
+ }
+ if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
+ op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
+ ssa_vars[op->op2_use].use_chain = idx;
+ }
+ if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
+ op->res_use_chain = ssa_vars[op->result_use].use_chain;
+ ssa_vars[op->result_use].use_chain = idx;
+ }
+ if (op->op1_def >= 0) {
+ ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
+ ssa_vars[op->op1_def].definition = idx;
+ }
+ if (op->op2_def >= 0) {
+ ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
+ ssa_vars[op->op2_def].definition = idx;
+ }
+ if (op->result_def >= 0) {
+ ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
+ ssa_vars[op->result_def].definition = idx;
+ }
+ op--;
+ idx--;
+ }
+
+ /* 4. Type inference */
+ op_array = trace_buffer->op_array;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ ssa = &jit_extension->func_info.ssa;
+
+ tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
+
+ if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
+ i = 0;
+ while (i < op_array->last_var) {
+ if (!ssa->var_info
+ || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
+ if (i < op_array->num_args) {
+ if (op_array->arg_info) {
+ zend_arg_info *arg_info = &op_array->arg_info[i];
+ zend_class_entry *ce;
+ uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
+
+ if (ZEND_ARG_SEND_MODE(arg_info)) {
+ tmp |= MAY_BE_REF;
+ }
+ ssa_var_info[i].type = tmp;
+ } else {
+ ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ }
+ } else {
+ ssa_var_info[i].type = MAY_BE_UNDEF;
+ }
+ }
+ i++;
+ }
+ } else {
+ int parent_vars_count = 0;
+ zend_jit_trace_stack *parent_stack = NULL;
+
+ i = 0;
+ if (parent_trace) {
+ parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
+ op_array->last_var + op_array->T);
+ if (parent_vars_count) {
+ parent_stack =
+ zend_jit_traces[parent_trace].stack_map +
+ zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
+ }
+ }
+ while (i < op_array->last_var + op_array->T) {
+ if (!ssa->var_info
+ || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
+ if (i < op_array->last_var) {
+ ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ } else {
+ ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ }
+ }
+ if (i < parent_vars_count) {
+ /* Initialize TSSA variable from parent trace */
+ zend_uchar op_type = parent_stack[i];
+
+ if (op_type != IS_UNKNOWN) {
+ ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
+ }
+ }
+ i++;
+ }
+ }
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ /* Propagae initial value through Phi functions */
+ zend_ssa_phi *phi = tssa->blocks[1].phis;
+
+ while (phi) {
+ if (!ssa->var_info
+ || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
+ ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
+ }
+ phi = phi->next;
+ }
+ }
+
+ frame = JIT_G(current_frame);
+ top = zend_jit_trace_call_frame(frame, op_array);
+ frame->call = NULL;
+ frame->prev = NULL;
+ frame->func = (const zend_function*)op_array;
+ frame->return_ssa_var = -1;
+ for (i = 0; i < op_array->last_var + op_array->T; i++) {
+ frame->stack[i] = -1;
+ }
+ memset(&return_value_info, 0, sizeof(return_value_info));
+
+ p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
+ idx = 0;
+ level = 0;
+ for (;;p++) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ uint8_t op1_type, op2_type, op3_type;
+
+ // TODO: use types recorded in trace (add guards) ???
+ // TODO: range inference ???
+ opline = p->opline;
+
+
+ op1_type = p->op1_type;
+ op2_type = p->op2_type;
+ op3_type = p->op3_type;
+ if (op1_type & IS_TRACE_REFERENCE) {
+ op1_type = IS_UNKNOWN;
+ }
+ if (op2_type & IS_TRACE_REFERENCE) {
+ op2_type = IS_UNKNOWN;
+ }
+ if (op3_type & IS_TRACE_REFERENCE) {
+ op3_type = IS_UNKNOWN;
+ }
+
+ if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
+ // TODO: support for recorded classes ???
+ p++;
+ }
+ if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ // TODO: support for recorded classes ???
+ p++;
+ }
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_DIM_OP:
+ case ZEND_ASSIGN_DIM:
+ if (tssa->ops[idx+1].op1_use >= 0 && op3_type != IS_UNKNOWN) {
+ zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx+1].op1_use];
+ if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op3_type)) {
+ info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op3_type, info->type);
+ }
+ }
+ /* break missing intentionally */
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+// case ZEND_DIV: // TODO: check for division by zero ???
+ case ZEND_CONCAT:
+ case ZEND_FAST_CONCAT:
+ case ZEND_ASSIGN_OP:
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_CASE:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ case ZEND_FETCH_DIM_R:
+ case ZEND_FETCH_DIM_IS:
+ case ZEND_ISSET_ISEMPTY_DIM_OBJ:
+ if (tssa->ops[idx].op2_use >= 0 && op2_type != IS_UNKNOWN) {
+ zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op2_use];
+ if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op2_type)) {
+ info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op2_type, info->type);
+ }
+ }
+ /* break missing intentionally */
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ case ZEND_QM_ASSIGN:
+ case ZEND_BOOL:
+ case ZEND_BOOL_NOT:
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ if (tssa->ops[idx].op1_use >= 0 && op1_type != IS_UNKNOWN) {
+ zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_use];
+ if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op1_type)) {
+ info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op1_type, info->type);
+ }
+ }
+ break;
+ case ZEND_ASSIGN:
+ if (tssa->ops[idx].op2_use >= 0 && op2_type != IS_UNKNOWN) {
+ zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op2_use];
+ if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op2_type)) {
+ info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op2_type, info->type);
+ }
+ }
+ break;
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAL_EX:
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ if (tssa->ops[idx].op1_use >= 0 && op1_type != IS_UNKNOWN) {
+ zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_use];
+ if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op1_type)) {
+ info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op1_type, info->type);
+ }
+ }
+ /* Propagate argument type */
+ if (frame->call
+ && frame->call->func->type == ZEND_USER_FUNCTION
+ && opline->op2.num <= frame->call->func->op_array.num_args) {
+ uint32_t info;
+
+ if (opline->op1_type == IS_CONST) {
+ info = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
+ } else {
+ ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
+ info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
+ }
+ if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ zend_arg_info *arg_info;
+
+ ZEND_ASSERT(frame->call->func->op_array.arg_info);
+ arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
+ if (ZEND_TYPE_IS_SET(arg_info->type)) {
+ info &= ZEND_TYPE_FULL_MASK(arg_info->type);
+ if (!info) {
+ break;
+ }
+ }
+ }
+ if (info & MAY_BE_UNDEF) {
+ info |= MAY_BE_NULL;
+ info &= ~MAY_BE_UNDEF;
+ }
+ if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
+ info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
+ }
+ frame->call->stack[opline->op2.num - 1] = info;
+ }
+ break;
+ case ZEND_RETURN:
+ if (tssa->ops[idx].op1_use >= 0 && op1_type != IS_UNKNOWN) {
+ zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_use];
+ if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op1_type)) {
+ info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op1_type, info->type);
+ }
+ }
+ /* Propagate return value types */
+ if (opline->op1_type == IS_UNUSED) {
+ return_value_info.type = MAY_BE_NULL;
+ } else if (opline->op1_type == IS_CONST) {
+ return_value_info.type = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
+ } else {
+ ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
+ return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
+ return_value_info.type &= ~MAY_BE_GUARD;
+ }
+ break;
+ default:
+ break;
+ }
+ len = zend_jit_trace_op_len(opline);
+ if (ssa->var_info) {
+ /* Add statically inferred ranges */
+ if (ssa_ops[idx].op1_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
+ }
+ if (ssa_ops[idx].op2_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
+ }
+ if (ssa_ops[idx].result_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
+ }
+ if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
+ if (ssa_ops[idx+1].op1_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
+ }
+ if (ssa_ops[idx+1].op2_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
+ }
+ if (ssa_ops[idx+1].result_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
+ }
+ }
+ }
+ if (opline->opcode == ZEND_RECV_INIT
+ && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ /* RECV_INIT always copy the constant */
+ ssa_var_info[ssa_ops[idx].result_def].type = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op2)));
+ } else {
+ if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
+ // TODO:
+ assert(0);
+ }
+ }
+ if (ssa->var_info) {
+ /* Add statically inferred restrictions */
+ if (ssa_ops[idx].op1_def >= 0) {
+ if ((opline->opcode == ZEND_SEND_VAR_EX
+ || opline->opcode == ZEND_FETCH_DIM_FUNC_ARG
+ || opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG)
+ && frame
+ && frame->call
+ && frame->call->func
+ && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
+ ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
+ if (opline->opcode == ZEND_SEND_VAR_EX) {
+ ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
+ }
+ if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
+ ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
+ }
+ } else {
+ zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
+ }
+ }
+ if (ssa_ops[idx].op2_def >= 0) {
+ zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
+ }
+ if (ssa_ops[idx].result_def >= 0) {
+ zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
+ }
+ }
+ idx++;
+ while (len > 1) {
+ opline++;
+ if (opline->opcode != ZEND_OP_DATA) {
+ if (ssa->var_info) {
+ /* Add statically inferred ranges */
+ if (ssa_ops[idx].op1_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
+ }
+ if (ssa_ops[idx].op2_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
+ }
+ if (ssa_ops[idx].result_def >= 0) {
+ zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
+ }
+ }
+ if (opline->opcode == ZEND_RECV_INIT
+ && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ /* RECV_INIT always copy the constant */
+ ssa_var_info[ssa_ops[idx].result_def].type = zend_jit_trace_type_to_info(Z_TYPE_P(RT_CONSTANT(opline, opline->op2)));
+ } else {
+ if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
+ // TODO:
+ assert(0);
+ }
+ }
+ }
+ if (ssa->var_info) {
+ /* Add statically inferred restrictions */
+ if (ssa_ops[idx].op1_def >= 0) {
+ zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
+ }
+ if (ssa_ops[idx].op2_def >= 0) {
+ zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
+ }
+ if (ssa_ops[idx].result_def >= 0) {
+ zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
+ }
+ }
+ idx++;
+ len--;
+ }
+
+ } else if (p->op == ZEND_JIT_TRACE_ENTER) {
+ op_array = p->op_array;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ ssa = &jit_extension->func_info.ssa;
+
+ call = frame->call;
+ if (!call) {
+ /* Trace missed INIT_FCALL opcode */
+ call = top;
+ call->call = NULL;
+ call->prev = NULL;
+ call->func = (const zend_function*)op_array;
+ top = zend_jit_trace_call_frame(top, op_array);
+ for (i = 0; i < op_array->last_var + op_array->T; i++) {
+ call->stack[i] = -1;
+ }
+ } else {
+ ZEND_ASSERT(&call->func->op_array == op_array);
+ }
+ frame->call = call->prev;
+ call->prev = frame;
+ call->return_ssa_var = find_return_ssa_var(p - 1, ssa_ops + (idx -1));
+ frame = call;
+
+ level++;
+ i = 0;
+ v = p->first_ssa_var;
+ while (i < op_array->last_var) {
+ if (!ssa->var_info
+ || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
+ if (i < op_array->num_args) {
+ if (op_array->arg_info) {
+ zend_arg_info *arg_info = &op_array->arg_info[i];
+ zend_class_entry *ce;
+ uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
+
+ if (ZEND_ARG_SEND_MODE(arg_info)) {
+ tmp |= MAY_BE_REF;
+ }
+ ssa_var_info[v].type = tmp;
+ } else {
+ ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ }
+ } else {
+ ssa_var_info[v].type = MAY_BE_UNDEF;
+ }
+ }
+ if (i < op_array->num_args) {
+ /* Propagate argument type */
+ ssa_var_info[v].type &= frame->stack[i];
+ }
+ i++;
+ v++;
+ }
+
+ } else if (p->op == ZEND_JIT_TRACE_BACK) {
+ op_array = p->op_array;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ ssa = &jit_extension->func_info.ssa;
+ if (level == 0) {
+ i = 0;
+ v = p->first_ssa_var;
+ while (i < op_array->last_var) {
+ if (!ssa->var_info
+ || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
+ ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ }
+ i++;
+ v++;
+ }
+ while (i < op_array->last_var + op_array->T) {
+ if (!ssa->var_info
+ || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
+ ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
+ }
+ i++;
+ v++;
+ }
+ if (return_value_info.type != 0) {
+ if ((p+1)->op == ZEND_JIT_TRACE_VM) {
+ const zend_op *opline = (p+1)->opline - 1;
+ if (opline->result_type != IS_UNUSED) {
+ ssa_var_info[
+ p->first_ssa_var +
+ EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
+ }
+ }
+ memset(&return_value_info, 0, sizeof(return_value_info));
+ }
+ } else {
+ level--;
+ if (return_value_info.type != 0) {
+ if ((p+1)->op == ZEND_JIT_TRACE_VM) {
+ const zend_op *opline = (p+1)->opline - 1;
+ if (opline->result_type != IS_UNUSED) {
+ if (frame->return_ssa_var >= 0) {
+ ssa_var_info[frame->return_ssa_var] = return_value_info;
+ }
+ }
+ }
+ memset(&return_value_info, 0, sizeof(return_value_info));
+ }
+ }
+
+ top = frame;
+ if (frame->prev) {
+ frame = frame->prev;
+ ZEND_ASSERT(&frame->func->op_array == op_array);
+ } else {
+ frame = zend_jit_trace_ret_frame(frame, op_array);
+ frame->call = NULL;
+ frame->prev = NULL;
+ frame->func = (const zend_function*)op_array;
+ frame->return_ssa_var = -1;
+ for (i = 0; i < op_array->last_var + op_array->T; i++) {
+ frame->stack[i] = -1;
+ }
+ }
+
+ } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
+ call = top;
+ call->call = NULL;
+ call->prev = frame->call;
+ call->func = p->func;
+ frame->call = call;
+ top = zend_jit_trace_call_frame(top, p->op_array);
+ if (p->func->type == ZEND_USER_FUNCTION) {
+ for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
+ call->stack[i] = -1;
+ }
+ }
+ } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
+ call = frame->call;
+ if (call) {
+ top = call;
+ frame->call = call->prev;
+ }
+ } else if (p->op == ZEND_JIT_TRACE_END) {
+ break;
+ }
+ }
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ /* Propagate guards through Phi sources */
+ zend_ssa_phi *phi = tssa->blocks[1].phis;
+
+ while (phi) {
+ uint32_t t = ssa_var_info[phi->ssa_var].type;
+ uint32_t t0 = ssa_var_info[phi->sources[0]].type;
+ uint32_t t1 = ssa_var_info[phi->sources[1]].type;
+
+ if (t & MAY_BE_GUARD) {
+ if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
+ if (!((t0 | t1) & MAY_BE_GUARD)) {
+ ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
+ }
+ } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
+ if (!(t1 & MAY_BE_GUARD)) {
+ ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
+ ssa_var_info[phi->sources[0]].type = t | MAY_BE_GUARD;
+ }
+ } else {
+ if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
+ ssa_var_info[phi->sources[0]].type = MAY_BE_GUARD | (t & t0);
+ }
+ if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
+ if (is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
+ ssa_var_info[phi->sources[1]].type = MAY_BE_GUARD | (t & t1);
+ ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
+ }
+ }
+ }
+ }
+ phi = phi->next;
+ }
+ }
+
+ if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_TSSA)) {
+ zend_jit_dump_trace(trace_buffer, tssa);
+ fprintf(stderr, "---- TRACE analysed\n");
+ }
+
+ return tssa;
+}
+
+static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
+{
+ const void *handler = NULL;
+ dasm_State* dasm_state = NULL;
+ zend_script *script = NULL;
+ zend_lifetime_interval **ra = NULL;
+ zend_string *name = NULL;
+ void *checkpoint;
+ const zend_op_array *op_array;
+ zend_ssa *ssa, *op_array_ssa;
+ zend_jit_trace_rec *p;
+ int call_level = -1; // TODO: proper support for inlined functions ???
+ zend_jit_op_array_trace_extension *jit_extension;
+ int num_op_arrays = 0;
+ zend_jit_trace_info *t;
+ const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
+ zend_uchar smart_branch_opcode;
+ const void *exit_addr;
+ uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
+ zend_bool send_result = 0;
+ zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
+ zend_class_entry *ce;
+ uint32_t i;
+ zend_jit_trace_stack_frame *frame, *top, *call;
+ zend_jit_trace_stack *stack;
+ zend_uchar res_type = IS_UNKNOWN;
+ const zend_op *opline, *orig_opline;
+ const zend_ssa_op *ssa_op, *orig_ssa_op;
+
+ checkpoint = zend_arena_checkpoint(CG(arena));
+
+ ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
+
+ p = trace_buffer;
+ ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
+ op_array = p->op_array;
+ frame = JIT_G(current_frame);
+ top = zend_jit_trace_call_frame(frame, op_array);
+ frame->call = NULL;
+ frame->prev = NULL;
+ frame->func = (const zend_function*)op_array;
+ frame->return_value_used = -1;
+ frame->nested = 0;
+ frame->num_args = -1;
+ stack = frame->stack;
+
+ if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
+ i = 0;
+ while (i < op_array->last_var) {
+ if (!(ssa->var_info[i].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[i].type)) {
+ stack[i] = concrete_type(ssa->var_info[i].type);
+ } else if (i < op_array->num_args) {
+ stack[i] = IS_UNKNOWN;
+ } else {
+ stack[i] = IS_UNDEF;
+ }
+ i++;
+ }
+ } else {
+ int parent_vars_count = 0;
+ zend_jit_trace_stack *parent_stack = NULL;
+
+ i = 0;
+ if (parent_trace) {
+ parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
+ op_array->last_var + op_array->T);
+ if (parent_vars_count) {
+ parent_stack =
+ zend_jit_traces[parent_trace].stack_map +
+ zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
+ }
+ }
+ while (i < op_array->last_var + op_array->T) {
+ if (!(ssa->var_info[i].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[i].type)) {
+ stack[i] = concrete_type(ssa->var_info[i].type);
+ } else if (i < parent_vars_count
+ && (zend_uchar)parent_stack[i] != IS_UNKNOWN) {
+ stack[i] = parent_stack[i];
+ } else {
+ stack[i] = IS_UNKNOWN;
+ }
+ i++;
+ }
+ }
+
+ opline = ((zend_jit_trace_start_rec*)p)->opline;
+ name = zend_jit_trace_name(op_array, opline->lineno);
+ p += ZEND_JIT_TRACE_START_REC_SIZE;
+
+ dasm_init(&dasm_state, DASM_MAXSECTION);
+ dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
+ dasm_setup(&dasm_state, dasm_actions);
+
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ op_array_ssa = &jit_extension->func_info.ssa;
+
+ // TODO: register allocation ???
+
+ dasm_growpc(&dasm_state, 1); /* trace needs just one global lable for loop */
+
+ zend_jit_align_func(&dasm_state);
+ if (!parent_trace) {
+ zend_jit_prologue(&dasm_state);
+ }
+ zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM);
+
+ if (!parent_trace) {
+ zend_jit_set_opline(&dasm_state, opline);
+ } else {
+ if (!((uintptr_t)zend_jit_traces[parent_trace].exit_info[exit_num].opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
+ zend_jit_trace_opline_guard(&dasm_state, opline);
+ zend_jit_set_opline(&dasm_state, opline);
+ } else {
+ zend_jit_reset_opline(&dasm_state, opline);
+ }
+ }
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
+ || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
+ || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ /* Check loop-invariant varaible types */
+ for (i = 0; i < op_array->last_var + op_array->T; i++) {
+ uint32_t info = ssa->var_info[i].type;
+
+ ZEND_ASSERT(ssa->vars[i].definition == -1);
+ ZEND_ASSERT(ssa->vars[i].definition_phi == NULL);
+ ZEND_ASSERT(ssa->vars[i].var == i);
+
+ if (info & MAY_BE_GUARD) {
+ if (ssa->vars[i].use_chain != -1
+ || (ssa->vars[i].phi_use_chain
+ && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD))) {
+ if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
+ goto jit_failure;
+ }
+ ssa->var_info[i].type = info & ~MAY_BE_GUARD;
+ stack[i] = concrete_type(info);
+ }
+ }
+ }
+ }
+
+ zend_jit_label(&dasm_state, 0); /* start of of trace loop */
+
+ if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+ // TODO: interupt exit may require deoptimization through side exit ???
+ zend_jit_check_timeout(&dasm_state, opline);
+ }
+ }
+
+ ssa_op = ssa->ops;
+ for (;;p++) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ uint8_t op1_type = p->op1_type;
+ uint8_t op2_type = p->op2_type;
+ uint8_t op3_type = p->op3_type;
+
+ opline = p->opline;
+ if (op1_type & IS_TRACE_REFERENCE) {
+ op1_type = IS_UNKNOWN;
+ }
+ if (op2_type & IS_TRACE_REFERENCE) {
+ op2_type = IS_UNKNOWN;
+ }
+ if (op3_type & IS_TRACE_REFERENCE) {
+ op3_type = IS_UNKNOWN;
+ }
+
+ if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
+ // TODO: support for recorded classes ???
+ p++;
+ }
+ if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ // TODO: support for recorded classes ???
+ p++;
+ }
+
+#if 0
+ // TODO: call level calculation doesn't work for traces ???
+ switch (opline->opcode) {
+ case ZEND_INIT_FCALL:
+ case ZEND_INIT_FCALL_BY_NAME:
+ case ZEND_INIT_NS_FCALL_BY_NAME:
+ case ZEND_INIT_METHOD_CALL:
+ case ZEND_INIT_DYNAMIC_CALL:
+ case ZEND_INIT_STATIC_METHOD_CALL:
+ case ZEND_INIT_USER_CALL:
+ case ZEND_NEW:
+ call_level++;
+ }
+#endif
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) {
+ switch (opline->opcode) {
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ if (opline->op1_type != IS_CV) {
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ if (!(op1_info & MAY_BE_LONG)) {
+ break;
+ }
+ if (opline->result_type != IS_UNUSED) {
+ res_use_info = RES_USE_INFO_EX();
+ USE_RES_TRACE_TYPE();
+ res_info = RES_INFO_EX();
+ res_addr = RES_REG_ADDR();
+ } else {
+ res_use_info = -1;
+ res_info = -1;
+ res_addr = 0;
+ }
+ op1_def_info = OP1_DEF_INFO_EX();
+ if (!zend_jit_inc_dec(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(),
+ op1_def_info, OP1_DEF_REG_ADDR(),
+ res_use_info, res_info,
+ res_addr,
+ (op1_def_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) {
+ ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
+ if (opline->result_type != IS_UNUSED) {
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+ }
+ }
+ goto done;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ break;
+ }
+ if (!(op1_info & MAY_BE_LONG)
+ || !(op2_info & MAY_BE_LONG)) {
+ break;
+ }
+ if (opline->result_type == IS_TMP_VAR
+ && (p+1)->op == ZEND_JIT_TRACE_VM
+ && (p+1)->opline == opline + 1
+ && (opline+1)->opcode == ZEND_SEND_VAL
+ && (opline+1)->op1_type == IS_TMP_VAR
+ && (opline+1)->op1.var == opline->result.var) {
+ p++;
+ if (frame->call) {
+ uint8_t res_type = p->op1_type;
+ if (res_type & IS_TRACE_REFERENCE) {
+ res_type = IS_UNKNOWN;;
+ }
+ if (res_type != IS_UNKNOWN) {
+ zend_jit_trace_send_type(opline+1, frame->call, res_type);
+ }
+ }
+ while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
+ (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ p++;
+ }
+ send_result = 1;
+ res_use_info = -1;
+ res_addr = 0; /* set inside backend */
+ } else {
+ send_result = 0;
+ res_use_info = RES_USE_INFO_EX();
+ USE_RES_TRACE_TYPE();
+ res_addr = RES_REG_ADDR();
+ }
+ res_info = RES_INFO_EX();
+ if (!zend_jit_long_math(&dasm_state, opline, op_array,
+ op1_info, OP1_RANGE_EX(), OP1_REG_ADDR(),
+ op2_info, OP2_RANGE_EX(), OP2_REG_ADDR(),
+ res_use_info, res_info, res_addr,
+ send_result,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+// case ZEND_DIV: // TODO: check for division by zero ???
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ break;
+ }
+ if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
+ !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ break;
+ }
+ if (opline->result_type == IS_TMP_VAR
+ && (p+1)->op == ZEND_JIT_TRACE_VM
+ && (p+1)->opline == opline + 1
+ && (opline+1)->opcode == ZEND_SEND_VAL
+ && (opline+1)->op1_type == IS_TMP_VAR
+ && (opline+1)->op1.var == opline->result.var) {
+ p++;
+ if (frame->call
+ && frame->call->func->type == ZEND_USER_FUNCTION) {
+ uint8_t res_type = p->op1_type;
+ if (res_type & IS_TRACE_REFERENCE) {
+ res_type = IS_UNKNOWN;;
+ }
+ if (res_type != IS_UNKNOWN) {
+ zend_jit_trace_send_type(opline+1, frame->call, res_type);
+ }
+ }
+ while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
+ (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ p++;
+ }
+ send_result = 1;
+ res_use_info = -1;
+ res_addr = 0; /* set inside backend */
+ } else {
+ send_result = 0;
+ res_use_info = RES_USE_INFO_EX();
+ USE_RES_TRACE_TYPE();
+ res_addr = RES_REG_ADDR();
+ }
+ res_info = RES_INFO_EX();
+ if (!zend_jit_math(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(),
+ op2_info, OP2_REG_ADDR(),
+ res_use_info, res_info, res_addr,
+ send_result,
+ (res_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow_ex(opline, ssa_op, op_array, ssa),
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ if ((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) {
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+ }
+ goto done;
+ case ZEND_CONCAT:
+ case ZEND_FAST_CONCAT:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ break;
+ }
+ if (!(op1_info & MAY_BE_STRING) ||
+ !(op2_info & MAY_BE_STRING)) {
+ break;
+ }
+ if (opline->result_type == IS_TMP_VAR
+ && (p+1)->op == ZEND_JIT_TRACE_VM
+ && (p+1)->opline == opline + 1
+ && (opline+1)->opcode == ZEND_SEND_VAL
+ && (opline+1)->op1_type == IS_TMP_VAR
+ && (opline+1)->op1.var == opline->result.var) {
+ p++;
+ if (frame->call
+ && frame->call->func->type == ZEND_USER_FUNCTION) {
+ uint8_t res_type = p->op1_type;
+ if (res_type & IS_TRACE_REFERENCE) {
+ res_type = IS_UNKNOWN;;
+ }
+ if (res_type != IS_UNKNOWN) {
+ zend_jit_trace_send_type(opline+1, frame->call, res_type);
+ }
+ }
+ while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
+ (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ p++;
+ }
+ send_result = 1;
+ } else {
+ send_result = 0;
+ }
+ res_info = RES_INFO_EX();
+ if (!zend_jit_concat(&dasm_state, opline, op_array,
+ op1_info, op2_info, res_info, send_result,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ASSIGN_OP:
+ if (opline->extended_value == ZEND_POW
+ || opline->extended_value == ZEND_DIV) {
+ // TODO: check for division by zero ???
+ break;
+ }
+ if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ break;
+ }
+ if (opline->extended_value == ZEND_ADD
+ || opline->extended_value == ZEND_SUB
+ || opline->extended_value == ZEND_MUL
+ || opline->extended_value == ZEND_DIV) {
+ if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
+ || !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ break;
+ }
+ } else if (opline->extended_value == ZEND_BW_OR
+ || opline->extended_value == ZEND_BW_AND
+ || opline->extended_value == ZEND_BW_XOR
+ || opline->extended_value == ZEND_SL
+ || opline->extended_value == ZEND_SR
+ || opline->extended_value == ZEND_MOD) {
+ if (!(op1_info & MAY_BE_LONG)
+ || !(op2_info & MAY_BE_LONG)) {
+ break;
+ }
+ } else if (opline->extended_value == ZEND_CONCAT) {
+ if (!(op1_info & MAY_BE_STRING)
+ || !(op2_info & MAY_BE_STRING)) {
+ break;
+ }
+ }
+ op1_def_info = OP1_DEF_INFO_EX();
+ if (!zend_jit_assign_op(&dasm_state, opline, op_array,
+ op1_info, op1_def_info, OP1_RANGE_EX(),
+ op2_info, OP2_RANGE_EX(),
+ (op1_def_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow_ex(opline, ssa_op, op_array, ssa),
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) {
+ ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
+ if (opline->result_type != IS_UNUSED) {
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+ }
+ }
+ goto done;
+ case ZEND_ASSIGN_DIM_OP:
+ if (opline->extended_value == ZEND_POW
+ || opline->extended_value == ZEND_DIV) {
+ // TODO: check for division by zero ???
+ break;
+ }
+ if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ op1_data_info = OP1_DATA_INFO_EX();
+ CHECK_OP1_DATA_TRACE_TYPE();
+ op1_def_info = OP1_DEF_INFO_EX();
+ if (!zend_jit_assign_dim_op(&dasm_state, opline, op_array,
+ op1_info, op1_def_info, op2_info,
+ op1_data_info, OP1_DATA_RANGE(),
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ASSIGN_DIM:
+ if (opline->op1_type != IS_CV) {
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ op1_data_info = OP1_DATA_INFO_EX();
+ CHECK_OP1_DATA_TRACE_TYPE();
+ if (!zend_jit_assign_dim(&dasm_state, opline, op_array,
+ op1_info, op2_info, op1_data_info,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ASSIGN:
+ if (opline->op1_type != IS_CV) {
+ break;
+ }
+ if (opline->result_type == IS_UNUSED) {
+ res_addr = 0;
+ res_info = -1;
+ } else {
+ res_addr = RES_REG_ADDR();
+ res_info = RES_INFO_EX();
+ }
+ op2_addr = OP2_REG_ADDR();
+ if (ra
+ && ssa_op->op2_def >= 0
+ && !ssa->vars[ssa_op->op2_def].no_val) {
+ op2_def_addr = OP2_DEF_REG_ADDR();
+ } else {
+ op2_def_addr = op2_addr;
+ }
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ op1_info = OP1_INFO_EX();
+ op1_def_info = OP1_DEF_INFO_EX();
+ USE_OP1_TRACE_TYPE();
+ if (!zend_jit_assign(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(),
+ op1_def_info, OP1_DEF_REG_ADDR(),
+ op2_info, op2_addr, op2_def_addr,
+ res_info, res_addr,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_QM_ASSIGN:
+ op1_addr = OP1_REG_ADDR();
+ if (ra
+ && ssa_op->op1_def >= 0
+ && !ssa->vars[ssa_op->op1_def].no_val) {
+ op1_def_addr = OP1_DEF_REG_ADDR();
+ } else {
+ op1_def_addr = op1_addr;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();//???USE_OP1_TRACE_TYPE();
+ res_info = RES_INFO_EX();
+ if (!zend_jit_qm_assign(&dasm_state, opline, op_array,
+ op1_info, op1_addr, op1_def_addr,
+ res_info, RES_REG_ADDR())) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_INIT_FCALL:
+ case ZEND_INIT_FCALL_BY_NAME:
+ if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, op_array_ssa, call_level, p + 1)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAL_EX:
+ if (opline->opcode == ZEND_SEND_VAL_EX
+ && opline->op2.num > MAX_ARG_FLAG_NUM) {
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE(); //???USE_OP1_TRACE_TYPE();
+ if (!zend_jit_send_val(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR())) {
+ goto jit_failure;
+ }
+ if (frame->call
+ && frame->call->func->type == ZEND_USER_FUNCTION) {
+ if (opline->op1_type == IS_CONST) {
+ zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
+ } else if (op1_type != IS_UNKNOWN) {
+ zend_jit_trace_send_type(opline, frame->call, op1_type);
+ }
+ }
+ goto done;
+ case ZEND_SEND_REF:
+ op1_info = OP1_INFO_EX();
+ USE_OP1_TRACE_TYPE();
+ if (!zend_jit_send_ref(&dasm_state, opline, op_array,
+ op1_info, 0)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ if ((opline->opcode == ZEND_SEND_VAR_EX
+ || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
+ && opline->op2.num > MAX_ARG_FLAG_NUM) {
+ break;
+ }
+ op1_addr = OP1_REG_ADDR();
+ if (ra
+ && ssa_op->op1_def >= 0
+ && !ssa->vars[ssa_op->op1_def].no_val) {
+ op1_def_addr = OP1_DEF_REG_ADDR();
+ } else {
+ op1_def_addr = op1_addr;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE(); //???USE_OP1_TRACE_TYPE();
+ if (!zend_jit_send_var(&dasm_state, opline, op_array,
+ op1_info, op1_addr, op1_def_addr)) {
+ goto jit_failure;
+ }
+ if (frame->call
+ && frame->call->func->type == ZEND_USER_FUNCTION) {
+ if (opline->opcode == ZEND_SEND_VAR_EX
+ && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
+ // TODO: this may require invalidation, if caller is changed ???
+ goto done;
+ }
+ if (op1_type != IS_UNKNOWN) {
+ zend_jit_trace_send_type(opline, frame->call, op1_type);
+ }
+ }
+ goto done;
+ case ZEND_DO_UCALL:
+ case ZEND_DO_ICALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ case ZEND_DO_FCALL:
+ if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, call_level, -1, p + 1)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_CASE:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ zend_bool exit_if_true = 0;
+ const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ goto jit_failure;
+ }
+ smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
+ } else {
+ smart_branch_opcode = 0;
+ exit_addr = NULL;
+ }
+ if (!zend_jit_cmp(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(),
+ op2_info, OP2_REG_ADDR(),
+ RES_REG_ADDR(),
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa),
+ smart_branch_opcode, -1, -1, exit_addr)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ zend_bool exit_if_true = 0;
+ const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ goto jit_failure;
+ }
+ if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
+ exit_if_true = !exit_if_true;
+ }
+ smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
+ } else {
+ smart_branch_opcode = 0;
+ exit_addr = NULL;
+ }
+ if (!zend_jit_identical(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(),
+ op2_info, OP2_REG_ADDR(),
+ RES_REG_ADDR(),
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa),
+ smart_branch_opcode, -1, -1, exit_addr)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_DEFINED:
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ zend_bool exit_if_true = 0;
+ const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ goto jit_failure;
+ }
+ smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
+ } else {
+ smart_branch_opcode = 0;
+ exit_addr = NULL;
+ }
+ if (!zend_jit_defined(&dasm_state, opline, op_array, smart_branch_opcode, -1, -1, exit_addr)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_TYPE_CHECK:
+ if (opline->extended_value == MAY_BE_RESOURCE) {
+ // TODO: support for is_resource() ???
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ USE_OP1_TRACE_TYPE();
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ zend_bool exit_if_true = 0;
+ const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ goto jit_failure;
+ }
+ smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
+ } else {
+ smart_branch_opcode = 0;
+ exit_addr = NULL;
+ }
+ if (!zend_jit_type_check(&dasm_state, opline, op_array, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_RETURN:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ if (opline->op1_type == IS_CONST) {
+ res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
+ } else if (op1_type != IS_UNKNOWN) {
+ res_type = op1_type;
+ }
+ if (op_array->type == ZEND_EVAL_CODE
+ // TODO: support for top-level code
+ || !op_array->function_name
+ // TODO: support for IS_UNDEF ???
+ || (op1_info & MAY_BE_UNDEF)) {
+ if (!zend_jit_tail_handler(&dasm_state, opline)) {
+ goto jit_failure;
+ }
+ } else {
+ int j;
+
+ if (!zend_jit_return(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR())) {
+ goto jit_failure;
+ }
+ if (!zend_jit_leave_frame(&dasm_state)) {
+ goto jit_failure;
+ }
+ for (j = 0 ; j < op_array->last_var; j++) {
+ // TODO: get info from trace ???
+ uint32_t info = zend_ssa_cv_info(opline, op_array, op_array_ssa, j);
+ zend_uchar type = stack[j];
+
+ info = zend_jit_trace_type_to_info_ex(type, info);
+ if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ if (!zend_jit_free_cv(&dasm_state, opline, op_array, info, j)) {
+ goto jit_failure;
+ }
+ }
+ }
+ if (!zend_jit_leave_func(&dasm_state, opline, op_array, p + 1)) {
+ goto jit_failure;
+ }
+ }
+ goto done;
+ case ZEND_BOOL:
+ case ZEND_BOOL_NOT:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
+ -1, -1,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa),
+ opline->opcode, NULL)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ if (/*opline > op_array->opcodes + ssa->cfg.blocks[b].start && ??? */
+ opline->op1_type == IS_TMP_VAR &&
+ ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ /* smart branch */
+ break;
+ }
+ /* break missing intentionally */
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ if (opline->result_type == IS_UNDEF) {
+ res_addr = 0;
+ } else {
+ res_addr = RES_REG_ADDR();
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
+ const zend_op *exit_opline = NULL;
+ uint32_t exit_point;
+
+ if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
+ /* taken branch */
+ if (opline->opcode == ZEND_JMPNZ_EX) {
+ smart_branch_opcode = ZEND_JMPZ_EX;
+ } else if (opline->opcode == ZEND_JMPZ_EX) {
+ smart_branch_opcode = ZEND_JMPNZ_EX;
+ } else if (opline->opcode == ZEND_JMPNZ) {
+ smart_branch_opcode = ZEND_JMPZ;
+ } else {
+ smart_branch_opcode = ZEND_JMPNZ;
+ }
+ exit_opline = (opline->opcode == ZEND_JMPZNZ) ?
+ ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
+ opline + 1;
+ } else if (opline->opcode == ZEND_JMPZNZ) {
+ ZEND_ASSERT((p+1)->opline == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value));
+ smart_branch_opcode = ZEND_JMPZ;
+ exit_opline = OP_JMP_ADDR(opline, opline->op2);
+ } else if ((p+1)->opline == opline + 1) {
+ /* not taken branch */
+ smart_branch_opcode = opline->opcode;
+ exit_opline = OP_JMP_ADDR(opline, opline->op2);
+ } else {
+ ZEND_ASSERT(0);
+ }
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p+1);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ goto jit_failure;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ if (!zend_jit_bool_jmpznz(&dasm_state, opline, op_array,
+ op1_info, OP1_REG_ADDR(), res_addr,
+ -1, -1,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa),
+ smart_branch_opcode, exit_addr)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_FETCH_DIM_R:
+ case ZEND_FETCH_DIM_IS:
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ res_info = RES_INFO_EX();
+ if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array,
+ op1_info, op2_info, res_info,
+ (
+ (op1_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
+ (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) != 0 ||
+ ((op1_info & MAY_BE_UNDEF) != 0 &&
+ opline->opcode == ZEND_FETCH_DIM_R) ||
+ ((opline->op1_type & (IS_TMP_VAR|IS_VAR)) != 0 &&
+ (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) != 0) ||
+ (op2_info & MAY_BE_UNDEF) != 0 ||
+ ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) != 0 &&
+ (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) != 0)))) {
+ goto jit_failure;
+ }
+ goto done;
+ goto done;
+ case ZEND_ISSET_ISEMPTY_DIM_OBJ:
+ if ((opline->extended_value & ZEND_ISEMPTY)) {
+ // TODO: support for empty() ???
+ break;
+ }
+ op1_info = OP1_INFO_EX();
+ CHECK_OP1_TRACE_TYPE();
+ op2_info = OP2_INFO_EX();
+ CHECK_OP2_TRACE_TYPE();
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
+ zend_bool exit_if_true = 0;
+ const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ goto jit_failure;
+ }
+ smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
+ } else {
+ smart_branch_opcode = 0;
+ exit_addr = NULL;
+ }
+ if (!zend_jit_isset_isempty_dim(&dasm_state, opline, op_array,
+ op1_info, op2_info,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa),
+ smart_branch_opcode, -1, -1,
+ exit_addr)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_FETCH_OBJ_R:
+ case ZEND_FETCH_OBJ_IS:
+ if (opline->op2_type != IS_CONST
+ || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
+ || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
+ break;
+ }
+ ce = NULL;
+ if (opline->op1_type == IS_UNUSED) {
+ op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
+ ce = op_array->scope;
+ } else {
+ op1_info = OP1_INFO_EX();
+ if (ssa->var_info && ssa->ops) {
+ if (ssa_op->op1_use >= 0) {
+ zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
+ if (op1_ssa->ce && !op1_ssa->is_instanceof && !op1_ssa->ce->create_object) {
+ ce = op1_ssa->ce;
+ }
+ }
+ }
+ }
+ if (!(op1_info & MAY_BE_OBJECT)) {
+ break;
+ }
+ if (!zend_jit_fetch_obj_read(&dasm_state, opline, op_array,
+ op1_info, ce,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_BIND_GLOBAL:
+ orig_opline = opline;
+ orig_ssa_op = ssa_op;
+ while (1) {
+ if (!ssa->ops || !ssa->var_info) {
+ op1_info = MAY_BE_ANY|MAY_BE_REF;
+ } else {
+ op1_info = OP1_INFO_EX();
+ }
+ if (!zend_jit_bind_global(&dasm_state, opline, op_array, op1_info)) {
+ goto jit_failure;
+ }
+ if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
+ opline++;
+ ssa_op++;
+ } else {
+ break;
+ }
+ }
+ opline = orig_opline;
+ ssa_op = orig_ssa_op;
+ goto done;
+ case ZEND_RECV:
+ if (!zend_jit_recv(&dasm_state, opline, op_array)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_RECV_INIT:
+ orig_opline = opline;
+ orig_ssa_op = ssa_op;
+ while (1) {
+ if (!zend_jit_recv_init(&dasm_state, opline, op_array,
+ (opline + 1)->opcode != ZEND_RECV_INIT,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ if ((opline+1)->opcode == ZEND_RECV_INIT) {
+ opline++;
+ ssa_op++;
+ } else {
+ break;
+ }
+ }
+ opline = orig_opline;
+ ssa_op = orig_ssa_op;
+ goto done;
+ case ZEND_FREE:
+ case ZEND_FE_FREE:
+ op1_info = OP1_INFO_EX();
+ USE_OP1_TRACE_TYPE();
+ if (!zend_jit_free(&dasm_state, opline, op_array, op1_info,
+ zend_may_throw_ex(opline, ssa_op, op_array, ssa))) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ECHO:
+ if (opline->op1_type != IS_CONST
+ || Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) != IS_STRING) {
+ break;
+ }
+ if (!zend_jit_echo(&dasm_state, opline, op_array)) {
+ goto jit_failure;
+ }
+ goto done;
+#if 0
+ case ZEND_SWITCH_LONG:
+ case ZEND_SWITCH_STRING:
+ if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+#endif
+// case ZEND_INIT_NS_FCALL_BY_NAME:
+ // TODO: we may need a guard after INIT_NS_FCALL???
+ case ZEND_INIT_METHOD_CALL:
+ case ZEND_INIT_DYNAMIC_CALL:
+ if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) {
+ goto jit_failure;
+ }
+ if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL) {
+ if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) {
+ goto jit_failure;
+ }
+ }
+ goto done;
+ case ZEND_INIT_STATIC_METHOD_CALL:
+ if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) {
+ goto jit_failure;
+ }
+ if ((opline->op1_type != IS_CONST
+ || opline->op2_type != IS_CONST)
+ && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL) {
+ if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) {
+ goto jit_failure;
+ }
+ }
+ goto done;
+ case ZEND_INIT_USER_CALL:
+ if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) {
+ goto jit_failure;
+ }
+ if (opline->op2_type != IS_CONST
+ && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL) {
+ if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) {
+ goto jit_failure;
+ }
+ }
+ goto done;
+ case ZEND_NEW:
+ if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) {
+ goto jit_failure;
+ }
+ if (opline->op1_type != IS_CONST
+ && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL) {
+ if (!zend_jit_init_fcall_guard(&dasm_state, opline, (p+1)->func)) {
+ goto jit_failure;
+ }
+ }
+ goto done;
+ default:
+ break;
+ }
+ }
+
+ if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
+ if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw_ex(opline, ssa_op, op_array, ssa), p + 1)) {
+ goto jit_failure;
+ }
+ }
+
+done:
+#if 0
+ // TODO: call level calculation doesn't work for traces ???
+ switch (opline->opcode) {
+ case ZEND_DO_FCALL:
+ case ZEND_DO_ICALL:
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ call_level--;
+ }
+#endif
+
+ /* Keep information about known types on abstract stack */
+ if (ssa_op->result_def >= 0) {
+ zend_uchar type = IS_UNKNOWN;
+
+ if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
+ || send_result) {
+ /* we didn't set result variable */
+ type = IS_UNKNOWN;
+ } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
+ type = concrete_type(ssa->var_info[ssa_op->result_def].type);
+ } else if (opline->opcode == ZEND_QM_ASSIGN) {
+ if (opline->op1_type != IS_CONST) {
+ /* copy */
+ type = STACK_VAR_TYPE(opline->op1.var);
+ }
+ } else if (opline->opcode == ZEND_ASSIGN) {
+ if (opline->op2_type != IS_CONST) {
+ /* copy */
+ type = STACK_VAR_TYPE(opline->op2.var);
+ }
+ } else if (opline->opcode == ZEND_POST_INC
+ || opline->opcode == ZEND_POST_DEC) {
+ /* copy */
+ type = STACK_VAR_TYPE(opline->op1.var);
+ }
+ SET_RES_STACK_VAR_TYPE(type);
+ if (type != IS_UNKNOWN) {
+ ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
+ }
+ }
+ if (ssa_op->op1_def >= 0) {
+ zend_uchar type = IS_UNKNOWN;
+
+ if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
+ type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
+ } else if (opline->opcode == ZEND_ASSIGN) {
+ if (!(OP1_INFO_EX() & MAY_BE_REF)
+ || STACK_VAR_TYPE(opline->op1.var) != IS_UNKNOWN) {
+ if (opline->op2_type != IS_CONST) {
+ /* copy */
+ type = STACK_VAR_TYPE(opline->op2.var);
+ }
+ }
+ } else if (opline->opcode == ZEND_SEND_VAR
+ || opline->opcode == ZEND_CAST
+ || opline->opcode == ZEND_QM_ASSIGN
+ || opline->opcode == ZEND_JMP_SET
+ || opline->opcode == ZEND_COALESCE
+ || opline->opcode == ZEND_FE_RESET_R) {
+ /* keep old value */
+ type = STACK_VAR_TYPE(opline->op1.var);
+ }
+ SET_OP1_STACK_VAR_TYPE(type);
+ if (type != IS_UNKNOWN) {
+ ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
+ }
+ }
+ if (ssa_op->op2_def >= 0) {
+ zend_uchar type = IS_UNKNOWN;
+
+ if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
+ type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
+ } else if (opline->opcode == ZEND_ASSIGN) {
+ /* keep old value */
+ type = STACK_VAR_TYPE(opline->op2.var);
+ }
+ SET_OP2_STACK_VAR_TYPE(type);
+ if (type != IS_UNKNOWN) {
+ ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
+ }
+ }
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_DIM:
+ case ZEND_ASSIGN_OBJ:
+ case ZEND_ASSIGN_STATIC_PROP:
+ case ZEND_ASSIGN_DIM_OP:
+ case ZEND_ASSIGN_OBJ_OP:
+ case ZEND_ASSIGN_STATIC_PROP_OP:
+ case ZEND_ASSIGN_OBJ_REF:
+ case ZEND_ASSIGN_STATIC_PROP_REF:
+ /* OP_DATA */
+ ssa_op++;
+ opline++;
+ if (ssa_op->op1_def >= 0) {
+ zend_uchar type = IS_UNKNOWN;
+
+ if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
+ type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
+ } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
+ || (opline-1)->opcode == ZEND_ASSIGN_OBJ
+ || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
+ /* keep old value */
+ type = STACK_VAR_TYPE(opline->op1.var);
+ }
+ SET_OP1_STACK_VAR_TYPE(type);
+ if (type != IS_UNKNOWN) {
+ ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
+ }
+ }
+ ssa_op++;
+ break;
+ case ZEND_RECV_INIT:
+ ssa_op++;
+ opline++;
+ while (opline->opcode == ZEND_RECV_INIT) {
+ if (ssa_op->result_def >= 0) {
+ zend_uchar type = IS_UNKNOWN;
+
+ if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
+ type = concrete_type(ssa->var_info[ssa_op->result_def].type);
+ }
+ SET_RES_STACK_VAR_TYPE(type);
+ }
+ ssa_op++;
+ opline++;
+ }
+ break;
+ case ZEND_BIND_GLOBAL:
+ ssa_op++;
+ opline++;
+ while (opline->opcode == ZEND_BIND_GLOBAL) {
+ if (ssa_op->op1_def >= 0) {
+ zend_uchar type = IS_UNKNOWN;
+
+ if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
+ type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
+ }
+ SET_OP1_STACK_VAR_TYPE(type);
+ }
+ ssa_op++;
+ opline++;
+ }
+ break;
+ default:
+ ssa_op += zend_jit_trace_op_len(opline);
+ break;
+ }
+
+ if (send_result) {
+ ssa_op++;
+ send_result = 0;
+ }
+
+ } else if (p->op == ZEND_JIT_TRACE_ENTER) {
+ op_array = (zend_op_array*)p->op_array;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ op_array_ssa = &jit_extension->func_info.ssa;
+ call = frame->call;
+ if (!call) {
+ /* Trace missed INIT_FCALL opcode */
+ call = top;
+ call->call = NULL;
+ call->prev = NULL;
+ call->func = (const zend_function*)op_array;
+ call->nested = 0;
+ call->num_args = -1; // TODO: should be possible to get the real number ???
+ top = zend_jit_trace_call_frame(top, op_array);
+ i = 0;
+ while (i < p->op_array->num_args) {
+ /* Initialize abstract stack using SSA */
+ if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) {
+ call->stack[i] = concrete_type(ssa->var_info[p->first_ssa_var + i].type);
+ } else {
+ call->stack[i] = IS_UNKNOWN;
+ }
+ i++;
+ }
+ while (i < p->op_array->last_var) {
+ call->stack[i] = IS_UNDEF;
+ i++;
+ }
+ while (i < p->op_array->last_var + p->op_array->T) {
+ call->stack[i] = IS_UNKNOWN;
+ i++;
+ }
+ } else {
+ ZEND_ASSERT(&call->func->op_array == op_array);
+ }
+ frame->call = call->prev;
+ call->prev = frame;
+ call->return_value_used = p->return_value_used;
+ JIT_G(current_frame) = frame = call;
+ stack = frame->stack;
+ zend_jit_set_opline(&dasm_state, (p+1)->opline);
+ } else if (p->op == ZEND_JIT_TRACE_BACK) {
+ op_array = (zend_op_array*)p->op_array;
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ op_array_ssa = &jit_extension->func_info.ssa;
+ top = frame;
+ if (frame->prev) {
+ frame = frame->prev;
+ stack = frame->stack;
+ ZEND_ASSERT(&frame->func->op_array == op_array);
+ } else {
+ frame = zend_jit_trace_ret_frame(frame, op_array);
+ frame->call = NULL;
+ frame->prev = NULL;
+ frame->func = (const zend_function*)op_array;
+ frame->return_value_used = -1;
+ frame->nested = 0;
+ frame->num_args = -1;
+ stack = frame->stack;
+ for (i = 0; i < op_array->last_var + op_array->T; i++) {
+ /* Initialize abstract stack using SSA */
+ if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD)
+ && has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) {
+ stack[i] = concrete_type(ssa->var_info[p->first_ssa_var + i].type);
+ } else {
+ stack[i] = IS_UNKNOWN;
+ }
+ }
+ }
+ JIT_G(current_frame) = frame;
+ if (res_type != IS_UNKNOWN
+ && (p+1)->op == ZEND_JIT_TRACE_VM) {
+ const zend_op *opline = (p+1)->opline - 1;
+ if (opline->result_type != IS_UNUSED) {
+ SET_RES_STACK_VAR_TYPE(res_type);
+ }
+ }
+ res_type = IS_UNKNOWN;
+ } else if (p->op == ZEND_JIT_TRACE_END) {
+ break;
+ } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
+ call = top;
+ call->call = NULL;
+ call->prev = frame->call;
+ call->func = p->func;
+ call->nested = 1;
+ call->num_args = find_call_num_args(p-1);
+ frame->call = call;
+ top = zend_jit_trace_call_frame(top, p->op_array);
+ if (p->func->type == ZEND_USER_FUNCTION) {
+ i = 0;
+ while (i < p->op_array->num_args) {
+ /* Types of arguments are going to be stored in abstract stack when proseccin SEV onstruction */
+ call->stack[i] = IS_UNKNOWN;
+ i++;
+ }
+ while (i < p->op_array->last_var) {
+ call->stack[i] = IS_UNDEF;
+ i++;
+ }
+ while (i < p->op_array->last_var + p->op_array->T) {
+ call->stack[i] = IS_UNKNOWN;
+ i++;
+ }
+ }
+ if (p->fake) {
+ if (!zend_jit_init_fcall_guard(&dasm_state, NULL, p->func)) {
+ goto jit_failure;
+ }
+ }
+ } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
+ call = frame->call;
+ if (call) {
+ top = call;
+ frame->call = call->prev;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ }
+
+ ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
+
+ t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
+
+ if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
+ || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
+ || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+ if (p->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ if (!zend_jit_set_valid_ip(&dasm_state, p->opline)) {
+ goto jit_failure;
+ }
+ }
+ t->link = ZEND_JIT_TRACE_NUM;
+ zend_jit_jmp(&dasm_state, 0); /* jump back to start of the trace loop */
+ } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
+ if (!zend_jit_set_valid_ip(&dasm_state, p->opline)) {
+ goto jit_failure;
+ }
+ t->link = zend_jit_find_trace(p->opline->handler);
+ zend_jit_trace_link_to_root(&dasm_state, p->opline->handler);
+ } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
+ zend_jit_trace_return(&dasm_state);
+ } else {
+ // TODO: not implemented ???
+ ZEND_ASSERT(0 && p->stop);
+ }
+
+ if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= ZEND_JIT_TRACE_MAX_EXIT_COUNTERS) {
+ goto jit_failure;
+ }
+
+ handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), 1);
+
+jit_failure:
+ dasm_free(&dasm_state);
+
+ if (name) {
+ zend_string_release(name);
+ }
+
+ /* Clenup used op_arrays */
+ while (num_op_arrays > 0) {
+ op_array = op_arrays[--num_op_arrays];
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+
+ memset(&jit_extension->func_info, 0, sizeof(jit_extension->func_info));
+ jit_extension->func_info.num_args = -1;
+ jit_extension->func_info.return_value_used = -1;
+ }
+
+ zend_arena_release(&CG(arena), checkpoint);
+
+ JIT_G(current_frame) = NULL;
+
+ return handler;
+}
+
+static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
+{
+ const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
+
+ opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED));
+ if (opline) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
+{
+ const void *handler = NULL;
+ dasm_State* dasm_state = NULL;
+ void *checkpoint;
+ char name[32];
+ const zend_op *opline;
+
+ if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
+ return dasm_labels[zend_lbtrace_escape];
+ }
+
+ checkpoint = zend_arena_checkpoint(CG(arena));;
+
+ sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
+
+ dasm_init(&dasm_state, DASM_MAXSECTION);
+ dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
+ dasm_setup(&dasm_state, dasm_actions);
+ dasm_growpc(&dasm_state, 0);
+
+ zend_jit_align_func(&dasm_state);
+
+ // TODO: Generate deoptimization code ???
+
+ opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
+ opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED));
+ if (opline) {
+ zend_jit_set_ip(&dasm_state, opline);
+ }
+
+ zend_jit_trace_return(&dasm_state);
+
+ handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 1);
+
+ dasm_free(&dasm_state);
+
+ zend_arena_release(&CG(arena), checkpoint);
+
+ return handler;
+}
+
+static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
+{
+ zend_jit_trace_stop ret;
+ const void *handler;
+ zend_jit_trace_info *t;
+ zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
+
+ zend_shared_alloc_lock();
+
+ /* Checks under lock */
+ if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
+ ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
+ } else if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) {
+ ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
+ } else {
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
+
+ t->id = ZEND_JIT_TRACE_NUM;
+ t->root = ZEND_JIT_TRACE_NUM;
+ t->parent = 0;
+ t->link = 0;
+ t->exit_count = 0;
+ t->child_count = 0;
+ t->stack_map_size = 0;
+ t->exit_info = exit_info;
+ t->stack_map = NULL;
+
+ handler = zend_jit_trace(trace_buffer, 0, 0);
+
+ if (handler) {
+ zend_jit_trace_exit_info *shared_exit_info = NULL;
+
+ t->exit_info = NULL;
+ if (t->exit_count) {
+ /* reallocate exit_info into shared memory */
+ shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
+ sizeof(zend_jit_trace_exit_info) * t->exit_count);
+
+ if (!shared_exit_info) {
+ if (t->stack_map) {
+ efree(t->stack_map);
+ t->stack_map = NULL;
+ }
+ ret = ZEND_JIT_TRACE_STOP_NO_SHM;
+ goto exit;
+ }
+ memcpy(shared_exit_info, exit_info,
+ sizeof(zend_jit_trace_exit_info) * t->exit_count);
+ t->exit_info = shared_exit_info;
+ }
+
+ if (t->stack_map_size) {
+ zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
+ if (!shared_stack_map) {
+ ret = ZEND_JIT_TRACE_STOP_NO_SHM;
+ goto exit;
+ }
+ memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
+ efree(t->stack_map);
+ t->stack_map = shared_stack_map;
+ }
+
+ t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
+ ZEND_JIT_EXIT_COUNTERS += t->exit_count;
+
+ ((zend_op*)opline)->handler = handler;
+
+ ZEND_JIT_TRACE_NUM++;
+ ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
+
+ ret = ZEND_JIT_TRACE_STOP_COMPILED;
+ } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
+ ZEND_JIT_EXIT_COUNTERS + t->exit_count >= ZEND_JIT_TRACE_MAX_EXIT_COUNTERS) {
+ if (t->stack_map) {
+ efree(t->stack_map);
+ t->stack_map = NULL;
+ }
+ ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
+ } else {
+ if (t->stack_map) {
+ efree(t->stack_map);
+ t->stack_map = NULL;
+ }
+ ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
+ }
+
+exit:
+ zend_jit_protect();
+ SHM_PROTECT();
+ }
+
+ zend_shared_alloc_unlock();
+
+ return ret;
+}
+
+static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
+{
+ zend_shared_alloc_lock();
+
+ if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ ((zend_op*)opline)->handler =
+ ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
+
+ ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
+
+ zend_jit_protect();
+ SHM_PROTECT();
+ }
+
+ zend_shared_alloc_unlock();
+}
+
+static zend_bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
+{
+ const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
+ uint8_t *cache_count = JIT_G(bad_root_cache_count);
+ uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
+ uint32_t cache_slot = JIT_G(bad_root_slot);
+ uint32_t i;
+
+ for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
+ if (cache_opline[i] == opline) {
+ if (cache_count[i] >= ZEND_JIT_TRACE_MAX_ROOT_FAILURES - 1) {
+ cache_opline[i] = NULL;
+ return 1;
+ } else {
+#if 0
+ if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
+ *ZEND_OP_TRACE_INFO(opline, offset)->counter =
+ random() % ZEND_JIT_TRACE_COUNTER_MAX;
+ }
+#endif
+ cache_count[i]++;
+ cache_stop[i] = stop;
+ return 0;
+ }
+ }
+ }
+ i = cache_slot;
+ cache_opline[i] = opline;
+ cache_count[i] = 1;
+ cache_stop[i] = stop;
+ cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
+ JIT_G(bad_root_slot) = cache_slot;
+ return 0;
+}
+
+#define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
+ description,
+
+static const char * zend_jit_trace_stop_description[] = {
+ ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
+};
+
+static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
+{
+ zend_jit_trace_rec *p = trace_buffer;
+ const zend_op_array *op_array;
+ const zend_op *opline;
+ uint32_t level = 1 + trace_buffer[0].level;
+ int idx, len, i, v, vars_count, call_level;
+
+ ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
+ op_array = p->op_array;
+ p += ZEND_JIT_TRACE_START_REC_SIZE;
+ idx = 0;
+ call_level = 0;
+
+ if (tssa && tssa->var_info) {
+ if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
+ vars_count = op_array->last_var;
+ } else {
+ vars_count = op_array->last_var + op_array->T;
+ }
+ for (i = 0; i < vars_count; i++) {
+ if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
+ fprintf(stderr, " %*c;", level, ' ');
+ zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
+ fprintf(stderr, "\n");
+ }
+ }
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
+ || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
+ || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+ fprintf(stderr, "LOOP:\n");
+ if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+ zend_ssa_phi *p = tssa->blocks[1].phis;
+
+ while (p) {
+ fprintf(stderr, " ;");
+ zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
+ fprintf(stderr, " = Phi(");
+ zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
+ fprintf(stderr, ", ");
+ zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
+ fprintf(stderr, ")\n");
+ p = p->next;
+ }
+ }
+ }
+ }
+
+ while (1) {
+ if (p->op == ZEND_JIT_TRACE_VM) {
+ uint8_t op1_type, op2_type, op3_type;
+
+ opline = p->opline;
+ fprintf(stderr, "%04d%*c",
+ (int)(opline - op_array->opcodes),
+ level, ' ');
+ zend_dump_op(op_array, NULL, opline, ZEND_DUMP_NUMERIC_OPLINES|ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
+
+ op1_type = p->op1_type;
+ op2_type = p->op2_type;
+ op3_type = p->op3_type;
+ if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
+ fprintf(stderr, " ;");
+ if (op1_type != IS_UNKNOWN) {
+ const char *ref = (op1_type & IS_TRACE_REFERENCE) ? "&" : "";
+ if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
+ p++;
+ fprintf(stderr, " op1(%sobject of class %s)", ref,
+ ZSTR_VAL(p->ce->name));
+ } else {
+ const char *type = (op1_type == 0) ? "undef" : zend_get_type_by_const(op1_type & ~IS_TRACE_REFERENCE);
+ fprintf(stderr, " op1(%s%s)", ref, type);
+ }
+ }
+ if (op2_type != IS_UNKNOWN) {
+ const char *ref = (op2_type & IS_TRACE_REFERENCE) ? "&" : "";
+ if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
+ p++;
+ fprintf(stderr, " op2(%sobject of class %s)", ref,
+ ZSTR_VAL(p->ce->name));
+ } else {
+ const char *type = (op2_type == 0) ? "undef" : zend_get_type_by_const(op2_type & ~IS_TRACE_REFERENCE);
+ fprintf(stderr, " op2(%s%s)", ref, type);
+ }
+ }
+ if (op3_type != IS_UNKNOWN) {
+ const char *ref = (op3_type & IS_TRACE_REFERENCE) ? "&" : "";
+ const char *type = (op3_type == 0) ? "undef" : zend_get_type_by_const(op3_type & ~IS_TRACE_REFERENCE);
+ fprintf(stderr, " op3(%s%s)", ref, type);
+ }
+ }
+ fprintf(stderr, "\n");
+ idx++;
+
+ len = zend_jit_trace_op_len(opline);
+ while (len > 1) {
+ opline++;
+ fprintf(stderr, "%04d%*c;",
+ (int)(opline - op_array->opcodes),
+ level, ' ');
+ zend_dump_op(op_array, NULL, opline, ZEND_DUMP_NUMERIC_OPLINES|ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
+ idx++;
+ len--;
+ fprintf(stderr, "\n");
+ }
+ } else if (p->op == ZEND_JIT_TRACE_ENTER) {
+ op_array = p->op_array;
+ fprintf(stderr, " %*c>enter %s%s%s\n",
+ level, ' ',
+ op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
+ op_array->scope ? "::" : "",
+ op_array->function_name ?
+ ZSTR_VAL(op_array->function_name) :
+ ZSTR_VAL(op_array->filename));
+ level++;
+ if (tssa && tssa->var_info) {
+ call_level++;
+ v = p->first_ssa_var;
+ vars_count = op_array->last_var;
+ for (i = 0; i < vars_count; i++, v++) {
+ if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
+ fprintf(stderr, " %*c;", level, ' ');
+ zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
+ fprintf(stderr, "\n");
+ }
+ }
+ }
+ } else if (p->op == ZEND_JIT_TRACE_BACK) {
+ op_array = p->op_array;
+ level--;
+ fprintf(stderr, " %*c<back %s%s%s\n",
+ level, ' ',
+ op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
+ op_array->scope ? "::" : "",
+ op_array->function_name ?
+ ZSTR_VAL(op_array->function_name) :
+ ZSTR_VAL(op_array->filename));
+ if (tssa && tssa->var_info) {
+ if (call_level == 0) {
+ v = p->first_ssa_var;
+ vars_count = op_array->last_var + op_array->T;
+ for (i = 0; i < vars_count; i++, v++) {
+ if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
+ fprintf(stderr, " %*c;", level, ' ');
+ zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
+ fprintf(stderr, "\n");
+ }
+ }
+ } else {
+ call_level--;
+ }
+ }
+ } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
+ if (p->func != (zend_function*)&zend_pass_function) {
+ fprintf(stderr, p->fake ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
+ level, ' ',
+ p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
+ p->func->common.scope ? "::" : "",
+ ZSTR_VAL(p->func->common.function_name));
+ } else {
+ fprintf(stderr, " %*c>skip\n",
+ level, ' ');
+ }
+ } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
+ if (p->func != (zend_function*)&zend_pass_function) {
+ fprintf(stderr, " %*c>call %s%s%s\n",
+ level, ' ',
+ p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
+ p->func->common.scope ? "::" : "",
+ ZSTR_VAL(p->func->common.function_name));
+ } else {
+ fprintf(stderr, " %*c>skip\n",
+ level, ' ');
+ }
+ } else if (p->op == ZEND_JIT_TRACE_END) {
+ break;
+ }
+ p++;
+ }
+}
+
+static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
+{
+ if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
+ return "loop";
+ } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
+ return "enter";
+ } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
+ return "return";
+ } else {
+ ZEND_ASSERT(0);
+ return "???";
+ }
+}
+
+int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
+{
+ const zend_op *orig_opline;
+ zend_jit_trace_stop stop;
+ zend_op_array *op_array;
+ zend_jit_op_array_trace_extension *jit_extension;
+ size_t offset;
+ uint32_t trace_num;
+ zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
+
+ ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
+ ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
+ opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
+
+repeat:
+ trace_num = ZEND_JIT_TRACE_NUM;
+ orig_opline = opline;
+ op_array = &EX(func)->op_array;
+ jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ offset = jit_extension->offset;
+
+ EX(opline) = opline;
+
+ /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
+ if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
+ return 0;
+ }
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) {
+ fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n",
+ trace_num,
+ zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
+ EX(func)->op_array.function_name ?
+ ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
+ ZSTR_VAL(EX(func)->op_array.filename),
+ opline->lineno);
+ }
+
+ if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) {
+ stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
+ goto abort;
+ }
+
+ stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
+ ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK);
+
+ if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) {
+ /* op_array may be already deallocated */
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) {
+ fprintf(stderr, "---- TRACE %d abort (%s)\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+ goto blacklist;
+ }
+
+ if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
+ zend_jit_dump_trace(trace_buffer, NULL);
+ }
+
+ if (ZEND_JIT_TRACE_STOP_OK(stop)) {
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) {
+ if (stop == ZEND_JIT_TRACE_STOP_LINK) {
+ uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);;
+ fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
+ trace_num,
+ link_to);
+ } else {
+ fprintf(stderr, "---- TRACE %d stop (%s)\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+ }
+ stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
+ if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) {
+ fprintf(stderr, "---- TRACE %d %s\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+ } else {
+ goto abort;
+ }
+ } else {
+abort:
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) {
+ fprintf(stderr, "---- TRACE %d abort (%s)\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+blacklist:
+ if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
+ || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
+ fprintf(stderr, "---- TRACE %d blacklisted\n",
+ trace_num);
+ }
+ zend_jit_blacklist_root_trace(orig_opline, offset);
+ }
+ if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
+ execute_data = EG(current_execute_data);
+ opline = EX(opline);
+ goto repeat;
+ }
+ }
+
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
+ fprintf(stderr, "\n");
+ }
+
+ return (stop == ZEND_JIT_TRACE_STOP_HALT) ? -1 : 0;
+}
+
+static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
+{
+ const void *handler;
+
+ zend_shared_alloc_lock();
+
+ if (!((uintptr_t)zend_jit_traces[trace_num].exit_info[exit_num].opline & ZEND_JIT_EXIT_BLACKLISTED)) {
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
+
+ if (handler) {
+ zend_jit_link_side_trace(
+ zend_jit_traces[trace_num].code_start,
+ zend_jit_traces[trace_num].code_size,
+ exit_num,
+ handler);
+ }
+
+ zend_jit_traces[trace_num].exit_info[exit_num].opline = (const zend_op*)
+ ((uintptr_t)zend_jit_traces[trace_num].exit_info[exit_num].opline | ZEND_JIT_EXIT_BLACKLISTED);
+
+ zend_jit_protect();
+ SHM_PROTECT();
+ }
+
+ zend_shared_alloc_unlock();
+}
+
+static zend_bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
+{
+ uint8_t *counter = JIT_G(exit_counters) +
+ zend_jit_traces[trace_num].exit_counters + exit_num;
+
+ if (*counter + 1 >= ZEND_JIT_TRACE_HOT_SIDE_COUNT + ZEND_JIT_TRACE_MAX_SIDE_FAILURES) {
+ return 1;
+ }
+ (*counter)++;
+ return 0;
+}
+
+static zend_bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
+{
+ uint8_t *counter = JIT_G(exit_counters) +
+ zend_jit_traces[trace_num].exit_counters + exit_num;
+
+ if (*counter + 1 >= ZEND_JIT_TRACE_HOT_SIDE_COUNT) {
+ return 1;
+ }
+ (*counter)++;
+ return 0;
+}
+
+static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num)
+{
+ zend_jit_trace_stop ret;
+ const void *handler;
+ zend_jit_trace_info *t;
+ zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
+
+ zend_shared_alloc_lock();
+
+ /* Checks under lock */
+ if (((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline & ZEND_JIT_EXIT_JITED)) {
+ ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
+ } else if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) {
+ ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
+ } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= ZEND_JIT_TRACE_MAX_SIDE_TRACES) {
+ ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
+ } else {
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
+
+ t->id = ZEND_JIT_TRACE_NUM;
+ t->root = zend_jit_traces[parent_num].root;
+ t->parent = parent_num;
+ t->link = 0;
+ t->exit_count = 0;
+ t->child_count = 0;
+ t->stack_map_size = 0;
+ t->exit_info = exit_info;
+ t->stack_map = NULL;
+
+ handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
+
+ if (handler) {
+ zend_jit_trace_exit_info *shared_exit_info = NULL;
+
+ t->exit_info = NULL;
+ if (t->exit_count) {
+ /* reallocate exit_info into shared memory */
+ shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
+ sizeof(zend_jit_trace_exit_info) * t->exit_count);
+
+ if (!shared_exit_info) {
+ if (t->stack_map) {
+ efree(t->stack_map);
+ t->stack_map = NULL;
+ }
+ ret = ZEND_JIT_TRACE_STOP_NO_SHM;
+ goto exit;
+ }
+ memcpy(shared_exit_info, exit_info,
+ sizeof(zend_jit_trace_exit_info) * t->exit_count);
+ t->exit_info = shared_exit_info;
+ }
+
+ if (t->stack_map_size) {
+ zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
+ if (!shared_stack_map) {
+ efree(t->stack_map);
+ ret = ZEND_JIT_TRACE_STOP_NO_SHM;
+ goto exit;
+ }
+ memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
+ efree(t->stack_map);
+ t->stack_map = shared_stack_map;
+ }
+
+ zend_jit_link_side_trace(
+ zend_jit_traces[parent_num].code_start,
+ zend_jit_traces[parent_num].code_size,
+ exit_num,
+ handler);
+
+ t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
+ ZEND_JIT_EXIT_COUNTERS += t->exit_count;
+
+ zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
+ ZEND_JIT_TRACE_NUM++;
+ zend_jit_traces[parent_num].exit_info[exit_num].opline = (const zend_op*)
+ ((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline | ZEND_JIT_EXIT_JITED);
+
+ ret = ZEND_JIT_TRACE_STOP_COMPILED;
+ } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
+ ZEND_JIT_EXIT_COUNTERS + t->exit_count >= ZEND_JIT_TRACE_MAX_EXIT_COUNTERS) {
+ if (t->stack_map) {
+ efree(t->stack_map);
+ t->stack_map = NULL;
+ }
+ ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
+ } else {
+ if (t->stack_map) {
+ efree(t->stack_map);
+ t->stack_map = NULL;
+ }
+ ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
+ }
+
+exit:
+ zend_jit_protect();
+ SHM_PROTECT();
+ }
+
+ zend_shared_alloc_unlock();
+
+ return ret;
+}
+
+int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
+{
+ zend_jit_trace_stop stop;
+ uint32_t trace_num;
+ zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
+
+ trace_num = ZEND_JIT_TRACE_NUM;
+
+ /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
+ if ((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
+ return 0;
+ }
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) {
+ fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s() %s:%d\n",
+ trace_num, parent_num, exit_num,
+ EX(func)->op_array.function_name ?
+ ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
+ ZSTR_VAL(EX(func)->op_array.filename),
+ EX(opline)->lineno);
+ }
+
+ if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) {
+ stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
+ goto abort;
+ }
+
+ if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= ZEND_JIT_TRACE_MAX_SIDE_TRACES) {
+ stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
+ goto abort;
+ }
+
+ stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE);
+
+ if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) {
+ /* op_array may be already deallocated */
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) {
+ fprintf(stderr, "---- TRACE %d abort (%s)\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+ goto blacklist;
+ }
+
+ if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
+ zend_jit_dump_trace(trace_buffer, NULL);
+ }
+
+ if (ZEND_JIT_TRACE_STOP_OK(stop)) {
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) {
+ if (stop == ZEND_JIT_TRACE_STOP_LINK) {
+ uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);;
+ fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
+ trace_num,
+ link_to);
+ } else {
+ fprintf(stderr, "---- TRACE %d stop (%s)\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+ }
+ if (EXPECTED(stop != ZEND_JIT_TRACE_STOP_LOOP)) {
+ stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num);
+ } else {
+ const zend_op_array *op_array = trace_buffer[0].op_array;
+ zend_jit_op_array_trace_extension *jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ const zend_op *opline = trace_buffer[1].opline;
+
+ stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
+ }
+ if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) {
+ fprintf(stderr, "---- TRACE %d %s\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+ } else {
+ goto abort;
+ }
+ } else {
+abort:
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) {
+ fprintf(stderr, "---- TRACE %d abort (%s)\n",
+ trace_num,
+ zend_jit_trace_stop_description[stop]);
+ }
+blacklist:
+ if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
+ || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
+ zend_jit_blacklist_trace_exit(parent_num, exit_num);
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
+ fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
+ parent_num, exit_num);
+ }
+ }
+ }
+
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
+ fprintf(stderr, "\n");
+ }
+
+ return (stop == ZEND_JIT_TRACE_STOP_HALT) ? -1 : 0;
+}
+
+int ZEND_FASTCALL zend_jit_trace_exit(uint32_t trace_num, uint32_t exit_num)
+{
+ zend_execute_data *execute_data = EG(current_execute_data);
+ const zend_op *opline;
+ zend_jit_trace_info *t = &zend_jit_traces[trace_num];
+
+ // TODO: Deoptimizatoion of VM stack state ???
+
+ /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
+ // TODO: We may remoive this, becaus of the same check in zend_jit_trace_hot_side() ???
+ opline = t->exit_info[exit_num].opline;
+ if ((uintptr_t)opline & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
+ opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED));
+ if (opline) {
+ /* Set VM opline to continue interpretation */
+ EX(opline) = opline;
+ }
+ return 0;
+ }
+
+ if (opline) {
+ /* Set VM opline to continue interpretation */
+ EX(opline) = opline;
+ }
+
+ ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
+ ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
+ EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT) {
+ fprintf(stderr, " TRACE %d exit %d %s() %s:%d\n",
+ trace_num,
+ exit_num,
+ EX(func)->op_array.function_name ?
+ ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
+ ZSTR_VAL(EX(func)->op_array.filename),
+ EX(opline)->lineno);
+ }
+
+ if (zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
+ return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
+ }
+
+ return 0;
+}
+
+static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
+{
+ switch (opline->opcode) {
+ case ZEND_CATCH:
+ case ZEND_FAST_CALL:
+ case ZEND_FAST_RET:
+ case ZEND_GENERATOR_CREATE:
+ case ZEND_GENERATOR_RETURN:
+ case ZEND_EXIT:
+ case ZEND_YIELD:
+ case ZEND_YIELD_FROM:
+ case ZEND_INCLUDE_OR_EVAL:
+ return ZEND_JIT_TRACE_UNSUPPORTED;
+ default:
+ return ZEND_JIT_TRACE_SUPPORTED;
+ }
+}
+
+static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
+{
+ zend_op *opline;
+ zend_jit_op_array_trace_extension *jit_extension;
+ zend_cfg cfg;
+ uint32_t i;
+
+ ZEND_ASSERT(zend_jit_func_counter_handler != NULL);
+ ZEND_ASSERT(zend_jit_ret_counter_handler != NULL);
+ ZEND_ASSERT(zend_jit_loop_counter_handler != NULL);
+ ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
+
+ if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+ jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
+ memset(&jit_extension->func_info, 0, sizeof(jit_extension->func_info));
+ jit_extension->func_info.num_args = -1;
+ jit_extension->func_info.return_value_used = -1;
+ jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
+ for (i = 0; i < op_array->last; i++) {
+ jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
+ jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
+ jit_extension->trace_info[i].counter = NULL;
+ jit_extension->trace_info[i].trace_flags =
+ zend_jit_trace_supported(&op_array->opcodes[i]);
+ }
+ ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
+
+ opline = op_array->opcodes;
+ if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
+ opline++;
+ }
+ }
+
+ if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
+ /* function entry */
+ opline->handler = (const void*)zend_jit_func_counter_handler;
+ ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
+ &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
+ ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
+ ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
+ ZEND_JIT_TRACE_START_ENTER;
+ }
+
+ for (i = 0; i < cfg.blocks_count; i++) {
+ if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
+ if (cfg.blocks[i].flags & ZEND_BB_ENTRY) {
+ /* continuation after return from function call */
+ opline = op_array->opcodes + cfg.blocks[i].start;
+ if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
+ opline->handler = (const void*)zend_jit_ret_counter_handler;
+ if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
+ ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
+ &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
+ ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
+ }
+ ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
+ ZEND_JIT_TRACE_START_RETURN;
+ }
+ }
+ if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
+ /* loop header */
+ opline = op_array->opcodes + cfg.blocks[i].start;
+ if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
+ opline->handler = (const void*)zend_jit_loop_counter_handler;
+ if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
+ ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
+ &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
+ ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
+ }
+ ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
+ ZEND_JIT_TRACE_START_LOOP;
+ }
+ }
+ }
+ }
+
+ return SUCCESS;
+}
+
+static void zend_jit_trace_init_caches(void)
+{
+ memset(JIT_G(bad_root_cache_opline), 0, sizeof(JIT_G(bad_root_cache_opline)));
+ memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
+ memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
+ JIT_G(bad_root_slot) = 0;
+
+ memset(JIT_G(exit_counters), 0, sizeof(JIT_G(exit_counters)));
+}
+
+static void zend_jit_trace_reset_caches(void)
+{
+}
#include <ZendAccelerator.h>
#include "Optimizer/zend_func_info.h"
+#include "Optimizer/zend_call_graph.h"
#include "zend_jit.h"
#include "zend_jit_internal.h"
#endif
}
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC)
+{
+ if (call_info & ZEND_CALL_TOP) {
+ ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
+ } else {
+ ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
+ }
+}
+
void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)
{
zend_op_array *op_array = &EX(func)->op_array;
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
- zend_jit_op_array_extension *jit_extension =
- (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
+ zend_jit_op_array_hot_extension *jit_extension =
+ (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
#ifndef HAVE_GCC_GLOBAL_REGS
const zend_op *opline = EX(opline);
#endif
*(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST;
if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
+ *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT;
zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN();
} else {
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
{
- zend_jit_op_array_extension *jit_extension =
- (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
+ zend_jit_op_array_hot_extension *jit_extension =
+ (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
#ifndef HAVE_GCC_GLOBAL_REGS
const zend_op *opline = EX(opline);
#endif
*(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST;
if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
+ *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT;
zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN();
} else {
{
return _zend_quick_get_constant(key, 0, 1);
}
+
+static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)
+{
+ zend_jit_op_array_trace_extension *jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
+ size_t offset = jit_extension->offset;
+#ifndef HAVE_GCC_GLOBAL_REGS
+ const zend_op *opline = EX(opline);
+#endif
+
+ *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= ZEND_JIT_TRACE_FUNC_COST;
+
+ if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
+ *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_TRACE_COUNTER_INIT;
+ if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
+#ifndef HAVE_GCC_GLOBAL_REGS
+ return -1;
+#endif
+ }
+#ifdef HAVE_GCC_GLOBAL_REGS
+ execute_data = EG(current_execute_data);
+ opline = EX(opline);
+ return;
+#else
+ return 1;
+#endif
+ } else {
+ zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
+ ZEND_OPCODE_TAIL_CALL(handler);
+ }
+}
+
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
+{
+ ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_FUNC_COST);
+}
+
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
+{
+ ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_RET_COST);
+}
+
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
+{
+ ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_LOOP_COST);
+}
+
+#define TRACE_RECORD(_op, _ptr) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].ptr = _ptr; \
+ idx++; \
+ if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \
+ stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
+ break; \
+ }
+
+#define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].op1_type = _op1_type; \
+ trace_buffer[idx].op2_type = _op2_type; \
+ trace_buffer[idx].op3_type = _op3_type; \
+ trace_buffer[idx].ptr = _ptr; \
+ idx++; \
+ if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \
+ stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
+ break; \
+ }
+
+#define TRACE_RECORD_ENTER(_op, _return_value_used, _ptr) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].return_value_used = _return_value_used; \
+ trace_buffer[idx].ptr = _ptr; \
+ idx++; \
+ if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \
+ stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
+ break; \
+ }
+
+#define TRACE_RECORD_INIT(_op, _fake, _ptr) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].fake = _fake; \
+ trace_buffer[idx].ptr = _ptr; \
+ idx++; \
+ if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \
+ stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
+ break; \
+ }
+
+#define TRACE_RECORD_BACK(_op, _recursive, _ptr) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].recursive = _recursive; \
+ trace_buffer[idx].ptr = _ptr; \
+ idx++; \
+ if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \
+ stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
+ break; \
+ }
+
+#define TRACE_RECORD_TYPE(_op, _ptr) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].ptr = _ptr; \
+ idx++; \
+ if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 1) { \
+ stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
+ break; \
+ }
+
+#define TRACE_START(_op, _start, _ptr) \
+ trace_buffer[0].op = _op; \
+ trace_buffer[0].start = _start; \
+ trace_buffer[0].level = 0; \
+ trace_buffer[0].ptr = _ptr; \
+ idx = ZEND_JIT_TRACE_START_REC_SIZE;
+
+#define TRACE_END(_op, _stop, _ptr) \
+ trace_buffer[idx].op = _op; \
+ trace_buffer[idx].start = trace_buffer[idx].start; \
+ trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
+ trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
+ trace_buffer[idx].ptr = _ptr;
+
+#ifndef ZEND_JIT_RECORD_RECURSIVE_RETURN
+# define ZEND_JIT_RECORD_RECURSIVE_RETURN 1
+#endif
+
+static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
+{
+ int i;
+ int count = 0;
+
+ for (i = ret_level; i < level; i++) {
+ count += (unrolled_calls[i] == op_array);
+ }
+ return count;
+}
+
+static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
+{
+ int i;
+ int count = 0;
+
+ for (i = 0; i < ret_level; i++) {
+ count += (unrolled_calls[i] == op_array);
+ }
+ return count;
+}
+
+static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
+{
+ while (ex != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
+ if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
+ return 1;
+ }
+ ex = ex->prev_execute_data;
+ ret_level++;
+ }
+ return 0;
+}
+
+static int zend_jit_trace_bad_inner_loop(const zend_op *opline)
+{
+ const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
+ uint8_t *cache_count = JIT_G(bad_root_cache_count);
+ uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
+ uint32_t i;
+
+ for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
+ if (cache_opline[i] == opline) {
+ if ((cache_stop[i] == ZEND_JIT_TRACE_STOP_INNER_LOOP
+ || cache_stop[i] == ZEND_JIT_TRACE_STOP_LOOP_EXIT)
+ && cache_count[i] > ZEND_JIT_TRACE_MAX_ROOT_FAILURES / 2) {
+ return 1;
+ }
+ break;
+ }
+ }
+ return 0;
+}
+
+static int zend_jit_trace_bad_compiled_loop(const zend_op *opline)
+{
+ const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
+ uint8_t *cache_count = JIT_G(bad_root_cache_count);
+ uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
+ uint32_t i;
+
+ for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
+ if (cache_opline[i] == opline) {
+ if (cache_stop[i] == ZEND_JIT_TRACE_STOP_COMPILED_LOOP
+ && cache_count[i] >= ZEND_JIT_TRACE_MAX_ROOT_FAILURES - 1) {
+ return 1;
+ }
+ break;
+ }
+ }
+ return 0;
+}
+
+static int zend_jit_trace_bad_loop_exit(const zend_op *opline)
+{
+ const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
+ uint8_t *cache_count = JIT_G(bad_root_cache_count);
+ uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
+ uint32_t i;
+
+ for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
+ if (cache_opline[i] == opline) {
+ if (cache_stop[i] == ZEND_JIT_TRACE_STOP_LOOP_EXIT
+ && cache_count[i] >= ZEND_JIT_TRACE_MAX_ROOT_FAILURES - 1) {
+ return 1;
+ }
+ break;
+ }
+ }
+ return 0;
+}
+
+static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx)
+{
+ zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
+
+ do {
+ if (call->prev_execute_data) {
+ idx = zend_jit_trace_record_fake_init_call(call->prev_execute_data, trace_buffer, idx);
+ }
+ TRACE_RECORD_INIT(ZEND_JIT_TRACE_INIT_CALL, 1, call->func);
+ } while (0);
+ return idx;
+}
+
+/*
+ * Trace Linking Rules
+ * ===================
+ *
+ * flags
+ * +----------+----------+----------++----------+----------+----------+
+ * | || JIT |
+ * +----------+----------+----------++----------+----------+----------+
+ * start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN |
+ * +========+==========+==========+==========++==========+==========+==========+
+ * | LOOP | loop | | loop-ret || COMPILED | LINK | LINK |
+ * +--------+----------+----------+----------++----------+----------+----------+
+ * | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK |
+ * +--------+----------+----------+----------++----------+----------+----------+
+ * | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK |
+ * +--------+----------+----------+----------++----------+----------+----------+
+ * | SIDE | unroll | | return || LINK | LINK | LINK |
+ * +--------+----------+----------+----------++----------+----------+----------+
+ *
+ * loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP
+ * INNER_LOOP: abort recording and start new one (wit for loop)
+ * COMPILED: abort recording (wait while side exit creates outer loop)
+ * unroll: continue recording while unroll limit reached
+ * rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue
+ * loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop)
+ * return: RETURN if level == 0
+ * rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
+ *
+ */
+
+zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start)
+{
+#ifdef HAVE_GCC_GLOBAL_REGS
+ zend_execute_data *save_execute_data = execute_data;
+ const zend_op *save_opline = opline;
+#endif
+ const zend_op *orig_opline;
+ zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
+ int level = 0;
+ int ret_level = 0;
+ zend_vm_opcode_handler_t handler;
+ zend_jit_op_array_trace_extension *jit_extension;
+ size_t offset;
+ int idx, count;
+ uint8_t trace_flags, op1_type, op2_type, op3_type;
+ int backtrack_recursion = -1;
+ int backtrack_ret_recursion = -1;
+ int backtrack_ret_recursion_level = 0;
+ int loop_unroll_limit = 0;
+ const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
+#if ZEND_JIT_DETECT_UNROLLED_LOOPS
+ uint32_t unrolled_loops[ZEND_JIT_TRACE_MAX_UNROLL_LOOPS];
+#endif
+ zend_bool is_toplevel;
+#ifdef HAVE_GCC_GLOBAL_REGS
+ zend_execute_data *prev_execute_data = ex;
+
+ execute_data = ex;
+ opline = EX(opline) = op;
+#else
+ int rc;
+ zend_execute_data *execute_data = ex;
+ const zend_op *opline = EX(opline);
+#endif
+ zend_execute_data *prev_call = EX(call);
+
+ orig_opline = opline;
+
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
+ offset = jit_extension->offset;
+
+ TRACE_START(ZEND_JIT_TRACE_START, start, &EX(func)->op_array);
+ ((zend_jit_trace_start_rec*)trace_buffer)->opline = opline;
+ is_toplevel = EX(func)->op_array.function_name == NULL;
+
+
+ if (prev_call) {
+ idx = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx);
+ }
+
+ while (1) {
+ if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
+ /* Abort trace because of exception */
+ stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
+ break;
+ }
+
+ op1_type = op2_type = op3_type = IS_UNKNOWN;
+ if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
+ && (opline->opcode != ZEND_ROPE_ADD && opline->opcode != ZEND_ROPE_END)) {
+ zval *zv = EX_VAR(opline->op1.var);
+ op1_type = Z_TYPE_P(zv);
+ if (op1_type == IS_INDIRECT) {
+ zv = Z_INDIRECT_P(zv);
+ op1_type = Z_TYPE_P(zv);
+ }
+ if (op1_type == IS_REFERENCE) {
+ op1_type = Z_TYPE_P(Z_REFVAL_P(zv)) | IS_TRACE_REFERENCE;
+ }
+ }
+ if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
+ zval *zv = EX_VAR(opline->op2.var);
+ op2_type = Z_TYPE_P(zv);
+ if (op2_type == IS_INDIRECT) {
+ zv = Z_INDIRECT_P(zv);
+ op2_type = Z_TYPE_P(zv);
+ }
+ if (op2_type == IS_REFERENCE) {
+ op2_type = Z_TYPE_P(Z_REFVAL_P(zv)) | IS_TRACE_REFERENCE;
+ }
+ }
+ if (opline->opcode == ZEND_ASSIGN_DIM ||
+ opline->opcode == ZEND_ASSIGN_OBJ ||
+ opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
+ opline->opcode == ZEND_ASSIGN_DIM_OP ||
+ opline->opcode == ZEND_ASSIGN_OBJ_OP ||
+ opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
+ opline->opcode == ZEND_ASSIGN_OBJ_REF ||
+ opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
+ if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
+ zval *zv = EX_VAR((opline+1)->op1.var);
+ op3_type = Z_TYPE_P(zv);
+ if (op3_type == IS_INDIRECT) {
+ zv = Z_INDIRECT_P(zv);
+ op3_type = Z_TYPE_P(zv);
+ }
+ if (op3_type == IS_REFERENCE) {
+ op3_type = Z_TYPE_P(Z_REFVAL_P(zv)) | IS_TRACE_REFERENCE;
+ }
+ }
+ }
+
+ TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
+
+ if (opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
+ zval *var = EX_VAR(opline->op1.var);
+ uint8_t type = Z_TYPE_P(var);
+
+ if (type == IS_OBJECT) {
+ TRACE_RECORD_TYPE(ZEND_JIT_TRACE_OP1_TYPE, Z_OBJCE_P(var));
+ }
+ }
+
+ if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
+ zval *var = EX_VAR(opline->op2.var);
+ uint8_t type = Z_TYPE_P(var);
+
+ if (type == IS_OBJECT) {
+ TRACE_RECORD_TYPE(ZEND_JIT_TRACE_OP2_TYPE, Z_OBJCE_P(var));
+ }
+ }
+
+ switch (opline->opcode) {
+ case ZEND_DO_FCALL:
+ case ZEND_DO_ICALL:
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
+ TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, EX(call)->func);
+ }
+ break;
+ default:
+ break;
+ }
+
+ handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
+#ifdef HAVE_GCC_GLOBAL_REGS
+ handler();
+ if (UNEXPECTED(opline == zend_jit_halt_op)) {
+ stop = ZEND_JIT_TRACE_STOP_HALT;
+ break;
+ }
+ if (UNEXPECTED(execute_data != prev_execute_data)) {
+ if (execute_data->prev_execute_data == prev_execute_data) {
+#else
+ rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
+ if (rc != 0) {
+ if (rc < 0) {
+ stop = ZEND_JIT_TRACE_STOP_HALT;
+ break;
+ }
+ execute_data = EG(current_execute_data);
+ opline = EX(opline);
+ if (rc == 1) {
+#endif
+ /* Enter into function */
+ if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
+ stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
+ break;
+ }
+
+ if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
+ /* TODO: Can we continue recording ??? */
+ stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
+ break;
+ }
+
+ TRACE_RECORD_ENTER(ZEND_JIT_TRACE_ENTER, EX(return_value) != NULL, &EX(func)->op_array);
+
+ count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
+
+ if (opline == orig_opline) {
+ if (count + 1 >= ZEND_JIT_TRACE_MAX_RECURSION) {
+ stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
+ break;
+ }
+ backtrack_recursion = idx;
+ } else if (count >= ZEND_JIT_TRACE_MAX_RECURSION) {
+ stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
+ break;
+ }
+
+ unrolled_calls[ret_level + level] = &EX(func)->op_array;
+ level++;
+ } else {
+ /* Return from function */
+ if (level == 0) {
+ if (is_toplevel) {
+ stop = ZEND_JIT_TRACE_STOP_TOPLEVEL;
+ break;
+#if ZEND_JIT_RECORD_RECURSIVE_RETURN
+ } else if (start == ZEND_JIT_TRACE_START_RETURN
+ && execute_data->prev_execute_data
+ && execute_data->prev_execute_data->func
+ && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
+ && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
+ if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
+ stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
+ break;
+ }
+ TRACE_RECORD_BACK(ZEND_JIT_TRACE_BACK, 1, &EX(func)->op_array);
+ count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
+ if (opline == orig_opline) {
+ if (count + 1 >= ZEND_JIT_TRACE_MAX_RECURSION) {
+ stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
+ break;
+ }
+ backtrack_ret_recursion = idx;
+ backtrack_ret_recursion_level = ret_level;
+ } else if (count >= ZEND_JIT_TRACE_MAX_RECURSION) {
+ stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
+ break;
+ }
+
+ unrolled_calls[ret_level] = &EX(func)->op_array;
+ ret_level++;
+ is_toplevel = EX(func)->op_array.function_name == NULL;
+#endif
+ } else if (start & ZEND_JIT_TRACE_START_LOOP
+ && !zend_jit_trace_bad_loop_exit(orig_opline)) {
+ /* Fail to try close the loop.
+ If this doesn't work terminate it. */
+ stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
+ break;
+ } else {
+ stop = ZEND_JIT_TRACE_STOP_RETURN;
+ break;
+ }
+ } else {
+ level--;
+ TRACE_RECORD_BACK(ZEND_JIT_TRACE_BACK, 0, &EX(func)->op_array);
+ }
+ }
+#ifdef HAVE_GCC_GLOBAL_REGS
+ prev_execute_data = execute_data;
+#endif
+ jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
+ if (UNEXPECTED(!jit_extension)) {
+ stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
+ break;
+ }
+ offset = jit_extension->offset;
+ }
+ if (EX(call) != prev_call) {
+ if (trace_buffer[idx-1].op != ZEND_JIT_TRACE_BACK
+ && EX(call)
+ && EX(call)->prev_execute_data == prev_call) {
+ if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
+ /* TODO: Can we continue recording ??? */
+ stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
+ break;
+ }
+ TRACE_RECORD_INIT(ZEND_JIT_TRACE_INIT_CALL, 0, EX(call)->func);
+ }
+ prev_call = EX(call);
+ }
+
+#ifndef HAVE_GCC_GLOBAL_REGS
+ opline = EX(opline);
+#endif
+
+ trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
+ if (trace_flags) {
+ if (trace_flags & ZEND_JIT_TRACE_JITED) {
+ if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
+ if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
+ && level + ret_level == 0
+ && !zend_jit_trace_bad_compiled_loop(orig_opline)) {
+ /* Fail to try close outer loop throgh side exit.
+ If this doesn't work just link. */
+ stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
+ break;
+ } else {
+ stop = ZEND_JIT_TRACE_STOP_LINK;
+ break;
+ }
+ } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
+ if (start != ZEND_JIT_TRACE_START_RETURN) {
+ // TODO: We may try to inline function ???
+ stop = ZEND_JIT_TRACE_STOP_LINK;
+ break;
+ }
+ } else {
+ stop = ZEND_JIT_TRACE_STOP_LINK;
+ break;
+ }
+ } else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
+ stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
+ break;
+ } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
+ if (start != ZEND_JIT_TRACE_START_SIDE) {
+ if (opline == orig_opline && level + ret_level == 0) {
+ stop = ZEND_JIT_TRACE_STOP_LOOP;
+ break;
+ }
+ /* Fail to try creating a trace for inner loop first.
+ If this doesn't work try unroling loop. */
+ if (!zend_jit_trace_bad_inner_loop(opline)) {
+ stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
+ break;
+ }
+ }
+ if (loop_unroll_limit < ZEND_JIT_TRACE_MAX_UNROLL_LOOPS) {
+ loop_unroll_limit++;
+ } else {
+ stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
+ break;
+ }
+ } else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
+ TRACE_RECORD(ZEND_JIT_TRACE_VM, opline);
+ stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
+ break;
+ }
+ }
+ }
+
+ if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
+ if (backtrack_recursion > 0) {
+ idx = backtrack_recursion;
+ stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
+ } else if (backtrack_ret_recursion > 0) {
+ idx = backtrack_ret_recursion;
+ ret_level = backtrack_ret_recursion_level;
+ stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
+ }
+ }
+
+ TRACE_END(ZEND_JIT_TRACE_END, stop, opline);
+
+#ifdef HAVE_GCC_GLOBAL_REGS
+ if (stop != ZEND_JIT_TRACE_STOP_HALT) {
+ EX(opline) = opline;
+ }
+#endif
+
+#ifdef HAVE_GCC_GLOBAL_REGS
+ execute_data = save_execute_data;
+ opline = save_opline;
+#endif
+
+ return stop;
+}
|.define HYBRID_SPAD, 16 // padding for stack alignment
+#define DASM_ALIGNMENT 16
+
/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to
* guarantee proper alignment of 128-bit SSE data allocated on stack.
* With broken alignment any execution of SSE code, including calls to
#include "Zend/zend_cpuinfo.h"
#include "jit/zend_jit_x86.h"
+#ifdef HAVE_VALGRIND
+# include <valgrind/valgrind.h>
+#endif
+
/* The generated code may contain tautological comparisons, ignore them. */
#if defined(__clang__)
# pragma clang diagnostic push
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
-|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != MAY_BE_DOUBLE) {
+|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE
|| }
-|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
+|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
-|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != MAY_BE_DOUBLE) {
+|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE
|| }
-|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
+|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
-|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
+|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
|| zend_uchar type = concrete_type(src_info);
| SET_ZVAL_TYPE_INFO dst_addr, type
|| }
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| zend_uchar type = concrete_type(src_info);
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
-|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
+|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
| SET_ZVAL_TYPE_INFO dst_addr, type
|| }
|| }
static uint32_t delayed_call_level;
static const zend_op *last_valid_opline;
static int jit_return_label;
+static uint32_t current_trace_num;
/* bit helpers */
| jmp aword [r1+r2*4+4]
| .endif
|1:
+ | mov word [r2], ZEND_JIT_HOT_COUNTER_INIT
| mov FCARG1a, FP
| GET_IP FCARG2a
| EXT_CALL zend_jit_hot_func, r0
| jmp aword [r1+r2*4+4]
| .endif
|1:
+ | mov word [r2], ZEND_JIT_HOT_COUNTER_INIT
| mov FCARG1a, FP
| GET_IP FCARG2a
| EXT_CALL zend_jit_hot_func, r0
return 1;
}
+static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
+{
+ | mov r0, EX->func
+ | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+ | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)]
+ | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)]
+ | sub word [r2], cost
+ | jle >1
+ | jmp aword [IP + r1]
+ |1:
+ | mov word [r2], ZEND_JIT_TRACE_COUNTER_INIT
+ | mov FCARG1a, FP
+ | GET_IP FCARG2a
+ | EXT_CALL zend_jit_trace_hot_root, r0
+ | test eax, eax // TODO : remove this check at least for HYBRID VM ???
+ | jl >1
+ | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
+ | LOAD_OPLINE
+ | JMP_IP
+ |1:
+ | EXT_JMP zend_jit_halt_op->handler, r0
+ return 1;
+}
+
+static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst)
+{
+ |->hybrid_func_counter:
+
+ return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_FUNC_COST);
+}
+
+static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst)
+{
+ |->hybrid_ret_counter:
+
+ return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_RET_COST);
+}
+
+static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst)
+{
+ |->hybrid_loop_counter:
+
+ return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_LOOP_COST);
+}
+
+static int zend_jit_trace_halt_stub(dasm_State **Dst)
+{
+ |->trace_halt:
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | EXT_JMP zend_jit_halt_op->handler, r0
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | ret // PC must be zero
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, -1 // ZEND_VM_RETURN
+ | ret
+ }
+ return 1;
+}
+
+static int zend_jit_trace_exit_stub(dasm_State **Dst)
+{
+ |->trace_exit:
+ |
+ | // TODO: Save CPU registers ???
+ |
+ | // trace_num = EG(reserved)[zend_func_info_rid]
+ | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, reserved[zend_func_info_rid], r0
+ | // exit_num = POP
+ | pop FCARG2a
+ | // EX(opline) = opline
+ | SAVE_OPLINE
+ | // zend_jit_trace_exit(trace_num, exit_num)
+ | EXT_CALL zend_jit_trace_exit, r0
+ | // execute_data = EG(current_excute_data)
+ | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
+ | test eax, eax
+ | jl ->trace_halt
+ | // opline = EX(opline)
+ | LOAD_OPLINE
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+#if 1
+ //TODO: this doesn't work for exit from first instruction ???
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+#else
+ | mov r0, EX->func
+ | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+ | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)]
+ | jmp aword [IP + r1]
+#endif
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | JMP_IP
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 1 // ZEND_VM_ENTER
+ | ret
+ }
+
+ return 1;
+}
+
+static int zend_jit_trace_escape_stub(dasm_State **Dst)
+{
+ |->trace_escape:
+ |
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | JMP_IP
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 1 // ZEND_VM_ENTER
+ | ret
+ }
+
+ return 1;
+}
+
+/* Keep 32 exit points in a single code block */
+#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
+#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
+
+static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n)
+{
+ uint32_t i;
+
+ for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) {
+ | push byte i
+ | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1
+ }
+ | push byte i
+ |// 1:
+ | .byte 0x81, 0x04, 0x24; .dword n // add aword [r4], n
+ | jmp ->trace_exit
+
+ return 1;
+}
+
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
{
JIT_STUB(undefined_function),
JIT_STUB(negative_shift),
JIT_STUB(mod_by_zero),
+ JIT_STUB(trace_halt),
+ JIT_STUB(trace_exit),
+ JIT_STUB(trace_escape),
JIT_STUB(double_one),
#ifdef CONTEXT_THREADED_JIT
JIT_STUB(context_threaded_call),
return SUCCESS;
}
+static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst)
+{
+ | int3
+ return 1;
+}
+
static int zend_jit_align_func(dasm_State **Dst)
{
reuse_ip = 0;
return zend_jit_check_exception(Dst);
}
+static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num)
+{
+ current_trace_num = trace_num;
+
+ | //EG(reserved)[zend_func_info_rid] = trace_num;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, reserved[zend_func_info_rid], trace_num, r0
+
+ return 1;
+}
+
+static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline)
+{
+ uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, NULL, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ | CMP_IP opline
+ | jne &exit_addr
+
+ return 1;
+}
+
+/* This taken from LuaJIT. Thanks to Mike Pall. */
+static uint32_t _asm_x86_inslen(const uint8_t* p)
+{
+ static const uint8_t map_op1[256] = {
+ 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20,
+ 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,
+ 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
+ 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
+#if defined(__x86_64__) || defined(_M_X64)
+ 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14,
+#else
+ 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
+#endif
+ 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
+ 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51,
+ 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
+ 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
+ 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51,
+#if defined(__x86_64__) || defined(_M_X64)
+ 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
+#else
+ 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
+#endif
+ 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05,
+ 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51,
+ 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
+ 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51,
+ 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92
+ };
+ static const uint8_t map_op2[256] = {
+ 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93,
+ 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
+ 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52
+ };
+ uint32_t result = 0;
+ uint32_t prefixes = 0;
+ uint32_t x = map_op1[*p];
+
+ for (;;) {
+ switch (x >> 4) {
+ case 0:
+ return result + x + (prefixes & 4);
+ case 1:
+ prefixes |= x;
+ x = map_op1[*++p];
+ result++;
+ break;
+ case 2:
+ x = map_op2[*++p];
+ break;
+ case 3:
+ p++;
+ goto mrm;
+ case 4:
+ result -= (prefixes & 2);
+ /* fallthrough */
+ case 5:
+ return result + (x & 15);
+ case 6: /* Group 3. */
+ if (p[1] & 0x38) {
+ x = 2;
+ } else if ((prefixes & 2) && (x == 0x66)) {
+ x = 4;
+ }
+ goto mrm;
+ case 7: /* VEX c4/c5. */
+#if !defined(__x86_64__) && !defined(_M_X64)
+ if (p[1] < 0xc0) {
+ x = 2;
+ goto mrm;
+ }
+#endif
+ if (x == 0x70) {
+ x = *++p & 0x1f;
+ result++;
+ if (x >= 2) {
+ p += 2;
+ result += 2;
+ goto mrm;
+ }
+ }
+ p++;
+ result++;
+ x = map_op2[*++p];
+ break;
+ case 8:
+ result -= (prefixes & 2);
+ /* fallthrough */
+ case 9:
+mrm:
+ /* ModR/M and possibly SIB. */
+ result += (x & 15);
+ x = *++p;
+ switch (x >> 6) {
+ case 0:
+ if ((x & 7) == 5) {
+ return result + 4;
+ }
+ break;
+ case 1:
+ result++;
+ break;
+ case 2:
+ result += 4;
+ break;
+ case 3:
+ return result;
+ }
+ if ((x & 7) == 4) {
+ result++;
+ if (x < 0x40 && (p[1] & 7) == 5) {
+ result += 4;
+ }
+ }
+ return result;
+ }
+ }
+}
+
+static int zend_jit_patch(const void *code, size_t size, const void *from_addr, const void *to_addr)
+{
+ int ret = 0;
+ uint8_t *p = (uint8_t*)code;
+ uint8_t *end = p + size - 5;
+
+ while (p < end) {
+ if ((*(uint16_t*)p & 0xf0ff) == 0x800f && p + *(int32_t*)(p+2) == (uint8_t*)from_addr - 6) {
+ *(int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6));
+ ret++;
+ } else if (*p == 0xe9 && p + *(int32_t*)(p+1) == (uint8_t*)from_addr - 5) {
+ *(int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5));
+ ret++;
+ }
+ p += _asm_x86_inslen(p);
+ }
+#ifdef HAVE_VALGRIND
+ VALGRIND_DISCARD_TRANSLATIONS(code, size);
+#endif
+ return ret;
+}
+
+static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t exit_num, const void *addr)
+{
+ return zend_jit_patch(code, size, zend_jit_trace_get_exit_addr(exit_num), addr);
+}
+
+static int zend_jit_trace_link_to_root(dasm_State **Dst, const void *code)
+{
+ const void *exit_addr;
+ size_t prologue_size;
+
+ /* Skip prologue. */
+ // TODO: don't hardcode this ???
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ // sub r4, HYBRID_SPAD
+#if defined(__x86_64__) || defined(_M_X64)
+ prologue_size = 4;
+#else
+ prologue_size = 3;
+#endif
+ } else if (GCC_GLOBAL_REGS) {
+ // sub r4, SPAD // stack alignment
+#if defined(__x86_64__) || defined(_M_X64)
+ prologue_size = 4;
+#else
+ prologue_size = 3;
+#endif
+ } else {
+ // sub r4, NR_SPAD // stack alignment
+ // mov aword T2, FP // save FP
+ // mov aword T3, RX // save IP
+ // mov FP, FCARG1a
+#if defined(__x86_64__) || defined(_M_X64)
+ prologue_size = 17;
+#else
+ prologue_size = 12;
+#endif
+ }
+ exit_addr = (const void*)((const char*)code + prologue_size);
+
+ | jmp &exit_addr
+ return 1;
+}
+
+static int zend_jit_trace_return(dasm_State **Dst)
+{
+#if 0
+ | jmp ->trace_escape
+#else
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | JMP_IP
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 2 // ZEND_VM_LEAVE
+ | ret
+ }
+#endif
+ return 1;
+}
+
+static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type)
+{
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ | IF_NOT_Z_TYPE FP + var, type, &exit_addr
+
+ return 1;
+}
+
+static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
+{
+ zend_jit_op_array_trace_extension *jit_extension =
+ (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+ size_t offset = jit_extension->offset;
+ const void *handler =
+ (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
+
+ if (!zend_jit_set_valid_ip(Dst, opline)) {
+ return 0;
+ }
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, FP
+ }
+ | EXT_CALL handler, r0
+ if (may_throw) {
+ zend_jit_check_exception(Dst);
+ }
+
+ if (!GCC_GLOBAL_REGS) {
+ if (opline->opcode == ZEND_RETURN ||
+ opline->opcode == ZEND_DO_UCALL ||
+ opline->opcode == ZEND_DO_FCALL_BY_NAME ||
+ opline->opcode == ZEND_DO_FCALL) {
+ | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
+ }
+ }
+
+ if (zend_jit_trace_may_exit(op_array, opline, trace)) {
+ // TODO: try to avoid this check ???
+ if (opline->opcode == ZEND_RETURN) {
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | cmp IP, zend_jit_halt_op
+ | je ->trace_halt
+ } else if (GCC_GLOBAL_REGS) {
+ | test IP, IP
+ | je ->trace_halt
+ } else {
+ | test eax, eax
+ | jl ->trace_halt
+ }
+ }
+ while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
+ trace++;
+ }
+ if (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN) {
+ const zend_op *next_opline = trace->opline;
+ const zend_op *exit_opline = NULL;
+ uint32_t exit_point;
+ const void *exit_addr;
+
+ if (zend_is_smart_branch(opline)) {
+ zend_bool exit_if_true = 0;
+ exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
+ } else {
+ switch (opline->opcode) {
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_FE_RESET_R:
+ case ZEND_FE_RESET_RW:
+ exit_opline = (trace->opline == opline + 1) ?
+ OP_JMP_ADDR(opline, opline->op2) :
+ opline + 1;
+ break;
+ case ZEND_JMPZNZ:
+ exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ?
+ ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
+ OP_JMP_ADDR(opline, opline->op2);
+ break;
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ exit_opline = (trace->opline == opline + 1) ?
+ ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
+ opline + 1;
+ break;
+
+ }
+ }
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, trace);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ | CMP_IP next_opline
+ | jne &exit_addr
+ }
+ } else {
+ while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
+ trace++;
+ }
+ // TODO: remove this ???
+ if (opline->opcode == ZEND_RETURN
+ && trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_RETURN) {
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | cmp IP, zend_jit_halt_op
+ | je ->trace_halt
+ } else if (GCC_GLOBAL_REGS) {
+ | test IP, IP
+ | je ->trace_halt
+ } else {
+ | test eax, eax
+ | jl ->trace_halt
+ }
+ }
+ }
+
+ last_valid_opline = trace->opline;
+
+ return 1;
+}
+
static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw)
{
const void *handler;
| LONG_OP_WITH_CONST sub, op1_def_addr, Z_L(1)
}
- if (may_overflow) {
+ if (may_overflow && (op1_def_info & MAY_BE_GUARD)) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ | jo &exit_addr
+ } else if (may_overflow) {
| jo >1
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
}
}
if (may_overflow) {
- | jo >1
+ if (res_info & MAY_BE_GUARD) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ | jo &exit_addr
+ } else {
+ | jo >1
+ }
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_LVAL res_addr, Ra(result_reg)
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
- if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
- if (may_overflow) {
+ if (may_overflow && !(res_info & MAY_BE_GUARD)) {
zend_reg tmp_reg1 = ZREG_XMM0;
zend_reg tmp_reg2 = ZREG_XMM1;
| SSE_SET_ZVAL_DVAL res_addr, tmp_reg1
} while (0);
- if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
| jmp >2
| SSE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
- if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
- if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
- if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
| SET_ZVAL_LVAL res_addr, Ra(result_reg)
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
- if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
{
zend_jit_addr op2_addr = OP2_ADDR();
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
+ const void *exit_addr = NULL;
+
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ }
if (op2_info & MAY_BE_LONG) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
|.endif
if (type == BP_JIT_IS) {
| jbe >9 // NOT_FOUND
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) {
+ | jbe &exit_addr
} else {
| jbe >2 // NOT_FOUND
}
|.endif
if (type == BP_JIT_IS) {
| jbe >9 // NOT_FOUND
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) {
+ | jbe &exit_addr
} else {
| jbe >2 // NOT_FOUND
}
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) {
- | jmp >2 // NOT_FOUND
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ | jmp &exit_addr
+ } else {
+ | jmp >2 // NOT_FOUND
+ }
}
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ | jmp &exit_addr
} else {
| jmp >2 // NOT_FOUND
}
}
| EXT_CALL zend_hash_index_find, r0
| test r0, r0
- | jz >2 // NOT_FOUND
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ | jz &exit_addr
+ } else {
+ | jz >2 // NOT_FOUND
+ }
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
- | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
- | // retval = &EG(uninitialized_zval);
- | UNDEFINED_OFFSET opline
- | jmp >9
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) {
+ | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
+ | // retval = &EG(uninitialized_zval);
+ | UNDEFINED_OFFSET opline
+ | jmp >9
+ }
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
break;
case BP_VAR_RW:
|2:
- | SAVE_VALID_OPLINE opline
- | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
- | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval));
- | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) {
+ | SAVE_VALID_OPLINE opline
+ | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
+ | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval));
+ | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0
+ }
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
- | jmp >8
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) {
+ | jmp >8
+ }
|4:
| SAVE_VALID_OPLINE opline
| EXT_CALL zend_jit_hash_index_lookup_rw, r0
| EXT_CALL _zend_hash_find_known_hash, r0
}
| test r0, r0
- | jz >2 // NOT_FOUND
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
+ | jz &exit_addr
+ } else {
+ | jz >2 // NOT_FOUND
+ }
| // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT))
| IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
- // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key));
- | UNDEFINED_INDEX opline
- | jmp >9
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) {
+ // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key));
+ | UNDEFINED_INDEX opline
+ | jmp >9
+ }
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
return result;
}
-static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_bool swap = 0;
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
- | jne => target_label
+ if (exit_addr) {
+ | jne &exit_addr
+ } else {
+ | jne => target_label
+ }
break;
case ZEND_IS_NOT_EQUAL:
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | je => target_label
+ }
+ break;
case ZEND_IS_NOT_IDENTICAL:
- | je => target_label
+ if (exit_addr) {
+ | jne &exit_addr
+ } else {
+ | je => target_label
+ }
break;
case ZEND_IS_SMALLER:
if (swap) {
- | jle => target_label
+ if (exit_addr) {
+ | jle &exit_addr
+ } else {
+ | jle => target_label
+ }
} else {
- | jge => target_label
+ if (exit_addr) {
+ | jge &exit_addr
+ } else {
+ | jge => target_label
+ }
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
- | jl => target_label
+ if (exit_addr) {
+ | jl &exit_addr
+ } else {
+ | jl => target_label
+ }
} else {
- | jg => target_label
+ if (exit_addr) {
+ | jg &exit_addr
+ } else {
+ | jg => target_label
+ }
}
break;
default:
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
- | je => target_label
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | je => target_label
+ }
break;
case ZEND_IS_NOT_EQUAL:
+ if (exit_addr) {
+ | jne &exit_addr
+ } else {
+ | jne => target_label
+ }
+ break;
case ZEND_IS_NOT_IDENTICAL:
- | jne => target_label
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | jne => target_label
+ }
break;
case ZEND_IS_SMALLER:
if (swap) {
- | jg => target_label
+ if (exit_addr) {
+ | jg &exit_addr
+ } else {
+ | jg => target_label
+ }
} else {
- | jl => target_label
+ if (exit_addr) {
+ | jl &exit_addr
+ } else {
+ | jl => target_label
+ }
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
- | jge => target_label
+ if (exit_addr) {
+ | jge &exit_addr
+ } else {
+ | jge => target_label
+ }
} else {
- | jle => target_label
+ if (exit_addr) {
+ | jle &exit_addr
+ } else {
+ | jle => target_label
+ }
}
break;
default:
return 1;
}
-static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
- | jne => target_label
- | jp => target_label
+ if (exit_addr) {
+ | jne &exit_addr
+ | jp &exit_addr
+ } else {
+ | jne => target_label
+ | jp => target_label
+ }
break;
case ZEND_IS_NOT_EQUAL:
- case ZEND_IS_NOT_IDENTICAL:
| jp >1
- | je => target_label
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | je => target_label
+ }
|1:
break;
+ case ZEND_IS_NOT_IDENTICAL:
+ if (exit_addr) {
+ | jne &exit_addr
+ | jp &exit_addr
+ } else {
+ | jp >1
+ | je => target_label
+ |1:
+ }
+ break;
case ZEND_IS_SMALLER:
if (swap) {
- | jbe => target_label
+ if (exit_addr) {
+ | jbe &exit_addr
+ } else {
+ | jbe => target_label
+ }
} else {
- | jae => target_label
- | jp => target_label
+ if (exit_addr) {
+ | jae &exit_addr
+ | jp &exit_addr
+ } else {
+ | jae => target_label
+ | jp => target_label
+ }
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
- | jb => target_label
+ if (exit_addr) {
+ | jb &exit_addr
+ } else {
+ | jb => target_label
+ }
} else {
- | ja => target_label
- | jp => target_label
+ if (exit_addr) {
+ | ja &exit_addr
+ | jp &exit_addr
+ } else {
+ | ja => target_label
+ | jp => target_label
+ }
}
break;
default:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
| jp >1
- | je => target_label
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | je => target_label
+ }
|1:
break;
case ZEND_IS_NOT_EQUAL:
+ if (exit_addr) {
+ | jne &exit_addr
+ | jp &exit_addr
+ } else {
+ | jne => target_label
+ | jp => target_label
+ }
+ break;
case ZEND_IS_NOT_IDENTICAL:
- | jne => target_label
- | jp => target_label
+ if (exit_addr) {
+ |1:
+ | je &exit_addr
+ |1:
+ } else {
+ | jne => target_label
+ | jp => target_label
+ }
break;
case ZEND_IS_SMALLER:
if (swap) {
- | ja => target_label
+ if (exit_addr) {
+ | ja &exit_addr
+ } else {
+ | ja => target_label
+ }
} else {
| jp >1
- | jb => target_label
+ if (exit_addr) {
+ | jb &exit_addr
+ } else {
+ | jb => target_label
+ }
|1:
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
- | jae => target_label
+ if (exit_addr) {
+ | jae &exit_addr
+ } else {
+ | jae => target_label
+ }
} else {
| jp >1
- | jbe => target_label
+ if (exit_addr) {
+ | jbe &exit_addr
+ } else {
+ | jbe => target_label
+ }
|1:
}
break;
return 1;
}
-static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_XMM0;
| SSE_GET_ZVAL_LVAL tmp_reg, op1_addr
| SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr
- return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2);
+ return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr);
}
-static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_XMM0;
| SSE_GET_ZVAL_LVAL tmp_reg, op2_addr
| SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr
- return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2);
+ return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr);
}
-static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_bool swap = 0;
| SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr
}
- return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2);
+ return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr);
}
-static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
| LONG_OP_WITH_CONST cmp, res_addr, Z_L(0)
if (smart_branch_opcode) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
- | jne => target_label
+ if (exit_addr) {
+ | jne &exit_addr
+ } else {
+ | jne => target_label
+ }
break;
case ZEND_IS_NOT_EQUAL:
- | je => target_label
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | je => target_label
+ }
break;
case ZEND_IS_SMALLER:
- | jge => target_label
+ if (exit_addr) {
+ | jge &exit_addr
+ } else {
+ | jge => target_label
+ }
break;
case ZEND_IS_SMALLER_OR_EQUAL:
- | jg => target_label
+ if (exit_addr) {
+ | jg &exit_addr
+ } else {
+ | jg => target_label
+ }
break;
default:
ZEND_ASSERT(0);
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
- | je => target_label
+ if (exit_addr) {
+ | je &exit_addr
+ } else {
+ | je => target_label
+ }
break;
case ZEND_IS_NOT_EQUAL:
- | jne => target_label
+ if (exit_addr) {
+ | jne &exit_addr
+ } else {
+ | jne => target_label
+ }
break;
case ZEND_IS_SMALLER:
- | jl => target_label
+ if (exit_addr) {
+ | jl &exit_addr
+ } else {
+ | jl => target_label
+ }
break;
case ZEND_IS_SMALLER_OR_EQUAL:
- | jle => target_label
+ if (exit_addr) {
+ | jle &exit_addr
+ } else {
+ | jle => target_label
+ }
break;
default:
ZEND_ASSERT(0);
return 1;
}
-static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var);
zend_bool has_slow;
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
- if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
}
- if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
}
- if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
- if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
}
- if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
- if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
}
- if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
}
- if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
- if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2)) {
+ if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (has_slow) {
return 1;
}
-static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t identical_label = (uint32_t)-1;
uint32_t not_identical_label = (uint32_t)-1;
- if (smart_branch_opcode) {
+ if (smart_branch_opcode && !exit_addr) {
if (opline->opcode == ZEND_IS_IDENTICAL) {
if (smart_branch_opcode == ZEND_JMPZ) {
not_identical_label = target_label;
}
}
+ if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
+ (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
+ if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
+ return 0;
+ }
+ return 1;
+ } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
+ (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
+ if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
+ return 0;
+ }
+ return 1;
+ }
+
if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) {
op1_info |= MAY_BE_NULL;
op2_info |= MAY_BE_NULL;
}
if (smart_branch_opcode) {
zend_jit_check_exception_undef_result(Dst, opline);
- if (not_identical_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ }
+ } else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else {
concrete_type(op1_info) == concrete_type(op2_info) &&
concrete_type(op1_info) <= IS_TRUE) {
if (smart_branch_opcode) {
- if (identical_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ }
+ } else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else {
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
if (smart_branch_opcode) {
- if (identical_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ }
+ } else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else {
}
} else {
if (smart_branch_opcode) {
- if (not_identical_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ }
+ } else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else {
| SAVE_VALID_OPLINE opline
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline
zend_jit_check_exception_undef_result(Dst, opline);
- if (identical_label != (uint32_t)-1) {
+ if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ } else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
} else {
| jmp >9
}
|8:
+ } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
+ | je &exit_addr
} else if (identical_label != (uint32_t)-1) {
| je =>identical_label
} else {
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline
zend_jit_check_exception_undef_result(Dst, opline);
}
- if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ }
+ } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
| SAVE_VALID_OPLINE opline
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
zend_jit_check_exception_undef_result(Dst, opline);
- if (identical_label != (uint32_t)-1) {
+ if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ } else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
} else {
| jmp >9
}
|8:
+ } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
+ | je &exit_addr
} else if (identical_label != (uint32_t)-1) {
| je =>identical_label
} else {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
zend_jit_check_exception_undef_result(Dst, opline);
}
- if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
- | jmp =>not_identical_label
- }
- } else if ((op1_info & MAY_BE_ANY) == MAY_BE_LONG &&
- (op2_info & MAY_BE_ANY) == MAY_BE_LONG) {
- if (!zend_jit_cmp_long_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
- return 0;
- }
- } else if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE &&
- (op2_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
- if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2)) {
- return 0;
+ if (smart_branch_opcode) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ }
+ } else if (not_identical_label != (uint32_t)-1) {
+ | jmp =>not_identical_label
+ }
}
} else {
if (opline->op1_type == IS_CONST) {
}
if (smart_branch_opcode) {
| test al, al
- if (not_identical_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jnz &exit_addr
+ } else {
+ | jz &exit_addr
+ }
+ } else if (not_identical_label != (uint32_t)-1) {
| jz =>not_identical_label
if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
if (may_throw) {
zend_jit_check_exception(Dst);
}
-
return 1;
}
-static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw)
+static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr)
{
uint32_t true_label = -1;
uint32_t false_label = -1;
zend_bool set_bool_not = 0;
zend_bool jmp_done = 0;
- if (opline->opcode == ZEND_JMPZ) {
+ if (branch_opcode == ZEND_BOOL) {
+ set_bool = 1;
+ } else if (branch_opcode == ZEND_BOOL_NOT) {
+ set_bool = 1;
+ set_bool_not = 1;
+ } else if (branch_opcode == ZEND_JMPZ) {
false_label = target_label;
- } else if (opline->opcode == ZEND_JMPNZ) {
+ } else if (branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
- } else if (opline->opcode == ZEND_JMPZNZ) {
+ } else if (branch_opcode == ZEND_JMPZNZ) {
true_label = target_label2;
false_label = target_label;
- } else if (opline->opcode == ZEND_BOOL) {
- set_bool = 1;
- } else if (opline->opcode == ZEND_BOOL_NOT) {
- set_bool = 1;
- set_bool_not = 1;
- } else if (opline->opcode == ZEND_JMPZ_EX) {
+ } else if (branch_opcode == ZEND_JMPZ_EX) {
set_bool = 1;
false_label = target_label;
- } else if (opline->opcode == ZEND_JMPNZ_EX) {
+ } else if (branch_opcode == ZEND_JMPNZ_EX) {
set_bool = 1;
true_label = target_label;
} else {
if ((op1_info & MAY_BE_LONG) &&
!(op1_info & MAY_BE_UNDEF) &&
!set_bool) {
- if (false_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ) {
+ | jl >9
+ } else {
+ | jl &exit_addr
+ }
+ } else if (false_label != (uint32_t)-1) {
| jl =>false_label
} else {
| jl >9
}
}
} else {
- if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (set_bool) {
+ | jne >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jmp &exit_addr
+ } else {
+ | jmp >9
+ }
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
+ if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
+ | jne &exit_addr
+ }
+ }
+ } else {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | je &exit_addr
+ } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
+ | jne &exit_addr
+ } else {
+ | je >9
+ }
+ }
+ } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (set_bool) {
| jne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
}
- if (false_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
+ | jmp &exit_addr
+ }
+ } else if (false_label != (uint32_t)-1) {
| jmp =>false_label
}
if (op1_info & MAY_BE_ANY) {
- if (false_label == (uint32_t)-1) {
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jmp >9
+ }
+ } else if (false_label == (uint32_t)-1) {
| jmp >9
}
|.code
}
if (!jmp_done) {
- if (false_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jmp >9
+ } else if (op1_info & MAY_BE_LONG) {
+ | jmp &exit_addr
+ }
+ } else if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if (op1_info & MAY_BE_LONG) {
| jmp >9
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
- if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jne &exit_addr
+ } else {
+ | je &exit_addr
+ }
+ } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr
if (set_bool) {
- if (false_label != (uint32_t)-1) { // JMPZ_EX
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jp >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | jne &exit_addr
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jp &exit_addr
+ | je &exit_addr
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ }
+ } else if (false_label != (uint32_t)-1) { // JMPZ_EX
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jp >1
| je => false_label
}
} else {
ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1);
- if (false_label != (uint32_t)-1) {
- | jp =>false_label
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jp >1
+ | jne &exit_addr
+ |1:
+ } else {
+ | jp &exit_addr
+ | je &exit_addr
+ }
} else {
- | jp >1
- }
- if (true_label != (uint32_t)-1) {
- | jne =>true_label
if (false_label != (uint32_t)-1) {
- | jmp =>false_label
+ | jp =>false_label
+ } else {
+ | jp >1
}
- } else {
- | je =>false_label
+ if (true_label != (uint32_t)-1) {
+ | jne =>true_label
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ }
+ } else {
+ | je =>false_label
+ }
+ |1:
}
- |1:
}
} else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| add eax, 2
}
| SET_ZVAL_TYPE_INFO res_addr, eax
- if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
+ if (exit_addr) {
+ | CMP_ZVAL_TYPE res_addr, IS_FALSE
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jne &exit_addr
+ } else {
+ | je &exit_addr
+ }
+ } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE
if (true_label != (uint32_t)-1) {
| jne =>true_label
}
} else {
| test r0, r0
- if (true_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
+ | jne &exit_addr
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ | jmp >9
+ }
+ } else {
+ | je &exit_addr
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ | jmp >9
+ }
+ }
+ } else if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
return 1;
}
-static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func)
+static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, const void *exit_addr)
{
uint32_t used_stack;
} else {
| cmp r2, FCARG1a
}
- | jb >1
- | // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
- |.cold_code
- |1:
- | SAVE_VALID_OPLINE opline
- if (func) {
- | mov FCARG1d, used_stack
- }
+
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ | jb &exit_addr
+ } else {
+ | jb >1
+ | // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
+ |.cold_code
+ |1:
+ | SAVE_VALID_OPLINE opline
+ if (func) {
+ | mov FCARG1d, used_stack
+ }
#ifdef _WIN32
- if (0) {
+ if (0) {
#else
- if (func && func->type == ZEND_INTERNAL_FUNCTION) {
+ if (func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
- | EXT_CALL zend_jit_int_extend_stack_helper, r0
- } else {
- | mov FCARG2a, r0
- | EXT_CALL zend_jit_extend_stack_helper, r0
- }
- | mov RX, r0
- | jmp >1
- |.code
+ | EXT_CALL zend_jit_int_extend_stack_helper, r0
+ } else {
+ | mov FCARG2a, r0
+ | EXT_CALL zend_jit_extend_stack_helper, r0
+ }
+ | mov RX, r0
+ | jmp >1
+ |.code
+ }
if (func) {
| MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2
}
}
-static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, int call_level)
+static int zend_jit_init_fcall_guard(dasm_State **Dst, const zend_op *opline, const zend_function *func)
+{
+ int32_t exit_point;
+ const void *exit_addr;
+
+ if (func->type == ZEND_INTERNAL_FUNCTION) {
+#ifdef ZEND_WIN32
+ // TODO: ASLR may cause different addresses in different workers ???
+ return 0;
+#endif
+ } else if (func->type == ZEND_USER_FUNCTION) {
+ if (!zend_accel_in_shm(func->op_array.opcodes)) {
+ /* op_array and op_array->opcodes are not persistent. We can't link. */
+ return 0;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ return 0;
+ }
+
+ exit_point = zend_jit_trace_get_exit_point(opline, opline ? (opline+1) : NULL, NULL);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ if (func->type == ZEND_USER_FUNCTION
+ && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
+ const zend_op *opcodes = func->op_array.opcodes;
+
+ | // call = EX(call);
+ | mov r1, EX->call
+ | mov r1, aword EX:r1->func
+ | .if X64
+ || if (!IS_SIGNED_32BIT(opcodes)) {
+ | mov64 r0, ((ptrdiff_t)opcodes)
+ | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r0
+ || } else {
+ | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
+ || }
+ | .else
+ | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
+ | .endif
+ | jne &exit_addr
+ } else {
+ | // call = EX(call);
+ | mov r1, EX->call
+ | .if X64
+ || if (!IS_SIGNED_32BIT(func)) {
+ | mov64 r0, ((ptrdiff_t)func)
+ | cmp aword EX:r1->func, r0
+ || } else {
+ | cmp aword EX:r1->func, func
+ || }
+ | .else
+ | cmp aword EX:r1->func, func
+ | .endif
+ | jne &exit_addr
+ }
+
+ return 1;
+}
+
+static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, int call_level, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
+ const void *exit_addr = NULL;
+
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ }
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
}
}
+ if (!func
+ && trace
+ && trace->op == ZEND_JIT_TRACE_INIT_CALL
+ && (opline->opcode == ZEND_INIT_FCALL || opline->opcode == ZEND_INIT_FCALL_BY_NAME)) {
+ /* TODO: add guard ??? */
+ func = (zend_function*)trace->func;
+ }
+
#ifdef _WIN32
if (0) {
#else
| mov aword [r1 + opline->result.num], r0
| test r0, r0
| jnz >3
- | // SAVE_OPLINE();
- | SAVE_VALID_OPLINE opline
- | jmp ->undefined_function
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ | jmp &exit_addr
+ } else {
+ | // SAVE_OPLINE();
+ | SAVE_VALID_OPLINE opline
+ | jmp ->undefined_function
+ }
}
|.code
|3:
}
- if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) {
+ if (!zend_jit_push_call_frame(Dst, opline, op_array, func, exit_addr)) {
return 0;
}
- if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) {
+ if (trace || zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
return 1;
}
-static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, zend_call_info *call_info)
+static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
{
uint32_t num_args = 0;
zend_function *func = call_info->callee_func;
return num_args;
}
-static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block)
+static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
- zend_function *func = NULL;
+ const zend_function *func = NULL;
uint32_t i;
zend_jit_addr res_addr;
+ uint32_t call_num_args = 0;
+ zend_bool unknown_num_args = 0;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
#endif
}
+ if ((opline-1)->opcode == ZEND_SEND_UNPACK|| (opline-1)->opcode == ZEND_SEND_ARRAY) {
+ unknown_num_args = 1;
+ }
+
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_call_opline != opline) {
/* resolve function ar run time */
} else if (func->type == ZEND_USER_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
- if (call_info->num_args > func->op_array.num_args ||
- (opline-1)->opcode == ZEND_SEND_UNPACK ||
- (opline-1)->opcode == ZEND_SEND_ARRAY) {
- goto fallback;
- }
+ call_num_args = call_info->num_args;
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
+ call_num_args = call_info->num_args;
#if ZEND_DEBUG
if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
- goto fallback;
+ // TODO: Mow most internal functions have type hints ???
+ if (!trace) {
+ goto fallback;
+ }
}
#endif
- if ((opline-1)->opcode == ZEND_SEND_UNPACK || (opline-1)->opcode == ZEND_SEND_ARRAY) {
- goto fallback;
- }
} else {
ZEND_ASSERT(0);
}
+ if (trace && !func) {
+ if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
+ ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
+#ifndef ZEND_WIN32
+ // TODO: ASLR may cause different addresses in different workers ???
+ func = trace->func;
+ if (JIT_G(current_frame) &&
+ JIT_G(current_frame)->call &&
+ JIT_G(current_frame)->call->num_args >= 0) {
+ call_num_args = JIT_G(current_frame)->call->num_args;
+ } else {
+ unknown_num_args = 1;
+ }
+#endif
+ } else if (trace->op == ZEND_JIT_TRACE_ENTER) {
+ ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
+ if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
+ func = trace->func;
+ if (JIT_G(current_frame) &&
+ JIT_G(current_frame)->call &&
+ JIT_G(current_frame)->call->num_args >= 0) {
+ call_num_args = JIT_G(current_frame)->call->num_args;
+ } else {
+ unknown_num_args = 1;
+ }
+ }
+ }
+ }
+
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
- | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
- | jnz >1
- |.cold_code
- |1:
- if (!GCC_GLOBAL_REGS) {
- | mov FCARG1a, RX
+ if (trace) {
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
+ | jnz &exit_addr
+ } else {
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
+ | jnz >1
+ |.cold_code
+ |1:
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_helper, r0
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ | mov r0, EX:RX->func // reload
+ | jmp >1
+ |.code
+ |1:
}
- | EXT_CALL zend_jit_deprecated_helper, r0
- | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
- | jne ->exception_handler
- | mov r0, EX:RX->func // reload
- | jmp >1
- |.code
- |1:
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
| EXT_CALL zend_jit_deprecated_helper, r0
| MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
| jne ->exception_handler
- | mov r0, EX:RX->func // reload
}
}
| mov aword EX:RX->return_value, 0
}
- if (func) {
- for (i = call_info->num_args; i < func->op_array.last_var; i++) {
+ if (func
+ && !unknown_num_args
+ && call_num_args <= func->op_array.num_args) {
+ for (i = call_num_args; i < func->op_array.last_var; i++) {
uint32_t n = EX_NUM_TO_VAR(i);
| SET_Z_TYPE_INFO RX + n, IS_UNDEF
}
if (!func || func->op_array.cache_size) {
if (func && op_array == &func->op_array) {
/* recursive call */
- if (func->op_array.cache_size > sizeof(void*)) {
+ if (trace || func->op_array.cache_size > sizeof(void*)) {
| mov r2, EX->run_time_cache
| mov EX:RX->run_time_cache, r2
}
| mov FP, RX
| // opline = op_array->opcodes;
- if (func) {
+ if (func
+ && !unknown_num_args
+ && call_num_args <= func->op_array.num_args) {
uint32_t num_args;
- if (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
+ && call_info) {
num_args = skip_valid_arguments(op_array, ssa, call_info);
} else {
- num_args = call_info->num_args;
+ num_args = call_num_args;
}
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes + num_args)
}
}
- if (op_array == &func->op_array) {
+ if (!trace && op_array == &func->op_array) {
/* recursive call */
#ifdef CONTEXT_THREADED_JIT
| call >1
#endif
return 1;
}
+ } else if (func
+ && !unknown_num_args
+ && call_num_args > func->op_array.num_args) {
+ if (func && zend_accel_in_shm(func->op_array.opcodes)) {
+ | LOAD_IP_ADDR (func->op_array.opcodes)
+ } else if (GCC_GLOBAL_REGS) {
+ | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
+ } else {
+ | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
+ | mov aword EX->opline, FCARG1a
+ }
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, FP
+ }
+ | EXT_CALL zend_jit_copy_extra_args_helper, r0
+ for (i = call_num_args; i < func->op_array.last_var; i++) {
+ uint32_t n = EX_NUM_TO_VAR(i);
+ | SET_Z_TYPE_INFO FP + n, IS_UNDEF
+ }
} else {
| // opline = op_array->opcodes
- if (GCC_GLOBAL_REGS) {
+ if (func && zend_accel_in_shm(func->op_array.opcodes)) {
+ | LOAD_IP_ADDR (func->op_array.opcodes)
+ } else if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
| mov aword EX->opline, FCARG1a
}
- | // first_extra_arg = op_array->num_args;
- | mov edx, dword [r0 + offsetof(zend_op_array, num_args)]
- | // num_args = EX_NUM_ARGS();
- | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
- | // if (UNEXPECTED(num_args > first_extra_arg))
- | cmp edx, ecx
- | jl >1
+ if (func) {
+ | // num_args = EX_NUM_ARGS();
+ | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
+ | // if (UNEXPECTED(num_args > first_extra_arg))
+ | cmp ecx, (func->op_array.num_args)
+ } else {
+ | // first_extra_arg = op_array->num_args;
+ | mov edx, dword [r0 + offsetof(zend_op_array, num_args)]
+ | // num_args = EX_NUM_ARGS();
+ | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
+ | // if (UNEXPECTED(num_args > first_extra_arg))
+ | cmp ecx, edx
+ }
+ | jg >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, r0
- | mov r0, EX->func // reload
+ if (!func) {
+ | mov r0, EX->func // reload
+ }
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload
| jmp >1
|.code
- | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
- | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS
- | jnz >1
- | // opline += num_args;
- |.if X64
- | movsxd r2, ecx
- | imul r2, r2, sizeof(zend_op)
- |.else
- | imul r2, ecx, sizeof(zend_op)
- |.endif
- | ADD_IP r2
+ if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
+ if (!func) {
+ | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS
+ | jnz >1
+ }
+ | // opline += num_args;
+ |.if X64
+ | movsxd r2, ecx
+ | imul r2, r2, sizeof(zend_op)
+ |.else
+ | imul r2, ecx, sizeof(zend_op)
+ |.endif
+ | ADD_IP r2
+ }
|1:
| // if (EXPECTED((int)num_args < op_array->last_var)) {
- | mov edx, dword [r0 + offsetof(zend_op_array, last_var)]
+ if (func) {
+ | mov edx, (func->op_array.last_var)
+ } else {
+ | mov edx, dword [r0 + offsetof(zend_op_array, last_var)]
+ }
| sub edx, ecx
| jle >3 //???
| // zval *var = EX_VAR_NUM(num_args);
|3:
}
+ if (trace) {
+ if (!func && (opline->opcode != ZEND_DO_UCALL)) {
+ | jmp >9
+ }
+ } else {
#ifdef CONTEXT_THREADED_JIT
- | call ->context_threaded_call
- if (!func && (opline->opcode != ZEND_DO_UCALL)) {
- | jmp >9
- }
+ | call ->context_threaded_call
+ if (!func && (opline->opcode != ZEND_DO_UCALL)) {
+ | jmp >9
+ }
+ | call ->context_threaded_call
+ if (!func) {
+ | jmp >9
+ }
#else
- if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
- | add r4, HYBRID_SPAD
- | JMP_IP
- } else if (GCC_GLOBAL_REGS) {
- | add r4, SPAD // stack alignment
- | JMP_IP
- } else {
- | mov FP, aword T2 // restore FP
- | mov RX, aword T3 // restore IP
- | add r4, NR_SPAD // stack alignment
- | mov r0, 1 // ZEND_VM_ENTER
- | ret
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | JMP_IP
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 1 // ZEND_VM_ENTER
+ | ret
+ }
}
#endif
}
}
if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
- | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
- | jnz >1
- |.cold_code
- |1:
- if (!GCC_GLOBAL_REGS) {
- | mov FCARG1a, RX
+ if (trace) {
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
+ | jnz &exit_addr
+ } else {
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
+ | jnz >1
+ |.cold_code
+ |1:
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_helper, r0
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ | mov r0, EX:RX->func // reload
+ | jmp >1
+ |.code
+ |1:
}
- | EXT_CALL zend_jit_deprecated_helper, r0
- | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
- | jne ->exception_handler
- | mov r0, EX:RX->func // reload
- | jmp >1
- |.code
- |1:
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
| call aword [r0 + offsetof(zend_internal_function, handler)]
}
+ if (trace) {
+ // TODO: This is a qucik dirty fix ??????
+ //
+ // Internal function may call another trace that,
+ // replaces EG(trace_id) and the following side exit
+ // from this trace is going to be mad !!!!!!
+ //
+ // Lets set EG(trace_id) once again...
+ zend_jit_trace_begin(Dst, current_trace_num);
+ }
+
| // EG(current_execute_data) = execute_data;
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0
| // zend_vm_stack_free_args(call);
- if (func) {
- for (i = 0; i < call_info->num_args; i++ ) {
+ if (func && call_info && !unknown_num_args) {
+ for (i = 0; i < call_num_args; i++ ) {
uint32_t offset = EX_NUM_TO_VAR(i);
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 0, opline
}
|2:
}
- | // zend_vm_stack_free_call_frame(call);
- | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16)
- | jnz >1
- |.cold_code
- |1:
- | mov FCARG1a, RX
- | EXT_CALL zend_jit_free_call_frame, r0
- | jmp >1
- |.code
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE ||
+ !JIT_G(current_frame) ||
+ !JIT_G(current_frame)->call ||
+ !JIT_G(current_frame)->call->nested) {
+
+ | // zend_vm_stack_free_call_frame(call);
+ | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16)
+ | jnz >1
+ |.cold_code
+ |1:
+ | mov FCARG1a, RX
+ | EXT_CALL zend_jit_free_call_frame, r0
+ | jmp >1
+ |.code
+ }
| MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0
|1:
}
return 1;
-
+#if ZEND_DEBUG
fallback:
/* fallback to subroutine threading */
+ if (trace) {
+ return zend_jit_trace_handler(Dst, op_array, opline, zend_may_throw(opline, op_array, ssa), trace);
+ }
if (opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ){
} else {
return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
}
+#endif
}
static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
if (opline->opcode == ZEND_SEND_VAL_EX) {
uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
- | mov r0, EX:RX->func
- if (arg_num <= MAX_ARG_FLAG_NUM) {
+ ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
+
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ /* Don't generate code that always throws exception */
+ return 0;
+ }
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ | mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
- | jnz >1
+ | jnz &exit_addr
} else {
- ZEND_ASSERT(0);
+ | mov r0, EX:RX->func
+ | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
+ | jnz >1
+ |.cold_code
+ |1:
+ | SAVE_VALID_OPLINE opline
+ | jmp ->throw_cannot_pass_by_ref
+ |.code
+
}
- |.cold_code
- |1:
- | SAVE_VALID_OPLINE opline
- | jmp ->throw_cannot_pass_by_ref
- |.code
}
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
- | ZVAL_COPY_CONST arg_addr, -1, -1, zv, r0
+ | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, r0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else {
- | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
}
return 1;
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
| mov r1, aword T1 // restore
- | ZVAL_COPY_VALUE ref_addr, -1, val_addr, op1_info, ZREG_R2, ZREG_R2
+ | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2
| SET_ZVAL_PTR val_addr, r0
| SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX
} else {
- | ZVAL_COPY_VALUE ref_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2
+ | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
}
| mov RX, EX->call
}
- if (opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
- uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
+ if (opline->opcode == ZEND_SEND_VAR_EX) {
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
+ return 0;
+ }
+ return 1;
+ }
+ } else {
+ uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
- | mov r0, EX:RX->func
- if (arg_num <= MAX_ARG_FLAG_NUM) {
+ | mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
- } else {
- ZEND_ASSERT(0);
+ |.cold_code
+ |1:
+ if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
+ return 0;
+ }
+ | jmp >7
+ |.code
}
+ } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE
+ && JIT_G(current_frame)
+ && JIT_G(current_frame)->call
+ && JIT_G(current_frame)->call->func) {
+ if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
- |.cold_code
- |1:
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
- if (opline->opcode == ZEND_SEND_VAR_EX) {
- if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
- return 0;
+ if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
+ if (!(op1_info & MAY_BE_REF)) {
+ /* Don't generate code that always throws exception */
+ return 0;
+ } else {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ | cmp cl, IS_REFERENCE
+ | jne &exit_addr
+ }
+ }
+ return 1;
}
- } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
+ } else {
+ uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
+
+ | mov r0, EX:RX->func
+ | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
+ | jnz >1
+ |.cold_code
+ |1:
+
mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
- | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (op1_info & MAY_BE_REF) {
| cmp cl, IS_REFERENCE
| je >7
}
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >7
- | SAVE_VALID_OPLINE opline
- | LOAD_ZVAL_ADDR FCARG1a, arg_addr
- | EXT_CALL zend_jit_only_vars_by_reference, r0
- if (!zend_jit_check_exception(Dst)) {
- return 0;
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ | jmp &exit_addr
+ } else {
+ | SAVE_VALID_OPLINE opline
+ | LOAD_ZVAL_ADDR FCARG1a, arg_addr
+ | EXT_CALL zend_jit_only_vars_by_reference, r0
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ | jmp >7
}
- } else {
- ZEND_ASSERT(0);
- }
- | jmp >7
- |.code
+ |.code
+ }
}
if (op1_info & MAY_BE_UNDEF) {
}
if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
- | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (op1_info & MAY_BE_REF) {
| cmp cl, IS_REFERENCE
| je >7
}
- | SAVE_VALID_OPLINE opline
- | LOAD_ZVAL_ADDR FCARG1a, arg_addr
- | EXT_CALL zend_jit_only_vars_by_reference, r0
- if (!zend_jit_check_exception(Dst)) {
- return 0;
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ | jmp &exit_addr
+ } else {
+ | SAVE_VALID_OPLINE opline
+ | LOAD_ZVAL_ADDR FCARG1a, arg_addr
+ | EXT_CALL zend_jit_only_vars_by_reference, r0
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
}
} else if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
- | ZVAL_COPY_VALUE arg_addr, -1, val_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2
| TRY_ADDREF op1_info, ah, r2
} else {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8);
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR FCARG1a, op1_addr
| // ZVAL_COPY_VALUE(return_value, &ref->value);
- | ZVAL_COPY_VALUE arg_addr, -1, ref_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2
| GC_DELREF FCARG1a
| je >1
| IF_NOT_REFCOUNTED ah, >2
| EFREE_REG_24 op_array, opline
| jmp >2
|.code
- | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
|2:
}
} else {
op1_addr= op1_def_addr;
}
}
- | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, ah, r2
}
return 1;
}
-static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t defined_label = (uint32_t)-1;
uint32_t undefined_label = (uint32_t)-1;
zval *zv = RT_CONSTANT(opline, opline->op1);
- zend_jit_addr res_addr;
+ zend_jit_addr res_addr = 0;
- if (smart_branch_opcode) {
+ if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
undefined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a
| shr r0, 1
| cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax
+
if (smart_branch_opcode) {
- if (undefined_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jz &exit_addr
+ } else {
+ | jz >3
+ }
+ } else if (undefined_label != (uint32_t)-1) {
| jz =>undefined_label
} else {
| jz >3
| LOAD_ADDR FCARG1a, zv
| EXT_CALL zend_jit_check_constant, r0
| test r0, r0
- if (smart_branch_opcode) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jnz >3
+ } else {
+ | jz >3
+ }
+ | jmp &exit_addr
+ } else if (smart_branch_opcode) {
if (undefined_label != (uint32_t)-1) {
| jnz =>undefined_label
} else {
}
|.code
if (smart_branch_opcode) {
- if (defined_label != (uint32_t)-1) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ }
+ } else if (defined_label != (uint32_t)-1) {
| jmp =>defined_label
}
} else {
return 1;
}
-static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t mask;
zend_uchar type;
| EXT_CALL zend_jit_undefined_op_helper, r0
zend_jit_check_exception_undef_result(Dst, opline);
if (opline->extended_value & MAY_BE_NULL) {
- if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
+ | jmp >7
+ }
+ } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else {
- if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
+ | jmp >7
+ }
+ } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) {
return 0;
}
}
type = 0;
}
- if (!(op1_info & (MAY_BE_ANY - mask))) {
+ if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
- if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ }
+ } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
- } else if (!(op1_info & mask)) {
+ } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
- if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ }
+ } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) {
return 0;
}
} else {
| mov eax, 1
| shl eax, cl
| test eax, mask
- if (smart_branch_opcode) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jne &exit_addr
+ } else {
+ | je &exit_addr
+ }
+ } else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| je =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| cmp byte [FP + opline->op1.var + 8], type
}
}
- if (smart_branch_opcode) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | je &exit_addr
+ } else {
+ | jne &exit_addr
+ }
+ } else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jne =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
return 1;
}
-static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+static uint32_t zend_ssa_cv_info(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
{
- uint32_t i, j, info;
+ uint32_t j, info;
- // Use type inference to avoid useless zval_ptr_dtor()
- for (i = 0 ; i < op_array->last_var; i++) {
- if (ssa->vars && ssa->var_info) {
- info = ssa->var_info[i].type;
- for (j = op_array->last_var; j < ssa->vars_count; j++) {
- if (ssa->vars[j].var == i) {
- info |= ssa->var_info[j].type;
- }
+ if (ssa->vars && ssa->var_info) {
+ info = ssa->var_info[var].type;
+ for (j = op_array->last_var; j < ssa->vars_count; j++) {
+ if (ssa->vars[j].var == var) {
+ info |= ssa->var_info[j].type;
}
- } else {
- info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF;
}
+ } else {
+ info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF;
+ }
#ifdef ZEND_JIT_USE_RC_INFERENCE
- /* Refcount may be increased by RETURN opcode */
- if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
- for (j = 0; j < ssa->cfg.blocks_count; j++) {
- if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
- ssa->cfg.blocks[j].len > 0) {
- const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
-
- if (opline->opcode == ZEND_RETURN) {
- if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(i)) {
- info |= MAY_BE_RCN;
- break;
- }
+ /* Refcount may be increased by RETURN opcode */
+ if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
+ for (j = 0; j < ssa->cfg.blocks_count; j++) {
+ if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
+ ssa->cfg.blocks[j].len > 0) {
+ const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
+
+ if (opline->opcode == ZEND_RETURN) {
+ if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
+ info |= MAY_BE_RCN;
+ break;
}
}
}
}
+ }
#endif
- if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
- uint32_t offset = EX_NUM_TO_VAR(i);
- | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 0, opline
- }
- }
- return 1;
+ return info;
}
-static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa)
+static int zend_jit_leave_frame(dasm_State **Dst)
{
- // Avoid multiple leave sequences
- if (jit_return_label >= 0) {
- | jmp =>jit_return_label
- return 1;
- }
-
- jit_return_label = ssa->cfg.blocks_count * 2;
-
- |=>jit_return_label:
-
| // EG(current_execute_data) = EX(prev_execute_data);
| mov r0, EX->prev_execute_data
| MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2
+ return 1;
+}
- // i_free_compiled_variables(execute_data);
- if (!zend_jit_free_compiled_variables(Dst, opline, op_array, ssa)) {
- return 0;
+static int zend_jit_free_cv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t info, uint32_t var)
+{
+ if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ uint32_t offset = EX_NUM_TO_VAR(var);
+ | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 0, opline
}
+ return 1;
+}
+static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_jit_trace_rec *trace)
+{
/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
| mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)]
| test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_FAKE_CLOSURE)
- | jnz ->leave_function_handler
+ if (trace && trace->op != ZEND_JIT_TRACE_END) {
+ | jnz >1
+ |.cold_code
+ |1:
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG2a, FP
+ }
+ | EXT_CALL zend_jit_leave_func_helper, r0
+
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE ||
+ !JIT_G(current_frame) ||
+ !JIT_G(current_frame)->nested) {
+ // TODO: try to avoid this check ???
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | cmp IP, zend_jit_halt_op
+ | je ->trace_halt
+ } else if (GCC_GLOBAL_REGS) {
+ | test IP, IP
+ | je ->trace_halt
+ } else {
+ | test eax, eax
+ | jl ->trace_halt
+ }
+ }
+
+ if (!GCC_GLOBAL_REGS) {
+ | // execute_data = EG(current_execute_data)
+ | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
+ }
+ | jmp >8
+ |.code
+ } else {
+ | jnz ->leave_function_handler
+ }
if ((op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) ||
(op_array->fn_flags & ZEND_ACC_CLOSURE)) {
| MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0
| // execute_data = EX(prev_execute_data);
| mov FP, EX->prev_execute_data
- | // if (EG(exception))
- | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
- | LOAD_OPLINE
- | jne ->leave_throw_handler
- | // opline = EX(opline) + 1
- | ADD_IP sizeof(zend_op)
+
+ |9:
+ if (trace) {
+ if (trace->op != ZEND_JIT_TRACE_END
+ && (trace->op != ZEND_JIT_TRACE_BACK || !trace->recursive)) {
+ zend_jit_reset_opline(Dst, NULL);
+ } else {
+ // TODO: exception handling for tracing JIT ???
+ | LOAD_OPLINE
+ | ADD_IP sizeof(zend_op)
+ }
+
+ |8:
+
+ if (opline->opcode == ZEND_RETURN
+ && trace->op == ZEND_JIT_TRACE_BACK
+ && trace->recursive) {
+ const zend_op *next_opline = trace->opline;
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, NULL, trace);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ trace++;
+ ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
+ next_opline = trace->opline;
+ exit_point = zend_jit_trace_get_exit_point(opline, NULL, trace);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+ | // TODO: exception handling ???
+ | CMP_IP next_opline
+ | jne &exit_addr
+
+ last_valid_opline = trace->opline;
+
+ return 1;
+ }
+ return 1;
+ } else {
+ | // if (EG(exception))
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | LOAD_OPLINE
+ | jne ->leave_throw_handler
+ | // opline = EX(opline) + 1
+ | ADD_IP sizeof(zend_op)
+ }
+
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD
#ifdef CONTEXT_THREADED_JIT
return 1;
}
-static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, uint32_t op1_info, zend_jit_addr op1_addr)
+static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr ret_addr;
+ int8_t return_value_used;
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
+ return_value_used = JIT_G(current_frame)->return_value_used;
+ } else {
+ return_value_used = -1;
+ }
+
// if (!EX(return_value))
if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) {
- | mov r2, EX->return_value
- | test r2, r2
+ if (return_value_used != 0) {
+ | mov r2, EX->return_value
+ }
+ if (return_value_used == -1) {
+ | test r2, r2
+ }
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
} else {
- | mov r1, EX->return_value
- | test r1, r1
+ if (return_value_used != 0) {
+ | mov r1, EX->return_value
+ }
+ if (return_value_used == -1) {
+ | test r1, r1
+ }
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
- | jz >1
- |.cold_code
- |1:
- if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
- if (jit_return_label >= 0) {
- | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label
- } else {
- | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9
- }
+ if (return_value_used == -1) {
+ | jz >1
+ |.cold_code
+ |1:
}
- | GET_ZVAL_PTR FCARG1a, op1_addr
- | GC_DELREF FCARG1a
- if (RC_MAY_BE_1(op1_info)) {
- if (RC_MAY_BE_N(op1_info)) {
+ if (return_value_used != 1) {
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (jit_return_label >= 0) {
- | jnz =>jit_return_label
+ | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label
} else {
- | jnz >9
+ | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9
}
}
- | //SAVE_OPLINE()
- | ZVAL_DTOR_FUNC op1_info, opline
- | //????mov r1, EX->return_value // reload ???
- }
- if (jit_return_label >= 0) {
- | jmp =>jit_return_label
- } else {
- | jmp >9
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ | GC_DELREF FCARG1a
+ if (RC_MAY_BE_1(op1_info)) {
+ if (RC_MAY_BE_N(op1_info)) {
+ if (jit_return_label >= 0) {
+ | jnz =>jit_return_label
+ } else {
+ | jnz >9
+ }
+ }
+ | //SAVE_OPLINE()
+ | ZVAL_DTOR_FUNC op1_info, opline
+ | //????mov r1, EX->return_value // reload ???
+ }
+ if (return_value_used == -1) {
+ if (jit_return_label >= 0) {
+ | jmp =>jit_return_label
+ } else {
+ | jmp >9
+ }
+ |.code
+ }
}
- |.code
- } else {
+ } else if (return_value_used == -1) {
if (jit_return_label >= 0) {
| jz =>jit_return_label
} else {
}
}
+ if (return_value_used == 0) {
+ return 1;
+ }
+
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
- | ZVAL_COPY_CONST ret_addr, -1, -1, zv, r0
+ | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, r0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else if (opline->op1_type == IS_TMP_VAR) {
- | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR r0, op1_addr
| ZVAL_DEREF r0, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
}
- | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
| // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ???
| TRY_ADDREF op1_info, ah, r2
} else {
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR r0, op1_addr
| // ZVAL_COPY_VALUE(return_value, &ref->value);
- | ZVAL_COPY_VALUE ret_addr, -1, ref_addr, op1_info, ZREG_R2, ZREG_R2
+ | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2
| GC_DELREF r0
| je >2
| // if (IS_REFCOUNTED())
}
|.code
}
- | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
}
|9:
- //JIT: ZEND_VM_DISPATCH_TO_HELPER(zend_leave_helper);
- return zend_jit_leave_func(Dst, opline, op_array, ssa);
+ return 1;
}
static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, uint32_t op2_info, uint32_t res_info, int may_throw)
return 1;
}
-static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, uint32_t op2_info, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
+static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, uint32_t op2_info, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_jit_addr op1_addr, op2_addr, res_addr;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
- if (smart_branch_opcode) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPNZ) {
+ | jmp &exit_addr
+ } else {
+ | jmp >8
+ }
+ } else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label2
} else if (smart_branch_opcode == ZEND_JMPNZ) {
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
- if (smart_branch_opcode) {
+ if (exit_addr) {
+ if (smart_branch_opcode == ZEND_JMPZ) {
+ | jmp &exit_addr
+ }
+ } else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
- | cmp dword EX->This.u2.num_args, arg_num
- | jae >5
+ if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE ||
+ (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ | cmp dword EX->This.u2.num_args, arg_num
+ | jae >5
+ }
| ZVAL_COPY_CONST res_addr, -1, -1, zv, r0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
return 0;
}
-static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_class_entry *ce, int may_throw)
+static int zend_jit_fetch_obj_read(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_class_entry *ce, int may_throw)
{
zval *member;
uint32_t offset;
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
- | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
+ } else {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7
+ }
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
| jmp >9
}
- if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) {
+ if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) {
|7:
if (opline->opcode == ZEND_FETCH_OBJ_R) {
| SAVE_VALID_OPLINE opline
projects / php / commitdiff