level = 0;
for (;;p++) {
if (p->op == ZEND_JIT_TRACE_VM) {
- uint8_t orig_op1_type, op1_type, op2_type, op3_type;
+ uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
+// zend_class_entry *op1_ce = NULL;
+ zend_class_entry *op2_ce = NULL;
// TODO: range inference ???
opline = p->opline;
op1_type = orig_op1_type = p->op1_type;
- op2_type = p->op2_type;
+ op2_type = orig_op2_type = p->op2_type;
op3_type = p->op3_type;
if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
op1_type = IS_UNKNOWN;
}
if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
- // TODO: support for recorded classes ???
+// op1_ce = (zend_class_entry*)(p+1)->ce;
p++;
}
if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
- // TODO: support for recorded classes ???
+ op2_ce = (zend_class_entry*)(p+1)->ce;
p++;
}
}
ADD_OP1_TRACE_GUARD();
break;
+ case ZEND_INIT_DYNAMIC_CALL:
+ if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
+ ADD_OP2_TRACE_GUARD();
+ }
+ break;
default:
break;
}
uint8_t op2_type = p->op2_type;
uint8_t op3_type = p->op3_type;
uint8_t orig_op1_type = op1_type;
+ uint8_t orig_op2_type = op2_type;
zend_bool op1_indirect;
zend_class_entry *op1_ce = NULL;
+ zend_class_entry *op2_ce = NULL;
opline = p->opline;
if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
p++;
}
if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
- // TODO: support for recorded classes ???
+ op2_ce = (zend_class_entry*)(p+1)->ce;
p++;
}
goto jit_failure;
}
goto done;
- case ZEND_INIT_METHOD_CALL:
case ZEND_INIT_DYNAMIC_CALL:
+ if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
+ op2_info = OP2_INFO();
+ CHECK_OP2_TRACE_TYPE();
+ if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) {
+ goto jit_failure;
+ }
+ goto done;
+ }
+ /* break missing intentionally */
+ case ZEND_INIT_METHOD_CALL:
if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
goto jit_failure;
}
if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
fprintf(stderr, "/CALL");
}
- if (t->exit_info[i].flags & ZEND_JIT_EXIT_POLYMORPHISM) {
+ if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_DYNAMIC_CALL)) {
fprintf(stderr, "/POLY");
}
if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
int ret = 0;
uint32_t trace_num;
zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
- zend_bool is_megamorphic = 0;
+ uint32_t is_megamorphic = 0;
uint32_t polymorphism = 0;
trace_num = ZEND_JIT_TRACE_NUM;
goto abort;
}
- if (EX(call)
- && JIT_G(max_polymorphic_calls) > 0
- && (zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)) {
- if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
- is_megamorphic = 1;
- } else if (!zend_jit_traces[parent_num].polymorphism) {
- polymorphism = 1;
- } else if (exit_num == 0) {
- polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
+ if (JIT_G(max_polymorphic_calls) > 0) {
+ if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_DYNAMIC_CALL)
+ || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
+ && EX(call))) {
+ if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
+ is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
+ (ZEND_JIT_EXIT_DYNAMIC_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
+ } else if (!zend_jit_traces[parent_num].polymorphism) {
+ polymorphism = 1;
+ } else if (exit_num == 0) {
+ polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
+ }
}
}
return 0;
}
-static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t init_level, uint32_t *call_level)
+static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t init_level, uint32_t *call_level)
{
zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
}
func = (zend_function*)jit_extension->op_array;
}
- if (is_megamorphic
+ if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
/* TODO: use more accurate check ??? */
&& ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
|| func->common.scope)) {
return idx;
}
-static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level)
+static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t *megamorphic, uint32_t level)
{
uint32_t call_level = 0;
*
*/
-zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphic)
+zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphic)
{
#ifdef HAVE_GCC_GLOBAL_REGS
if (JIT_G(max_polymorphic_calls) == 0
&& zend_jit_may_be_polymorphic_call(opline - 1)) {
func = NULL;
+ } else if (is_megamorphic == ZEND_JIT_EXIT_DYNAMIC_CALL
+ && trace_buffer[1].opline == opline - 1) {
+ func = NULL;
}
call_level = zend_jit_trace_call_level(EX(call));
ZEND_ASSERT(ret_level + level + call_level < 32);
return 1;
}
-static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func)
+/* copy of hidden zend_closure */
+typedef struct _zend_closure {
+ zend_object std;
+ zend_function func;
+ zval this_ptr;
+ zend_class_entry *called_scope;
+ zif_handler orig_internal_handler;
+} zend_closure;
+
+static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func, zend_bool is_closure)
{
uint32_t used_stack;
used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval);
| // if (EXPECTED(ZEND_USER_CODE(func->type))) {
- | test byte [r0 + offsetof(zend_function, type)], 1
- | mov FCARG1a, used_stack
- | jnz >1
+ if (!is_closure) {
+ | test byte [r0 + offsetof(zend_function, type)], 1
+ | mov FCARG1a, used_stack
+ | jnz >1
+ } else {
+ | mov FCARG1a, used_stack
+ }
| // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
| mov edx, opline->extended_value
- | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
- | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
- | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)]
- | sub edx, dword [r0 + offsetof(zend_function, op_array.T)]
+ if (!is_closure) {
+ | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
+ | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
+ | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)]
+ | sub edx, dword [r0 + offsetof(zend_function, op_array.T)]
+ } else {
+ | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)]
+ | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)]
+ | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)]
+ | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)]
+ }
| shl edx, 5
|.if X64
| movsxd r2, edx
| // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
|.cold_code
|1:
- | SET_EX_OPLINE opline, r0
if (func) {
| mov FCARG1d, used_stack
}
#else
if (func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
+ | SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_int_extend_stack_helper, r0
} else {
- | mov FCARG2a, r0
+ if (!is_closure) {
+ | mov FCARG2a, r0
+ } else {
+ | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)]
+ }
+ | SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_extend_stack_helper, r0
}
| mov RX, r0
| // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)
- | // call->func = func;
#ifdef _WIN32
if (0) {
#else
if (func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
+ | // call->func = func;
|1:
| ADDR_OP2_2 mov, aword EX:RX->func, func, r1
} else {
- | mov aword EX:RX->func, r0
+ if (!is_closure) {
+ | // call->func = func;
+ | mov aword EX:RX->func, r0
+ } else {
+ | // call->func = &closure->func;
+ | lea r1, aword [r0 + offsetof(zend_closure, func)]
+ | mov aword EX:RX->func, r1
+ }
+ |1:
+ }
+ if (!is_closure) {
+ | // Z_CE(call->This) = called_scope;
+ | mov aword EX:RX->This.value.ptr, 0
+ } else {
+ if (opline->op2_type == IS_CV) {
+ | // GC_ADDREF(closure);
+ | add dword [r0], 1
+ }
+ | // object_or_called_scope = closure->called_scope;
+ | mov r1, aword [r0 + offsetof(zend_closure, called_scope)]
+ | mov aword EX:RX->This.value.ptr, r1
+ | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
+ | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
+ | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)]
+ | and edx, ZEND_ACC_FAKE_CLOSURE
+ | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE)
+ | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
+ | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF
+ | jz >1
+ | // call_info |= ZEND_CALL_HAS_THIS;
+ | or edx, ZEND_CALL_HAS_THIS
+ | // object_or_called_scope = Z_OBJ(closure->this_ptr);
+ | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)]
+ |1:
+ | // ZEND_SET_CALL_INFO(call, 0, call_info);
+ | or dword EX:RX->This.u1.type_info, edx
+ | // Z_PTR(call->This) = object_or_called_scope;
+ | mov aword EX:RX->This.value.ptr, r1
+ | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0
+ | jnz >1
+ | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)]
+ | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0
|1:
}
- | // Z_CE(call->This) = called_scope;
- | mov aword EX:RX->This.value.ptr, 0
| // ZEND_CALL_NUM_ARGS(call) = num_args;
| mov dword EX:RX->This.u2.num_args, opline->extended_value
return 1;
|3:
}
- if (!zend_jit_push_call_frame(Dst, opline, func)) {
+ if (!zend_jit_push_call_frame(Dst, opline, func, 0)) {
return 0;
}
return 1;
}
+static int zend_jit_init_closure_call(dasm_State **Dst,
+ const zend_op *opline,
+ uint32_t b,
+ const zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_ssa_op *ssa_op,
+ int call_level,
+ zend_jit_trace_rec *trace)
+{
+ zend_function *func = NULL;
+ zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+
+ | GET_ZVAL_PTR r0, op2_addr
+
+ if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
+ && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
+ |.if X64
+ || if (!IS_SIGNED_32BIT(zend_ce_closure)) {
+ | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure)
+ | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a
+ || } else {
+ | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure
+ || }
+ |.else
+ | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure
+ |.endif
+ | jne &exit_addr
+ if (ssa->var_info && ssa_op->op2_use >= 0) {
+ ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
+ ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
+ ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
+ }
+ }
+
+ if (trace
+ && trace->op == ZEND_JIT_TRACE_INIT_CALL
+ && trace->func
+ && trace->func->type == ZEND_USER_FUNCTION) {
+ const zend_op *opcodes;
+ int32_t exit_point;
+ const void *exit_addr;
+
+ func = (zend_function*)trace->func;
+ opcodes = func->op_array.opcodes;
+ exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_DYNAMIC_CALL);
+ exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+ if (!exit_addr) {
+ return 0;
+ }
+
+ | .if X64
+ || if (!IS_SIGNED_32BIT(opcodes)) {
+ | mov64 FCARG1a, ((ptrdiff_t)opcodes)
+ | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a
+ || } else {
+ | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes
+ || }
+ | .else
+ | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes
+ | .endif
+ | jne &exit_addr
+ }
+
+ if (delayed_call_chain) {
+ if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ if (!zend_jit_push_call_frame(Dst, opline, func, 1)) {
+ return 0;
+ }
+
+ if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, trace)) {
+ if (!zend_jit_save_call_chain(Dst, call_level)) {
+ return 0;
+ }
+ } else {
+ delayed_call_chain = 1;
+ delayed_call_level = call_level;
+ }
+
+ if (trace
+ && trace->op == ZEND_JIT_TRACE_END
+ && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
+ if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
{
uint32_t num_args = 0;
projects / php / commitdiff