static void ZEND_FASTCALL zend_runtime_jit(void);
static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline, zend_jit_trace_rec *trace);
-static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace);
+static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace, uint32_t flags);
static const void *zend_jit_trace_get_exit_addr(uint32_t n);
static void zend_jit_trace_add_code(const void *start, uint32_t size);
#define ZEND_JIT_EXIT_JITED (1<<0)
#define ZEND_JIT_EXIT_BLACKLISTED (1<<1)
+#define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */
typedef union _zend_op_trace_info {
zend_op dummy; /* the size of this structure must be the same as zend_op */
typedef struct _zend_jit_trace_exit_info {
const zend_op *opline; /* opline where VM should continue execution */
const zend_op_array *op_array;
+ uint32_t flags; /* set of ZEND_JIT_EXIT_... */
uint32_t stack_size;
uint32_t stack_offset;
} zend_jit_trace_exit_info;
((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
}
-static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace)
+static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace, uint32_t flags)
{
zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
uint32_t exit_point;
|| (t->exit_info[i].stack_size >= stack_size
&& memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
stack_offset = t->exit_info[i].stack_offset;
- if (t->exit_info[i].opline == to_opline) {
+ if (t->exit_info[i].opline == to_opline
+ && t->exit_info[i].flags == flags) {
return i;
}
}
t->exit_count++;
t->exit_info[exit_point].opline = to_opline;
t->exit_info[exit_point].op_array = op_array;
+ t->exit_info[exit_point].flags = flags;
t->exit_info[exit_point].stack_size = stack_size;
t->exit_info[exit_point].stack_offset = stack_offset;
}
if (!parent_trace) {
zend_jit_set_opline(&dasm_state, opline);
} else {
- if (!((uintptr_t)zend_jit_traces[parent_trace].exit_info[exit_num].opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
+ if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
zend_jit_trace_opline_guard(&dasm_state, opline);
zend_jit_set_opline(&dasm_state, opline);
} else {
const void *exit_addr = NULL;
if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
- uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, NULL, NULL);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, NULL, NULL, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
if (ra) {
zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
}
- exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
goto jit_failure;
if (ra) {
zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
}
- exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
goto jit_failure;
if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
zend_bool exit_if_true = 0;
const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
- uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
if (ra) {
zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
}
- exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
goto jit_failure;
if (ra) {
zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
}
- exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p+1);
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p+1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
goto jit_failure;
if (ra) {
zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
}
- exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1);
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, p + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
goto jit_failure;
uint32_t stack_size;
zend_jit_trace_stack *stack;
- opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED));
if (opline) {
return 1;
}
}
opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
- opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED));
if (opline) {
zend_jit_set_ip(&dasm_state, opline);
if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
} else {
fprintf(stderr, "----/0");
}
+ if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
+ fprintf(stderr, "/VM");
+ }
for (j = 0; j < stack_size; j++) {
zend_uchar type = STACK_TYPE(stack, j);
if (type != IS_UNKNOWN) {
zend_shared_alloc_lock();
- if (!((uintptr_t)zend_jit_traces[trace_num].exit_info[exit_num].opline & ZEND_JIT_EXIT_BLACKLISTED)) {
+ if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
SHM_UNPROTECT();
zend_jit_unprotect();
handler);
}
- zend_jit_traces[trace_num].exit_info[exit_num].opline = (const zend_op*)
- ((uintptr_t)zend_jit_traces[trace_num].exit_info[exit_num].opline | ZEND_JIT_EXIT_BLACKLISTED);
+ zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
zend_jit_protect();
SHM_PROTECT();
zend_shared_alloc_lock();
/* Checks under lock */
- if (((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline & ZEND_JIT_EXIT_JITED)) {
+ if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
} else if (ZEND_JIT_TRACE_NUM >= ZEND_JIT_TRACE_MAX_TRACES) {
ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
ZEND_JIT_TRACE_NUM++;
- zend_jit_traces[parent_num].exit_info[exit_num].opline = (const zend_op*)
- ((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline | ZEND_JIT_EXIT_JITED);
+ zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
ret = ZEND_JIT_TRACE_STOP_COMPILED;
} else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
trace_num = ZEND_JIT_TRACE_NUM;
/* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
- if ((uintptr_t)zend_jit_traces[parent_num].exit_info[exit_num].opline & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
+ if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
return 0;
}
}
}
- /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
- // TODO: We may remove this, because of the same check in zend_jit_trace_hot_side() ???
opline = t->exit_info[exit_num].opline;
- if (EG(vm_interrupt) || ((uintptr_t)opline & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
- opline = (const zend_op*)((uintptr_t)opline & ~(ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED));
- if (opline) {
- /* Set VM opline to continue interpretation */
- EX(opline) = opline;
- }
- return 0;
- }
-
if (opline) {
/* Set VM opline to continue interpretation */
EX(opline) = opline;
}
+ if (EG(vm_interrupt)) {
+ return 0;
+ /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
+ } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
+ return 0;
+ }
+
ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
EX(opline)->lineno);
}
- if (zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
+ if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
+ if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
+ zend_jit_blacklist_trace_exit(trace_num, exit_num);
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
+ fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
+ trace_num, exit_num);
+ }
+ }
+ } else if (zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
}
static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline)
{
- uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, NULL, NULL);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, NULL, NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type)
{
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
}
}
- exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, trace);
+
+ exit_point = zend_jit_trace_get_exit_point(opline, exit_opline, trace, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
}
if (may_overflow && ((op1_def_info & MAY_BE_GUARD) || (opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD)))) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| jo &exit_addr
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
}
if (may_overflow) {
if (res_info & MAY_BE_GUARD) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| jo &exit_addr
} else {
const void *exit_addr = NULL;
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
return 1;
}
-static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, const void *exit_addr)
+static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func)
{
uint32_t used_stack;
}
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
| jb &exit_addr
} else {
| jb >1
return 0;
}
- exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL);
+ exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
- const void *exit_addr = NULL;
-
- if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
- exit_addr = zend_jit_trace_get_exit_addr(exit_point);
- if (!exit_addr) {
- return 0;
- }
- }
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
| mov r1, EX->run_time_cache
| mov aword [r1 + opline->result.num], r0
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0);
+ const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+ if (!exit_addr) {
+ return 0;
+ }
+
if (!func || opline->opcode == ZEND_INIT_FCALL) {
| test r0, r0
| jnz >3
|3:
}
- if (!zend_jit_push_call_frame(Dst, opline, op_array, func, exit_addr)) {
+ if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) {
return 0;
}
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (trace) {
- uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (trace) {
- uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ uint32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
/* Don't generate code that always throws exception */
return 0;
} else {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >7
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
| je >7
}
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
next_opline = trace->opline;
current_frame = JIT_G(current_frame);
JIT_G(current_frame) = NULL;
- exit_point = zend_jit_trace_get_exit_point(opline, NULL, trace);
+ exit_point = zend_jit_trace_get_exit_point(opline, NULL, trace, 0);
JIT_G(current_frame) = current_frame;
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
- int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL);
+ int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
| cmp byte EX->This.u1.v.type, IS_OBJECT