Free room for information about register allocation at astact stack and at trace_exit...

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index 54eaca15c6bc1ea32cd70337c2256a6cc96891b5..25fedaa93db769c2c0cd242edb496daf7aedc183 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -307,7 +307,35 @@ typedef struct _zend_jit_trace_exit_info {
        uint32_t       stack_offset;
 } zend_jit_trace_exit_info;
 
-typedef int32_t zend_jit_trace_stack;
+typedef union _zend_jit_trace_stack {
+       int32_t      __ssa_var;
+       uint32_t     __info;
+       struct {
+               uint8_t  __type;
+               int8_t   __reg;
+               uint16_t __flags;
+       };
+} zend_jit_trace_stack;
+
+#define STACK_VAR(_stack, _slot) \
+       (_stack)[_slot].__ssa_var
+#define STACK_INFO(_stack, _slot) \
+       (_stack)[_slot].__info
+#define STACK_TYPE(_stack, _slot) \
+       (_stack)[_slot].__type
+#define STACK_REG(_stack, _slot) \
+       (_stack)[_slot].__reg
+#define SET_STACK_VAR(_stack, _slot, _ssa_var) do { \
+               (_stack)[_slot].__ssa_var = _ssa_var; \
+       } while (0)
+#define SET_STACK_INFO(_stack, _slot, _info) do { \
+               (_stack)[_slot].__info = _info; \
+       } while (0)
+#define SET_STACK_TYPE(_stack, _slot, _type) do { \
+               (_stack)[_slot].__type = _type; \
+               (_stack)[_slot].__reg = ZREG_NONE; \
+               (_stack)[_slot].__flags = 0; \
+       } while (0)
 
 typedef struct _zend_jit_trace_info {
        uint32_t                  id;            /* trace id */

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index f24ae1c2a131f3bb3abe7de70202f929a945f591..171ecd7d51b7f58866244fc00072c7b89294cd8e 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -134,7 +134,7 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const
        if (stack_size) {
                stack = JIT_G(current_frame)->stack;
                do {
-                       if (stack[stack_size-1] != IS_UNKNOWN) {
+                       if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN) {
                                break;
                        }
                        stack_size--;
@@ -320,10 +320,10 @@ static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type)
 }
 
 #define STACK_VAR_TYPE(_var) \
-       ((zend_uchar)stack[EX_VAR_TO_NUM(_var)])
+       STACK_TYPE(stack, EX_VAR_TO_NUM(_var))
 
 #define SET_STACK_VAR_TYPE(_var, _type) do { \
-               stack[EX_VAR_TO_NUM(_var)] = _type; \
+               SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), _type); \
        } while (0)
 
 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
@@ -575,13 +575,13 @@ static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ss
                                ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
                                sizeof(void*) * 2);
                        phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
-                       phi->sources[0] = stack[k];
+                       phi->sources[0] = STACK_VAR(stack, k);
                        phi->sources[1] = -1;
                        phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
                        phi->pi = -1;
                        phi->var = k;
                        phi->ssa_var = ssa_vars_count;
-                       stack[k] = ssa_vars_count;
+                       SET_STACK_VAR(stack, k, ssa_vars_count);
                        ssa_vars_count++;
                        phi->block = 1;
                        if (prev) {
@@ -957,7 +957,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
        }
        stack = frame->stack;
        for (i = 0; i < ssa_vars_count; i++) {
-               stack[i] = i;
+               SET_STACK_VAR(stack, i, i);
        }
 
        if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
@@ -973,14 +973,14 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                if (p->op == ZEND_JIT_TRACE_VM) {
                        opline = p->opline;
                        ssa_opcodes[idx] = opline;
-                       ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, stack);
+                       ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
                        idx++;
                        len = zend_jit_trace_op_len(p->opline);
                        while (len > 1) {
                                opline++;
                                ssa_opcodes[idx] = opline;
                                if (opline->opcode != ZEND_OP_DATA) {
-                                       ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, stack);
+                                       ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
                                }
                                idx++;
                                len--;
@@ -993,7 +993,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                        ZEND_ASSERT(ssa_vars_count < 0xff);
                        p->first_ssa_var = ssa_vars_count;
                        for (i = 0; i < op_array->last_var; i++) {
-                               stack[i] = ssa_vars_count++;
+                               SET_STACK_VAR(stack, i, ssa_vars_count++);
                        }
                } else if (p->op == ZEND_JIT_TRACE_BACK) {
                        op_array = p->op_array;
@@ -1003,7 +1003,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                                ZEND_ASSERT(ssa_vars_count <= 0xff);
                                p->first_ssa_var = ssa_vars_count;
                                for (i = 0; i < op_array->last_var + op_array->T; i++) {
-                                       stack[i] = ssa_vars_count++;
+                                       SET_STACK_VAR(stack, i, ssa_vars_count++);
                                }
                        } else {
                                level--;
@@ -1042,7 +1042,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                zend_ssa_phi *phi = tssa->blocks[1].phis;
 
                while (phi) {
-                       phi->sources[1] = stack[phi->var];
+                       phi->sources[1] = STACK_VAR(stack, phi->var);
                        ssa_vars[phi->ssa_var].var = phi->var;
                        ssa_vars[phi->ssa_var].definition_phi = phi;
                        ssa_vars[phi->sources[0]].phi_use_chain = phi;
@@ -1143,7 +1143,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                        }
                        if (i < parent_vars_count) {
                                /* Initialize TSSA variable from parent trace */
-                               zend_uchar op_type = parent_stack[i];
+                               zend_uchar op_type = STACK_TYPE(parent_stack, i);
 
                                if (op_type != IS_UNKNOWN) {
                                        ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
@@ -1171,7 +1171,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
        TRACE_FRAME_INIT(frame, op_array, 0, 0);
        TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
        for (i = 0; i < op_array->last_var + op_array->T; i++) {
-               frame->stack[i] = -1;
+               SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN);
        }
        memset(&return_value_info, 0, sizeof(return_value_info));
 
@@ -1334,7 +1334,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                                                if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
                                                        info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
                                                }
-                                               frame->call->stack[opline->op2.num - 1] = info;
+                                               SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
                                        }
                                        break;
                                case ZEND_RETURN:
@@ -1476,7 +1476,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                                TRACE_FRAME_INIT(call, op_array, 0, 0);
                                top = zend_jit_trace_call_frame(top, op_array);
                                for (i = 0; i < op_array->last_var + op_array->T; i++) {
-                                       call->stack[i] = -1;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
                                }
                        } else {
                                ZEND_ASSERT(&call->func->op_array == op_array);
@@ -1515,7 +1515,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                                if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
                                 && i < op_array->num_args) {
                                        /* Propagate argument type */
-                                       ssa_var_info[v].type &= frame->stack[i];
+                                       ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
                                }
                                i++;
                                v++;
@@ -1582,7 +1582,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                                TRACE_FRAME_INIT(frame, op_array, 0, 0);
                                TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
                                for (i = 0; i < op_array->last_var + op_array->T; i++) {
-                                       frame->stack[i] = -1;
+                                       SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN);
                                }
                        }
 
@@ -1594,7 +1594,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                        top = zend_jit_trace_call_frame(top, p->op_array);
                        if (p->func->type == ZEND_USER_FUNCTION) {
                                for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
-                                       call->stack[i] = -1;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
                                }
                        }
                } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
@@ -1724,11 +1724,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                while (i < op_array->last_var) {
                        if (!(ssa->var_info[i].type & MAY_BE_GUARD)
                         && has_concrete_type(ssa->var_info[i].type)) {
-                               stack[i] = concrete_type(ssa->var_info[i].type);
+                               SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[i].type));
                        } else if (i < op_array->num_args) {
-                               stack[i] = IS_UNKNOWN;
+                               SET_STACK_TYPE(stack, i, IS_UNKNOWN);
                        } else {
-                               stack[i] = IS_UNDEF;
+                               SET_STACK_TYPE(stack, i, IS_UNDEF);
                        }
                        i++;
                }
@@ -1749,12 +1749,12 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                while (i < op_array->last_var + op_array->T) {
                        if (!(ssa->var_info[i].type & MAY_BE_GUARD)
                         && has_concrete_type(ssa->var_info[i].type)) {
-                               stack[i] = concrete_type(ssa->var_info[i].type);
+                               SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[i].type));
                        } else if (i < parent_vars_count
-                        && (zend_uchar)parent_stack[i] != IS_UNKNOWN) {
-                               stack[i] = parent_stack[i];
+                        && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
+                               SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
                        } else {
-                               stack[i] = IS_UNKNOWN;
+                               SET_STACK_TYPE(stack, i, IS_UNKNOWN);
                        }
                        i++;
                }
@@ -1814,7 +1814,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                                        goto jit_failure;
                                                }
                                                ssa->var_info[i].type = info & ~MAY_BE_GUARD;
-                                               stack[i] = concrete_type(info);
+                                               SET_STACK_TYPE(stack, i, concrete_type(info));
                                        }
                                }
                        }
@@ -2449,7 +2449,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                                        for (j = 0 ; j < op_array->last_var; j++) {
                                                                // TODO: get info from trace ???
                                                                uint32_t info = zend_ssa_cv_info(opline, op_array, op_array_ssa, j);
-                                                               zend_uchar type = stack[j];
+                                                               zend_uchar type = STACK_TYPE(stack, j);
 
                                                                info = zend_jit_trace_type_to_info_ex(type, info);
                                                                if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
@@ -2949,18 +2949,18 @@ done:
                                        /* Initialize abstract stack using SSA */
                                        if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD)
                                         && has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) {
-                                               call->stack[i] = concrete_type(ssa->var_info[p->first_ssa_var + i].type);
+                                               SET_STACK_TYPE(call->stack, i, concrete_type(ssa->var_info[p->first_ssa_var + i].type));
                                        } else {
-                                               call->stack[i] = IS_UNKNOWN;
+                                               SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
                                        }
                                        i++;
                                }
                                while (i < p->op_array->last_var) {
-                                       call->stack[i] = IS_UNDEF;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNDEF);
                                        i++;
                                }
                                while (i < p->op_array->last_var + p->op_array->T) {
-                                       call->stack[i] = IS_UNKNOWN;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
                                        i++;
                                }
                        } else {
@@ -2994,9 +2994,9 @@ done:
                                        /* Initialize abstract stack using SSA */
                                        if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD)
                                         && has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) {
-                                               stack[i] = concrete_type(ssa->var_info[p->first_ssa_var + i].type);
+                                               SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[p->first_ssa_var + i].type));
                                        } else {
-                                               stack[i] = IS_UNKNOWN;
+                                               SET_STACK_TYPE(stack, i, IS_UNKNOWN);
                                        }
                                }
                        }
@@ -3024,15 +3024,15 @@ done:
                                i = 0;
                                while (i < p->op_array->num_args) {
                                        /* Types of arguments are going to be stored in abstract stack when proseccin SEV onstruction */
-                                       call->stack[i] = IS_UNKNOWN;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
                                        i++;
                                }
                                while (i < p->op_array->last_var) {
-                                       call->stack[i] = IS_UNDEF;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNDEF);
                                        i++;
                                }
                                while (i < p->op_array->last_var + p->op_array->T) {
-                                       call->stack[i] = IS_UNKNOWN;
+                                       SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
                                        i++;
                                }
                        }
@@ -3886,7 +3886,22 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf
        const zend_op *opline;
        zend_jit_trace_info *t = &zend_jit_traces[trace_num];
 
-       // TODO: Deoptimizatoion of VM stack state ???
+       /* Deoptimizatoion of VM stack state */
+       uint32_t i;
+       uint32_t stack_size = t->exit_info[exit_num].stack_size;
+       zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
+
+       for (i = 0; i < stack_size; i++) {
+               if (STACK_REG(stack, i) != ZREG_NONE) {
+                       if (STACK_TYPE(stack, i) == IS_LONG) {
+                               ZVAL_LONG(EX_VAR_NUM(i), regs->r[STACK_REG(stack, i)]);
+                       } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
+                               ZVAL_DOUBLE(EX_VAR_NUM(i), regs->xmm[STACK_REG(stack, i) - ZREG_XMM0]);
+                       } else {
+                               ZEND_ASSERT(0);
+                       }
+               }
+       }
 
        /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
        // TODO: We may remoive this, becaus of the same check in zend_jit_trace_hot_side() ???