FETCH_FIM_R/IS JIT improvement

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 39bf3578c14545d58c25dd3c276989c77ba3bc8a..475265bf7d42cb995d4f0bc419bfb29b6b1646d5 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -1014,7 +1014,7 @@ static void* dasm_labels[zend_lb_MAX];
 |.endmacro
 
 /* the same as above, but "src" may overlap with "tmp_reg1" */
-|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
+|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2, src_loaded
 ||     if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
 ||             if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
 ||                     if (Z_MODE(src_addr) == IS_REG) {
@@ -1024,7 +1024,9 @@ static void* dasm_labels[zend_lb_MAX];
 ||                     } else if (Z_MODE(dst_addr) == IS_REG) {
 |                              GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
 ||                     } else {
-|                              GET_ZVAL_LVAL tmp_reg2, src_addr
+||                             if (!src_loaded) {
+|                                      GET_ZVAL_LVAL tmp_reg2, src_addr
+||                             }
 |                              SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
 ||                     }
 ||             } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
@@ -1037,20 +1039,26 @@ static void* dasm_labels[zend_lb_MAX];
 |                              SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
 ||                     }
 ||             } else if (!(src_info & MAY_BE_DOUBLE)) {
-|                      GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+||                     if (!src_loaded) {
+|                              GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+||                     }
 |                      SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
 ||             } else {
 |                      .if X64
-|                              GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+||                             if (!src_loaded) {
+|                                      GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+||                             }
 |                              SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
 |                      .else
-||                             if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) {
+||                             if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) && !src_loaded) {
 |                                      GET_ZVAL_W2 Ra(tmp_reg2), src_addr
 |                                      SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
 |                                      GET_ZVAL_PTR Ra(tmp_reg2), src_addr
 |                                      SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
 ||                             } else {
-|                                      GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+||                                     if (!src_loaded) {
+|                                              GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+||                                     }
 |                                      GET_ZVAL_W2 Ra(tmp_reg1), src_addr
 |                                      SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
 |                                      SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
@@ -3545,7 +3553,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
                |       IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
        }
        if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
-               |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+               |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1, 0
        }
        if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, MAY_BE_LONG)) {
                return 0;
@@ -3595,7 +3603,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
 
                if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
                    opline->result_type != IS_UNUSED) {
-                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1, 0
                }
 
                SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
@@ -3606,7 +3614,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
                |       jo >1
                if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
                    opline->result_type != IS_UNUSED) {
-                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1, 0
                }
                |.cold_code
                |1:
@@ -3632,14 +3640,14 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
                }
                if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
                    opline->result_type != IS_UNUSED) {
-                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1
+                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1, 0
                }
                |       jmp >3
                |.code
        } else {
                if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
                    opline->result_type != IS_UNUSED) {
-                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+                       |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1, 0
                }
        }
        if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
@@ -3701,7 +3709,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
                        if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
                                zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
 
-                               |       ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2
+                               |       ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2, 0
                                |       TRY_ADDREF op1_info, ah, r2
                        }
                        if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
@@ -3726,7 +3734,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
                        zend_reg tmp_reg;
 
                        if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
-                               |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2
+                               |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2, 0
                        }
                        if (Z_MODE(op1_def_addr) == IS_REG) {
                                tmp_reg = Z_REG(op1_def_addr);
@@ -3752,7 +3760,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
                        |       SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg
                        if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
                            opline->result_type != IS_UNUSED) {
-                               |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1
+                               |       ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1, 0
                                |       TRY_ADDREF op1_def_info, ah, r1
                        }
                }
@@ -5178,7 +5186,7 @@ static int zend_jit_simple_assign(dasm_State    **Dst,
                                |       // ZVAL_COPY_VALUE(return_value, &ref->value);
                                ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8);
                                if (!res_addr) {
-                                       |       ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, ZREG_R0
+                                       |       ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, ZREG_R0, 0
                                } else {
                                        |       ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, ZREG_R0
                                }
@@ -5207,7 +5215,7 @@ static int zend_jit_simple_assign(dasm_State    **Dst,
                }
 
                if (!res_addr) {
-                       |       ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, ZREG_R0
+                       |       ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, ZREG_R0, 0
                } else {
                        |       ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, ZREG_R0
                }
@@ -8874,7 +8882,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend
                        |       ADDREF_CONST zv, r0
                }
        } else {
-               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
+               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2, 0
        }
 
        return 1;
@@ -8943,11 +8951,11 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend
                        zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
 
                        |       mov r1, aword T1 // restore
-                       |       ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2
+                       |       ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2, 0
                        |       SET_ZVAL_PTR val_addr, r0
                        |       SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX
                } else {
-                       |       ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
+                       |       ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2, 0
                        |       SET_ZVAL_PTR op1_addr, r0
                        |       SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
                }
@@ -9011,7 +9019,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
                 && JIT_G(current_frame)->call->func) {
                        if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
 
-                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
+                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2, 0
 
                                if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
                                        if (!(op1_info & MAY_BE_REF)) {
@@ -9040,7 +9048,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
 
                        mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
 
-                       |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
+                       |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2, 0
                        if (op1_info & MAY_BE_REF) {
                                |       cmp cl, IS_REFERENCE
                                |       je >7
@@ -9112,7 +9120,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
        }
 
        if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
-               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
+               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2, 0
                if (op1_info & MAY_BE_REF) {
                        |       cmp cl, IS_REFERENCE
                        |       je >7
@@ -9139,7 +9147,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
 
                                |       LOAD_ZVAL_ADDR FCARG1a, op1_addr
                                |       ZVAL_DEREF FCARG1a, op1_info
-                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2
+                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2, 0
                                |       TRY_ADDREF op1_info, ah, r2
                        } else {
                                zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8);
@@ -9150,7 +9158,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
                                |       // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
                                |       GET_ZVAL_PTR FCARG1a, op1_addr
                                |       // ZVAL_COPY_VALUE(return_value, &ref->value);
-                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2
+                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2, 0
                                |       GC_DELREF FCARG1a
                                |       je >1
                                |       IF_NOT_REFCOUNTED ah, >2
@@ -9160,7 +9168,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
                                |       EFREE_REG_24 op_array, opline
                                |       jmp >2
                                |.code
-                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
+                               |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2, 0
                                |2:
                        }
                } else {
@@ -9172,7 +9180,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
                                        op1_addr= op1_def_addr;
                                }
                        }
-                       |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
+                       |       ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2, 0
                        if (opline->op1_type == IS_CV) {
                                |       TRY_ADDREF op1_info, ah, r2
                        }
@@ -9945,14 +9953,14 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o
                        |       ADDREF_CONST zv, r0
                }
        } else if (opline->op1_type == IS_TMP_VAR) {
-               |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
+               |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2, 0
        } else if (opline->op1_type == IS_CV) {
                if (op1_info & MAY_BE_REF) {
                        |       LOAD_ZVAL_ADDR r0, op1_addr
                        |       ZVAL_DEREF r0, op1_info
                        op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
                }
-               |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
+               |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2, 0
                if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op1_info & MAY_BE_REF) || (return_value_used != 1)) {
                        |       // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ???
                        |       TRY_ADDREF op1_info, ah, r2
@@ -9967,7 +9975,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o
                        |       // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
                        |       GET_ZVAL_PTR r0, op1_addr
                        |       // ZVAL_COPY_VALUE(return_value, &ref->value);
-                       |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2
+                       |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2, 0
                        |       GC_DELREF r0
                        |       je >2
                        |       // if (IS_REFCOUNTED())
@@ -9993,7 +10001,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o
                        }
                        |.code
                }
-               |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
+               |       ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2, 0
        }
 
        |9:
@@ -10168,19 +10176,19 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, cons
                |8:
                if (op1_info & MAY_BE_ARRAY_OF_REF) {
                        |       // ZVAL_COPY_DEREF
+                       |       GET_Z_PTR r2, r0
                        |       IF_NOT_ZVAL_REFCOUNTED val_addr, >2
                        |       IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1
-                       |       GET_Z_PTR r0, r0
-                       |       add r0, offsetof(zend_reference, val)
+                       |       lea r0, [r2 + offsetof(zend_reference, val)]
+                       |       GET_Z_PTR r2, r0
                        |       IF_NOT_ZVAL_REFCOUNTED val_addr, >2
                        |1:
-                       |       GET_Z_PTR r1, r0
-                       |       GC_ADDREF r1
+                       |       GC_ADDREF r2
                        |2:
-                       |       ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2
+                       |       ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2, 1
                } else  {
                        |       // ZVAL_COPY
-                       |       ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2
+                       |       ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2, 0
                        |       TRY_ADDREF res_info, ch, r2
                }
        }