--- /dev/null
- case ZEND_DECLARE_ANON_INHERITED_CLASS:
+/*
+ +----------------------------------------------------------------------+
+ | Zend JIT |
+ +----------------------------------------------------------------------+
+ | Copyright (c) The PHP Group |
+ +----------------------------------------------------------------------+
+ | This source file is subject to version 3.01 of the PHP license, |
+ | that is bundled with this package in the file LICENSE, and is |
+ | available through the world-wide-web at the following url: |
+ | http://www.php.net/license/3_01.txt |
+ | If you did not receive a copy of the PHP license and are unable to |
+ | obtain it through the world-wide-web, please send a note to |
+ | license@php.net so we can mail you a copy immediately. |
+ +----------------------------------------------------------------------+
+ | Authors: Dmitry Stogov <dmitry@php.net> |
+ +----------------------------------------------------------------------+
+*/
+
+#include <ZendAccelerator.h>
+#include "zend_shared_alloc.h"
+#include "Zend/zend_execute.h"
+#include "Zend/zend_vm.h"
+#include "Zend/zend_exceptions.h"
+#include "Zend/zend_constants.h"
+#include "zend_smart_str.h"
+#include "jit/zend_jit.h"
+#include "jit/zend_jit_internal.h"
+
+#ifdef HAVE_JIT
+
+#include "Optimizer/zend_func_info.h"
+#include "Optimizer/zend_ssa.h"
+#include "Optimizer/zend_inference.h"
+#include "Optimizer/zend_call_graph.h"
+#include "Optimizer/zend_dump.h"
+
+//#define CONTEXT_THREADED_JIT
+#define ZEND_JIT_USE_RC_INFERENCE
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+# define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
+# define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
+# define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
+#else
+# define ZEND_SSA_RC_INFERENCE_FLAG 0
+# define RC_MAY_BE_1(info) 1
+# define RC_MAY_BE_N(info) 1
+#endif
+
+#define JIT_PREFIX "JIT$"
+#define JIT_STUB_PREFIX "JIT$$"
+
+#define DASM_M_GROW(ctx, t, p, sz, need) \
+ do { \
+ size_t _sz = (sz), _need = (need); \
+ if (_sz < _need) { \
+ if (_sz < 16) _sz = 16; \
+ while (_sz < _need) _sz += _sz; \
+ (p) = (t *)erealloc((p), _sz); \
+ (sz) = _sz; \
+ } \
+ } while(0)
+
+#define DASM_M_FREE(ctx, p, sz) efree(p)
+
+#include "dynasm/dasm_proto.h"
+
+typedef struct _zend_jit_stub {
+ const char *name;
+ int (*stub)(dasm_State **Dst);
+} zend_jit_stub;
+
+#define JIT_STUB(name) \
+ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub}
+
+static zend_uchar zend_jit_level = 0;
+static zend_uchar zend_jit_trigger = 0;
+static zend_uchar zend_jit_reg_alloc = 0;
+static zend_uchar zend_jit_cpu_flags = 0;
+
+zend_ulong zend_jit_profile_counter = 0;
+int zend_jit_profile_counter_rid = -1;
+
+int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
+
+const zend_op *zend_jit_halt_op = NULL;
+static int zend_jit_vm_kind = 0;
+
+static void *dasm_buf = NULL;
+static void *dasm_end = NULL;
+static void **dasm_ptr = NULL;
+
+static size_t dasm_size = 0;
+
+static const void *zend_jit_runtime_jit_handler = NULL;
+static const void *zend_jit_profile_jit_handler = NULL;
+static const void *zend_jit_func_counter_handler = NULL;
+static const void *zend_jit_loop_counter_handler = NULL;
+
+static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa);
+static void ZEND_FASTCALL zend_runtime_jit(void);
+
+static zend_bool zend_ssa_is_last_use(zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
+{
+ if (ssa->vars[var].phi_use_chain) {
+ zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
+ do {
+ if (!ssa->vars[phi->ssa_var].no_val) {
+ return 0;
+ }
+ phi = zend_ssa_next_use_phi(ssa, var, phi);
+ } while (phi);
+ }
+
+ use = zend_ssa_next_use(ssa->ops, var, use);
+ return use < 0 || zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var);
+}
+
+static zend_bool zend_is_commutative(zend_uchar opcode)
+{
+ return
+ opcode == ZEND_ADD ||
+ opcode == ZEND_MUL ||
+ opcode == ZEND_BW_OR ||
+ opcode == ZEND_BW_AND ||
+ opcode == ZEND_BW_XOR ||
+ opcode == ZEND_ASSIGN_ADD ||
+ opcode == ZEND_ASSIGN_MUL||
+ opcode == ZEND_ASSIGN_BW_OR ||
+ opcode == ZEND_ASSIGN_BW_AND ||
+ opcode == ZEND_ASSIGN_BW_XOR;
+}
+
+static zend_bool zend_long_is_power_of_two(zend_long x)
+{
+ return (x > 0) && !(x & (x - 1));
+}
+
+#include "dynasm/dasm_x86.h"
+#include "jit/zend_jit_x86.h"
+#include "jit/zend_jit_helpers.c"
+#include "jit/zend_jit_x86.c"
+#include "jit/zend_jit_disasm_x86.c"
+#ifndef _WIN32
+#include "jit/zend_jit_gdb.c"
+#include "jit/zend_jit_perf_dump.c"
+#endif
+#ifdef HAVE_OPROFILE
+# include "jit/zend_jit_oprofile.c"
+#endif
+#include "jit/zend_jit_vtune.c"
+
+#if _WIN32
+# include <Windows.h>
+#else
+# include <sys/mman.h>
+# if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
+# define MAP_ANONYMOUS MAP_ANON
+# endif
+#endif
+
+#define DASM_ALIGNMENT 16
+
+ZEND_EXT_API void zend_jit_status(zval *ret)
+{
+ zval stats;
+ array_init(&stats);
+ add_assoc_long(&stats, "level", zend_jit_level);
+ add_assoc_long(&stats, "trigger", zend_jit_trigger);
+ add_assoc_long(&stats, "reg-alloc", zend_jit_reg_alloc);
+ if (dasm_buf) {
+ add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
+ add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
+ } else {
+ add_assoc_long(&stats, "buffer_size", 0);
+ add_assoc_long(&stats, "buffer_free", 0);
+ }
+ add_assoc_zval(ret, "jit", &stats);
+}
+
+static zend_string *zend_jit_func_name(zend_op_array *op_array)
+{
+ smart_str buf = {0};
+
+ if (op_array->function_name) {
+ if (op_array->scope) {
+ smart_str_appends(&buf, JIT_PREFIX);
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
+ smart_str_appends(&buf, "::");
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
+ smart_str_0(&buf);
+ return buf.s;
+ } else {
+ smart_str_appends(&buf, JIT_PREFIX);
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
+ smart_str_0(&buf);
+ return buf.s;
+ }
+ } else if (op_array->filename) {
+ smart_str_appends(&buf, JIT_PREFIX);
+ smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
+ smart_str_0(&buf);
+ return buf.s;
+ } else {
+ return NULL;
+ }
+}
+
+static void *dasm_link_and_encode(dasm_State **dasm_state,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_op *rt_opline,
+ zend_lifetime_interval **ra,
+ const char *name)
+{
+ size_t size;
+ int ret;
+ void *entry;
+#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
+ zend_string *str = NULL;
+#endif
+
+ if (rt_opline && ssa && ssa->cfg.map) {
+ /* Create additional entry point, to switch from interpreter to JIT-ed
+ * code at run-time.
+ */
+ int b = ssa->cfg.map[rt_opline - op_array->opcodes];
+
+//#ifdef CONTEXT_THREADED_JIT
+// if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
+//#else
+ if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
+//#endif
+ zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
+ zend_jit_prologue(dasm_state);
+ if (ra) {
+ int i;
+ zend_lifetime_interval *ival;
+ zend_life_range *range;
+ uint32_t pos = rt_opline - op_array->opcodes;
+
+ for (i = 0; i < ssa->vars_count; i++) {
+ ival = ra[i];
+
+ if (ival && ival->reg != ZREG_NONE) {
+ range = &ival->range;
+
+ if (pos >= range->start && pos <= range->end) {
+ if (!zend_jit_load_ssa_var(dasm_state, ssa, i, ival->reg)) {
+ return NULL;
+ }
+ break;
+ }
+ range = range->next;
+ }
+ }
+ }
+ zend_jit_jmp(dasm_state, b);
+ }
+ }
+
+ if (dasm_link(dasm_state, &size) != DASM_S_OK) {
+ // TODO: dasm_link() failed ???
+ return NULL;
+ }
+
+ if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
+ *dasm_ptr = dasm_end; //prevent further try
+ // TODO: jit_buffer_size overflow ???
+ return NULL;
+ }
+
+ ret = dasm_encode(dasm_state, *dasm_ptr);
+
+ if (ret != DASM_S_OK) {
+ // TODO: dasm_encode() failed ???
+ return NULL;
+ }
+
+ entry = *dasm_ptr;
+ *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
+
+ if (op_array && ssa) {
+ int b;
+
+ for (b = 0; b < ssa->cfg.blocks_count; b++) {
+//#ifdef CONTEXT_THREADED_JIT
+// if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
+//#else
+ if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
+//#endif
+ zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
+ int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
+
+ if (offset >= 0) {
+ opline->handler = (void*)(((char*)entry) + offset);
+ }
+ }
+ }
+ if (rt_opline && ssa && ssa->cfg.map) {
+ int b = ssa->cfg.map[rt_opline - op_array->opcodes];
+ zend_op *opline = (zend_op*)rt_opline;
+ int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
+
+ if (offset >= 0) {
+ opline->handler = (void*)(((char*)entry) + offset);
+ }
+ }
+ }
+
+#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
+ if (!name) {
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
+ str = zend_jit_func_name(op_array);
+ if (str) {
+ name = ZSTR_VAL(str);
+ }
+ }
+#ifdef HAVE_DISASM
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) {
+ zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
+ zend_jit_disasm(
+ name,
+ (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
+ op_array,
+ &ssa->cfg,
+ entry,
+ size);
+ }
+ } else {
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
+ zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) {
+ zend_jit_disasm(
+ name,
+ (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
+ op_array,
+ &ssa->cfg,
+ entry,
+ size);
+ }
+ }
+# endif
+ }
+#endif
+
+#ifdef HAVE_GDB
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) {
+ if (name) {
+ zend_jit_gdb_register(
+ name,
+ op_array,
+ entry,
+ size);
+ }
+ }
+#endif
+
+#ifdef HAVE_OPROFILE
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) {
+ zend_jit_oprofile_register(
+ name,
+ entry,
+ size);
+ }
+#endif
+
+#ifdef HAVE_PERFTOOLS
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
+ if (name) {
+ zend_jit_perf_map_register(
+ name,
+ entry,
+ size);
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) {
+ zend_jit_perf_jitdump_register(
+ name,
+ entry,
+ size);
+ }
+ }
+ }
+#endif
+
+#ifdef HAVE_VTUNE
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_VTUNE) {
+ if (name) {
+ zend_jit_vtune_register(
+ name,
+ entry,
+ size);
+ }
+ }
+#endif
+
+#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
+ if (str) {
+ zend_string_release(str);
+ }
+#endif
+
+ return entry;
+}
+
+static int zend_may_overflow(const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t num;
+ int res;
+
+ if (!ssa->ops || !ssa->var_info) {
+ return 1;
+ }
+ switch (opline->opcode) {
+ case ZEND_PRE_INC:
+ case ZEND_POST_INC:
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].op1_def;
+ return (res < 0 ||
+ !ssa->var_info[res].has_range ||
+ ssa->var_info[res].range.overflow);
+ case ZEND_PRE_DEC:
+ case ZEND_POST_DEC:
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].op1_def;
+ return (res < 0 ||
+ !ssa->var_info[res].has_range ||
+ ssa->var_info[res].range.underflow);
+ case ZEND_ADD:
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].result_def;
+ if (res < 0 ||
+ !ssa->var_info[res].has_range) {
+ return 1;
+ }
+ if (ssa->var_info[res].range.underflow) {
+ zend_long op1_min, op2_min, res_min;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_min = OP1_MIN_RANGE();
+ op2_min = OP2_MIN_RANGE();
+ res_min = op1_min + op2_min;
+ if (op1_min < 0 && op2_min < 0 && res_min >= 0) {
+ return 1;
+ }
+ }
+ if (ssa->var_info[res].range.overflow) {
+ zend_long op1_max, op2_max, res_max;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_max = OP1_MAX_RANGE();
+ op2_max = OP2_MAX_RANGE();
+ res_max = op1_max + op2_max;
+ if (op1_max > 0 && op2_max > 0 && res_max <= 0) {
+ return 1;
+ }
+ }
+ return 0;
+ case ZEND_SUB:
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].result_def;
+ if (res < 0 ||
+ !ssa->var_info[res].has_range) {
+ return 1;
+ }
+ if (ssa->var_info[res].range.underflow) {
+ zend_long op1_min, op2_max, res_min;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_min = OP1_MIN_RANGE();
+ op2_max = OP2_MAX_RANGE();
+ res_min = op1_min - op2_max;
+ if (op1_min < 0 && op2_max > 0 && res_min >= 0) {
+ return 1;
+ }
+ }
+ if (ssa->var_info[res].range.overflow) {
+ zend_long op1_max, op2_min, res_max;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_max = OP1_MAX_RANGE();
+ op2_min = OP2_MIN_RANGE();
+ res_max = op1_max - op2_min;
+ if (op1_max > 0 && op2_min < 0 && res_max <= 0) {
+ return 1;
+ }
+ }
+ return 0;
+ case ZEND_MUL:
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].result_def;
+ return (res < 0 ||
+ !ssa->var_info[res].has_range ||
+ ssa->var_info[res].range.underflow ||
+ ssa->var_info[res].range.overflow);
+ case ZEND_ASSIGN_ADD:
+ if (opline->extended_value != 0) {
+ return 1;
+ }
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].op1_def;
+ if (res < 0 ||
+ !ssa->var_info[res].has_range) {
+ return 1;
+ }
+ if (ssa->var_info[res].range.underflow) {
+ zend_long op1_min, op2_min, res_min;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_min = OP1_MIN_RANGE();
+ op2_min = OP2_MIN_RANGE();
+ res_min = op1_min + op2_min;
+ if (op1_min < 0 && op2_min < 0 && res_min >= 0) {
+ return 1;
+ }
+ }
+ if (ssa->var_info[res].range.overflow) {
+ zend_long op1_max, op2_max, res_max;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_max = OP1_MAX_RANGE();
+ op2_max = OP2_MAX_RANGE();
+ res_max = op1_max + op2_max;
+ if (op1_max > 0 && op2_max > 0 && res_max <= 0) {
+ return 1;
+ }
+ }
+ return 0;
+ case ZEND_ASSIGN_SUB:
+ if (opline->extended_value != 0) {
+ return 1;
+ }
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].op1_def;
+ if (res < 0 ||
+ !ssa->var_info[res].has_range) {
+ return 1;
+ }
+ if (ssa->var_info[res].range.underflow) {
+ zend_long op1_min, op2_max, res_min;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_min = OP1_MIN_RANGE();
+ op2_max = OP2_MAX_RANGE();
+ res_min = op1_min - op2_max;
+ if (op1_min < 0 && op2_max > 0 && res_min >= 0) {
+ return 1;
+ }
+ }
+ if (ssa->var_info[res].range.overflow) {
+ zend_long op1_max, op2_min, res_max;
+
+ if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
+ return 1;
+ }
+ op1_max = OP1_MAX_RANGE();
+ op2_min = OP2_MIN_RANGE();
+ res_max = op1_max - op2_min;
+ if (op1_max > 0 && op2_min < 0 && res_max <= 0) {
+ return 1;
+ }
+ }
+ return 0;
+ case ZEND_ASSIGN_MUL:
+ if (opline->extended_value != 0) {
+ return 1;
+ }
+ num = opline - op_array->opcodes;
+ res = ssa->ops[num].op1_def;
+ return (res < 0 ||
+ !ssa->var_info[res].has_range ||
+ ssa->var_info[res].range.underflow ||
+ ssa->var_info[res].range.overflow);
+ default:
+ return 1;
+ }
+}
+
+static int zend_jit_build_cfg(zend_op_array *op_array, zend_cfg *cfg)
+{
+ uint32_t flags;
+
+ flags = ZEND_CFG_STACKLESS | ZEND_RT_CONSTANTS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
+
+ if (zend_build_cfg(&CG(arena), op_array, flags, cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+ /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
+ * generated code, some of our analysis is recursive and will stack overflow with many
+ * blocks. */
+ if (cfg->blocks_count > 100000) {
+ return FAILURE;
+ }
+
+ if (zend_cfg_build_predecessors(&CG(arena), cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+ /* Compute Dominators Tree */
+ if (zend_cfg_compute_dominators_tree(op_array, cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+ /* Identify reducible and irreducible loops */
+ if (zend_cfg_identify_loops(op_array, cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+ return SUCCESS;
+}
+
+static int zend_jit_op_array_analyze1(zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
+{
+ if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+#if 0
+ /* TODO: debugger and profiler supports? */
+ if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
+ return FAILURE;
+ }
+#endif
+
+ if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC)
+ && ssa->cfg.blocks
+ && op_array->last_try_catch == 0
+ && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
+ && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
+ if (zend_build_ssa(&CG(arena), script, op_array, ZEND_RT_CONSTANTS | ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
+ return FAILURE;
+ }
+
+ if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa) != SUCCESS) {
+ return FAILURE;
+ }
+
+ if (zend_ssa_find_false_dependencies(op_array, ssa) != SUCCESS) {
+ return FAILURE;
+ }
+
+ if (zend_ssa_find_sccs(op_array, ssa) != SUCCESS){
+ return FAILURE;
+ }
+ } else {
+ ssa->rt_constants = 1;
+ }
+
+ return SUCCESS;
+}
+
+static int zend_jit_op_array_analyze2(zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
+{
+ if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC)
+ && ssa->cfg.blocks
+ && op_array->last_try_catch == 0
+ && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
+ && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
+
+ uint32_t optimization_level = ZCG(accel_directives).optimization_level;
+ if (zend_ssa_inference(&CG(arena), op_array, script, ssa, optimization_level) != SUCCESS) {
+ return FAILURE;
+ }
+ }
+
+ return SUCCESS;
+}
+
+static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
+{
+ zend_lifetime_interval *ival = intervals[var];
+
+ if (!ival) {
+ ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
+ if (!ival) {
+ return FAILURE;
+ }
+ ival->ssa_var = var;
+ ival->reg = ZREG_NONE;
+ ival->split = 0;
+ ival->store = 0;
+ ival->load = 0;
+ ival->range.start = from;
+ ival->range.end = to;
+ ival->range.next = NULL;
+ ival->hint = NULL;
+ ival->used_as_hint = NULL;
+ intervals[var] = ival;
+ } else if (ival->range.start > to + 1) {
+ zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
+
+ if (!range) {
+ return FAILURE;
+ }
+ range->start = ival->range.start;
+ range->end = ival->range.end;
+ range->next = ival->range.next;
+ ival->range.start = from;
+ ival->range.end = to;
+ ival->range.next = range;
+ } else if (ival->range.start == to + 1) {
+ ival->range.start = from;
+ } else {
+ zend_life_range *range = &ival->range;
+ zend_life_range *last = NULL;
+
+ do {
+ if (range->start > to + 1) {
+ break;
+ } else if (range->end + 1 >= from) {
+ if (range->start > from) {
+ range->start = from;
+ }
+ last = range;
+ range = range->next;
+ while (range) {
+ if (range->start > to + 1) {
+ break;
+ }
+ last->end = range->end;
+ range = range->next;
+ last->next = range;
+ }
+ if (to > last->end) {
+ last->end = to;
+ }
+ return SUCCESS;
+ }
+ last = range;
+ range = range->next;
+ } while (range);
+
+ range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
+ if (!range) {
+ return FAILURE;
+ }
+ range->start = from;
+ range->end = to;
+ range->next = last->next;
+ last->next = range;
+ }
+
+ return SUCCESS;
+}
+
+static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
+{
+ if (block_start != from && intervals[var]) {
+ zend_life_range *range = &intervals[var]->range;
+
+ do {
+ if (from >= range->start && from <= range->end) {
+ if (range->start == block_start) {
+ range->start = from;
+ } else {
+ zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
+ if (!r) {
+ return FAILURE;
+ }
+ r->start = from;
+ r->end = range->end;
+ r->next = range->next;
+ range->end = block_start - 1;
+ range->next = r;
+ }
+ return SUCCESS;
+ }
+ range = range->next;
+ } while (range);
+ }
+
+ // dead store
+ return zend_jit_add_range(intervals, var, from, from);
+}
+
+static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
+{
+ while (1) {
+ if (*list == NULL) {
+ *list = ival;
+ ival->list_next = NULL;
+ return;
+ } else if (ival->range.start < (*list)->range.start) {
+ ival->list_next = *list;
+ *list = ival;
+ return;
+ }
+ list = &(*list)->list_next;
+ }
+}
+
+static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
+{
+ zend_lifetime_interval *ival;
+ zend_life_range *range = ¤t->range;
+ zend_life_range *prev = NULL;
+
+ if (*free) {
+ ival = *free;
+ *free = ival->list_next;
+ } else {
+ ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
+
+ if (!ival) {
+ return FAILURE;
+ }
+ }
+
+ current->store = 1;
+
+ ival->ssa_var = current->ssa_var;
+ ival->reg = ZREG_NONE;
+ ival->split = 1;
+ ival->store = 0;
+ ival->load = 1;
+ ival->hint = NULL;
+
+ do {
+ if (pos >= range->start && pos <= range->end) {
+ break;
+ }
+ prev = range;
+ range = range->next;
+ } while(range);
+
+ ZEND_ASSERT(range != NULL);
+
+ ival->range.start = pos;
+ ival->range.end = range->end;
+ ival->range.next = range->next;
+
+ if (pos == range->start) {
+ ZEND_ASSERT(prev != NULL);
+ prev->next = NULL;
+ } else {
+ range->end = pos - 1;
+ }
+
+ zend_jit_insert_interval(list, ival);
+
+ return SUCCESS;
+}
+
+static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
+{
+ zend_lifetime_interval *list, *last;
+ int i;
+
+ list = NULL;
+ i = 0;
+ while (i < count) {
+ list = intervals[i];
+ i++;
+ if (list) {
+ last = list;
+ last->list_next = NULL;
+ break;
+ }
+ }
+
+ while (i < count) {
+ zend_lifetime_interval *ival = intervals[i];
+
+ i++;
+ if (ival) {
+ if ((ival->range.start > last->range.start) ||
+ (ival->range.start == last->range.start &&
+ ival->range.end > last->range.end)) {
+ last->list_next = ival;
+ last = ival;
+ ival->list_next = NULL;
+ } else {
+ zend_lifetime_interval **p = &list;
+
+ while (1) {
+ if (*p == NULL) {
+ *p = last = ival;
+ ival->list_next = NULL;
+ break;
+ } else if ((ival->range.start < (*p)->range.start) ||
+ (ival->range.start == (*p)->range.start &&
+ ival->range.end < (*p)->range.end)) {
+ ival->list_next = *p;
+ *p = ival;
+ break;
+ }
+ p = &(*p)->list_next;
+ }
+ }
+ }
+ }
+
+ return list;
+}
+
+static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
+{
+ zend_reg reg;
+ int first = 1;
+
+ ZEND_REGSET_FOREACH(regset, reg) {
+ if (first) {
+ first = 0;
+ fprintf(stderr, "%s", zend_reg_name[reg]);
+ } else {
+ fprintf(stderr, ", %s", zend_reg_name[reg]);
+ }
+ } ZEND_REGSET_FOREACH_END();
+}
+
+static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
+{
+ zend_basic_block *b = ssa->cfg.blocks + n;
+
+tail_call:
+ *block_order = n;
+ block_order++;
+
+ n = b->children;
+ while (n >= 0) {
+ b = ssa->cfg.blocks + n;
+ if (b->next_child < 0) {
+ goto tail_call;
+ }
+ block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
+ n = b->next_child;
+ }
+
+ return block_order;
+}
+
+static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
+{
+ int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
+
+ return end - block_order;
+}
+
+static zend_bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
+{
+ while (b->loop_header >= 0) {
+ if (b->loop_header == header) {
+ return 1;
+ }
+ b = ssa->cfg.blocks + b->loop_header;
+ }
+ return 0;
+}
+
+static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
+{
+ zend_basic_block *b = ssa->cfg.blocks + n;
+ uint32_t i;
+
+tail_call:
+ if (b->len) {
+ for (i = b->start; i < b->start + b->len; i++) {
+ zend_bitset_incl(loop_body, i);
+ }
+ }
+
+ n = b->children;
+ while (n >= 0) {
+ b = ssa->cfg.blocks + n;
+ if (zend_jit_in_loop(ssa, header, b)) {
+ if (b->next_child < 0) {
+ goto tail_call;
+ }
+ zend_jit_compute_loop_body(ssa, header, n, loop_body);
+ }
+ n = b->next_child;
+ }
+}
+
+static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
+{
+ if (intervals[dst]->range.start < intervals[src]->range.start) {
+ int tmp = src;
+ src = dst;
+ dst = tmp;
+ }
+ while (1) {
+ if (intervals[dst]->hint) {
+ if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
+ int tmp = src;
+ src = intervals[dst]->hint->ssa_var;
+ dst = tmp;
+ } else {
+ dst = intervals[dst]->hint->ssa_var;
+ }
+ } else {
+ if (dst != src) {
+ intervals[dst]->hint = intervals[src];
+ }
+ return;
+ }
+ }
+}
+
+/* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
+ Michael Franz, CGO'10 (2010), Figure 4. */
+static int zend_jit_compute_liveness(zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
+{
+ int set_size, i, j, k, l;
+ uint32_t n;
+ zend_bitset live, live_in, pi_vars, loop_body;
+ int *block_order;
+ zend_ssa_phi *phi;
+ zend_lifetime_interval **intervals;
+ size_t mem_size;
+ ALLOCA_FLAG(use_heap);
+
+ set_size = zend_bitset_len(ssa->vars_count);
+ mem_size =
+ ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
+ ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
+ ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
+ ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
+ ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
+ ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
+ intervals = do_alloca(mem_size, use_heap);
+ if (!intervals) {
+ *list = NULL;
+ return FAILURE;
+ }
+
+ live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
+ live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
+ pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
+ loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
+ block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
+
+ memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
+ zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
+
+ /* TODO: Provide a linear block order where all dominators of a block
+ * are before this block, and where all blocks belonging to the same loop
+ * are contiguous ???
+ */
+ for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
+ zend_basic_block *b;
+
+ i = block_order[l];
+ b = ssa->cfg.blocks + i;
+
+ /* live = UNION of successor.liveIn for each successor of b */
+ /* live.add(phi.inputOf(b)) for each phi of successors of b */
+ zend_bitset_clear(live, set_size);
+ for (j = 0; j < b->successors_count; j++) {
+ int succ = b->successors[j];
+
+ zend_bitset_union(live, live_in + set_size * succ, set_size);
+ zend_bitset_clear(pi_vars, set_size);
+ for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
+ if (ssa->vars[phi->ssa_var].no_val) {
+ /* skip */
+ } else if (phi->pi >= 0) {
+ if (phi->pi == i && phi->sources[0] >= 0) {
+ if (zend_bitset_in(candidates, phi->sources[0])) {
+ zend_bitset_incl(live, phi->sources[0]);
+ }
+ zend_bitset_incl(pi_vars, phi->var);
+ }
+ } else if (!zend_bitset_in(pi_vars, phi->var)) {
+ for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
+ if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
+ if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
+ zend_bitset_incl(live, phi->sources[k]);
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ /* addRange(var, b.from, b.to) for each var in live */
+ ZEND_BITSET_FOREACH(live, set_size, j) {
+ if (zend_bitset_in(candidates, j)) {
+ if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
+ goto failure;
+ }
+ }
+ } ZEND_BITSET_FOREACH_END();
+
+ /* for each operation op of b in reverse order */
+ for (n = b->start + b->len; n > b->start;) {
+ zend_ssa_op *op;
+ const zend_op *opline;
+ uint32_t num;
+
+ n--;
+ op = ssa->ops + n;
+ opline = op_array->opcodes + n;
+
+ if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
+ num = n - 1;
+ } else {
+ num = n;
+ }
+
+ /* for each output operand opd of op do */
+ /* setFrom(opd, op) */
+ /* live.remove(opd) */
+ if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
+ if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
+ goto failure;
+ }
+ zend_bitset_excl(live, op->op1_def);
+ }
+ if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
+ if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
+ goto failure;
+ }
+ zend_bitset_excl(live, op->op2_def);
+ }
+ if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
+ if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
+ goto failure;
+ }
+ zend_bitset_excl(live, op->result_def);
+ }
+
+ /* for each input operand opd of op do */
+ /* live.add(opd) */
+ /* addRange(opd, b.from, op) */
+ if (op->op1_use >= 0
+ && zend_bitset_in(candidates, op->op1_use)
+ && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
+ zend_bitset_incl(live, op->op1_use);
+ if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
+ goto failure;
+ }
+ }
+ if (op->op2_use >= 0
+ && zend_bitset_in(candidates, op->op2_use)
+ && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
+ zend_bitset_incl(live, op->op2_use);
+ if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
+ goto failure;
+ }
+ }
+ if (op->result_use >= 0
+ && zend_bitset_in(candidates, op->result_use)
+ && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
+ zend_bitset_incl(live, op->result_use);
+ if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
+ goto failure;
+ }
+ }
+ }
+
+ /* live.remove(phi.output) for each phi of b */
+ for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
+ zend_bitset_excl(live, phi->ssa_var);
+ }
+
+ /* b.liveIn = live */
+ zend_bitset_copy(live_in + set_size * i, live, set_size);
+ }
+
+ for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
+ zend_basic_block *b = ssa->cfg.blocks + i;
+
+ /* if b is loop header */
+ if ((b->flags & ZEND_BB_LOOP_HEADER)) {
+ live = live_in + set_size * i;
+
+ if (!zend_bitset_empty(live, set_size)) {
+ uint32_t set_size2 = zend_bitset_len(op_array->last);
+
+ zend_bitset_clear(loop_body, set_size2);
+ zend_jit_compute_loop_body(ssa, i, i, loop_body);
+ while (!zend_bitset_empty(loop_body, set_size2)) {
+ uint32_t from = zend_bitset_first(loop_body, set_size2);
+ uint32_t to = from;
+
+ do {
+ zend_bitset_excl(loop_body, to);
+ to++;
+ } while (zend_bitset_in(loop_body, to));
+ to--;
+
+ ZEND_BITSET_FOREACH(live, set_size, j) {
+ if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
+ goto failure;
+ }
+ } ZEND_BITSET_FOREACH_END();
+ }
+ }
+ }
+
+ }
+
+ if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) {
+ /* Register hinting (a cheap way for register coalescing) */
+ for (i = 0; i < ssa->vars_count; i++) {
+ if (intervals[i]) {
+ int var = intervals[i]->ssa_var;
+ int src;
+
+ if (ssa->vars[var].definition_phi) {
+ zend_ssa_phi *phi = ssa->vars[var].definition_phi;
+
+ if (phi->pi >= 0) {
+ src = phi->sources[0];
+ if (intervals[src]) {
+ zend_jit_add_hint(intervals, i, src);
+ }
+ } else {
+ for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
+ src = phi->sources[k];
+ if (src >= 0) {
+ if (ssa->vars[src].definition_phi
+ && ssa->vars[src].definition_phi->pi >= 0
+ && phi->block == ssa->vars[src].definition_phi->block) {
+ /* Skip zero-lenght interval for Pi variable */
+ src = ssa->vars[src].definition_phi->sources[0];
+ }
+ if (intervals[src]) {
+ zend_jit_add_hint(intervals, i, src);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ for (i = 0; i < ssa->vars_count; i++) {
+ if (intervals[i] && !intervals[i]->hint) {
+ int var = intervals[i]->ssa_var;
+
+ if (ssa->vars[var].definition >= 0) {
+ uint32_t line = ssa->vars[var].definition;
+ const zend_op *opline = op_array->opcodes + line;
+
+ switch (opline->opcode) {
+ case ZEND_QM_ASSIGN:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ if (ssa->ops[line].op1_use >= 0 &&
+ intervals[ssa->ops[line].op1_use] &&
+ (var == ssa->ops[line].op1_def ||
+ (var == ssa->ops[line].result_def &&
+ (ssa->ops[line].op1_def < 0 ||
+ !intervals[ssa->ops[line].op1_def])))) {
+ zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
+ }
+ break;
+ case ZEND_SEND_VAR:
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ if (var == ssa->ops[line].op1_def &&
+ ssa->ops[line].op1_use >= 0 &&
+ intervals[ssa->ops[line].op1_use]) {
+ zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
+ }
+ break;
+ case ZEND_ASSIGN:
+ if (ssa->ops[line].op2_use >= 0 &&
+ intervals[ssa->ops[line].op2_use] &&
+ (var == ssa->ops[line].op2_def ||
+ (var == ssa->ops[line].op1_def &&
+ (ssa->ops[line].op2_def < 0 ||
+ !intervals[ssa->ops[line].op2_def])) ||
+ (var == ssa->ops[line].result_def &&
+ (ssa->ops[line].op2_def < 0 ||
+ !intervals[ssa->ops[line].op2_def]) &&
+ (ssa->ops[line].op1_def < 0 ||
+ !intervals[ssa->ops[line].op1_def])))) {
+ zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
+
+ if (*list) {
+ zend_lifetime_interval *ival = *list;
+ while (ival) {
+ if (ival->hint) {
+ ival->hint->used_as_hint = ival;
+ }
+ ival = ival->list_next;
+ }
+ }
+
+ free_alloca(intervals, use_heap);
+ return SUCCESS;
+
+failure:
+ *list = NULL;
+ free_alloca(intervals, use_heap);
+ return FAILURE;
+}
+
+static uint32_t zend_interval_end(zend_lifetime_interval *ival)
+{
+ zend_life_range *range = &ival->range;
+
+ while (range->next) {
+ range = range->next;
+ }
+ return range->end;
+}
+
+static zend_bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
+{
+ zend_life_range *range = &ival->range;
+
+ do {
+ if (position >= range->start && position <= range->end) {
+ return 1;
+ }
+ range = range->next;
+ } while (range);
+
+ return 0;
+}
+
+static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
+{
+ zend_life_range *r1 = &ival1->range;
+ zend_life_range *r2 = &ival2->range;
+
+ do {
+ if (r1->start <= r2->end) {
+ if (r2->start <= r1->end) {
+ return MAX(r1->start, r2->start);
+ } else {
+ r2 = r2->next;
+ }
+ } else {
+ r1 = r1->next;
+ }
+ } while (r1 && r2);
+
+ return 0xffffffff;
+}
+
+/* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
+ Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
+static int zend_jit_try_allocate_free_reg(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
+{
+ zend_lifetime_interval *it;
+ uint32_t freeUntilPos[ZREG_NUM];
+ uint32_t pos, pos2;
+ zend_reg i, reg, reg2;
+ zend_reg hint = ZREG_NONE;
+ zend_regset low_priority_regs;
+ zend_life_range *range;
+
+ if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
+ } else {
+ available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
+ }
+
+ /* TODO: Allow usage of preserved registers ???
+ * Their values have to be stored in prologuee and restored in epilogue
+ */
+ available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
+
+ if (ZEND_REGSET_IS_EMPTY(available)) {
+ return 0;
+ }
+
+ /* Set freeUntilPos of all physical registers to maxInt */
+ for (i = 0; i < ZREG_NUM; i++) {
+ freeUntilPos[i] = 0xffffffff;
+ }
+
+ /* for each interval it in active do */
+ /* freeUntilPos[it.reg] = 0 */
+ it = active;
+ if (ssa->vars[current->ssa_var].definition == current->range.start) {
+ while (it) {
+ if (current->range.start != zend_interval_end(it)) {
+ freeUntilPos[it->reg] = 0;
+ } else if (zend_jit_may_reuse_reg(op_array, ssa, current->range.start, current->ssa_var, it->ssa_var)) {
+ if (!ZEND_REGSET_IN(*hints, it->reg) &&
+ /* TODO: Avoid most often scratch registers. Find a better way ??? */
+ (!current->used_as_hint ||
+ (it->reg != ZREG_R0 && it->reg != ZREG_R1 && it->reg != ZREG_XMM0 && it->reg != ZREG_XMM1))) {
+ hint = it->reg;
+ }
+ } else {
+ freeUntilPos[it->reg] = 0;
+ }
+ it = it->list_next;
+ }
+ } else {
+ while (it) {
+ freeUntilPos[it->reg] = 0;
+ it = it->list_next;
+ }
+ }
+ if (current->hint) {
+ hint = current->hint->reg;
+ if (current->hint->used_as_hint == current) {
+ ZEND_REGSET_EXCL(*hints, hint);
+ }
+ }
+
+ /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
+ Michael Franz, CGO'10 (2010), Figure 6. */
+ if (current->split) {
+ /* for each interval it in inactive intersecting with current do */
+ /* freeUntilPos[it.reg] = next intersection of it with current */
+ it = inactive;
+ while (it) {
+ uint32_t next = zend_interval_intersection(current, it);
+
+ //ZEND_ASSERT(next != 0xffffffff && !current->split);
+ if (next < freeUntilPos[it->reg]) {
+ freeUntilPos[it->reg] = next;
+ }
+ it = it->list_next;
+ }
+ }
+
+ /* Handle Scratch Registers */
+ /* TODO: Optimize ??? */
+ range = ¤t->range;
+ do {
+ uint32_t line = range->start;
+ zend_regset regset;
+ zend_reg reg;
+
+ if (ssa->ops[line].op1_def == current->ssa_var ||
+ ssa->ops[line].op2_def == current->ssa_var ||
+ ssa->ops[line].result_def == current->ssa_var) {
+ line++;
+ }
+ while (line <= range->end) {
+ regset = zend_jit_get_scratch_regset(op_array, ssa, line, current->ssa_var);
+ ZEND_REGSET_FOREACH(regset, reg) {
+ if (line < freeUntilPos[reg]) {
+ freeUntilPos[reg] = line;
+ }
+ } ZEND_REGSET_FOREACH_END();
+ line++;
+ }
+ range = range->next;
+ } while (range);
+
+#if 0
+ /* Coalesing */
+ if (ssa->vars[current->ssa_var].definition == current->start) {
+ zend_op *opline = op_array->opcodes + current->start;
+ int hint = -1;
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN:
+ hint = ssa->ops[current->start].op2_use;
+ case ZEND_QM_ASSIGN:
+ hint = ssa->ops[current->start].op1_use;
+ break;
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ hint = ssa->ops[current->start].op1_use;
+ break;
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+ if (opline->extended_value) {
+ hint = ssa->ops[current->start].op1_use;
+ }
+ break;
+ }
+ if (hint >= 0) {
+ }
+ }
+#endif
+
+ if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
+ current->reg = hint;
+ if (current->used_as_hint) {
+ ZEND_REGSET_INCL(*hints, hint);
+ }
+ return 1;
+ }
+
+ pos = 0; reg = ZREG_NONE;
+ pos2 = 0; reg2 = ZREG_NONE;
+ low_priority_regs = *hints;
+ if (current->used_as_hint) {
+ /* TODO: Avoid most often scratch registers. Find a better way ??? */
+ ZEND_REGSET_INCL(low_priority_regs, ZREG_R0);
+ ZEND_REGSET_INCL(low_priority_regs, ZREG_R1);
+ ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM0);
+ ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM1);
+ }
+
+ ZEND_REGSET_FOREACH(available, i) {
+ if (ZEND_REGSET_IN(low_priority_regs, i)) {
+ if (freeUntilPos[i] > pos2) {
+ reg2 = i;
+ pos2 = freeUntilPos[i];
+ }
+ } else if (freeUntilPos[i] > pos) {
+ reg = i;
+ pos = freeUntilPos[i];
+ }
+ } ZEND_REGSET_FOREACH_END();
+
+ if (reg == ZREG_NONE) {
+ if (reg2 != ZREG_NONE) {
+ reg = reg2;
+ pos = pos2;
+ reg2 = ZREG_NONE;
+ }
+ }
+
+ if (reg == ZREG_NONE) {
+ /* no register available without spilling */
+ return 0;
+ } else if (zend_interval_end(current) < pos) {
+ /* register available for the whole interval */
+ current->reg = reg;
+ if (current->used_as_hint) {
+ ZEND_REGSET_INCL(*hints, reg);
+ }
+ return 1;
+#if 0
+ // TODO: allow low prioirity register usage
+ } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
+ /* register available for the whole interval */
+ current->reg = reg2;
+ if (current->used_as_hint) {
+ ZEND_REGSET_INCL(*hints, reg2);
+ }
+ return 1;
+#endif
+ } else {
+ /* TODO: enable interval splitting ??? */
+ /* register available for the first part of the interval */
+ if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
+ return 0;
+ }
+ current->reg = reg;
+ if (current->used_as_hint) {
+ ZEND_REGSET_INCL(*hints, reg);
+ }
+ return 1;
+ }
+}
+
+/* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
+ Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
+ and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
+ Michael Franz, CGO'10 (2010), Figure 6. */
+static int zend_jit_allocate_blocked_reg(void)
+{
+ /* TODO: ??? */
+ return 0;
+}
+
+/* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
+ Christian Wimmer VEE'10 (2005), Figure 2. */
+static zend_lifetime_interval* zend_jit_linear_scan(zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval *list)
+{
+ zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
+ zend_lifetime_interval *current, **p, *q;
+ uint32_t position;
+ zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
+ zend_regset hints = ZEND_REGSET_EMPTY;
+
+ unhandled = list;
+ /* active = inactive = handled = free = {} */
+ active = inactive = handled = free = NULL;
+ while (unhandled != NULL) {
+ current = unhandled;
+ unhandled = unhandled->list_next;
+ position = current->range.start;
+
+ p = &active;
+ while (*p) {
+ uint32_t end = zend_interval_end(*p);
+
+ q = *p;
+ if (end < position) {
+ /* move ival from active to handled */
+ ZEND_REGSET_INCL(available, q->reg);
+ *p = q->list_next;
+ q->list_next = handled;
+ handled = q;
+ } else if (!zend_interval_covers(q, position)) {
+ /* move ival from active to inactive */
+ ZEND_REGSET_INCL(available, q->reg);
+ *p = q->list_next;
+ q->list_next = inactive;
+ inactive = q;
+ } else {
+ p = &q->list_next;
+ }
+ }
+
+ p = &inactive;
+ while (*p) {
+ uint32_t end = zend_interval_end(*p);
+
+ q = *p;
+ if (end < position) {
+ /* move ival from inactive to handled */
+ *p = q->list_next;
+ q->list_next = handled;
+ handled = q;
+ } else if (zend_interval_covers(q, position)) {
+ /* move ival from inactive to active */
+ ZEND_REGSET_EXCL(available, q->reg);
+ *p = q->list_next;
+ q->list_next = active;
+ active = q;
+ } else {
+ p = &q->list_next;
+ }
+ }
+
+ if (zend_jit_try_allocate_free_reg(op_array, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
+ zend_jit_allocate_blocked_reg()) {
+ ZEND_REGSET_EXCL(available, current->reg);
+ current->list_next = active;
+ active = current;
+ } else {
+ current->list_next = free;
+ free = current;
+ }
+ }
+
+ /* move active to handled */
+ while (active) {
+ current = active;
+ active = active->list_next;
+ current->list_next = handled;
+ handled = current;
+ }
+
+ /* move inactive to handled */
+ while (inactive) {
+ current = inactive;
+ inactive = inactive->list_next;
+ current->list_next = handled;
+ handled = current;
+ }
+
+ return handled;
+}
+
+static zend_lifetime_interval** zend_jit_allocate_registers(zend_op_array *op_array, zend_ssa *ssa)
+{
+ void *checkpoint;
+ int set_size, candidates_count, i;
+ zend_bitset candidates = NULL;
+ zend_lifetime_interval *list, *ival;
+ zend_lifetime_interval **intervals;
+ ALLOCA_FLAG(use_heap);
+
+ if (!ssa->var_info) {
+ return NULL;
+ }
+
+ /* Identify SSA variables suitable for register allocation */
+ set_size = zend_bitset_len(ssa->vars_count);
+ candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
+ if (!candidates) {
+ return NULL;
+ }
+ candidates_count = 0;
+ zend_bitset_clear(candidates, set_size);
+ for (i = 0; i < ssa->vars_count; i++) {
+ if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
+ zend_bitset_incl(candidates, i);
+ candidates_count++;
+ }
+ }
+ if (!candidates_count) {
+ free_alloca(candidates, use_heap);
+ return NULL;
+ }
+
+ checkpoint = zend_arena_checkpoint(CG(arena));
+
+ /* Find life-time intervals */
+ if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
+ goto failure;
+ }
+
+ if (list) {
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) {
+ fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
+ ival = list;
+ while (ival) {
+ zend_life_range *range;
+ int var_num = ssa->vars[ival->ssa_var].var;
+
+ fprintf(stderr, "#%d.", ival->ssa_var);
+ zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
+ fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
+ range = ival->range.next;
+ while (range) {
+ fprintf(stderr, ", %u-%u", range->start, range->end);
+ range = range->next;
+ }
+ if (ival->load) {
+ fprintf(stderr, " load");
+ }
+ if (ival->store) {
+ fprintf(stderr, " store");
+ }
+ if (ival->hint) {
+ var_num = ssa->vars[ival->hint->ssa_var].var;
+ fprintf(stderr, " hint=#%d.", ival->hint->ssa_var);
+ zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
+ }
+ fprintf(stderr, "\n");
+ ival = ival->list_next;
+ }
+ fprintf(stderr, "\n");
+ }
+
+ /* Linear Scan Register Allocation */
+ list = zend_jit_linear_scan(op_array, ssa, list);
+
+ if (list) {
+ intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
+ if (!intervals) {
+ goto failure;
+ }
+
+ ival = list;
+ while (ival != NULL) {
+ zend_lifetime_interval *next = ival->list_next;
+
+ ival->list_next = intervals[ival->ssa_var];
+ intervals[ival->ssa_var] = ival;
+ ival = next;
+ }
+
+ if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) {
+ /* Naive SSA resolution */
+ for (i = 0; i < ssa->vars_count; i++) {
+ if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
+ zend_ssa_phi *phi = ssa->vars[i].definition_phi;
+ int k, src;
+
+ if (phi->pi >= 0) {
+ if (!ssa->vars[i].phi_use_chain
+ || ssa->vars[i].phi_use_chain->block != phi->block) {
+ src = phi->sources[0];
+ if (intervals[i]) {
+ if (!intervals[src]) {
+ intervals[i]->load = 1;
+ } else if (intervals[i]->reg != intervals[src]->reg) {
+ intervals[i]->load = 1;
+ intervals[src]->store = 1;
+ }
+ } else if (intervals[src]) {
+ intervals[src]->store = 1;
+ }
+ }
+ } else {
+ int need_move = 0;
+
+ for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
+ src = phi->sources[k];
+ if (src >= 0) {
+ if (ssa->vars[src].definition_phi
+ && ssa->vars[src].definition_phi->pi >= 0
+ && phi->block == ssa->vars[src].definition_phi->block) {
+ /* Skip zero-lenght interval for Pi variable */
+ src = ssa->vars[src].definition_phi->sources[0];
+ }
+ if (intervals[i]) {
+ if (!intervals[src]) {
+ need_move = 1;
+ } else if (intervals[i]->reg != intervals[src]->reg) {
+ need_move = 1;
+ }
+ } else if (intervals[src]) {
+ need_move = 1;
+ }
+ }
+ }
+ if (need_move) {
+ if (intervals[i]) {
+ intervals[i]->load = 1;
+ }
+ for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
+ src = phi->sources[k];
+ if (src >= 0) {
+ if (ssa->vars[src].definition_phi
+ && ssa->vars[src].definition_phi->pi >= 0
+ && phi->block == ssa->vars[src].definition_phi->block) {
+ /* Skip zero-lenght interval for Pi variable */
+ src = ssa->vars[src].definition_phi->sources[0];
+ }
+ if (intervals[src]) {
+ intervals[src]->store = 1;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ /* Remove useless register allocation */
+ for (i = 0; i < ssa->vars_count; i++) {
+ if (intervals[i] &&
+ (intervals[i]->load ||
+ (intervals[i]->store && ssa->vars[i].definition >= 0)) &&
+ ssa->vars[i].use_chain < 0) {
+ zend_bool may_remove = 1;
+ zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
+
+ while (phi) {
+ if (intervals[phi->ssa_var] &&
+ !intervals[phi->ssa_var]->load) {
+ may_remove = 0;
+ break;
+ }
+ phi = zend_ssa_next_use_phi(ssa, i, phi);
+ }
+ if (may_remove) {
+ intervals[i] = NULL;
+ }
+ }
+ }
+ /* Remove intervals used once */
+ for (i = 0; i < ssa->vars_count; i++) {
+ if (intervals[i] &&
+ intervals[i]->load &&
+ intervals[i]->store &&
+ (ssa->vars[i].use_chain < 0 ||
+ zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
+ zend_bool may_remove = 1;
+ zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
+
+ while (phi) {
+ if (intervals[phi->ssa_var] &&
+ !intervals[phi->ssa_var]->load) {
+ may_remove = 0;
+ break;
+ }
+ phi = zend_ssa_next_use_phi(ssa, i, phi);
+ }
+ if (may_remove) {
+ intervals[i] = NULL;
+ }
+ }
+ }
+ }
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) {
+ fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
+ for (i = 0; i < ssa->vars_count; i++) {
+ ival = intervals[i];
+ while (ival) {
+ zend_life_range *range;
+ int var_num = ssa->vars[ival->ssa_var].var;
+
+ fprintf(stderr, "#%d.", ival->ssa_var);
+ zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
+ fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
+ range = ival->range.next;
+ while (range) {
+ fprintf(stderr, ", %u-%u", range->start, range->end);
+ range = range->next;
+ }
+ fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
+ if (ival->load) {
+ fprintf(stderr, " load");
+ }
+ if (ival->store) {
+ fprintf(stderr, " store");
+ }
+ if (ival->hint) {
+ var_num = ssa->vars[ival->hint->ssa_var].var;
+ fprintf(stderr, " hint=#%d.", ival->hint->ssa_var);
+ zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
+ if (ival->hint->reg != ZREG_NONE) {
+ fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
+ }
+ }
+ fprintf(stderr, "\n");
+ ival = ival->list_next;
+ }
+ }
+ fprintf(stderr, "\n");
+ }
+
+ free_alloca(candidates, use_heap);
+ return intervals;
+ }
+ }
+
+failure:
+ zend_arena_release(&CG(arena), checkpoint);
+ free_alloca(candidates, use_heap);
+ return NULL;
+}
+
+static void zend_calc_checked_this_r(zend_bitset checked_this, zend_op_array *op_array, zend_cfg *cfg, int b, int checked)
+{
+ zend_op *opline = &op_array->opcodes[cfg->blocks[b].start];
+ zend_op *end = opline + cfg->blocks[b].len;
+ int old_checked = checked;
+ int i;
+
+ for (; opline < end; opline++) {
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+ case ZEND_ASSIGN_DIV:
+ case ZEND_ASSIGN_MOD:
+ case ZEND_ASSIGN_SL:
+ case ZEND_ASSIGN_SR:
+ case ZEND_ASSIGN_CONCAT:
+ case ZEND_ASSIGN_BW_OR:
+ case ZEND_ASSIGN_BW_AND:
+ case ZEND_ASSIGN_BW_XOR:
+ case ZEND_ASSIGN_POW:
+ if (opline->extended_value != ZEND_ASSIGN_OBJ) {
+ break;
+ }
+ case ZEND_PRE_INC_OBJ:
+ case ZEND_PRE_DEC_OBJ:
+ case ZEND_POST_INC_OBJ:
+ case ZEND_POST_DEC_OBJ:
+ case ZEND_FETCH_OBJ_R:
+ case ZEND_FETCH_OBJ_W:
+ case ZEND_FETCH_OBJ_RW:
+ case ZEND_FETCH_OBJ_IS:
+ case ZEND_FETCH_OBJ_FUNC_ARG:
+ case ZEND_FETCH_OBJ_UNSET:
+ case ZEND_ASSIGN_OBJ:
+ case ZEND_ASSIGN_OBJ_REF:
+ case ZEND_INIT_METHOD_CALL:
+ case ZEND_CLONE:
+ case ZEND_UNSET_OBJ:
+ case ZEND_ISSET_ISEMPTY_PROP_OBJ:
+ if (opline->op1_type != IS_UNUSED) {
+ break;
+ }
+ case ZEND_FETCH_THIS:
+ if (checked) {
+ zend_bitset_incl(checked_this, (opline - op_array->opcodes));
+ } else {
+ checked = 1;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (cfg->blocks[b].flags & ZEND_BB_TRY) {
+ checked = old_checked;
+ }
+
+ for (i = cfg->blocks[b].children; i >= 0; i = cfg->blocks[i].next_child) {
+ zend_calc_checked_this_r(checked_this, op_array, cfg, i, checked);
+ }
+}
+
+static zend_bitset zend_calc_checked_this(zend_arena **arena, zend_op_array *op_array, zend_cfg *cfg)
+{
+ uint32_t bitset_len = zend_bitset_len(op_array->last);
+ zend_bitset checked_this = zend_arena_calloc(arena, bitset_len, ZEND_BITSET_ELM_SIZE);
+
+ zend_calc_checked_this_r(checked_this, op_array, cfg, 0, 0);
+
+ return checked_this;
+}
+
+static int zend_jit(zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
+{
+ int b, i, end;
+ zend_op *opline;
+ dasm_State* dasm_state = NULL;
+ void *handler;
+ int call_level = 0;
+ void *checkpoint = NULL;
+ zend_lifetime_interval **ra = NULL;
+ zend_bitset checked_this = NULL;
+ zend_bool is_terminated = 1; /* previous basic block is terminated by jump */
+ zend_bool recv_emitted = 0; /* emitted at least one RECV opcode */
+
+ if (zend_jit_reg_alloc) {
+ checkpoint = zend_arena_checkpoint(CG(arena));
+ ra = zend_jit_allocate_registers(op_array, ssa);
+ }
+
+ /* mark hidden branch targets */
+ for (b = 0; b < ssa->cfg.blocks_count; b++) {
+ if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
+ ssa->cfg.blocks[b].len > 1) {
+
+ opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
+ if (opline->opcode == ZEND_DO_FCALL &&
+ (opline-1)->opcode == ZEND_NEW) {
+ ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
+ }
+ }
+ }
+
+ dasm_init(&dasm_state, DASM_MAXSECTION);
+ dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
+ dasm_setup(&dasm_state, dasm_actions);
+
+ dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
+
+ zend_jit_align_func(&dasm_state);
+ for (b = 0; b < ssa->cfg.blocks_count; b++) {
+ if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
+ continue;
+ }
+//#ifndef CONTEXT_THREADED_JIT
+ if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
+ if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
+ /* pass */
+ } else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE &&
+ ssa->cfg.blocks[b].len == 1 &&
+ (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
+ op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
+ /* don't generate code for BB with single opcode */
+ continue;
+ }
+ if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
+ if (!is_terminated) {
+ zend_jit_jmp(&dasm_state, b);
+ }
+ }
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
+ zend_jit_prologue(&dasm_state);
+ } else
+//#endif
+ if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
+ opline = op_array->opcodes + ssa->cfg.blocks[b].start;
+ if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
+ if (opline->opcode == ZEND_RECV_INIT) {
+ if (opline == op_array->opcodes ||
+ (opline-1)->opcode != ZEND_RECV_INIT) {
+ if (recv_emitted) {
+ zend_jit_jmp(&dasm_state, b);
+ }
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
+ for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
+ }
+ zend_jit_prologue(&dasm_state);
+ }
+ recv_emitted = 1;
+ } else if (opline->opcode == ZEND_RECV) {
+ if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ /* skip */
+ continue;
+ } else if (recv_emitted) {
+ zend_jit_jmp(&dasm_state, b);
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
+ zend_jit_prologue(&dasm_state);
+ } else {
+ zend_arg_info *arg_info;
+
+ if (opline->op1.num <= op_array->num_args) {
+ arg_info = &op_array->arg_info[opline->op1.num - 1];
+ } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
+ arg_info = &op_array->arg_info[op_array->num_args];
+ } else {
+ /* skip */
+ continue;
+ }
+ if (!ZEND_TYPE_IS_SET(arg_info->type)) {
+ /* skip */
+ continue;
+ }
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
+ zend_jit_prologue(&dasm_state);
+ recv_emitted = 1;
+ }
+ } else {
+ if (recv_emitted) {
+ zend_jit_jmp(&dasm_state, b);
+ } else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE &&
+ ssa->cfg.blocks[b].len == 1 &&
+ (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
+ /* don't generate code for BB with single opcode */
+ dasm_free(&dasm_state);
+
+ if (zend_jit_reg_alloc) {
+ zend_arena_release(&CG(arena), checkpoint);
+ }
+ return SUCCESS;
+ }
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
+ zend_jit_prologue(&dasm_state);
+ recv_emitted = 1;
+ }
+ } else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE &&
+ ssa->cfg.blocks[b].len == 1 &&
+ (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
+ /* don't generate code for BB with single opcode */
+ dasm_free(&dasm_state);
+
+ if (zend_jit_reg_alloc) {
+ zend_arena_release(&CG(arena), checkpoint);
+ }
+ return SUCCESS;
+ } else {
+ zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
+ zend_jit_prologue(&dasm_state);
+ }
+ }
+
+ is_terminated = 0;
+
+ zend_jit_label(&dasm_state, b);
+ if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) {
+ if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
+ && ssa->cfg.blocks[b].start != 0
+ && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
+ || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
+ || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING)) {
+ if (!zend_jit_reset_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)
+ || !zend_jit_set_valid_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
+ goto jit_failure;
+ }
+ } else {
+ if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
+ goto jit_failure;
+ }
+ }
+ } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
+ if (!zend_jit_reset_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
+ goto jit_failure;
+ }
+ } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
+ if (!zend_jit_set_opline(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
+ goto jit_failure;
+ }
+ }
+ if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
+ if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
+ goto jit_failure;
+ }
+ }
+ if (!ssa->cfg.blocks[b].len) {
+ continue;
+ }
+ if ((zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
+ zend_ssa_phi *phi = ssa->blocks[b].phis;
+
+ while (phi) {
+ zend_lifetime_interval *ival = ra[phi->ssa_var];
+
+ if (ival) {
+ if (ival->load) {
+ ZEND_ASSERT(ival->reg != ZREG_NONE);
+
+ if (!zend_jit_load_ssa_var(&dasm_state, ssa, phi->ssa_var, ival->reg)) {
+ goto jit_failure;
+ }
+ } else if (ival->store) {
+ ZEND_ASSERT(ival->reg != ZREG_NONE);
+
+ if (!zend_jit_store_ssa_var(&dasm_state, ssa, phi->ssa_var, ival->reg)) {
+ goto jit_failure;
+ }
+ }
+ }
+ phi = phi->next;
+ }
+ }
+ end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
+ for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
+ opline = op_array->opcodes + i;
+ switch (opline->opcode) {
+ case ZEND_INIT_FCALL:
+ case ZEND_INIT_FCALL_BY_NAME:
+ case ZEND_INIT_NS_FCALL_BY_NAME:
+ case ZEND_INIT_METHOD_CALL:
+ case ZEND_INIT_DYNAMIC_CALL:
+ case ZEND_INIT_STATIC_METHOD_CALL:
+ case ZEND_INIT_USER_CALL:
+ case ZEND_NEW:
+ call_level++;
+ }
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) {
+ switch (opline->opcode) {
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ if (!zend_jit_inc_dec(&dasm_state, opline, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ if (!zend_jit_long_math(&dasm_state, opline, &i, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+// case ZEND_DIV: // TODO: check for division by zero ???
+ if (!zend_jit_math(&dasm_state, opline, &i, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_CONCAT:
+ case ZEND_FAST_CONCAT:
+ if (!zend_jit_concat(&dasm_state, opline, &i, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+// case ZEND_ASSIGN_DIV: // TODO: check for division by zero ???
+ case ZEND_ASSIGN_CONCAT:
+ case ZEND_ASSIGN_BW_OR:
+ case ZEND_ASSIGN_BW_AND:
+ case ZEND_ASSIGN_BW_XOR:
+ case ZEND_ASSIGN_SL:
+ case ZEND_ASSIGN_SR:
+ case ZEND_ASSIGN_MOD:
+ if (!zend_jit_assign_op(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ASSIGN_DIM:
+ if (!zend_jit_assign_dim(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ASSIGN:
+ if (!zend_jit_assign(&dasm_state, opline, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_QM_ASSIGN:
+ if (!zend_jit_qm_assign(&dasm_state, opline, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_INIT_FCALL:
+ case ZEND_INIT_FCALL_BY_NAME:
+ if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAL_EX:
+ if (!zend_jit_send_val(&dasm_state, opline, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_SEND_REF:
+ if (!zend_jit_send_ref(&dasm_state, opline, op_array, ssa, 0)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ if (!zend_jit_send_var(&dasm_state, opline, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_DO_UCALL:
+ is_terminated = 1;
+ /* break missing intentionally */
+ case ZEND_DO_ICALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ case ZEND_DO_FCALL:
+ if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_CASE:
+ if (!zend_jit_cmp(&dasm_state, opline, b, &i, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ if (!zend_jit_identical(&dasm_state, opline, b, &i, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_DEFINED:
+ if (!zend_jit_defined(&dasm_state, opline, b, &i, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_TYPE_CHECK:
+ if (!zend_jit_type_check(&dasm_state, opline, b, &i, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_RETURN:
+ if (!zend_jit_return(&dasm_state, opline, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_BOOL:
+ case ZEND_BOOL_NOT:
+ if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ if (i != ssa->cfg.blocks[b].start &&
+ ((opline-1)->opcode == ZEND_IS_EQUAL ||
+ (opline-1)->opcode == ZEND_IS_NOT_EQUAL ||
+ (opline-1)->opcode == ZEND_IS_SMALLER ||
+ (opline-1)->opcode == ZEND_IS_SMALLER_OR_EQUAL ||
+ (opline-1)->opcode == ZEND_CASE)) {
+ /* skip */
+ } else if (i != ssa->cfg.blocks[b].start &&
+ (opline->opcode == ZEND_JMPZ ||
+ (opline->opcode == ZEND_JMPNZ)) &&
+ zend_is_smart_branch(opline-1)) {
+ /* smart branch */
+ if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
+ goto jit_failure;
+ }
+ } else {
+ if (!zend_jit_bool_jmpznz(&dasm_state, opline, b, op_array, ssa, ra)) {
+ goto jit_failure;
+ }
+ }
+ goto done;
+ case ZEND_FETCH_DIM_R:
+ case ZEND_FETCH_DIM_IS:
+ if (!zend_jit_fetch_dim_read(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ISSET_ISEMPTY_DIM_OBJ:
+ if (!zend_jit_isset_isempty_dim(&dasm_state, opline, b, &i, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_FETCH_OBJ_R:
+ case ZEND_FETCH_OBJ_IS:
+ if (opline->op1_type == IS_UNUSED && !checked_this) {
+ checked_this = zend_calc_checked_this(&CG(arena), op_array, &ssa->cfg);
+ }
+ if (!zend_jit_fetch_obj_read(&dasm_state, opline, op_array, ssa, checked_this)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_BIND_GLOBAL:
+ if (!zend_jit_bind_global(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_RECV:
+ if (!zend_jit_recv(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_RECV_INIT:
+ if (!zend_jit_recv_init(&dasm_state, opline, op_array, (opline + 1)->opcode != ZEND_RECV_INIT, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_FREE:
+ case ZEND_FE_FREE:
+ if (!zend_jit_free(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_ECHO:
+ if (!zend_jit_echo(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ case ZEND_SWITCH_LONG:
+ case ZEND_SWITCH_STRING:
+ if (!zend_jit_switch(&dasm_state, opline, op_array, ssa)) {
+ goto jit_failure;
+ }
+ goto done;
+ default:
+ break;
+ }
+ }
+
+ switch (opline->opcode) {
+ case ZEND_RECV_INIT:
+ case ZEND_BIND_GLOBAL:
+ if (opline == op_array->opcodes ||
+ opline->opcode != op_array->opcodes[i-1].opcode) {
+ /* repeatable opcodes */
+ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) {
+ goto jit_failure;
+ }
+ }
+ zend_jit_set_opline(&dasm_state, opline+1);
+ break;
+ case ZEND_NOP:
+ case ZEND_OP_DATA:
+ case ZEND_SWITCH_LONG:
+ case ZEND_SWITCH_STRING:
+ break;
+ case ZEND_JMP:
+ if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) {
+ const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
+
+ if (!zend_jit_set_ip(&dasm_state, target)) {
+ goto jit_failure;
+ }
+ }
+ if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
+ goto jit_failure;
+ }
+ is_terminated = 1;
+ break;
+ case ZEND_CATCH:
+ case ZEND_FAST_CALL:
+ case ZEND_FAST_RET:
+ case ZEND_GENERATOR_CREATE:
+ case ZEND_GENERATOR_RETURN:
+ case ZEND_RETURN_BY_REF:
+ case ZEND_RETURN:
+ case ZEND_EXIT:
+ /* switch through trampoline */
+ case ZEND_YIELD:
+ case ZEND_YIELD_FROM:
+ if (!zend_jit_tail_handler(&dasm_state, opline)) {
+ goto jit_failure;
+ }
+ is_terminated = 1;
+ break;
+ /* stackless execution */
+ case ZEND_INCLUDE_OR_EVAL:
+ case ZEND_DO_FCALL:
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ if (!zend_jit_call(&dasm_state, opline, b + 1)) {
+ goto jit_failure;
+ }
+ is_terminated = 1;
+ break;
+ case ZEND_JMPZNZ:
+ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) ||
+ !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) ||
+ !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
+ goto jit_failure;
+ }
+ is_terminated = 1;
+ break;
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ if (i != ssa->cfg.blocks[b].start) {
+ if (zend_is_smart_branch(opline-1)) {
+ /* smart branch */
+ if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
+ goto jit_failure;
+ }
+ break;
+ }
+ }
+ /* break missing intentionally */
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_FE_RESET_R:
+ case ZEND_FE_RESET_RW:
+ case ZEND_ASSERT_CHECK:
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ case ZEND_DECLARE_ANON_CLASS:
+ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa)) ||
+ !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
+ goto jit_failure;
+ }
+ break;
+ case ZEND_NEW:
+ if (!zend_jit_handler(&dasm_state, opline, 1)) {
+ return 0;
+ }
+ if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
+ zend_class_entry *ce = NULL;
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) {
+ if (ssa->ops && ssa->var_info) {
+ zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
+ if (res_ssa->ce && !res_ssa->is_instanceof) {
+ ce = res_ssa->ce;
+ }
+ }
+ } else {
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+ if (Z_TYPE_P(zv) == IS_STRING) {
+ zval *lc = zv + 1;
+ ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
+ }
+ }
+ }
+
+ i++;
+
+ if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
+ const zend_op *next_opline = opline + 1;
+
+ zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
+ if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) {
+ zend_jit_call(&dasm_state, next_opline, b + 1);
+ is_terminated = 1;
+ } else {
+ zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1);
+ }
+ }
+ }
+ break;
+ default:
+ if (!zend_jit_handler(&dasm_state, opline, zend_may_throw(opline, op_array, ssa))) {
+ goto jit_failure;
+ }
+ }
+done:
+ switch (opline->opcode) {
+ case ZEND_DO_FCALL:
+ case ZEND_DO_ICALL:
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ call_level--;
+ }
+ }
+ }
+
+ handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL);
+ if (!handler) {
+ goto jit_failure;
+ }
+ dasm_free(&dasm_state);
+
+ if (zend_jit_reg_alloc) {
+ zend_arena_release(&CG(arena), checkpoint);
+ }
+ return SUCCESS;
+
+jit_failure:
+ if (dasm_state) {
+ dasm_free(&dasm_state);
+ }
+ if (zend_jit_reg_alloc) {
+ zend_arena_release(&CG(arena), checkpoint);
+ }
+ return FAILURE;
+}
+
+static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
+{
+ zend_func_info *func_info =
+ zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
+
+ ZEND_SET_FUNC_INFO(op_array, func_info);
+ func_info->num_args = -1;
+ func_info->return_value_used = -1;
+ return zend_analyze_calls(&CG(arena), script, ZEND_RT_CONSTANTS | ZEND_CALL_TREE, op_array, func_info);
+}
+
+static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
+{
+ zend_ssa ssa;
+ void *checkpoint;
+
+ if (*dasm_ptr == dasm_end) {
+ return FAILURE;
+ }
+
+ checkpoint = zend_arena_checkpoint(CG(arena));
+
+ /* Build SSA */
+ memset(&ssa, 0, sizeof(zend_ssa));
+
+ if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
+ goto jit_failure;
+ }
+
+ if (zend_jit_op_array_analyze2(op_array, script, &ssa) != SUCCESS) {
+ goto jit_failure;
+ }
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) {
+ zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &ssa);
+ }
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) {
+ if (zend_jit_collect_calls(op_array, script) != SUCCESS) {
+ ZEND_SET_FUNC_INFO(op_array, NULL);
+ goto jit_failure;
+ }
+ }
+
+ if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
+ goto jit_failure;
+ }
+
+ if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) {
+ ZEND_SET_FUNC_INFO(op_array, NULL);
+ }
+
+ zend_arena_release(&CG(arena), checkpoint);
+ return SUCCESS;
+
+jit_failure:
+ zend_arena_release(&CG(arena), checkpoint);
+ return FAILURE;
+}
+
+/* Run-time JIT handler */
+static void ZEND_FASTCALL zend_runtime_jit(void)
+{
+ zend_execute_data *execute_data = EG(current_execute_data);
+ zend_op_array *op_array = &EX(func)->op_array;
+ zend_op *opline = op_array->opcodes;
+
+ zend_shared_alloc_lock();
+
+ if (ZEND_FUNC_INFO(op_array)) {
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ /* restore original opcode handlers */
+ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
+ opline++;
+ }
+ opline->handler = ZEND_FUNC_INFO(op_array);
+ ZEND_SET_FUNC_INFO(op_array, NULL);
+
+ /* perform real JIT for this function */
+ zend_real_jit_func(op_array, NULL, NULL);
+
+ zend_jit_protect();
+ SHM_PROTECT();
+ }
+
+ zend_shared_alloc_unlock();
+
+ /* JIT-ed code is going to be called by VM */
+}
+
+void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) {
+ zend_op *opline;
+ zend_function *func;
+ zend_op_array *op_array;
+ uintptr_t counter;
+
+ ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) {
+ if (func->type == ZEND_INTERNAL_FUNCTION) {
+ break;
+ }
+ op_array = &func->op_array;
+ opline = op_array->opcodes;
+ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
+ opline++;
+ }
+ if (opline->handler == zend_jit_profile_jit_handler) {
+ if (!RUN_TIME_CACHE(op_array)) {
+ continue;
+ }
+ counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
+ ZEND_COUNTER_INFO(op_array) = 0;
+ opline->handler = ZEND_FUNC_INFO(op_array);
+ ZEND_SET_FUNC_INFO(op_array, NULL);
+ if (((double)counter / (double)zend_jit_profile_counter) > ZEND_JIT_PROF_THRESHOLD) {
+ zend_real_jit_func(op_array, NULL, NULL);
+ }
+ }
+ } ZEND_HASH_FOREACH_END();
+}
+
+void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
+{
+ zend_op_array *op_array = &EX(func)->op_array;
+ const void **orig_handlers;
+ uint32_t i;
+
+ zend_shared_alloc_lock();
+ orig_handlers = (const void**)ZEND_FUNC_INFO(op_array);
+
+ if (orig_handlers) {
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ for (i = 0; i < op_array->last; i++) {
+ op_array->opcodes[i].handler = orig_handlers[i];
+ }
+ ZEND_SET_FUNC_INFO(op_array, NULL);
+
+ /* perform real JIT for this function */
+ zend_real_jit_func(op_array, NULL, opline);
+
+ zend_jit_protect();
+ SHM_PROTECT();
+ }
+
+ zend_shared_alloc_unlock();
+
+ /* JIT-ed code is going to be called by VM */
+}
+
+static int zend_jit_setup_hot_counters(zend_op_array *op_array)
+{
+ zend_op *opline = op_array->opcodes;
+ const void **orig_handlers;
+ zend_cfg cfg;
+ uint32_t i;
+
+ if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
+ return FAILURE;
+ }
+
+ orig_handlers = (const void**)zend_shared_alloc(op_array->last * sizeof(void*));
+ for (i = 0; i < op_array->last; i++) {
+ orig_handlers[i] = op_array->opcodes[i].handler;
+ }
+ ZEND_SET_FUNC_INFO(op_array, (void*)orig_handlers);
+
+ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
+ opline++;
+ }
+
+ opline->handler = (const void*)zend_jit_func_counter_handler;
+
+ for (i = 0; i < cfg.blocks_count; i++) {
+ if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) &&
+ (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
+ op_array->opcodes[cfg.blocks[i].start].handler =
+ (const void*)zend_jit_loop_counter_handler;
+ }
+ }
+
+ return SUCCESS;
+}
+
+static int zend_needs_manual_jit(const zend_op_array *op_array)
+{
+ if (op_array->doc_comment) {
+ const char *s = ZSTR_VAL(op_array->doc_comment);
+ const char *p = strstr(s, "@jit");
+
+ if (p) {
+ size_t l = ZSTR_LEN(op_array->doc_comment);
+
+ if ((p == s + 3 || *(p-1) <= ' ') &&
+ (p + 6 == s + l || *(p+4) <= ' ')) {
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
+ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
+{
+ if (dasm_ptr == NULL) {
+ return FAILURE;
+ }
+
+ if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) {
+ zend_op *opline = op_array->opcodes;
+
+ /* Set run-time JIT handler */
+ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
+ opline++;
+ }
+ ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler);
+ opline->handler = (const void*)zend_jit_runtime_jit_handler;
+
+ return SUCCESS;
+ } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) {
+ zend_op *opline = op_array->opcodes;
+
+ if (op_array->function_name) {
+ while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
+ opline++;
+ }
+ ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler);
+ opline->handler = (const void*)zend_jit_profile_jit_handler;
+ }
+
+ return SUCCESS;
+ } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
+ return zend_jit_setup_hot_counters(op_array);
+ } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) {
+ return zend_real_jit_func(op_array, script, NULL);
+ } else if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) {
+ if (zend_needs_manual_jit(op_array)) {
+ return zend_real_jit_func(op_array, script, NULL);
+ } else {
+ return SUCCESS;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+}
+
+ZEND_EXT_API int zend_jit_script(zend_script *script)
+{
+ void *checkpoint;
+ zend_call_graph call_graph;
+ zend_func_info *info;
+ int i;
+
+ if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
+ return FAILURE;
+ }
+
+ checkpoint = zend_arena_checkpoint(CG(arena));
+
+ call_graph.op_arrays_count = 0;
+ if (zend_build_call_graph(&CG(arena), script, ZEND_RT_CONSTANTS, &call_graph) != SUCCESS) {
+ goto jit_failure;
+ }
+
+ if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC ||
+ zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST ||
+ zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
+ if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
+ goto jit_failure;
+ }
+ }
+ } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD ||
+ zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) {
+
+ if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) {
+ int do_jit = 0;
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ if (zend_needs_manual_jit(call_graph.op_arrays[i])) {
+ do_jit = 1;
+ break;
+ }
+ }
+ if (!do_jit) {
+ goto jit_failure;
+ }
+ }
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
+ if (info) {
+ if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
+ goto jit_failure;
+ }
+ info->flags = info->ssa.cfg.flags;
+ }
+ }
+
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
+ if (info) {
+ info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
+ if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
+ zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
+ }
+ }
+ }
+
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT &&
+ !zend_needs_manual_jit(call_graph.op_arrays[i])) {
+ continue;
+ }
+ info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
+ if (info) {
+ if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
+ goto jit_failure;
+ }
+ info->flags = info->ssa.cfg.flags;
+ }
+ }
+
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) {
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT &&
+ !zend_needs_manual_jit(call_graph.op_arrays[i])) {
+ continue;
+ }
+ info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
+ if (info) {
+ zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA|ZEND_DUMP_RT_CONSTANTS, "JIT", &info->ssa);
+ }
+ }
+ }
+
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT &&
+ !zend_needs_manual_jit(call_graph.op_arrays[i])) {
+ continue;
+ }
+ info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
+ if (info) {
+ if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
+ goto jit_failure;
+ }
+ }
+ }
+
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+
+ zend_arena_release(&CG(arena), checkpoint);
+ return SUCCESS;
+
+jit_failure:
+ for (i = 0; i < call_graph.op_arrays_count; i++) {
+ ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
+ }
+ zend_arena_release(&CG(arena), checkpoint);
+ return FAILURE;
+}
+
+ZEND_EXT_API void zend_jit_unprotect(void)
+{
+#ifdef HAVE_MPROTECT
+ if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
+ if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE) != 0) {
+ fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
+ }
+ }
+#elif _WIN32
+ if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
+ DWORD old;
+
+ if (!VirtualProtect(dasm_buf, dasm_size, PAGE_READWRITE, &old)) {
+ fprintf(stderr, "VirtualProtect() failed\n");
+ }
+ }
+#endif
+}
+
+ZEND_EXT_API void zend_jit_protect(void)
+{
+#ifdef HAVE_MPROTECT
+ if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
+ if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
+ fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
+ }
+ }
+#elif _WIN32
+ if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
+ DWORD old;
+
+ if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
+ fprintf(stderr, "VirtualProtect() failed\n");
+ }
+ }
+#endif
+}
+
+static int zend_jit_make_stubs(void)
+{
+ dasm_State* dasm_state = NULL;
+ uint32_t i;
+
+ dasm_init(&dasm_state, DASM_MAXSECTION);
+ dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
+
+ for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_stubs[i].stub(&dasm_state)) {
+ return 0;
+ }
+ if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name)) {
+ return 0;
+ }
+ }
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit");
+ if (!zend_jit_runtime_jit_handler) {
+ return 0;
+ }
+
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit");
+ if (!zend_jit_profile_jit_handler) {
+ return 0;
+ }
+
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter");
+ if (!zend_jit_func_counter_handler) {
+ return 0;
+ }
+
+ dasm_setup(&dasm_state, dasm_actions);
+ if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) {
+ return 0;
+ }
+ zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter");
+ if (!zend_jit_loop_counter_handler) {
+ return 0;
+ }
+ } else {
+ zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
+ zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
+ zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper;
+ zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper;
+ }
+
+ dasm_free(&dasm_state);
+ return 1;
+}
+
+ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bool reattached)
+{
+ int ret;
+
+ zend_jit_level = ZEND_JIT_LEVEL(jit);
+ zend_jit_trigger = ZEND_JIT_TRIGGER(jit);
+ zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit);
+ zend_jit_cpu_flags = ZEND_JIT_CPU_FLAGS(jit);
+
+ zend_jit_vm_kind = zend_vm_kind();
+ if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
+ zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
+ // TODO: error reporting and cleanup ???
+ return FAILURE;
+ }
+
+ zend_jit_halt_op = zend_get_halt_op();
+
+ if (zend_jit_setup() != SUCCESS) {
+ // TODO: error reporting and cleanup ???
+ return FAILURE;
+ }
+
+ if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) {
+ zend_jit_profile_counter_rid = zend_get_op_array_extension_handle();
+ }
+
+#ifdef HAVE_GDB
+ zend_jit_gdb_init();
+#endif
+
+#ifdef HAVE_OPROFILE
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) {
+ if (!zend_jit_oprofile_startup()) {
+ // TODO: error reporting and cleanup ???
+ return FAILURE;
+ }
+ }
+#endif
+
+ dasm_buf = buf;
+ dasm_size = size;
+
+#ifdef HAVE_MPROTECT
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
+ if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
+ fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
+ }
+ } else {
+ if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
+ fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
+ }
+ }
+#elif _WIN32
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
+ DWORD old;
+
+ if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
+ fprintf(stderr, "VirtualProtect() failed\n");
+ }
+ } else {
+ DWORD old;
+
+ if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
+ fprintf(stderr, "VirtualProtect() failed\n");
+ }
+ }
+#endif
+
+ dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr));
+ if (!reattached) {
+ zend_jit_unprotect();
+ *dasm_ptr = dasm_buf;
+#if _WIN32
+ /* reserve space for global labels */
+ *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
+#endif
+ zend_jit_protect();
+ }
+
+#ifdef HAVE_DISASM
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
+ if (!zend_jit_disasm_init()) {
+ // TODO: error reporting and cleanup ???
+ return FAILURE;
+ }
+ }
+#endif
+
+#ifdef HAVE_PERFTOOLS
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) {
+ zend_jit_perf_jitdump_open();
+ }
+#endif
+
+ if (!reattached) {
+ zend_jit_unprotect();
+ ret = zend_jit_make_stubs();
+#if _WIN32
+ /* save global labels */
+ memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
+#endif
+ zend_jit_protect();
+ if (!ret) {
+ // TODO: error reporting and cleanup ???
+ return FAILURE;
+ }
+ } else {
+#if _WIN32
+ /* restore global labels */
+ memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
+#endif
+ }
+
+ return SUCCESS;
+}
+
+ZEND_EXT_API void zend_jit_shutdown(void)
+{
+#ifdef HAVE_OPROFILE
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) {
+ zend_jit_oprofile_shutdown();
+ }
+#endif
+
+#ifdef HAVE_GDB
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) {
+ zend_jit_gdb_unregister();
+ }
+#endif
+
+#ifdef HAVE_DISASM
+ if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
+ zend_jit_disasm_shutdown();
+ }
+#endif
+
+#ifdef HAVE_PERFTOOLS
+ if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) {
+ zend_jit_perf_jitdump_close();
+ }
+#endif
+}
+
+ZEND_EXT_API void zend_jit_activate(void)
+{
+ if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
+ int i;
+
+ for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
+ zend_jit_hot_counters[i] = ZEND_JIT_HOT_COUNTER_INIT;
+ }
+ }
+}
+
+ZEND_EXT_API void zend_jit_deactivate(void)
+{
+ if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) {
+ if (!zend_jit_profile_counter) {
+ return;
+ } else {
+ zend_class_entry *ce;
+
+ zend_shared_alloc_lock();
+ SHM_UNPROTECT();
+ zend_jit_unprotect();
+
+ zend_jit_check_funcs(EG(function_table), 0);
+ ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) {
+ if (ce->type == ZEND_INTERNAL_CLASS) {
+ break;
+ }
+ zend_jit_check_funcs(&ce->function_table, 1);
+ } ZEND_HASH_FOREACH_END();
+
+ zend_jit_protect();
+ SHM_PROTECT();
+ zend_shared_alloc_unlock();
+
+ zend_jit_profile_counter = 0;
+ }
+ }
+}
+
+#else /* HAVE_JIT */
+
+ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
+{
+ return FAILURE;
+}
+
+ZEND_EXT_API int zend_jit_script(zend_script *script)
+{
+ return FAILURE;
+}
+
+ZEND_EXT_API void zend_jit_unprotect(void)
+{
+}
+
+ZEND_EXT_API void zend_jit_protect(void)
+{
+}
+
+ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size)
+{
+ return FAILURE;
+}
+
+ZEND_EXT_API void zend_jit_shutdown(void)
+{
+}
+
+ZEND_EXT_API void zend_jit_activate(void)
+{
+}
+
+ZEND_EXT_API void zend_jit_deactivate(void)
+{
+}
+
+#endif /* HAVE_JIT */
--- /dev/null
- case ZEND_DECLARE_ANON_INHERITED_CLASS:
+/*
+ * +----------------------------------------------------------------------+
+ * | Zend JIT |
+ * +----------------------------------------------------------------------+
+ * | Copyright (c) The PHP Group |
+ * +----------------------------------------------------------------------+
+ * | This source file is subject to version 3.01 of the PHP license, |
+ * | that is bundled with this package in the file LICENSE, and is |
+ * | available through the world-wide-web at the following url: |
+ * | http://www.php.net/license/3_01.txt |
+ * | If you did not receive a copy of the PHP license and are unable to |
+ * | obtain it through the world-wide-web, please send a note to |
+ * | license@php.net so we can mail you a copy immediately. |
+ * +----------------------------------------------------------------------+
+ * | Authors: Dmitry Stogov <dmitry@php.net> |
+ * | Xinchen Hui <laruence@php.net> |
+ * +----------------------------------------------------------------------+
+ */
+
+|.if X64
+ |.arch x64
+|.else
+ |.arch x86
+|.endif
+
+|.if X64WIN
+ |.define FP, r14
+ |.define IP, r15
+ |.define IPl, r15d
+ |.define RX, r15 // the same as VM IP reused as a general purpos reg
+ |.define CARG1, rcx // x64/POSIX C call arguments.
+ |.define CARG2, rdx
+ |.define CARG3, r8
+ |.define CARG4, r9
+ |.define CARG1d, ecx
+ |.define CARG2d, edx
+ |.define CARG3d, r8d
+ |.define CARG4d, r9d
+ |.define FCARG1a, CARG1 // Simulate x86 fastcall.
+ |.define FCARG2a, CARG2
+ |.define FCARG1d, CARG1d
+ |.define FCARG2d, CARG2d
+ |.define SPAD, 0x08 // padding for CPU stack alignment
+ |.define NR_SPAD, 0x58 // padding for CPU stack alignment
+ |.define SSE, 1
+ |.define T3, [r4+0x50] // Used to store old value of IP
+ |.define T2, [r4+0x48] // Used to store old value of FP
+ |.define T1, [r4+0x40]
+ |.define A6, [r4+0x28] // preallocated slot for 6-th argument
+ |.define A5, [r4+0x20] // preallocated slot for 5-th argument
+|.elif X64
+ |.define FP, r14
+ |.define IP, r15
+ |.define IPl, r15d
+ |.define RX, r15 // the same as VM IP reused as a general purpos reg
+ |.define CARG1, rdi // x64/POSIX C call arguments.
+ |.define CARG2, rsi
+ |.define CARG3, rdx
+ |.define CARG4, rcx
+ |.define CARG5, r8
+ |.define CARG6, r9
+ |.define CARG1d, edi
+ |.define CARG2d, esi
+ |.define CARG3d, edx
+ |.define CARG4d, ecx
+ |.define CARG5d, r8d
+ |.define CARG6d, r9d
+ |.define FCARG1a, CARG1 // Simulate x86 fastcall.
+ |.define FCARG2a, CARG2
+ |.define FCARG1d, CARG1d
+ |.define FCARG2d, CARG2d
+ |.define SPAD, 0x08 // padding for CPU stack alignment
+ |.define NR_SPAD, 0x18 // padding for CPU stack alignment
+ |.define SSE, 1
+ |.define T3, [r4+0x10] // Used to store old value of IP (CALL VM only)
+ |.define T2, [r4+0x08] // Used to store old value of FP (CALL VM only)
+ |.define T1, [r4]
+|.else
+ |.define FP, esi
+ |.define IP, edi
+ |.define IPl, edi
+ |.define RX, edi // the same as VM IP reused as a general purpos reg
+ |.define FCARG1a, ecx // x86 fastcall arguments.
+ |.define FCARG2a, edx
+ |.define FCARG1d, ecx
+ |.define FCARG2d, edx
+ |.define SPAD, 12 // padding for CPU stack alignment
+ |.define NR_SPAD, 12 // padding for CPU stack alignment
+ |.define SSE, 1
+ |.define T3, [r4+0x10] // Used to store old value of IP (CALL VM only)
+ |.define T2, [r4+0x08] // Used to store old value of FP (CALL VM only)
+ |.define T1, [r4]
+|.endif
+
+|.define HYBRID_SPAD, 16 // padding for stack alignment
+
+/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to
+ * guarantee proper alignment of 128-bit SSE data allocated on stack.
+ * With broken alignment any execution of SSE code, including calls to
+ * memcpy() and others, may lead to crash.
+ */
+
+#include "Zend/zend_cpuinfo.h"
+#include "jit/zend_jit_x86.h"
+
+const char* zend_reg_name[] = {
+#if defined(__x86_64__) || defined(_M_X64)
+ "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
+ "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
+ "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xm15"
+#else
+ "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
+#endif
+};
+
+#ifdef HAVE_GCC_GLOBAL_REGS
+# define GCC_GLOBAL_REGS 1
+#else
+# define GCC_GLOBAL_REGS 0
+#endif
+
+static uint32_t zend_jit_x86_flags = 0;
+
+#if ZTS
+static size_t tsrm_ls_cache_tcb_offset = 0;
+static size_t tsrm_tls_index;
+static size_t tsrm_tls_offset;
+#endif
+
+|.type EX, zend_execute_data, FP
+|.type OP, zend_op
+|.type ZVAL, zval
+
+|.actionlist dasm_actions
+
+|.globals zend_lb
+static void* dasm_labels[zend_lb_MAX];
+
+|.section code, cold_code
+
+#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff)
+
+#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1)))
+
+#define BP_JIT_IS 6
+
+|.macro LOAD_ADDR, reg, addr
+| .if X64
+|| if (IS_32BIT(addr)) {
+| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit>
+|| } else {
+| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit>
+|| }
+| .else
+| mov reg, ((ptrdiff_t)addr)
+| .endif
+|.endmacro
+
+|.macro LOAD_TSRM_CACHE, reg
+| .if X64WIN
+| gs
+| mov reg, aword [0x58]
+| mov reg, aword [reg + tsrm_tls_index]
+| mov reg, aword [reg + tsrm_tls_offset]
+| .elif WIN
+| fs
+| mov reg, aword [0x2c]
+| mov reg, aword [reg + tsrm_tls_index]
+| mov reg, aword [reg + tsrm_tls_offset]
+| .elif X64
+| fs
+|| if (tsrm_ls_cache_tcb_offset) {
+| mov reg, aword [tsrm_ls_cache_tcb_offset]
+|| } else {
+| mov reg, [0x8]
+| mov reg, aword [reg + tsrm_tls_index]
+| mov reg, aword [reg + tsrm_tls_offset]
+|| }
+| .else
+| gs
+|| if (tsrm_ls_cache_tcb_offset) {
+| mov reg, aword [tsrm_ls_cache_tcb_offset]
+|| } else {
+| mov reg, [0x4]
+| mov reg, aword [reg + tsrm_tls_index]
+| mov reg, aword [reg + tsrm_tls_offset]
+|| }
+| .endif
+|.endmacro
+
+|.macro LOAD_ADDR_ZTS, reg, struct, field
+| .if ZTS
+| LOAD_TSRM_CACHE reg
+| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))]
+| .else
+| LOAD_ADDR reg, &struct.field
+| .endif
+|.endmacro
+
+|.macro SAVE_OPLINE
+|| if (GCC_GLOBAL_REGS) {
+| mov aword EX->opline, IP
+|| }
+|.endmacro
+
+|.macro LOAD_OPLINE
+|| if (GCC_GLOBAL_REGS) {
+| mov IP, aword EX->opline
+|| }
+|.endmacro
+
+|.macro LOAD_IP_ADDR, addr
+|| if (GCC_GLOBAL_REGS) {
+| LOAD_ADDR IP, addr
+|| } else {
+| LOAD_ADDR RX, addr
+| mov aword EX->opline, RX
+|| }
+|.endmacro
+
+|.macro LOAD_IP_ADDR_ZTS, struct, field
+| .if ZTS
+|| if (GCC_GLOBAL_REGS) {
+| LOAD_TSRM_CACHE IP
+| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))]
+|| } else {
+| LOAD_TSRM_CACHE RX
+| mov RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))]
+| mov aword EX->opline, RX
+|| }
+| .else
+| LOAD_IP_ADDR &struct.field
+| .endif
+|.endmacro
+
+|.macro GET_IP, reg
+|| if (GCC_GLOBAL_REGS) {
+| mov reg, IP
+|| } else {
+| mov reg, aword EX->opline
+|| }
+|.endmacro
+
+|.macro ADD_IP, val
+|| if (GCC_GLOBAL_REGS) {
+| add IP, val
+|| } else {
+| add aword EX->opline, val
+|| }
+|.endmacro
+
+|.macro JMP_IP
+|| if (GCC_GLOBAL_REGS) {
+| jmp aword [IP]
+|| } else {
+| mov r0, aword EX:FCARG1a->opline
+| jmp aword [r0]
+|| }
+|.endmacro
+
+/* In 64-bit build we compare only low 32-bits.
+ * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full
+ * comparison would require additinal load of 64-bit address into register.
+ * This is not a problem at all, while JIT buffer size is less than 4GB.
+ */
+|.macro CMP_IP, addr
+|| if (GCC_GLOBAL_REGS) {
+| cmp IPl, addr
+|| } else {
+| cmp dword EX->opline, addr
+|| }
+|.endmacro
+
+|.macro ADDR_OP1, addr_ins, addr, tmp_reg
+| .if X64
+|| if (IS_32BIT(addr)) {
+| addr_ins ((ptrdiff_t)addr)
+|| } else {
+| mov64 tmp_reg, ((ptrdiff_t)addr)
+| addr_ins tmp_reg
+|| }
+| .else
+| addr_ins ((ptrdiff_t)addr)
+| .endif
+|.endmacro
+
+|.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg
+| .if X64
+|| if (IS_32BIT(addr)) {
+| addr_ins op1, ((ptrdiff_t)addr)
+|| } else {
+| mov64 tmp_reg, ((ptrdiff_t)addr)
+| addr_ins op1, tmp_reg
+|| }
+| .else
+| addr_ins op1, ((ptrdiff_t)addr)
+| .endif
+|.endmacro
+
+|.macro PUSH_ADDR, addr, tmp_reg
+| ADDR_OP1 push, addr, tmp_reg
+|.endmacro
+
+|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg
+| .if ZTS
+| LOAD_TSRM_CACHE tmp_reg
+| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))]
+| push tmp_reg
+| .else
+| ADDR_OP1 push, &struct.field, tmp_reg
+| .endif
+|.endmacro
+
+|.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg
+| .if X64
+|| if (IS_32BIT(addr)) {
+| mem_ins prefix [addr]
+|| } else {
+| mov64 tmp_reg, ((ptrdiff_t)addr)
+| mem_ins prefix [tmp_reg]
+|| }
+| .else
+| mem_ins prefix [addr]
+| .endif
+|.endmacro
+
+|.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg
+| .if X64
+|| if (IS_32BIT(addr)) {
+| mem_ins prefix [addr], op2
+|| } else {
+| mov64 tmp_reg, ((ptrdiff_t)addr)
+| mem_ins prefix [tmp_reg], op2
+|| }
+| .else
+| mem_ins prefix [addr], op2
+| .endif
+|.endmacro
+
+|.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg
+| .if X64
+|| if (IS_32BIT(addr)) {
+| mem_ins op1, prefix [addr]
+|| } else {
+| mov64 tmp_reg, ((ptrdiff_t)addr)
+| mem_ins op1, prefix [tmp_reg]
+|| }
+| .else
+| mem_ins op1, prefix [addr]
+| .endif
+|.endmacro
+
+|.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg
+| .if ZTS
+| LOAD_TSRM_CACHE tmp_reg
+| mem_ins prefix [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))], op2
+| .else
+| MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg
+| .endif
+|.endmacro
+
+|.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg
+| .if ZTS
+| LOAD_TSRM_CACHE tmp_reg
+| mem_ins op1, prefix [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))]
+| .else
+| MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg
+| .endif
+|.endmacro
+
+|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg
+| .if X64
+|| if (IS_32BIT(addr)) {
+| mem_ins op1, op2, prefix [addr]
+|| } else {
+| mov64 tmp_reg, ((ptrdiff_t)addr)
+| mem_ins op1, op2, prefix [tmp_reg]
+|| }
+| .else
+| mem_ins op1, op2, prefix [addr]
+| .endif
+|.endmacro
+
+|.macro LOAD_BASE_ADDR, reg, base, offset
+|| if (offset) {
+| lea reg, qword [Ra(base)+offset]
+|| } else {
+| mov reg, Ra(base)
+|| }
+|.endmacro
+
+|.macro PUSH_BASE_ADDR, base, offset, tmp_reg
+|| if (offset) {
+| lea tmp_reg, qword [Ra(base)+offset]
+| push tmp_reg
+|| } else {
+| push Ra(base)
+|| }
+|.endmacro
+
+|.macro EXT_CALL, func, tmp_reg
+| .if X64
+|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) {
+| call qword &func
+|| } else {
+| LOAD_ADDR tmp_reg, func
+| call tmp_reg
+|| }
+| .else
+| call dword &func
+| .endif
+|.endmacro
+
+|.macro EXT_JMP, func, tmp_reg
+| .if X64
+|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) {
+| jmp qword &func
+|| } else {
+| LOAD_ADDR tmp_reg, func
+| jmp tmp_reg
+|| }
+| .else
+| jmp dword &func
+| .endif
+|.endmacro
+
+|.macro LOAD_ZVAL_ADDR, reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| LOAD_ADDR reg, Z_ZV(addr)
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr)
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro PUSH_ZVAL_ADDR, addr, tmp_reg
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| PUSH_ADDR Z_ZV(addr), tmp_reg
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro GET_Z_TYPE_INFO, reg, zv
+| mov reg, dword [zv+offsetof(zval,u1.type_info)]
+|.endmacro
+
+|.macro SET_Z_TYPE_INFO, zv, type
+| mov dword [zv+offsetof(zval,u1.type_info)], type
+|.endmacro
+
+|.macro GET_ZVAL_TYPE_INFO, reg, addr
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)]
+|.endmacro
+
+|.macro SET_ZVAL_TYPE_INFO, addr, type
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type
+|.endmacro
+
+|.macro GET_Z_PTR, reg, zv
+| mov reg, aword [zv]
+|.endmacro
+
+|.macro SET_Z_PTR, zv, val
+| mov aword [zv], val
+|.endmacro
+
+|.macro GET_ZVAL_PTR, reg, addr
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|.endmacro
+
+|.macro SET_ZVAL_PTR, addr, val
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val
+|.endmacro
+
+|.macro GET_ZVAL_W2, reg, addr
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4]
+|.endmacro
+
+|.macro SET_ZVAL_W2, addr, val
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val
+|.endmacro
+
+|.macro FPU_OP, fp_ins, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| MEM_OP1 fp_ins, qword, Z_ZV(addr), r0
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| fp_ins qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro FPU_GET_ZVAL_DVAL, addr
+| FPU_OP fld, addr
+|.endmacro
+
+|.macro FPU_MATH, opcode, addr
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| FPU_OP fadd, addr
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| FPU_OP fsub, addr
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| FPU_OP fmul, addr
+|| break;
+|| case ZEND_DIV:
+|| case ZEND_ASSIGN_DIV:
+| FPU_OP fdiv, addr
+|| break;
+|| }
+|.endmacro
+
+|.macro FPU_SET_ZVAL_DVAL, addr
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| fstp qword [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)]
+|.endmacro
+
+|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| avx_ins op1, op2
+|| } else {
+| sse_ins op1, op2
+|| }
+|.endmacro
+
+|.macro SSE_OP, sse_ins, reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), r0
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| .if X64
+|| if (IS_32BIT(Z_ZV(addr))) {
+| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
+|| } else {
+| LOAD_ADDR r0, Z_ZV(addr)
+| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0]
+|| }
+| .else
+| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
+| .endif
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro SSE_GET_ZVAL_LVAL, reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+|| if (Z_LVAL_P(Z_ZV(addr)) == 0) {
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
+|| } else {
+| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
+|| }
+|| } else {
+|.if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
+| mov64 r0, Z_LVAL_P(Z_ZV(addr))
+|| } else {
+| mov r0, Z_LVAL_P(Z_ZV(addr))
+|| }
+|.else
+| mov r0, Z_LVAL_P(Z_ZV(addr))
+|.endif
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0
+|| } else {
+| cvtsi2sd, xmm(reg-ZREG_XMM0), r0
+|| }
+|| }
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else {
+| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| }
+|| } else if (Z_MODE(addr) == IS_REG) {
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
+|| } else {
+| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
+|| }
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro SSE_GET_ZVAL_DVAL, reg, addr
+|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) {
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| .if X64
+|| if (IS_32BIT(Z_ZV(addr))) {
+| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
+|| } else {
+| LOAD_ADDR r0, Z_ZV(addr)
+| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0]
+|| }
+| .else
+| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
+| .endif
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+| SSE_AVX_INS movsd, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|| }
+|.endmacro
+
+|.macro SSE_MATH, opcode, reg, addr
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| SSE_OP addsd, reg, addr
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| SSE_OP subsd, reg, addr
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| SSE_OP mulsd, reg, addr
+|| break;
+|| case ZEND_DIV:
+|| case ZEND_ASSIGN_DIV:
+| SSE_OP divsd, reg, addr
+|| break;
+|| }
+|.endmacro
+
+|.macro SSE_MATH_REG, opcode, dst_reg, src_reg
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| case ZEND_DIV:
+|| case ZEND_ASSIGN_DIV:
+| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| }
+|.endmacro
+
+|.macro SSE_SET_ZVAL_DVAL, addr, reg
+|| if (Z_MODE(addr) == IS_REG) {
+|| if (reg != Z_REG(addr)) {
+| SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0)
+|| }
+|| } else {
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0)
+|| }
+|.endmacro
+
+|.macro AVX_OP, avx_ins, reg, op1_reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), r0
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro AVX_MATH, opcode, reg, op1_reg, addr
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| AVX_OP vaddsd, reg, op1_reg, addr
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| AVX_OP vsubsd, reg, op1_reg, addr
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| AVX_OP vmulsd, reg, op1_reg, addr
+|| break;
+|| case ZEND_DIV:
+|| case ZEND_ASSIGN_DIV:
+| AVX_OP vdivsd, reg, op1_reg, addr
+|| break;
+|| }
+|.endmacro
+
+|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| case ZEND_DIV:
+|| case ZEND_ASSIGN_DIV:
+| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
+|| break;
+|| }
+|.endmacro
+
+|.macro LONG_OP, long_ins, reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| .if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
+|| if (reg != ZREG_R0) {
+| mov64 r0, Z_LVAL_P(Z_ZV(addr))
+| long_ins Ra(reg), r0
+|| } else {
+| mov64 r1, Z_LVAL_P(Z_ZV(addr))
+| long_ins Ra(reg), r1
+|| }
+|| } else {
+| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr))
+|| }
+| .else
+| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr))
+| .endif
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+| long_ins Ra(reg), Ra(Z_REG(addr))
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval
+|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) {
+| .if X64
+|| if (!IS_SIGNED_32BIT(lval)) {
+| mov64 r0, lval
+| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0
+|| } else {
+| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
+|| }
+| .else
+| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
+| .endif
+|| } else if (Z_MODE(op1_addr) == IS_REG) {
+| .if X64
+|| if (!IS_SIGNED_32BIT(lval)) {
+| mov64 r0, lval
+| long_ins Ra(Z_REG(op1_addr)), r0
+|| } else {
+| long_ins Ra(Z_REG(op1_addr)), lval
+|| }
+| .else
+| long_ins Ra(Z_REG(op1_addr)), lval
+| .endif
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro GET_ZVAL_LVAL, reg, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+|| if (Z_LVAL_P(Z_ZV(addr)) == 0) {
+| xor Ra(reg), Ra(reg)
+|| } else {
+| .if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
+| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr))
+|| } else {
+| mov Ra(reg), Z_LVAL_P(Z_ZV(addr))
+|| }
+| .else
+| mov Ra(reg), Z_LVAL_P(Z_ZV(addr))
+| .endif
+|| }
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+|| if (reg != Z_REG(addr)) {
+| mov Ra(reg), Ra(Z_REG(addr))
+|| }
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro LONG_MATH, opcode, reg, addr
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| LONG_OP add, reg, addr
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| LONG_OP sub, reg, addr
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| LONG_OP imul, reg, addr
+|| break;
+|| case ZEND_BW_OR:
+|| case ZEND_ASSIGN_BW_OR:
+| LONG_OP or, reg, addr
+|| break;
+|| case ZEND_BW_AND:
+|| case ZEND_ASSIGN_BW_AND:
+| LONG_OP and, reg, addr
+|| break;
+|| case ZEND_BW_XOR:
+|| case ZEND_ASSIGN_BW_XOR:
+| LONG_OP xor, reg, addr
+|| break;
+|| default:
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro LONG_MATH_REG, opcode, dst_reg, src_reg
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| add dst_reg, src_reg
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| sub dst_reg, src_reg
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| imul dst_reg, src_reg
+|| break;
+|| case ZEND_BW_OR:
+|| case ZEND_ASSIGN_BW_OR:
+| or dst_reg, src_reg
+|| break;
+|| case ZEND_BW_AND:
+|| case ZEND_ASSIGN_BW_AND:
+| and dst_reg, src_reg
+|| break;
+|| case ZEND_BW_XOR:
+|| case ZEND_ASSIGN_BW_XOR:
+| xor dst_reg, src_reg
+|| break;
+|| default:
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro SET_ZVAL_LVAL, addr, lval
+|| if (Z_MODE(addr) == IS_REG) {
+| mov Ra(Z_REG(addr)), lval
+|| } else {
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval
+|| }
+|.endmacro
+
+|.macro FPU_LONG_OP, fp_ins, addr
+|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
+| MEM_OP1 fp_ins, aword, Z_ZV(addr), r0
+|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
+| fp_ins aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
+|| } else if (Z_MODE(addr) == IS_REG) {
+| fp_ins Ra(Z_REG(addr))
+|| } else {
+|| ZEND_ASSERT(0);
+|| }
+|.endmacro
+
+|.macro FPU_GET_ZVAL_LVAL, addr
+| FPU_LONG_OP fild, addr
+|.endmacro
+
+|.macro FPU_MATH_REG, opcode, reg
+|| switch (opcode) {
+|| case ZEND_ADD:
+|| case ZEND_ASSIGN_ADD:
+| fadd reg
+|| break;
+|| case ZEND_SUB:
+|| case ZEND_ASSIGN_SUB:
+| fsub reg
+|| break;
+|| case ZEND_MUL:
+|| case ZEND_ASSIGN_MUL:
+| fmul reg
+|| break;
+|| case ZEND_DIV:
+|| case ZEND_ASSIGN_DIV:
+| fdiv reg
+|| break;
+|| }
+|.endmacro
+
+|.macro ZVAL_COPY_CONST, dst_addr, dst_info, zv, tmp_reg
+|| if (Z_TYPE_P(zv) > IS_TRUE) {
+|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
+|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0;
+| .if X64 or SSE
+|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
+|| } else {
+| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
+|| }
+| .if X64
+|| } else if (!IS_32BIT(zv)) {
+| mov64 tmp_reg, ((uintptr_t)zv)
+| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg]
+| .endif
+|| } else {
+| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)]
+|| }
+| SSE_SET_ZVAL_DVAL dst_addr, dst_reg
+| .else
+|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
+| fldz
+|| } else if (Z_DVAL_P(zv) == 1.0) {
+| fld1
+|| } else {
+| fld qword [zv]
+|| }
+| FPU_SET_ZVAL_DVAL dst_addr
+| .endif
+|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) {
+| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr))
+|| } else {
+| .if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
+|| if (Z_MODE(dst_addr) == IS_REG) {
+| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv))
+|| } else {
+| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
+| SET_ZVAL_LVAL dst_addr, tmp_reg
+|| }
+|| } else {
+| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
+|| }
+| .else
+| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
+| .endif
+|| }
+|| }
+|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
+|| if (((dst_info & MAY_BE_ANY) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
+| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
+|| }
+|| }
+|.endmacro
+
+|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, zv, tmp_reg
+|| if (Z_TYPE_P(zv) > IS_TRUE) {
+|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
+|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ?
+|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0);
+| .if X64 or SSE
+|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
+|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
+|| } else {
+| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
+|| }
+| .if X64
+|| } else if (!IS_32BIT(zv)) {
+| mov64 tmp_reg, ((uintptr_t)zv)
+| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg]
+| .endif
+|| } else {
+| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)]
+|| }
+| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
+| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
+| .else
+|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
+| fldz
+|| } else if (Z_DVAL_P(zv) == 1.0) {
+| fld1
+|| } else {
+| fld qword [zv]
+|| }
+| FPU_SET_ZVAL_DVAL dst_addr
+|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
+| fldz
+|| } else if (Z_DVAL_P(zv) == 1.0) {
+| fld1
+|| } else {
+| fld qword [zv]
+|| }
+| FPU_SET_ZVAL_DVAL res_addr
+| .endif
+|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) {
+|| if (Z_MODE(dst_addr) == IS_REG) {
+| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr))
+| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
+|| } else {
+| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr))
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
+|| }
+|| } else {
+| .if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
+|| if (Z_MODE(dst_addr) == IS_REG) {
+| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv))
+| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
+|| } else if (Z_MODE(res_addr) == IS_REG) {
+| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv))
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
+|| } else {
+| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
+| SET_ZVAL_LVAL dst_addr, tmp_reg
+| SET_ZVAL_LVAL res_addr, tmp_reg
+|| }
+|| } else if (Z_MODE(dst_addr) == IS_REG) {
+| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
+| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
+|| } else if (Z_MODE(res_addr) == IS_REG) {
+| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
+|| } else {
+| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
+| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
+|| }
+| .else
+|| if (Z_MODE(dst_addr) == IS_REG) {
+| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
+| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
+|| } else if (Z_MODE(res_addr) == IS_REG) {
+| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
+|| } else {
+| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
+| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
+|| }
+| .endif
+|| }
+|| }
+|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
+|| if (((dst_info & MAY_BE_ANY) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
+| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
+|| }
+|| }
+|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv)
+|| }
+|.endmacro
+
+/* the same as above, but "src" may overlap with "tmp_reg1" */
+|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
+|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
+|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
+|| if (Z_MODE(src_addr) == IS_REG) {
+|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr))
+|| }
+|| } else if (Z_MODE(dst_addr) == IS_REG) {
+| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
+|| } else {
+| GET_ZVAL_LVAL tmp_reg2, src_addr
+| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
+|| }
+|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+| .if X64 or SSE
+|| if (Z_MODE(src_addr) == IS_REG) {
+| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr)
+|| } else if (Z_MODE(dst_addr) == IS_REG) {
+| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr
+|| } else {
+| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr
+| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
+|| }
+| .else
+| FPU_GET_ZVAL_DVAL src_addr
+| FPU_SET_ZVAL_DVAL dst_addr
+| .endif
+|| } else if (!(src_info & MAY_BE_DOUBLE)) {
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+|| } else {
+| .if X64
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+| .else
+|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) {
+| GET_ZVAL_W2 Ra(tmp_reg2), src_addr
+| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+|| } else {
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| GET_ZVAL_W2 Ra(tmp_reg1), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
+|| }
+| .endif
+|| }
+|| }
+|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+|| has_concrete_type(src_info & MAY_BE_ANY)) {
+|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
+|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
+|| zend_uchar type = concrete_type(src_info);
+| SET_ZVAL_TYPE_INFO dst_addr, type
+|| }
+|| }
+|| } else {
+| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr
+| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1)
+|| }
+|.endmacro
+
+|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2
+|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
+|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
+|| if (Z_MODE(src_addr) == IS_REG) {
+|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr))
+|| }
+|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) {
+| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr))
+|| }
+|| } else if (Z_MODE(dst_addr) == IS_REG) {
+| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
+|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) {
+| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
+|| }
+|| } else if (Z_MODE(res_addr) == IS_REG) {
+| GET_ZVAL_LVAL Z_REG(res_addr), src_addr
+| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
+|| } else {
+| GET_ZVAL_LVAL tmp_reg2, src_addr
+| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2)
+|| }
+|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+| .if X64 or SSE
+|| if (Z_MODE(src_addr) == IS_REG) {
+| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr)
+| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr)
+|| } else if (Z_MODE(dst_addr) == IS_REG) {
+| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr
+| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr)
+|| } else if (Z_MODE(res_addr) == IS_REG) {
+| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr
+| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr)
+|| } else {
+| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr
+| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
+| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
+|| }
+| .else
+| FPU_GET_ZVAL_DVAL src_addr
+| FPU_STROE dst_addr
+| FPU_GET_ZVAL_DVAL src_addr
+| FPU_SET_ZVAL_DVAL res_addr
+| .endif
+|| } else if (!(src_info & MAY_BE_DOUBLE)) {
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
+|| } else {
+| .if X64
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
+| .else
+|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) {
+| GET_ZVAL_W2 Ra(tmp_reg2), src_addr
+| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_W2 res_addr, Ra(tmp_reg2)
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
+|| } else {
+| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
+| GET_ZVAL_W2 Ra(tmp_reg1), src_addr
+| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
+| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
+| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
+| SET_ZVAL_W2 res_addr, Ra(tmp_reg1)
+|| }
+| .endif
+|| }
+|| }
+|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+|| has_concrete_type(src_info & MAY_BE_ANY)) {
+|| zend_uchar type = concrete_type(src_info);
+|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
+|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
+| SET_ZVAL_TYPE_INFO dst_addr, type
+|| }
+|| }
+|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+| SET_ZVAL_TYPE_INFO res_addr, type
+|| }
+|| } else {
+| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr
+| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1)
+| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1)
+|| }
+|.endmacro
+
+|.macro IF_TYPE, type, val, label
+| cmp type, val
+| je label
+|.endmacro
+
+|.macro IF_NOT_TYPE, type, val, label
+| cmp type, val
+| jne label
+|.endmacro
+
+|.macro IF_Z_TYPE, zv, val, label
+| IF_TYPE byte [zv + offsetof(zval, u1.v.type)], val, label
+|.endmacro
+
+|.macro IF_NOT_Z_TYPE, zv, val, label
+| IF_NOT_TYPE byte [zv + offsetof(zval, u1.v.type)], val, label
+|.endmacro
+
+|.macro CMP_ZVAL_TYPE, addr, val
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val
+|.endmacro
+
+|.macro IF_ZVAL_TYPE, addr, val, label
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label
+|.endmacro
+
+|.macro IF_NOT_ZVAL_TYPE, addr, val, label
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label
+|.endmacro
+
+|.macro IF_FLAGS, type_flags, mask, label
+| test type_flags, mask
+| jnz label
+|.endmacro
+
+|.macro IF_NOT_FLAGS, type_flags, mask, label
+| test type_flags, mask
+| jz label
+|.endmacro
+
+|.macro IF_REFCOUNTED, type_flags, label
+| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label
+|.endmacro
+
+|.macro IF_NOT_REFCOUNTED, type_flags, label
+| IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label
+|.endmacro
+
+|.macro IF_ZVAL_FLAGS, addr, mask, label
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label
+|.endmacro
+
+|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label
+|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
+| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label
+|.endmacro
+
+|.macro IF_ZVAL_REFCOUNTED, addr, label
+| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label
+|.endmacro
+
+|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label
+| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label
+|.endmacro
+
+|.macro GC_ADDREF, zv
+| add dword [zv], 1
+|.endmacro
+
+|.macro GC_DELREF, zv
+| sub dword [zv], 1
+|.endmacro
+
+|.macro IF_GC_MAY_NOT_LEAK, ptr, tmp_reg, label
+| mov tmp_reg, dword [ptr + 4]
+| and tmp_reg, (GC_INFO_MASK | (GC_COLLECTABLE << GC_FLAGS_SHIFT))
+| cmp tmp_reg, (GC_COLLECTABLE << GC_FLAGS_SHIFT)
+| jne label
+|.endmacro
+
+|.macro ADDREF_CONST, zv, tmp_reg
+| .if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
+| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
+| add dword [tmp_reg], 1
+|| } else {
+| add dword [Z_LVAL_P(zv)], 1
+|| }
+| .else
+| add dword [Z_LVAL_P(zv)], 1
+| .endif
+|.endmacro
+
+|.macro ADDREF_CONST_2, zv, tmp_reg
+| .if X64
+|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
+| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
+| add dword [tmp_reg], 2
+|| } else {
+| add dword [Z_LVAL_P(zv)], 2
+|| }
+| .else
+| add dword [Z_LVAL_P(zv)], 2
+| .endif
+|.endmacro
+
+|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg
+|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+| IF_NOT_REFCOUNTED type_flags_reg, >1
+|| }
+| GC_ADDREF value_ptr_reg
+|1:
+|| }
+|.endmacro
+
+|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg
+|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+| IF_NOT_REFCOUNTED type_flags_reg, >1
+|| }
+| add dword [value_ptr_reg], 2
+|1:
+|| }
+|.endmacro
+
+|.macro ZVAL_DEREF, reg, info
+|| if (info & MAY_BE_REF) {
+| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1
+| GET_Z_PTR reg, reg
+| add reg, offsetof(zend_reference, val)
+|1:
+|| }
+|.endmacro
+
+|.macro SAVE_VALID_OPLINE, op
+|| if (op == last_valid_opline) {
+| SAVE_OPLINE
+|| } else {
+| ADDR_OP2_2 mov, aword EX->opline, op, r0
+|| }
+|.endmacro
+
+// zval should be in FCARG1a
+|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a
+|| do {
+|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
+|| if (type == IS_STRING && !ZEND_DEBUG) {
+| EXT_CALL _efree, r0
+|| break;
+|| } else if (type == IS_ARRAY) {
+|| if (opline) {
+| SAVE_VALID_OPLINE opline
+|| }
+| EXT_CALL zend_array_destroy, r0
+|| break;
+|| } else if (type == IS_OBJECT) {
+|| if (opline) {
+| SAVE_VALID_OPLINE opline
+|| }
+| EXT_CALL zend_objects_store_del, r0
+|| break;
+|| }
+|| }
+|| if (opline) {
+| SAVE_VALID_OPLINE opline
+|| }
+| EXT_CALL rc_dtor_func, r0
+|| } while(0);
+|.endmacro
+
+|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, safe, opline
+|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+| // if (Z_REFCOUNTED_P(cv)) {
+|| if (cold) {
+| IF_ZVAL_REFCOUNTED addr, >1
+|.cold_code
+|1:
+|| } else {
+| IF_NOT_ZVAL_REFCOUNTED addr, >4
+|| }
+|| }
+| // if (!Z_DELREF_P(cv)) {
+| GET_ZVAL_PTR FCARG1a, addr
+| GC_DELREF FCARG1a
+|| if (RC_MAY_BE_1(op_info)) {
+|| if (RC_MAY_BE_N(op_info)) {
+|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
+| jnz >3
+|| } else {
+| jnz >4
+|| }
+|| }
+|| if (safe) {
+| // ZVAL_NULL(cv);
+| SET_ZVAL_TYPE_INFO addr, IS_NULL
+|| }
+| // zval_dtor_func(r);
+| ZVAL_DTOR_FUNC op_info, opline
+|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
+| jmp >4
+|| }
+|3:
+|| }
+|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
+|| if ((op_info) & MAY_BE_REF) {
+|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val));
+| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1
+| IF_NOT_ZVAL_REFCOUNTED ref_addr, >4
+| GET_ZVAL_PTR FCARG1a, ref_addr
+|1:
+|| }
+| IF_GC_MAY_NOT_LEAK FCARG1a, eax, >4
+| // gc_possible_root(Z_COUNTED_P(z))
+| EXT_CALL gc_possible_root, r0
+|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) {
+| jmp >4
+|.code
+|| }
+|| } else if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) {
+| jmp >4
+|.code
+|| }
+|4:
+|| }
+|.endmacro
+
+|.macro FREE_OP, op_type, op, op_info, cold, op_array, opline
+|| if (op_type & (IS_VAR|IS_TMP_VAR)) {
+| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, 0, opline
+|| }
+|.endmacro
+
+|.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold
+|| if ((op_info & MAY_BE_ARRAY) && RC_MAY_BE_N(op_info)) {
+|| if (cold) {
+| IF_ZVAL_TYPE addr, IS_ARRAY, >1
+|.cold_code
+|1:
+|| } else {
+| IF_NOT_ZVAL_TYPE addr, IS_ARRAY, >2
+|| }
+| GET_ZVAL_PTR r0, addr
+|| if (RC_MAY_BE_1(op_info)) {
+| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1)
+| jbe >2
+|| }
+| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, >1
+| GC_DELREF r0
+|1:
+|| if (Z_REG(addr) == ZREG_FCARG1a) {
+| mov aword T1, FCARG1a // save
+|| } else {
+| LOAD_ZVAL_ADDR FCARG1a, addr
+|| }
+| EXT_CALL zval_copy_ctor_func, r0
+|| if (Z_REG(addr) == ZREG_FCARG1a) {
+| mov FCARG1a, aword T1 // restore
+|| }
+|| if (cold) {
+| jmp >2
+|.code
+|| }
+|2:
+|| }
+|.endmacro
+
+|.macro SEPARATE_ARRAY, addr, op_info, cold
+|| if (RC_MAY_BE_N(op_info)) {
+|| zend_reg tmp_reg;
+||
+|| tmp_reg = (Z_REG(addr) == ZREG_FCARG1a) ? ZREG_R0 : ZREG_FCARG1a;
+| GET_ZVAL_LVAL tmp_reg, addr
+|| if (RC_MAY_BE_1(op_info)) {
+| cmp dword [Ra(tmp_reg)], 1 // if (GC_REFCOUNTED() > 1)
+|| if (cold) {
+| ja >1
+|.cold_code
+|1:
+|| } else {
+| jbe >2
+|| }
+|| }
+| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, >1
+| GC_DELREF Ra(tmp_reg)
+|1:
+|| if (Z_REG(addr) == ZREG_FCARG1a) {
+| mov aword T1, FCARG1a // save
+|| } else {
+| LOAD_ZVAL_ADDR FCARG1a, addr
+|| }
+| EXT_CALL zval_copy_ctor_func, r0
+|| if (Z_REG(addr) == ZREG_FCARG1a) {
+| mov FCARG1a, aword T1 // restore
+|| }
+|| if (RC_MAY_BE_1(op_info)) {
+|| if (cold) {
+| jmp >2
+|.code
+|| }
+|| }
+|2:
+|| }
+| GET_ZVAL_LVAL ZREG_FCARG1a, addr
+|.endmacro
+
+|.macro EFREE_REG_24, op_array, opline
+||#if ZEND_DEBUG
+|| const char *filename = op_array->filename ? op_array->filename->val : NULL;
+| LOAD_ADDR FCARG2a, filename
+| .if X64WIN
+| mov CARG3d, opline->lineno
+| xor CARG4, CARG4
+| mov aword A5, 0
+| EXT_CALL _efree, r0
+| .elif X64
+| mov CARG3d, opline->lineno
+| xor CARG4, CARG4
+| xor CARG5, CARG5
+| EXT_CALL _efree, r0
+| .else
+| sub r4, 4
+| push 0
+| push 0
+| push opline->lineno
+| EXT_CALL _efree, r0
+| add r4, 4
+| .endif
+||#else
+||#ifdef HAVE_BUILTIN_CONSTANT_P
+| EXT_CALL _efree_24, r0
+||#else
+| EXT_CALL _efree, r0
+||#endif
+||#endif
+|.endmacro
+
+|.macro EFREE_24, ptr, op_array, opline
+| mov FCARG1a, ptr
+| EFREE_REG_24 op_array, opline
+|.endmacro
+
+|.macro EMALLOC, size, op_array, opline
+||#if ZEND_DEBUG
+|| const char *filename = op_array->filename ? op_array->filename->val : NULL;
+| mov FCARG1a, size
+| LOAD_ADDR FCARG2a, filename
+| .if X64WIN
+| mov CARG3d, opline->lineno
+| xor CARG4, CARG4
+| mov aword A5, 0
+| EXT_CALL _emalloc, r0
+| .elif X64
+| mov CARG3d, opline->lineno
+| xor CARG4, CARG4
+| xor CARG5, CARG5
+| EXT_CALL _emalloc, r0
+| .else
+| sub r4, 4
+| push 0
+| push 0
+| push opline->lineno
+| EXT_CALL _emalloc, r0
+| add r4, 4
+| .endif
+||#else
+||#ifdef HAVE_BUILTIN_CONSTANT_P
+|| if (size == 24) {
+| EXT_CALL _emalloc_24, r0
+|| } else {
+| mov FCARG1a, size
+| EXT_CALL _emalloc, r0
+|| }
+||#else
+| mov FCARG1a, size
+| EXT_CALL _emalloc, r0
+||#endif
+||#endif
+|.endmacro
+
+|.macro OBJ_RELEASE, reg, tmp_reg, exit_label
+| GC_DELREF reg
+| jne >1
+| // zend_objects_store_del(obj);
+| mov FCARG1a, reg
+| EXT_CALL zend_objects_store_del, r0
+| jmp exit_label
+|1:
+| IF_GC_MAY_NOT_LEAK reg, tmp_reg, >1
+| // gc_possible_root(obj)
+| mov FCARG1a, reg
+| EXT_CALL gc_possible_root, r0
+|1:
+|.endmacro
+
+|.macro UNDEFINED_OFFSET, opline
+|| if (opline == last_valid_opline) {
+| call ->undefined_offset_ex
+|| } else {
+| SAVE_VALID_OPLINE, opline
+| call ->undefined_offset
+|| }
+|.endmacro
+
+|.macro UNDEFINED_INDEX, opline
+|| if (opline == last_valid_opline) {
+| call ->undefined_index_ex
+|| } else {
+| SAVE_VALID_OPLINE, opline
+| call ->undefined_index
+|| }
+|.endmacro
+
+|.macro CANNOT_ADD_ELEMENT, opline
+|| if (opline == last_valid_opline) {
+| call ->cannot_add_element_ex
+|| } else {
+| SAVE_VALID_OPLINE, opline
+| call ->cannot_add_element
+|| }
+|.endmacro
+
+static zend_bool reuse_ip;
+static zend_bool delayed_call_chain;
+static uint32_t delayed_call_level;
+static const zend_op *last_valid_opline;
+static int jit_return_label;
+
+/* bit helpers */
+
+/* from http://aggregate.org/MAGIC/ */
+static uint32_t ones32(uint32_t x)
+{
+ x -= ((x >> 1) & 0x55555555);
+ x = (((x >> 2) & 0x33333333) + (x & 0x33333333));
+ x = (((x >> 4) + x) & 0x0f0f0f0f);
+ x += (x >> 8);
+ x += (x >> 16);
+ return x & 0x0000003f;
+}
+
+static uint32_t floor_log2(uint32_t x)
+{
+ x |= (x >> 1);
+ x |= (x >> 2);
+ x |= (x >> 4);
+ x |= (x >> 8);
+ x |= (x >> 16);
+ return ones32(x) - 1;
+}
+
+static zend_bool is_power_of_two(uint32_t x)
+{
+ return !(x & (x - 1));
+}
+
+static zend_bool has_concrete_type(uint32_t value_type)
+{
+ if (value_type & MAY_BE_UNDEF) {
+ value_type |= MAY_BE_NULL;
+ }
+ value_type &= MAY_BE_ANY;
+ return is_power_of_two (value_type);
+}
+
+static uint32_t concrete_type(uint32_t value_type)
+{
+ return floor_log2(value_type & MAY_BE_ANY);
+}
+
+static inline zend_bool is_signed(double d)
+{
+ return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0;
+}
+
+static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
+{
+ |->interrupt_handler:
+ | //EG(vm_interrupt) = 0;
+ | MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0
+ | //if (EG(timed_out)) {
+ | MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0
+ | je >1
+ | //zend_timeout(0);
+ | xor FCARG1d, FCARG1d
+ | EXT_CALL zend_timeout, r0
+ |1:
+ | //} else if (zend_interrupt_function) {
+ if (zend_interrupt_function) {
+ | SAVE_OPLINE
+ | //zend_interrupt_function(execute_data);
+ |.if X64
+ | mov CARG1, FP
+ | EXT_CALL zend_interrupt_function, r0
+ |.else
+ | sub r4, 12
+ | push FP
+ | EXT_CALL zend_interrupt_function, r0
+ | add r4, 16
+ |.endif
+ | //ZEND_VM_ENTER();
+ | //execute_data = EG(current_execute_data);
+ | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0
+ | LOAD_OPLINE
+ }
+ | //ZEND_VM_CONTINUE()
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | JMP_IP
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 1 // ZEND_VM_ENTER
+ | ret
+ }
+
+ return 1;
+}
+
+static int zend_jit_exception_handler_stub(dasm_State **Dst)
+{
+ |->exception_handler:
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ const void *handler = zend_get_opcode_handler_func(EG(exception_op));
+
+ | add r4, HYBRID_SPAD
+ | EXT_CALL handler, r0
+ | JMP_IP
+ } else {
+ const void *handler = EG(exception_op)->handler;
+
+ if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ } else {
+ | mov FCARG1a, FP
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ }
+ | EXT_JMP handler, r0
+ }
+
+ return 1;
+}
+
+static int zend_jit_exception_handler_undef_stub(dasm_State **Dst)
+{
+ |->exception_handler_undef:
+ | SET_Z_TYPE_INFO FP + r0, IS_UNDEF
+ | jmp ->exception_handler
+
+ return 1;
+}
+
+static int zend_jit_leave_function_stub(dasm_State **Dst)
+{
+ |->leave_function_handler:
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | test FCARG1d, ZEND_CALL_TOP
+ | jnz >1
+ | EXT_CALL zend_jit_leave_nested_func_helper, r0
+ | add r4, HYBRID_SPAD // stack alignment
+ | JMP_IP
+ |1:
+ | EXT_CALL zend_jit_leave_top_func_helper, r0
+ | add r4, HYBRID_SPAD // stack alignment
+ | JMP_IP
+ } else {
+ if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD
+ } else {
+ | mov FCARG2a, FP
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD
+ }
+ | test FCARG1d, ZEND_CALL_TOP
+ | jnz >1
+ | EXT_JMP zend_jit_leave_nested_func_helper, r0
+ |1:
+ | EXT_JMP zend_jit_leave_top_func_helper, r0
+ }
+
+ return 1;
+}
+
+static int zend_jit_leave_throw_stub(dasm_State **Dst)
+{
+ |->leave_throw_handler:
+ | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
+ if (GCC_GLOBAL_REGS) {
+ | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION
+ | je >5
+ | // EG(opline_before_exception) = opline;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0
+ |5:
+ | // opline = EG(exception_op);
+ | LOAD_IP_ADDR_ZTS executor_globals, exception_op
+ | // HANDLE_EXCEPTION()
+ | jmp ->exception_handler
+ } else {
+ | GET_IP FCARG1a
+ | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION
+ | je >5
+ | // EG(opline_before_exception) = opline;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0
+ |5:
+ | // opline = EG(exception_op);
+ | LOAD_IP_ADDR_ZTS executor_globals, exception_op
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 2 // ZEND_VM_LEAVE
+ | ret
+ }
+
+ return 1;
+}
+
+static int zend_jit_icall_throw_stub(dasm_State **Dst)
+{
+ |->icall_throw_handler:
+ | // zend_throw_exception_internal(NULL);
+ |.if X64
+ | xor CARG1, CARG1
+ | EXT_CALL zend_throw_exception_internal, r0
+ |.else
+ | sub r4, 12
+ | push 0
+ | EXT_CALL zend_throw_exception_internal, r0
+ | add r4, 16
+ |.endif
+ | // HANDLE_EXCEPTION()
+ | jmp ->exception_handler
+
+ return 1;
+}
+
+static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst)
+{
+ |->throw_cannot_pass_by_ref:
+ | mov r0, EX->opline
+ |.if X64
+ | movsxd r1, dword OP:r0->result.var
+ |.else
+ | mov r1, OP:r0->result.var
+ |.endif
+ | SET_Z_TYPE_INFO RX+r1, IS_UNDEF
+ | mov RX, r0
+ |.if X64
+ | xor CARG1, CARG1
+ | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference"
+ | mov CARG3d, dword OP:r0->op2.num
+ | EXT_CALL zend_throw_error, r0
+ |.else
+ | mov r1, dword OP:r0->op2.num
+ | sub r4, 4
+ | push r1
+ | push "Cannot pass parameter %d by reference"
+ | push 0
+ | EXT_CALL zend_throw_error, r0
+ | add r4, 16
+ |.endif
+ | cmp byte OP:RX->op1_type, IS_TMP_VAR
+ | jne >9
+ |.if X64
+ | movsxd r0, dword OP:RX->op1.var
+ |.else
+ | mov r0, OP:RX->op1.var
+ |.endif
+ | add r0, FP
+ | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL
+ |9:
+ | jmp ->exception_handler
+
+ return 1;
+}
+
+static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst)
+{
+ |->undefined_offset_ex:
+ | SAVE_OPLINE
+ | jmp ->undefined_offset
+
+ return 1;
+}
+
+static int zend_jit_undefined_offset_stub(dasm_State **Dst)
+{
+ |->undefined_offset:
+ |.if X64WIN
+ | sub r4, 0x28
+ |.elif X64
+ | sub r4, 8
+ |.else
+ | sub r4, 12
+ |.endif
+ | mov r0, EX->opline
+ |.if X64
+ | movsxd r1, dword OP:r0->result.var
+ |.else
+ | mov r1, OP:r0->result.var
+ |.endif
+ | cmp byte OP:r0->op2_type, IS_CONST
+ | SET_Z_TYPE_INFO FP + r1, IS_NULL
+ | jne >2
+ |.if X64
+ | movsxd r1, dword OP:r0->op2.constant
+ | add r0, r1
+ |.else
+ | mov r0, aword OP:r0->op2.zv
+ |.endif
+ | jmp >3
+ |2:
+ |.if X64
+ | movsxd r0, dword OP:r0->op2.var
+ |.else
+ | mov r0, OP:r0->op2.var
+ |.endif
+ | add r0, FP
+ |3:
+ |.if X64WIN
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT
+ | mov CARG3, aword [r0]
+ | EXT_CALL zend_error, r0
+ | add r4, 0x28 // stack alignment
+ |.elif X64
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT
+ | mov CARG3, aword [r0]
+ | EXT_CALL zend_error, r0
+ | add r4, 8 // stack alignment
+ |.else
+ | sub r4, 4
+ | push aword [r0]
+ | push "Undefined offset: " ZEND_LONG_FMT
+ | push E_NOTICE
+ | EXT_CALL zend_error, r0
+ | add r4, 28
+ |.endif
+ | ret
+
+ return 1;
+}
+
+static int zend_jit_undefined_index_ex_stub(dasm_State **Dst)
+{
+ |->undefined_index_ex:
+ | SAVE_OPLINE
+ | jmp ->undefined_index
+
+ return 1;
+}
+
+static int zend_jit_undefined_index_stub(dasm_State **Dst)
+{
+ |->undefined_index:
+ |.if X64WIN
+ | sub r4, 0x28
+ |.elif X64
+ | sub r4, 8
+ |.else
+ | sub r4, 12
+ |.endif
+ | mov r0, EX->opline
+ |.if X64
+ | movsxd r1, dword OP:r0->result.var
+ |.else
+ | mov r1, OP:r0->result.var
+ |.endif
+ | cmp byte OP:r0->op2_type, IS_CONST
+ | SET_Z_TYPE_INFO FP + r1, IS_NULL
+ | jne >2
+ |.if X64
+ | movsxd r1, dword OP:r0->op2.constant
+ | add r0, r1
+ |.else
+ | mov r0, aword OP:r0->op2.zv
+ |.endif
+ | jmp >3
+ |2:
+ |.if X64
+ | movsxd r0, dword OP:r0->op2.var
+ |.else
+ | mov r0, OP:r0->op2.var
+ |.endif
+ | add r0, FP
+ |3:
+ |.if X64WIN
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Undefined index: %s"
+ | mov CARG3, aword [r0]
+ | add CARG3, offsetof(zend_string, val)
+ | EXT_CALL zend_error, r0
+ | add r4, 0x28
+ |.elif X64
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Undefined index: %s"
+ | mov CARG3, aword [r0]
+ | add CARG3, offsetof(zend_string, val)
+ | EXT_CALL zend_error, r0
+ | add r4, 8
+ |.else
+ | sub r4, 4
+ | mov r0, aword [r0]
+ | add r0, offsetof(zend_string, val)
+ | push r0
+ | push "Undefined index: %s"
+ | push E_NOTICE
+ | EXT_CALL zend_error, r0
+ | add r4, 28
+ |.endif
+ | ret
+
+ return 1;
+}
+
+static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst)
+{
+ |->cannot_add_element_ex:
+ | SAVE_OPLINE
+ | jmp ->cannot_add_element
+
+ return 1;
+}
+
+static int zend_jit_cannot_add_element_stub(dasm_State **Dst)
+{
+ |->cannot_add_element:
+ |.if X64WIN
+ | sub r4, 0x28
+ |.elif X64
+ | sub r4, 8
+ |.else
+ | sub r4, 12
+ |.endif
+ | mov r0, EX->opline
+ | cmp byte OP:r0->result_type, IS_UNUSED
+ | jz >1
+ |.if X64
+ | movsxd r0, dword OP:r0->result.var
+ |.else
+ | mov r0, OP:r0->result.var
+ |.endif
+ | SET_Z_TYPE_INFO FP + r0, IS_NULL
+ |1:
+ |.if X64WIN
+ | mov CARG1, E_WARNING
+ | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
+ | EXT_CALL zend_error, r0
+ | add r4, 0x28
+ |.elif X64
+ | mov CARG1, E_WARNING
+ | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
+ | EXT_CALL zend_error, r0
+ | add r4, 8
+ |.else
+ | sub r4, 8
+ | push "Cannot add element to the array as the next element is already occupied"
+ | push E_WARNING
+ | EXT_CALL zend_error, r0
+ | add r4, 28
+ |.endif
+ | ret
+
+ return 1;
+}
+
+static int zend_jit_not_obj_stub(dasm_State **Dst)
+{
+ |->not_obj:
+ |.if X64
+ | xor CARG1, CARG1
+ | LOAD_ADDR CARG2, "Using $this when not in object context"
+ | EXT_CALL zend_throw_error, r0
+ |.else
+ | sub r4, 8
+ | push "Using $this when not in object context"
+ | push 0
+ | EXT_CALL zend_throw_error, r0
+ | add r4, 16
+ |.endif
+ | jmp ->exception_handler
+ return 1;
+}
+
+static int zend_jit_undefined_function_stub(dasm_State **Dst)
+{
+ |->undefined_function:
+ | mov r0, aword EX->opline
+ |.if X64
+ | xor CARG1, CARG1
+ | LOAD_ADDR CARG2, "Call to undefined function %s()"
+ | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)]
+ | mov CARG3, aword [r0 + CARG3]
+ | add CARG3, offsetof(zend_string, val)
+ | EXT_CALL zend_throw_error, r0
+ |.else
+ | sub r4, 4
+ | mov r0, aword [r0 + offsetof(zend_op, op2.zv)]
+ | mov r0, aword [r0]
+ | add r0, offsetof(zend_string, val)
+ | push r0
+ | push "Call to undefined function %s()"
+ | push 0
+ | EXT_CALL zend_throw_error, r0
+ | add r4, 16
+ |.endif
+ | jmp ->exception_handler
+ return 1;
+}
+
+static int zend_jit_negative_shift_stub(dasm_State **Dst)
+{
+ |->negative_shift:
+ |.if X64
+ |.if WIN
+ | LOAD_ADDR CARG1, &zend_ce_arithmetic_error
+ | mov CARG1, aword [CARG1]
+ |.else
+ | LOAD_ADDR CARG1, zend_ce_arithmetic_error
+ |.endif
+ | LOAD_ADDR CARG2, "Bit shift by negative number"
+ | EXT_CALL zend_throw_error, r0
+ |.else
+ | sub r4, 8
+ | push "Bit shift by negative number"
+ |.if WIN
+ | LOAD_ADDR r0, &zend_ce_arithmetic_error
+ | push aword [r0]
+ |.else
+ | PUSH_ADDR zend_ce_arithmetic_error, r0
+ |.endif
+ | EXT_CALL zend_throw_error, r0
+ | add r4, 16
+ |.endif
+ | jmp ->exception_handler
+ return 1;
+}
+
+static int zend_jit_mod_by_zero_stub(dasm_State **Dst)
+{
+ |->mod_by_zero:
+ |.if X64
+ |.if WIN
+ | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error
+ | mov CARG1, aword [CARG1]
+ |.else
+ | LOAD_ADDR CARG1, zend_ce_division_by_zero_error
+ |.endif
+ | LOAD_ADDR CARG2, "Modulo by zero"
+ | EXT_CALL zend_throw_error, r0
+ |.else
+ | sub r4, 8
+ | push "Modulo by zero"
+ |.if WIN
+ | LOAD_ADDR r0, &zend_ce_division_by_zero_error
+ | push aword [r0]
+ |.else
+ | PUSH_ADDR zend_ce_division_by_zero_error, r0
+ |.endif
+ | EXT_CALL zend_throw_error, r0
+ | add r4, 16
+ |.endif
+ | jmp ->exception_handler
+ return 1;
+}
+
+static int zend_jit_double_one_stub(dasm_State **Dst)
+{
+ |->one:
+ |.dword 0, 0x3ff00000
+ return 1;
+}
+
+static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
+{
+ |->hybrid_runtime_jit:
+ | EXT_CALL zend_runtime_jit, r0
+ | JMP_IP
+ return 1;
+}
+
+static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
+{
+ |->hybrid_profile_jit:
+ | // ++zend_jit_profile_counter;
+ | .if X64
+ | LOAD_ADDR r0, &zend_jit_profile_counter
+ | inc aword [r0]
+ | .else
+ | inc aword [&zend_jit_profile_counter]
+ | .endif
+ | // op_array = (zend_op_array*)EX(func);
+ | mov r0, EX->func
+ | // ++ZEND_COUNTER_INFO(op_array)
+ | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)]
+#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR
+ | mov r2, aword [r2]
+#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET
+ | xor r1, r1
+ | test r2, 1
+ | jz >1
+ | MEM_OP2_2_ZTS mov, r1, aword, compiler_globals, map_ptr_base, r1
+ | sub r1, 1
+ |1:
+ | mov r2, aword [r1 + r2]
+#else
+# error "Unknown ZEND_MAP_PTR_KIND"
+#endif
+ | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)]
+ | // handler = (const void*)ZEND_FUNC_INFO(op_array);
+ | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+ | // return ((zend_vm_opcode_handler_t)handler)();
+ | jmp r0
+ return 1;
+}
+
+/*
+ * This code is based Mike Pall's "Hashed profile counters" idea, implemented
+ * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual
+ * property disclosure and research opportunities" email
+ * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html
+ *
+ * In addition we use a variation of Knuth's multiplicative hash function
+ * described at https://code.i-harness.com/en/q/a21ce
+ *
+ * uint64_t hash(uint64_t x) {
+ * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
+ * x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
+ * x = x ^ (x >> 31);
+ * return x;
+ * }
+ *
+ * uint_32_t hash(uint32_t x) {
+ * x = ((x >> 16) ^ x) * 0x45d9f3b;
+ * x = ((x >> 16) ^ x) * 0x45d9f3b;
+ * x = (x >> 16) ^ x;
+ * return x;
+ * }
+ *
+ */
+static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst)
+{
+ |->hybrid_func_counter:
+ | mov r0, EX->func
+ | mov r1, aword [r0 + offsetof(zend_op_array, function_name)]
+ | test r1, r1
+ | jne >1
+ | mov r0, aword [r0 + offsetof(zend_op_array, filename)]
+ |1:
+ | shr r0, 3
+ | mov r1, r0
+ | .if X64
+ | shr r0, 30
+ | xor r0, r1
+ | mov64 r1, 0xbf58476d1ce4e5b9
+ | imul r0, r1
+ | mov r1, r0
+ | shr r0, 27
+ | xor r0, r1
+ | mov64 r1, 0x94d049bb133111eb
+ | imul r0, r1
+ | mov r1, r0
+ | shr r0, 31
+ | xor r1, r0
+ | and r1, ZEND_HOT_COUNTERS_COUNT-1
+ | LOAD_ADDR r0, &zend_jit_hot_counters
+ | sub word [r0+r1*2], ZEND_JIT_HOT_FUNC_COST
+ | .else
+ | shr r0, 16
+ | xor r0, r1
+ | imul r0, 0x45d9f3b
+ | mov r1, r0
+ | shr r0, 16
+ | xor r0, r1
+ | imul r0, 0x45d9f3b
+ | mov r1, r0
+ | shr r0, 16
+ | xor r1, r0
+ | and r1, ZEND_HOT_COUNTERS_COUNT-1
+ | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_FUNC_COST
+ | .endif
+ | jle >1
+ | mov r1, EX->func
+ | GET_IP r0
+ | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)]
+ | // divide by sizeof(zend_op)
+ | .if X64
+ || ZEND_ASSERT(sizeof(zend_op) == 32);
+ | sar r0, 5
+ | .else
+ || ZEND_ASSERT(sizeof(zend_op) == 28);
+ | sar r0, 2
+ | imul r0, 0xb6db6db7
+ | .endif
+ | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+ | .if X64
+ | jmp aword [r1+r0*8]
+ | .else
+ | jmp aword [r1+r0*4]
+ | .endif
+ |1:
+ | .if X64
+ | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT
+ | .else
+ | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT
+ | .endif
+ | mov FCARG1a, FP
+ | GET_IP FCARG2a
+ | EXT_CALL zend_jit_hot_func, r0
+ | JMP_IP
+ return 1;
+}
+
+static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst)
+{
+ |->hybrid_loop_counter:
+ | mov r0, EX->func
+ | mov r1, aword [r0 + offsetof(zend_op_array, function_name)]
+ | test r1, r1
+ | jne >1
+ | mov r0, aword [r0 + offsetof(zend_op_array, filename)]
+ |1:
+ | shr r0, 3
+ | mov r1, r0
+ | .if X64
+ | shr r0, 30
+ | xor r0, r1
+ | mov64 r1, 0xbf58476d1ce4e5b9
+ | imul r0, r1
+ | mov r1, r0
+ | shr r0, 27
+ | xor r0, r1
+ | mov64 r1, 0x94d049bb133111eb
+ | imul r0, r1
+ | mov r1, r0
+ | shr r0, 31
+ | xor r1, r0
+ | and r1, ZEND_HOT_COUNTERS_COUNT-1
+ | LOAD_ADDR r0, &zend_jit_hot_counters
+ | sub word [r0+r1*2], ZEND_JIT_HOT_LOOP_COST
+ | .else
+ | shr r0, 16
+ | xor r0, r1
+ | imul r0, 0x45d9f3b
+ | mov r1, r0
+ | shr r0, 16
+ | xor r0, r1
+ | imul r0, 0x45d9f3b
+ | mov r1, r0
+ | shr r0, 16
+ | xor r1, r0
+ | and r1, ZEND_HOT_COUNTERS_COUNT-1
+ | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_LOOP_COST
+ | .endif
+ | jle >1
+ | mov r1, EX->func
+ | GET_IP r0
+ | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)]
+ | // divide by sizeof(zend_op)
+ | .if X64
+ || ZEND_ASSERT(sizeof(zend_op) == 32);
+ | sar r0, 5
+ | .else
+ || ZEND_ASSERT(sizeof(zend_op) == 28);
+ | sar r0, 2
+ | imul r0, 0xb6db6db7
+ | .endif
+ | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+ | .if X64
+ | jmp aword [r1+r0*8]
+ | .else
+ | jmp aword [r1+r0*4]
+ | .endif
+ |1:
+ | .if X64
+ | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT
+ | .else
+ | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT
+ | .endif
+ | mov FCARG1a, FP
+ | GET_IP FCARG2a
+ | EXT_CALL zend_jit_hot_func, r0
+ | JMP_IP
+ return 1;
+}
+
+#ifdef CONTEXT_THREADED_JIT
+static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
+{
+ |->context_threaded_call:
+ | pop r0
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | jmp aword [IP]
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | jmp aword [IP]
+ } else {
+ ZEND_ASSERT(0);
+ // TODO: context threading can't work without GLOBAL REGS because we have to change
+ // the value of execute_data in execute_ex()
+ | mov FCARG1a, FP
+ | mov r0, aword [FP]
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | jmp aword [r0]
+ }
+ return 1;
+}
+#endif
+
+static const zend_jit_stub zend_jit_stubs[] = {
+ JIT_STUB(interrupt_handler),
+ JIT_STUB(exception_handler),
+ JIT_STUB(exception_handler_undef),
+ JIT_STUB(leave_function),
+ JIT_STUB(leave_throw),
+ JIT_STUB(icall_throw),
+ JIT_STUB(throw_cannot_pass_by_ref),
+ JIT_STUB(undefined_offset),
+ JIT_STUB(undefined_index),
+ JIT_STUB(cannot_add_element),
+ JIT_STUB(undefined_offset_ex),
+ JIT_STUB(undefined_index_ex),
+ JIT_STUB(cannot_add_element_ex),
+ JIT_STUB(not_obj),
+ JIT_STUB(undefined_function),
+ JIT_STUB(negative_shift),
+ JIT_STUB(mod_by_zero),
+ JIT_STUB(double_one),
+#ifdef CONTEXT_THREADED_JIT
+ JIT_STUB(context_threaded_call),
+#endif
+};
+
+#if ZTS && defined(ZEND_WIN32)
+extern uint32_t _tls_index;
+extern char *_tls_start;
+extern char *_tls_end;
+#endif
+
+static int zend_jit_setup(void)
+{
+ |.if SSE
+ if (!zend_cpu_supports(ZEND_CPU_FEATURE_SSE2)) {
+ zend_error(E_CORE_ERROR, "CPU doesn't support SSE2");
+ return FAILURE;
+ }
+ if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) {
+ if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) {
+ zend_jit_x86_flags |= ZEND_JIT_CPU_AVX;
+ }
+ }
+ |.endif
+
+#if ZTS
+# ifdef _WIN64
+ tsrm_tls_index = _tls_index * sizeof(void*);
+
+ /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
+ /* Probably, it might be better solution */
+ do {
+ void ***tls_mem = ((void***)__readgsqword(0x58))[_tls_index];
+ void *val = _tsrm_ls_cache;
+ size_t offset = 0;
+ size_t size = (char*)&_tls_end - (char*)&_tls_start;
+
+ while (offset < size) {
+ if (*tls_mem == val) {
+ tsrm_tls_offset = offset;
+ break;
+ }
+ tls_mem++;
+ offset += sizeof(void*);
+ }
+ if (offset >= size) {
+ // TODO: error message ???
+ return FAILURE;
+ }
+ } while(0);
+# elif ZEND_WIN32
+ tsrm_tls_index = _tls_index * sizeof(void*);
+
+ /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
+ /* Probably, it might be better solution */
+ do {
+ void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index];
+ void *val = _tsrm_ls_cache;
+ size_t offset = 0;
+ size_t size = (char*)&_tls_end - (char*)&_tls_start;
+
+ while (offset < size) {
+ if (*tls_mem == val) {
+ tsrm_tls_offset = offset;
+ break;
+ }
+ tls_mem++;
+ offset += sizeof(void*);
+ }
+ if (offset >= size) {
+ // TODO: error message ???
+ return FAILURE;
+ }
+ } while(0);
+# elif defined(__GNUC__) && defined(__x86_64__)
+ tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
+ if (tsrm_ls_cache_tcb_offset == 0) {
+ size_t *ti;
+
+ __asm__(
+ "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
+ : "=a" (ti));
+ tsrm_tls_offset = ti[1];
+ tsrm_tls_index = ti[0] * 16;
+ }
+# elif defined(__GNUC__) && defined(__i386__)
+ tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
+ if (tsrm_ls_cache_tcb_offset == 0) {
+#if 1
+ size_t ret;
+
+ asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
+ : "=a" (ret));
+ tsrm_ls_cache_tcb_offset = ret;
+#else
+ size_t *ti, _ebx, _ecx, _edx;
+
+ __asm__(
+ "call 1f\n"
+ ".subsection 1\n"
+ "1:\tmovl (%%esp), %%ebx\n\t"
+ "ret\n"
+ ".previous\n\t"
+ "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
+ "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
+ "call ___tls_get_addr@plt\n\t"
+ "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
+ : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
+ tsrm_tls_offset = ti[1];
+ tsrm_tls_index = ti[0] * 8;
+#endif
+ }
+# endif
+#endif
+
+ return SUCCESS;
+}
+
+static int zend_jit_align_func(dasm_State **Dst)
+{
+ reuse_ip = 0;
+ delayed_call_chain = 0;
+ last_valid_opline = NULL;
+ jit_return_label = -1;
+ |.align 16
+ return 1;
+}
+
+static int zend_jit_prologue(dasm_State **Dst)
+{
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | sub r4, HYBRID_SPAD
+ } else if (GCC_GLOBAL_REGS) {
+ | sub r4, SPAD // stack alignment
+ } else {
+ | sub r4, NR_SPAD // stack alignment
+ | mov aword T2, FP // save FP
+ | mov aword T3, RX // save IP
+ | mov FP, FCARG1a
+ }
+ return 1;
+}
+
+static int zend_jit_label(dasm_State **Dst, unsigned int label)
+{
+ |=>label:
+ return 1;
+}
+
+static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
+{
+ | // call->prev_execute_data = EX(call);
+ if (call_level == 1) {
+ | mov aword EX:RX->prev_execute_data, 0
+ } else {
+ | mov r0, EX->call
+ | mov EX:RX->prev_execute_data, r0
+ }
+ | // EX(call) = call;
+ | mov EX->call, RX
+
+ delayed_call_chain = 0;
+
+ return 1;
+}
+
+static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
+{
+ if (!last_valid_opline) {
+ | LOAD_IP_ADDR opline
+ } else if (last_valid_opline != opline) {
+ if (GCC_GLOBAL_REGS) {
+ | ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
+ } else {
+ | LOAD_IP_ADDR opline
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline)
+{
+ if (delayed_call_chain) {
+ if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
+ return 0;
+ }
+ }
+ if (!zend_jit_set_ip(Dst, opline)) {
+ return 0;
+ }
+ last_valid_opline = opline;
+ reuse_ip = 0;
+ return 1;
+}
+
+static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline)
+{
+ if (zend_interrupt_function) {
+#if 0
+ if (!zend_jit_set_valid_ip(Dst, opline)) {
+ return 0;
+ }
+ | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
+ | jne ->interrupt_handler
+#else
+ | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
+ if (last_valid_opline == opline) {
+ | jne ->interrupt_handler
+ } else {
+ | jne >1
+ |.cold_code
+ |1:
+ | LOAD_IP_ADDR opline
+ | jmp ->interrupt_handler
+ |.code
+ }
+ return 1;
+#endif
+ }
+ return 1;
+}
+
+static int zend_jit_check_exception(dasm_State **Dst)
+{
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ return 1;
+}
+
+static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline)
+{
+ if (opline->result_type & (IS_TMP_VAR|IS_VAR)) {
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | mov r0, opline->result.var
+ | jne ->exception_handler_undef
+ return 1;
+ }
+ return zend_jit_check_exception(Dst);
+}
+
+static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw)
+{
+ const void *handler;
+
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ handler = zend_get_opcode_handler_func(opline);
+ } else {
+ handler = opline->handler;
+ }
+
+ if (!zend_jit_set_valid_ip(Dst, opline)) {
+ return 0;
+ }
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, FP
+ }
+ | EXT_CALL handler, r0
+ if (may_throw) {
+ zend_jit_check_exception(Dst);
+ }
+ last_valid_opline++;
+
+ /* Skip the following OP_DATA */
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_DIM:
+ case ZEND_ASSIGN_OBJ:
+ case ZEND_ASSIGN_STATIC_PROP:
+ case ZEND_ASSIGN_STATIC_PROP_REF:
+ case ZEND_ASSIGN_OBJ_REF:
+ last_valid_opline++;
+ break;
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+ case ZEND_ASSIGN_DIV:
+ case ZEND_ASSIGN_MOD:
+ case ZEND_ASSIGN_SL:
+ case ZEND_ASSIGN_SR:
+ case ZEND_ASSIGN_CONCAT:
+ case ZEND_ASSIGN_BW_OR:
+ case ZEND_ASSIGN_BW_AND:
+ case ZEND_ASSIGN_BW_XOR:
+ case ZEND_ASSIGN_POW:
+ if (opline->extended_value) {
+ last_valid_opline++;
+ }
+ break;
+ default:
+ break;
+ }
+
+ return 1;
+}
+
+static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
+{
+ if (!zend_jit_set_valid_ip(Dst, opline)) {
+ return 0;
+ }
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ if (opline->opcode == ZEND_DO_UCALL ||
+ opline->opcode == ZEND_DO_FCALL_BY_NAME ||
+ opline->opcode == ZEND_DO_FCALL ||
+ opline->opcode == ZEND_RETURN) {
+
+ /* Use inlined HYBRID VM handler */
+ const void *handler = opline->handler;
+
+ | add r4, HYBRID_SPAD
+ | EXT_JMP handler, r0
+ } else {
+ const void *handler = zend_get_opcode_handler_func(opline);
+
+ | EXT_CALL handler, r0
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+ }
+ } else {
+ const void *handler = opline->handler;
+
+ if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ } else {
+ | mov FCARG1a, FP
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ }
+ | EXT_JMP handler, r0
+ }
+ last_valid_opline = NULL;
+ return 1;
+}
+
+static int zend_jit_set_opline(dasm_State **Dst, const zend_op *target_opline)
+{
+ if (!reuse_ip) {
+ last_valid_opline = target_opline;
+ }
+ return 1;
+}
+
+static int zend_jit_reset_opline(dasm_State **Dst, const zend_op *target_opline)
+{
+ last_valid_opline = NULL;
+ return 1;
+}
+
+static void zend_jit_start_reuse_ip(void) {
+ last_valid_opline = NULL;
+ reuse_ip = 1;
+}
+
+static void zend_jit_stop_reuse_ip(void) {
+ reuse_ip = 0;
+}
+
+static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label)
+{
+ | jmp =>target_label
+ return 1;
+}
+
+static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label)
+{
+ | CMP_IP next_opline
+ | jne =>target_label
+
+ last_valid_opline = next_opline;
+
+ return 1;
+}
+
+#ifdef CONTEXT_THREADED_JIT
+static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
+{
+ if (!zend_jit_handler(Dst, opline, 1)) return 0;
+ if (opline->opcode == ZEND_DO_UCALL) {
+ | call ->context_threaded_call
+ } else {
+ const zend_op *next_opline = opline + 1;
+
+ | CMP_IP next_opline
+ | je =>next_block
+ | call ->context_threaded_call
+ }
+ return 1;
+}
+#endif
+
+static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
+{
+#ifdef CONTEXT_THREADED_JIT
+ return zend_jit_context_threaded_call(Dst, opline, next_block);
+#else
+ return zend_jit_tail_handler(Dst, opline);
+#endif
+}
+
+static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type)
+{
+ ZEND_ASSERT(Z_MODE(src) == IS_REG);
+ ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ | SET_ZVAL_LVAL dst, Ra(Z_REG(src))
+ if (set_type) {
+ | SET_ZVAL_TYPE_INFO dst, IS_LONG
+ }
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ | .if SSE
+ | SSE_SET_ZVAL_DVAL dst, Z_REG(src)
+ | .else
+ || ZEND_ASSERT(0);
+ | .endif
+ if (set_type) {
+ | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ return 1;
+}
+
+static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
+{
+ ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
+ ZEND_ASSERT(Z_MODE(dst) == IS_REG);
+
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ | GET_ZVAL_LVAL Z_REG(dst), src
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ | .if SSE
+ | SSE_GET_ZVAL_DVAL Z_REG(dst), src
+ | .else
+ || ZEND_ASSERT(0);
+ | .endif
+ } else {
+ ZEND_ASSERT(0);
+ }
+ return 1;
+}
+
+static int zend_jit_store_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_reg reg)
+{
+ zend_jit_addr src = ZEND_ADDR_REG(reg);
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var));
+ uint32_t info = ssa->var_info[var].type;
+
+ return zend_jit_spill_store(Dst, src, dst, info, 1);
+}
+
+static int zend_jit_store_ssa_var_if_necessary(dasm_State **Dst, zend_ssa *ssa, zend_lifetime_interval **ra, zend_jit_addr src, int var, int old_var)
+{
+ if (Z_MODE(src) == IS_REG && ra[var] && ra[var]->store) {
+ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var));
+ uint32_t info = ssa->var_info[var].type;
+ zend_bool set_type = 1;
+
+ if (old_var >= 0) {
+ uint32_t old_info = ssa->var_info[old_var].type;
+ if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
+ (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
+ if (!ra[old_var] || ra[old_var]->load || ra[old_var]->store) {
+ set_type = 0;
+ }
+ }
+ }
+ return zend_jit_spill_store(Dst, src, dst, info, set_type);
+ }
+ return 1;
+}
+
+static int zend_jit_load_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_reg reg)
+{
+ zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var));
+ zend_jit_addr dst = ZEND_ADDR_REG(reg);
+ uint32_t info = ssa->var_info[var].type;
+
+ return zend_jit_load_reg(Dst, src, dst, info);
+}
+
+static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_lifetime_interval *src_ival)
+{
+ if (src != dst) {
+ if (Z_MODE(src) == IS_REG) {
+ if (Z_MODE(dst) == IS_REG) {
+ if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
+ | mov Ra(Z_REG(dst)), Ra(Z_REG(src))
+ } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ | .if SSE
+ | SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0)
+ | .else
+ || ZEND_ASSERT(0);
+ | .endif
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else if (Z_MODE(dst) == IS_MEM_ZVAL) {
+ if (!src_ival->load && !src_ival->store) {
+ if (!zend_jit_spill_store(Dst, src, dst, info, 1)) {
+ return 0;
+ }
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else if (Z_MODE(src) == IS_MEM_ZVAL) {
+ if (Z_MODE(dst) == IS_REG) {
+ if (!zend_jit_load_reg(Dst, src, dst, info)) {
+ return 0;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info, op1_def_info, res_use_info = 0;
+ zend_jit_addr op1_addr, op1_def_addr, res_addr = 0;
+
+ op1_info = OP1_INFO();
+ if (opline->op1_type != IS_CV || !(op1_info & MAY_BE_LONG)) {
+ goto fallback;
+ }
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1);
+ if (opline->result_type != IS_UNUSED) {
+ res_use_info = RES_USE_INFO();
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1);
+ }
+
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
+ }
+ if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+ }
+ if (op1_addr != op1_def_addr) {
+ if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op1_def_addr) == IS_REG &&
+ (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) {
+ if (!zend_jit_update_regs(Dst, res_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) {
+ return 0;
+ }
+ } else {
+ if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) {
+ return 0;
+ }
+ }
+ }
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ | LONG_OP_WITH_CONST add, op1_def_addr, Z_L(1)
+ } else {
+ | LONG_OP_WITH_CONST sub, op1_def_addr, Z_L(1)
+ }
+ op1_def_info = OP1_DEF_INFO();
+
+ if ((op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) {
+ | jo >1
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+ }
+ |.cold_code
+ |1:
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ |.if X64
+ | mov64 rax, 0x43e0000000000000
+ | SET_ZVAL_LVAL op1_def_addr, rax
+ |.else
+ | SET_ZVAL_LVAL op1_def_addr, 0
+ | SET_ZVAL_W2 op1_def_addr, 0x41e00000
+ |.endif
+ } else {
+ |.if X64
+ | mov64 rax, 0xc3e0000000000000
+ | SET_ZVAL_LVAL op1_def_addr, rax
+ |.else
+ | SET_ZVAL_LVAL op1_def_addr, 0x00200000
+ | SET_ZVAL_W2 op1_def_addr, 0xc1e00000
+ |.endif
+ }
+ | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1
+ }
+ | jmp >3
+ |.code
+ } else {
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
+ }
+ }
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ |.cold_code
+ |2:
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | SAVE_VALID_OPLINE opline
+ if (op1_info & MAY_BE_UNDEF) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
+ op1_info |= MAY_BE_NULL;
+ }
+ |2:
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+
+ | // ZVAL_DEREF(var_ptr);
+ if (op1_info & MAY_BE_REF) {
+ | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2
+ | GET_Z_PTR FCARG2a, FCARG1a
+ | cmp aword [FCARG2a + offsetof(zend_reference, sources.ptr)], 0
+ | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
+ | jz >2
+ |.if X64
+ if (RETURN_VALUE_USED(opline)) {
+ | LOAD_ZVAL_ADDR CARG3, res_addr
+ } else {
+ | mov CARG3, 0
+ }
+ |.else
+ | sub r4, 12
+ if (RETURN_VALUE_USED(opline)) {
+ | PUSH_ZVAL_ADDR res_addr, r0
+ } else {
+ | push 0
+ }
+ |.endif
+ if (opline->opcode == ZEND_PRE_INC) {
+ | EXT_CALL zend_jit_pre_inc_typed_ref, r0
+ } else if (opline->opcode == ZEND_PRE_DEC) {
+ | EXT_CALL zend_jit_pre_dec_typed_ref, r0
+ } else if (opline->opcode == ZEND_POST_INC) {
+ | EXT_CALL zend_jit_post_inc_typed_ref, r0
+ } else if (opline->opcode == ZEND_POST_DEC) {
+ | EXT_CALL zend_jit_post_dec_typed_ref, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ zend_jit_check_exception(Dst);
+ | jmp >3
+ |2:
+ }
+
+ if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) {
+ if (opline->result_type != IS_UNUSED) {
+ zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+
+ | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2
+ | TRY_ADDREF op1_info, ah, r2
+ }
+ }
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ | EXT_CALL increment_function, r0
+ } else {
+ | EXT_CALL decrement_function, r0
+ }
+ } else {
+ zend_reg tmp_reg;
+
+ if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) {
+ if (opline->result_type != IS_UNUSED) {
+ | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2
+ }
+ }
+ |.if SSE
+ if (Z_MODE(op1_def_addr) == IS_REG) {
+ tmp_reg = Z_REG(op1_def_addr);
+ } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) {
+ tmp_reg = Z_REG(op1_addr);
+ } else {
+ tmp_reg = ZREG_XMM0;
+ }
+ | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
+ } else {
+ | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
+ }
+ } else {
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
+ } else {
+ | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
+ }
+ }
+ | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg
+ |.else
+ | FPU_GET_ZVAL_DVAL op1_addr
+ | fld1
+ if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
+ | fadd
+ } else {
+ | fsub
+ }
+ | FPU_SET_ZVAL_DVAL op1_def_addr
+ |.endif
+ }
+ if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
+ opline->result_type != IS_UNUSED) {
+ | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1
+ | TRY_ADDREF op1_def_info, ah, r1
+ }
+ | jmp >3
+ |.code
+ }
+ |3:
+ if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, op1_def_addr, ssa->ops[opline - op_array->opcodes].op1_def, ssa->ops[opline - op_array->opcodes].op1_use)) {
+ return 0;
+ }
+ if (opline->result_type != IS_UNUSED) {
+ if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) {
+ return 0;
+ }
+ }
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_math_long_long(dasm_State **Dst,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_op *opline,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_info,
+ uint32_t res_use_info)
+{
+ zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ zend_reg result_reg;
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ result_reg = Z_REG(res_addr);
+ } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) {
+ result_reg = Z_REG(op1_addr);
+ } else {
+ result_reg = ZREG_R0;
+ }
+
+ if (opline->opcode == ZEND_MUL &&
+ ((Z_MODE(op2_addr) == IS_CONST_ZVAL &&
+ IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) &&
+ is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) ||
+ (Z_MODE(op1_addr) == IS_CONST_ZVAL &&
+ IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) &&
+ is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
+ } else {
+ | GET_ZVAL_LVAL result_reg, op2_addr
+ | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr)))
+ }
+ } else if (opline->opcode == ZEND_DIV &&
+ (Z_MODE(op2_addr) == IS_CONST_ZVAL &&
+ is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ | shr Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
+ } else {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ if (same_ops && opline->opcode != ZEND_DIV) {
+ | LONG_MATH_REG opline->opcode, Ra(result_reg), Ra(result_reg)
+ } else {
+ | LONG_MATH opline->opcode, result_reg, op2_addr
+ }
+ }
+ if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) {
+ | jo >1
+ |.cold_code
+ |1:
+ |.if SSE
+ zend_reg tmp_reg1 = ZREG_XMM0;
+ zend_reg tmp_reg2 = ZREG_XMM1;
+
+ | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr
+ | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ | AVX_MATH_REG opline->opcode, tmp_reg1, tmp_reg1, tmp_reg2
+ } else {
+ | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2
+ }
+ | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1
+ |.else
+ | FPU_GET_ZVAL_LVAL op2_addr
+ | FPU_GET_ZVAL_LVAL op1_addr
+ | FPU_MATH_REG opline->opcode, st1
+ | FPU_SET_ZVAL_DVAL res_addr
+ |.endif
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
+ }
+ | jmp >2
+ |.code
+ | SET_ZVAL_LVAL res_addr, Ra(result_reg)
+ if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_LONG
+ }
+ }
+ |2:
+ } else if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+ | SET_ZVAL_LVAL res_addr, Ra(result_reg)
+ if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_LONG
+ }
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math_long_double(dasm_State **Dst,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_op *opline,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_use_info)
+{
+ |.if SSE
+ zend_reg result_reg =
+ (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0;
+
+ | SSE_GET_ZVAL_LVAL result_reg, op1_addr
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ | AVX_MATH opline->opcode, result_reg, result_reg, op2_addr
+ } else {
+ | SSE_MATH opline->opcode, result_reg, op2_addr
+ }
+ | SSE_SET_ZVAL_DVAL res_addr, result_reg
+ |.else
+ | FPU_GET_ZVAL_LVAL op1_addr
+ | FPU_MATH opline->opcode, op2_addr
+ | FPU_SET_ZVAL_DVAL res_addr
+ |.endif
+
+ if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math_double_long(dasm_State **Dst,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_op *opline,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_use_info)
+{
+ |.if SSE
+ zend_reg result_reg;
+
+ if (zend_is_commutative(opline->opcode)) {
+ if (Z_MODE(res_addr) == IS_REG) {
+ result_reg = Z_REG(res_addr);
+ } else {
+ result_reg = ZREG_XMM0;
+ }
+ | SSE_GET_ZVAL_LVAL result_reg, op2_addr
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ | AVX_MATH opline->opcode, result_reg, result_reg, op1_addr
+ } else {
+ | SSE_MATH opline->opcode, result_reg, op1_addr
+ }
+ } else {
+ zend_reg tmp_reg;
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ result_reg = Z_REG(res_addr);
+ tmp_reg = ZREG_XMM0;
+ } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) {
+ result_reg = Z_REG(op1_addr);
+ tmp_reg = ZREG_XMM0;
+ } else {
+ result_reg = ZREG_XMM0;
+ tmp_reg = ZREG_XMM1;
+ }
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ zend_reg op1_reg;
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ op1_reg = Z_REG(op1_addr);
+ } else {
+ | SSE_GET_ZVAL_DVAL result_reg, op1_addr
+ op1_reg = result_reg;
+ }
+ | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr
+ | AVX_MATH_REG opline->opcode, result_reg, op1_reg, tmp_reg
+ } else {
+ | SSE_GET_ZVAL_DVAL result_reg, op1_addr
+ | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr
+ | SSE_MATH_REG opline->opcode, result_reg, tmp_reg
+ }
+ }
+ | SSE_SET_ZVAL_DVAL res_addr, result_reg
+ |.else
+ | FPU_GET_ZVAL_LVAL op2_addr
+ if (zend_is_commutative(opline->opcode)) {
+ | FPU_MATH opline->opcode, op1_addr
+ } else {
+ | FPU_GET_ZVAL_DVAL op1_addr
+ | FPU_MATH_REG opline->opcode, st1
+ }
+ | FPU_SET_ZVAL_DVAL res_addr
+ |.endif
+
+ if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+ if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
+ }
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math_double_double(dasm_State **Dst,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ const zend_op *opline,
+ zend_jit_addr op1_addr,
+ zend_jit_addr op2_addr,
+ zend_jit_addr res_addr,
+ uint32_t res_use_info)
+{
+ zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+
+ |.if SSE
+ zend_reg result_reg;
+
+ if (Z_MODE(res_addr) == IS_REG) {
+ result_reg = Z_REG(res_addr);
+ } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) {
+ result_reg = Z_REG(op1_addr);
+ } else {
+ result_reg = ZREG_XMM0;
+ }
+
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ zend_reg op1_reg;
+ zend_jit_addr val_addr;
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ op1_reg = Z_REG(op1_addr);
+ val_addr = op2_addr;
+ } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opline->opcode)) {
+ op1_reg = Z_REG(op2_addr);
+ val_addr = op1_addr;
+ } else {
+ | SSE_GET_ZVAL_DVAL result_reg, op1_addr
+ op1_reg = result_reg;
+ val_addr = op2_addr;
+ }
+ if ((opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) &&
+ Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
+ | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg
+ } else {
+ | AVX_MATH opline->opcode, result_reg, op1_reg, val_addr
+ }
+ } else {
+ zend_jit_addr val_addr;
+
+ if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opline->opcode)) {
+ | SSE_GET_ZVAL_DVAL result_reg, op2_addr
+ val_addr = op1_addr;
+ } else {
+ | SSE_GET_ZVAL_DVAL result_reg, op1_addr
+ val_addr = op2_addr;
+ }
+ if (same_ops) {
+ | SSE_MATH_REG opline->opcode, result_reg, result_reg
+ } else if ((opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) &&
+ Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
+ | SSE_MATH_REG ZEND_ADD, result_reg, result_reg
+ } else {
+ | SSE_MATH opline->opcode, result_reg, val_addr
+ }
+ }
+ | SSE_SET_ZVAL_DVAL res_addr, result_reg
+ |.else
+ | FPU_GET_ZVAL_DVAL op1_addr
+ | FPU_MATH opline->opcode, op2_addr
+ | FPU_SET_ZVAL_DVAL res_addr
+ |.endif
+
+ if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+ if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
+ }
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math_helper(dasm_State **Dst,
+ const zend_op *opline,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ zend_uchar op1_type,
+ znode_op op1,
+ zend_jit_addr op1_addr,
+ uint32_t op1_info,
+ zend_uchar op2_type,
+ znode_op op2,
+ zend_jit_addr op2_addr,
+ uint32_t op2_info,
+ uint32_t res_var,
+ zend_jit_addr res_addr,
+ uint32_t res_info,
+ uint32_t res_use_info)
+/* Labels: 1,2,3,4,5,6 */
+{
+ zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
+ if (op1_info & MAY_BE_DOUBLE) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
+ } else {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
+ }
+ }
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
+ if (op2_info & MAY_BE_DOUBLE) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1
+ |.cold_code
+ |1:
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
+ }
+ if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ | jmp >5
+ |.code
+ } else {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
+ }
+ }
+ if (!zend_jit_math_long_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_info, res_use_info)) {
+ return 0;
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ |.cold_code
+ |3:
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if (!same_ops) {
+ | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1
+ } else {
+ | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6
+ }
+ }
+ if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ | jmp >5
+ }
+ if (!same_ops) {
+ |1:
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
+ }
+ if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ | jmp >5
+ }
+ |.code
+ }
+ } else if ((op1_info & MAY_BE_DOUBLE) &&
+ !(op1_info & MAY_BE_LONG) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if (!same_ops && (op2_info & MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1
+ } else {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
+ }
+ }
+ if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ }
+ if (!same_ops && (op2_info & MAY_BE_LONG)) {
+ if (op2_info & MAY_BE_DOUBLE) {
+ |.cold_code
+ }
+ |1:
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
+ }
+ if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ | jmp >5
+ |.code
+ }
+ }
+ } else if ((op2_info & MAY_BE_DOUBLE) &&
+ !(op2_info & MAY_BE_LONG) &&
+ (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if (!same_ops && (op1_info & MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1
+ } else {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
+ }
+ }
+ if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ }
+ if (!same_ops && (op1_info & MAY_BE_LONG)) {
+ if (op1_info & MAY_BE_DOUBLE) {
+ |.cold_code
+ }
+ |1:
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
+ }
+ if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) {
+ return 0;
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ | jmp >5
+ |.code
+ }
+ }
+ }
+
+ |5:
+
+ if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ |.cold_code
+ }
+ |6:
+ | SAVE_VALID_OPLINE opline
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
+ | LOAD_ZVAL_ADDR FCARG1a, real_addr
+ } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ }
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
+ if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op1_addr
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
+ if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op2_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR op2_addr, r0
+ |.endif
+ if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) {
+ | EXT_CALL add_function, r0
+ } else if (opline->opcode == ZEND_SUB || opline->opcode == ZEND_ASSIGN_SUB) {
+ | EXT_CALL sub_function, r0
+ } else if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) {
+ | EXT_CALL mul_function, r0
+ } else if (opline->opcode == ZEND_DIV || opline->opcode == ZEND_ASSIGN_DIV) {
+ | EXT_CALL div_function, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ | FREE_OP op1_type, op1, op1_info, 0, op_array, opline
+ | FREE_OP op2_type, op2, op2_info, 0, op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception(Dst);
+ }
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
+ if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
+ return 0;
+ }
+ }
+ if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | jmp <5
+ |.code
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info, op2_info, res_use_info;
+ zend_jit_addr op1_addr, op2_addr, res_addr;
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ res_use_info = RES_USE_INFO();
+
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ goto fallback;
+ }
+
+ if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
+ !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ goto fallback;
+ }
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1);
+
+ if (opline->result_type == IS_TMP_VAR &&
+ (opline+1)->opcode == ZEND_SEND_VAL &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ /* Eliminate the following SEND_VAL */
+ (*opnum)++;
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
+ res_use_info = -1;
+ } else {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def);
+ }
+
+ if (!zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info)) {
+ return 0;
+ }
+ if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) {
+ return 0;
+ }
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_long_math_helper(dasm_State **Dst,
+ const zend_op *opline,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ zend_uchar op1_type,
+ znode_op op1,
+ zend_jit_addr op1_addr,
+ uint32_t op1_info,
+ int op1_ssa_var,
+ zend_uchar op2_type,
+ znode_op op2,
+ zend_jit_addr op2_addr,
+ uint32_t op2_info,
+ int op2_ssa_var,
+ uint32_t res_var,
+ zend_jit_addr res_addr,
+ uint32_t res_info,
+ uint32_t res_use_info)
+/* Labels: 6 */
+{
+ zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
+ zend_reg result_reg;
+ zend_uchar opcode = opline->opcode;
+ zval tmp;
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
+ }
+ if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
+ }
+
+ if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL &&
+ op1_ssa_var >= 0 &&
+ ssa->var_info[op1_ssa_var].has_range &&
+ ssa->var_info[op1_ssa_var].range.min >= 0) {
+ zend_long l = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (zend_long_is_power_of_two(l)) {
+ /* Optimisation for mod of power of 2 */
+ opcode = ZEND_BW_AND;
+ ZVAL_LONG(&tmp, l - 1);
+ op2_addr = ZEND_ADDR_CONST_ZVAL(&tmp);
+ }
+ }
+
+ if (opcode == ZEND_MOD) {
+ result_reg = ZREG_RAX;
+ } else if (Z_MODE(res_addr) == IS_REG) {
+ result_reg = Z_REG(res_addr);
+ } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) {
+ result_reg = Z_REG(op1_addr);
+ } else {
+ result_reg = ZREG_R0;
+ }
+
+ if (opcode == ZEND_SL || opcode == ZEND_ASSIGN_SL) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
+ if (EXPECTED(op2_lval > 0)) {
+ | xor Ra(result_reg), Ra(result_reg)
+ } else {
+ | SAVE_VALID_OPLINE opline
+ | jmp ->negative_shift
+ }
+ } else {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ | shl Ra(result_reg), op2_lval
+ }
+ } else {
+ if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) {
+ | GET_ZVAL_LVAL ZREG_RCX, op2_addr
+ }
+ if (op2_ssa_var < 0 ||
+ !ssa->var_info[op2_ssa_var].has_range ||
+ ssa->var_info[op2_ssa_var].range.min < 0 ||
+ ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) {
+ | cmp r1, (SIZEOF_ZEND_LONG*8)
+ | jae >1
+ |.cold_code
+ |1:
+ | cmp r1, 0
+ | mov Ra(result_reg), 0
+ | jg >1
+ | SAVE_VALID_OPLINE opline
+ | jmp ->negative_shift
+ |.code
+ }
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ | shl Ra(result_reg), cl
+ |1:
+ }
+ } else if (opcode == ZEND_SR || opcode == ZEND_ASSIGN_SR) {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
+ if (EXPECTED(op2_lval > 0)) {
+ | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1
+ } else {
+ | SAVE_VALID_OPLINE opline
+ | jmp ->negative_shift
+ }
+ } else {
+ | sar Ra(result_reg), op2_lval
+ }
+ } else {
+ if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) {
+ | GET_ZVAL_LVAL ZREG_RCX, op2_addr
+ }
+ if (op2_ssa_var < 0 ||
+ !ssa->var_info[op2_ssa_var].has_range ||
+ ssa->var_info[op2_ssa_var].range.min < 0 ||
+ ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) {
+ | cmp r1, (SIZEOF_ZEND_LONG*8)
+ | jae >1
+ |.cold_code
+ |1:
+ | cmp r1, 0
+ | mov r1, (SIZEOF_ZEND_LONG * 8) - 1
+ | jg >1
+ | SAVE_VALID_OPLINE opline
+ | jmp ->negative_shift
+ |.code
+ }
+ |1:
+ | sar Ra(result_reg), cl
+ }
+ } else if (opcode == ZEND_MOD || opcode == ZEND_ASSIGN_MOD) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
+
+ if (op2_lval == 0) {
+ | SAVE_VALID_OPLINE opline
+ | jmp ->mod_by_zero
+ } else if (op2_lval == -1) {
+ | xor Ra(result_reg), Ra(result_reg)
+ } else {
+ result_reg = ZREG_RDX;
+ | GET_ZVAL_LVAL ZREG_RAX, op1_addr
+ | GET_ZVAL_LVAL ZREG_RCX, op2_addr
+ |.if X64
+ | cqo
+ |.else
+ | cdq
+ |.endif
+ | idiv Ra(ZREG_RCX)
+ }
+ } else {
+ if (op2_ssa_var < 0 ||
+ !ssa->var_info[op2_ssa_var].has_range ||
+ (ssa->var_info[op2_ssa_var].range.min <= 0 &&
+ ssa->var_info[op2_ssa_var].range.max >= 0)) {
+ if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
+ | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0
+ } else if (Z_MODE(op2_addr) == IS_REG) {
+ | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr))
+ }
+ | jz >1
+ |.cold_code
+ |1:
+ | SAVE_VALID_OPLINE opline
+ | jmp ->mod_by_zero
+ |.code
+ }
+
+ //TODO: add check for -1 ???
+
+ result_reg = ZREG_RDX;
+ | GET_ZVAL_LVAL ZREG_RAX, op1_addr
+ |.if X64
+ | cqo
+ |.else
+ | cdq
+ |.endif
+ if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
+ | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)]
+ } else if (Z_MODE(op2_addr) == IS_REG) {
+ | idiv Ra(Z_REG(op2_addr))
+ }
+ }
+ } else if (same_ops) {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg)
+ } else {
+ | GET_ZVAL_LVAL result_reg, op1_addr
+ | LONG_MATH opcode, result_reg, op2_addr
+ }
+
+ | SET_ZVAL_LVAL res_addr, Ra(result_reg)
+ if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
+ if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
+ if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_LONG
+ }
+ }
+ }
+
+ if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ if ((op1_info & MAY_BE_LONG) &&
+ (op2_info & MAY_BE_LONG)) {
+ |5:
+ |.cold_code
+ }
+ |6:
+ | SAVE_VALID_OPLINE opline
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
+ | LOAD_ZVAL_ADDR FCARG1a, real_addr
+ } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ }
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
+ if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op1_addr
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
+ if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op2_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR op2_addr, r0
+ |.endif
+ if (opline->opcode == ZEND_BW_OR || opline->opcode == ZEND_ASSIGN_BW_OR) {
+ | EXT_CALL bitwise_or_function, r0
+ } else if (opline->opcode == ZEND_BW_AND || opline->opcode == ZEND_ASSIGN_BW_AND) {
+ | EXT_CALL bitwise_and_function, r0
+ } else if (opline->opcode == ZEND_BW_XOR || opline->opcode == ZEND_ASSIGN_BW_XOR) {
+ | EXT_CALL bitwise_xor_function, r0
+ } else if (opline->opcode == ZEND_SL || opline->opcode == ZEND_ASSIGN_SL) {
+ | EXT_CALL shift_left_function, r0
+ } else if (opline->opcode == ZEND_SR || opline->opcode == ZEND_ASSIGN_SR) {
+ | EXT_CALL shift_right_function, r0
+ } else if (opline->opcode == ZEND_MOD || opline->opcode == ZEND_ASSIGN_MOD) {
+ | EXT_CALL mod_function, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ | FREE_OP op1_type, op1, op1_info, 0, op_array, opline
+ | FREE_OP op2_type, op2, op2_info, 0, op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception(Dst);
+ }
+ if (Z_MODE(res_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
+ if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
+ return 0;
+ }
+ }
+ if ((op1_info & MAY_BE_LONG) &&
+ (op2_info & MAY_BE_LONG)) {
+ | jmp <5
+ |.code
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info, op2_info, res_use_info;
+ zend_jit_addr op1_addr, op2_addr, res_addr;
+ int op1_ssa_var, op2_ssa_var;
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_ssa_var = ssa->ops[opline - op_array->opcodes].op1_use;
+ op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use;
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ res_use_info = RES_USE_INFO();
+
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ goto fallback;
+ }
+
+ if (!(op1_info & MAY_BE_LONG) ||
+ !(op2_info & MAY_BE_LONG)) {
+ goto fallback;
+ }
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1);
+
+ if (opline->result_type == IS_TMP_VAR &&
+ (opline+1)->opcode == ZEND_SEND_VAL &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ /* Eliminate the following SEND_VAL */
+ (*opnum)++;
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
+ res_use_info = -1;
+ } else {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def);
+ }
+
+ if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info)) {
+ return 0;
+ }
+ if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) {
+ return 0;
+ }
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_concat_helper(dasm_State **Dst,
+ const zend_op *opline,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ zend_uchar op1_type,
+ znode_op op1,
+ zend_jit_addr op1_addr,
+ uint32_t op1_info,
+ zend_uchar op2_type,
+ znode_op op2,
+ zend_jit_addr op2_addr,
+ uint32_t op2_info,
+ zend_jit_addr res_addr,
+ uint32_t res_info)
+{
+#if 1
+ if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
+ }
+ if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6
+ }
+ if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) {
+ if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ | EXT_CALL zend_jit_fast_assign_concat_helper, r0
+ } else {
+ if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op1_addr
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op2_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR op2_addr, r0
+ |.endif
+ | EXT_CALL zend_jit_fast_concat_helper, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ }
+ | FREE_OP op1_type, op1, op1_info, 0, op_array, opline
+ | FREE_OP op2_type, op2, op2_info, 0, op_array, opline
+ |5:
+ }
+ if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
+ (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
+ if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
+ |.cold_code
+ |6:
+ }
+#endif
+ | SAVE_VALID_OPLINE opline
+ if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op1_addr
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op2_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR op2_addr, r0
+ |.endif
+ | EXT_CALL concat_function, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ | FREE_OP op1_type, op1, op1_info, 0, op_array, opline
+ | FREE_OP op2_type, op2, op2_info, 0, op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception(Dst);
+ }
+#if 1
+ if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
+ | jmp <5
+ |.code
+ }
+ }
+#endif
+
+ return 1;
+}
+
+static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, op2_info;
+ zend_jit_addr op1_addr, op2_addr, res_addr;
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ goto fallback;
+ }
+
+ if (!(op1_info & MAY_BE_STRING) ||
+ !(op2_info & MAY_BE_STRING)) {
+ goto fallback;
+ }
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+
+ if (opline->result_type == IS_TMP_VAR &&
+ (opline+1)->opcode == ZEND_SEND_VAL &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ /* Eliminate the following SEND_VAL */
+ (*opnum)++;
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
+ } else {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+ }
+ return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO());
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint32_t found, uint32_t not_found)
+/* Labels: 1,2,3,4,5 */
+{
+ zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ if (op2_info & MAY_BE_LONG) {
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
+ | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
+ }
+ | // hval = Z_LVAL_P(dim);
+ | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long val = Z_LVAL_P(Z_ZV(op2_addr));
+ if (val >= 0 && val < HT_MAX_SIZE) {
+ | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
+ | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
+ | jz >4 // HASH_FIND
+ | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
+ |.if X64
+ | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)]
+ if (val == 0) {
+ | test r0, r0
+ } else {
+ | cmp r0, val
+ }
+ |.else
+ | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val
+ |.endif
+ if (type == BP_JIT_IS) {
+ | jbe >9 // NOT_FOUND
+ } else {
+ | jbe >2 // NOT_FOUND
+ }
+ | // _ret = &_ht->arData[_h].val;
+ | mov r0, aword [FCARG1a + offsetof(zend_array, arData)]
+ if (val != 0) {
+ | add r0, val * sizeof(Bucket)
+ }
+ if (type == BP_JIT_IS) {
+ | jmp >5
+ } else {
+ | IF_NOT_Z_TYPE r0, IS_UNDEF, >8
+ }
+ }
+ } else {
+ | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
+ | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
+ | jz >4 // HASH_FIND
+ | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
+ |.if X64
+ | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)]
+ | cmp r0, FCARG2a
+ |.else
+ | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a
+ |.endif
+ if (type == BP_JIT_IS) {
+ | jbe >9 // NOT_FOUND
+ } else {
+ | jbe >2 // NOT_FOUND
+ }
+ | // _ret = &_ht->arData[_h].val;
+ |.if X64
+ | mov r0, FCARG2a
+ | shl r0, 5
+ |.else
+ | imul r0, FCARG2a, sizeof(Bucket)
+ |.endif
+ | add r0, aword [FCARG1a + offsetof(zend_array, arData)]
+ if (type == BP_JIT_IS) {
+ | jmp >5
+ } else {
+ | IF_NOT_Z_TYPE r0, IS_UNDEF, >8
+ }
+ }
+ }
+ switch (type) {
+ case BP_JIT_IS:
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ |4:
+ }
+ | EXT_CALL zend_hash_index_find, r0
+ | test r0, r0
+ | jz >9 // NOT_FOUND
+ if (op2_info & MAY_BE_STRING) {
+ | jmp >5
+ }
+ break;
+ case BP_VAR_R:
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ zend_long val = Z_LVAL_P(Z_ZV(op2_addr));
+ if (val >= 0 && val < HT_MAX_SIZE) {
+ | jmp >2 // NOT_FOUND
+ }
+ } else {
+ | jmp >2 // NOT_FOUND
+ }
+ |4:
+ }
+ | EXT_CALL zend_hash_index_find, r0
+ | test r0, r0
+ | jz >2 // NOT_FOUND
+ |.cold_code
+ |2:
+ switch (type) {
+ case BP_VAR_R:
+ | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
+ | // retval = &EG(uninitialized_zval);
+ | UNDEFINED_OFFSET opline
+ | jmp >9
+ break;
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ | // retval = &EG(uninitialized_zval);
+ | SET_ZVAL_TYPE_INFO res_addr, IS_NULL
+ | jmp >9
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ |.code
+ break;
+ case BP_VAR_RW:
+ |2:
+ | SAVE_VALID_OPLINE opline
+ | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
+ | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval));
+ | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ | jmp >8
+ |4:
+ | SAVE_VALID_OPLINE opline
+ | EXT_CALL zend_jit_hash_index_lookup_rw, r0
+ }
+ break;
+ case BP_VAR_W:
+ |2:
+ | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval));
+ |.if X64
+ | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval
+ |.else
+ | sub r4, 12
+ | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0
+ |.endif
+ | EXT_CALL zend_hash_index_add_new, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
+ | jmp >8
+ |4:
+ | EXT_CALL zend_jit_hash_index_lookup_w, r0
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+
+ if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) {
+ | jmp >8
+ }
+ }
+
+ if (op2_info & MAY_BE_STRING) {
+ |3:
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
+ | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3
+ }
+ | // offset_key = Z_STR_P(dim);
+ | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
+ | // retval = zend_hash_find(ht, offset_key);
+ switch (type) {
+ case BP_JIT_IS:
+ if (opline->op2_type != IS_CONST) {
+ | cmp byte [FCARG2a + offsetof(zend_string, val)], '9'
+ | jle >1
+ |.cold_code
+ |1:
+ | EXT_CALL zend_jit_symtable_find, r0
+ | jmp >1
+ |.code
+ | EXT_CALL zend_hash_find, r0
+ |1:
+ } else {
+ | EXT_CALL _zend_hash_find_known_hash, r0
+ }
+ | test r0, r0
+ | jz >9 // NOT_FOUND
+ | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT))
+ | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1
+ | GET_Z_PTR r0, r0
+ |1:
+ break;
+ case BP_VAR_R:
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ if (opline->op2_type != IS_CONST) {
+ | cmp byte [FCARG2a + offsetof(zend_string, val)], '9'
+ | jle >1
+ |.cold_code
+ |1:
+ | EXT_CALL zend_jit_symtable_find, r0
+ | jmp >1
+ |.code
+ | EXT_CALL zend_hash_find, r0
+ |1:
+ } else {
+ | EXT_CALL _zend_hash_find_known_hash, r0
+ }
+ | test r0, r0
+ | jz >2 // NOT_FOUND
+ | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT))
+ | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW
+ |.cold_code
+ |1:
+ | // retval = Z_INDIRECT_P(retval);
+ | GET_Z_PTR r0, r0
+ | IF_NOT_Z_TYPE r0, IS_UNDEF, >8
+ |2:
+ switch (type) {
+ case BP_VAR_R:
+ // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key));
+ | UNDEFINED_INDEX opline
+ | jmp >9
+ break;
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ | // retval = &EG(uninitialized_zval);
+ | SET_ZVAL_TYPE_INFO res_addr, IS_NULL
+ | jmp >9
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ |.code
+ break;
+ case BP_VAR_RW:
+ | SAVE_VALID_OPLINE opline
+ if (opline->op2_type != IS_CONST) {
+ | EXT_CALL zend_jit_symtable_lookup_rw, r0
+ } else {
+ | EXT_CALL zend_jit_hash_lookup_rw, r0
+ }
+ break;
+ case BP_VAR_W:
+ if (opline->op2_type != IS_CONST) {
+ | EXT_CALL zend_jit_symtable_lookup_w, r0
+ } else {
+ | EXT_CALL zend_jit_hash_lookup_w, r0
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ }
+
+ if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) {
+ |5:
+ if (op1_info & MAY_BE_ARRAY_OF_REF) {
+ | ZVAL_DEREF r0, MAY_BE_REF
+ }
+ | cmp byte [r0 + 8], IS_NULL
+ | jle >9 // NOT FOUND
+ }
+
+ if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
+ if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
+ |.cold_code
+ |3:
+ }
+ | SAVE_VALID_OPLINE opline
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ switch (type) {
+ case BP_VAR_R:
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, res_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR res_addr, r0
+ |.endif
+ | EXT_CALL zend_jit_fetch_dim_r_helper, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ | jmp >9
+ break;
+ case BP_JIT_IS:
+ | EXT_CALL zend_jit_fetch_dim_isset_helper, r0
+ | test r0, r0
+ | jne >8
+ | jmp >9
+ break;
+ case BP_VAR_IS:
+ case BP_VAR_UNSET:
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, res_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR res_addr, r0
+ |.endif
+ | EXT_CALL zend_jit_fetch_dim_is_helper, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ | jmp >9
+ break;
+ case BP_VAR_RW:
+ | EXT_CALL zend_jit_fetch_dim_rw_helper, r0
+ | test r0, r0
+ | jne >8
+ | jmp >9
+ break;
+ case BP_VAR_W:
+ | EXT_CALL zend_jit_fetch_dim_w_helper, r0
+ | test r0, r0
+ | jne >8
+ | jmp >9
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
+ |.code
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_simple_assign(dasm_State **Dst,
+ const zend_op *opline,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ zend_jit_addr var_addr,
+ uint32_t var_info,
+ zend_uchar val_type,
+ znode_op val,
+ zend_jit_addr val_addr,
+ uint32_t val_info,
+ zend_jit_addr res_addr,
+ int in_cold)
+/* Labels: 1,2,3 */
+{
+ ZEND_ASSERT(Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0);
+ if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
+ zval *zv = Z_ZV(val_addr);
+
+ if (!res_addr) {
+ | ZVAL_COPY_CONST var_addr, var_info, zv, r0
+ } else {
+ | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, zv, r0
+ }
+ if (Z_REFCOUNTED_P(zv)) {
+ if (!res_addr) {
+ | ADDREF_CONST zv, r0
+ } else {
+ | ADDREF_CONST_2 zv, r0
+ }
+ }
+ } else {
+ if (val_info & MAY_BE_UNDEF) {
+ if (in_cold) {
+ | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2
+ } else {
+ | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ }
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ if (Z_REG(var_addr) != ZREG_FP) {
+ | mov aword T1, Ra(Z_REG(var_addr)) // save
+ }
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1d, val.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ if (Z_REG(var_addr) != ZREG_FP) {
+ | mov Ra(Z_REG(var_addr)), aword T1 // restore
+ }
+ | SET_ZVAL_TYPE_INFO var_addr, IS_NULL
+ if (res_addr) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_NULL
+ }
+ | jmp >3
+ if (in_cold) {
+ |2:
+ } else {
+ |.code
+ }
+ }
+ if (val_info & MAY_BE_REF) {
+ if (val_type == IS_CV) {
+ ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2);
+ | LOAD_ZVAL_ADDR r2, val_addr
+ | ZVAL_DEREF r2, val_info
+ val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
+ } else {
+ zend_jit_addr ref_addr;
+
+ if (in_cold) {
+ | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1
+ } else {
+ | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1
+ |.cold_code
+ |1:
+ }
+ | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
+ | GET_ZVAL_PTR r2, val_addr
+ | GC_DELREF r2
+ | // ZVAL_COPY_VALUE(return_value, &ref->value);
+ ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8);
+ if (!res_addr) {
+ | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, ZREG_R0
+ } else {
+ | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, ZREG_R0
+ }
+ | je >2
+ | IF_NOT_REFCOUNTED dh, >3
+ if (!res_addr) {
+ | GC_ADDREF r0
+ } else {
+ | add dword [r0], 2
+ }
+ | jmp >3
+ |2:
+ if (res_addr) {
+ | IF_NOT_REFCOUNTED dh, >2
+ | GC_ADDREF r0
+ |2:
+ }
+ | EFREE_24 aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)], op_array, opline
+ | jmp >3
+ if (in_cold) {
+ |1:
+ } else {
+ |.code
+ }
+ }
+ }
+
+ if (!res_addr) {
+ | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, ZREG_R0
+ } else {
+ | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, ZREG_R0
+ }
+
+ if (val_type == IS_CV) {
+ if (!res_addr) {
+ | TRY_ADDREF val_info, dh, r0
+ } else {
+ | TRY_ADDREF_2 val_info, dh, r0
+ }
+ } else {
+ if (res_addr) {
+ | TRY_ADDREF val_info, dh, r0
+ }
+ }
+ |3:
+ }
+ return 1;
+}
+
+static int zend_jit_assign_to_variable(dasm_State **Dst,
+ const zend_op *opline,
+ zend_op_array *op_array,
+ zend_ssa *ssa,
+ zend_jit_addr var_addr,
+ uint32_t var_info,
+ zend_uchar val_type,
+ znode_op val,
+ zend_jit_addr val_addr,
+ uint32_t val_info,
+ zend_jit_addr res_addr)
+/* Labels: 1,2,3,4,5,8 */
+{
+ //ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL);
+ if (var_info & MAY_BE_REF) {
+ if (Z_MODE(var_addr) != IS_REG || Z_REG(var_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, var_addr
+ var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+ | // if (Z_ISREF_P(variable_ptr)) {
+ | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1
+ | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
+ | GET_Z_PTR FCARG1a, FCARG1a
+ | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
+ | jnz >2
+ | add FCARG1a, offsetof(zend_reference, val)
+ |.cold_code
+ |2:
+ | LOAD_ZVAL_ADDR FCARG2a, val_addr
+ if (val_type == IS_CONST) {
+ | EXT_CALL zend_jit_assign_const_to_typed_ref, r0
+ } else if (val_type == IS_TMP_VAR) {
+ | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0
+ } else if (val_type == IS_VAR) {
+ | EXT_CALL zend_jit_assign_var_to_typed_ref, r0
+ } else if (val_type == IS_CV) {
+ | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ | jmp >8
+ |.code
+ |1:
+ }
+ if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ int in_cold = 0;
+
+ ZEND_ASSERT(Z_REG(var_addr) != ZREG_R0);
+ if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ | IF_ZVAL_REFCOUNTED var_addr, >1
+ |.cold_code
+ |1:
+ in_cold = 1;
+ }
+ | // TODO: support for object->set
+ | // TODO: support for assignment to itself
+ | GET_ZVAL_PTR r0, var_addr
+ | GC_DELREF r0
+ if (RC_MAY_BE_1(var_info)) {
+ if (RC_MAY_BE_N(var_info)) {
+ | jnz >4
+ }
+ | mov aword T1, r0 // save
+ if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, in_cold)) {
+ return 0;
+ }
+ | mov FCARG1a, aword T1 // restore
+ if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
+ | cmp dword [FCARG1a], 0
+ | jnz >8
+ }
+ | ZVAL_DTOR_FUNC var_info, opline
+ | jmp >8
+ |4:
+ }
+ if (RC_MAY_BE_N(var_info)) {
+ if (Z_REG(var_addr) == ZREG_FP) {
+ | GET_ZVAL_PTR FCARG1a, var_addr
+ | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5
+ } else if (Z_REG(var_addr) != ZREG_FCARG1a) {
+ | GET_ZVAL_PTR FCARG1a, var_addr
+ | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5
+ | mov T1, Ra(Z_REG(var_addr)) // save
+ } else {
+ | GET_ZVAL_PTR r0, var_addr
+ | IF_GC_MAY_NOT_LEAK r0, eax, >5
+ | mov T1, Ra(Z_REG(var_addr)) // save
+ | GET_ZVAL_PTR FCARG1a, var_addr
+ }
+ | EXT_CALL gc_possible_root, r0
+ if (Z_REG(var_addr) != ZREG_FP) {
+ | mov Ra(Z_REG(var_addr)), T1 // restore
+ }
+ if (in_cold) {
+ | jmp >5
+ |.code
+ }
+ } else if (in_cold) {
+ ZEND_ASSERT(RC_MAY_BE_1(var_info));
+ |.code
+ }
+ |5:
+ }
+
+ if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, 0)) {
+ return 0;
+ }
+ |8:
+
+ return 1;
+}
+
+static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, op2_info, val_info;
+ zend_jit_addr op1_addr, op2_addr, op3_addr, res_addr;
+
+ if (opline->op1_type != IS_CV) {
+ goto fallback;
+ }
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ val_info = OP1_DATA_INFO();
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+ op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline+1, NULL, -1);
+ if (opline->result_type == IS_UNUSED) {
+ res_addr = 0;
+ } else {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
+ }
+ | SEPARATE_ARRAY op1_addr, op1_info, 1
+ } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
+ | CMP_ZVAL_TYPE op1_addr, IS_FALSE
+ | jg >7
+ }
+ | // ZVAL_ARR(container, zend_new_array(8));
+ if (Z_REG(op1_addr) != ZREG_FP) {
+ | mov T1, Ra(Z_REG(op1_addr)) // save
+ }
+ | EXT_CALL _zend_new_array_0, r0
+ if (Z_REG(op1_addr) != ZREG_FP) {
+ | mov Ra(Z_REG(op1_addr)), T1 // restore
+ }
+ | SET_ZVAL_LVAL op1_addr, r0
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
+ | mov FCARG1a, r0
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
+ |6:
+ if (opline->op2_type == IS_UNUSED) {
+ | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
+ | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
+ | EXT_CALL zend_hash_next_index_insert, r0
+ | // if (UNEXPECTED(!var_ptr)) {
+ | test r0, r0
+ | jz >1
+ |.cold_code
+ |1:
+ | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied");
+ | CANNOT_ADD_ELEMENT opline
+ | //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
+ | jmp >9
+ |.code
+ | mov FCARG1a, r0
+ } else {
+ if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_W, op1_info, op2_info, 8, 8)) {
+ return 0;
+ }
+
+ |8:
+ | mov FCARG1a, r0
+ }
+
+ if (opline->op2_type == IS_UNUSED) {
+ uint32_t var_info = zend_array_element_type(op1_info, 0, 0);
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+
+ if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
+ var_info |= MAY_BE_REF;
+ }
+ if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr, 0)) {
+ return 0;
+ }
+ } else {
+ uint32_t var_info = zend_array_element_type(op1_info, 0, 0);
+ zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+
+ if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
+ var_info |= MAY_BE_REF;
+ }
+ | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
+ if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr)) {
+ return 0;
+ }
+ }
+ }
+
+ if (((op1_info & MAY_BE_ARRAY) &&
+ (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) ||
+ (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) {
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
+ |.cold_code
+ |7:
+ }
+
+ if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) &&
+ (op1_info & MAY_BE_ARRAY)) {
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
+ | CMP_ZVAL_TYPE op1_addr, IS_FALSE
+ | jg >2
+ }
+ | // ZVAL_ARR(container, zend_new_array(8));
+ if (Z_REG(op1_addr) != ZREG_FP) {
+ | mov T1, Ra(Z_REG(op1_addr)) // save
+ }
+ | EXT_CALL _zend_new_array_0, r0
+ if (Z_REG(op1_addr) != ZREG_FP) {
+ | mov Ra(Z_REG(op1_addr)), T1 // restore
+ }
+ | SET_ZVAL_LVAL op1_addr, r0
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
+ | mov FCARG1a, r0
+ | // ZEND_VM_C_GOTO(assign_dim_op_new_array);
+ | jmp <6
+ |2:
+ }
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
+ | SAVE_VALID_OPLINE opline
+ if (Z_REG(op1_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ if (opline->op2_type == IS_UNUSED) {
+ | xor FCARG2a, FCARG2a
+ } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
+ } else {
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ |.if not(X64)
+ | sub r4, 8
+ |.endif
+ if (opline->result_type == IS_UNUSED) {
+ |.if X64
+ | xor CARG4, CARG4
+ |.else
+ | push 0
+ |.endif
+ } else {
+ zend_jit_addr res_addr;
+
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG4, res_addr
+ |.else
+ | PUSH_ZVAL_ADDR res_addr, r0
+ |.endif
+ }
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op3_addr
+ |.else
+ | PUSH_ZVAL_ADDR op3_addr, r0
+ |.endif
+ | EXT_CALL zend_jit_assign_dim_helper, r0
+ |.if not(X64)
+ | add r4, 8
+ |.endif
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
+ /* ASSIGN_DIM may increase refcount of the value */
+ val_info |= MAY_BE_RCN;
+ }
+#endif
+
+ | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, op_array, opline
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
+ | jmp >9 // END
+ }
+ |.code
+ }
+ }
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
+ /* ASSIGN_DIM may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ |9:
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline
+
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception(Dst);
+ }
+
+ return 1;
+
+fallback:
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, op2_info;
+ zend_jit_addr op1_addr, op2_addr, op3_addr, var_addr;
+ int op3_ssa_var;
+
+ if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
+ goto fallback;
+ }
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op3_ssa_var = ssa->ops[opline - op_array->opcodes + 1].op1_use;
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+ op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline+1, NULL, -1);
+
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
+ }
+ | SEPARATE_ARRAY op1_addr, op1_info, 1
+ }
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
+ if (op1_info & MAY_BE_ARRAY) {
+ |.cold_code
+ |7:
+ }
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
+ | CMP_ZVAL_TYPE op1_addr, IS_FALSE
+ | jg >7
+ }
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ }
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1a, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ |1:
+ }
+ | // ZVAL_ARR(container, zend_new_array(8));
+ if (Z_REG(op1_addr) != ZREG_FP) {
+ | mov T1, Ra(Z_REG(op1_addr)) // save
+ }
+ | EXT_CALL _zend_new_array_0, r0
+ if (Z_REG(op1_addr) != ZREG_FP) {
+ | mov Ra(Z_REG(op1_addr)), T1 // restore
+ }
+ | SET_ZVAL_LVAL op1_addr, r0
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
+ | mov FCARG1a, r0
+ if (op1_info & MAY_BE_ARRAY) {
+ | jmp >1
+ |.code
+ |1:
+ }
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
+ uint32_t var_info = zend_array_element_type(op1_info, 0, 0);
+ uint32_t var_def_info = zend_array_element_type(OP1_DEF_INFO(), 1, 0);
+
+ if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
+ var_info |= MAY_BE_REF;
+ }
+ |6:
+ if (opline->op2_type == IS_UNUSED) {
+ | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
+ | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
+ | EXT_CALL zend_hash_next_index_insert, r0
+ | // if (UNEXPECTED(!var_ptr)) {
+ | test r0, r0
+ | jz >1
+ |.cold_code
+ |1:
+ | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied");
+ | CANNOT_ADD_ELEMENT opline
+ | //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
+ | jmp >9
+ |.code
+ | mov FCARG1a, r0
+ } else {
+ if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op1_info, op2_info, 8, 8)) {
+ return 0;
+ }
+
+ |8:
+ | mov FCARG1a, r0
+ if (op1_info & (MAY_BE_ARRAY_OF_REF)) {
+ | ZVAL_DEREF FCARG1a, MAY_BE_REF
+ }
+ }
+
+ var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+ case ZEND_ASSIGN_DIV:
+ if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, var_def_info, var_info)) {
+ return 0;
+ }
+ break;
+ case ZEND_ASSIGN_BW_OR:
+ case ZEND_ASSIGN_BW_AND:
+ case ZEND_ASSIGN_BW_XOR:
+ case ZEND_ASSIGN_SL:
+ case ZEND_ASSIGN_SR:
+ case ZEND_ASSIGN_MOD:
+ if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, -1, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info)) {
+ return 0;
+ }
+ break;
+ case ZEND_ASSIGN_CONCAT:
+ if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO())) {
+ return 0;
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ }
+
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) {
+ binary_op_type binary_op;
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
+ |.cold_code
+ |7:
+ }
+
+ | SAVE_VALID_OPLINE opline
+ if (Z_REG(op1_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ if (opline->op2_type == IS_UNUSED) {
+ | xor FCARG2a, FCARG2a
+ } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
+ } else {
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ binary_op = get_binary_op(opline->opcode);
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op3_addr
+ | LOAD_ADDR CARG4, binary_op
+ |.else
+ | sub r4, 8
+ | PUSH_ADDR binary_op, r0
+ | PUSH_ZVAL_ADDR op3_addr, r0
+ |.endif
+ | EXT_CALL zend_jit_assign_dim_op_helper, r0
+ |.if not(X64)
+ | add r4, 8
+ |.endif
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) {
+ | jmp >9 // END
+ |.code
+ }
+ }
+
+ |9:
+
+ return 1;
+
+fallback:
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, op2_info;
+ zend_jit_addr op1_addr, op2_addr;
+ int op1_ssa_var, op2_ssa_var;
+
+ if (opline->extended_value == ZEND_ASSIGN_DIM) {
+ return zend_jit_assign_dim_op(Dst, opline, op_array, ssa);
+ } else if (opline->extended_value == ZEND_ASSIGN_OBJ || opline->extended_value == ZEND_ASSIGN_STATIC_PROP) {
+ goto fallback;
+ }
+
+ if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
+ goto fallback;
+ }
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_ssa_var = ssa->ops[opline - op_array->opcodes].op1_use;
+ op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use;
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+
+ if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
+ goto fallback;
+ }
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+ case ZEND_ASSIGN_DIV:
+ if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
+ !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ goto fallback;
+ }
+ break;
+ case ZEND_ASSIGN_BW_OR:
+ case ZEND_ASSIGN_BW_AND:
+ case ZEND_ASSIGN_BW_XOR:
+ case ZEND_ASSIGN_SL:
+ case ZEND_ASSIGN_SR:
+ case ZEND_ASSIGN_MOD:
+ if (!(op1_info & MAY_BE_LONG) ||
+ !(op2_info & MAY_BE_LONG)) {
+ goto fallback;
+ }
+ break;
+ case ZEND_ASSIGN_CONCAT:
+ if (!(op1_info & MAY_BE_STRING) ||
+ !(op2_info & MAY_BE_STRING)) {
+ goto fallback;
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+
+ switch (opline->opcode) {
+ case ZEND_ASSIGN_ADD:
+ case ZEND_ASSIGN_SUB:
+ case ZEND_ASSIGN_MUL:
+ case ZEND_ASSIGN_DIV:
+ return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info);
+ case ZEND_ASSIGN_BW_OR:
+ case ZEND_ASSIGN_BW_AND:
+ case ZEND_ASSIGN_BW_XOR:
+ case ZEND_ASSIGN_SL:
+ case ZEND_ASSIGN_SR:
+ case ZEND_ASSIGN_MOD:
+ return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info);
+ case ZEND_ASSIGN_CONCAT:
+ return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO());
+ default:
+ ZEND_ASSERT(0);
+ }
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr)
+{
+ unsigned int target_label;
+ zend_bool swap = 0;
+
+ if (Z_MODE(op1_addr) == IS_REG) {
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
+ | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr))
+ } else {
+ | LONG_OP cmp, Z_REG(op1_addr), op2_addr
+ }
+ } else if (Z_MODE(op2_addr) == IS_REG) {
+ if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) {
+ | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr))
+ } else {
+ | LONG_OP cmp, Z_REG(op2_addr), op1_addr
+ }
+ swap = 1;
+ } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) {
+ | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr))
+ swap = 1;
+ } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) {
+ | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr))
+ } else {
+ | GET_ZVAL_LVAL ZREG_R0, op1_addr
+ if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
+ | test r0, r0
+ } else {
+ | LONG_OP cmp, ZREG_R0, op2_addr
+ }
+ }
+ if (((opline+1)->opcode == ZEND_JMPZ_EX ||
+ (opline+1)->opcode == ZEND_JMPNZ_EX) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | sete al
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | setne al
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | setg al
+ } else {
+ | setl al
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | setge al
+ } else {
+ | setle al
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ | movzx eax, al
+ | lea eax, [eax + 2]
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPZ_EX) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jne => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | je => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | jle => target_label
+ } else {
+ | jge => target_label
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | jl => target_label
+ } else {
+ | jg => target_label
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if (((opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPNZ_EX) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | je => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | jne => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | jg => target_label
+ } else {
+ | jl => target_label
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | jge => target_label
+ } else {
+ | jle => target_label
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jne => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | je => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | jle => target_label
+ } else {
+ | jge => target_label
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | jl => target_label
+ } else {
+ | jg => target_label
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp => target_label
+ } else {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | sete al
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | setne al
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | setg al
+ } else {
+ | setl al
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | setge al
+ } else {
+ | setle al
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ | movzx eax, al
+ | add eax, 2
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+
+ return 1;
+}
+
+static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_bool swap)
+{
+ unsigned int target_label;
+
+ if ((opline+1)->opcode == ZEND_JMPZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jne => target_label
+ | jp => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | jp >1
+ | je => target_label
+ |1:
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | jbe => target_label
+ } else {
+ | jae => target_label
+ | jp => target_label
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | jb => target_label
+ } else {
+ | ja => target_label
+ | jp => target_label
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jp >1
+ | je => target_label
+ |1:
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | jne => target_label
+ | jp => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | ja => target_label
+ } else {
+ | jp >1
+ | jb => target_label
+ |1:
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | jae => target_label
+ } else {
+ | jp >1
+ | jbe => target_label
+ |1:
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ unsigned int target_label2 = ssa->cfg.blocks[b].successors[1];
+
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jne => target_label
+ | jp => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | jp => target_label2
+ | je => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | jbe => target_label
+ } else {
+ | jae => target_label
+ | jp => target_label
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | jb => target_label
+ } else {
+ | ja => target_label
+ | jp => target_label
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ | jmp => target_label2
+ } else if ((opline+1)->opcode == ZEND_JMPZ_EX &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jne => target_label
+ | jp => target_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | jp >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | je => target_label
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jbe => target_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jae => target_label
+ | jp => target_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jb => target_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | ja => target_label
+ | jp => target_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPNZ_EX &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jp >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | je => target_label
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | jne => target_label
+ | jp => target_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | seta al
+ | movzx eax, al
+ | lea eax, [eax + 2]
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ | ja => target_label
+ } else {
+ | jp >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | jb => target_label
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | setae al
+ | movzx eax, al
+ | lea eax, [eax + 2]
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ | jae => target_label
+ } else {
+ | jp >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | jbe => target_label
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_CASE:
+ | jp >1
+ | mov eax, IS_TRUE
+ | je >2
+ |1:
+ | mov eax, IS_FALSE
+ |2:
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ | jp >1
+ | mov eax, IS_FALSE
+ | je >2
+ |1:
+ | mov eax, IS_TRUE
+ |2:
+ break;
+ case ZEND_IS_SMALLER:
+ if (swap) {
+ | seta al
+ | movzx eax, al
+ | add eax, 2
+ } else {
+ | jp >1
+ | mov eax, IS_TRUE
+ | jb >2
+ |1:
+ | mov eax, IS_FALSE
+ |2:
+ }
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ if (swap) {
+ | setae al
+ | movzx eax, al
+ | add eax, 2
+ } else {
+ | jp >1
+ | mov eax, IS_TRUE
+ | jbe >2
+ |1:
+ | mov eax, IS_FALSE
+ |2:
+ }
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+
+ return 1;
+}
+
+static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr)
+{
+ |.if SSE
+ zend_reg tmp_reg = ZREG_XMM0;
+
+ | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr
+ | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr
+ |.else
+ | FPU_GET_ZVAL_DVAL op2_addr
+ | FPU_GET_ZVAL_LVAL op1_addr
+ | fucomip st1
+ | fstp st0
+ |.endif
+
+ return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, 0);
+}
+
+static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr)
+{
+ zend_bool swap = 0;
+
+ |.if SSE
+ zend_reg tmp_reg = ZREG_XMM0;
+
+ | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr
+ | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr
+ swap = 1;
+ |.else
+ | FPU_GET_ZVAL_LVAL op2_addr
+ | FPU_GET_ZVAL_DVAL op1_addr
+ | fucomip st1
+ | fstp st0
+ |.endif
+
+ return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap);
+}
+
+static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr)
+{
+ zend_bool swap = 0;
+
+ |.if SSE
+ if (Z_MODE(op1_addr) == IS_REG) {
+ | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr
+ } else if (Z_MODE(op2_addr) == IS_REG) {
+ | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr
+ swap = 1;
+ } else {
+ zend_reg tmp_reg = ZREG_XMM0;
+
+ | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr
+ | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr
+ }
+ |.else
+ | FPU_GET_ZVAL_DVAL op2_addr
+ | FPU_GET_ZVAL_DVAL op1_addr
+ | fucomip st1
+ | fstp st0
+ |.endif
+
+ return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap);
+}
+
+static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa)
+{
+ unsigned int target_label;
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0)
+ if (((opline+1)->opcode == ZEND_JMPZ_EX ||
+ (opline+1)->opcode == ZEND_JMPNZ_EX) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_CASE:
+ | sete al
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ | setne al
+ break;
+ case ZEND_IS_SMALLER:
+ | setl al
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ | setle al
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ | movzx eax, al
+ | lea eax, [eax + 2]
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPZ_EX) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_CASE:
+ | jne => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ | je => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ | jge => target_label
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ | jg => target_label
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if (((opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPNZ_EX) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_CASE:
+ | je => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ | jne => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ | jl => target_label
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ | jle => target_label
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_CASE:
+ | jne => target_label
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ | je => target_label
+ break;
+ case ZEND_IS_SMALLER:
+ | jge => target_label
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ | jg => target_label
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp => target_label
+ } else {
+ switch (opline->opcode) {
+ case ZEND_IS_EQUAL:
+ case ZEND_CASE:
+ | sete al
+ break;
+ case ZEND_IS_NOT_EQUAL:
+ | setne al
+ break;
+ case ZEND_IS_SMALLER:
+ | setl al
+ break;
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ | setle al
+ break;
+ default:
+ ZEND_ASSERT(0);
+ }
+ | movzx eax, al
+ | add eax, 2
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+
+ return 1;
+}
+
+static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info, op2_info;
+ zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var);
+ zend_bool has_slow;
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1);
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1);
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+
+ has_slow =
+ (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
+ ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
+
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
+ if (op1_info & MAY_BE_DOUBLE) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4
+ } else {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
+ }
+ }
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
+ if (op2_info & MAY_BE_DOUBLE) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
+ |.cold_code
+ |3:
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
+ }
+ if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ | jmp >6
+ |.code
+ } else {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
+ }
+ }
+ if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ |.cold_code
+ |4:
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if (!same_ops) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5
+ } else {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
+ }
+ }
+ if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ | jmp >6
+ }
+ if (!same_ops) {
+ |5:
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
+ }
+ if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ | jmp >6
+ }
+ |.code
+ }
+ } else if ((op1_info & MAY_BE_DOUBLE) &&
+ !(op1_info & MAY_BE_LONG) &&
+ (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if (!same_ops && (op2_info & MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3
+ } else {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
+ }
+ }
+ if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ }
+ if (!same_ops && (op2_info & MAY_BE_LONG)) {
+ if (op2_info & MAY_BE_DOUBLE) {
+ |.cold_code
+ }
+ |3:
+ if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
+ }
+ if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ if (op2_info & MAY_BE_DOUBLE) {
+ | jmp >6
+ |.code
+ }
+ }
+ } else if ((op2_info & MAY_BE_DOUBLE) &&
+ !(op2_info & MAY_BE_LONG) &&
+ (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
+ if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
+ if (!same_ops && (op1_info & MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3
+ } else {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
+ }
+ }
+ if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ }
+ if (!same_ops && (op1_info & MAY_BE_LONG)) {
+ if (op1_info & MAY_BE_DOUBLE) {
+ |.cold_code
+ }
+ |3:
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
+ }
+ if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ if (op1_info & MAY_BE_DOUBLE) {
+ | jmp >6
+ |.code
+ }
+ }
+ }
+
+ if (has_slow ||
+ (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
+ (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ if (has_slow) {
+ |.cold_code
+ |9:
+ }
+ | SAVE_VALID_OPLINE opline
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op1_addr
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1
+ | mov FCARG1a, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
+ |1:
+ }
+ if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
+ | mov T1, FCARG2a // save
+ | mov FCARG1a, opline->op2.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ | mov FCARG2a, T1 // restore
+ |.if X64
+ | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval
+ |.else
+ | sub r4, 12
+ | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0
+ |.endif
+ | jmp >2
+ |1:
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op2_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR op2_addr, r0
+ |.endif
+ |2:
+ } else {
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, op2_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR op2_addr, r0
+ |.endif
+ }
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ | EXT_CALL compare_function, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ if (opline->opcode != ZEND_CASE) {
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline
+ }
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ if (!zend_jit_cmp_slow(Dst, opline, b, op_array, ssa)) {
+ return 0;
+ }
+ if (has_slow) {
+ | jmp >6
+ |.code
+ }
+ }
+
+ |6:
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPZ_EX ||
+ (opline+1)->opcode == ZEND_JMPNZ_EX ||
+ (opline+1)->opcode == ZEND_JMPZNZ) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ (*opnum)++;
+ }
+
+ return 1;
+}
+
+static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ zend_bool smart_branch = 0;
+ uint32_t identical_label = (uint32_t)-1;
+ uint32_t not_identical_label = (uint32_t)-1;
+ uint32_t op1_info, op2_info;
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1);
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1);
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPZNZ) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ (*opnum)++;
+ smart_branch = 1;
+ }
+
+ if (smart_branch) {
+ if (opline->opcode == ZEND_IS_IDENTICAL) {
+ if ((opline+1)->opcode == ZEND_JMPZ) {
+ not_identical_label = ssa->cfg.blocks[b].successors[0];
+ } else if ((opline+1)->opcode == ZEND_JMPNZ) {
+ identical_label = ssa->cfg.blocks[b].successors[0];
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ) {
+ not_identical_label = ssa->cfg.blocks[b].successors[0];
+ identical_label = ssa->cfg.blocks[b].successors[1];
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
+ if ((opline+1)->opcode == ZEND_JMPZ) {
+ identical_label = ssa->cfg.blocks[b].successors[0];
+ } else if ((opline+1)->opcode == ZEND_JMPNZ) {
+ not_identical_label = ssa->cfg.blocks[b].successors[0];
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ) {
+ identical_label = ssa->cfg.blocks[b].successors[0];
+ not_identical_label = ssa->cfg.blocks[b].successors[1];
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ }
+
+ if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) {
+ op1_info |= MAY_BE_NULL;
+ op2_info |= MAY_BE_NULL;
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | IF_Z_TYPE FCARG1a, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
+ | jmp >1
+ |.code
+ |1:
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ | IF_Z_TYPE FCARG2a, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ | SAVE_VALID_OPLINE opline
+ | mov aword T1, FCARG1a // save
+ | mov FCARG1d, opline->op2.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ | mov FCARG1a, aword T1 // restore
+ | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
+ | jmp >1
+ |.code
+ |1:
+ } else if (op1_info & MAY_BE_UNDEF) {
+ op1_info |= MAY_BE_NULL;
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | IF_Z_TYPE FCARG1a, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
+ | jmp >1
+ |.code
+ |1:
+ if (opline->op2_type != IS_CONST) {
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ } else if (op2_info & MAY_BE_UNDEF) {
+ op2_info |= MAY_BE_NULL;
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ | IF_Z_TYPE FCARG2a, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1d, opline->op2.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
+ | jmp >1
+ |.code
+ |1:
+ if (opline->op1_type != IS_CONST) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ } else {
+ if (opline->op1_type != IS_CONST) {
+ if (Z_MODE(op1_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+ if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
+ return 0;
+ }
+ op1_addr = real_addr;
+ }
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ if (opline->op2_type != IS_CONST) {
+ if (Z_MODE(op2_addr) == IS_REG) {
+ zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
+ if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
+ return 0;
+ }
+ op2_addr = real_addr;
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ }
+ if (opline->op1_type & (IS_CV|IS_VAR)) {
+ | ZVAL_DEREF FCARG1a, op1_info
+ }
+ if (opline->op2_type & (IS_CV|IS_VAR)) {
+ | ZVAL_DEREF FCARG2a, op2_info
+ }
+
+ if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
+ if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
+ ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
+ | SAVE_VALID_OPLINE opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline
+ }
+ if (smart_branch) {
+ zend_jit_check_exception_undef_result(Dst, opline);
+ if (not_identical_label != (uint32_t)-1) {
+ | jmp =>not_identical_label
+ }
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE)
+ zend_jit_check_exception(Dst);
+ }
+ } else if (has_concrete_type(op1_info) &&
+ has_concrete_type(op2_info) &&
+ concrete_type(op1_info) == concrete_type(op2_info) &&
+ concrete_type(op1_info) <= IS_TRUE) {
+ if (smart_branch) {
+ if (identical_label != (uint32_t)-1) {
+ | jmp =>identical_label
+ }
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE)
+ }
+ } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
+ if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
+ if (smart_branch) {
+ if (identical_label != (uint32_t)-1) {
+ | jmp =>identical_label
+ }
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE)
+ }
+ } else {
+ if (smart_branch) {
+ if (not_identical_label != (uint32_t)-1) {
+ | jmp =>not_identical_label
+ }
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE)
+ }
+ }
+ } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
+ zval *val = Z_ZV(op1_addr);
+
+ | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val)
+ if (smart_branch) {
+ if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) {
+ | jne >8
+ | SAVE_VALID_OPLINE opline
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline
+ zend_jit_check_exception_undef_result(Dst, opline);
+ if (identical_label != (uint32_t)-1) {
+ | jmp =>identical_label
+ } else {
+ | jmp >9
+ }
+ |8:
+ } else if (identical_label != (uint32_t)-1) {
+ | je =>identical_label
+ } else {
+ | je >9
+ }
+ } else {
+ if (opline->opcode == ZEND_IS_IDENTICAL) {
+ | sete al
+ } else {
+ | setne al
+ }
+ | movzx eax, al
+ | lea eax, [eax + 2]
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
+ | SAVE_VALID_OPLINE opline
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ if (smart_branch && not_identical_label != (uint32_t)-1) {
+ | jmp =>not_identical_label
+ }
+ } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
+ zval *val = Z_ZV(op2_addr);
+
+ | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val)
+ if (smart_branch) {
+ if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) {
+ | jne >8
+ | SAVE_VALID_OPLINE opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ zend_jit_check_exception_undef_result(Dst, opline);
+ if (identical_label != (uint32_t)-1) {
+ | jmp =>identical_label
+ } else {
+ | jmp >9
+ }
+ |8:
+ } else if (identical_label != (uint32_t)-1) {
+ | je =>identical_label
+ } else {
+ | je >9
+ }
+ } else {
+ if (opline->opcode == ZEND_IS_IDENTICAL) {
+ | sete al
+ } else {
+ | setne al
+ }
+ | movzx eax, al
+ | lea eax, [eax + 2]
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
+ | SAVE_VALID_OPLINE opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ zend_jit_check_exception_undef_result(Dst, opline);
+ }
+ if (smart_branch && not_identical_label != (uint32_t)-1) {
+ | jmp =>not_identical_label
+ }
+ } else if ((op1_info & MAY_BE_ANY) == MAY_BE_LONG &&
+ (op2_info & MAY_BE_ANY) == MAY_BE_LONG) {
+ if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ } else if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE &&
+ (op2_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) {
+ return 0;
+ }
+ } else {
+ if (opline->op1_type == IS_CONST) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ if (opline->op2_type == IS_CONST) {
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ | EXT_CALL zend_is_identical, r0
+ if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
+ ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
+ | mov aword T1, r0 // save
+ | SAVE_VALID_OPLINE opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline
+ zend_jit_check_exception_undef_result(Dst, opline);
+ | mov r0, aword T1 // restore
+ }
+ if (smart_branch) {
+ | test al, al
+ if (not_identical_label != (uint32_t)-1) {
+ | jz =>not_identical_label
+ if (identical_label != (uint32_t)-1) {
+ | jmp =>identical_label
+ }
+ } else if (identical_label != (uint32_t)-1) {
+ | jnz =>identical_label
+ }
+ } else {
+ | movzx eax, al
+ if (opline->opcode == ZEND_IS_IDENTICAL) {
+ | lea eax, [eax + 2]
+ } else {
+ | neg eax
+ | lea eax, [eax + 3]
+ }
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ }
+
+ |9:
+
+ return 1;
+}
+
+static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info = OP1_INFO();
+ uint32_t true_label = -1;
+ uint32_t false_label = -1;
+ zend_bool set_bool = 0;
+ zend_bool set_bool_not = 0;
+ zend_bool jmp_done = 0;
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ zend_jit_addr res_addr = 0;
+
+ if (opline->opcode == ZEND_JMPZ) {
+ false_label = ssa->cfg.blocks[b].successors[0];
+ } else if (opline->opcode == ZEND_JMPNZ) {
+ true_label = ssa->cfg.blocks[b].successors[0];
+ } else if (opline->opcode == ZEND_JMPZNZ) {
+ true_label = ssa->cfg.blocks[b].successors[1];
+ false_label = ssa->cfg.blocks[b].successors[0];
+ } else if (opline->opcode == ZEND_BOOL) {
+ set_bool = 1;
+ } else if (opline->opcode == ZEND_BOOL_NOT) {
+ set_bool = 1;
+ set_bool_not = 1;
+ } else if (opline->opcode == ZEND_JMPZ_EX) {
+ set_bool = 1;
+ false_label = ssa->cfg.blocks[b].successors[0];
+ } else if (opline->opcode == ZEND_JMPNZ_EX) {
+ set_bool = 1;
+ true_label = ssa->cfg.blocks[b].successors[0];
+ } else {
+ ZEND_ASSERT(0);
+ }
+
+ if (set_bool) {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1);
+ }
+
+ if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
+ if (zend_is_true(Z_ZV(op1_addr))) {
+ /* Always TRUE */
+ if (set_bool) {
+ if (set_bool_not) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ }
+ }
+ if (true_label != (uint32_t)-1) {
+ | jmp =>true_label;
+ }
+ } else {
+ /* Always FALSE */
+ if (set_bool) {
+ if (set_bool_not) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ }
+ }
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label;
+ }
+ }
+ return 1;
+ }
+
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
+ if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
+ /* Always TRUE */
+ if (set_bool) {
+ if (set_bool_not) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ }
+ }
+ if (true_label != (uint32_t)-1) {
+ | jmp =>true_label;
+ }
+ } else {
+ if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
+ /* Always FALSE */
+ if (set_bool) {
+ if (set_bool_not) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ }
+ }
+ } else {
+ | CMP_ZVAL_TYPE op1_addr, IS_TRUE
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
+ if ((op1_info & MAY_BE_LONG) &&
+ !(op1_info & MAY_BE_UNDEF) &&
+ !set_bool) {
+ if (false_label != (uint32_t)-1) {
+ | jl =>false_label
+ } else {
+ | jl >9
+ }
+ jmp_done = 1;
+ } else {
+ | jg >2
+ }
+ }
+ if (!(op1_info & MAY_BE_TRUE)) {
+ /* It's FALSE */
+ if (set_bool) {
+ if (set_bool_not) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ }
+ }
+ } else {
+ if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
+ if (set_bool) {
+ | jne >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ if (true_label != (uint32_t)-1) {
+ | jmp =>true_label
+ } else {
+ | jmp >9
+ }
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ } else {
+ if (true_label != (uint32_t)-1) {
+ | je =>true_label
+ } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
+ | jne =>false_label
+ jmp_done = 1;
+ } else {
+ | je >9
+ }
+ }
+ } else if (set_bool) {
+ | sete al
+ | movzx eax, al
+ if (set_bool_not) {
+ | neg eax
+ | add eax, 3
+ } else {
+ | add eax, 2
+ }
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ }
+ }
+
+ /* It's FALSE, but may be UNDEF */
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & MAY_BE_ANY) {
+ | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ }
+ | mov FCARG1d, opline->op1.var
+ | SAVE_VALID_OPLINE opline
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception_undef_result(Dst, opline)) {
+ return 0;
+ }
+ }
+
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ }
+ if (op1_info & MAY_BE_ANY) {
+ if (false_label == (uint32_t)-1) {
+ | jmp >9
+ }
+ |.code
+ }
+ }
+
+ if (!jmp_done) {
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ } else if (op1_info & MAY_BE_LONG) {
+ | jmp >9
+ }
+ }
+ }
+ }
+
+ if (op1_info & MAY_BE_LONG) {
+ |2:
+ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
+ }
+ if (Z_MODE(op1_addr) == IS_REG) {
+ | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr))
+ } else {
+ | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0)
+ }
+ if (set_bool) {
+ | setne al
+ | movzx eax, al
+ if (set_bool_not) {
+ | neg eax
+ | add eax, 3
+ } else {
+ | lea eax, [eax + 2]
+ }
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
+ if (true_label != (uint32_t)-1) {
+ | jne =>true_label
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ }
+ } else {
+ | je =>false_label
+ }
+ }
+ }
+
+ if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
+ |.if SSE
+ if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) {
+ | vxorps xmm0, xmm0, xmm0
+ } else {
+ | xorps xmm0, xmm0
+ }
+ | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr
+ |.else
+ | FPU_GET_ZVAL_DVAL op1_addr
+ | fldz
+ | fucomip st1
+ | fstp st0
+ |.endif
+
+ if (set_bool) {
+ if (false_label != (uint32_t)-1) { // JMPZ_EX
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jp >1
+ | je => false_label
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ |1:
+ } else if (true_label != (uint32_t)-1) { // JMPNZ_EX
+ | jp >1
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | jne => true_label
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ } else if (set_bool_not) { // BOOL_NOT
+ | jp >1
+ | mov eax, IS_TRUE
+ | je >2
+ |1:
+ | mov eax, IS_FALSE
+ |2:
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ } else { // BOOL
+ | jp >1
+ | mov eax, IS_TRUE
+ | jne >2
+ |1:
+ | mov eax, IS_FALSE
+ |2:
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ }
+ } else {
+ ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1);
+ if (false_label != (uint32_t)-1) {
+ | jp =>false_label
+ } else {
+ | jp >1
+ }
+ if (true_label != (uint32_t)-1) {
+ | jne =>true_label
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ }
+ } else {
+ | je =>false_label
+ }
+ |1:
+ }
+ } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ |.cold_code
+ |2:
+ }
+ if (Z_REG(op1_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ | SAVE_VALID_OPLINE opline
+ | EXT_CALL zend_is_true, r0
+
+ if (set_bool) {
+ if (set_bool_not) {
+ | neg eax
+ | add eax, 3
+ } else {
+ | add eax, 2
+ }
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ | FREE_OP opline->op1_type, opline->op1, op1_info, !(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)), op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception_undef_result(Dst, opline)) {
+ return 0;
+ }
+ }
+ if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
+ | CMP_ZVAL_TYPE res_addr, IS_FALSE
+ if (true_label != (uint32_t)-1) {
+ | jne =>true_label
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ | jmp >9
+ }
+ } else {
+ | je =>false_label
+ }
+ }
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ | jmp >9
+ |.code
+ }
+ } else {
+
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
+
+ | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ | GC_DELREF FCARG1a
+ | jnz >3
+ | mov aword T1, r0 // save
+ | ZVAL_DTOR_FUNC op1_info, opline
+ | mov r0, aword T1 // restore
+ |3:
+ }
+ if (zend_may_throw(opline, op_array, ssa)) {
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1
+ | jne ->exception_handler_undef
+ }
+
+ | test r0, r0
+ if (true_label != (uint32_t)-1) {
+ | jne =>true_label
+ if (false_label != (uint32_t)-1) {
+ | jmp =>false_label
+ } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ | jmp >9
+ }
+ } else {
+ | je =>false_label
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ | jmp >9
+ }
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
+ |.code
+ }
+ }
+ }
+
+ |9:
+
+ return 1;
+}
+
+static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info = OP1_INFO();
+ zend_jit_addr op1_addr, res_addr;
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1);
+
+ if (ra
+ && ssa->ops[opline - op_array->opcodes].op1_def >= 0
+ && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
+ zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def);
+
+ if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info, ra[ssa->ops[opline - op_array->opcodes].op1_use])) {
+ return 0;
+ }
+ }
+
+ if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0)) {
+ return 0;
+ }
+ if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) {
+ return 0;
+ }
+ return 1;
+}
+
+static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info, op2_info;
+ zend_jit_addr op1_addr, op2_addr, res_addr;
+
+ if (opline->op1_type != IS_CV || !ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1);
+ if (opline->result_type == IS_UNUSED) {
+ res_addr = 0;
+ } else {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1);
+ }
+
+ if (ra
+ && ssa->ops[opline - op_array->opcodes].op2_def >= 0
+ && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
+ zend_jit_addr op2_def_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_def);
+
+ if (!zend_jit_update_regs(Dst, op2_addr, op2_def_addr, op2_info, ra[ssa->ops[opline - op_array->opcodes].op2_use])) {
+ return 0;
+ }
+ }
+
+ if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr)) {
+ return 0;
+ }
+
+ if (zend_may_throw(opline, op_array, ssa)) {
+ zend_jit_check_exception(Dst);
+ }
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_function *func)
+{
+ uint32_t used_stack;
+
+ if (func) {
+ used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
+ } else {
+ used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval);
+
+ | // if (EXPECTED(ZEND_USER_CODE(func->type))) {
+ | test byte [r0 + offsetof(zend_function, type)], 1
+ | mov FCARG1a, used_stack
+ | jnz >1
+ | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
+ | mov edx, opline->extended_value
+ | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
+ | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
+ | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)]
+ | sub edx, dword [r0 + offsetof(zend_function, op_array.T)]
+ | shl edx, 5
+ |.if X64
+ | movsxd r2, edx
+ |.endif
+ | sub FCARG1a, r2
+ |1:
+ }
+
+ zend_jit_start_reuse_ip();
+
+ | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
+ | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX
+ | // Check Stack Overflow
+ | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2
+ | sub r2, RX
+ if (func) {
+ | cmp r2, used_stack
+ } else {
+ | cmp r2, FCARG1a
+ }
+ | jb >1
+ | // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
+ |.cold_code
+ |1:
+ if (func) {
+ | mov FCARG1d, used_stack
+ }
+#ifdef _WIN32
+ if (0) {
+#else
+ if (func && func->type == ZEND_INTERNAL_FUNCTION) {
+#endif
+ | EXT_CALL zend_jit_int_extend_stack_helper, r0
+ } else {
+ | mov FCARG2a, r0
+ | EXT_CALL zend_jit_extend_stack_helper, r0
+ }
+ | mov RX, r0
+ | jmp >1
+ |.code
+
+ if (func) {
+ | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2
+ } else {
+ | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2
+ }
+ | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
+ | // ZEND_SET_CALL_INFO(call, 0, call_info);
+ | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)
+ | // call->func = func;
+#ifdef _WIN32
+ if (0) {
+#else
+ if (func && func->type == ZEND_INTERNAL_FUNCTION) {
+#endif
+ |1:
+ | ADDR_OP2_2 mov, aword EX:RX->func, func, r1
+ } else {
+ | mov aword EX:RX->func, r0
+ |1:
+ }
+ | // Z_CE(call->This) = called_scope;
+ | mov aword EX:RX->This.value.ptr, 0
+ | // ZEND_CALL_NUM_ARGS(call) = num_args;
+ | mov dword EX:RX->This.u2.num_args, opline->extended_value
+ return 1;
+}
+
+static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline)
+{
+ int skip;
+
+ if (!call_info) {
+ const zend_op *end = op_array->opcodes + op_array->last;
+
+ opline++;
+ skip = 1;
+ while (opline != end) {
+ if (!skip) {
+ if (zend_may_throw(opline, op_array, ssa)) {
+ return 1;
+ }
+ }
+ switch (opline->opcode) {
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAL_EX:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_FUNC_ARG:
+ case ZEND_SEND_REF:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ skip = 0;
+ break;
+ case ZEND_SEND_ARRAY:
+ case ZEND_SEND_USER:
+ case ZEND_SEND_UNPACK:
+ case ZEND_INIT_FCALL:
+ case ZEND_INIT_METHOD_CALL:
+ case ZEND_INIT_STATIC_METHOD_CALL:
+ case ZEND_INIT_FCALL_BY_NAME:
+ case ZEND_INIT_NS_FCALL_BY_NAME:
+ case ZEND_INIT_DYNAMIC_CALL:
+ case ZEND_NEW:
+ case ZEND_INIT_USER_CALL:
+ case ZEND_FAST_CALL:
+ case ZEND_JMP:
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ case ZEND_FE_RESET_R:
+ case ZEND_FE_RESET_RW:
+ case ZEND_JMP_SET:
+ case ZEND_COALESCE:
+ case ZEND_ASSERT_CHECK:
+ case ZEND_CATCH:
+ case ZEND_DECLARE_ANON_CLASS:
+ case ZEND_FE_FETCH_R:
+ case ZEND_FE_FETCH_RW:
+ return 1;
+ case ZEND_DO_ICALL:
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ case ZEND_DO_FCALL:
+ end = opline;
+ if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
+ /* INIT_FCALL and DO_FCALL in different BasicBlocks */
+ return 1;
+ }
+ return 0;
+ }
+ opline++;
+ }
+
+ return 1;
+ } else {
+ const zend_op *end = call_info->caller_call_opline;
+
+ if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
+ /* INIT_FCALL and DO_FCALL in different BasicBlocks */
+ return 1;
+ }
+
+ opline++;
+ skip = 1;
+ while (opline != end) {
+ if (skip) {
+ switch (opline->opcode) {
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAL_EX:
+ case ZEND_SEND_VAR_EX:
+ case ZEND_SEND_FUNC_ARG:
+ case ZEND_SEND_REF:
+ case ZEND_SEND_VAR_NO_REF:
+ case ZEND_SEND_VAR_NO_REF_EX:
+ skip = 0;
+ break;
+ case ZEND_SEND_ARRAY:
+ case ZEND_SEND_USER:
+ case ZEND_SEND_UNPACK:
+ return 1;
+ }
+ } else {
+ if (zend_may_throw(opline, op_array, ssa)) {
+ return 1;
+ }
+ }
+ opline++;
+ }
+
+ return 0;
+ }
+}
+
+static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, int call_level)
+{
+ zend_func_info *info = ZEND_FUNC_INFO(op_array);
+ zend_call_info *call_info = NULL;
+ zend_function *func = NULL;
+
+ if (delayed_call_chain) {
+ if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
+ return 0;
+ }
+ }
+
+ if (info) {
+ call_info = info->callee_info;
+ while (call_info && call_info->caller_init_opline != opline) {
+ call_info = call_info->next_callee;
+ }
+ if (call_info && call_info->callee_func) {
+ func = call_info->callee_func;
+ }
+ }
+
+#ifdef _WIN32
+ if (0) {
+#else
+ if (func && func->type == ZEND_INTERNAL_FUNCTION) {
+#endif
+ /* load constant address later */
+ } else if (func && op_array == &func->op_array) {
+ /* recursive call */
+ | mov r0, EX->func
+ } else {
+ | // if (CACHED_PTR(opline->result.num))
+ | mov r0, EX->run_time_cache
+ | mov r0, aword [r0 + opline->result.num]
+ | test r0, r0
+ | jz >1
+ |.cold_code
+ |1:
+ if (func && func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
+ | LOAD_ADDR FCARG1a, func
+ | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0
+ | mov r1, EX->run_time_cache
+ | mov aword [r1 + opline->result.num], r0
+ | jmp >3
+ } else {
+ zval *zv = RT_CONSTANT(opline, opline->op2);
+
+ if (opline->opcode == ZEND_INIT_FCALL) {
+ | LOAD_ADDR FCARG1a, Z_STR_P(zv);
+ } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
+ | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1);
+ } else {
+ ZEND_ASSERT(0);
+ }
+ | EXT_CALL zend_jit_find_func_helper, r0
+ | // CACHE_PTR(opline->result.num, fbc);
+ | mov r1, EX->run_time_cache
+ | mov aword [r1 + opline->result.num], r0
+ | test r0, r0
+ | jnz >3
+ | // SAVE_OPLINE();
+ | SAVE_VALID_OPLINE opline
+ | jmp ->undefined_function
+ }
+ |.code
+ |3:
+ }
+
+ if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) {
+ return 0;
+ }
+
+ if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) {
+ if (!zend_jit_save_call_chain(Dst, call_level)) {
+ return 0;
+ }
+ } else {
+ delayed_call_chain = 1;
+ delayed_call_level = call_level;
+ }
+
+ return 1;
+}
+
+static uint32_t skip_valid_arguments(zend_op_array *op_array, zend_ssa *ssa, zend_call_info *call_info)
+{
+ uint32_t num_args = 0;
+ zend_function *func = call_info->callee_func;
+
+ while (num_args < call_info->num_args) {
+ zend_arg_info *arg_info = func->op_array.arg_info + num_args;
+
+ if (ZEND_TYPE_IS_SET(arg_info->type)) {
+ if (!ZEND_TYPE_IS_CLASS(arg_info->type)) {
+ unsigned char code = ZEND_TYPE_CODE(arg_info->type);
+ uint32_t info = _ssa_op1_info(op_array, ssa, call_info->arg_info[num_args].opline);
+
+ if (code == _IS_BOOL) {
+ if (info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_FALSE|MAY_BE_TRUE))) {
+ break;
+ }
+ } else if (code <= IS_RESOURCE) {
+ if (info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (1 << code))) {
+ break;
+ }
+ } else {
+ break;
+ }
+ } else {
+ break;
+ }
+ }
+ num_args++;
+ }
+ return num_args;
+}
+
+static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block)
+{
+ zend_func_info *info = ZEND_FUNC_INFO(op_array);
+ zend_call_info *call_info = NULL;
+ zend_function *func = NULL;
+ uint32_t i;
+ zend_jit_addr res_addr;
+
+ if (RETURN_VALUE_USED(opline)) {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+ } else {
+#ifdef _WIN64
+ /* Reuse reserved arguments stack */
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 0x20);
+#else
+ /* CPU stack alocated temorary zval */
+ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 8);
+#endif
+ }
+
+ if (info) {
+ call_info = info->callee_info;
+ while (call_info && call_info->caller_call_opline != opline) {
+ call_info = call_info->next_callee;
+ }
+ if (call_info && call_info->callee_func) {
+ func = call_info->callee_func;
+ }
+ }
+ if (!func) {
+ /* resolve function ar run time */
+ } else if (func->type == ZEND_USER_FUNCTION) {
+ if (call_info->num_args > func->op_array.num_args ||
+ (opline-1)->opcode == ZEND_SEND_UNPACK ||
+ (opline-1)->opcode == ZEND_SEND_ARRAY) {
+ goto fallback;
+ }
+ } else if (func->type == ZEND_INTERNAL_FUNCTION) {
+#if ZEND_DEBUG
+ if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
+ goto fallback;
+ }
+#endif
+ if ((opline-1)->opcode == ZEND_SEND_UNPACK || (opline-1)->opcode == ZEND_SEND_ARRAY) {
+ goto fallback;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+ zend_jit_stop_reuse_ip();
+
+ | // fbc = call->func;
+ | // mov r2, EX:RX->func ???
+ | // SAVE_OPLINE();
+ | SAVE_VALID_OPLINE opline
+
+ if (!delayed_call_chain) {
+ if (call_level == 1) {
+ | mov aword EX->call, 0
+ } else {
+ | //EX(call) = call->prev_execute_data;
+ | mov r0, EX:RX->prev_execute_data
+ | mov EX->call, r0
+ }
+ }
+ delayed_call_chain = 0;
+
+ | //call->prev_execute_data = execute_data;
+ | mov EX:RX->prev_execute_data, EX
+
+ if (!func) {
+ | mov r0, EX:RX->func
+ }
+
+ if (opline->opcode == ZEND_DO_FCALL) {
+ if (!func) {
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], (ZEND_ACC_DEPRECATED|ZEND_ACC_ABSTRACT)
+ | jnz >1
+ |.cold_code
+ |1:
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ | mov r0, EX:RX->func // reload
+ | jmp >1
+ |.code
+ |1:
+ } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) {
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0
+ | jmp ->exception_handler
+ } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ | mov r0, EX:RX->func // reload
+ }
+ }
+
+ if (!func) {
+ | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION
+ | jne >8
+ }
+
+ if (!func || func->type == ZEND_USER_FUNCTION) {
+ | // EX(call) = NULL;
+ | mov aword EX:RX->call, 0
+
+ if (RETURN_VALUE_USED(opline)) {
+ | // EX(return_value) = EX_VAR(opline->result.var);
+ | LOAD_ZVAL_ADDR r2, res_addr
+ | mov aword EX:RX->return_value, r2
+ } else {
+ | // EX(return_value) = 0;
+ | mov aword EX:RX->return_value, 0
+ }
+
+ if (func) {
+ for (i = call_info->num_args; i < func->op_array.last_var; i++) {
+ uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i);
+ | SET_Z_TYPE_INFO RX + n, IS_UNDEF
+ }
+ }
+
+ //EX_LOAD_RUN_TIME_CACHE(op_array);
+ if (!func || func->op_array.cache_size) {
+ if (func && op_array == &func->op_array) {
+ /* recursive call */
+ if (func->op_array.cache_size > sizeof(void*)) {
+ | mov r2, EX->run_time_cache
+ | mov EX:RX->run_time_cache, r2
+ }
+ } else {
+ if (func) {
+ | mov r0, EX:RX->func
+ }
+ | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)]
+#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR
+ | mov r2, aword [r2]
+#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET
+ | xor r1, r1
+ | test r2, 1
+ | jz >1
+ | MEM_OP2_2_ZTS mov, r1, aword, compiler_globals, map_ptr_base, r1
+ | sub r1, 1
+ |1:
+ | mov r2, aword [r1 + r2]
+#else
+# error "Unknown ZEND_MAP_PTR_KIND"
+#endif
+ | mov EX:RX->run_time_cache, r2
+ }
+ }
+
+ | // EG(current_execute_data) = execute_data;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1
+ | mov FP, RX
+
+ | // opline = op_array->opcodes;
+ if (func) {
+ uint32_t num_args;
+
+ if (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ num_args = skip_valid_arguments(op_array, ssa, call_info);
+ } else {
+ num_args = call_info->num_args;
+ }
+ if (func && zend_accel_in_shm(func->op_array.opcodes)) {
+ | LOAD_IP_ADDR (func->op_array.opcodes + num_args)
+ } else {
+ if (func) {
+ | mov r0, EX->func
+ }
+ if (GCC_GLOBAL_REGS) {
+ | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
+ if (num_args) {
+ | add IP, (num_args * sizeof(zend_op))
+ }
+ } else {
+ | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
+ if (num_args) {
+ | add FCARG1a, (num_args * sizeof(zend_op))
+ }
+ | mov aword EX->opline, FCARG1a
+ }
+ }
+
+ if (op_array == &func->op_array) {
+ /* recursive call */
+#ifdef CONTEXT_THREADED_JIT
+ | call >1
+ |.cold_code
+ |1:
+ | pop r0
+ | jmp =>num_args
+ |.code
+#else
+ | jmp =>num_args
+#endif
+ return 1;
+ }
+ } else {
+ | // opline = op_array->opcodes
+ if (GCC_GLOBAL_REGS) {
+ | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
+ } else {
+ | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
+ | mov aword EX->opline, FCARG1a
+ }
+ | // first_extra_arg = op_array->num_args;
+ | mov edx, dword [r0 + offsetof(zend_op_array, num_args)]
+ | // num_args = EX_NUM_ARGS();
+ | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
+ | // if (UNEXPECTED(num_args > first_extra_arg))
+ | cmp edx, ecx
+ | jl >1
+ |.cold_code
+ |1:
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, FP
+ }
+ | EXT_CALL zend_jit_copy_extra_args_helper, r0
+ | mov r0, EX->func // reload
+ | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload
+ | jmp >1
+ |.code
+ | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS
+ | jnz >1
+ | // opline += num_args;
+ |.if X64
+ | movsxd r2, ecx
+ | imul r2, r2, sizeof(zend_op)
+ |.else
+ | imul r2, ecx, sizeof(zend_op)
+ |.endif
+ | ADD_IP r2
+ |1:
+ | // if (EXPECTED((int)num_args < op_array->last_var)) {
+ | mov edx, dword [r0 + offsetof(zend_op_array, last_var)]
+ | sub edx, ecx
+ | jle >3 //???
+ | // zval *var = EX_VAR_NUM(num_args);
+ |.if X64
+ | movsxd r1, ecx
+ |.endif
+ | shl r1, 4
+ | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))]
+ |2:
+ | SET_Z_TYPE_INFO r1, IS_UNDEF
+ | sub edx, 1
+ | lea r1, [r1 + 16]
+ | jne <2
+ |3:
+ }
+
+#ifdef CONTEXT_THREADED_JIT
+ | call ->context_threaded_call
+ if (!func) {
+ | jmp >9
+ }
+#else
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+ | JMP_IP
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+ | JMP_IP
+ } else {
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 1 // ZEND_VM_ENTER
+ | ret
+ }
+#endif
+ }
+
+ if (!func || func->type == ZEND_INTERNAL_FUNCTION) {
+ if (!func) {
+ |8:
+ }
+ if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
+ if (!func) {
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], (ZEND_ACC_DEPRECATED|ZEND_ACC_ABSTRACT)
+ | jnz >1
+ |.cold_code
+ |1:
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ | mov r0, EX:RX->func // reload
+ | jmp >1
+ |.code
+ |1:
+ } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) {
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0
+ | jmp ->exception_handler
+ } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
+ if (!GCC_GLOBAL_REGS) {
+ | mov FCARG1a, RX
+ }
+ | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne ->exception_handler
+ | mov r0, EX:RX->func // reload
+ }
+ }
+
+ if (!RETURN_VALUE_USED(opline)) {
+ |.if not(X64WIN)
+ | sub r4, 16 /* alloca() */
+ |.endif
+ }
+
+ | // ZVAL_NULL(EX_VAR(opline->result.var));
+ | LOAD_ZVAL_ADDR FCARG2a, res_addr
+ | SET_Z_TYPE_INFO FCARG2a, IS_NULL
+
+ | // EG(current_execute_data) = execute_data;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1
+
+ if (!func || (func->common.fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
+ if (!func) {
+ | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS
+ | jnz >1
+ |.cold_code
+ }
+ |1:
+ | mov FCARG1a, RX
+ | EXT_CALL zend_jit_verify_internal_arg_types_helper, r0
+ | test r0, r0
+ | je >8
+ | LOAD_ZVAL_ADDR FCARG2a, res_addr // reload
+ if (!func) {
+ | mov r0, EX:RX->func // reload
+ | jmp >1
+ |.code
+ }
+ |1:
+ }
+
+ zend_jit_reset_opline(Dst, NULL);
+
+ | // fbc->internal_function.handler(call, ret);
+ | mov FCARG1a, RX
+ if (func) {
+ | EXT_CALL func->internal_function.handler, r0
+ } else {
+ | call aword [r0 + offsetof(zend_internal_function, handler)]
+ }
+
+ | // EG(current_execute_data) = execute_data;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0
+
+ | // zend_vm_stack_free_args(call);
+ if (func) {
+ for (i = 0; i < call_info->num_args; i++ ) {
+ uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i);
+ | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, opline
+ }
+ } else {
+ | mov FCARG1a, RX
+ | EXT_CALL zend_jit_vm_stack_free_args_helper, r0
+ }
+
+ |8:
+ if (opline->opcode == ZEND_DO_FCALL) {
+ // TODO: optimize ???
+ | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
+ | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16)
+ | jnz >1
+ |.cold_code
+ |1:
+ | GET_Z_PTR r0, RX + offsetof(zend_execute_data, This)
+ | // OBJ_RELEASE(object);
+ | OBJ_RELEASE r0, ecx, >2
+ | jmp >2
+ |.code
+ |2:
+ }
+
+ | // zend_vm_stack_free_call_frame(call);
+ | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16)
+ | jnz >1
+ |.cold_code
+ |1:
+ | mov FCARG1a, RX
+ | EXT_CALL zend_jit_free_call_frame, r0
+ | jmp >1
+ |.code
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0
+ |1:
+
+ if (!RETURN_VALUE_USED(opline)) {
+ uint32_t func_info = call_info ?
+ zend_get_func_info(call_info, ssa) :
+ (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
+
+ if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, 0, opline
+ }
+ |.if not(X64WIN)
+ | add r4, 16 /* revert alloca() */
+ |.endif
+ }
+
+ | // if (UNEXPECTED(EG(exception) != NULL)) {
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | jne >1
+ |.cold_code
+ |1:
+ | LOAD_IP_ADDR opline
+ | jmp ->icall_throw_handler
+ |.code
+
+ // TODO: Can we avoid checking for interrupts after each call ???
+ if (!zend_jit_check_timeout(Dst, opline + 1)) {
+ return 0;
+ }
+ if (opline->opcode != ZEND_DO_ICALL) {
+ | LOAD_IP_ADDR (opline + 1)
+ }
+ }
+
+ if (!func) {
+ |9:
+ }
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ if (opline->opcode == ZEND_DO_FCALL ||
+ opline->opcode == ZEND_DO_UCALL ||
+ opline->opcode == ZEND_DO_FCALL_BY_NAME ){
+ return zend_jit_call(Dst, opline, next_block);
+ } else {
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+ }
+}
+
+static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info;
+ uint32_t arg_num = opline->op2.num;
+ zend_jit_addr op1_addr, arg_addr;
+
+ if (opline->opcode == ZEND_SEND_VAL_EX && arg_num > MAX_ARG_FLAG_NUM) {
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+
+ if (opline->opcode == ZEND_SEND_VAL_EX) {
+ uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
+
+ | mov r0, EX:RX->func
+ if (arg_num <= MAX_ARG_FLAG_NUM) {
+ | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
+ | jnz >1
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.cold_code
+ |1:
+ | SAVE_VALID_OPLINE opline
+ | jmp ->throw_cannot_pass_by_ref
+ |.code
+ }
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
+
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+
+ | ZVAL_COPY_CONST arg_addr, -1, zv, r0
+ if (Z_REFCOUNTED_P(zv)) {
+ | ADDREF_CONST zv, r0
+ }
+ } else {
+ | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ }
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int cold)
+{
+ uint32_t op1_info;
+ zend_jit_addr op1_addr, arg_addr, ref_addr;
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
+ op1_info = OP1_INFO();
+
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+
+ if (opline->op1_type == IS_VAR) {
+ | LOAD_ZVAL_ADDR r0, op1_addr
+ | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) {
+ | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1
+ | // ret = Z_INDIRECT_P(ret);
+ | GET_Z_PTR r0, r0
+ |1:
+ if (op1_info & MAY_BE_ERROR) {
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ if (cold) {
+ | IF_NOT_Z_TYPE r0, _IS_ERROR, >1
+ } else {
+ | IF_Z_TYPE r0, _IS_ERROR, >1
+ |.cold_code
+ |1:
+ }
+ }
+
+ | // ZVAL_NEW_EMPTY_REF(arg);
+ | EMALLOC sizeof(zend_reference), op_array, opline
+ | SET_ZVAL_PTR arg_addr, r0
+ | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
+ | mov dword [r0], 1
+ | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], IS_REFERENCE
+ | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0
+ ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8);
+ | // ZVAL_NULL(Z_REFVAL_P(arg));
+ | SET_ZVAL_TYPE_INFO ref_addr, IS_NULL
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ | jmp >7
+ if (cold) {
+ |1:
+ } else {
+ |.code
+ }
+ }
+ }
+ } else if (opline->op1_type == IS_CV) {
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
+ | jmp >2
+ |1:
+ }
+ op1_info &= ~MAY_BE_UNDEF;
+ op1_info |= MAY_BE_NULL;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+
+ if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
+ if (op1_info & MAY_BE_REF) {
+ if (opline->op1_type == IS_VAR) {
+ | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2
+ | GET_Z_PTR r1, r0
+ } else {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2
+ | GET_ZVAL_PTR r1, op1_addr
+ }
+ | GC_ADDREF r1
+ | SET_ZVAL_PTR arg_addr, r1
+ | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
+ | jmp >6
+ }
+ |2:
+ | // ZVAL_NEW_REF(arg, varptr);
+ if (opline->op1_type == IS_VAR) {
+ | mov aword T1, r0 // save
+ }
+ | EMALLOC sizeof(zend_reference), op_array, opline
+ | mov dword [r0], 2
+ | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], IS_REFERENCE
+ | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0
+ ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8);
+ if (opline->op1_type == IS_VAR) {
+ zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
+
+ | mov r1, aword T1 // restore
+ | ZVAL_COPY_VALUE ref_addr, -1, val_addr, op1_info, ZREG_R2, ZREG_R2
+ | SET_ZVAL_PTR val_addr, r0
+ | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX
+ } else {
+ | ZVAL_COPY_VALUE ref_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2
+ | SET_ZVAL_PTR op1_addr, r0
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
+ }
+ | SET_ZVAL_PTR arg_addr, r0
+ | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
+ }
+
+ |6:
+ | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, op_array, opline
+ |7:
+
+ return 1;
+}
+
+static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info;
+ uint32_t arg_num = opline->op2.num;
+ zend_jit_addr op1_addr, arg_addr;
+
+ if ((opline->opcode == ZEND_SEND_VAR_EX ||
+ opline->opcode == ZEND_SEND_VAR_NO_REF_EX) &&
+ arg_num > MAX_ARG_FLAG_NUM) {
+ goto fallback;
+ }
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
+ op1_info = OP1_INFO();
+
+ if (!reuse_ip) {
+ zend_jit_start_reuse_ip();
+ | // call = EX(call);
+ | mov RX, EX->call
+ }
+
+ if (opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
+ uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
+
+ | mov r0, EX:RX->func
+ if (arg_num <= MAX_ARG_FLAG_NUM) {
+ | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
+ | jnz >1
+ } else {
+ ZEND_ASSERT(0);
+ }
+
+ |.cold_code
+ |1:
+
+ if (opline->opcode == ZEND_SEND_VAR_EX) {
+ if (!zend_jit_send_ref(Dst, opline, op_array, ssa, 1)) {
+ return 0;
+ }
+ } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
+ mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
+
+ | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2
+ if (op1_info & MAY_BE_REF) {
+ | cmp cl, IS_REFERENCE
+ | je >7
+ }
+ | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
+ | jnz >7
+ | SAVE_VALID_OPLINE opline
+ |.if X64
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Only variables should be passed by reference"
+ | EXT_CALL zend_error, r0
+ |.else
+ | sub r4, 8
+ | push "Only variables should be passed by reference"
+ | push E_NOTICE
+ | EXT_CALL zend_error, r0
+ | add r4, 16
+ |.endif
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+
+ | jmp >7
+ |.code
+ }
+
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ }
+
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ | jmp >7
+ |.code
+ }
+ }
+
+ if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
+ | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2
+ if (op1_info & MAY_BE_REF) {
+ | cmp cl, IS_REFERENCE
+ | je >7
+ }
+ | SAVE_VALID_OPLINE opline
+ |.if X64
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Only variables should be passed by reference"
+ | EXT_CALL zend_error, r0
+ |.else
+ | sub r4, 8
+ | push "Only variables should be passed by reference"
+ | push E_NOTICE
+ | EXT_CALL zend_error, r0
+ | add r4, 16
+ |.endif
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ } else if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ if (op1_info & MAY_BE_REF) {
+ if (opline->op1_type == IS_CV) {
+ zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ | ZVAL_COPY_VALUE arg_addr, -1, val_addr, op1_info, ZREG_R0, ZREG_R2
+ | TRY_ADDREF op1_info, ah, r2
+ } else {
+ zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8);
+
+ | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
+ |.cold_code
+ |1:
+ | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ | // ZVAL_COPY_VALUE(return_value, &ref->value);
+ | ZVAL_COPY_VALUE arg_addr, -1, ref_addr, op1_info, ZREG_R0, ZREG_R2
+ | GC_DELREF FCARG1a
+ | je >1
+ | IF_NOT_REFCOUNTED ah, >2
+ | GC_ADDREF r2
+ | jmp >2
+ |1:
+ | EFREE_REG_24 op_array, opline
+ | jmp >2
+ |.code
+ | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ |2:
+ }
+ } else {
+ if (ra
+ && ssa->ops[opline - op_array->opcodes].op1_def >= 0
+ && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
+ zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def);
+
+ if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info, ra[ssa->ops[opline - op_array->opcodes].op1_use])) {
+ return 0;
+ }
+ if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
+ op1_addr= op1_def_addr;
+ }
+ }
+ | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ if (opline->op1_type == IS_CV) {
+ | TRY_ADDREF op1_info, ah, r2
+ }
+ }
+ }
+ |7:
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, int jmp)
+{
+ uint32_t target_label;
+
+ if ((opline+1)->opcode == ZEND_JMPZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ if (jmp) {
+ | jmp >7
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | jmp =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp =>target_label
+ } else {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ if (jmp) {
+ | jmp >7
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, int jmp)
+{
+ uint32_t target_label;
+
+ if ((opline+1)->opcode == ZEND_JMPZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | jmp =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ if (jmp) {
+ | jmp >7
+ }
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | jmp =>target_label
+ } else {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ if (jmp) {
+ | jmp >7
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa)
+{
+ zend_bool smart_branch = 0;
+ uint32_t defined_label = (uint32_t)-1;
+ uint32_t undefined_label = (uint32_t)-1;
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+ zend_jit_addr res_addr;
+
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPZNZ) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ (*opnum)++;
+ smart_branch = 1;
+ }
+
+ if (smart_branch) {
+ if ((opline+1)->opcode == ZEND_JMPZ) {
+ undefined_label = ssa->cfg.blocks[b].successors[0];
+ } else if ((opline+1)->opcode == ZEND_JMPNZ) {
+ defined_label = ssa->cfg.blocks[b].successors[0];
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ) {
+ undefined_label = ssa->cfg.blocks[b].successors[0];
+ defined_label = ssa->cfg.blocks[b].successors[1];
+ } else {
+ ZEND_ASSERT(0);
+ }
+ }
+
+ | // if (CACHED_PTR(opline->extended_value)) {
+ | mov r0, EX->run_time_cache
+ | mov r0, aword [r0 + opline->extended_value]
+ | test r0, r0
+ | jz >1
+ | test r0, 0x1
+ | jnz >4
+ |.cold_code
+ |4:
+ | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a
+ | shl r0, 1
+ | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax
+ if (smart_branch) {
+ if (undefined_label != (uint32_t)-1) {
+ | jz =>undefined_label
+ } else {
+ | jz >3
+ }
+ } else {
+ | jz >2
+ }
+ |1:
+ | SAVE_VALID_OPLINE opline
+ | LOAD_ADDR FCARG1a, zv
+ | EXT_CALL zend_jit_check_constant, r0
+ | test r0, r0
+ if (smart_branch) {
+ if (undefined_label != (uint32_t)-1) {
+ | jnz =>undefined_label
+ } else {
+ | jnz >3
+ }
+ if (defined_label != (uint32_t)-1) {
+ | jmp =>defined_label
+ } else {
+ | jmp >3
+ }
+ } else {
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+ | jz >1
+ |2:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ | jmp >3
+ }
+ |.code
+ if (smart_branch) {
+ if (defined_label != (uint32_t)-1) {
+ | jmp =>defined_label
+ }
+ } else {
+ |1:
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ }
+ |3:
+
+ return 1;
+}
+
+static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, mask;
+ uint32_t target_label;
+ zend_uchar type;
+ zend_bool smart_branch = 0;
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+
+ if (opline->extended_value & MAY_BE_RESOURCE) {
+ // TODO: support for is_resource() ???
+ goto fallback;
+ }
+
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPZNZ) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ (*opnum)++;
+ smart_branch = 1;
+ }
+
+ op1_info = OP1_INFO();
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ }
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ if (opline->extended_value & MAY_BE_NULL) {
+ if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) {
+ return 0;
+ }
+ } else {
+ if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) {
+ return 0;
+ }
+ }
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ |.code
+ }
+ }
+
+ if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
+ mask = opline->extended_value;
+ switch (mask) {
+ case MAY_BE_NULL: type = IS_NULL; break;
+ case MAY_BE_FALSE: type = IS_FALSE; break;
+ case MAY_BE_TRUE: type = IS_TRUE; break;
+ case MAY_BE_LONG: type = IS_LONG; break;
+ case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
+ case MAY_BE_STRING: type = IS_STRING; break;
+ case MAY_BE_ARRAY: type = IS_ARRAY; break;
+ case MAY_BE_OBJECT: type = IS_OBJECT; break;
+ default:
+ type = 0;
+ }
+
+ if (!(op1_info & (MAY_BE_ANY - mask))) {
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, 0)) {
+ return 0;
+ }
+ } else if (!(op1_info & mask)) {
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, 0)) {
+ return 0;
+ }
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR r0, op1_addr
+ | ZVAL_DEREF r0, op1_info
+ }
+ if (type == 0) {
+ if (smart_branch &&
+ (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ | // if (Z_REFCOUNTED_P(cv)) {
+ | IF_ZVAL_REFCOUNTED op1_addr, >1
+ |.cold_code
+ |1:
+ }
+ | // if (!Z_DELREF_P(cv)) {
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ | GC_DELREF FCARG1a
+ if (RC_MAY_BE_1(op1_info)) {
+ if (RC_MAY_BE_N(op1_info)) {
+ | jnz >3
+ }
+ if (op1_info & MAY_BE_REF) {
+ | mov al, byte [r0 + 8]
+ } else {
+ | mov al, byte [FP + opline->op1.var + 8]
+ }
+ | mov byte T1, al // save
+ | // zval_dtor_func(r);
+ | ZVAL_DTOR_FUNC op1_info, opline
+ | mov cl, byte T1 // restore
+ |jmp >2
+ }
+ if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if (!RC_MAY_BE_1(op1_info)) {
+ | jmp >3
+ }
+ |.code
+ }
+ |3:
+ if (op1_info & MAY_BE_REF) {
+ | mov cl, byte [r0 + 8]
+ } else {
+ | mov cl, byte [FP + opline->op1.var + 8]
+ }
+ |2:
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ | mov cl, byte [r0 + 8]
+ } else {
+ | mov cl, byte [FP + opline->op1.var + 8]
+ }
+ }
+ | mov eax, 1
+ | shl eax, cl
+ | test eax, mask
+ if ((opline+1)->opcode == ZEND_JMPZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | je =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | jne =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | je =>target_label
+ target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp =>target_label
+ } else {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | setne al
+ | movzx eax, al
+ | add eax, 2
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ }
+ } else {
+ if (smart_branch &&
+ (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ | // if (Z_REFCOUNTED_P(cv)) {
+ | IF_ZVAL_REFCOUNTED op1_addr, >1
+ |.cold_code
+ |1:
+ }
+ | // if (!Z_DELREF_P(cv)) {
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ | GC_DELREF FCARG1a
+ if (RC_MAY_BE_1(op1_info)) {
+ if (RC_MAY_BE_N(op1_info)) {
+ | jnz >3
+ }
+ if (op1_info & MAY_BE_REF) {
+ | mov al, byte [r0 + 8]
+ } else {
+ | mov al, byte [FP + opline->op1.var + 8]
+ }
+ | mov byte T1, al // save
+ | // zval_dtor_func(r);
+ | ZVAL_DTOR_FUNC op1_info, opline
+ | mov cl, byte T1 // restore
+ |jmp >2
+ }
+ if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if (!RC_MAY_BE_1(op1_info)) {
+ | jmp >3
+ }
+ |.code
+ }
+ |3:
+ if (op1_info & MAY_BE_REF) {
+ | mov cl, byte [r0 + 8]
+ } else {
+ | mov cl, byte [FP + opline->op1.var + 8]
+ }
+ |2:
+ | cmp cl, type
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ | cmp byte [r0 + 8], type
+ } else {
+ | cmp byte [FP + opline->op1.var + 8], type
+ }
+ }
+ if ((opline+1)->opcode == ZEND_JMPZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | jne =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | je =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ target_label = ssa->cfg.blocks[b].successors[0];
+ | jne =>target_label
+ target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp =>target_label
+ } else {
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | sete al
+ | movzx eax, al
+ | add eax, 2
+ | SET_ZVAL_TYPE_INFO res_addr, eax
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+ }
+ }
+ }
+ }
+
+ |7:
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t i, j, info;
+
+ // Use type inference to avoid useless zval_ptr_dtor()
+ for (i = 0 ; i < op_array->last_var; i++) {
+ if (ssa->vars && ssa->var_info) {
+ info = ssa->var_info[i].type;
+ for (j = op_array->last_var; j < ssa->vars_count; j++) {
+ if (ssa->vars[j].var == i) {
+ info |= ssa->var_info[j].type;
+ }
+ }
+ } else {
+ info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF;
+ }
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ /* Refcount may be increased by RETRUN opcode */
+ if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
+ for (j = 0; j < ssa->cfg.blocks_count; j++) {
+ if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
+ ssa->cfg.blocks[j].len > 0) {
+ const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
+
+ if (opline->opcode == ZEND_RETURN) {
+ if (opline->op1_type == IS_CV &&
+ opline->op1.var == (uint32_t)(uintptr_t)(ZEND_CALL_VAR_NUM(NULL, i))) {
+ info |= MAY_BE_RCN;
+ break;
+ }
+ }
+ }
+ }
+ }
+#endif
+
+ if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i);
+ | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 1, opline
+ }
+ }
+ return 1;
+}
+
+static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ // Avoid multiple leave sequnces
+ if (jit_return_label >= 0) {
+ | jmp =>jit_return_label
+ return 1;
+ }
+
+ jit_return_label = ssa->cfg.blocks_count * 2;
+
+ |=>jit_return_label:
+
+ // i_free_compiled_variables(execute_data);
+ if (!zend_jit_free_compiled_variables(Dst, opline, op_array, ssa)) {
+ return 0;
+ }
+
+ /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
+ | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)]
+ | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_FAKE_CLOSURE)
+ | jnz ->leave_function_handler
+
+ if ((op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) ||
+ (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
+ | // EG(current_execute_data) = EX(prev_execute_data);
+ | mov r0, EX->prev_execute_data
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2
+ if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
+ | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
+ | mov r0, EX->func
+ | sub r0, sizeof(zend_object)
+ | OBJ_RELEASE r0, ecx, >4
+ } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
+ | // if (call_info & ZEND_CALL_RELEASE_THIS)
+ | test FCARG1d, ZEND_CALL_RELEASE_THIS
+ | je >4
+ | // zend_object *object = Z_OBJ(execute_data->This);
+ | mov r0, EX->This.value.obj
+ | // OBJ_RELEASE(object);
+ | OBJ_RELEASE r0, ecx, >4
+ }
+ |4:
+ | // EG(vm_stack_top) = (zval*)execute_data;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0
+ | // execute_data = EX(prev_execute_data);
+ | mov FP, EX->prev_execute_data
+ } else {
+ | // EG(vm_stack_top) = (zval*)execute_data;
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0
+ | // execute_data = EX(prev_execute_data);
+ | mov FP, EX->prev_execute_data
+ | // EG(current_execute_data) = execute_data
+ | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0
+ }
+ | // if (EG(exception))
+ | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0
+ | LOAD_OPLINE
+ | jne ->leave_throw_handler
+ | // opline = EX(opline) + 1
+ | ADD_IP sizeof(zend_op)
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
+ | add r4, HYBRID_SPAD
+#ifdef CONTEXT_THREADED_JIT
+ | push aword [IP]
+ | ret
+#else
+ | JMP_IP
+#endif
+ } else if (GCC_GLOBAL_REGS) {
+ | add r4, SPAD // stack alignment
+#ifdef CONTEXT_THREADED_JIT
+ | push aword [IP]
+ | ret
+#else
+ | JMP_IP
+#endif
+ } else {
+#ifdef CONTEXT_THREADED_JIT
+ ZEND_ASSERT(0);
+ // TODO: context threading can't work without GLOBAL REGS because we have to change
+ // the value of execute_data in execute_ex()
+ | mov FCARG1a, FP
+ | mov r0, aword [FP]
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | push aword [r0]
+ | ret
+#else
+ | mov FP, aword T2 // restore FP
+ | mov RX, aword T3 // restore IP
+ | add r4, NR_SPAD // stack alignment
+ | mov r0, 2 // ZEND_VM_LEAVE
+ | ret
+#endif
+ }
+
+ return 1;
+}
+
+static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra)
+{
+ uint32_t op1_info;
+ zend_jit_addr op1_addr, ret_addr;
+
+ if (op_array->type == ZEND_EVAL_CODE || !op_array->function_name || !ssa->ops || !ssa->var_info) {
+ // TODO: support for top-level code
+ return zend_jit_tail_handler(Dst, opline);
+ }
+
+ op1_info = OP1_INFO();
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1);
+ if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
+ // TODO: support for IS_UNDEF ???
+ return zend_jit_tail_handler(Dst, opline);
+ }
+
+ // if (!EX(return_value))
+ if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) {
+ | mov r2, EX->return_value
+ | test r2, r2
+ ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
+ } else {
+ | mov r1, EX->return_value
+ | test r1, r1
+ ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
+ }
+ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
+ (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ | jz >1
+ |.cold_code
+ |1:
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ if (jit_return_label >= 0) {
+ | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label
+ } else {
+ | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9
+ }
+ }
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ | GC_DELREF FCARG1a
+ if (RC_MAY_BE_1(op1_info)) {
+ if (RC_MAY_BE_N(op1_info)) {
+ if (jit_return_label >= 0) {
+ | jnz =>jit_return_label
+ } else {
+ | jnz >9
+ }
+ }
+ | //SAVE_OPLINE()
+ | ZVAL_DTOR_FUNC op1_info, opline
+ | //????mov r1, EX->return_value // reload ???
+ }
+ if (jit_return_label >= 0) {
+ | jmp =>jit_return_label
+ } else {
+ | jmp >9
+ }
+ |.code
+ } else {
+ if (jit_return_label >= 0) {
+ | jz =>jit_return_label
+ } else {
+ | jz >9
+ }
+ }
+
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+ | ZVAL_COPY_CONST ret_addr, -1, zv, r0
+ if (Z_REFCOUNTED_P(zv)) {
+ | ADDREF_CONST zv, r0
+ }
+ } else if (opline->op1_type == IS_TMP_VAR) {
+ | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ } else if (opline->op1_type == IS_CV) {
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR r0, op1_addr
+ | ZVAL_DEREF r0, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
+ }
+ | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ???
+ | TRY_ADDREF op1_info, ah, r2
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val));
+
+ | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
+ |.cold_code
+ |1:
+ | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
+ | GET_ZVAL_PTR r0, op1_addr
+ | // ZVAL_COPY_VALUE(return_value, &ref->value);
+ | ZVAL_COPY_VALUE ret_addr, -1, ref_addr, op1_info, ZREG_R2, ZREG_R2
+ | GC_DELREF r0
+ | je >2
+ | // if (IS_REFCOUNTED())
+ if (jit_return_label >= 0) {
+ | IF_NOT_REFCOUNTED dh, =>jit_return_label
+ } else {
+ | IF_NOT_REFCOUNTED dh, >9
+ }
+ | // ADDREF
+ | GET_ZVAL_PTR r2, ret_addr // reload
+ | GC_ADDREF r2
+ if (jit_return_label >= 0) {
+ | jmp =>jit_return_label
+ } else {
+ | jmp >9
+ }
+ |2:
+ | EFREE_24 r0, op_array, opline
+ if (jit_return_label >= 0) {
+ | jmp =>jit_return_label
+ } else {
+ | jmp >9
+ }
+ |.code
+ }
+ | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2
+ }
+
+ |9:
+ //JIT: ZEND_VM_DISPATCH_TO_HELPER(zend_leave_helper);
+ return zend_jit_leave_func(Dst, opline, op_array, ssa);
+}
+
+static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, op2_info, res_info;
+ zend_jit_addr op1_addr, op2_addr, res_addr;
+
+ if (!ssa->ops || !ssa->var_info) {
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ res_info = RES_INFO();
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
+ }
+ | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr
+ if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, 8, 9)) {
+ return 0;
+ }
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
+ if (op1_info & MAY_BE_ARRAY) {
+ |.cold_code
+ |7:
+ }
+
+ if (op1_info & MAY_BE_STRING) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
+ }
+ | SAVE_VALID_OPLINE opline
+ if (Z_REG(op1_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, res_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR res_addr, r0
+ |.endif
+ if (opline->opcode == ZEND_FETCH_DIM_R) {
+ | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0
+ } else if (opline->opcode == ZEND_FETCH_DIM_IS) {
+ | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ if ((op1_info & MAY_BE_ARRAY) ||
+ (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) {
+ | jmp >9 // END
+ }
+ |6:
+ }
+
+ if (op1_info & MAY_BE_OBJECT) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6
+ }
+ | SAVE_VALID_OPLINE opline
+ if (Z_REG(op1_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
+ } else {
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, res_addr
+ |.else
+ | sub r4, 12
+ | PUSH_ZVAL_ADDR res_addr, r0
+ |.endif
+ if (opline->opcode == ZEND_FETCH_DIM_R) {
+ | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0
+ } else if (opline->opcode == ZEND_FETCH_DIM_IS) {
+ | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ if ((op1_info & MAY_BE_ARRAY) ||
+ (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) {
+ | jmp >9 // END
+ }
+ |6:
+ }
+
+ if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
+ | SAVE_VALID_OPLINE opline
+ if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ |1:
+ }
+
+ if (op2_info & MAY_BE_UNDEF) {
+ | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
+ | mov FCARG1d, opline->op2.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ |1:
+ }
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_NULL
+ if (op1_info & MAY_BE_ARRAY) {
+ | jmp >9 // END
+ }
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ |.code
+ }
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
+
+ |8:
+ if (op1_info & MAY_BE_ARRAY_OF_REF) {
+ | // ZVAL_COPY_DEREF
+ | IF_NOT_ZVAL_REFCOUNTED val_addr, >2
+ | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1
+ | GET_Z_PTR r0, r0
+ | add r0, offsetof(zend_reference, val)
+ | IF_NOT_ZVAL_REFCOUNTED val_addr, >2
+ |1:
+ | GET_Z_PTR r1, r0
+ | GC_ADDREF r1
+ |2:
+ | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2
+ } else {
+ | // ZVAL_COPY
+ | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2
+ | TRY_ADDREF res_info, ch, r2
+ }
+ }
+ |9: // END
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
+ /* Magic offsetGet() may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline
+
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info, op2_info;
+ zend_jit_addr op1_addr, op2_addr, res_addr;
+
+ if (!ssa->ops || !ssa->var_info || (opline->extended_value & ZEND_ISEMPTY)) {
+ // TODO: support for empty() ???
+ goto fallback;
+ }
+
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+
+ op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1);
+ res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | ZVAL_DEREF FCARG1a, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ }
+
+ if (op1_info & MAY_BE_ARRAY) {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
+ }
+ | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr
+ if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_JIT_IS, op1_info, op2_info, 8, 9)) {
+ return 0;
+ }
+ }
+
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
+ if (op1_info & MAY_BE_ARRAY) {
+ |.cold_code
+ |7:
+ }
+
+ | SAVE_VALID_OPLINE opline
+ if (Z_REG(op1_addr) != ZREG_FCARG1a) {
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ }
+ if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
+ ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
+ | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
+ } else {
+ | LOAD_ZVAL_ADDR FCARG2a, op2_addr
+ }
+ | EXT_CALL zend_jit_isset_dim_helper, r0
+ | test r0, r0
+ | jz >9
+ | jmp >8
+
+ if (op1_info & MAY_BE_ARRAY) {
+ |.code
+ }
+ }
+
+#ifdef ZEND_JIT_USE_RC_INFERENCE
+ if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
+ /* Magic offsetExists() may increase refcount of the key */
+ op2_info |= MAY_BE_RCN;
+ }
+#endif
+
+ |8:
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception_undef_result(Dst, opline)) {
+ return 0;
+ }
+ }
+ if (!(opline->extended_value & ZEND_ISEMPTY)) {
+ if ((opline+1)->opcode == ZEND_JMPZ) {
+ unsigned int target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPNZ) {
+ unsigned int target_label = ssa->cfg.blocks[b].successors[0];
+ | jmp =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ) {
+ unsigned int target_label = ssa->cfg.blocks[b].successors[1];
+ | jmp =>target_label
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
+ | jmp >8
+ }
+ } else {
+ | //????
+ | int3
+ }
+
+ |9: // not found
+ | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception_undef_result(Dst, opline)) {
+ return 0;
+ }
+ }
+ if (!(opline->extended_value & ZEND_ISEMPTY)) {
+ if ((opline+1)->opcode == ZEND_JMPZ) {
+ unsigned int target_label = ssa->cfg.blocks[b].successors[0];
+ | jmp =>target_label
+ } else if ((opline+1)->opcode == ZEND_JMPNZ) {
+ } else if ((opline+1)->opcode == ZEND_JMPZNZ) {
+ unsigned int target_label = ssa->cfg.blocks[b].successors[0];
+ | jmp =>target_label
+ } else {
+ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
+ }
+ } else {
+ | //????
+ | int3
+ }
+
+ |8:
+
+ if (((opline+1)->opcode == ZEND_JMPZ ||
+ (opline+1)->opcode == ZEND_JMPNZ ||
+ (opline+1)->opcode == ZEND_JMPZNZ) &&
+ (opline+1)->op1_type == IS_TMP_VAR &&
+ (opline+1)->op1.var == opline->result.var) {
+ (*opnum)++;
+ }
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info;
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ zval *varname = RT_CONSTANT(opline, opline->op2);
+
+ if (!ssa->ops || !ssa->var_info) {
+ op1_info = MAY_BE_ANY|MAY_BE_REF;
+ } else {
+ op1_info = OP1_INFO();
+ }
+
+ //idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1;
+ | mov r0, EX->run_time_cache
+ | mov r0, aword [r0 + opline->extended_value]
+ | sub r0, 1
+ //if (EXPECTED(idx < EG(symbol_table).nNumUsed))
+ | MEM_OP2_2_ZTS cmp, eax, dword, executor_globals, symbol_table.nNumUsed, r1
+ | jae >9
+ //Bucket *p = EG(symbol_table).arData + idx;
+ |.if X64
+ | shl r0, 5
+ |.else
+ | imul r0, sizeof(Bucket)
+ |.endif
+ | MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1
+ | IF_Z_TYPE r0, IS_UNDEF, >9
+ // (EXPECTED(p->key == Z_STR_P(varname))
+ | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname), r1
+ | jne >1
+ |.cold_code
+ |1:
+ //(EXPECTED(p->h == ZSTR_H(Z_STR_P(varname)))
+ | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)), r1
+ | jne >9
+ //EXPECTED(p->key != NULL)
+ | mov r1, [r0 + offsetof(Bucket, key)]
+ | test r1, r1
+ | jz >9
+ //EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(varname))
+ | ADDR_OP2_2 cmp, aword [r1 + offsetof(zend_string, len)], Z_STRLEN_P(varname), r2
+ | jne >9
+ //EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(varname), Z_STRLEN_P(varname)) == 0)
+ | add r1, offsetof(zend_string, val)
+ | mov T1, r0
+ |.if X64
+ | mov CARG1, r1
+ | LOAD_ADDR CARG2, Z_STRVAL_P(varname)
+ | mov CARG3, Z_STRLEN_P(varname)
+ | EXT_CALL memcmp, r0
+ |.else
+ | sub r4, 4
+ | push Z_STRLEN_P(varname)
+ | push Z_STRVAL_P(varname)
+ | push r1
+ | call &memcmp
+ | add r4, 16
+ |.endif
+ | test al, al
+ | mov r0, aword T1
+ | jnz >9
+ | jmp >2
+ |.code
+ |2:
+ // if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT))
+ | mov cl, byte [r0 + 8]
+ | cmp cl, IS_INDIRECT
+ | je >1
+ |.cold_code
+ |1:
+ //value = Z_INDIRECT_P(value)
+ | mov r0, [r0]
+ | mov cl, byte [r0 + 8]
+ | test cl, cl // cmp cl, IS_UNDEF
+ | jne >2
+ | SET_Z_TYPE_INFO r0, IS_NULL
+ | jmp >8
+ |.code
+ |2:
+ | cmp cl, IS_REFERENCE
+ | jne >8
+ |1:
+ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ //stash this for later use
+ | mov r2, r0
+ }
+ | GET_Z_PTR r0, r0
+ | GC_ADDREF r0
+ //if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
+ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
+ | IF_ZVAL_REFCOUNTED op1_addr, >2
+ }
+ |.cold_code
+ //zval_dtor_func(Z_COUNTED_P(variable_ptr))
+ |2:
+ //if (EXPECTED(variable_ptr != value))
+ | LOAD_ZVAL_ADDR FCARG1a, op1_addr
+ | cmp FCARG1a, r2
+ | je >4
+ | GET_Z_PTR FCARG1a, FCARG1a
+ | GC_DELREF FCARG1a
+ if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
+ | jnz >3
+ }
+ | mov aword T1, r0 // save
+ | ZVAL_DTOR_FUNC op1_info, opline
+ | mov r0, aword T1 // restore
+ | jmp >5
+ if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
+ |3:
+ // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
+ | IF_GC_MAY_NOT_LEAK FCARG1a, edx, >5
+ | mov aword T1, r0 //save
+ | EXT_CALL gc_possible_root, r1
+ | mov r0, aword T1 // restore
+ | jmp >5
+ }
+ |4:
+ | GET_Z_PTR FCARG1a, FCARG1a
+ | GC_DELREF FCARG1a
+ | jmp >5
+ |.code
+ }
+ |5:
+ //ZVAL_REF(variable_ptr, ref)
+ | SET_ZVAL_PTR op1_addr, r0
+ | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
+ //END of handler
+
+ |.cold_code
+ |8:
+ | mov FCARG1a, r0
+ | EXT_CALL zend_jit_new_ref_helper, r0
+ | jmp <1
+ |9:
+ | mov FCARG1a, FP
+ | LOAD_ADDR FCARG2a, (ptrdiff_t)varname
+ |.if X64
+ | mov CARG3, opline->extended_value
+ |.else
+ | sub r4, 12
+ | push opline->extended_value
+ |.endif
+ | EXT_CALL zend_jit_fetch_global_helper, r0
+ |.if not(X64)
+ | add r4, 12
+ |.endif
+ | jmp <1
+ |.code
+
+ return 1;
+}
+
+static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t arg_num = opline->op1.num;
+
+ if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ zend_arg_info *arg_info = NULL;
+
+ if (EXPECTED(arg_num <= op_array->num_args)) {
+ arg_info = &op_array->arg_info[arg_num-1];
+ } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
+ arg_info = &op_array->arg_info[op_array->num_args];
+ }
+ if (arg_info) {
+ zend_type type = arg_info->type;
+
+ if (ZEND_TYPE_IS_SET(type)) {
+ // Type check
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | LOAD_ZVAL_ADDR r0, res_addr
+ if (arg_info->pass_by_reference) {
+ | GET_Z_PTR r0, r0
+ | add r0, offsetof(zend_reference, val)
+ }
+ if (!ZEND_TYPE_IS_CLASS(type)) {
+ unsigned char code = ZEND_TYPE_CODE(type);
+
+ if (code == _IS_BOOL) {
+ | cmp byte [r0 + 8], IS_FALSE
+ | je >1
+ | cmp byte [r0 + 8], IS_TRUE
+ | jne >8
+ |1:
+ } else {
+ | cmp byte [r0 + 8], code
+ | jne >8
+ }
+ } else {
+ | SAVE_VALID_OPLINE opline
+ | cmp byte [r0 + 8], IS_OBJECT
+ | jne >9
+ | mov FCARG1a, r0
+ | mov r0, EX->run_time_cache
+ | add r0, opline->op2.num
+ | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array
+ |.if X64WIN
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov aword A5, r0
+ | EXT_CALL zend_jit_verify_arg_object, r0
+ |.elif X64
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov CARG5, r0
+ | EXT_CALL zend_jit_verify_arg_object, r0
+ |.else
+ | sub r4, 4
+ | push r0
+ | push (ptrdiff_t)arg_info
+ | push arg_num
+ | EXT_CALL zend_jit_verify_arg_object, r0
+ | add r4, 4
+ |.endif
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+
+ |.cold_code
+ |8:
+ | SAVE_VALID_OPLINE opline
+ |9:
+ | mov FCARG1a, r0
+ | mov r0, EX->run_time_cache
+ | add r0, opline->op2.num
+ | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array
+ |.if X64WIN
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov aword A5, r0
+ | mov aword A6, 0
+ | EXT_CALL zend_jit_verify_arg_slow, r0
+ |.elif X64
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov CARG5, r0
+ | xor CARG6, CARG6
+ | EXT_CALL zend_jit_verify_arg_slow, r0
+ |.else
+ | push 0
+ | push r0
+ | push (ptrdiff_t)arg_info
+ | push arg_num
+ | EXT_CALL zend_jit_verify_arg_slow, r0
+ |.endif
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ | jmp >1
+ |.code
+ |1:
+
+ if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
+ last_valid_opline = NULL;
+ if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+ }
+ }
+ }
+
+ | cmp dword EX->This.u2.num_args, arg_num
+ | jb >1
+ |.cold_code
+ |1:
+ | SAVE_VALID_OPLINE opline
+ | mov FCARG1a, FP
+ | EXT_CALL zend_missing_arg_error, r0
+ | jmp ->exception_handler
+ |.code
+
+ if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
+ last_valid_opline = NULL;
+ if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_bool is_last, zend_ssa *ssa)
+{
+ zend_arg_info *arg_info = NULL;
+ zend_bool has_slow = 0;
+ uint32_t arg_num = opline->op1.num;
+ zval *zv = RT_CONSTANT(opline, opline->op2);
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+
+ | cmp dword EX->This.u2.num_args, arg_num
+ | jae >5
+ | ZVAL_COPY_CONST res_addr, -1, zv, r0
+ if (Z_REFCOUNTED_P(zv)) {
+ | ADDREF_CONST zv, r0
+ }
+ if (Z_CONSTANT_P(zv)) {
+ has_slow = 1;
+ | SAVE_VALID_OPLINE opline
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG1, res_addr
+ | mov r0, EX->func
+ | mov CARG2, [r0 + offsetof(zend_op_array, scope)]
+ | EXT_CALL zval_update_constant_ex, r0
+ |.else
+ | sub r4, 8
+ | mov r0, EX->func
+ | push dword [r0 + offsetof(zend_op_array, scope)]
+ | LOAD_ZVAL_ADDR r0, res_addr
+ | push r0
+ | EXT_CALL zval_update_constant_ex, r0
+ | add r4, 16
+ |.endif
+ | test al, al
+ | jnz >7
+ }
+ |5:
+ if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
+ do {
+ if (arg_num <= op_array->num_args) {
+ arg_info = &op_array->arg_info[arg_num-1];
+ } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
+ arg_info = &op_array->arg_info[op_array->num_args];
+ } else {
+ break;
+ }
+ if (!ZEND_TYPE_IS_SET(arg_info->type)) {
+ break;
+ }
+ has_slow += 2;
+ | LOAD_ZVAL_ADDR r0, res_addr
+ | ZVAL_DEREF r0, MAY_BE_REF
+ if (!ZEND_TYPE_IS_CLASS(arg_info->type)) {
+ | cmp byte [r0 + 8], ZEND_TYPE_CODE(arg_info->type)
+ | jne >8
+ } else {
+ | cmp byte [r0 + 8], IS_OBJECT
+ | jne >8
+ | mov FCARG1a, r0
+ | mov r0, EX->run_time_cache
+ | lea r0, [r0 + opline->extended_value]
+ | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array
+ |.if X64WIN
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov aword A5, r0
+ | EXT_CALL zend_jit_verify_arg_object, r0
+ |.elif X64
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov CARG5, r0
+ | EXT_CALL zend_jit_verify_arg_object, r0
+ |.else
+ | sub r4, 4
+ | push r0
+ | push (ptrdiff_t)arg_info
+ | push arg_num
+ | EXT_CALL zend_jit_verify_arg_object, r0
+ | add r4, 4
+ |.endif
+ }
+ } while (0);
+ }
+ |9:
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+ if (is_last) {
+ | LOAD_IP_ADDR (opline + 1)
+ last_valid_opline = (opline + 1);
+ }
+
+ if (has_slow) {
+ |.cold_code
+ if (has_slow & 1) {
+ |7:
+ | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, opline
+ | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+ | jmp <5
+ }
+ if (has_slow & 2) {
+ |8:
+ | mov FCARG1a, r0
+ | mov r0, EX->run_time_cache
+ | lea r0, [r0 + opline->extended_value]
+ | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array
+ |.if X64WIN
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov aword A5, r0
+ | ADDR_OP2_2 mov, aword A6, zv, r0
+ | EXT_CALL zend_jit_verify_arg_slow, r0
+ |.elif X64
+ | mov CARG3, arg_num
+ | LOAD_ADDR CARG4, (ptrdiff_t)arg_info
+ | mov CARG5, r0
+ | LOAD_ADDR CARG6, zv
+ | EXT_CALL zend_jit_verify_arg_slow, r0
+ |.else
+ | push zv
+ | push r0
+ | push (ptrdiff_t)arg_info
+ | push arg_num
+ | EXT_CALL zend_jit_verify_arg_slow, r0
+ |.endif
+ | jmp <9
+ }
+ |.code
+ }
+
+ return 1;
+}
+
+#define ZEND_WRONG_PROPERTY_OFFSET 0
+
+static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename)
+{
+ zend_property_info *info;
+
+ if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || (ce->ce_flags & ZEND_ACC_TRAIT)) {
+ return ZEND_WRONG_PROPERTY_OFFSET;
+ }
+
+ if (ce->info.user.filename != filename) {
+ /* class declaration might be changed infdependently */
+ return ZEND_WRONG_PROPERTY_OFFSET;
+ }
+
+ if (ce->ce_flags & ZEND_ACC_INHERITED) {
+ if (!ce->parent) {
+ /* propery offests may be changed by inheritance */
+ return ZEND_WRONG_PROPERTY_OFFSET;
+ } else {
+ zend_class_entry *parent = ce->parent;
+
+ do {
+ if (parent->type == ZEND_INTERNAL_CLASS) {
+ break;
+ } else if (parent->info.user.filename != filename) {
+ /* some of parents class declarations might be changed infdependently */
+ /* TODO: this check may be not enough, because even
+ * in the same it's possible to conditionnaly define
+ * few classes with the same name, and "parent" may
+ * change from request to request.
+ */
+ return ZEND_WRONG_PROPERTY_OFFSET;
+ }
+ parent = parent->parent;
+ } while (parent);
+ }
+ }
+
+ info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
+ if (info == NULL ||
+ info->offset == ZEND_WRONG_PROPERTY_OFFSET ||
+ (info->flags & ZEND_ACC_STATIC)) {
+ return ZEND_WRONG_PROPERTY_OFFSET;
+ }
+
+ if (!(info->flags & ZEND_ACC_PUBLIC) &&
+ (!on_this || info->ce != ce)) {
+ return ZEND_WRONG_PROPERTY_OFFSET;
+ }
+
+ return info->offset;
+}
+
+static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename)
+{
+ zend_property_info *info;
+
+ if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) {
+ return 1;
+ }
+
+ if (ce->info.user.filename != filename) {
+ /* class declaration might be changed infdependently */
+ return 1;
+ }
+
+ info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
+ if (info == NULL ||
+ info->offset == ZEND_WRONG_PROPERTY_OFFSET ||
+ (info->flags & ZEND_ACC_STATIC)) {
+ return 1;
+ }
+
+ if (!(info->flags & ZEND_ACC_PUBLIC) &&
+ (!on_this || info->ce != ce)) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_bitset checked_this)
+{
+ uint32_t op1_info;
+ zend_class_entry *ce = NULL;
+ zval *member;
+ uint32_t offset;
+ zend_bool may_be_dynamic = 1;
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1);
+ zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+ zend_jit_addr prop_addr;
+
+ if (opline->op2_type != IS_CONST) {
+ goto fallback;
+ }
+
+ member = RT_CONSTANT(opline, opline->op2);
+ if (Z_TYPE_P(member) != IS_STRING || Z_STRVAL_P(member)[0] == '\0') {
+ goto fallback;
+ }
+
+ if (opline->op1_type == IS_UNUSED) {
+ op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
+ ce = op_array->scope;
+ } else {
+ op1_info = OP1_INFO();
+ if (ssa->var_info && ssa->ops) {
+ zend_ssa_var_info *op1_ssa =
+ ssa->var_info + ssa->ops[opline - op_array->opcodes].op1_use;
+
+ if (op1_ssa->ce && !op1_ssa->is_instanceof && !op1_ssa->ce->create_object) {
+ ce = op1_ssa->ce;
+ }
+ }
+ }
+
+ if (!(op1_info & MAY_BE_OBJECT)) {
+ goto fallback;
+ }
+
+ offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
+
+ if (opline->op1_type == IS_UNUSED) {
+ if (!checked_this || !zend_bitset_in(checked_this, (opline - op_array->opcodes))) {
+ | IF_ZVAL_TYPE this_addr, IS_UNDEF, >1
+ |.cold_code
+ |1:
+ | SAVE_VALID_OPLINE opline
+ | jmp ->not_obj
+ |.code
+ }
+ | GET_ZVAL_PTR FCARG1a, this_addr
+ } else {
+ if (op1_info & MAY_BE_REF) {
+ | LOAD_ZVAL_ADDR r0, op1_addr
+ | ZVAL_DEREF r0, op1_info
+ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
+ }
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7
+ }
+ | GET_ZVAL_PTR FCARG1a, op1_addr
+ }
+
+ if (offset == ZEND_WRONG_PROPERTY_OFFSET) {
+ | mov r0, EX->run_time_cache
+ | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)]
+ | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
+ | jne >5
+ | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)]
+ may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
+ if (may_be_dynamic) {
+ | test r0, r0
+ | jl >8 // dynamic property
+ }
+ | mov edx, dword [FCARG1a + r0 + 8]
+ | IF_TYPE dl, IS_UNDEF, >5
+ | add FCARG1a, r0
+ prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+ } else {
+ prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offset);
+ | mov edx, dword [FCARG1a + offset + 8]
+ | IF_TYPE dl, IS_UNDEF, >5
+ }
+ | GET_ZVAL_PTR r0, prop_addr
+ | IF_NOT_REFCOUNTED dh, >2
+ if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) {
+ | IF_TYPE dl, IS_REFERENCE, >6
+ }
+ |1:
+ | GC_ADDREF r0
+ |2:
+ |.if X64
+ | SET_ZVAL_PTR res_addr, r0
+ |.else
+ | SET_ZVAL_PTR res_addr, r0
+ | GET_ZVAL_W2 r0, prop_addr
+ | SET_ZVAL_W2 res_addr, r0
+ |.endif
+ | SET_ZVAL_TYPE_INFO res_addr, edx
+
+ |.cold_code
+ |5:
+ | LOAD_ADDR FCARG2a, member
+ |.if X64
+ | LOAD_ZVAL_ADDR CARG3, res_addr
+ | mov CARG4, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)
+ |.else
+ | sub r4, 8
+ | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)
+ | PUSH_ZVAL_ADDR res_addr, r0
+ |.endif
+ | SAVE_VALID_OPLINE opline
+ if (opline->opcode == ZEND_FETCH_OBJ_R) {
+ | EXT_CALL zend_jit_fetch_obj_r_slow, r0
+ } else if (opline->opcode == ZEND_FETCH_OBJ_IS) {
+ | EXT_CALL zend_jit_fetch_obj_is_slow, r0
+ } else {
+ ZEND_ASSERT(0);
+ }
+ |.if not(X64)
+ | add r4, 8
+ |.endif
+ | jmp >9
+
+ if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) {
+ |6:
+ if (offset == ZEND_WRONG_PROPERTY_OFFSET) {
+ | mov FCARG2a, FCARG1a
+ } else {
+ | lea FCARG2a, [FCARG1a + offset]
+ }
+ | LOAD_ZVAL_ADDR FCARG1a, res_addr
+ | EXT_CALL zend_jit_zval_copy_deref_helper, r0
+ | jmp >9
+ }
+
+ if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) {
+ |7:
+ if (opline->opcode == ZEND_FETCH_OBJ_R) {
+ | SAVE_VALID_OPLINE opline
+ if (op1_info & MAY_BE_UNDEF) {
+ if (op1_info & MAY_BE_ANY) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
+ }
+ | mov FCARG1d, opline->op1.var
+ | EXT_CALL zend_jit_undefined_op_helper, r0
+ |1:
+ }
+ |.if X64
+ | mov CARG1, E_NOTICE
+ | LOAD_ADDR CARG2, "Trying to get property '%s' of non-object"
+ | LOAD_ADDR CARG3, Z_STRVAL_P(member)
+ | EXT_CALL zend_error, r0
+ |.else
+ | sub r4, 4
+ | push Z_STRVAL_P(member)
+ | push "Trying to get property '%s' of non-object"
+ | push E_NOTICE
+ | EXT_CALL zend_error, r0
+ | add r4, 16
+ |.endif
+ }
+ | SET_ZVAL_TYPE_INFO res_addr, IS_NULL
+ | jmp >9
+ }
+
+ if (offset == ZEND_WRONG_PROPERTY_OFFSET && may_be_dynamic) {
+ |8:
+ | mov FCARG2a, r0
+ |.if X64WIN
+ | LOAD_ADDR CARG3, member
+ | LOAD_ZVAL_ADDR CARG4, res_addr
+ | mov aword A5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)
+ |.elif X64
+ | LOAD_ADDR CARG3, member
+ | LOAD_ZVAL_ADDR CARG4, res_addr
+ | mov CARG5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)
+ |.else
+ | sub r4, 4
+ | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)
+ | PUSH_ZVAL_ADDR res_addr, r0
+ | PUSH_ADDR member, r0
+ |.endif
+ | SAVE_VALID_OPLINE opline
+ if (opline->opcode == ZEND_FETCH_OBJ_R) {
+ | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0
+ } else if (opline->opcode == ZEND_FETCH_OBJ_IS) {
+ | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0
+ }
+ |.if not(X64)
+ | add r4, 4
+ |.endif
+ | jmp >9
+ }
+
+ |.code;
+ |9: // END
+ | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline
+
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+
+ return 1;
+
+fallback:
+ /* fallback to subroutine threading */
+ return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa));
+}
+
+static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ uint32_t op1_info = OP1_INFO();
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+
+ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
+ if (zend_may_throw(opline, op_array, ssa)) {
+ | SAVE_VALID_OPLINE, opline
+ }
+ if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
+ if (op1_info & MAY_BE_ARRAY) {
+ | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7
+ }
+ | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)]
+ | cmp FCARG1d, -1
+ | je >7
+ | EXT_CALL zend_hash_iterator_del, r0
+ |7:
+ }
+ | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, 0, opline
+ if (zend_may_throw(opline, op_array, ssa)) {
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+ }
+
+ return 1;
+}
+
+static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+
+ if (Z_TYPE_P(zv) == IS_STRING) {
+ size_t len = Z_STRLEN_P(zv);
+
+ if (len > 0) {
+ const char *str = Z_STRVAL_P(zv);
+
+ | SAVE_VALID_OPLINE opline
+ |.if X64
+ | LOAD_ADDR CARG1, str
+ | LOAD_ADDR CARG2, len
+ | EXT_CALL zend_write, r0
+ |.else
+ | sub r4, 8
+ | push len
+ | push str
+ | EXT_CALL zend_write, r0
+ | add r4, 16
+ |.endif
+ if (!zend_jit_check_exception(Dst)) {
+ return 0;
+ }
+ }
+ return 1;
+ }
+ }
+
+ if (!zend_jit_handler(Dst, opline, 1)) {
+ return 0;
+ }
+
+ return 1;
+}
+
+static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa)
+{
+ HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
+
+ if (sizeof(void*) == 8 && !IS_32BIT(dasm_end)) {
+ // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ return 1;
+ }
+ if (opline->op1_type == IS_CONST) {
+ zval *zv = RT_CONSTANT(opline, opline->op1);
+ zval *jump_zv;
+ int b;
+
+ if (opline->opcode == ZEND_SWITCH_LONG) {
+ if (Z_TYPE_P(zv) == IS_LONG) {
+ jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
+ if (jump_zv != NULL) {
+ b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
+ } else {
+ b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+ }
+ | jmp =>b
+ }
+ } else if (opline->opcode == ZEND_SWITCH_STRING) {
+ if (Z_TYPE_P(zv) == IS_STRING) {
+ jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1);
+ if (jump_zv != NULL) {
+ b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
+ } else {
+ b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+ }
+ | jmp =>b
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ } else {
+ uint32_t op1_info = OP1_INFO();
+ zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1);
+ int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
+ zval *val;
+
+ if (opline->opcode == ZEND_SWITCH_LONG) {
+ if (op1_info & MAY_BE_LONG) {
+ if (op1_info & MAY_BE_REF) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1
+ | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr
+ |.cold_code
+ |1:
+ | // ZVAL_DEREF(op)
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
+ | GET_ZVAL_PTR FCARG2a, op1_addr
+ | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3
+ | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)]
+ | jmp >2
+ |.code
+ |2:
+ } else {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
+ }
+ | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr
+ }
+ if (HT_IS_PACKED(jumptable)) {
+ uint32_t count = jumptable->nNumUsed;
+ Bucket *p = jumptable->arData;
+
+ | cmp FCARG2a, jumptable->nNumUsed
+ | jae >3
+ |.if X64
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | movsxd r0, dword [FCARG2a * 4 + >4]
+ | jmp r0
+ |.else
+ | jmp aword [FCARG2a * 4 + >4]
+ |.endif
+ |3:
+ |.cold_code
+ |4:
+ p = jumptable->arData;
+ do {
+ if (Z_TYPE(p->val) == IS_UNDEF) {
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | .aword =>b
+ } else {
+ int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes];
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | .aword =>b
+ }
+ p++;
+ count--;
+ } while (count);
+ |.code
+ } else {
+ | LOAD_ADDR FCARG1a, jumptable
+ | EXT_CALL zend_hash_index_find, r0
+ | test r0, r0
+ | jz =>b
+ | LOAD_ADDR FCARG1a, jumptable
+ | sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
+ |.if X64
+ | cqo
+ |.else
+ | cdq
+ |.endif
+ | idiv FCARG1a
+ |.if X64
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | movsxd r0, dword [r0 + >4]
+ | jmp r0
+ |.else
+ | jmp dword [r0 + >4]
+ |.endif
+ |3:
+ |.cold_code
+ |4:
+ ZEND_HASH_FOREACH_VAL(jumptable, val) {
+ b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | .aword =>b
+ } ZEND_HASH_FOREACH_END();
+ |.code
+ }
+ }
+ } else if (opline->opcode == ZEND_SWITCH_STRING) {
+ if (op1_info & MAY_BE_STRING) {
+ if (op1_info & MAY_BE_REF) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1
+ | GET_ZVAL_PTR FCARG2a, op1_addr
+ |.cold_code
+ |1:
+ | // ZVAL_DEREF(op)
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
+ | GET_ZVAL_PTR FCARG2a, op1_addr
+ | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3
+ | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)]
+ | jmp >2
+ |.code
+ |2:
+ } else {
+ if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
+ | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
+ }
+ | GET_ZVAL_PTR FCARG2a, op1_addr
+ }
+ | LOAD_ADDR FCARG1a, jumptable
+ | EXT_CALL zend_hash_find, r0
+ | test r0, r0
+ | jz =>b
+ | LOAD_ADDR FCARG1a, jumptable
+ | sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
+ |.if X64
+ | cqo
+ |.else
+ | cdq
+ |.endif
+ | idiv FCARG1a
+ |.if X64
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ | movsxd r0, dword [r0 + >4]
+ | jmp r0
+ |.else
+ | jmp dword [r0 + >4]
+ |.endif
+ |3:
+ |.cold_code
+ |4:
+ ZEND_HASH_FOREACH_VAL(jumptable, val) {
+ b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
+ | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
+ |.if X64
+ | .aword =>b
+ |.else
+ | .aword =>b
+ |.endif
+ } ZEND_HASH_FOREACH_END();
+ |.code
+ }
+ } else {
+ ZEND_ASSERT(0);
+ }
+ }
+ return 1;
+}
+
+static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, uint32_t position, int def_var, int use_var)
+{
+ if (ssa->var_info[def_var].type != ssa->var_info[use_var].type) {
+ return 0;
+ }
+
+ switch (op_array->opcodes[position].opcode) {
+ case ZEND_QM_ASSIGN:
+ case ZEND_SEND_VAR:
+ case ZEND_ASSIGN:
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ return 1;
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ if (def_var == ssa->ops[position].result_def &&
+ use_var == ssa->ops[position].op1_use) {
+ return 1;
+ }
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int var)
+{
+ uint32_t op1_info, op2_info;
+
+ switch (opline->opcode) {
+ case ZEND_QM_ASSIGN:
+ case ZEND_SEND_VAR:
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAL_EX:
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ case ZEND_CASE:
+ case ZEND_RETURN:
+ return 1;
+ case ZEND_ASSIGN:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ return
+ opline->op1_type == IS_CV &&
+ !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) &&
+ !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)));
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if ((op1_info | op2_info) & MAY_BE_UNDEF) {
+ return 0;
+ }
+ return (op1_info & op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) != 0;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ case ZEND_SL:
+ case ZEND_SR:
+ case ZEND_MOD:
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if ((op1_info | op2_info) & MAY_BE_UNDEF) {
+ return 0;
+ }
+ return (op1_info & op2_info & MAY_BE_LONG) != 0;
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ op1_info = OP1_INFO();
+ return
+ opline->op1_type == IS_CV &&
+ (op1_info & MAY_BE_LONG);
+ case ZEND_BOOL:
+ case ZEND_BOOL_NOT:
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ return 1;
+ }
+ return 0;
+}
+
+static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int var)
+{
+ if (ssa->vars[var].no_val) {
+ /* we don't need the value */
+ return 0;
+ }
+
+ if (zend_jit_reg_alloc < ZEND_JIT_REG_ALLOC_GLOBAL) {
+ /* Disable global register allocation,
+ * register allocation forSAA variables connected through Phi functions
+ */
+ if (ssa->vars[var].definition_phi) {
+ return 0;
+ }
+ if (ssa->vars[var].phi_use_chain) {
+ zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
+ do {
+ if (!ssa->vars[phi->ssa_var].no_val) {
+ return 0;
+ }
+ phi = zend_ssa_next_use_phi(ssa, var, phi);
+ } while (phi);
+ }
+ }
+
+ if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
+ ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
+ /* bad type */
+ return 0;
+ }
+
+ if (ssa->vars[var].definition >= 0) {
+ if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) {
+ return 0;
+ }
+ }
+
+ if (ssa->vars[var].use_chain >= 0) {
+ int use = ssa->vars[var].use_chain;
+
+ do {
+ if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
+ !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) {
+ return 0;
+ }
+ use = zend_ssa_next_use(ssa->ops, var, use);
+ } while (use >= 0);
+ }
+
+ return 1;
+}
+
+static zend_bool zend_needs_extra_reg_for_const(zend_op_array *op_array, const zend_op *opline, zend_uchar op_type, znode_op op)
+{
+|.if X64
+|| if (op_type == IS_CONST) {
+|| zval *zv = RT_CONSTANT(opline, op);
+|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_32BIT(zv)) {
+|| return 1;
+|| }
+|| }
+|.endif
+ return 0;
+}
+
+static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, uint32_t line, int current_var)
+{
+ const zend_op *opline = op_array->opcodes + line;
+ uint32_t op1_info, op2_info, res_info;
+ zend_regset regset = ZEND_REGSET_SCRATCH;
+
+ switch (opline->opcode) {
+ case ZEND_NOP:
+ case ZEND_OP_DATA:
+ case ZEND_JMP:
+ case ZEND_RETURN:
+ regset = ZEND_REGSET_EMPTY;
+ break;
+ case ZEND_QM_ASSIGN:
+ if (ssa->ops[line].op1_def == current_var ||
+ ssa->ops[line].result_def == current_var) {
+ regset = ZEND_REGSET_EMPTY;
+ break;
+ }
+ /* break missing intentionally */
+ case ZEND_SEND_VAL:
+ case ZEND_SEND_VAL_EX:
+ if (ssa->ops[line].op1_use == current_var) {
+ regset = ZEND_REGSET(ZREG_R0);
+ break;
+ }
+ op1_info = OP1_INFO();
+ if (!(op1_info & MAY_BE_UNDEF)) {
+ if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
+ regset = ZEND_REGSET(ZREG_XMM0);
+ } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
+ regset = ZEND_REGSET(ZREG_R0);
+ } else {
+ regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
+ }
+ }
+ break;
+ case ZEND_SEND_VAR:
+ if (ssa->ops[line].op1_use == current_var ||
+ ssa->ops[line].op1_def == current_var) {
+ regset = ZEND_REGSET_EMPTY;
+ break;
+ }
+ op1_info = OP1_INFO();
+ if (!(op1_info & MAY_BE_UNDEF)) {
+ if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
+ regset = ZEND_REGSET(ZREG_XMM0);
+ } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
+ } else {
+ regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
+ if (op1_info & MAY_BE_REF) {
+ ZEND_REGSET_INCL(regset, ZREG_R1);
+ }
+ }
+ }
+ break;
+ case ZEND_ASSIGN:
+ if (ssa->ops[line].op2_use == current_var ||
+ ssa->ops[line].op2_def == current_var ||
+ ssa->ops[line].op1_def == current_var ||
+ ssa->ops[line].result_def == current_var) {
+ regset = ZEND_REGSET_EMPTY;
+ break;
+ }
+ op1_info = OP1_INFO();
+ op2_info = OP2_INFO();
+ if (opline->op1_type == IS_CV
+ && !(op2_info & MAY_BE_UNDEF)
+ && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF))) {
+ if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
+ regset = ZEND_REGSET(ZREG_XMM0);
+ } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
+ regset = ZEND_REGSET(ZREG_R0);
+ } else {
+ regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
+ }
+ }
+ break;
+ case ZEND_PRE_INC:
+ case ZEND_PRE_DEC:
+ case ZEND_POST_INC:
+ case ZEND_POST_DEC:
+ if (ssa->ops[line].op1_use == current_var ||
+ ssa->ops[line].op1_def == current_var ||
+ ssa->ops[line].result_def == current_var) {
+ regset = ZEND_REGSET_EMPTY;
+ break;
+ }
+ op1_info = OP1_INFO();
+ if (opline->op1_type == IS_CV
+ && (op1_info & MAY_BE_LONG)
+ && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ regset = ZEND_REGSET_EMPTY;
+ if (op1_info & MAY_BE_DOUBLE) {
+ regset = ZEND_REGSET(ZREG_XMM0);
+ }
+ }
+ break;
+ case ZEND_ADD:
+ case ZEND_SUB:
+ case ZEND_MUL:
+ op1_info = OP1_INFO();
+ op2_info = OP1_INFO();
+ if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
+ !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+
+ regset = ZEND_REGSET_EMPTY;
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
+ if (ssa->ops[line].result_def != current_var &&
+ (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ res_info = OP1_INFO();
+ if (res_info & MAY_BE_DOUBLE) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ ZEND_REGSET_INCL(regset, ZREG_XMM1);
+ }
+ }
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
+ if (ssa->ops[line].result_def != current_var) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ }
+ if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
+ if (zend_is_commutative(opline->opcode)) {
+ if (ssa->ops[line].result_def != current_var) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ } else {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ if (ssa->ops[line].result_def != current_var &&
+ (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM1);
+ }
+ }
+ }
+ if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
+ if (ssa->ops[line].result_def != current_var &&
+ (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ }
+ if (zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op1) ||
+ zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op2)) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ }
+ break;
+ case ZEND_BW_OR:
+ case ZEND_BW_AND:
+ case ZEND_BW_XOR:
+ op1_info = OP1_INFO();
+ op2_info = OP1_INFO();
+ if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
+ !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ regset = ZEND_REGSET_EMPTY;
+ if (ssa->ops[line].result_def != current_var &&
+ (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ }
+ break;
+ case ZEND_SL:
+ case ZEND_SR:
+ op1_info = OP1_INFO();
+ op2_info = OP1_INFO();
+ if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
+ !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ regset = ZEND_REGSET_EMPTY;
+ if (ssa->ops[line].result_def != current_var &&
+ (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ if (opline->op2_type != IS_CONST && ssa->ops[line].op2_use != current_var) {
+ ZEND_REGSET_INCL(regset, ZREG_R1);
+ }
+ }
+ break;
+ case ZEND_MOD:
+ op1_info = OP1_INFO();
+ op2_info = OP1_INFO();
+ if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
+ !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
+ regset = ZEND_REGSET_EMPTY;
+ if (opline->op2_type == IS_CONST &&
+ Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG &&
+ zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)))) {
+ if (ssa->ops[line].result_def != current_var &&
+ (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ } else {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ ZEND_REGSET_INCL(regset, ZREG_R1);
+ }
+ }
+ break;
+ case ZEND_IS_SMALLER:
+ case ZEND_IS_SMALLER_OR_EQUAL:
+ case ZEND_IS_EQUAL:
+ case ZEND_IS_NOT_EQUAL:
+ case ZEND_IS_IDENTICAL:
+ case ZEND_IS_NOT_IDENTICAL:
+ case ZEND_CASE:
+ op1_info = OP1_INFO();
+ op2_info = OP1_INFO();
+ if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
+ !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ regset = ZEND_REGSET_EMPTY;
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) &&
+ opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) {
+ if (ssa->ops[line].op1_use != current_var &&
+ ssa->ops[line].op2_use != current_var) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ }
+ if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
+ if (ssa->ops[line].op1_use != current_var &&
+ ssa->ops[line].op2_use != current_var) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ }
+ if (zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op1) ||
+ zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op2)) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ }
+ break;
+ case ZEND_BOOL:
+ case ZEND_BOOL_NOT:
+ case ZEND_JMPZ:
+ case ZEND_JMPNZ:
+ case ZEND_JMPZNZ:
+ case ZEND_JMPZ_EX:
+ case ZEND_JMPNZ_EX:
+ op1_info = OP1_INFO();
+ if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) {
+ regset = ZEND_REGSET_EMPTY;
+ if (op1_info & MAY_BE_DOUBLE) {
+ ZEND_REGSET_INCL(regset, ZREG_XMM0);
+ }
+ if (opline->opcode == ZEND_BOOL ||
+ opline->opcode == ZEND_BOOL_NOT ||
+ opline->opcode == ZEND_JMPZ_EX ||
+ opline->opcode == ZEND_JMPNZ_EX) {
+ ZEND_REGSET_INCL(regset, ZREG_R0);
+ }
+ }
+ break;
+ case ZEND_DO_UCALL:
+ case ZEND_DO_FCALL:
+ case ZEND_DO_FCALL_BY_NAME:
+ case ZEND_INCLUDE_OR_EVAL:
+ case ZEND_GENERATOR_CREATE:
+ case ZEND_YIELD:
+ case ZEND_YIELD_FROM:
+ regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
+ break;
+ default:
+ break;
+ }
+
+ return regset;
+}
+
+/*
+ * Local variables:
+ * tab-width: 4
+ * c-basic-offset: 4
+ * indent-tabs-mode: t
+ * End:
+ */
projects / php / commitdiff