Avoid hash value recalculation on each counter update

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index bba076cac7f1f99591c82639ae0b4a39be09a7c2..3509c584c10f6308593e97a433d81a627b8262fa 100644 (file)
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -2693,18 +2693,19 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) {
 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
 {
        zend_op_array *op_array = &EX(func)->op_array;
-       const void **orig_handlers;
+       zend_jit_op_array_extension *jit_extension;
        uint32_t i;
 
        zend_shared_alloc_lock();
-       orig_handlers = (const void**)ZEND_FUNC_INFO(op_array);
+       jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
 
-       if (orig_handlers) {
+       if (jit_extension) {
                SHM_UNPROTECT();
                zend_jit_unprotect();
 
+               *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT;
                for (i = 0; i < op_array->last; i++) {
-                       op_array->opcodes[i].handler = orig_handlers[i];
+                       op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
                }
                ZEND_SET_FUNC_INFO(op_array, NULL);
 
@@ -2723,7 +2724,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend
 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
 {
        zend_op *opline = op_array->opcodes;
-       const void **orig_handlers;
+       zend_jit_op_array_extension *jit_extension;
        zend_cfg cfg;
        uint32_t i;
 
@@ -2731,11 +2732,12 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array)
                return FAILURE;
        }
 
-       orig_handlers = (const void**)zend_shared_alloc(op_array->last * sizeof(void*));
+       jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension) + (op_array->last - 1) * sizeof(void*));
+       jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
        for (i = 0; i < op_array->last; i++) {
-               orig_handlers[i] = op_array->opcodes[i].handler;
+               jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
        }
-       ZEND_SET_FUNC_INFO(op_array, (void*)orig_handlers);
+       ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
 
        while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
                opline++;

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index 603db0ab4b6dd64f33135f9131696873d719c291..f327fd5128067f0f2e1f171d081d9f6b6d226a52 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -28,12 +28,35 @@ extern int zend_jit_profile_counter_rid;
        ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid)
 
 /* Hot Counters */
+
 #define ZEND_HOT_COUNTERS_COUNT 128
 
 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
 
 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
 
+typedef struct _zend_jit_op_array_extension {
+       int16_t    *counter;
+       const void *orig_handlers[1];
+} zend_jit_op_array_extension;
+
+static zend_always_inline zend_long zend_jit_op_array_hash(const zend_op_array *op_array)
+{
+       uintptr_t x;
+
+       x = (uintptr_t)op_array->opcodes >> 3;
+#if SIZEOF_SIZE_T == 4
+       x = ((x >> 16) ^ x) * 0x45d9f3b;
+       x = ((x >> 16) ^ x) * 0x45d9f3b;
+       x = (x >> 16) ^ x;
+#elif SIZEOF_SIZE_T == 8
+       x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
+       x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
+       x = x ^ (x >> 31);
+#endif
+       return x;
+}
+
 extern const zend_op *zend_jit_halt_op;
 
 #ifdef HAVE_GCC_GLOBAL_REGS

diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c
index 51dbc75828f49fd59dd19eb2d65dbb120d9998a7..346d50ec59f452f727029908cfe33a01a6b19324 100644 (file)
--- a/ext/opcache/jit/zend_jit_vm_helpers.c
+++ b/ext/opcache/jit/zend_jit_vm_helpers.c
@@ -182,67 +182,40 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLE
        ZEND_OPCODE_TAIL_CALL(handler);
 }
 
-static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array)
-{
-       uintptr_t x;
-
-       if (op_array->function_name) {
-               x = (uintptr_t)op_array >> 3;
-       } else {
-               x = (uintptr_t)op_array->filename >> 3;
-       }
-#if SIZEOF_SIZE_T == 4
-       x = ((x >> 16) ^ x) * 0x45d9f3b;
-       x = ((x >> 16) ^ x) * 0x45d9f3b;
-       x = (x >> 16) ^ x;
-#elif SIZEOF_SIZE_T == 8
-       x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
-       x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
-       x = x ^ (x >> 31);
-#endif
-       return x;
-}
-
 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
 {
+       zend_jit_op_array_extension *jit_extension =
+               (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
 #ifndef HAVE_GCC_GLOBAL_REGS
        const zend_op *opline = EX(opline);
 #endif
-       unsigned int n = _op_array_hash(&EX(func)->op_array)  %
-               (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0]));
 
-       zend_jit_hot_counters[n] -= ZEND_JIT_HOT_FUNC_COST;
+       *(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST;
 
-       if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) {
-               zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT;
+       if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
                zend_jit_hot_func(execute_data, opline);
                ZEND_OPCODE_RETURN();
        } else {
-               zend_vm_opcode_handler_t *handlers =
-                       (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array);
-               zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes];
+               zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
                ZEND_OPCODE_TAIL_CALL(handler);
        }
 }
 
 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
 {
+       zend_jit_op_array_extension *jit_extension =
+               (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
 #ifndef HAVE_GCC_GLOBAL_REGS
        const zend_op *opline = EX(opline);
 #endif
-       unsigned int n = _op_array_hash(&EX(func)->op_array)  %
-               (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0]));
 
-       zend_jit_hot_counters[n] -= ZEND_JIT_HOT_LOOP_COST;
+       *(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST;
 
-       if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) {
-               zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT;
+       if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
                zend_jit_hot_func(execute_data, opline);
                ZEND_OPCODE_RETURN();
        } else {
-               zend_vm_opcode_handler_t *handlers =
-                       (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array);
-               zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes];
+               zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
                ZEND_OPCODE_TAIL_CALL(handler);
        }
 }

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 2473be4198254bfc9451989f931c378caac86ed9..aa47e5e97c07d8ee0d4efecee09cca4578235565 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -2176,68 +2176,27 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst)
 {
        |->hybrid_func_counter:
        |       mov r0, EX->func
-       |       mov r1, aword [r0 + offsetof(zend_op_array, function_name)]
-       |       test r1, r1
-       |       jne     >1
-       |       mov r0, aword [r0 + offsetof(zend_op_array, filename)]
-       |1:
-       |       shr r0, 3
-       |       mov r1, r0
-       |       .if X64
-       |               shr r0, 30
-       |               xor r0, r1
-       |               mov64 r1, 0xbf58476d1ce4e5b9
-       |               imul r0, r1
-       |               mov r1, r0
-       |               shr r0, 27
-       |               xor r0, r1
-       |               mov64 r1, 0x94d049bb133111eb
-       |               imul r0, r1
-       |               mov r1, r0
-       |               shr r0, 31
-       |               xor r1, r0
-       |               and r1, ZEND_HOT_COUNTERS_COUNT-1
-       |               LOAD_ADDR r0, &zend_jit_hot_counters
-       |               sub word [r0+r1*2], ZEND_JIT_HOT_FUNC_COST
-       |       .else
-       |               shr r0, 16
-       |               xor r0, r1
-       |               imul r0, 0x45d9f3b
-       |               mov r1, r0
-       |               shr r0, 16
-       |               xor r0, r1
-       |               imul r0, 0x45d9f3b
-       |               mov r1, r0
-       |               shr r0, 16
-       |               xor r1, r0
-       |               and r1, ZEND_HOT_COUNTERS_COUNT-1
-       |               sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_FUNC_COST
-       |       .endif
+       |       mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+       |       mov r2, aword [r1]
+       |       sub word [r2], ZEND_JIT_HOT_FUNC_COST
        |       jle >1
-       |       mov r1, EX->func
-       |       GET_IP r0
-       |       sub r0, aword [r1 + offsetof(zend_op_array, opcodes)]
+       |       GET_IP r2
+       |       sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
        |       // divide by sizeof(zend_op)
        |       .if X64
        ||              ZEND_ASSERT(sizeof(zend_op) == 32);
-       |               sar r0, 5
+       |               sar r2, 5
        |       .else
        ||              ZEND_ASSERT(sizeof(zend_op) == 28);
-       |               sar r0, 2
-       |               imul r0, 0xb6db6db7
+       |               sar r2, 2
+       |               imul r2, 0xb6db6db7
        |       .endif
-       |       mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
        |       .if X64
-       |               jmp aword [r1+r0*8]
+       |               jmp aword [r1+r2*8+8]
        |       .else
-       |               jmp aword [r1+r0*4]
+       |               jmp aword [r1+r2*4+4]
        |       .endif
        |1:
-       |       .if X64
-       |               mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT
-       |       .else
-       |               mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT
-       |       .endif
        |       mov FCARG1a, FP
        |       GET_IP FCARG2a
        |       EXT_CALL zend_jit_hot_func, r0
@@ -2249,68 +2208,27 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst)
 {
        |->hybrid_loop_counter:
        |       mov r0, EX->func
-       |       mov r1, aword [r0 + offsetof(zend_op_array, function_name)]
-       |       test r1, r1
-       |       jne     >1
-       |       mov r0, aword [r0 + offsetof(zend_op_array, filename)]
-       |1:
-       |       shr r0, 3
-       |       mov r1, r0
-       |       .if X64
-       |               shr r0, 30
-       |               xor r0, r1
-       |               mov64 r1, 0xbf58476d1ce4e5b9
-       |               imul r0, r1
-       |               mov r1, r0
-       |               shr r0, 27
-       |               xor r0, r1
-       |               mov64 r1, 0x94d049bb133111eb
-       |               imul r0, r1
-       |               mov r1, r0
-       |               shr r0, 31
-       |               xor r1, r0
-       |               and r1, ZEND_HOT_COUNTERS_COUNT-1
-       |               LOAD_ADDR r0, &zend_jit_hot_counters
-       |               sub word [r0+r1*2], ZEND_JIT_HOT_LOOP_COST
-       |       .else
-       |               shr r0, 16
-       |               xor r0, r1
-       |               imul r0, 0x45d9f3b
-       |               mov r1, r0
-       |               shr r0, 16
-       |               xor r0, r1
-       |               imul r0, 0x45d9f3b
-       |               mov r1, r0
-       |               shr r0, 16
-       |               xor r1, r0
-       |               and r1, ZEND_HOT_COUNTERS_COUNT-1
-       |               sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_LOOP_COST
-       |       .endif
+       |       mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+       |       mov r2, aword [r1]
+       |       sub word [r2], ZEND_JIT_HOT_LOOP_COST
        |       jle >1
-       |       mov r1, EX->func
-       |       GET_IP r0
-       |       sub r0, aword [r1 + offsetof(zend_op_array, opcodes)]
+       |       GET_IP r2
+       |       sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
        |       // divide by sizeof(zend_op)
        |       .if X64
        ||              ZEND_ASSERT(sizeof(zend_op) == 32);
-       |               sar r0, 5
+       |               sar r2, 5
        |       .else
        ||              ZEND_ASSERT(sizeof(zend_op) == 28);
-       |               sar r0, 2
-       |               imul r0, 0xb6db6db7
+       |               sar r2, 2
+       |               imul r2, 0xb6db6db7
        |       .endif
-       |       mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
        |       .if X64
-       |               jmp aword [r1+r0*8]
+       |               jmp aword [r1+r2*8+8]
        |       .else
-       |               jmp aword [r1+r0*4]
+       |               jmp aword [r1+r2*4+4]
        |       .endif
        |1:
-       |       .if X64
-       |               mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT
-       |       .else
-       |               mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT
-       |       .endif
        |       mov FCARG1a, FP
        |       GET_IP FCARG2a
        |       EXT_CALL zend_jit_hot_func, r0