From: Dmitry Stogov Date: Mon, 9 Sep 2019 18:25:54 +0000 (+0300) Subject: Avoid hash value recalculation on each counter update X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=f43fe067e929aee40cbf1036492550879311e772;p=php Avoid hash value recalculation on each counter update --- diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bba076cac7..3509c584c1 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2693,18 +2693,19 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) { zend_op_array *op_array = &EX(func)->op_array; - const void **orig_handlers; + zend_jit_op_array_extension *jit_extension; uint32_t i; zend_shared_alloc_lock(); - orig_handlers = (const void**)ZEND_FUNC_INFO(op_array); + jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); - if (orig_handlers) { + if (jit_extension) { SHM_UNPROTECT(); zend_jit_unprotect(); + *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; for (i = 0; i < op_array->last; i++) { - op_array->opcodes[i].handler = orig_handlers[i]; + op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } ZEND_SET_FUNC_INFO(op_array, NULL); @@ -2723,7 +2724,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend static int zend_jit_setup_hot_counters(zend_op_array *op_array) { zend_op *opline = op_array->opcodes; - const void **orig_handlers; + zend_jit_op_array_extension *jit_extension; zend_cfg cfg; uint32_t i; @@ -2731,11 +2732,12 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) return FAILURE; } - orig_handlers = (const void**)zend_shared_alloc(op_array->last * sizeof(void*)); + jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension) + (op_array->last - 1) * sizeof(void*)); + jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)]; for (i = 0; i < op_array->last; i++) { - orig_handlers[i] = op_array->opcodes[i].handler; + jit_extension->orig_handlers[i] = op_array->opcodes[i].handler; } - ZEND_SET_FUNC_INFO(op_array, (void*)orig_handlers); + ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { opline++; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 603db0ab4b..f327fd5128 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -28,12 +28,35 @@ extern int zend_jit_profile_counter_rid; ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid) /* Hot Counters */ + #define ZEND_HOT_COUNTERS_COUNT 128 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); +typedef struct _zend_jit_op_array_extension { + int16_t *counter; + const void *orig_handlers[1]; +} zend_jit_op_array_extension; + +static zend_always_inline zend_long zend_jit_op_array_hash(const zend_op_array *op_array) +{ + uintptr_t x; + + x = (uintptr_t)op_array->opcodes >> 3; +#if SIZEOF_SIZE_T == 4 + x = ((x >> 16) ^ x) * 0x45d9f3b; + x = ((x >> 16) ^ x) * 0x45d9f3b; + x = (x >> 16) ^ x; +#elif SIZEOF_SIZE_T == 8 + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + x = x ^ (x >> 31); +#endif + return x; +} + extern const zend_op *zend_jit_halt_op; #ifdef HAVE_GCC_GLOBAL_REGS diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 51dbc75828..346d50ec59 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -182,67 +182,40 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLE ZEND_OPCODE_TAIL_CALL(handler); } -static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array) -{ - uintptr_t x; - - if (op_array->function_name) { - x = (uintptr_t)op_array >> 3; - } else { - x = (uintptr_t)op_array->filename >> 3; - } -#if SIZEOF_SIZE_T == 4 - x = ((x >> 16) ^ x) * 0x45d9f3b; - x = ((x >> 16) ^ x) * 0x45d9f3b; - x = (x >> 16) ^ x; -#elif SIZEOF_SIZE_T == 8 - x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; - x = (x ^ (x >> 27)) * 0x94d049bb133111eb; - x = x ^ (x >> 31); -#endif - return x; -} - ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS) { + zend_jit_op_array_extension *jit_extension = + (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif - unsigned int n = _op_array_hash(&EX(func)->op_array) % - (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); - zend_jit_hot_counters[n] -= ZEND_JIT_HOT_FUNC_COST; + *(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST; - if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) { - zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT; + if (UNEXPECTED(*(jit_extension->counter) <= 0)) { zend_jit_hot_func(execute_data, opline); ZEND_OPCODE_RETURN(); } else { - zend_vm_opcode_handler_t *handlers = - (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array); - zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes]; + zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes]; ZEND_OPCODE_TAIL_CALL(handler); } } ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS) { + zend_jit_op_array_extension *jit_extension = + (zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif - unsigned int n = _op_array_hash(&EX(func)->op_array) % - (sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0])); - zend_jit_hot_counters[n] -= ZEND_JIT_HOT_LOOP_COST; + *(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST; - if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) { - zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT; + if (UNEXPECTED(*(jit_extension->counter) <= 0)) { zend_jit_hot_func(execute_data, opline); ZEND_OPCODE_RETURN(); } else { - zend_vm_opcode_handler_t *handlers = - (zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array); - zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes]; + zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes]; ZEND_OPCODE_TAIL_CALL(handler); } } diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 2473be4198..aa47e5e97c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2176,68 +2176,27 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) { |->hybrid_func_counter: | mov r0, EX->func - | mov r1, aword [r0 + offsetof(zend_op_array, function_name)] - | test r1, r1 - | jne >1 - | mov r0, aword [r0 + offsetof(zend_op_array, filename)] - |1: - | shr r0, 3 - | mov r1, r0 - | .if X64 - | shr r0, 30 - | xor r0, r1 - | mov64 r1, 0xbf58476d1ce4e5b9 - | imul r0, r1 - | mov r1, r0 - | shr r0, 27 - | xor r0, r1 - | mov64 r1, 0x94d049bb133111eb - | imul r0, r1 - | mov r1, r0 - | shr r0, 31 - | xor r1, r0 - | and r1, ZEND_HOT_COUNTERS_COUNT-1 - | LOAD_ADDR r0, &zend_jit_hot_counters - | sub word [r0+r1*2], ZEND_JIT_HOT_FUNC_COST - | .else - | shr r0, 16 - | xor r0, r1 - | imul r0, 0x45d9f3b - | mov r1, r0 - | shr r0, 16 - | xor r0, r1 - | imul r0, 0x45d9f3b - | mov r1, r0 - | shr r0, 16 - | xor r1, r0 - | and r1, ZEND_HOT_COUNTERS_COUNT-1 - | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_FUNC_COST - | .endif + | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | mov r2, aword [r1] + | sub word [r2], ZEND_JIT_HOT_FUNC_COST | jle >1 - | mov r1, EX->func - | GET_IP r0 - | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] + | GET_IP r2 + | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 || ZEND_ASSERT(sizeof(zend_op) == 32); - | sar r0, 5 + | sar r2, 5 | .else || ZEND_ASSERT(sizeof(zend_op) == 28); - | sar r0, 2 - | imul r0, 0xb6db6db7 + | sar r2, 2 + | imul r2, 0xb6db6db7 | .endif - | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | .if X64 - | jmp aword [r1+r0*8] + | jmp aword [r1+r2*8+8] | .else - | jmp aword [r1+r0*4] + | jmp aword [r1+r2*4+4] | .endif |1: - | .if X64 - | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT - | .else - | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT - | .endif | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0 @@ -2249,68 +2208,27 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) { |->hybrid_loop_counter: | mov r0, EX->func - | mov r1, aword [r0 + offsetof(zend_op_array, function_name)] - | test r1, r1 - | jne >1 - | mov r0, aword [r0 + offsetof(zend_op_array, filename)] - |1: - | shr r0, 3 - | mov r1, r0 - | .if X64 - | shr r0, 30 - | xor r0, r1 - | mov64 r1, 0xbf58476d1ce4e5b9 - | imul r0, r1 - | mov r1, r0 - | shr r0, 27 - | xor r0, r1 - | mov64 r1, 0x94d049bb133111eb - | imul r0, r1 - | mov r1, r0 - | shr r0, 31 - | xor r1, r0 - | and r1, ZEND_HOT_COUNTERS_COUNT-1 - | LOAD_ADDR r0, &zend_jit_hot_counters - | sub word [r0+r1*2], ZEND_JIT_HOT_LOOP_COST - | .else - | shr r0, 16 - | xor r0, r1 - | imul r0, 0x45d9f3b - | mov r1, r0 - | shr r0, 16 - | xor r0, r1 - | imul r0, 0x45d9f3b - | mov r1, r0 - | shr r0, 16 - | xor r1, r0 - | and r1, ZEND_HOT_COUNTERS_COUNT-1 - | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_LOOP_COST - | .endif + | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | mov r2, aword [r1] + | sub word [r2], ZEND_JIT_HOT_LOOP_COST | jle >1 - | mov r1, EX->func - | GET_IP r0 - | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] + | GET_IP r2 + | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 || ZEND_ASSERT(sizeof(zend_op) == 32); - | sar r0, 5 + | sar r2, 5 | .else || ZEND_ASSERT(sizeof(zend_op) == 28); - | sar r0, 2 - | imul r0, 0xb6db6db7 + | sar r2, 2 + | imul r2, 0xb6db6db7 | .endif - | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | .if X64 - | jmp aword [r1+r0*8] + | jmp aword [r1+r2*8+8] | .else - | jmp aword [r1+r0*4] + | jmp aword [r1+r2*4+4] | .endif |1: - | .if X64 - | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT - | .else - | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT - | .endif | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0