Use fast SSE2 memcpy() for copying block of data from SHM to process memory

diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c
index 62e62e18214df228fa03fc67a4b6170802650aee..cdcbfeda560198eb9ad6635f242afb37cca83f5a 100644 (file)
--- a/ext/opcache/ZendAccelerator.c
+++ b/ext/opcache/ZendAccelerator.c
@@ -1163,7 +1163,13 @@ static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_scr
        memory_used = zend_accel_script_persist_calc(new_persistent_script, key, key_length);
 
        /* Allocate shared memory */
+#ifdef __SSE2__
+       /* Align to 64-byte boundary */
+       ZCG(mem) = zend_shared_alloc(memory_used + 64);
+       ZCG(mem) = (void*)(((zend_uintptr_t)ZCG(mem) + 63L) & ~63L);
+#else
        ZCG(mem) = zend_shared_alloc(memory_used);
+#endif
        if (!ZCG(mem)) {
                zend_accel_schedule_restart_if_necessary(ACCEL_RESTART_OOM);
                zend_shared_alloc_unlock();

diff --git a/ext/opcache/zend_accelerator_util_funcs.c b/ext/opcache/zend_accelerator_util_funcs.c
index 8a52a531034dfbcce57fc6774cca51d4a4f98d08..2e7f0c8da5af94de4ab8323bd6289f48e9e51e0b 100644 (file)
--- a/ext/opcache/zend_accelerator_util_funcs.c
+++ b/ext/opcache/zend_accelerator_util_funcs.c
@@ -771,6 +771,34 @@ failure:
        zend_error(E_ERROR, "Cannot redeclare class %s", ce1->name->val);
 }
 
+#ifdef __SSE2__
+#include <mmintrin.h>
+#include <emmintrin.h>
+
+static zend_always_inline void fast_memcpy(void *dest, const void *src, size_t size)
+{
+       __m128i *dqdest = (__m128i*)dest;
+       const __m128i *dqsrc  = (const __m128i*)src;
+       const __m128i *end  = (const __m128i*)((const char*)src + size);
+
+       do {
+               _mm_prefetch(dqsrc + 4, _MM_HINT_NTA);
+               _mm_prefetch(dqsrc + 6, _MM_HINT_NTA);
+
+               __m128i xmm0 = _mm_load_si128(dqsrc + 0);
+               __m128i xmm1 = _mm_load_si128(dqsrc + 1);
+               __m128i xmm2 = _mm_load_si128(dqsrc + 2);
+               __m128i xmm3 = _mm_load_si128(dqsrc + 3);
+               dqsrc  += 4;
+               _mm_stream_si128(dqdest + 0, xmm0);
+               _mm_stream_si128(dqdest + 1, xmm1);
+               _mm_stream_si128(dqdest + 2, xmm2);
+               _mm_stream_si128(dqdest + 3, xmm3);
+               dqdest += 4;
+       } while (dqsrc != end);
+}
+#endif
+
 zend_op_array* zend_accel_load_script(zend_persistent_script *persistent_script, int from_shared_memory)
 {
        zend_op_array *op_array;
@@ -784,8 +812,15 @@ zend_op_array* zend_accel_load_script(zend_persistent_script *persistent_script,
                ZCG(current_persistent_script) = persistent_script;
                ZCG(arena_mem) = NULL;
                if (EXPECTED(persistent_script->arena_size)) {
+#ifdef __SSE2__
+                       /* Target address must be aligned to 64-byte boundary */
+                       ZCG(arena_mem) = zend_arena_alloc(&CG(arena), persistent_script->arena_size + 64);
+                       ZCG(arena_mem) = (void*)(((zend_uintptr_t)ZCG(arena_mem) + 63L) & ~63L);
+                       fast_memcpy(ZCG(arena_mem), persistent_script->arena_mem, persistent_script->arena_size);
+#else
                        ZCG(arena_mem) = zend_arena_alloc(&CG(arena), persistent_script->arena_size);
                        memcpy(ZCG(arena_mem), persistent_script->arena_mem, persistent_script->arena_size);
+#endif
                }
 
                /* Copy all the necessary stuff from shared memory to regular memory, and protect the shared script */

diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c
index 1c440eceae037976f5f92a731f9750a0492f106e..eb442e97428696537bc1947e296c3a81dd5014df 100644 (file)
--- a/ext/opcache/zend_persist.c
+++ b/ext/opcache/zend_persist.c
@@ -734,6 +734,11 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script
        *key = zend_accel_memdup(*key, key_length + 1);
        zend_accel_store_string(script->full_path);
 
+#ifdef __SSE2__
+       /* Align to 64-byte boundary */
+       ZCG(mem) = (void*)(((zend_uintptr_t)ZCG(mem) + 63L) & ~63L);
+#endif
+
        script->arena_mem = ZCG(arena_mem) = ZCG(mem);
        ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size);
 

diff --git a/ext/opcache/zend_persist_calc.c b/ext/opcache/zend_persist_calc.c
index 2f211e43cb8d1a7bf92f0ff90d3a0c05e316e9f5..7d77d62b8dc9dc73320023d99769d5447e22bf99 100644 (file)
--- a/ext/opcache/zend_persist_calc.c
+++ b/ext/opcache/zend_persist_calc.c
@@ -361,10 +361,20 @@ uint zend_accel_script_persist_calc(zend_persistent_script *new_persistent_scrip
        ADD_DUP_SIZE(key, key_length + 1);
        ADD_STRING(new_persistent_script->full_path);
 
+#ifdef __SSE2__
+       /* Align size to 64-byte boundary */
+       new_persistent_script->size = (new_persistent_script->size + 63) & ~63;
+#endif
+
        zend_accel_persist_class_table_calc(&new_persistent_script->class_table);
        zend_hash_persist_calc(&new_persistent_script->function_table, zend_persist_op_array_calc);
        zend_persist_op_array_calc_ex(&new_persistent_script->main_op_array);
 
+#ifdef __SSE2__
+       /* Align size to 64-byte boundary */
+       new_persistent_script->arena_size = (new_persistent_script->arena_size + 63) & ~63;
+#endif
+
        new_persistent_script->size += new_persistent_script->arena_size;
 
        ZCG(current_persistent_script) = NULL;