Support JIT jumptables on x64

diff --git a/ext/opcache/jit/dynasm/dasm_x86.h b/ext/opcache/jit/dynasm/dasm_x86.h
index 77945ac65540135d75b371fca79e992a865aea4e..39449c15afdb3ca5ad77c2a6b45b98b40e13e3aa 100644 (file)
--- a/ext/opcache/jit/dynasm/dasm_x86.h
+++ b/ext/opcache/jit/dynasm/dasm_x86.h
@@ -17,11 +17,11 @@
 
 /* Action definitions. DASM_STOP must be 255. */
 enum {
-  DASM_DISP = 233,
+  DASM_DISP = 232,
   DASM_IMM_S, DASM_IMM_B, DASM_IMM_W, DASM_IMM_D, DASM_IMM_WB, DASM_IMM_DB,
   DASM_VREG, DASM_SPACE, DASM_SETLABEL, DASM_REL_A, DASM_REL_LG, DASM_REL_PC,
-  DASM_IMM_LG, DASM_IMM_PC, DASM_LABEL_LG, DASM_LABEL_PC, DASM_ALIGN,
-  DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
+  DASM_IMM_LG, DASM_IMM_PC, DASM_IMM_PC64, DASM_LABEL_LG, DASM_LABEL_PC,
+  DASM_ALIGN, DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
 };
 
 /* Maximum number of section buffer positions for a single dasm_put() call. */
@@ -228,6 +228,7 @@ void dasm_put(Dst_DECL, int start, ...)
        pl -= 246; n = *pl;
        if (n < 0) n = 0;  /* Start new chain for fwd rel if label exists. */
        goto linkrel;
+         case DASM_IMM_PC64: ofs += 4;
       case DASM_REL_PC:
       case DASM_IMM_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC);
       putrel:
@@ -335,7 +336,8 @@ int dasm_link(Dst_DECL, size_t *szp)
        case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++;
        case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W:
        case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB:
-       case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break;
+       case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: case DASM_IMM_PC64:
+         pos++; break;
        case DASM_LABEL_LG: p++;
        case DASM_LABEL_PC: b[pos++] += ofs; break; /* Fix label offset. */
        case DASM_ALIGN: ofs -= (b[pos++]+ofs)&*p++; break; /* Adjust ofs. */
@@ -359,10 +361,13 @@ int dasm_link(Dst_DECL, size_t *szp)
 #ifndef DASM_ALIGNED_WRITES
 typedef ZEND_SET_ALIGNED(1, unsigned short unaligned_short);
 typedef ZEND_SET_ALIGNED(1, unsigned int unaligned_int);
+typedef ZEND_SET_ALIGNED(1, uint64_t unaligned_uint64_t);
 #define dasmw(x) \
   do { *((unaligned_short *)cp) = (unsigned short)(x); cp+=2; } while (0)
 #define dasmd(x) \
   do { *((unaligned_int *)cp) = (unsigned int)(x); cp+=4; } while (0)
+#define dasmq(x) \
+  do { *((unaligned_uint64_t *)cp) = (uint64_t)(x); cp+=8; } while (0)
 #else
 #define dasmw(x)       do { dasmb(x); dasmb((x)>>8); } while (0)
 #define dasmd(x)       do { dasmw(x); dasmw((x)>>16); } while (0)
@@ -442,6 +447,11 @@ int dasm_encode(Dst_DECL, void *buffer)
          n = *pb < 0 ? pb[1] : (*pb + (int)(ptrdiff_t)base);
          goto wd;
        }
+       case DASM_IMM_PC64: {
+         int *pb = DASM_POS2PTR(D, n);
+         dasmq(*pb < 0 ? pb[1] : (*pb + (ptrdiff_t)base));
+         break;
+       }
        case DASM_LABEL_LG: {
          int idx = *p++;
          if (idx >= 10)

diff --git a/ext/opcache/jit/dynasm/dasm_x86.lua b/ext/opcache/jit/dynasm/dasm_x86.lua
index a5efd98fb5f39e0a547e0bdccc511a54815ca1d8..f7d81d838f96232b5941d84295df749ec58943c6 100644 (file)
--- a/ext/opcache/jit/dynasm/dasm_x86.lua
+++ b/ext/opcache/jit/dynasm/dasm_x86.lua
@@ -47,7 +47,7 @@ local action_names = {
   -- action arg (1 byte) or int arg, 2 buffer pos (link, offset):
   "REL_LG", "REL_PC",
   -- action arg (1 byte) or int arg, 1 buffer pos (link):
-  "IMM_LG", "IMM_PC",
+  "IMM_LG", "IMM_PC", "IMM_PC64",
   -- action arg (1 byte) or int arg, 1 buffer pos (offset):
   "LABEL_LG", "LABEL_PC",
   -- action arg (1 byte), 1 buffer pos (offset):
@@ -434,7 +434,11 @@ local function wputlabel(aprefix, imm, num)
     end
     wputxb(imm)
   else
-    waction(aprefix.."PC", imm, num)
+    if aprefix == "IMM_" and x64 then
+      waction("IMM_PC64", imm, num)
+    else
+      waction(aprefix.."PC", imm, num)
+    end
   end
 end
 

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index 478244b6d736186f744e041082444f3a796d04bf..0031dc6731b3885e8a156329154541bbba7103bf 100644 (file)
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -309,6 +309,9 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 
        if (ret != DASM_S_OK) {
                // TODO: dasm_encode() failed ???
+#if ZEND_DEBUG
+               ZEND_UNREACHABLE();
+#endif
                return NULL;
        }
 

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 12dccc00ff6935e79a337374748031dd9a7db491..abcea54aa521cb35189e789b02d3e1081258cf82 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -11377,10 +11377,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 {
        HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
 
-       if (sizeof(void*) == 8 && !IS_32BIT(dasm_end)) {
-               // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-               return 1;
-       }
        if (opline->op1_type == IS_CONST) {
                zval *zv = RT_CONSTANT(opline, opline->op1);
                zval *jump_zv;
@@ -11444,9 +11440,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |       cmp FCARG2a, jumptable->nNumUsed
                                        |       jae >3
                                        |.if X64
-                                       |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-                                       |       movsxd r0, dword [FCARG2a * 4 + >4]
-                                       |       jmp r0
+                                       |       jmp aword [FCARG2a * 8 + >4]
                                        |.else
                                        |       jmp aword [FCARG2a * 4 + >4]
                                        |.endif
@@ -11456,11 +11450,9 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        p = jumptable->arData;
                                        do {
                                                if (Z_TYPE(p->val) == IS_UNDEF) {
-                                                       |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
                                                        |       .aword =>b
                                                } else {
                                                        int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes];
-                                                       |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
                                                        |       .aword =>b
                                                }
                                                p++;
@@ -11474,7 +11466,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |       jz =>b
                                        |       LOAD_ADDR FCARG1a, jumptable
                                        |       sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
-                                       |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
                                        |       mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
                                        |.if X64
                                        |       cqo
@@ -11482,19 +11473,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                        |       cdq
                                        |.endif
                                        |       idiv FCARG1a
-                                       |.if X64
-                                       |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-                                       |       movsxd r0, dword [r0 + >4]
-                                       |       jmp r0
-                                       |.else
-                                       |       jmp dword [r0 + >4]
-                                       |.endif
+                                       |       jmp aword [r0 + >4]
                                        |3:
                                        |.cold_code
                                        |4:
                                        ZEND_HASH_FOREACH_VAL(jumptable, val) {
                                                b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
-                                               |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
                                                |       .aword =>b
                                        } ZEND_HASH_FOREACH_END();
                                        |.code
@@ -11527,7 +11511,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                |       jz =>b
                                |       LOAD_ADDR FCARG1a, jumptable
                                |       sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
-                               |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
                                |       mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
                                |.if X64
                                |       cqo
@@ -11535,24 +11518,13 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
                                |       cdq
                                |.endif
                                |       idiv FCARG1a
-                               |.if X64
-                               |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-                               |       movsxd r0, dword [r0 + >4]
-                               |       jmp r0
-                               |.else
-                               |       jmp dword [r0 + >4]
-                               |.endif
+                               |       jmp aword [r0 + >4]
                                |3:
                                |.cold_code
                                |4:
                                ZEND_HASH_FOREACH_VAL(jumptable, val) {
                                        b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
-                                       |       // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-                                       |.if X64
                                        |       .aword =>b
-                                       |.else
-                                       |       .aword =>b
-                                       |.endif
                                } ZEND_HASH_FOREACH_END();
                                |.code
                        }