From: Dmitry Stogov <dmitry@zend.com>
Date: Mon, 17 Aug 2020 10:50:03 +0000 (+0300)
Subject: Properly patch jmp tables
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8202b4ed09fe4160ac9265780816cf380a34b480;p=php

Properly patch jmp tables
---

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index e2e85f3431..90bed235d2 100644
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -2925,7 +2925,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
 					case ZEND_SWITCH_LONG:
 					case ZEND_SWITCH_STRING:
 					case ZEND_MATCH:
-						if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL)) {
+						if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
 							goto jit_failure;
 						}
 						goto done;
diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index 7dd8786fde..b40553b03d 100644
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -350,6 +350,7 @@ typedef struct _zend_jit_trace_info {
 	uint32_t                  stack_map_size;
 	uint32_t                  flags;         /* See ZEND_JIT_TRACE_... defines above */
 	uint32_t                  polymorphism;  /* Counter of polymorphic calls */
+	uint32_t                  jmp_table_size;/* number of jmp_table slots */
 	const zend_op            *opline;        /* first opline */
 	const void               *code_start;    /* address of native code */
 	zend_jit_trace_exit_info *exit_info;     /* info about side exits */
diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index 96fa37abb7..24b4e7cd81 100644
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -4163,7 +4163,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 					case ZEND_SWITCH_LONG:
 					case ZEND_SWITCH_STRING:
 					case ZEND_MATCH:
-						if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1)) {
+						if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
 							goto jit_failure;
 						}
 						goto done;
@@ -4827,6 +4827,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace
 		t->stack_map_size = 0;
 		t->flags = 0;
 		t->polymorphism = 0;
+		t->jmp_table_size = 0;
 		t->opline = trace_buffer[1].opline;
 		t->exit_info = exit_info;
 		t->stack_map = NULL;
@@ -5353,6 +5354,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
 			zend_jit_link_side_trace(
 				zend_jit_traces[trace_num].code_start,
 				zend_jit_traces[trace_num].code_size,
+				zend_jit_traces[trace_num].jmp_table_size,
 				exit_num,
 				handler);
 		}
@@ -5421,6 +5423,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
 		t->stack_map_size = 0;
 		t->flags = 0;
 		t->polymorphism = polymorphism;
+		t->jmp_table_size = 0;
 		t->opline = NULL;
 		t->exit_info = exit_info;
 		t->stack_map = NULL;
@@ -5464,6 +5467,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
 			zend_jit_link_side_trace(
 				zend_jit_traces[parent_num].code_start,
 				zend_jit_traces[parent_num].code_size,
+				zend_jit_traces[parent_num].jmp_table_size,
 				exit_num,
 				handler);
 
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 32bd3b4677..520a3facfa 100644
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -153,7 +153,7 @@ static size_t tsrm_tls_offset;
 |.globals zend_lb
 static void* dasm_labels[zend_lb_MAX];
 
-|.section code, cold_code
+|.section code, cold_code, jmp_table
 
 #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff)
 
@@ -3093,12 +3093,26 @@ mrm:
 typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t);
 typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t);
 
-static int zend_jit_patch(const void *code, size_t size, const void *from_addr, const void *to_addr)
+static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
 {
 	int ret = 0;
-	uint8_t *p = (uint8_t*)code;
-	uint8_t *end = p + size - 5;
+	uint8_t *p, *end;
 
+	if (jmp_table_size) {
+		const void **jmp_slot = (const void **)((char*)code + size);
+
+		size -= jmp_table_size * sizeof(void*);
+		do {
+			jmp_slot--;
+			if (*jmp_slot == from_addr) {
+				*jmp_slot = to_addr;
+				ret++;
+			}
+		} while (--jmp_table_size);
+	}
+
+	p = (uint8_t*)code;
+	end = p + size - 5;
 	while (p < end) {
 		if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) {
 			*(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6));
@@ -3115,9 +3129,9 @@ static int zend_jit_patch(const void *code, size_t size, const void *from_addr,
 	return ret;
 }
 
-static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t exit_num, const void *addr)
+static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
 {
-	return zend_jit_patch(code, size, zend_jit_trace_get_exit_addr(exit_num), addr);
+	return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
 }
 
 static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t)
@@ -11782,7 +11796,7 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze
 	return 1;
 }
 
-static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace)
+static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
 {
 	HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
 	const zend_op *next_opline = NULL;
@@ -11920,9 +11934,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 					|.else
 					|	jmp aword [FCARG2a * 4 + >4]
 					|.endif
-					|.cold_code
+					|.jmp_table
 					|.align aword
 					|4:
+					if (trace_info) {
+						trace_info->jmp_table_size += count;
+					}
 					p = jumptable->arData;
 					do {
 						if (Z_TYPE(p->val) == IS_UNDEF) {
@@ -11981,9 +11998,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 					|.else
 					|	jmp aword [r0 + >4]
 					|.endif
-					|.cold_code
+					|.jmp_table
 					|.align aword
 					|4:
+					if (trace_info) {
+						trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
+					}
 					ZEND_HASH_FOREACH_VAL(jumptable, val) {
 						target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
 						if (!next_opline) {
@@ -12063,9 +12083,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 				|.else
 				|	jmp aword [r0 + >4]
 				|.endif
-				|.cold_code
+				|.jmp_table
 				|.align aword
 				|4:
+				if (trace_info) {
+					trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
+				}
 				ZEND_HASH_FOREACH_VAL(jumptable, val) {
 					target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val));
 					if (!next_opline) {