Skip to content

Commit 4f9ada9

Browse files
dstogovdstogov
committed
Switch to CPU stack spill slots
1 parent 1a1a49c commit 4f9ada9

File tree

6 files changed

+157
-57
lines changed

6 files changed

+157
-57
lines changed

‎ext/opcache/config.m4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ if test "$PHP_OPCACHE" != "no"; then
147147

148148
JIT_CFLAGS="-I@ext_builddir@/jit/ir -D${IR_TARGET} -DIR_PHP"
149149
if test "$ZEND_DEBUG" = "yes"; then
150-
JIT_CFLAGS="${JIT_CFLAGS} -DIR_DEBUG"
150+
JIT_CFLAGS="${JIT_CFLAGS} -DIR_DEBUG -DIR_DEBUG_MESSAGES"
151151
fi
152152
fi
153153

‎ext/opcache/config.w32‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ if (PHP_OPCACHE != "no") {
5353

5454
ADD_FLAG("CFLAGS_OPCACHE", "/I \"ext\\opcache\\jit\\ir\" /D "+ir_target+" /D IR_PHP");
5555
if (PHP_DEBUG == "yes") {
56-
ADD_FLAG("CFLAGS_OPCACHE", "/D IR_DEBUG");
56+
ADD_FLAG("CFLAGS_OPCACHE", "/D IR_DEBUG /D IR_DEBUG_MESSAGES");
5757
}
5858

5959
if (CHECK_HEADER_ADD_INCLUDE("capstone\\capstone.h", "CFLAGS_OPCACHE", PHP_OPCACHE+ ";" + PHP_PHP_BUILD + "\\include") &&

‎ext/opcache/jit/ir‎

‎ext/opcache/jit/zend_jit.h‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -169,15 +169,18 @@ ZEND_EXT_API void zend_jit_restart(void);
169169
#define ZREG_LOAD (1<<0)
170170
#define ZREG_STORE (1<<1)
171171
#define ZREG_LAST_USE (1<<2)
172+
172173
#define ZREG_PI (1<<3)
173174
#define ZREG_PHI (1<<4)
174175
#define ZREG_FORWARD (1<<5)
175-
#define ZREG_CONST (1<<6)
176-
#define ZREG_ZVAL_COPY (1<<7)
177176

178-
#define ZREG_TYPE_ONLY (1<<3)
179-
#define ZREG_ZVAL_ADDREF (1<<4)
180-
#define ZREG_THIS (1<<5)
177+
#define ZREG_SPILL_SLOT (1<<3)
178+
179+
#define ZREG_CONST (1<<4)
180+
#define ZREG_ZVAL_COPY (2<<4)
181+
#define ZREG_TYPE_ONLY (3<<4)
182+
#define ZREG_ZVAL_ADDREF (4<<4)
183+
#define ZREG_THIS (5<<4)
181184

182185
#define ZREG_NONE -1
183186

‎ext/opcache/jit/zend_jit_ir.c‎

Lines changed: 94 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@
2121

2222
#if defined(IR_TARGET_X86)
2323
# define IR_REG_SP 4 /* IR_REG_RSP */
24+
# define IR_REG_FP 5 /* IR_REG_RBP */
2425
# define ZREG_FP 6 /* IR_REG_RSI */
2526
# define ZREG_IP 7 /* IR_REG_RDI */
2627
# define ZREG_FIRST_FPR 8
2728
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
2829
#elif defined(IR_TARGET_X64)
2930
# define IR_REG_SP 4 /* IR_REG_RSP */
31+
# define IR_REG_FP 5 /* IR_REG_RBP */
3032
# define ZREG_FP 14 /* IR_REG_R14 */
3133
# define ZREG_IP 15 /* IR_REG_R15 */
3234
# define ZREG_FIRST_FPR 16
@@ -42,6 +44,7 @@
4244
# endif
4345
#elif defined(IR_TARGET_AARCH64)
4446
# define IR_REG_SP 31 /* IR_REG_RSP */
47+
# define IR_REG_FP 29 /* IR_REG_X29 */
4548
# define ZREG_FP 27 /* IR_REG_X27 */
4649
# define ZREG_IP 28 /* IR_REG_X28 */
4750
# define ZREG_FIRST_FPR 32
@@ -676,9 +679,17 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snaps
676679

677680
if (ref > 0) {
678681
if (reg != ZREG_NONE) {
679-
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
680682
if (reg & IR_REG_SPILL_LOAD) {
681-
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_LOAD;
683+
ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));
684+
/* spill slot on a CPU stack */
685+
t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;
686+
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
687+
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;
688+
} else if (reg & IR_REG_SPILL_SPECIAL) {
689+
/* spill slot on a VM stack */
690+
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
691+
} else {
692+
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
682693
}
683694
} else {
684695
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
@@ -1149,6 +1160,29 @@ static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref)
11491160
return ir_LOAD_D(ref);
11501161
}
11511162

1163+
static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val)
1164+
{
1165+
// if (jit->ctx.ir_base[val].op == IR_RLOAD) {
1166+
// /* Deoptimization */
1167+
// return 0;
1168+
// }
1169+
// if (jit->ctx.ir_base[val].op == IR_LOAD
1170+
// && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
1171+
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
1172+
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
1173+
// && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
1174+
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
1175+
// /* LOAD from the same location (the LOAD is pinned) */
1176+
// // TODO: should be anti-dependent with the following stores ???
1177+
// return 0;
1178+
// }
1179+
// if (jit->ssa->vars[var].var < jit->current_op_array->last_var) {
1180+
// /* IS_CV */
1181+
// return 0;
1182+
// }
1183+
return 1;
1184+
}
1185+
11521186
static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
11531187
{
11541188
int var;
@@ -1161,46 +1195,8 @@ static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
11611195
}
11621196
ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL);
11631197

1164-
/* Disable CSE for temporary variables */
1165-
/* TODO: This is a workarounf to fix ext/standard/tests/strings/htmlentities20.phpt failure with tracing JIT ??? */
1166-
if (0 && val > 0 && jit->ssa->vars[var].var >= jit->current_op_array->last_var) {
1167-
ir_insn *insn = &jit->ctx.ir_base[val];
1168-
ir_op op = insn->op;
1169-
1170-
if (op <= IR_LAST_FOLDABLE_OP && jit->ctx.prev_insn_chain[op]) {
1171-
if (jit->ctx.prev_insn_chain[op] == val) {
1172-
if (insn->prev_insn_offset) {
1173-
jit->ctx.prev_insn_chain[op] = val - (ir_ref)(uint32_t)insn->prev_insn_offset;
1174-
} else {
1175-
jit->ctx.prev_insn_chain[op] = IR_UNUSED;
1176-
}
1177-
} else {
1178-
ir_ref prev = jit->ctx.prev_insn_chain[op];
1179-
ir_ref tmp;
1180-
1181-
while (prev) {
1182-
insn = &jit->ctx.ir_base[prev];
1183-
if (!insn->prev_insn_offset) {
1184-
break;
1185-
}
1186-
tmp = prev - (ir_ref)(uint32_t)insn->prev_insn_offset;
1187-
if (tmp == val) {
1188-
ir_ref offset = jit->ctx.ir_base[tmp].prev_insn_offset;
1189-
1190-
if (!offset || prev - tmp + offset > 0xffff) {
1191-
insn->prev_insn_offset = 0;
1192-
} else {
1193-
insn->prev_insn_offset = prev - tmp + offset;
1194-
}
1195-
}
1196-
prev = tmp;
1197-
}
1198-
}
1199-
}
1200-
}
1201-
12021198
/* Negative "var" has special meaning for IR */
1203-
if (val > 0 && jit->ssa->vars[var].var < jit->current_op_array->last_var) {
1199+
if (val > 0 && !zend_jit_spilling_may_cause_conflict(jit, var, val)) {
12041200
val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val);
12051201
}
12061202
jit->ra[var].ref = val;
@@ -2623,7 +2619,7 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
26232619
jit->ctx.fixed_call_stack_size = 16;
26242620
#endif
26252621
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
2626-
jit->ctx.fixed_regset |= (1<<5); /* prevent %rbp (%r5) usage */
2622+
jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */
26272623
#endif
26282624
}
26292625
}
@@ -4148,6 +4144,43 @@ static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t
41484144
return 1;
41494145
}
41504146

4147+
static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type)
4148+
{
4149+
zend_jit_addr src;
4150+
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4151+
4152+
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4153+
src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
4154+
if (jit->ra && jit->ra[var].ref == IR_NULL) {
4155+
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4156+
} else {
4157+
jit_set_Z_LVAL(jit, dst, src);
4158+
if (set_type &&
4159+
(Z_REG(dst) != ZREG_FP ||
4160+
!JIT_G(current_frame) ||
4161+
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4162+
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4163+
}
4164+
}
4165+
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4166+
src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
4167+
if (jit->ra && jit->ra[var].ref == IR_NULL) {
4168+
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4169+
} else {
4170+
jit_set_Z_DVAL(jit, dst, src);
4171+
if (set_type &&
4172+
(Z_REG(dst) != ZREG_FP ||
4173+
!JIT_G(current_frame) ||
4174+
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4175+
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4176+
}
4177+
}
4178+
} else {
4179+
ZEND_UNREACHABLE();
4180+
}
4181+
return 1;
4182+
}
4183+
41514184
static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type)
41524185
{
41534186
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
@@ -6169,6 +6202,9 @@ static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
61696202
phi = ir_PHI_N(res_inputs->count, res_inputs->refs);
61706203
}
61716204
if (Z_MODE(var_addr) == IS_REG) {
6205+
if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) {
6206+
phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1);
6207+
}
61726208
zend_jit_def_reg(jit, var_addr, phi);
61736209
if (real_res_addr) {
61746210
if (var_def_info & MAY_BE_LONG) {
@@ -15547,6 +15583,20 @@ static void *zend_jit_finish(zend_jit_ctx *jit)
1554715583
}
1554815584
} else {
1554915585
/* Only for tracing JIT */
15586+
zend_jit_trace_info *t = jit->trace;
15587+
zend_jit_trace_stack *stack;
15588+
uint32_t i;
15589+
15590+
if (t) {
15591+
for (i = 0; i < t->stack_map_size; i++) {
15592+
stack = t->stack_map + i;
15593+
if (stack->flags & ZREG_SPILL_SLOT) {
15594+
stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
15595+
stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
15596+
}
15597+
}
15598+
}
15599+
1555015600
zend_jit_trace_add_code(entry, size);
1555115601

1555215602
#if ZEND_JIT_SUPPORT_CLDEMOTE
@@ -16023,12 +16073,12 @@ static int zend_jit_trace_start(zend_jit_ctx *jit,
1602316073

1602416074
if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) {
1602516075
/* op3 is used as a flag that the value is already stored in memory.
16026-
* In case the IR framework desides to spill the result of IR_LOAD,
16076+
* In case the IR framework decides to spill the result of IR_LOAD,
1602716077
* it doesn't have to store the value once again.
1602816078
*
1602916079
* See: insn->op3 check in ir_emit_rload()
1603016080
*/
16031-
ir_set_op(&jit->ctx, ref, 3, 1);
16081+
ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
1603216082
}
1603316083
}
1603416084
}

‎ext/opcache/jit/zend_jit_trace.c‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3862,15 +3862,18 @@ static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack
38623862
uint32_t i;
38633863

38643864
for (i = 0; i < stack_size; i++) {
3865-
#ifdef ZEND_JIT_IR
3866-
if (STACK_FLAGS(stack, i) & (ZREG_CONST|ZREG_ZVAL_COPY|ZREG_TYPE_ONLY|ZREG_ZVAL_ADDREF)) {
3867-
return 1;
3868-
}
3869-
#endif
3865+
#ifndef ZEND_JIT_IR
38703866
if (STACK_REG(stack, i) != ZREG_NONE
38713867
&& !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
38723868
return 1;
38733869
}
3870+
#else
3871+
if (STACK_FLAGS(stack, i) & ~(ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE)) {
3872+
return 1;
3873+
} else if (STACK_REG(stack, i) != ZREG_NONE) {
3874+
return 1;
3875+
}
3876+
#endif
38743877
}
38753878
return 0;
38763879
}
@@ -3999,6 +4002,29 @@ static int zend_jit_trace_deoptimization(
39994002
ZEND_ASSERT(reg != ZREG_NONE);
40004003
ZEND_ASSERT(check2 == -1);
40014004
check2 = i;
4005+
} else if (STACK_FLAGS(parent_stack, i) & ZREG_SPILL_SLOT) {
4006+
if (ssa && ssa->vars[i].no_val) {
4007+
/* pass */
4008+
} else {
4009+
uint8_t type = STACK_TYPE(parent_stack, i);
4010+
4011+
if (!zend_jit_store_spill_slot(jit, 1 << type, i, reg, STACK_REF(parent_stack, i),
4012+
STACK_MEM_TYPE(parent_stack, i) != type)) {
4013+
return 0;
4014+
}
4015+
if (stack) {
4016+
if (jit->ra && jit->ra[i].ref) {
4017+
SET_STACK_TYPE(stack, i, type, 0);
4018+
if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
4019+
SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
4020+
} else {
4021+
SET_STACK_REF(stack, i, jit->ra[i].ref);
4022+
}
4023+
} else {
4024+
SET_STACK_TYPE(stack, i, type, 1);
4025+
}
4026+
}
4027+
}
40024028
} else if (reg != ZREG_NONE) {
40034029
if (ssa && ssa->vars[i].no_val) {
40044030
/* pass */
@@ -8474,6 +8500,16 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
84748500
fprintf(stderr, "(zval_try_addref)");
84758501
} else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
84768502
fprintf(stderr, "zval_copy(%s)", zend_reg_name(STACK_REG(stack, j)));
8503+
} else if (STACK_FLAGS(stack, j) & ZREG_SPILL_SLOT) {
8504+
if (STACK_REG(stack, j) == ZREG_NONE) {
8505+
fprintf(stderr, "(spill=0x%x", STACK_REF(stack, j));
8506+
} else {
8507+
fprintf(stderr, "(spill=0x%x(%s)", STACK_REF(stack, j), zend_reg_name(STACK_REG(stack, j)));
8508+
}
8509+
if (STACK_FLAGS(stack, j) != 0) {
8510+
fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
8511+
}
8512+
fprintf(stderr, ")");
84778513
} else if (STACK_REG(stack, j) != ZREG_NONE) {
84788514
fprintf(stderr, "(%s", zend_reg_name(STACK_REG(stack, j)));
84798515
if (STACK_FLAGS(stack, j) != 0) {
@@ -9136,6 +9172,17 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf
91369172
} else {
91379173
ZVAL_COPY(EX_VAR_NUM(i), val);
91389174
}
9175+
} else if (STACK_FLAGS(stack, i) & ZREG_SPILL_SLOT) {
9176+
ZEND_ASSERT(STACK_REG(stack, i) != ZREG_NONE);
9177+
uintptr_t ptr = (uintptr_t)regs->gpr[STACK_REG(stack, i)] + STACK_REF(stack, i);
9178+
9179+
if (STACK_TYPE(stack, i) == IS_LONG) {
9180+
ZVAL_LONG(EX_VAR_NUM(i), *(zend_long*)ptr);
9181+
} else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
9182+
ZVAL_DOUBLE(EX_VAR_NUM(i), *(double*)ptr);
9183+
} else {
9184+
ZEND_UNREACHABLE();
9185+
}
91399186
} else if (STACK_REG(stack, i) != ZREG_NONE) {
91409187
if (STACK_TYPE(stack, i) == IS_LONG) {
91419188
zend_long val = regs->gpr[STACK_REG(stack, i)];

0 commit comments

Comments
 (0)