From acfba51b70dc5c0fb8feb604e2c55371e7a096e3 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 29 May 2024 21:37:58 +0200 Subject: [PATCH 1/3] Jit: Fix "skip redundant flushes" for skipped instructions Normally when an instruction is skipped (for instance due to branch merging or BLR optimization), the registers that would have been flushed at the end of that instruction will instead be flushed at the end of the next instruction, which is maybe not perfect, but usually good enough. However, since the addition of the "skip redundant flushes" logic in fd511a689f, Dolphin has accidentally skipped flushing those registers, which creates unnecessary register pressure. This commit restores the old behavior. --- Source/Core/Core/PowerPC/Jit64/Jit.cpp | 18 +++++++++++---- Source/Core/Core/PowerPC/JitArm64/Jit.cpp | 27 +++++++++++++++++------ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp index 1248b1ab01..f31ad18913 100644 --- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp +++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp @@ -992,6 +992,11 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) IntializeSpeculativeConstants(); } + BitSet32 previous_gpr_in_use{}; + BitSet32 previous_fpr_in_use{}; + BitSet32 previous_gpr_will_be_written{}; + BitSet32 previous_fpr_will_be_written{}; + // Translate instructions for (u32 i = 0; i < code_block.m_num_instructions; i++) { @@ -1233,12 +1238,17 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) gpr.Discard(op.gprDiscardable); fpr.Discard(op.fprDiscardable); } - gpr.Flush(~(op.gprWillBeRead | op.gprWillBeWritten) & (op.regsIn | op.regsOut), + gpr.Flush(~(op.gprWillBeRead | op.gprWillBeWritten) & previous_gpr_in_use, RegCache::FlushMode::Full); - fpr.Flush(~(op.fprWillBeRead | op.fprWillBeWritten) & (op.fregsIn | op.GetFregsOut()), + fpr.Flush(~(op.fprWillBeRead | op.fprWillBeWritten) & previous_fpr_in_use, RegCache::FlushMode::Full); - gpr.Flush(~op.gprWillBeWritten & op.regsOut, RegCache::FlushMode::Undirty); - fpr.Flush(~op.fprWillBeWritten & op.GetFregsOut(), RegCache::FlushMode::Undirty); + gpr.Flush(~op.gprWillBeWritten & previous_gpr_will_be_written, RegCache::FlushMode::Undirty); + fpr.Flush(~op.fprWillBeWritten & previous_fpr_will_be_written, RegCache::FlushMode::Undirty); + + previous_gpr_in_use = op.gprWillBeRead | op.gprWillBeWritten; + previous_fpr_in_use = op.fprWillBeRead | op.fprWillBeWritten; + previous_gpr_will_be_written = op.gprWillBeWritten; + previous_fpr_will_be_written = op.fprWillBeWritten; if (opinfo->flags & FL_LOADSTORE) ++js.numLoadStoreInst; diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index aba973346d..4258fb999d 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -1207,6 +1207,13 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) IntializeSpeculativeConstants(); } + BitSet32 previous_gpr_in_use{}; + BitSet32 previous_fpr_in_use{}; + BitSet8 previous_cr_in_use{}; + BitSet32 previous_gpr_will_be_written{}; + BitSet32 previous_fpr_will_be_written{}; + BitSet8 previous_cr_will_be_written{}; + // Translate instructions for (u32 i = 0; i < code_block.m_num_instructions; i++) { @@ -1416,16 +1423,22 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) fpr.DiscardRegisters(op.fprDiscardable); gpr.DiscardCRRegisters(op.crDiscardable); } - gpr.FlushRegisters(~(op.gprWillBeRead | op.gprWillBeWritten) & (op.regsIn | op.regsOut), + gpr.FlushRegisters(~(op.gprWillBeRead | op.gprWillBeWritten) & previous_gpr_in_use, FlushMode::Full); - fpr.FlushRegisters(~(op.fprWillBeRead | op.fprWillBeWritten) & - (op.fregsIn | op.GetFregsOut()), + fpr.FlushRegisters(~(op.fprWillBeRead | op.fprWillBeWritten) & previous_fpr_in_use, FlushMode::Full); - gpr.FlushCRRegisters(~(op.crWillBeRead | op.crWillBeWritten) & (op.crIn | op.crOut), + gpr.FlushCRRegisters(~(op.crWillBeRead | op.crWillBeWritten) & previous_cr_in_use, FlushMode::Full); - gpr.FlushRegisters(~op.gprWillBeWritten & op.regsOut, FlushMode::Undirty); - fpr.FlushRegisters(~op.fprWillBeWritten & op.GetFregsOut(), FlushMode::Undirty); - gpr.FlushCRRegisters(~op.crWillBeWritten & op.crOut, FlushMode::Undirty); + gpr.FlushRegisters(~op.gprWillBeWritten & previous_gpr_will_be_written, FlushMode::Undirty); + fpr.FlushRegisters(~op.fprWillBeWritten & previous_fpr_will_be_written, FlushMode::Undirty); + gpr.FlushCRRegisters(~op.crWillBeWritten & previous_cr_will_be_written, FlushMode::Undirty); + + previous_gpr_in_use = op.gprWillBeRead | op.gprWillBeWritten; + previous_fpr_in_use = op.fprWillBeRead | op.fprWillBeWritten; + previous_cr_in_use = op.crWillBeRead | op.crWillBeWritten; + previous_gpr_will_be_written = op.gprWillBeWritten; + previous_fpr_will_be_written = op.fprWillBeWritten; + previous_cr_will_be_written = op.crWillBeWritten; if (opinfo->flags & FL_LOADSTORE) ++js.numLoadStoreInst; From a2f56ed0828c7f2f4063e8f117ed18a7e3911aec Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 29 May 2024 22:07:15 +0200 Subject: [PATCH 2/3] Jit: Further improve flushing for skipped instructions Now we no longer have to wait until we've compiled the next instruction after a skipped instruction before we can flush registers. This commit is easier to read using "Hide whitespace". --- Source/Core/Core/PowerPC/Jit64/Jit.cpp | 433 +++++++++++----------- Source/Core/Core/PowerPC/JitArm64/Jit.cpp | 369 +++++++++--------- 2 files changed, 407 insertions(+), 395 deletions(-) diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp index f31ad18913..e612be567f 100644 --- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp +++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp @@ -1016,247 +1016,256 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) js.isLastInstruction = true; } - if (i != 0) + if (js.skipInstructions != 0) { - // Gather pipe writes using a non-immediate address are discovered by profiling. - const u32 prev_address = m_code_buffer[i - 1].address; - bool gatherPipeIntCheck = js.fifoWriteAddresses.contains(prev_address); - - // Gather pipe writes using an immediate address are explicitly tracked. - if (jo.optimizeGatherPipe && - (js.fifoBytesSinceCheck >= GPFifo::GATHER_PIPE_SIZE || js.mustCheckFifo)) - { - js.fifoBytesSinceCheck = 0; - js.mustCheckFifo = false; - BitSet32 registersInUse = CallerSavedRegistersInUse(); - ABI_PushRegistersAndAdjustStack(registersInUse, 0); - ABI_CallFunctionP(GPFifo::FastCheckGatherPipe, &m_system.GetGPFifo()); - ABI_PopRegistersAndAdjustStack(registersInUse, 0); - gatherPipeIntCheck = true; - } - - // Gather pipe writes can generate an exception; add an exception check. - // TODO: This doesn't really match hardware; the CP interrupt is - // asynchronous. - if (gatherPipeIntCheck) - { - TEST(32, PPCSTATE(Exceptions), Imm32(EXCEPTION_EXTERNAL_INT)); - FixupBranch extException = J_CC(CC_NZ, Jump::Near); - - SwitchToFarCode(); - SetJumpTarget(extException); - TEST(32, PPCSTATE(msr), Imm32(0x0008000)); - FixupBranch noExtIntEnable = J_CC(CC_Z, Jump::Near); - MOV(64, R(RSCRATCH), ImmPtr(&m_system.GetProcessorInterface().m_interrupt_cause)); - TEST(32, MatR(RSCRATCH), - Imm32(ProcessorInterface::INT_CAUSE_CP | ProcessorInterface::INT_CAUSE_PE_TOKEN | - ProcessorInterface::INT_CAUSE_PE_FINISH)); - FixupBranch noCPInt = J_CC(CC_Z, Jump::Near); - - { - RCForkGuard gpr_guard = gpr.Fork(); - RCForkGuard fpr_guard = fpr.Fork(); - - gpr.Flush(); - fpr.Flush(); - - MOV(32, PPCSTATE(pc), Imm32(op.address)); - WriteExternalExceptionExit(); - } - SwitchToNearCode(); - SetJumpTarget(noCPInt); - SetJumpTarget(noExtIntEnable); - } - } - - if (HandleFunctionHooking(op.address)) - break; - - if (op.skip) - { - if (IsBranchWatchEnabled()) - { - // The only thing that currently sets op.skip is the BLR following optimization. - // If any non-branch instruction starts setting that too, this will need to be changed. - ASSERT(op.inst.hex == 0x4e800020); - WriteBranchWatch(op.address, op.branchTo, op.inst, CallerSavedRegistersInUse()); - } + js.skipInstructions--; } else { - auto& cpu = m_system.GetCPU(); - auto& power_pc = m_system.GetPowerPC(); - if (IsDebuggingEnabled() && power_pc.GetBreakPoints().IsAddressBreakPoint(op.address) && - !cpu.IsStepping()) + if (i != 0) { - gpr.Flush(); - fpr.Flush(); + // Gather pipe writes using a non-immediate address are discovered by profiling. + const u32 prev_address = m_code_buffer[i - 1].address; + bool gatherPipeIntCheck = js.fifoWriteAddresses.contains(prev_address); - MOV(32, PPCSTATE(pc), Imm32(op.address)); - ABI_PushRegistersAndAdjustStack({}, 0); - ABI_CallFunctionP(PowerPC::CheckAndHandleBreakPointsFromJIT, &power_pc); - ABI_PopRegistersAndAdjustStack({}, 0); - MOV(64, R(RSCRATCH), ImmPtr(cpu.GetStatePtr())); - CMP(32, MatR(RSCRATCH), Imm32(std::to_underlying(CPU::State::Running))); - FixupBranch noBreakpoint = J_CC(CC_E); - - Cleanup(); - MOV(32, PPCSTATE(npc), Imm32(op.address)); - SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount)); - JMP(asm_routines.dispatcher_exit); - - SetJumpTarget(noBreakpoint); - } - - if ((opinfo->flags & FL_USE_FPU) && !js.firstFPInstructionFound) - { - // This instruction uses FPU - needs to add FP exception bailout - TEST(32, PPCSTATE(msr), Imm32(1 << 13)); // Test FP enabled bit - FixupBranch b1 = J_CC(CC_Z, Jump::Near); - - SwitchToFarCode(); - SetJumpTarget(b1); + // Gather pipe writes using an immediate address are explicitly tracked. + if (jo.optimizeGatherPipe && + (js.fifoBytesSinceCheck >= GPFifo::GATHER_PIPE_SIZE || js.mustCheckFifo)) { - RCForkGuard gpr_guard = gpr.Fork(); - RCForkGuard fpr_guard = fpr.Fork(); - - gpr.Flush(); - fpr.Flush(); - - // If a FPU exception occurs, the exception handler will read - // from PC. Update PC with the latest value in case that happens. - MOV(32, PPCSTATE(pc), Imm32(op.address)); - OR(32, PPCSTATE(Exceptions), Imm32(EXCEPTION_FPU_UNAVAILABLE)); - WriteExceptionExit(); + js.fifoBytesSinceCheck = 0; + js.mustCheckFifo = false; + BitSet32 registersInUse = CallerSavedRegistersInUse(); + ABI_PushRegistersAndAdjustStack(registersInUse, 0); + ABI_CallFunctionP(GPFifo::FastCheckGatherPipe, &m_system.GetGPFifo()); + ABI_PopRegistersAndAdjustStack(registersInUse, 0); + gatherPipeIntCheck = true; } - SwitchToNearCode(); - js.firstFPInstructionFound = true; + // Gather pipe writes can generate an exception; add an exception check. + // TODO: This doesn't really match hardware; the CP interrupt is + // asynchronous. + if (gatherPipeIntCheck) + { + TEST(32, PPCSTATE(Exceptions), Imm32(EXCEPTION_EXTERNAL_INT)); + FixupBranch extException = J_CC(CC_NZ, Jump::Near); + + SwitchToFarCode(); + SetJumpTarget(extException); + TEST(32, PPCSTATE(msr), Imm32(0x0008000)); + FixupBranch noExtIntEnable = J_CC(CC_Z, Jump::Near); + MOV(64, R(RSCRATCH), ImmPtr(&m_system.GetProcessorInterface().m_interrupt_cause)); + TEST(32, MatR(RSCRATCH), + Imm32(ProcessorInterface::INT_CAUSE_CP | ProcessorInterface::INT_CAUSE_PE_TOKEN | + ProcessorInterface::INT_CAUSE_PE_FINISH)); + FixupBranch noCPInt = J_CC(CC_Z, Jump::Near); + + { + RCForkGuard gpr_guard = gpr.Fork(); + RCForkGuard fpr_guard = fpr.Fork(); + + gpr.Flush(); + fpr.Flush(); + + MOV(32, PPCSTATE(pc), Imm32(op.address)); + WriteExternalExceptionExit(); + } + SwitchToNearCode(); + SetJumpTarget(noCPInt); + SetJumpTarget(noExtIntEnable); + } } - if (bJITRegisterCacheOff) - { - gpr.Flush(); - fpr.Flush(); - m_constant_propagation.Clear(); + if (HandleFunctionHooking(op.address)) + break; - CompileInstruction(op); + if (op.skip) + { + if (IsBranchWatchEnabled()) + { + // The only thing that currently sets op.skip is the BLR following optimization. + // If any non-branch instruction starts setting that too, this will need to be changed. + ASSERT(op.inst.hex == 0x4e800020); + WriteBranchWatch(op.address, op.branchTo, op.inst, CallerSavedRegistersInUse()); + } } else { - const JitCommon::ConstantPropagationResult constant_propagation_result = - m_constant_propagation.EvaluateInstruction(op.inst, opinfo->flags); - - if (!constant_propagation_result.instruction_fully_executed) + auto& cpu = m_system.GetCPU(); + auto& power_pc = m_system.GetPowerPC(); + if (IsDebuggingEnabled() && power_pc.GetBreakPoints().IsAddressBreakPoint(op.address) && + !cpu.IsStepping()) { - if (!bJITRegisterCacheOff) + gpr.Flush(); + fpr.Flush(); + + MOV(32, PPCSTATE(pc), Imm32(op.address)); + ABI_PushRegistersAndAdjustStack({}, 0); + ABI_CallFunctionP(PowerPC::CheckAndHandleBreakPointsFromJIT, &power_pc); + ABI_PopRegistersAndAdjustStack({}, 0); + MOV(64, R(RSCRATCH), ImmPtr(cpu.GetStatePtr())); + CMP(32, MatR(RSCRATCH), Imm32(std::to_underlying(CPU::State::Running))); + FixupBranch noBreakpoint = J_CC(CC_E); + + Cleanup(); + MOV(32, PPCSTATE(npc), Imm32(op.address)); + SUB(32, PPCSTATE(downcount), Imm32(js.downcountAmount)); + JMP(asm_routines.dispatcher_exit); + + SetJumpTarget(noBreakpoint); + } + + if ((opinfo->flags & FL_USE_FPU) && !js.firstFPInstructionFound) + { + // This instruction uses FPU - needs to add FP exception bailout + TEST(32, PPCSTATE(msr), Imm32(1 << 13)); // Test FP enabled bit + FixupBranch b1 = J_CC(CC_Z, Jump::Near); + + SwitchToFarCode(); + SetJumpTarget(b1); { - // If we have an input register that is going to be used again, load it pre-emptively, - // even if the instruction doesn't strictly need it in a register, to avoid redundant - // loads later. Of course, don't do this if we're already out of registers. - // As a bit of a heuristic, make sure we have at least one register left over for the - // output, which needs to be bound in the actual instruction compilation. - // TODO: make this smarter in the case that we're actually register-starved, i.e. - // prioritize the more important registers. - gpr.PreloadRegisters(op.regsIn & (op.gprWillBeRead | op.gprWillBeWritten) & - ~op.gprDiscardable); - fpr.PreloadRegisters(op.fregsIn & op.fprInXmm & ~op.fprDiscardable); + RCForkGuard gpr_guard = gpr.Fork(); + RCForkGuard fpr_guard = fpr.Fork(); + + gpr.Flush(); + fpr.Flush(); + + // If a FPU exception occurs, the exception handler will read + // from PC. Update PC with the latest value in case that happens. + MOV(32, PPCSTATE(pc), Imm32(op.address)); + OR(32, PPCSTATE(Exceptions), Imm32(EXCEPTION_FPU_UNAVAILABLE)); + WriteExceptionExit(); } + SwitchToNearCode(); + + js.firstFPInstructionFound = true; + } + + if (bJITRegisterCacheOff) + { + gpr.Flush(); + fpr.Flush(); + m_constant_propagation.Clear(); CompileInstruction(op); } - - m_constant_propagation.Apply(constant_propagation_result); - - if (constant_propagation_result.gpr >= 0) - { - // Mark the GPR as dirty in the register cache - gpr.SetImmediate32(constant_propagation_result.gpr, - constant_propagation_result.gpr_value); - } - - if (constant_propagation_result.instruction_fully_executed) - { - if (constant_propagation_result.carry) - FinalizeCarry(*constant_propagation_result.carry); - - if (constant_propagation_result.overflow) - GenerateConstantOverflow(*constant_propagation_result.overflow); - - // FinalizeImmediateRC is called last, because it may trigger branch merging - if (constant_propagation_result.compute_rc) - FinalizeImmediateRC(constant_propagation_result.gpr_value); - } - } - - js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; - - if (jo.memcheck && (opinfo->flags & FL_LOADSTORE)) - { - // If we have a fastmem loadstore, we can omit the exception check and let fastmem handle - // it. - FixupBranch memException; - ASSERT_MSG(DYNA_REC, !(js.fastmemLoadStore && js.fixupExceptionHandler), - "Fastmem loadstores shouldn't have exception handler fixups (PC={:x})!", - op.address); - if (!js.fastmemLoadStore && !js.fixupExceptionHandler) - { - TEST(32, PPCSTATE(Exceptions), Imm32(EXCEPTION_DSI)); - memException = J_CC(CC_NZ, Jump::Near); - } - - SwitchToFarCode(); - if (!js.fastmemLoadStore) - { - m_exception_handler_at_loc[js.fastmemLoadStore] = nullptr; - SetJumpTarget(js.fixupExceptionHandler ? js.exceptionHandler : memException); - } else { - m_exception_handler_at_loc[js.fastmemLoadStore] = GetWritableCodePtr(); + const JitCommon::ConstantPropagationResult constant_propagation_result = + m_constant_propagation.EvaluateInstruction(op.inst, opinfo->flags); + + if (!constant_propagation_result.instruction_fully_executed) + { + if (!bJITRegisterCacheOff) + { + // If we have an input register that is going to be used again, load it pre-emptively, + // even if the instruction doesn't strictly need it in a register, to avoid redundant + // loads later. Of course, don't do this if we're already out of registers. + // As a bit of a heuristic, make sure we have at least one register left over for the + // output, which needs to be bound in the actual instruction compilation. + // TODO: make this smarter in the case that we're actually register-starved, i.e. + // prioritize the more important registers. + gpr.PreloadRegisters(op.regsIn & (op.gprWillBeRead | op.gprWillBeWritten) & + ~op.gprDiscardable); + fpr.PreloadRegisters(op.fregsIn & op.fprInXmm & ~op.fprDiscardable); + } + + CompileInstruction(op); + } + + m_constant_propagation.Apply(constant_propagation_result); + + if (constant_propagation_result.gpr >= 0) + { + // Mark the GPR as dirty in the register cache + gpr.SetImmediate32(constant_propagation_result.gpr, + constant_propagation_result.gpr_value); + } + + if (constant_propagation_result.instruction_fully_executed) + { + if (constant_propagation_result.carry) + FinalizeCarry(*constant_propagation_result.carry); + + if (constant_propagation_result.overflow) + GenerateConstantOverflow(*constant_propagation_result.overflow); + + // FinalizeImmediateRC is called last, because it may trigger branch merging + if (constant_propagation_result.compute_rc) + FinalizeImmediateRC(constant_propagation_result.gpr_value); + } } - RCForkGuard gpr_guard = gpr.Fork(); - RCForkGuard fpr_guard = fpr.Fork(); + js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; - gpr.Revert(); - fpr.Revert(); - gpr.Flush(); - fpr.Flush(); + if (jo.memcheck && (opinfo->flags & FL_LOADSTORE)) + { + // If we have a fastmem loadstore, we can omit the exception check and let fastmem handle + // it. + FixupBranch memException; + ASSERT_MSG(DYNA_REC, !(js.fastmemLoadStore && js.fixupExceptionHandler), + "Fastmem loadstores shouldn't have exception handler fixups (PC={:x})!", + op.address); + if (!js.fastmemLoadStore && !js.fixupExceptionHandler) + { + TEST(32, PPCSTATE(Exceptions), Imm32(EXCEPTION_DSI)); + memException = J_CC(CC_NZ, Jump::Near); + } - MOV(32, PPCSTATE(pc), Imm32(op.address)); - WriteExceptionExit(); - SwitchToNearCode(); + SwitchToFarCode(); + if (!js.fastmemLoadStore) + { + m_exception_handler_at_loc[js.fastmemLoadStore] = nullptr; + SetJumpTarget(js.fixupExceptionHandler ? js.exceptionHandler : memException); + } + else + { + m_exception_handler_at_loc[js.fastmemLoadStore] = GetWritableCodePtr(); + } + + RCForkGuard gpr_guard = gpr.Fork(); + RCForkGuard fpr_guard = fpr.Fork(); + + gpr.Revert(); + fpr.Revert(); + gpr.Flush(); + fpr.Flush(); + + MOV(32, PPCSTATE(pc), Imm32(op.address)); + WriteExceptionExit(); + SwitchToNearCode(); + } + + gpr.Commit(); + fpr.Commit(); + + if (opinfo->flags & FL_LOADSTORE) + ++js.numLoadStoreInst; + + if (opinfo->flags & FL_USE_FPU) + ++js.numFloatingPointInst; } - - gpr.Commit(); - fpr.Commit(); - - // If we have a register that will never be used again, discard or flush it. - if (!bJITRegisterCacheOff) - { - gpr.Discard(op.gprDiscardable); - fpr.Discard(op.fprDiscardable); - } - gpr.Flush(~(op.gprWillBeRead | op.gprWillBeWritten) & previous_gpr_in_use, - RegCache::FlushMode::Full); - fpr.Flush(~(op.fprWillBeRead | op.fprWillBeWritten) & previous_fpr_in_use, - RegCache::FlushMode::Full); - gpr.Flush(~op.gprWillBeWritten & previous_gpr_will_be_written, RegCache::FlushMode::Undirty); - fpr.Flush(~op.fprWillBeWritten & previous_fpr_will_be_written, RegCache::FlushMode::Undirty); - - previous_gpr_in_use = op.gprWillBeRead | op.gprWillBeWritten; - previous_fpr_in_use = op.fprWillBeRead | op.fprWillBeWritten; - previous_gpr_will_be_written = op.gprWillBeWritten; - previous_fpr_will_be_written = op.fprWillBeWritten; - - if (opinfo->flags & FL_LOADSTORE) - ++js.numLoadStoreInst; - - if (opinfo->flags & FL_USE_FPU) - ++js.numFloatingPointInst; } + js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; + + // If we have a register that will never be used again, discard or flush it. + if (!bJITRegisterCacheOff) + { + gpr.Discard(op.gprDiscardable); + fpr.Discard(op.fprDiscardable); + } + gpr.Flush(~(op.gprWillBeRead | op.gprWillBeWritten) & previous_gpr_in_use, + RegCache::FlushMode::Full); + fpr.Flush(~(op.fprWillBeRead | op.fprWillBeWritten) & previous_fpr_in_use, + RegCache::FlushMode::Full); + gpr.Flush(~op.gprWillBeWritten & previous_gpr_will_be_written, RegCache::FlushMode::Undirty); + fpr.Flush(~op.fprWillBeWritten & previous_fpr_will_be_written, RegCache::FlushMode::Undirty); + + previous_gpr_in_use = op.gprWillBeRead | op.gprWillBeWritten; + previous_fpr_in_use = op.fprWillBeRead | op.fprWillBeWritten; + previous_gpr_will_be_written = op.gprWillBeWritten; + previous_fpr_will_be_written = op.fprWillBeWritten; + #if defined(_DEBUG) || defined(DEBUGFAST) if (!gpr.SanityCheck() || !fpr.SanityCheck()) { @@ -1264,8 +1273,6 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) NOTICE_LOG_FMT(DYNA_REC, "Unflushed register: {}", ppc_inst); } #endif - i += js.skipInstructions; - js.skipInstructions = 0; } if (code_block.m_broken) diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index 4258fb999d..6be3e2b535 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -1236,219 +1236,224 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) fpr_used[op.fregOut] = true; fpr.UpdateLastUsed(fpr_used); - if (i != 0) + if (js.skipInstructions != 0) { - // Gather pipe writes using a non-immediate address are discovered by profiling. - const u32 prev_address = m_code_buffer[i - 1].address; - bool gatherPipeIntCheck = js.fifoWriteAddresses.contains(prev_address); - - if (jo.optimizeGatherPipe && - (js.fifoBytesSinceCheck >= GPFifo::GATHER_PIPE_SIZE || js.mustCheckFifo)) - { - js.fifoBytesSinceCheck = 0; - js.mustCheckFifo = false; - - gpr.Lock(ARM64Reg::W30); - BitSet32 regs_in_use = gpr.GetCallerSavedUsed(); - BitSet32 fprs_in_use = fpr.GetCallerSavedUsed(); - regs_in_use[DecodeReg(ARM64Reg::W30)] = false; - - ABI_PushRegisters(regs_in_use); - m_float_emit.ABI_PushRegisters(fprs_in_use, ARM64Reg::X30); - ABI_CallFunction(&GPFifo::FastCheckGatherPipe, &m_system.GetGPFifo()); - m_float_emit.ABI_PopRegisters(fprs_in_use, ARM64Reg::X30); - ABI_PopRegisters(regs_in_use); - - gpr.Unlock(ARM64Reg::W30); - gatherPipeIntCheck = true; - } - // Gather pipe writes can generate an exception; add an exception check. - // TODO: This doesn't really match hardware; the CP interrupt is - // asynchronous. - if (jo.optimizeGatherPipe && gatherPipeIntCheck) - { - auto WA = gpr.GetScopedReg(); - ARM64Reg XA = EncodeRegTo64(WA); - - LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(Exceptions)); - FixupBranch no_ext_exception = TBZ(WA, MathUtil::IntLog2(EXCEPTION_EXTERNAL_INT)); - FixupBranch exception = B(); - SwitchToFarCode(); - const u8* done_here = GetCodePtr(); - FixupBranch exit = B(); - SetJumpTarget(exception); - LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(msr)); - TBZ(WA, 15, done_here); // MSR.EE - LDR(IndexType::Unsigned, WA, XA, - MOVPage2R(XA, &m_system.GetProcessorInterface().m_interrupt_cause)); - constexpr u32 cause_mask = ProcessorInterface::INT_CAUSE_CP | - ProcessorInterface::INT_CAUSE_PE_TOKEN | - ProcessorInterface::INT_CAUSE_PE_FINISH; - TST(WA, LogicalImm(cause_mask, GPRSize::B32)); - B(CC_EQ, done_here); - - gpr.Flush(FlushMode::MaintainState, WA); - fpr.Flush(FlushMode::MaintainState, ARM64Reg::INVALID_REG); - WriteExceptionExit(js.compilerPC, true, true); - SwitchToNearCode(); - SetJumpTarget(no_ext_exception); - SetJumpTarget(exit); - } - } - - if (HandleFunctionHooking(op.address)) - break; - - if (op.skip) - { - if (IsBranchWatchEnabled()) - { - // The only thing that currently sets op.skip is the BLR following optimization. - // If any non-branch instruction starts setting that too, this will need to be changed. - ASSERT(op.inst.hex == 0x4e800020); - WriteBranchWatch(op.address, op.branchTo, op.inst, gpr.GetCallerSavedUsed(), - fpr.GetCallerSavedUsed()); - } + js.skipInstructions--; } else { - if (IsDebuggingEnabled() && !cpu.IsStepping() && - m_system.GetPowerPC().GetBreakPoints().IsAddressBreakPoint(op.address)) + if (i != 0) { - FlushCarry(); - gpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); - fpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); + // Gather pipe writes using a non-immediate address are discovered by profiling. + const u32 prev_address = m_code_buffer[i - 1].address; + bool gatherPipeIntCheck = js.fifoWriteAddresses.contains(prev_address); - static_assert(PPCSTATE_OFF(pc) <= 252); - static_assert(PPCSTATE_OFF(pc) + 4 == PPCSTATE_OFF(npc)); - - MOVI2R(DISPATCHER_PC, op.address); - STP(IndexType::Signed, DISPATCHER_PC, DISPATCHER_PC, PPC_REG, PPCSTATE_OFF(pc)); - ABI_CallFunction(&PowerPC::CheckAndHandleBreakPointsFromJIT, &m_system.GetPowerPC()); - - LDR(IndexType::Unsigned, ARM64Reg::W0, ARM64Reg::X0, - MOVPage2R(ARM64Reg::X0, cpu.GetStatePtr())); - static_assert(std::to_underlying(CPU::State::Running) == 0); - FixupBranch no_breakpoint = CBZ(ARM64Reg::W0); - - Cleanup(); - if (IsProfilingEnabled()) + if (jo.optimizeGatherPipe && + (js.fifoBytesSinceCheck >= GPFifo::GATHER_PIPE_SIZE || js.mustCheckFifo)) { - ABI_CallFunction(&JitBlock::ProfileData::EndProfiling, b->profile_data.get(), - js.downcountAmount); + js.fifoBytesSinceCheck = 0; + js.mustCheckFifo = false; + + gpr.Lock(ARM64Reg::W30); + BitSet32 regs_in_use = gpr.GetCallerSavedUsed(); + BitSet32 fprs_in_use = fpr.GetCallerSavedUsed(); + regs_in_use[DecodeReg(ARM64Reg::W30)] = false; + + ABI_PushRegisters(regs_in_use); + m_float_emit.ABI_PushRegisters(fprs_in_use, ARM64Reg::X30); + ABI_CallFunction(&GPFifo::FastCheckGatherPipe, &m_system.GetGPFifo()); + m_float_emit.ABI_PopRegisters(fprs_in_use, ARM64Reg::X30); + ABI_PopRegisters(regs_in_use); + + gpr.Unlock(ARM64Reg::W30); + gatherPipeIntCheck = true; } - DoDownCount(); - B(dispatcher_exit); - - SetJumpTarget(no_breakpoint); - } - - if ((opinfo->flags & FL_USE_FPU) && !js.firstFPInstructionFound) - { - FixupBranch b1; - // This instruction uses FPU - needs to add FP exception bailout + // Gather pipe writes can generate an exception; add an exception check. + // TODO: This doesn't really match hardware; the CP interrupt is + // asynchronous. + if (jo.optimizeGatherPipe && gatherPipeIntCheck) { auto WA = gpr.GetScopedReg(); - LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(msr)); - b1 = TBNZ(WA, 13); // Test FP enabled bit + ARM64Reg XA = EncodeRegTo64(WA); - FixupBranch far_addr = B(); + LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(Exceptions)); + FixupBranch no_ext_exception = TBZ(WA, MathUtil::IntLog2(EXCEPTION_EXTERNAL_INT)); + FixupBranch exception = B(); SwitchToFarCode(); - SetJumpTarget(far_addr); + const u8* done_here = GetCodePtr(); + FixupBranch exit = B(); + SetJumpTarget(exception); + LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(msr)); + TBZ(WA, 15, done_here); // MSR.EE + LDR(IndexType::Unsigned, WA, XA, + MOVPage2R(XA, &m_system.GetProcessorInterface().m_interrupt_cause)); + constexpr u32 cause_mask = ProcessorInterface::INT_CAUSE_CP | + ProcessorInterface::INT_CAUSE_PE_TOKEN | + ProcessorInterface::INT_CAUSE_PE_FINISH; + TST(WA, LogicalImm(cause_mask, GPRSize::B32)); + B(CC_EQ, done_here); gpr.Flush(FlushMode::MaintainState, WA); fpr.Flush(FlushMode::MaintainState, ARM64Reg::INVALID_REG); - - LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(Exceptions)); - ORR(WA, WA, LogicalImm(EXCEPTION_FPU_UNAVAILABLE, GPRSize::B32)); - STR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(Exceptions)); + WriteExceptionExit(js.compilerPC, true, true); + SwitchToNearCode(); + SetJumpTarget(no_ext_exception); + SetJumpTarget(exit); } - - WriteExceptionExit(js.compilerPC, false, true); - - SwitchToNearCode(); - - SetJumpTarget(b1); - - js.firstFPInstructionFound = true; } - if (bJITRegisterCacheOff) - { - FlushCarry(); - gpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); - fpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); - m_constant_propagation.Clear(); + if (HandleFunctionHooking(op.address)) + break; - CompileInstruction(op); + if (op.skip) + { + if (IsBranchWatchEnabled()) + { + // The only thing that currently sets op.skip is the BLR following optimization. + // If any non-branch instruction starts setting that too, this will need to be changed. + ASSERT(op.inst.hex == 0x4e800020); + WriteBranchWatch(op.address, op.branchTo, op.inst, gpr.GetCallerSavedUsed(), + fpr.GetCallerSavedUsed()); + } } else { - const JitCommon::ConstantPropagationResult constant_propagation_result = - m_constant_propagation.EvaluateInstruction(op.inst, opinfo->flags); + if (IsDebuggingEnabled() && !cpu.IsStepping() && + m_system.GetPowerPC().GetBreakPoints().IsAddressBreakPoint(op.address)) + { + FlushCarry(); + gpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); + fpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); + + static_assert(PPCSTATE_OFF(pc) <= 252); + static_assert(PPCSTATE_OFF(pc) + 4 == PPCSTATE_OFF(npc)); + + MOVI2R(DISPATCHER_PC, op.address); + STP(IndexType::Signed, DISPATCHER_PC, DISPATCHER_PC, PPC_REG, PPCSTATE_OFF(pc)); + ABI_CallFunction(&PowerPC::CheckAndHandleBreakPointsFromJIT, &m_system.GetPowerPC()); + + LDR(IndexType::Unsigned, ARM64Reg::W0, ARM64Reg::X0, + MOVPage2R(ARM64Reg::X0, cpu.GetStatePtr())); + static_assert(std::to_underlying(CPU::State::Running) == 0); + FixupBranch no_breakpoint = CBZ(ARM64Reg::W0); + + Cleanup(); + if (IsProfilingEnabled()) + { + ABI_CallFunction(&JitBlock::ProfileData::EndProfiling, b->profile_data.get(), + js.downcountAmount); + } + DoDownCount(); + B(dispatcher_exit); + + SetJumpTarget(no_breakpoint); + } + + if ((opinfo->flags & FL_USE_FPU) && !js.firstFPInstructionFound) + { + FixupBranch b1; + // This instruction uses FPU - needs to add FP exception bailout + { + auto WA = gpr.GetScopedReg(); + LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(msr)); + b1 = TBNZ(WA, 13); // Test FP enabled bit + + FixupBranch far_addr = B(); + SwitchToFarCode(); + SetJumpTarget(far_addr); + + gpr.Flush(FlushMode::MaintainState, WA); + fpr.Flush(FlushMode::MaintainState, ARM64Reg::INVALID_REG); + + LDR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(Exceptions)); + ORR(WA, WA, LogicalImm(EXCEPTION_FPU_UNAVAILABLE, GPRSize::B32)); + STR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(Exceptions)); + } + + WriteExceptionExit(js.compilerPC, false, true); + + SwitchToNearCode(); + + SetJumpTarget(b1); + + js.firstFPInstructionFound = true; + } + + if (bJITRegisterCacheOff) + { + FlushCarry(); + gpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); + fpr.Flush(FlushMode::Full, ARM64Reg::INVALID_REG); + m_constant_propagation.Clear(); - if (!constant_propagation_result.instruction_fully_executed) CompileInstruction(op); - - m_constant_propagation.Apply(constant_propagation_result); - - if (constant_propagation_result.gpr >= 0) + } + else { - // Mark the GPR as dirty in the register cache - gpr.SetImmediate(constant_propagation_result.gpr, constant_propagation_result.gpr_value); + const JitCommon::ConstantPropagationResult constant_propagation_result = + m_constant_propagation.EvaluateInstruction(op.inst, opinfo->flags); + + if (!constant_propagation_result.instruction_fully_executed) + CompileInstruction(op); + + m_constant_propagation.Apply(constant_propagation_result); + + if (constant_propagation_result.gpr >= 0) + { + // Mark the GPR as dirty in the register cache + gpr.SetImmediate(constant_propagation_result.gpr, + constant_propagation_result.gpr_value); + } + + if (constant_propagation_result.instruction_fully_executed) + { + if (constant_propagation_result.carry) + ComputeCarry(*constant_propagation_result.carry); + + if (constant_propagation_result.overflow) + GenerateConstantOverflow(*constant_propagation_result.overflow); + + if (constant_propagation_result.compute_rc) + ComputeRC0(constant_propagation_result.gpr_value); + } } - if (constant_propagation_result.instruction_fully_executed) - { - if (constant_propagation_result.carry) - ComputeCarry(*constant_propagation_result.carry); + if (opinfo->flags & FL_LOADSTORE) + ++js.numLoadStoreInst; - if (constant_propagation_result.overflow) - GenerateConstantOverflow(*constant_propagation_result.overflow); - - if (constant_propagation_result.compute_rc) - ComputeRC0(constant_propagation_result.gpr_value); - } + if (opinfo->flags & FL_USE_FPU) + ++js.numFloatingPointInst; } - - js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; - - if (!CanMergeNextInstructions(1) || js.op[1].opinfo->type != ::OpType::Integer) - FlushCarry(); - - // If we have a register that will never be used again, discard or flush it. - if (!bJITRegisterCacheOff) - { - gpr.DiscardRegisters(op.gprDiscardable); - fpr.DiscardRegisters(op.fprDiscardable); - gpr.DiscardCRRegisters(op.crDiscardable); - } - gpr.FlushRegisters(~(op.gprWillBeRead | op.gprWillBeWritten) & previous_gpr_in_use, - FlushMode::Full); - fpr.FlushRegisters(~(op.fprWillBeRead | op.fprWillBeWritten) & previous_fpr_in_use, - FlushMode::Full); - gpr.FlushCRRegisters(~(op.crWillBeRead | op.crWillBeWritten) & previous_cr_in_use, - FlushMode::Full); - gpr.FlushRegisters(~op.gprWillBeWritten & previous_gpr_will_be_written, FlushMode::Undirty); - fpr.FlushRegisters(~op.fprWillBeWritten & previous_fpr_will_be_written, FlushMode::Undirty); - gpr.FlushCRRegisters(~op.crWillBeWritten & previous_cr_will_be_written, FlushMode::Undirty); - - previous_gpr_in_use = op.gprWillBeRead | op.gprWillBeWritten; - previous_fpr_in_use = op.fprWillBeRead | op.fprWillBeWritten; - previous_cr_in_use = op.crWillBeRead | op.crWillBeWritten; - previous_gpr_will_be_written = op.gprWillBeWritten; - previous_fpr_will_be_written = op.fprWillBeWritten; - previous_cr_will_be_written = op.crWillBeWritten; - - if (opinfo->flags & FL_LOADSTORE) - ++js.numLoadStoreInst; - - if (opinfo->flags & FL_USE_FPU) - ++js.numFloatingPointInst; } - i += js.skipInstructions; - js.skipInstructions = 0; + js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; + + if (!CanMergeNextInstructions(1) || js.op[1].opinfo->type != ::OpType::Integer) + FlushCarry(); + + // If we have a register that will never be used again, discard or flush it. + if (!bJITRegisterCacheOff) + { + gpr.DiscardRegisters(op.gprDiscardable); + fpr.DiscardRegisters(op.fprDiscardable); + gpr.DiscardCRRegisters(op.crDiscardable); + } + gpr.FlushRegisters(~(op.gprWillBeRead | op.gprWillBeWritten) & previous_gpr_in_use, + FlushMode::Full); + fpr.FlushRegisters(~(op.fprWillBeRead | op.fprWillBeWritten) & previous_fpr_in_use, + FlushMode::Full); + gpr.FlushCRRegisters(~(op.crWillBeRead | op.crWillBeWritten) & previous_cr_in_use, + FlushMode::Full); + gpr.FlushRegisters(~op.gprWillBeWritten & previous_gpr_will_be_written, FlushMode::Undirty); + fpr.FlushRegisters(~op.fprWillBeWritten & previous_fpr_will_be_written, FlushMode::Undirty); + gpr.FlushCRRegisters(~op.crWillBeWritten & previous_cr_will_be_written, FlushMode::Undirty); + + previous_gpr_in_use = op.gprWillBeRead | op.gprWillBeWritten; + previous_fpr_in_use = op.fprWillBeRead | op.fprWillBeWritten; + previous_cr_in_use = op.crWillBeRead | op.crWillBeWritten; + previous_gpr_will_be_written = op.gprWillBeWritten; + previous_fpr_will_be_written = op.fprWillBeWritten; + previous_cr_will_be_written = op.crWillBeWritten; } if (code_block.m_broken) From 1b9d0ba5729ac20b4185fb31e97dc5e70d600740 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 29 May 2024 22:14:25 +0200 Subject: [PATCH 3/3] Jit: Don't skip incrementing numLoadStoreInst/numFloatingPointInst This was technically wrong, but it only affected merged dcbt+dcbst, and I doubt any games care about it. --- Source/Core/Core/PowerPC/Jit64/Jit.cpp | 12 ++++++------ Source/Core/Core/PowerPC/JitArm64/Jit.cpp | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp index e612be567f..a1a85719cf 100644 --- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp +++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp @@ -1237,15 +1237,15 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) gpr.Commit(); fpr.Commit(); - - if (opinfo->flags & FL_LOADSTORE) - ++js.numLoadStoreInst; - - if (opinfo->flags & FL_USE_FPU) - ++js.numFloatingPointInst; } } + if (opinfo->flags & FL_LOADSTORE) + ++js.numLoadStoreInst; + + if (opinfo->flags & FL_USE_FPU) + ++js.numFloatingPointInst; + js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; // If we have a register that will never be used again, discard or flush it. diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index 6be3e2b535..548a3257a8 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -1417,15 +1417,15 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) ComputeRC0(constant_propagation_result.gpr_value); } } - - if (opinfo->flags & FL_LOADSTORE) - ++js.numLoadStoreInst; - - if (opinfo->flags & FL_USE_FPU) - ++js.numFloatingPointInst; } } + if (opinfo->flags & FL_LOADSTORE) + ++js.numLoadStoreInst; + + if (opinfo->flags & FL_USE_FPU) + ++js.numFloatingPointInst; + js.fpr_is_store_safe = op.fprIsStoreSafeAfterInst; if (!CanMergeNextInstructions(1) || js.op[1].opinfo->type != ::OpType::Integer)