From 7f5eb0cdb1ee202062b4b1d6411e4589fbf0dad2 Mon Sep 17 00:00:00 2001 From: Fabian Stuckmann Date: Wed, 22 Jul 2026 11:30:05 +0200 Subject: [PATCH 1/5] [AIE][NFC] Extract findFreeNonOverlappingPhysReg into AIERegAllocationUtils Pull the free-register search out of AIEWawRegRewriter into a shared Utils/AIERegAllocationUtils helper so a second reallocation pass can reuse it. Switch the WAW round-robin from std::list to SmallVector along the way. No functional change to WAW register rewriting. --- llvm/lib/Target/AIE/AIEWawRegRewriter.cpp | 66 ++++++++----------- .../AIE/Utils/AIERegAllocationUtils.cpp | 42 ++++++++++++ .../Target/AIE/Utils/AIERegAllocationUtils.h | 38 +++++++++++ llvm/lib/Target/AIE/Utils/CMakeLists.txt | 1 + 4 files changed, 108 insertions(+), 39 deletions(-) create mode 100644 llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.cpp create mode 100644 llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.h diff --git a/llvm/lib/Target/AIE/AIEWawRegRewriter.cpp b/llvm/lib/Target/AIE/AIEWawRegRewriter.cpp index 24715c54d09b..e5ed10e8b452 100644 --- a/llvm/lib/Target/AIE/AIEWawRegRewriter.cpp +++ b/llvm/lib/Target/AIE/AIEWawRegRewriter.cpp @@ -21,6 +21,7 @@ #include "AIESlotStatistics.h" #include "Utils/AIELoopOptionOverrides.h" #include "Utils/AIELoopUtils.h" +#include "Utils/AIERegAllocationUtils.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" @@ -96,7 +97,7 @@ static cl::opt MinIIBias("aie-realloc-ii-bias", cl::Hidden, cl::init(0), namespace { // Defines the next register to use in reallocation. -using RoundRobin = std::list; +using RoundRobin = SmallVector; // Record the candidates and their original allocation. using OriginalAllocation = @@ -790,46 +791,33 @@ MCPhysReg AIEWawRegRewriter::getReplacementPhysReg(const Register VReg, const TargetRegisterClass *RC = MRI->getRegClass(VReg); const LiveInterval &LI = LIS->getInterval(VReg); - // Find the least-recently assigned register to assign to VReg. - for (auto It = LRURegisters.begin(); It != LRURegisters.end(); ++It) { - MCPhysReg PhysReg = *It; - - if (!RC->contains(PhysReg)) { - continue; - } - LiveRegMatrix::InterferenceKind IK = LRM->checkInterference(LI, PhysReg); - if (IK == LiveRegMatrix::IK_Free) { - // If the chosen physical register has already been used and the vreg to - // allocate is defined at a point where another vreg gets killed, prefer - // reusing the assignment of the killed reg. - if (std::optional KilledReg = - getKilledRegAtSingledDefPoint(VReg, *MRI); - KilledReg && WasUsedForReassignment(PhysReg)) { - MCRegister KilledPhysReg = getAssignedPhysReg(*KilledReg); - if (KilledPhysReg && LRM->checkInterference(LI, KilledPhysReg) == - LiveRegMatrix::IK_Free) { - - LLVM_DEBUG(dbgs() << " re-use killed physreg for assigning: " - << printReg(VReg, TRI) << " to " - << TRI->getName(KilledPhysReg) << '\n'); - PhysReg = KilledPhysReg; - It = llvm::find(LRURegisters, KilledPhysReg); - assert(It != LRURegisters.end()); - } - } - - // Move it to the end of the list. We return, so don't have to - // care about invalidation - moveRegAndAliasesBack(PhysReg, LRURegisters, TRI); - for (MCRegUnit RU : TRI->regunits(PhysReg)) - UsedUnits.set(RU); - return PhysReg; + MCPhysReg PhysReg = AIERegAllocationUtils::findFreeNonOverlappingPhysReg( + LI, *RC, LRURegisters, MCRegister::NoRegister, BitVector(), *TRI, *LRM); + if (!PhysReg) + return MCRegister::NoRegister; + + assert(llvm::is_contained(LRURegisters, PhysReg)); + + // Prefer a previously used killed register when it remains free. + if (std::optional KilledReg = + getKilledRegAtSingledDefPoint(VReg, *MRI); + KilledReg && WasUsedForReassignment(PhysReg)) { + MCRegister KilledPhysReg = getAssignedPhysReg(*KilledReg); + if (KilledPhysReg && + LRM->checkInterference(LI, KilledPhysReg) == LiveRegMatrix::IK_Free) { + + LLVM_DEBUG(dbgs() << " re-use killed physreg for assigning: " + << printReg(VReg, TRI) << " to " + << TRI->getName(KilledPhysReg) << '\n'); + PhysReg = KilledPhysReg; + assert(llvm::is_contained(LRURegisters, KilledPhysReg)); } - LLVM_DEBUG(dbgs() << " Cannot assign " << printReg(VReg, TRI) - << " to " << TRI->getName(PhysReg) - << " due to interference\n"); } - return MCRegister::NoRegister; + + moveRegAndAliasesBack(PhysReg, LRURegisters, TRI); + for (MCRegUnit RU : TRI->regunits(PhysReg)) + UsedUnits.set(RU); + return PhysReg; } bool AIEWawRegRewriter::isIdentityCopy(const MachineInstr &MI) const { diff --git a/llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.cpp b/llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.cpp new file mode 100644 index 000000000000..fa6c515fd324 --- /dev/null +++ b/llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.cpp @@ -0,0 +1,42 @@ +//===-- AIERegAllocationUtils.cpp - AIE register allocation utils --------===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +// +//===----------------------------------------------------------------------===// + +#include "AIERegAllocationUtils.h" +#include "llvm/CodeGen/LiveInterval.h" +#include "llvm/CodeGen/LiveRegMatrix.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" + +using namespace llvm; + +MCPhysReg AIERegAllocationUtils::findFreeNonOverlappingPhysReg( + const LiveInterval &LI, const TargetRegisterClass &RC, + ArrayRef AllocationOrder, MCPhysReg AvoidPhysReg, + const BitVector &ReservedRegUnits, const TargetRegisterInfo &TRI, + LiveRegMatrix &LRM) { + for (MCPhysReg PhysReg : AllocationOrder) { + if (!RC.contains(PhysReg)) + continue; + + if (AvoidPhysReg && TRI.regsOverlap(PhysReg, AvoidPhysReg)) + continue; + + const bool HasReservedUnit = + llvm::any_of(TRI.regunits(PhysReg), [&](MCRegUnit Unit) { + return !ReservedRegUnits.empty() && ReservedRegUnits.test(Unit); + }); + if (HasReservedUnit) + continue; + + if (LRM.checkInterference(LI, PhysReg) == LiveRegMatrix::IK_Free) + return PhysReg; + } + + return MCRegister::NoRegister; +} diff --git a/llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.h b/llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.h new file mode 100644 index 000000000000..2ca1f0be3640 --- /dev/null +++ b/llvm/lib/Target/AIE/Utils/AIERegAllocationUtils.h @@ -0,0 +1,38 @@ +//===-- AIERegAllocationUtils.h - AIE register allocation utils -*- C++ -*-===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AIE_UTILS_AIEREGALLOCATIONUTILS_H +#define LLVM_LIB_TARGET_AIE_UTILS_AIEREGALLOCATIONUTILS_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/MC/MCRegister.h" + +namespace llvm { + +class LiveInterval; +class LiveRegMatrix; +class TargetRegisterClass; +class TargetRegisterInfo; + +namespace AIERegAllocationUtils { + +MCPhysReg findFreeNonOverlappingPhysReg(const LiveInterval &LI, + const TargetRegisterClass &RC, + ArrayRef AllocationOrder, + MCPhysReg AvoidPhysReg, + const BitVector &ReservedRegUnits, + const TargetRegisterInfo &TRI, + LiveRegMatrix &LRM); + +} // namespace AIERegAllocationUtils +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/AIE/Utils/CMakeLists.txt b/llvm/lib/Target/AIE/Utils/CMakeLists.txt index 21d44fca1b18..b7b66ece0f93 100644 --- a/llvm/lib/Target/AIE/Utils/CMakeLists.txt +++ b/llvm/lib/Target/AIE/Utils/CMakeLists.txt @@ -10,6 +10,7 @@ add_llvm_component_library(LLVMAIEUtils AIELoopUtils.cpp AIELoopOptionOverrides.cpp AIEMachineBasicBlockUtils.cpp + AIERegAllocationUtils.cpp LINK_COMPONENTS Analysis From 0a54c3c6652f1a0d3a254e17e72661f667cdafab Mon Sep 17 00:00:00 2001 From: Fabian Stuckmann Date: Wed, 22 Jul 2026 11:30:06 +0200 Subject: [PATCH 2/5] [AIE][NFC] Share physical-copy materialization across subtargets Add a common physical-copy materialization helper on AIEBaseInstrInfo and implement it for AIE1/AIE2/AIE2P/AIE2PS so downstream passes can emit register copies without duplicating per-subtarget copy-splitting logic. --- llvm/lib/Target/AIE/AIE2InstrInfo.cpp | 59 +++--- llvm/lib/Target/AIE/AIE2InstrInfo.h | 5 + llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp | 63 +++++- llvm/lib/Target/AIE/AIEBaseInstrInfo.h | 59 +++++- llvm/lib/Target/AIE/aie1/AIE1InstrInfo.cpp | 69 ++++--- llvm/lib/Target/AIE/aie1/AIE1InstrInfo.h | 5 + llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp | 186 ++++++++++-------- llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h | 5 + .../lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp | 91 +++++---- llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h | 5 + 10 files changed, 354 insertions(+), 193 deletions(-) diff --git a/llvm/lib/Target/AIE/AIE2InstrInfo.cpp b/llvm/lib/Target/AIE/AIE2InstrInfo.cpp index 539e294775a7..e9f8ada7b5ca 100644 --- a/llvm/lib/Target/AIE/AIE2InstrInfo.cpp +++ b/llvm/lib/Target/AIE/AIE2InstrInfo.cpp @@ -488,85 +488,92 @@ void AIE2InstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - bool RenamableDest, bool RenamableSrc) const { - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); - const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + bool /* RenamableDest */, + bool /* RenamableSrc */) const { + const TargetRegisterInfo &TRI = + *MBB.getParent()->getRegInfo().getTargetRegisterInfo(); + CopyMaterializer M(*this, TRI, MBB, MBBI, DL); + if (!materializeCopy(M, DstReg, SrcReg, KillSrc)) + llvm_unreachable("unhandled case in copyPhysReg"); +} + +bool AIE2InstrInfo::materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const { + const TargetRegisterInfo &TRI = M.getRegisterInfo(); // TODO : add support for 128-bit mask register if (AIE2::mMvSclSrcRegClass.contains(SrcReg) && AIE2::mMvSclDstRegClass.contains(DstReg)) { const unsigned MOVSclOpcode = getScalarMovOpcode(DstReg, SrcReg); - BuildMI(MBB, MBBI, DL, get(MOVSclOpcode), DstReg) + M.buildInstr(get(MOVSclOpcode), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2::eLRegClass.contains(SrcReg)) && (AIE2::eLRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::MOV_mv_scl), - TRI.getSubReg(DstReg, AIE2::sub_l_even)) + M.buildInstr(get(AIE2::MOV_mv_scl), TRI.getSubReg(DstReg, AIE2::sub_l_even)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_l_even), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2::MOV_mv_scl), - TRI.getSubReg(DstReg, AIE2::sub_l_odd)) + M.buildInstr(get(AIE2::MOV_mv_scl), TRI.getSubReg(DstReg, AIE2::sub_l_odd)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_l_odd), getKillRegState(KillSrc)); } else if ((AIE2::eDRegClass.contains(SrcReg)) && (AIE2::eDRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2::eDSRegClass.contains(SrcReg)) && (AIE2::eDSRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2::VEC128RegClass.contains(SrcReg) || AIE2::VEC256RegClass.contains(SrcReg) || AIE2::ACC256RegClass.contains(SrcReg)) && (AIE2::VEC128RegClass.contains(DstReg) || AIE2::VEC256RegClass.contains(DstReg) || AIE2::ACC256RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_w), DstReg) + M.buildInstr(get(AIE2::VMOV_mv_w), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2::VEC512RegClass.contains(SrcReg) || AIE2::ACC512RegClass.contains(SrcReg)) && (AIE2::VEC512RegClass.contains(DstReg) || AIE2::ACC512RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), DstReg) + M.buildInstr(get(AIE2::VMOV_mv_x), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2::VEC1024RegClass.contains(SrcReg)) && (AIE2::VEC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), - TRI.getSubReg(DstReg, AIE2::sub_512_lo)) + M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), - TRI.getSubReg(DstReg, AIE2::sub_512_hi)) + M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_hi), getKillRegState(KillSrc)); } else if ((AIE2::ACC1024RegClass.contains(SrcReg)) && (AIE2::ACC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_cm), DstReg) + M.buildInstr(get(AIE2::VMOV_mv_cm), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2::VEC1024RegClass.contains(SrcReg) || AIE2::ACC1024RegClass.contains(SrcReg)) && (AIE2::VEC1024RegClass.contains(DstReg) || AIE2::ACC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), - TRI.getSubReg(DstReg, AIE2::sub_512_lo)) + M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), - TRI.getSubReg(DstReg, AIE2::sub_512_hi)) + M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_hi), getKillRegState(KillSrc)); } else if ((AIE2::SPARSEVEC640RegClass.contains(SrcReg)) && (AIE2::SPARSEVEC640RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), - TRI.getSubReg(DstReg, AIE2::sub_sparse_x)) + M.buildInstr(get(AIE2::VMOV_mv_x), + TRI.getSubReg(DstReg, AIE2::sub_sparse_x)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_sparse_x), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_w), - TRI.getSubReg(DstReg, AIE2::sub_sparse_q)) + M.buildInstr(get(AIE2::VMOV_mv_w), + TRI.getSubReg(DstReg, AIE2::sub_sparse_q)) .addReg(TRI.getSubReg(SrcReg, AIE2::sub_sparse_q), getKillRegState(KillSrc)); } else { - llvm_unreachable("unhandled case in copyPhysReg"); + return false; } + + return true; } // Some AIE instructions like Load/Stores take compound register classes diff --git a/llvm/lib/Target/AIE/AIE2InstrInfo.h b/llvm/lib/Target/AIE/AIE2InstrInfo.h index 90160c59016f..7c9ce85debef 100644 --- a/llvm/lib/Target/AIE/AIE2InstrInfo.h +++ b/llvm/lib/Target/AIE/AIE2InstrInfo.h @@ -160,6 +160,11 @@ class AIE2InstrInfo : public AIE2GenInstrInfo { bool KillSrc, bool RenamableDest = false, bool RenamableSrc = false) const override; +protected: + bool materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const override; + +public: void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp b/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp index 914f38e7c619..f290d75e49fe 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp @@ -811,13 +811,10 @@ static void collectSubRegs(MCRegister Reg, SmallSet &SubRegs, } } -void AIEBaseInstrInfo::copyThroughSubRegs(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - const DebugLoc &DL, MCRegister DstReg, - MCRegister SrcReg, +bool AIEBaseInstrInfo::copyThroughSubRegs(CopyMaterializer &M, + MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const { - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); - const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + const TargetRegisterInfo &TRI = M.getRegisterInfo(); SmallSet SrcSubRegs; collectSubRegs(SrcReg, SrcSubRegs, TRI); @@ -825,8 +822,60 @@ void AIEBaseInstrInfo::copyThroughSubRegs(MachineBasicBlock &MBB, for (MCRegister SrcSubReg : SrcSubRegs) { unsigned SubRegIdx = TRI.getSubRegIndex(SrcReg, SrcSubReg); MCRegister DstSubReg = TRI.getSubReg(DstReg, SubRegIdx); - copyPhysReg(MBB, MBBI, DL, DstSubReg, SrcSubReg, KillSrc); + if (!M.copy(DstSubReg, SrcSubReg, KillSrc)) + return false; } + return true; +} + +AIEBaseInstrInfo::CopyInstrBuilder & +AIEBaseInstrInfo::CopyInstrBuilder::addReg(Register Reg, unsigned Flags) { + if (MI) + MachineInstrBuilder(*MF, MI).addReg(Reg, Flags); + return *this; +} + +AIEBaseInstrInfo::CopyInstrBuilder & +AIEBaseInstrInfo::CopyInstrBuilder::addImm(int64_t Imm) { + if (MI) + MachineInstrBuilder(*MF, MI).addImm(Imm); + return *this; +} + +AIEBaseInstrInfo::CopyMaterializer::CopyMaterializer( + const AIEBaseInstrInfo &TII, const TargetRegisterInfo &TRI, + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) + : TII(TII), TRI(TRI), InsertPt(InsertionPoint{MBB, MBBI, DL}) {} + +AIEBaseInstrInfo::CopyMaterializer::CopyMaterializer( + const AIEBaseInstrInfo &TII, const TargetRegisterInfo &TRI) + : TII(TII), TRI(TRI) {} + +AIEBaseInstrInfo::CopyInstrBuilder +AIEBaseInstrInfo::CopyMaterializer::buildInstr(const MCInstrDesc &Desc, + Register DstReg) { + ++NumInstructions; + if (!InsertPt) + return CopyInstrBuilder(); + return CopyInstrBuilder( + InsertPt->MBB.getParent(), + BuildMI(InsertPt->MBB, InsertPt->MBBI, InsertPt->DL, Desc, DstReg) + .getInstr()); +} + +bool AIEBaseInstrInfo::CopyMaterializer::copy(MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) { + return TII.materializeCopy(*this, DstReg, SrcReg, KillSrc); +} + +std::optional +AIEBaseInstrInfo::getCopyCost(const TargetRegisterInfo &TRI, MCRegister DstReg, + MCRegister SrcReg) const { + CopyMaterializer M(*this, TRI); + if (!M.copy(DstReg, SrcReg, false)) + return std::nullopt; + return M.getNumInstructions(); } static bool isPreRA(const MachineFunction &MF) { diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h index 045e15ac9fb3..539b63666766 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h @@ -1023,6 +1023,54 @@ struct AIEBaseInstrInfo : public TargetInstrInfo { } public: + class CopyInstrBuilder { + public: + CopyInstrBuilder(MachineFunction *MF = nullptr, MachineInstr *MI = nullptr) + : MF(MF), MI(MI) {} + + CopyInstrBuilder &addReg(Register Reg, unsigned Flags = 0); + CopyInstrBuilder &addImm(int64_t Imm); + + private: + MachineFunction *MF; + MachineInstr *MI; + }; + + /// Materializes physical-register copies either into a machine basic block + /// or into an instruction count. Target copy lowering uses this common + /// interface so both operations exercise the same implementation. + class CopyMaterializer { + public: + CopyMaterializer(const AIEBaseInstrInfo &TII, const TargetRegisterInfo &TRI, + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL); + CopyMaterializer(const AIEBaseInstrInfo &TII, + const TargetRegisterInfo &TRI); + + CopyInstrBuilder buildInstr(const MCInstrDesc &Desc, Register DstReg); + bool copy(MCRegister DstReg, MCRegister SrcReg, bool KillSrc); + unsigned getNumInstructions() const { return NumInstructions; } + const TargetRegisterInfo &getRegisterInfo() const { return TRI; } + + private: + struct InsertionPoint { + MachineBasicBlock &MBB; + MachineBasicBlock::iterator MBBI; + const DebugLoc &DL; + }; + + const AIEBaseInstrInfo &TII; + const TargetRegisterInfo &TRI; + std::optional InsertPt; + unsigned NumInstructions = 0; + }; + + /// Return the number of instructions materialized for an exact physical + /// register copy, or std::nullopt if the copy is unsupported. + std::optional getCopyCost(const TargetRegisterInfo &TRI, + MCRegister DstReg, + MCRegister SrcReg) const; + /// Expand a spill pseudo-instruction into actual target instructions. This /// will essentially split the register being handled into its sub-registers, /// until there is an actual instruction that can handle them. @@ -1077,6 +1125,11 @@ struct AIEBaseInstrInfo : public TargetInstrInfo { }; protected: + virtual bool materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const { + return false; + }; + struct AIERegOffsetSpillInstrInfo { /// Opcode for spill using register offset. unsigned SpillOpCode; @@ -1099,10 +1152,8 @@ struct AIEBaseInstrInfo : public TargetInstrInfo { } // Copy SrcReg to DstReg through their sub-registers. - void copyThroughSubRegs(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, const DebugLoc &DL, - MCRegister DstReg, MCRegister SrcReg, - bool KillSrc) const; + bool copyThroughSubRegs(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const; #if 0 // TODO. I guess this should wait for Davy's PR to land diff --git a/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.cpp b/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.cpp index 3636c14a5bd4..81f6305ea7a1 100644 --- a/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.cpp +++ b/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.cpp @@ -4,7 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// (c) Copyright 2023-2025 Advanced Micro Devices, Inc. or its affiliates +// (c) Copyright 2023-2026 Advanced Micro Devices, Inc. or its affiliates // //===----------------------------------------------------------------------===// // @@ -101,94 +101,103 @@ void AIEInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - bool RenamableDest, bool RenamableSrc) const { - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); - const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + bool /* RenamableDest */, + bool /* RenamableSrc */) const { + const TargetRegisterInfo &TRI = + *MBB.getParent()->getRegInfo().getTargetRegisterInfo(); + CopyMaterializer M(*this, TRI, MBB, MBBI, DL); + if (!materializeCopy(M, DstReg, SrcReg, KillSrc)) + assert(false && "unhandled case in copyPhysReg"); +} + +bool AIEInstrInfo::materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const { + const TargetRegisterInfo &TRI = M.getRegisterInfo(); // Generate a mov instruction if ((AIE::mMvSclRegClass.contains(SrcReg) && AIE::mMvSclRegClass.contains(DstReg)) || (AIE::PTRRegClass.contains(SrcReg) && AIE::PTRRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE::MOV), DstReg) + M.buildInstr(get(AIE::MOV), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::SPRRegClass.contains(SrcReg) && AIE::GPRRegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_SPECIAL2R), DstReg) + M.buildInstr(get(AIE::MV_SPECIAL2R), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::GPRRegClass.contains(SrcReg) && AIE::SPRRegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_R2SPECIAL), DstReg) + M.buildInstr(get(AIE::MV_R2SPECIAL), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::SPRRegClass.contains(SrcReg) && AIE::PTRRegClass.contains(DstReg)) { Register GPRReg = AIE::r15; - BuildMI(MBB, MBBI, DL, get(AIE::MV_SPECIAL2R), GPRReg) + M.buildInstr(get(AIE::MV_SPECIAL2R), GPRReg) .addReg(SrcReg, getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE::MOV), DstReg) + M.buildInstr(get(AIE::MOV), DstReg) .addReg(GPRReg, getKillRegState(KillSrc)); } else if (AIE::PTRRegClass.contains(SrcReg) && AIE::SPRRegClass.contains(DstReg)) { Register GPRReg = AIE::r15; - BuildMI(MBB, MBBI, DL, get(AIE::MOV), GPRReg) + M.buildInstr(get(AIE::MOV), GPRReg) .addReg(SrcReg, getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE::MV_R2SPECIAL), DstReg) + M.buildInstr(get(AIE::MV_R2SPECIAL), DstReg) .addReg(GPRReg, getKillRegState(KillSrc)); } else if (AIE::SPRRegClass.contains(SrcReg) && AIE::SPRRegClass.contains(DstReg)) { Register GPRReg = AIE::r15; - BuildMI(MBB, MBBI, DL, get(AIE::MV_SPECIAL2R), GPRReg) + M.buildInstr(get(AIE::MV_SPECIAL2R), GPRReg) .addReg(SrcReg, getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE::MV_R2SPECIAL), DstReg) + M.buildInstr(get(AIE::MV_R2SPECIAL), DstReg) .addReg(GPRReg, getKillRegState(KillSrc)); } else if (AIE::VEC128RegClass.contains(SrcReg) && AIE::VEC128RegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_V), DstReg) + M.buildInstr(get(AIE::MV_V), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::VEC256RegClass.contains(SrcReg) && AIE::VEC256RegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_W), DstReg) + M.buildInstr(get(AIE::MV_W), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::VEC512RegClass.contains(SrcReg) && AIE::VEC512RegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_X), DstReg) + M.buildInstr(get(AIE::MV_X), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::VEC1024RegClass.contains(SrcReg) && AIE::VEC1024RegClass.contains(DstReg)) { // The 1024-bit registers share their high-order bits. if (SrcReg == AIE::ya && DstReg == AIE::yd) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_X), AIE::xd) + M.buildInstr(get(AIE::MV_X), AIE::xd) .addReg(AIE::xa, getKillRegState(KillSrc)); } else if (SrcReg == AIE::yd && DstReg == AIE::ya) { - BuildMI(MBB, MBBI, DL, get(AIE::MV_X), AIE::xa) + M.buildInstr(get(AIE::MV_X), AIE::xa) .addReg(AIE::xd, getKillRegState(KillSrc)); } } else if (AIE::GPRRegClass.contains(SrcReg) && AIE::VEC256RegClass.contains(DstReg)) { // only for f32 type. - BuildMI(MBB, MBBI, DL, get(AIE::S2V_SHIFTW0_R32), DstReg) + M.buildInstr(get(AIE::S2V_SHIFTW0_R32), DstReg) .addReg(DstReg, RegState::Undef) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE::mMv0Cg20RegClass.contains(SrcReg) && AIE::VEC256RegClass.contains(DstReg)) { // only for f32 type. Bounce through GPR Register GPRReg = AIE::r15; - BuildMI(MBB, MBBI, DL, get(AIE::MOV), GPRReg) + M.buildInstr(get(AIE::MOV), GPRReg) .addReg(SrcReg, getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE::S2V_SHIFTW0_R32), DstReg) + M.buildInstr(get(AIE::S2V_SHIFTW0_R32), DstReg) .addReg(DstReg, RegState::Undef) .addReg(GPRReg, getKillRegState(KillSrc)); } else if (AIE::VEC256RegClass.contains(SrcReg) && AIE::GPRRegClass.contains(DstReg)) { // only for f32 type. - BuildMI(MBB, MBBI, DL, get(AIE::S2V_EXT_R32), DstReg) + M.buildInstr(get(AIE::S2V_EXT_R32), DstReg) .addReg(TRI.getSubReg(SrcReg, AIE::sub_128bit_lo), getKillRegState(true)) .addImm(0); } else if (AIE::VEC256RegClass.contains(SrcReg) && AIE::PTRRegClass.contains(DstReg)) { // only for f32 type. - BuildMI(MBB, MBBI, DL, get(AIE::S2V_EXT_P32), DstReg) + M.buildInstr(get(AIE::S2V_EXT_P32), DstReg) .addReg(TRI.getSubReg(SrcReg, AIE::sub_128bit_lo), getKillRegState(true)) .addImm(0); @@ -196,27 +205,29 @@ void AIEInstrInfo::copyPhysReg(MachineBasicBlock &MBB, AIE::mMv0Cg20RegClass.contains(DstReg)) { // only for f32 type. Bounce through GPR Register GPRReg = AIE::r15; - BuildMI(MBB, MBBI, DL, get(AIE::S2V_EXT_R32), GPRReg) + M.buildInstr(get(AIE::S2V_EXT_R32), GPRReg) .addReg(TRI.getSubReg(SrcReg, AIE::sub_128bit_lo), getKillRegState(true)) .addImm(0); - BuildMI(MBB, MBBI, DL, get(AIE::MOV), DstReg) + M.buildInstr(get(AIE::MOV), DstReg) .addReg(GPRReg, getKillRegState(KillSrc)); } else if (AIE::mCRegClass.contains(SrcReg) && AIE::mCRegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::MOV), TRI.getSubReg(DstReg, AIE::sub_32_lo)) + M.buildInstr(get(AIE::MOV), TRI.getSubReg(DstReg, AIE::sub_32_lo)) .addReg(TRI.getSubReg(SrcReg, AIE::sub_32_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE::MOV), TRI.getSubReg(DstReg, AIE::sub_32_hi)) + M.buildInstr(get(AIE::MOV), TRI.getSubReg(DstReg, AIE::sub_32_hi)) .addReg(TRI.getSubReg(SrcReg, AIE::sub_32_hi), getKillRegState(KillSrc)); } else if (AIE::ACC384RegClass.contains(SrcReg) && AIE::ACC384RegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE::ACCUMULATOR_MOVE), DstReg) + M.buildInstr(get(AIE::ACCUMULATOR_MOVE), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else { - assert(false && "unhandled case in copyPhysReg"); + return false; } + + return true; } // Sometimes the instruction set encodes smaller registers (256-bit or 512-bit) diff --git a/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.h b/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.h index 843ce0898868..3f8ed557832b 100644 --- a/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.h +++ b/llvm/lib/Target/AIE/aie1/AIE1InstrInfo.h @@ -46,6 +46,11 @@ class AIEInstrInfo : public AIEGenInstrInfo { bool KillSrc, bool RenamableDest = false, bool RenamableSrc = false) const override; +protected: + bool materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const override; + +public: bool expandPostRAPseudo(MachineInstr &MI) const override; void storeRegToStackSlot(MachineBasicBlock &MBB, diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp index 330f14560986..01286582859c 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp @@ -627,47 +627,58 @@ void AIE2PInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - bool RenamableDest, bool RenamableSrc) const { - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); - const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + bool /* RenamableDest */, + bool /* RenamableSrc */) const { + const TargetRegisterInfo &TRI = + *MBB.getParent()->getRegInfo().getTargetRegisterInfo(); + CopyMaterializer M(*this, TRI, MBB, MBBI, DL); + if (!materializeCopy(M, DstReg, SrcReg, KillSrc)) + llvm_unreachable("unhandled case in copyPhysReg"); +} + +bool AIE2PInstrInfo::materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const { + const TargetRegisterInfo &TRI = M.getRegisterInfo(); if (AIE2P::mMvSclSrcRegClass.contains(SrcReg) && AIE2P::mMvSclDstRegClass.contains(DstReg)) { // Build MultiSlotPseudo in preference const unsigned MOVSclOpcode = getScalarMovOpcode(DstReg, SrcReg); - BuildMI(MBB, MBBI, DL, get(MOVSclOpcode), DstReg) + M.buildInstr(get(MOVSclOpcode), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::eLRegClass.contains(SrcReg)) && (AIE2P::eLRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_scl), - TRI.getSubReg(DstReg, AIE2P::sub_l_even)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_scl), + TRI.getSubReg(DstReg, AIE2P::sub_l_even)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_l_even), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_scl), - TRI.getSubReg(DstReg, AIE2P::sub_l_odd)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_scl), + TRI.getSubReg(DstReg, AIE2P::sub_l_odd)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_l_odd), getKillRegState(KillSrc)); } else if ((AIE2P::eDRegClass.contains(SrcReg)) && (AIE2P::eDRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2P::eDSRegClass.contains(SrcReg)) && (AIE2P::eDSRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2P::mQQsaRegClass.contains(SrcReg)) && (AIE2P::mQQsaRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_q), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_q), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mWmRegClass.contains(SrcReg)) && (AIE2P::mQQsmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_w_to_q), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_w_to_q), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mQQsmRegClass.contains(SrcReg)) && (AIE2P::mWmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_q_to_w), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_q_to_w), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mWmRegClass.contains(SrcReg)) && (AIE2P::mWmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_w), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_w), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::VEC512RegClass.contains(SrcReg) || AIE2P::ACC512RegClass.contains(SrcReg) || @@ -675,206 +686,209 @@ void AIE2PInstrInfo::copyPhysReg(MachineBasicBlock &MBB, (AIE2P::VEC512RegClass.contains(DstReg) || AIE2P::ACC512RegClass.contains(DstReg) || AIE2P::FIFO512RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::VEC1024RegClass.contains(SrcReg)) && (AIE2P::VEC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_lo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_hi)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_hi), getKillRegState(KillSrc)); } else if ((AIE2P::ACC1024RegClass.contains(SrcReg)) && (AIE2P::ACC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_cm), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_cm), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (AIE2P::VEC1024RegClass.contains(SrcReg) && AIE2P::ACC1024RegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_acc_lo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_acc_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_acc_hi)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_acc_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_hi), getKillRegState(KillSrc)); } else if (AIE2P::ACC1024RegClass.contains(SrcReg) && AIE2P::VEC1024RegClass.contains(DstReg)) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_lo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_acc_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_hi)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_acc_hi), getKillRegState(KillSrc)); } else if ((AIE2P::ACC2048RegClass.contains(SrcReg)) && (AIE2P::ACC2048RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_D), DstReg) + M.buildInstr(get(AIE2P::VMOV_D), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mEXmRegClass.contains(SrcReg)) && (AIE2P::mEXmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_ex), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_ex), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::eEYRegClass.contains(SrcReg)) && (AIE2P::eEYRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_ex), - TRI.getSubReg(DstReg, AIE2P::sub_bfp576_lo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_ex), + TRI.getSubReg(DstReg, AIE2P::sub_bfp576_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_bfp576_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_ex), - TRI.getSubReg(DstReg, AIE2P::sub_bfp576_hi)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_ex), + TRI.getSubReg(DstReg, AIE2P::sub_bfp576_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_bfp576_hi), getKillRegState(KillSrc)); } else if ((AIE2P::SPARSEVEC640RegClass.contains(SrcReg)) && (AIE2P::SPARSEVEC640RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_qx), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_qx), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::SPARSEVEC1280RegClass.contains(SrcReg)) && (AIE2P::SPARSEVEC1280RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_qx), - TRI.getSubReg(DstReg, AIE2P::sub_even_vecmask_640)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_qx), + TRI.getSubReg(DstReg, AIE2P::sub_even_vecmask_640)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_even_vecmask_640), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_qx), - TRI.getSubReg(DstReg, AIE2P::sub_odd_vecmask_640)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_qx), + TRI.getSubReg(DstReg, AIE2P::sub_odd_vecmask_640)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_odd_vecmask_640), getKillRegState(KillSrc)); } else if ((AIE2P::mQEXsmRegClass.contains(SrcReg)) && (AIE2P::mQEXsmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_qex), DstReg) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_qex), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::eQEYsRegClass.contains(SrcReg)) && (AIE2P::eQEYsRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_qex), - TRI.getSubReg(DstReg, AIE2P::sub_even_vecmaskexp_704)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_qex), + TRI.getSubReg(DstReg, AIE2P::sub_even_vecmaskexp_704)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_even_vecmaskexp_704), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_qex), - TRI.getSubReg(DstReg, AIE2P::sub_odd_vecmaskexp_704)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_qex), + TRI.getSubReg(DstReg, AIE2P::sub_odd_vecmaskexp_704)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_odd_vecmaskexp_704), getKillRegState(KillSrc)); } else if ((AIE2P::FIFO1024RegClass.contains(SrcReg)) && (AIE2P::FIFO1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_lo_fifo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_lo_fifo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_lo_fifo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_hi_fifo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_hi_fifo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_hi_fifo), getKillRegState(KillSrc)); } else if ((AIE2P::VEC1024RegClass.contains(SrcReg)) && (AIE2P::FIFO1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_lo_fifo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_lo_fifo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_hi_fifo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_hi_fifo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_hi), getKillRegState(KillSrc)); } else if ((AIE2P::FIFO1024RegClass.contains(SrcReg)) && (AIE2P::VEC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_lo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_lo_fifo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_hi)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_hi_fifo), getKillRegState(KillSrc)); } else if ((AIE2P::ACC1024RegClass.contains(SrcReg)) && (AIE2P::FIFO1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_lo_fifo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_lo_fifo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_acc_lo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_hi_fifo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_hi_fifo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_512_acc_hi), getKillRegState(KillSrc)); } else if ((AIE2P::FIFO1024RegClass.contains(SrcReg)) && (AIE2P::ACC1024RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_acc_lo)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_acc_lo)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_lo_fifo), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::VMOV_alu_mv_mv_x), - TRI.getSubReg(DstReg, AIE2P::sub_512_acc_hi)) + M.buildInstr(get(AIE2P::VMOV_alu_mv_mv_x), + TRI.getSubReg(DstReg, AIE2P::sub_512_acc_hi)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_hi_fifo), getKillRegState(KillSrc)); } else if ((AIE2P::eLRegClass.contains(SrcReg)) && (AIE2P::EXPVEC64RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_el), - TRI.getSubReg(DstReg, AIE2P::sub_lo_exp)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_el), + TRI.getSubReg(DstReg, AIE2P::sub_lo_exp)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_l_even), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_eh), - TRI.getSubReg(DstReg, AIE2P::sub_hi_exp)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_eh), + TRI.getSubReg(DstReg, AIE2P::sub_hi_exp)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_l_odd), getKillRegState(KillSrc)); } else if ((AIE2P::EXPVEC64RegClass.contains(SrcReg)) && (AIE2P::eLRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_r), - TRI.getSubReg(DstReg, AIE2P::sub_l_even)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_r), + TRI.getSubReg(DstReg, AIE2P::sub_l_even)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_lo_exp), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_r), - TRI.getSubReg(DstReg, AIE2P::sub_l_odd)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_r), + TRI.getSubReg(DstReg, AIE2P::sub_l_odd)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_hi_exp), getKillRegState(KillSrc)); } else if ((AIE2P::EXPVEC64RegClass.contains(SrcReg)) && (AIE2P::EXPVEC64RegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_el), - TRI.getSubReg(DstReg, AIE2P::sub_lo_exp)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_el), + TRI.getSubReg(DstReg, AIE2P::sub_lo_exp)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_lo_exp), getKillRegState(KillSrc)); - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_eh), - TRI.getSubReg(DstReg, AIE2P::sub_hi_exp)) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_eh), + TRI.getSubReg(DstReg, AIE2P::sub_hi_exp)) .addReg(TRI.getSubReg(SrcReg, AIE2P::sub_hi_exp), getKillRegState(KillSrc)); } else if ((AIE2P::ePSRFLdFRegClass.contains(SrcReg)) && (AIE2P::ePSRFLdFRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2P::mEhmRegClass.contains(SrcReg)) && (AIE2P::mEhmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_eh), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_eh), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mElmRegClass.contains(SrcReg)) && (AIE2P::mElmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_el), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_el), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mElmRegClass.contains(SrcReg)) && (AIE2P::mEhmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_eh), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_eh), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mEhmRegClass.contains(SrcReg)) && (AIE2P::mElmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_el), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_el), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mEhmRegClass.contains(SrcReg)) && (AIE2P::eRRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_r), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_eh_to_r), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::eRRegClass.contains(SrcReg)) && (AIE2P::mEhmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_eh), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_eh), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::mElmRegClass.contains(SrcReg)) && (AIE2P::eRRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_r), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_el_to_r), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else if ((AIE2P::eRRegClass.contains(SrcReg)) && (AIE2P::mElmRegClass.contains(DstReg))) { - BuildMI(MBB, MBBI, DL, get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_el), DstReg) + M.buildInstr(get(AIE2P::MOV_alu_mv_mv_mv_e_mv_r_to_el), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else { - llvm_unreachable("unhandled case in copyPhysReg"); + return false; } + + return true; } // Some AIE instructions like Load/Stores take compound register classes diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h index 4337f2d9ee24..6f976c6ac64a 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h @@ -162,6 +162,11 @@ class AIE2PInstrInfo : public AIE2PGenInstrInfo { bool KillSrc, bool RenamableDest = false, bool RenamableSrc = false) const override; +protected: + bool materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const override; + +public: void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp index 3c6cad68e01f..70cfcc7afe80 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp @@ -322,21 +322,33 @@ void AIE2PSInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - bool RenamableDest, bool RenamableSrc) const { - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); - const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + bool /* RenamableDest */, + bool /* RenamableSrc */) const { + const TargetRegisterInfo &TRI = + *MBB.getParent()->getRegInfo().getTargetRegisterInfo(); + CopyMaterializer M(*this, TRI, MBB, MBBI, DL); + if (!materializeCopy(M, DstReg, SrcReg, KillSrc)) { + errs() << "copyPhysReg: cannot copy " << TRI.getName(SrcReg) << " -> " + << TRI.getName(DstReg) << '\n'; + llvm_unreachable("unhandled case in copyPhysReg"); + } +} + +bool AIE2PSInstrInfo::materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const { + const TargetRegisterInfo &TRI = M.getRegisterInfo(); if (AIE2PS::mMvSclSrcRegClass.contains(SrcReg) && AIE2PS::mMvSclDstRegClass.contains(DstReg)) { // Build MultiSlotPseudo in preference const unsigned MOVSclOpcode = getScalarMovOpcode(DstReg, SrcReg); - BuildMI(MBB, MBBI, DL, get(MOVSclOpcode), DstReg) + M.buildInstr(get(MOVSclOpcode), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)); // clang-format off #define HANDLE_MOV_CASE(SRC_CLASS, DST_CLASS, OPCODE) \ } else if ((AIE2PS::SRC_CLASS##RegClass.contains(SrcReg)) && \ (AIE2PS::DST_CLASS##RegClass.contains(DstReg))) { \ - BuildMI(MBB, MBBI, DL, get(AIE2PS::MOV_alu_mv_mv_mv_##OPCODE), DstReg) \ + M.buildInstr(get(AIE2PS::MOV_alu_mv_mv_mv_##OPCODE), DstReg) \ .addReg(SrcReg, getKillRegState(KillSrc)); HANDLE_MOV_CASE(eR, mSCm, sc_r) HANDLE_MOV_CASE(mSCm, eR, r_sc) @@ -360,18 +372,21 @@ void AIE2PSInstrInfo::copyPhysReg(MachineBasicBlock &MBB, // clang-format on } else if ((AIE2PS::eLRegClass.contains(SrcReg)) && (AIE2PS::eLRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2PS::eDRegClass.contains(SrcReg)) && (AIE2PS::eDRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else if ((AIE2PS::eDSRegClass.contains(SrcReg)) && (AIE2PS::eDSRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; // clang-format off #define HANDLE_VMOV_CASE(SRC_CLASS, DST_CLASS, OPCODE) \ } else if ((AIE2PS::SRC_CLASS##RegClass.contains(SrcReg)) && \ (AIE2PS::DST_CLASS##RegClass.contains(DstReg))) { \ - BuildMI(MBB, MBBI, DL, get(AIE2PS::VMOV_alu_mv_mv_mv_##OPCODE), DstReg) \ + M.buildInstr(get(AIE2PS::VMOV_alu_mv_mv_mv_##OPCODE), DstReg) \ .addReg(SrcReg, getKillRegState(KillSrc)); HANDLE_VMOV_CASE(mFm, mFm, f) HANDLE_VMOV_CASE(mWm, mWm, w) @@ -394,60 +409,54 @@ void AIE2PSInstrInfo::copyPhysReg(MachineBasicBlock &MBB, // clang-format on } else if ((AIE2PS::eLRegClass.contains(SrcReg)) && (AIE2PS::EXPVEC64RegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::EXPVEC64RegClass.contains(SrcReg)) && (AIE2PS::eLRegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::eLRegClass.contains(SrcReg) && AIE2PS::mGGaRegClass.contains(DstReg)) || (AIE2PS::mGGaRegClass.contains(SrcReg) && AIE2PS::eLRegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::mEYwRegClass.contains(SrcReg)) && (AIE2PS::mEYwRegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::mFEYwRegClass.contains(SrcReg)) && (AIE2PS::mFEYwRegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::ACC1024RegClass.contains(SrcReg) || AIE2PS::VEC1024RegClass.contains(SrcReg) || AIE2PS::FIFO1024RegClass.contains(SrcReg)) && (AIE2PS::ACC1024RegClass.contains(DstReg) || AIE2PS::VEC1024RegClass.contains(DstReg) || AIE2PS::FIFO1024RegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::ACC2048RegClass.contains(SrcReg)) && (AIE2PS::ACC2048RegClass.contains(DstReg))) { - copyPhysReg(MBB, MBBI, DL, getLoSubReg(TRI, DstReg), - getLoSubReg(TRI, SrcReg), KillSrc); - copyPhysReg(MBB, MBBI, DL, getHiSubReg(TRI, DstReg), - getHiSubReg(TRI, SrcReg), KillSrc); + if (!M.copy(getLoSubReg(TRI, DstReg), getLoSubReg(TRI, SrcReg), KillSrc) || + !M.copy(getHiSubReg(TRI, DstReg), getHiSubReg(TRI, SrcReg), KillSrc)) + return false; } else if ((AIE2PS::ePSRFLdFRegClass.contains(SrcReg)) && (AIE2PS::ePSRFLdFRegClass.contains(DstReg))) { - copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc); + if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc)) + return false; } else { - errs() << "copyPhysReg: cannot copy " << TRI.getName(SrcReg) << " -> " - << TRI.getName(DstReg) << '\n'; - llvm_unreachable("unhandled case in copyPhysReg"); + return false; } + + return true; } static const TargetRegisterClass * diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h index 99d263b87d10..5556c081cad7 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h @@ -76,6 +76,11 @@ class AIE2PSInstrInfo : public AIE2PSGenInstrInfo { bool KillSrc, bool RenamableDest = false, bool RenamableSrc = false) const override; +protected: + bool materializeCopy(CopyMaterializer &M, MCRegister DstReg, + MCRegister SrcReg, bool KillSrc) const override; + +public: Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override; Register isStoreToStackSlot(const MachineInstr &MI, From 332689460c194b1b22c436eacf1f2570d36af51e Mon Sep 17 00:00:00 2001 From: Fabian Stuckmann Date: Wed, 22 Jul 2026 13:39:18 +0200 Subject: [PATCH 3/5] [AIE] Add AIEEpilogueRegRewriter scaffold and baseline WAR-rename test Introduce the AIEEpilogueRegRewriter pass as a registered no-op and wire it into the AIE2/AIE2P/AIE2PS optimized register-allocation pipelines behind EnableEpilogueRegRewrite (default on), plus a second greedy pass. The pass declares its analysis dependencies but does not yet rename anything. Also register the --aie-epilogue-reg-rewrite-copy-budget and --aie-olp-war-rename options and add the epilogue-reg-rewrite MIR test with CHECK lines capturing the pre-rewrite (before) baseline, so the follow-up that implements the renaming shows a clean before/after diff. Pipeline listing tests updated to include the pass. --- llvm/lib/Target/AIE/AIE.h | 3 + llvm/lib/Target/AIE/AIE2TargetMachine.cpp | 7 +- llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp | 6 + .../lib/Target/AIE/AIEEpilogueRegRewriter.cpp | 89 ++ llvm/lib/Target/AIE/CMakeLists.txt | 1 + .../Target/AIE/aie2p/AIE2PTargetMachine.cpp | 7 +- .../Target/AIE/aie2ps/AIE2PSTargetMachine.cpp | 7 +- .../CodeGen/AIE/aie2/llc-pipeline-aie2.ll | 2 + .../CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll | 4 + .../CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll | 4 + .../AIE/aie2ps/ra/epilogue-reg-rewrite.mir | 862 ++++++++++++++++++ 11 files changed, 986 insertions(+), 6 deletions(-) create mode 100644 llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp create mode 100644 llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir diff --git a/llvm/lib/Target/AIE/AIE.h b/llvm/lib/Target/AIE/AIE.h index 5119c2e48702..3d95757b3d3b 100644 --- a/llvm/lib/Target/AIE/AIE.h +++ b/llvm/lib/Target/AIE/AIE.h @@ -60,6 +60,7 @@ MachineFunctionPass *createAIEEliminateDuplicatePHI(); FunctionPass *createAIEOutlineMemoryGEP(); FunctionPass *createAIESuperRegRewriter(); FunctionPass *createAIEWawRegRewriter(); +FunctionPass *createAIEEpilogueRegRewriter(); FunctionPass *createAIEUnallocatedSuperRegRewriter(); FunctionPass *createAIESpillSlotOptimization(); FunctionPass *createAIEPostSelectOptimize(); @@ -88,6 +89,8 @@ extern char &AIESuperRegRewriterID; void initializeAIESuperRegRewriterPass(PassRegistry &); extern char &AIEWawRegRewriterID; void initializeAIEWawRegRewriterPass(PassRegistry &); +extern char &AIEEpilogueRegRewriterID; +void initializeAIEEpilogueRegRewriterPass(PassRegistry &); extern char &AIEUnallocatedSuperRegRewriterID; void initializeAIEUnallocatedSuperRegRewriterPass(PassRegistry &); extern char &AIESpillSlotOptimizationID; diff --git a/llvm/lib/Target/AIE/AIE2TargetMachine.cpp b/llvm/lib/Target/AIE/AIE2TargetMachine.cpp index 59e30d38d107..678409844f59 100644 --- a/llvm/lib/Target/AIE/AIE2TargetMachine.cpp +++ b/llvm/lib/Target/AIE/AIE2TargetMachine.cpp @@ -53,6 +53,7 @@ extern cl::opt EnableSuperRegSplitting; extern cl::opt AllocateMRegsFirst; extern cl::opt EnablePreMISchedCoalescer; extern cl::opt EnableWAWRegRewrite; +extern cl::opt EnableEpilogueRegRewrite; extern cl::opt EnableAIEIfConversion; extern bool AIEDumpArtifacts; @@ -193,10 +194,12 @@ bool AIE2PassConfig::addRegAssignAndRewriteOptimized() { addPass(createAIESuperRegRewriter()); } addPass(createGreedyRegisterAllocator()); - if (EnableWAWRegRewrite) { + if (EnableWAWRegRewrite) addPass(createAIEWawRegRewriter()); + if (EnableEpilogueRegRewrite) + addPass(createAIEEpilogueRegRewriter()); + if (EnableWAWRegRewrite || EnableEpilogueRegRewrite) addPass(createGreedyRegisterAllocator()); - } addPass(createVirtRegRewriter()); return true; diff --git a/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp b/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp index bdf4cef46f3a..9aa40e9621bc 100644 --- a/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp +++ b/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp @@ -111,6 +111,11 @@ cl::opt cl::desc("Enable the WAW Register Renaming in loops"), cl::init(true), cl::Hidden); +cl::opt EnableEpilogueRegRewrite( + "aie-enable-epilogue-reg-rewrite", + cl::desc("Enable epilogue register renaming after allocation"), + cl::init(true), cl::Hidden); + cl::opt EnableSuperRegSplitting("aie-split-superregs", cl::Hidden, cl::init(true), cl::desc("Enable splitting super-regs into their " @@ -165,6 +170,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAIETarget() { initializeAIESuperRegRewriterPass(*PR); initializeAIEUnallocatedSuperRegRewriterPass(*PR); initializeAIEWawRegRewriterPass(*PR); + initializeAIEEpilogueRegRewriterPass(*PR); initializeAIEOutlineMemoryGEPPass(*PR); initializeAIEFinalizeBundlePass(*PR); initializeAIEMachineAlignmentPass(*PR); diff --git a/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp b/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp new file mode 100644 index 000000000000..5875992f7df3 --- /dev/null +++ b/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp @@ -0,0 +1,89 @@ +//===-- AIEEpilogueRegRewriter.cpp - Rename epilogue definitions --------===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +// +//===----------------------------------------------------------------------===// + +#include "AIE.h" +#include "AIEBaseInstrInfo.h" +#include "AIEBaseRegisterInfo.h" +#include "AIESuperRegUtils.h" +#include "Utils/AIELoopOptionOverrides.h" +#include "Utils/AIERegAllocationUtils.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/LiveDebugVariables.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/LiveRegMatrix.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +#define DEBUG_TYPE "aie-epilogue-reg-rewriter" + +static cl::opt EpilogueCopyBudget( + "aie-epilogue-reg-rewrite-copy-budget", cl::Hidden, cl::init(4), + cl::desc("Maximum materialized copy instructions per epilogue")); + +// Per-block opt-in: only rename WAR definitions in a block that carries this +// hint (see AIE::LoopOptionOverrides). +static cl::opt EnableOLPWarRename( + "aie-olp-war-rename", cl::Hidden, cl::init(false), + cl::desc("Enable WAR register renaming on blocks carrying the hint")); + +namespace { + +class AIEEpilogueRegRewriter : public MachineFunctionPass { +public: + static char ID; + + AIEEpilogueRegRewriter() : MachineFunctionPass(ID) {} + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &MF) override { return false; } +}; + +} // namespace + +char AIEEpilogueRegRewriter::ID = 0; +char &llvm::AIEEpilogueRegRewriterID = AIEEpilogueRegRewriter::ID; + +INITIALIZE_PASS_BEGIN(AIEEpilogueRegRewriter, DEBUG_TYPE, + "AIE epilogue register rewrite", false, false) +INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy) +INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy) +INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy) +INITIALIZE_PASS_END(AIEEpilogueRegRewriter, DEBUG_TYPE, + "AIE epilogue register rewrite", false, false) + +FunctionPass *llvm::createAIEEpilogueRegRewriter() { + return new AIEEpilogueRegRewriter(); +} diff --git a/llvm/lib/Target/AIE/CMakeLists.txt b/llvm/lib/Target/AIE/CMakeLists.txt index 79265658531a..75cf6bc8887b 100644 --- a/llvm/lib/Target/AIE/CMakeLists.txt +++ b/llvm/lib/Target/AIE/CMakeLists.txt @@ -128,6 +128,7 @@ add_llvm_target(AIECodeGen AIEDataDependenceHelper.cpp AIEDumpArtifacts.cpp AIEEliminateDuplicatePHI.cpp + AIEEpilogueRegRewriter.cpp AIEFinalizeBundle.cpp AIEGlobalCombiner.cpp AIEGlobalCombinerPtrMods.cpp diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PTargetMachine.cpp b/llvm/lib/Target/AIE/aie2p/AIE2PTargetMachine.cpp index 7d1703f1dfb4..787b8a4be74c 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PTargetMachine.cpp +++ b/llvm/lib/Target/AIE/aie2p/AIE2PTargetMachine.cpp @@ -27,6 +27,7 @@ extern cl::opt EnablePreMISchedCoalescer; extern cl::opt EnableAddressChaining; extern cl::opt EnableGlobalPtrModOptimizer; extern cl::opt EnableWAWRegRewrite; +extern cl::opt EnableEpilogueRegRewrite; extern cl::opt EnableAIEIfConversion; extern cl::opt EnableFineGrainedStagedRA; @@ -119,10 +120,12 @@ bool AIE2PPassConfig::addRegAssignAndRewriteOptimized() { addPass(createAIEUnallocatedSuperRegRewriter()); } addPass(createGreedyRegisterAllocator()); - if (EnableWAWRegRewrite) { + if (EnableWAWRegRewrite) addPass(createAIEWawRegRewriter()); + if (EnableEpilogueRegRewrite) + addPass(createAIEEpilogueRegRewriter()); + if (EnableWAWRegRewrite || EnableEpilogueRegRewrite) addPass(createGreedyRegisterAllocator()); - } addPass(createVirtRegRewriter()); return true; diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSTargetMachine.cpp b/llvm/lib/Target/AIE/aie2ps/AIE2PSTargetMachine.cpp index 1a151a900269..2985522ace64 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSTargetMachine.cpp +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSTargetMachine.cpp @@ -25,6 +25,7 @@ extern cl::opt AllocateMRegsFirst; extern cl::opt EnablePreMISchedCoalescer; extern cl::opt EnableFineGrainedStagedRA; extern cl::opt EnableWAWRegRewrite; +extern cl::opt EnableEpilogueRegRewrite; extern cl::opt EnableAddressChaining; extern cl::opt EnableGlobalPtrModOptimizer; @@ -134,10 +135,12 @@ bool AIE2PSPassConfig::addRegAssignAndRewriteOptimized() { addPass(createAIEUnallocatedSuperRegRewriter()); } addPass(createGreedyRegisterAllocator()); - if (EnableWAWRegRewrite) { + if (EnableWAWRegRewrite) addPass(createAIEWawRegRewriter()); + if (EnableEpilogueRegRewrite) + addPass(createAIEEpilogueRegRewriter()); + if (EnableWAWRegRewrite || EnableEpilogueRegRewrite) addPass(createGreedyRegisterAllocator()); - } addPass(createVirtRegRewriter()); return true; diff --git a/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll b/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll index f3c3cb8cd341..d8a64b371fda 100644 --- a/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll +++ b/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll @@ -231,6 +231,8 @@ ; AIE-O123-NEXT: AIE super-reg rewrite ; AIE-O123-NEXT: Greedy Register Allocator ; AIE-O123-NEXT: AIE waw-reg rewrite +; AIE-O123-NEXT: AIE epilogue register rewrite +; AIE-O123-NEXT: Live Stack Slot Analysis ; AIE-O123-NEXT: Greedy Register Allocator ; AIE-O123-NEXT: Virtual Register Rewriter ; AIE-O123-NEXT: Stack Slot Coloring diff --git a/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll b/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll index 08c3e0f15097..531d6ac3cff1 100644 --- a/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll +++ b/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll @@ -257,6 +257,8 @@ ; AIE-O1-NEXT: AIE unallocated super-reg rewrite ; AIE-O1-NEXT: Greedy Register Allocator ; AIE-O1-NEXT: AIE waw-reg rewrite +; AIE-O1-NEXT: AIE epilogue register rewrite +; AIE-O1-NEXT: Live Stack Slot Analysis ; AIE-O1-NEXT: Greedy Register Allocator ; AIE-O1-NEXT: Virtual Register Rewriter ; AIE-O1-NEXT: Stack Slot Coloring @@ -489,6 +491,8 @@ ; AIE-O23-NEXT: AIE unallocated super-reg rewrite ; AIE-O23-NEXT: Greedy Register Allocator ; AIE-O23-NEXT: AIE waw-reg rewrite +; AIE-O23-NEXT: AIE epilogue register rewrite +; AIE-O23-NEXT: Live Stack Slot Analysis ; AIE-O23-NEXT: Greedy Register Allocator ; AIE-O23-NEXT: Virtual Register Rewriter ; AIE-O23-NEXT: Stack Slot Coloring diff --git a/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll b/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll index ecebdb214ad5..a0dddb071b16 100644 --- a/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll +++ b/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll @@ -263,6 +263,8 @@ ; AIE-O1-NEXT: AIE unallocated super-reg rewrite ; AIE-O1-NEXT: Greedy Register Allocator ; AIE-O1-NEXT: AIE waw-reg rewrite +; AIE-O1-NEXT: AIE epilogue register rewrite +; AIE-O1-NEXT: Live Stack Slot Analysis ; AIE-O1-NEXT: Greedy Register Allocator ; AIE-O1-NEXT: Virtual Register Rewriter ; AIE-O1-NEXT: Stack Slot Coloring @@ -498,6 +500,8 @@ ; AIE-O23-NEXT: AIE unallocated super-reg rewrite ; AIE-O23-NEXT: Greedy Register Allocator ; AIE-O23-NEXT: AIE waw-reg rewrite +; AIE-O23-NEXT: AIE epilogue register rewrite +; AIE-O23-NEXT: Live Stack Slot Analysis ; AIE-O23-NEXT: Greedy Register Allocator ; AIE-O23-NEXT: Virtual Register Rewriter ; AIE-O23-NEXT: Stack Slot Coloring diff --git a/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir b/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir new file mode 100644 index 000000000000..91838feb21af --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir @@ -0,0 +1,862 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates + +# RUN: llc -mtriple=aie2ps --aie-wawreg-rewrite=false --aie-olp-war-rename \ +# RUN: --aie-enable-epilogue-reg-rewrite --start-before=greedy \ +# RUN: --stop-after=aie-epilogue-reg-rewriter %s -o - | \ +# RUN: FileCheck %s --check-prefix=DEFAULT +# RUN: llc -mtriple=aie2ps --aie-wawreg-rewrite=false --aie-olp-war-rename \ +# RUN: --aie-enable-epilogue-reg-rewrite \ +# RUN: --aie-epilogue-reg-rewrite-copy-budget=3 --start-before=greedy \ +# RUN: --stop-after=aie-epilogue-reg-rewriter %s -o - | \ +# RUN: FileCheck %s --check-prefix=BUDGET3 +# RUN: llc -mtriple=aie2ps --aie-wawreg-rewrite=false --aie-olp-war-rename \ +# RUN: --aie-enable-epilogue-reg-rewrite \ +# RUN: --aie-epilogue-reg-rewrite-copy-budget=8 --start-before=greedy \ +# RUN: --stop-after=aie-epilogue-reg-rewriter %s -o - | \ +# RUN: FileCheck %s --check-prefix=BUDGET8 +# RUN: llc -mtriple=aie2ps --aie-wawreg-rewrite=false --aie-olp-war-rename \ +# RUN: --aie-enable-epilogue-reg-rewrite \ +# RUN: --aie-epilogue-reg-rewrite-copy-budget=8 --start-before=greedy \ +# RUN: --stop-after=virtregrewriter %s -o - | \ +# RUN: FileCheck %s --check-prefix=PHYS + +# A register read then rewritten in a block carrying the aie-olp-war-rename hint +# is a WAR candidate: its defs are renamed and a repair copy restores the old +# name at the block end. The copy budget bounds how many candidates are taken; +# acc2048 costs four sub-register copies, so the default budget of four renames +# one candidate, budget three renames none, and budget eight renames two. Both +# VLDA- and COPY-defined accumulators are handled. +--- +name: rename_war_vlda_and_copy +alignment: 16 +legalized: true +isSSA: false +tracksRegLiveness: true +body: | + ; DEFAULT-LABEL: name: rename_war_vlda_and_copy + ; DEFAULT: bb.0: + ; DEFAULT-NEXT: successors: %bb.1(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF6:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF7:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF8:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.1 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.1: + ; DEFAULT-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF9:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF10:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF11:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.2 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.2: + ; DEFAULT-NEXT: successors: %bb.3(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: undef [[DEF9:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF9:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: undef [[DEF10:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] + ; DEFAULT-NEXT: [[DEF10:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] + ; DEFAULT-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] + ; DEFAULT-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] + ; DEFAULT-NEXT: undef [[DEF11:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF11:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.3 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.3: + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X3:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X4:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X5:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X3]], implicit [[VMAC_vmac_vmul_cm_core_X_X4]], implicit [[VMAC_vmac_vmul_cm_core_X_X5]] + ; + ; BUDGET3-LABEL: name: rename_war_vlda_and_copy + ; BUDGET3: bb.0: + ; BUDGET3-NEXT: successors: %bb.1(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF6:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF7:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF8:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.1: + ; BUDGET3-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF9:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF10:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF11:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.2: + ; BUDGET3-NEXT: successors: %bb.3(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: undef [[DEF9:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF9:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: undef [[DEF10:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] + ; BUDGET3-NEXT: [[DEF10:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] + ; BUDGET3-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] + ; BUDGET3-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] + ; BUDGET3-NEXT: undef [[DEF11:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF11:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.3: + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X3:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X4:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X5:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X3]], implicit [[VMAC_vmac_vmul_cm_core_X_X4]], implicit [[VMAC_vmac_vmul_cm_core_X_X5]] + ; + ; BUDGET8-LABEL: name: rename_war_vlda_and_copy + ; BUDGET8: bb.0: + ; BUDGET8-NEXT: successors: %bb.1(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF6:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF7:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF8:%[0-9]+]]:acc512 = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.1: + ; BUDGET8-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF9:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF10:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF11:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.2: + ; BUDGET8-NEXT: successors: %bb.3(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: undef [[DEF9:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF9:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: undef [[DEF10:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] + ; BUDGET8-NEXT: [[DEF10:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] + ; BUDGET8-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] + ; BUDGET8-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] + ; BUDGET8-NEXT: undef [[DEF11:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF11:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.3: + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X3:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X4:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X5:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X3]], implicit [[VMAC_vmac_vmul_cm_core_X_X4]], implicit [[VMAC_vmac_vmul_cm_core_X_X5]] + ; + ; PHYS-LABEL: name: rename_war_vlda_and_copy + ; PHYS: bb.0: + ; PHYS-NEXT: successors: %bb.1(0x80000000) + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $x0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $x2 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r8 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $p0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $bmll1 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $bmll2 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $bmll3 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $bmll4 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJ_jump_imm %bb.1 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.1: + ; PHYS-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; PHYS-NEXT: liveins: $bmll1, $bmll2, $bmll3, $bmll4, $p0, $r0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $dm0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm5 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm6 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJNZ renamable $r0, %bb.1 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.2: + ; PHYS-NEXT: successors: %bb.3(0x80000000) + ; PHYS-NEXT: liveins: $bmll1, $bmll2, $bmll3, $bmll4, $dm0:0x000000030000000C, $dm5:0x000000030000000C, $dm6:0x000000030000000C, $p0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm5, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm6, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $bmll0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmll5 = COPY killed renamable $bmll1 + ; PHYS-NEXT: renamable $bmlh5 = COPY killed renamable $bmll2 + ; PHYS-NEXT: renamable $bmhl5 = COPY killed renamable $bmll3 + ; PHYS-NEXT: renamable $bmhh5 = COPY killed renamable $bmll4 + ; PHYS-NEXT: renamable $bmll6, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh6, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl6, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh6, dead renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.3 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.3: + ; PHYS-NEXT: liveins: $dm0:0x000000030000000C, $dm5:0x000000030000000C, $dm6:0x000000030000000C, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm1 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm5, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm2 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm6, killed renamable $x0, killed renamable $x2, killed renamable $r8 + ; PHYS-NEXT: PseudoRET implicit $lr, implicit killed renamable $dm0, implicit killed renamable $dm1, implicit killed renamable $dm2 + bb.0: + successors: %bb.1 + %20:vec512 = IMPLICIT_DEF + %21:vec512 = IMPLICIT_DEF + %22:mrv8 = IMPLICIT_DEF + %23:ep_as_32bit = IMPLICIT_DEF + %24:mlockid_reg = IMPLICIT_DEF + %25:acc512 = IMPLICIT_DEF + %26:acc512 = IMPLICIT_DEF + %27:acc512 = IMPLICIT_DEF + %28:acc512 = IMPLICIT_DEF + PseudoJ_jump_imm %bb.1 + + bb.1: + successors: %bb.1, %bb.2 + %0:acc2048 = IMPLICIT_DEF + %1:acc2048 = IMPLICIT_DEF + %2:acc2048 = IMPLICIT_DEF + PseudoJNZ %24, %bb.1 + PseudoJ_jump_imm %bb.2 + + bb.2: + successors: %bb.3 + dead %30:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + dead %31:acc2048 = VMAC_vmac_vmul_cm_core_X_X %1, %20, %21, %22 + dead %32:acc2048 = VMAC_vmac_vmul_cm_core_X_X %2, %20, %21, %22 + undef %0.sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_512_acc_hi:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + undef %1.sub_512_acc_lo:acc2048 = COPY %25 + %1.sub_512_acc_hi:acc2048 = COPY %26 + %1.sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY %27 + %1.sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY %28 + undef %2.sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %2.sub_512_acc_hi:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %2.sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %2.sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + PseudoJ_jump_imm %bb.3 + + bb.3: + %40:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + %41:acc2048 = VMAC_vmac_vmul_cm_core_X_X %1, %20, %21, %22 + %42:acc2048 = VMAC_vmac_vmul_cm_core_X_X %2, %20, %21, %22 + PseudoRET implicit $lr, implicit %40, implicit %41, implicit %42 +... + +# A read of the accumulator after its rewrite in the same block would observe +# the old name (the repair copy only lands at the block end), so no rename. +--- +name: reject_use_after_def +alignment: 16 +legalized: true +isSSA: false +tracksRegLiveness: true +body: | + ; DEFAULT-LABEL: name: reject_use_after_def + ; DEFAULT: bb.0: + ; DEFAULT-NEXT: successors: %bb.1(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.1 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.1: + ; DEFAULT-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.2 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.2: + ; DEFAULT-NEXT: successors: %bb.3(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: undef [[DEF5:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.3 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.3: + ; DEFAULT-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X1]] + ; + ; BUDGET3-LABEL: name: reject_use_after_def + ; BUDGET3: bb.0: + ; BUDGET3-NEXT: successors: %bb.1(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.1: + ; BUDGET3-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.2: + ; BUDGET3-NEXT: successors: %bb.3(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: undef [[DEF5:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.3: + ; BUDGET3-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X1]] + ; + ; BUDGET8-LABEL: name: reject_use_after_def + ; BUDGET8: bb.0: + ; BUDGET8-NEXT: successors: %bb.1(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.1: + ; BUDGET8-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.2: + ; BUDGET8-NEXT: successors: %bb.3(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: undef [[DEF5:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.3: + ; BUDGET8-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X1]] + ; + ; PHYS-LABEL: name: reject_use_after_def + ; PHYS: bb.0: + ; PHYS-NEXT: successors: %bb.1(0x80000000) + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $x0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $x2 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r8 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $p0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r0 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJ_jump_imm %bb.1 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.1: + ; PHYS-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; PHYS-NEXT: liveins: $p0, $r0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $dm0 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJNZ renamable $r0, %bb.1 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.2: + ; PHYS-NEXT: successors: %bb.3(0x80000000) + ; PHYS-NEXT: liveins: $dm0:0x000000030000000C, $p0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $bmll0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh0, dead renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, killed renamable $x0, killed renamable $x2, killed renamable $r8 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.3 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.3: + ; PHYS-NEXT: liveins: $dm0 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: PseudoRET implicit $lr, implicit killed renamable $dm0 + bb.0: + successors: %bb.1 + %20:vec512 = IMPLICIT_DEF + %21:vec512 = IMPLICIT_DEF + %22:mrv8 = IMPLICIT_DEF + %23:ep_as_32bit = IMPLICIT_DEF + %24:mlockid_reg = IMPLICIT_DEF + PseudoJ_jump_imm %bb.1 + + bb.1: + successors: %bb.1, %bb.2 + %0:acc2048 = IMPLICIT_DEF + PseudoJNZ %24, %bb.1 + PseudoJ_jump_imm %bb.2 + + bb.2: + successors: %bb.3 + dead %30:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + undef %0.sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_512_acc_hi:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %31:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + PseudoJ_jump_imm %bb.3 + + bb.3: + PseudoRET implicit $lr, implicit %31 +... + +# An accumulator that is only written (never read before the write) in the block +# has no WAR hazard to break, so no rename. +--- +name: reject_no_use_before_def +alignment: 16 +legalized: true +isSSA: false +tracksRegLiveness: true +body: | + ; DEFAULT-LABEL: name: reject_no_use_before_def + ; DEFAULT: bb.0: + ; DEFAULT-NEXT: successors: %bb.1(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.1 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.1: + ; DEFAULT-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: dead [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.2 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.2: + ; DEFAULT-NEXT: successors: %bb.3(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.3 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.3: + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X]] + ; + ; BUDGET3-LABEL: name: reject_no_use_before_def + ; BUDGET3: bb.0: + ; BUDGET3-NEXT: successors: %bb.1(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.1: + ; BUDGET3-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: dead [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.2: + ; BUDGET3-NEXT: successors: %bb.3(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.3: + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X]] + ; + ; BUDGET8-LABEL: name: reject_no_use_before_def + ; BUDGET8: bb.0: + ; BUDGET8-NEXT: successors: %bb.1(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.1: + ; BUDGET8-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: dead [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.2: + ; BUDGET8-NEXT: successors: %bb.3(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.3: + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X]] + ; + ; PHYS-LABEL: name: reject_no_use_before_def + ; PHYS: bb.0: + ; PHYS-NEXT: successors: %bb.1(0x80000000) + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $x0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $x2 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r8 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $p0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r0 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJ_jump_imm %bb.1 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.1: + ; PHYS-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; PHYS-NEXT: liveins: $p0, $r0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: dead renamable $dm0 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJNZ renamable $r0, %bb.1 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.2: + ; PHYS-NEXT: successors: %bb.3(0x80000000) + ; PHYS-NEXT: liveins: $p0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $bmll0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh0, dead renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.3 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.3: + ; PHYS-NEXT: liveins: $dm0:0x000000030000000C, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, killed renamable $x0, killed renamable $x2, killed renamable $r8 + ; PHYS-NEXT: PseudoRET implicit $lr, implicit killed renamable $dm0 + bb.0: + successors: %bb.1 + %20:vec512 = IMPLICIT_DEF + %21:vec512 = IMPLICIT_DEF + %22:mrv8 = IMPLICIT_DEF + %23:ep_as_32bit = IMPLICIT_DEF + %24:mlockid_reg = IMPLICIT_DEF + PseudoJ_jump_imm %bb.1 + + bb.1: + successors: %bb.1, %bb.2 + %0:acc2048 = IMPLICIT_DEF + PseudoJNZ %24, %bb.1 + PseudoJ_jump_imm %bb.2 + + bb.2: + successors: %bb.3 + undef %0.sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_512_acc_hi:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + PseudoJ_jump_imm %bb.3 + + bb.3: + %40:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + PseudoRET implicit $lr, implicit %40 +... + +# The candidate is valid, but every other accumulator register is live across +# the rewrite, so no free non-overlapping physical register exists and the +# rename is skipped. +--- +name: reject_without_free_dm +alignment: 16 +legalized: true +isSSA: false +tracksRegLiveness: true +body: | + ; DEFAULT-LABEL: name: reject_without_free_dm + ; DEFAULT: bb.0: + ; DEFAULT-NEXT: successors: %bb.1(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.1 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.1: + ; DEFAULT-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF6:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF7:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF8:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF9:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF10:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF11:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: [[DEF12:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; DEFAULT-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.2 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.2: + ; DEFAULT-NEXT: successors: %bb.3(0x80000000) + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: undef [[DEF5:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF6]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF7]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X3:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF8]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X4:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X5:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X6:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X7:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF12]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.3 + ; DEFAULT-NEXT: {{ $}} + ; DEFAULT-NEXT: bb.3: + ; DEFAULT-NEXT: [[VMAC_vmac_vmul_cm_core_X_X8:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; DEFAULT-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X8]], implicit [[VMAC_vmac_vmul_cm_core_X_X1]], implicit [[VMAC_vmac_vmul_cm_core_X_X2]], implicit [[VMAC_vmac_vmul_cm_core_X_X3]], implicit [[VMAC_vmac_vmul_cm_core_X_X4]], implicit [[VMAC_vmac_vmul_cm_core_X_X5]], implicit [[VMAC_vmac_vmul_cm_core_X_X6]], implicit [[VMAC_vmac_vmul_cm_core_X_X7]] + ; + ; BUDGET3-LABEL: name: reject_without_free_dm + ; BUDGET3: bb.0: + ; BUDGET3-NEXT: successors: %bb.1(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.1: + ; BUDGET3-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF6:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF7:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF8:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF9:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF10:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF11:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: [[DEF12:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET3-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.2: + ; BUDGET3-NEXT: successors: %bb.3(0x80000000) + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: undef [[DEF5:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF6]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF7]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X3:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF8]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X4:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X5:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X6:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X7:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF12]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET3-NEXT: {{ $}} + ; BUDGET3-NEXT: bb.3: + ; BUDGET3-NEXT: [[VMAC_vmac_vmul_cm_core_X_X8:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET3-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X8]], implicit [[VMAC_vmac_vmul_cm_core_X_X1]], implicit [[VMAC_vmac_vmul_cm_core_X_X2]], implicit [[VMAC_vmac_vmul_cm_core_X_X3]], implicit [[VMAC_vmac_vmul_cm_core_X_X4]], implicit [[VMAC_vmac_vmul_cm_core_X_X5]], implicit [[VMAC_vmac_vmul_cm_core_X_X6]], implicit [[VMAC_vmac_vmul_cm_core_X_X7]] + ; + ; BUDGET8-LABEL: name: reject_without_free_dm + ; BUDGET8: bb.0: + ; BUDGET8-NEXT: successors: %bb.1(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF1:%[0-9]+]]:vec512 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF2:%[0-9]+]]:mrv8 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF3:%[0-9]+]]:ep_as_32bit = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF4:%[0-9]+]]:mlockid_reg = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.1 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.1: + ; BUDGET8-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF6:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF7:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF8:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF9:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF10:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF11:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: [[DEF12:%[0-9]+]]:acc2048 = IMPLICIT_DEF + ; BUDGET8-NEXT: PseudoJNZ [[DEF4]], %bb.1 + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.2 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.2: + ; BUDGET8-NEXT: successors: %bb.3(0x80000000) + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: undef [[DEF5:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF5:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF6]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF7]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X3:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF8]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X4:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X5:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X6:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X7:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF12]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.3 + ; BUDGET8-NEXT: {{ $}} + ; BUDGET8-NEXT: bb.3: + ; BUDGET8-NEXT: [[VMAC_vmac_vmul_cm_core_X_X8:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF5]], [[DEF]], [[DEF1]], [[DEF2]] + ; BUDGET8-NEXT: PseudoRET implicit $lr, implicit [[VMAC_vmac_vmul_cm_core_X_X8]], implicit [[VMAC_vmac_vmul_cm_core_X_X1]], implicit [[VMAC_vmac_vmul_cm_core_X_X2]], implicit [[VMAC_vmac_vmul_cm_core_X_X3]], implicit [[VMAC_vmac_vmul_cm_core_X_X4]], implicit [[VMAC_vmac_vmul_cm_core_X_X5]], implicit [[VMAC_vmac_vmul_cm_core_X_X6]], implicit [[VMAC_vmac_vmul_cm_core_X_X7]] + ; + ; PHYS-LABEL: name: reject_without_free_dm + ; PHYS: bb.0: + ; PHYS-NEXT: successors: %bb.1(0x80000000) + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $x0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $x2 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r8 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $p0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $r0 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJ_jump_imm %bb.1 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.1: + ; PHYS-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; PHYS-NEXT: liveins: $p0, $r0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $dm0 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm1 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm2 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm3 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm4 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm5 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm6 = IMPLICIT_DEF + ; PHYS-NEXT: renamable $dm7 = IMPLICIT_DEF + ; PHYS-NEXT: PseudoJNZ renamable $r0, %bb.1 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.2: + ; PHYS-NEXT: successors: %bb.3(0x80000000) + ; PHYS-NEXT: liveins: $dm0:0x000000030000000C, $dm1, $dm2, $dm3, $dm4, $dm5, $dm6, $dm7, $p0, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $bmll0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh0, dead renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $dm1 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm1, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm2 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm2, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm3 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm3, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm4 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm4, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm5 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm5, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm6 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm6, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: renamable $dm7 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm7, renamable $x0, renamable $x2, renamable $r8 + ; PHYS-NEXT: PseudoJ_jump_imm %bb.3 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: bb.3: + ; PHYS-NEXT: liveins: $dm0:0x000000030000000C, $dm1, $dm2, $dm3, $dm4, $dm5, $dm6, $dm7, $r8, $x0, $x2 + ; PHYS-NEXT: {{ $}} + ; PHYS-NEXT: renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, killed renamable $x0, killed renamable $x2, killed renamable $r8 + ; PHYS-NEXT: PseudoRET implicit $lr, implicit killed renamable $dm0, implicit killed renamable $dm1, implicit killed renamable $dm2, implicit killed renamable $dm3, implicit killed renamable $dm4, implicit killed renamable $dm5, implicit killed renamable $dm6, implicit killed renamable $dm7 + bb.0: + successors: %bb.1 + %20:vec512 = IMPLICIT_DEF + %21:vec512 = IMPLICIT_DEF + %22:mrv8 = IMPLICIT_DEF + %23:ep_as_32bit = IMPLICIT_DEF + %24:mlockid_reg = IMPLICIT_DEF + PseudoJ_jump_imm %bb.1 + + bb.1: + successors: %bb.1, %bb.2 + %0:acc2048 = IMPLICIT_DEF + %1:acc2048 = IMPLICIT_DEF + %2:acc2048 = IMPLICIT_DEF + %3:acc2048 = IMPLICIT_DEF + %4:acc2048 = IMPLICIT_DEF + %5:acc2048 = IMPLICIT_DEF + %6:acc2048 = IMPLICIT_DEF + %7:acc2048 = IMPLICIT_DEF + PseudoJNZ %24, %bb.1 + PseudoJ_jump_imm %bb.2 + + bb.2: + successors: %bb.3 + dead %30:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + undef %0.sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_512_acc_hi:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %0.sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead %23:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm %23, 64 + %31:acc2048 = VMAC_vmac_vmul_cm_core_X_X %1, %20, %21, %22 + %32:acc2048 = VMAC_vmac_vmul_cm_core_X_X %2, %20, %21, %22 + %33:acc2048 = VMAC_vmac_vmul_cm_core_X_X %3, %20, %21, %22 + %34:acc2048 = VMAC_vmac_vmul_cm_core_X_X %4, %20, %21, %22 + %35:acc2048 = VMAC_vmac_vmul_cm_core_X_X %5, %20, %21, %22 + %36:acc2048 = VMAC_vmac_vmul_cm_core_X_X %6, %20, %21, %22 + %37:acc2048 = VMAC_vmac_vmul_cm_core_X_X %7, %20, %21, %22 + PseudoJ_jump_imm %bb.3 + + bb.3: + %40:acc2048 = VMAC_vmac_vmul_cm_core_X_X %0, %20, %21, %22 + PseudoRET implicit $lr, implicit %40, implicit %31, implicit %32, implicit %33, implicit %34, implicit %35, implicit %36, implicit %37 +... From cf23bf320d4bbe5b5e90e047140b3db4a5797218 Mon Sep 17 00:00:00 2001 From: Fabian Stuckmann Date: Wed, 22 Jul 2026 13:39:37 +0200 Subject: [PATCH 4/5] [AIE] Implement WAR renaming in AIEEpilogueRegRewriter Fill in the epilogue register rewriter: collect def-driven WAR candidates (registers read before they are rewritten in a block carrying the aie-olp-war-rename hint), find a free non-overlapping physical register via the shared search helper, and materialize a repair copy that restores the old name at the block end. The copy budget bounds how many renames are taken. Both VLDA- and COPY-defined accumulators are handled. Updates the epilogue-reg-rewrite MIR test CHECK lines to the post-rewrite (after) state, so the diff against the baseline shows exactly what the pass changes. --- .../lib/Target/AIE/AIEEpilogueRegRewriter.cpp | 234 +++++++++++++++++- .../AIE/aie2ps/ra/epilogue-reg-rewrite.mir | 84 ++++--- 2 files changed, 279 insertions(+), 39 deletions(-) diff --git a/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp b/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp index 5875992f7df3..827687c0b130 100644 --- a/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp +++ b/llvm/lib/Target/AIE/AIEEpilogueRegRewriter.cpp @@ -45,6 +45,13 @@ static cl::opt EnableOLPWarRename( namespace { +struct RewriteCandidate { + Register OldReg; + // Sorted by def slot index, so Defs.front() is the first def in the block. + SmallVector Defs; + MachineBasicBlock *MBB; +}; + class AIEEpilogueRegRewriter : public MachineFunctionPass { public: static char ID; @@ -66,9 +73,234 @@ class AIEEpilogueRegRewriter : public MachineFunctionPass { MachineFunctionPass::getAnalysisUsage(AU); } - bool runOnMachineFunction(MachineFunction &MF) override { return false; } + bool runOnMachineFunction(MachineFunction &MF) override; + +private: + std::optional + collectCandidate(Register Reg, MachineBasicBlock &MBB, + MachineRegisterInfo &MRI, const AIEBaseRegisterInfo &TRI, + VirtRegMap &VRM, LiveIntervals &LIS) const; + + SmallVector + collectCandidates(MachineFunction &MF, MachineRegisterInfo &MRI, + const AIEBaseRegisterInfo &TRI, VirtRegMap &VRM, + LiveIntervals &LIS) const; + + MCPhysReg findReplacementPhysReg(const RewriteCandidate &Candidate, + MachineRegisterInfo &MRI, + const AIEBaseRegisterInfo &TRI, + VirtRegMap &VRM, LiveIntervals &LIS, + LiveRegMatrix &LRM, + BitVector &ReservedRegUnits) const; + + void commitRewrite(RewriteCandidate &Candidate, MCPhysReg NewPhys, + MachineRegisterInfo &MRI, const AIEBaseInstrInfo &TII, + VirtRegMap &VRM, LiveRegMatrix &LRM, LiveIntervals &LIS, + LiveDebugVariables &DebugVars) const; }; +std::optional AIEEpilogueRegRewriter::collectCandidate( + Register Reg, MachineBasicBlock &MBB, MachineRegisterInfo &MRI, + const AIEBaseRegisterInfo &TRI, VirtRegMap &VRM, LiveIntervals &LIS) const { + SmallVector Defs; + SmallVector Uses; + for (MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) { + if (MO.getParent()->getParent() != &MBB) + continue; + if (MO.isDef()) + Defs.push_back(&MO); + else if (MO.isUse()) + Uses.push_back(&MO); + } + // A WAR hazard only exists if the register is both read and written here. + if (Defs.empty() || Uses.empty()) + return std::nullopt; + + llvm::sort(Defs, + [&](const MachineOperand *Left, const MachineOperand *Right) { + return LIS.getInstructionIndex(*Left->getParent()) < + LIS.getInstructionIndex(*Right->getParent()); + }); + + // Only defs get renamed and the repair copy lands at the block end, so a use + // at or after the first rewritten def would see a partially renamed reg. + const SlotIndex FirstDefIndex = + LIS.getInstructionIndex(*Defs.front()->getParent()); + for (MachineOperand *Use : Uses) + if (LIS.getInstructionIndex(*Use->getParent()) >= FirstDefIndex) + return std::nullopt; + + LLVM_DEBUG(dbgs() << "Epilogue register rewrite: target " + << printReg(Reg, &TRI, 0, &MRI) << " in " + << MBB.getFullName() << ", old physical register " + << printReg(VRM.getPhys(Reg), &TRI) << '\n'); + return RewriteCandidate{Reg, std::move(Defs), &MBB}; +} + +SmallVector AIEEpilogueRegRewriter::collectCandidates( + MachineFunction &MF, MachineRegisterInfo &MRI, + const AIEBaseRegisterInfo &TRI, VirtRegMap &VRM, LiveIntervals &LIS) const { + SmallVector Candidates; + + for (MachineBasicBlock &MBB : MF) { + AIE::LoopOptionOverrides Overrides(MBB); + if (!Overrides.get(EnableOLPWarRename)) + continue; + + SmallSet SeenRegs; + for (MachineInstr &MI : MBB) { + for (const MachineOperand &Def : MI.defs()) { + if (!Def.getReg().isVirtual()) + continue; + + Register Reg = Def.getReg(); + if (!SeenRegs.insert(Reg).second) + continue; + if (auto Candidate = collectCandidate(Reg, MBB, MRI, TRI, VRM, LIS)) + Candidates.push_back(std::move(*Candidate)); + } + } + } + + return Candidates; +} + +MCPhysReg AIEEpilogueRegRewriter::findReplacementPhysReg( + const RewriteCandidate &Candidate, MachineRegisterInfo &MRI, + const AIEBaseRegisterInfo &TRI, VirtRegMap &VRM, LiveIntervals &LIS, + LiveRegMatrix &LRM, BitVector &ReservedRegUnits) const { + // Repair copy slot isn't allocated yet; MBB end is a stable, conservative + // superset of the true [FirstDef, copy] range the new reg will occupy. + const SlotIndex Boundary = LIS.getMBBEndIdx(Candidate.MBB); + const MachineOperand *FirstDefMO = Candidate.Defs.front(); + const SlotIndex FirstDef = LIS.getInstructionIndex(*FirstDefMO->getParent()) + .getRegSlot(FirstDefMO->isEarlyClobber()); + + // Defs live as one range up to the repair copy, and interference only + // consults the main range, so per-def subranges cannot change the result. + LiveInterval ProspectiveLI(Candidate.OldReg, 0.0F); + VNInfo *MainValue = + ProspectiveLI.getNextValue(FirstDef, LIS.getVNInfoAllocator()); + ProspectiveLI.addSegment(LiveRange::Segment(FirstDef, Boundary, MainValue)); + + LRM.invalidateVirtRegs(); + const TargetRegisterClass *RC = MRI.getRegClass(Candidate.OldReg); + const MCPhysReg OldPhys = VRM.getPhys(Candidate.OldReg); + MCPhysReg NewPhys = AIERegAllocationUtils::findFreeNonOverlappingPhysReg( + ProspectiveLI, *RC, RC->getRegisters(), OldPhys, ReservedRegUnits, TRI, + LRM); + LLVM_DEBUG(dbgs() << "Epilogue register rewrite: replacement for " + << printReg(Candidate.OldReg, &TRI) << " (" + << printReg(OldPhys, &TRI) << ") is " + << printReg(NewPhys, &TRI) << '\n'); + return NewPhys; +} + +void AIEEpilogueRegRewriter::commitRewrite( + RewriteCandidate &Candidate, MCPhysReg NewPhys, MachineRegisterInfo &MRI, + const AIEBaseInstrInfo &TII, VirtRegMap &VRM, LiveRegMatrix &LRM, + LiveIntervals &LIS, LiveDebugVariables &DebugVars) const { + Register NewReg = MRI.cloneVirtualRegister(Candidate.OldReg); + VRM.grow(); + for (MachineOperand *Def : Candidate.Defs) + Def->setReg(NewReg); + + MachineInstr *Copy = + BuildMI(*Candidate.MBB, Candidate.MBB->getFirstTerminator(), DebugLoc(), + TII.get(TargetOpcode::COPY), Candidate.OldReg) + .addReg(NewReg) + .getInstr(); + LIS.InsertMachineInstrInMaps(*Copy); + LLVM_DEBUG( + dbgs() << "Epilogue register rewrite: rewrite " + << printReg(Candidate.OldReg, MRI.getTargetRegisterInfo(), 0, &MRI) + << " to " << printReg(NewReg, MRI.getTargetRegisterInfo(), 0, &MRI) + << " in " << Candidate.MBB->getFullName() << '\n'); + + SmallSet RegistersToRepair; + RegistersToRepair.insert(Candidate.OldReg); + AIESuperRegUtils::repairLiveIntervals(RegistersToRepair, VRM, LRM, LIS); + + LiveInterval &NewLI = LIS.createAndComputeVirtRegInterval(NewReg); + LIS.shrinkToUses(&NewLI); + + LRM.invalidateVirtRegs(); +#ifndef NDEBUG + assert(LRM.checkInterference(NewLI, NewPhys) == LiveRegMatrix::IK_Free && + "Prevalidated physical register became unavailable"); +#endif + + VRM.setRequiredPhys(NewReg, NewPhys); + LRM.assign(NewLI, NewPhys); + SmallVector SplitRegs{Candidate.OldReg, NewReg}; + DebugVars.splitRegister(Candidate.OldReg, SplitRegs, LIS); +} + +bool AIEEpilogueRegRewriter::runOnMachineFunction(MachineFunction &MF) { + MachineRegisterInfo &MRI = MF.getRegInfo(); + const auto &TRI = + *static_cast(MRI.getTargetRegisterInfo()); + const auto &TII = + *static_cast(MF.getSubtarget().getInstrInfo()); + VirtRegMap &VRM = getAnalysis().getVRM(); + LiveIntervals &LIS = getAnalysis().getLIS(); + LiveRegMatrix &LRM = getAnalysis().getLRM(); + LiveDebugVariables &DebugVars = + getAnalysis().getLDV(); + + SmallVector Candidates = + collectCandidates(MF, MRI, TRI, VRM, LIS); + if (Candidates.empty()) { + LLVM_DEBUG(dbgs() << "Epilogue register rewrite: no candidates in " + << MF.getName() << '\n'); + return false; + } + + LLVM_DEBUG(dbgs() << "Epilogue register rewrite: " << Candidates.size() + << " candidate(s) in " << MF.getName() << '\n'); + + BitVector ReservedRegUnits(TRI.getNumRegUnits()); + DenseMap SpentBudget; + bool Changed = false; + + for (RewriteCandidate &Candidate : Candidates) { + unsigned &Spent = SpentBudget[Candidate.MBB]; + if (Spent >= EpilogueCopyBudget) { + LLVM_DEBUG(dbgs() << " skip " + << printReg(Candidate.OldReg, &TRI, 0, &MRI) + << ": epilogue copy budget exhausted\n"); + continue; + } + + MCPhysReg NewPhys = findReplacementPhysReg(Candidate, MRI, TRI, VRM, LIS, + LRM, ReservedRegUnits); + if (!NewPhys) { + LLVM_DEBUG(dbgs() << " skip " + << printReg(Candidate.OldReg, &TRI, 0, &MRI) + << ": no free non-overlapping physical register\n"); + continue; + } + + const std::optional CopyCost = + TII.getCopyCost(TRI, VRM.getPhys(Candidate.OldReg), NewPhys); + if (!CopyCost || *CopyCost > EpilogueCopyBudget - Spent) { + LLVM_DEBUG(dbgs() << " skip " + << printReg(Candidate.OldReg, &TRI, 0, &MRI) + << ": copy cost does not fit the remaining budget\n"); + continue; + } + + // Block replaced registers from being selected again. + for (MCRegUnit Unit : TRI.regunits(NewPhys)) + ReservedRegUnits.set(Unit); + Spent += *CopyCost; + commitRewrite(Candidate, NewPhys, MRI, TII, VRM, LRM, LIS, DebugVars); + Changed = true; + } + + return Changed; +} + } // namespace char AIEEpilogueRegRewriter::ID = 0; diff --git a/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir b/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir index 91838feb21af..041fa5a5920e 100644 --- a/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir +++ b/llvm/test/CodeGen/AIE/aie2ps/ra/epilogue-reg-rewrite.mir @@ -68,18 +68,20 @@ body: | ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] ; DEFAULT-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] - ; DEFAULT-NEXT: undef [[DEF9:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; DEFAULT-NEXT: [[DEF9:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; DEFAULT-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; DEFAULT-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; DEFAULT-NEXT: undef [[DEF10:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] - ; DEFAULT-NEXT: [[DEF10:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] - ; DEFAULT-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] - ; DEFAULT-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] + ; DEFAULT-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: undef [[COPY:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] + ; DEFAULT-NEXT: [[COPY:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] + ; DEFAULT-NEXT: [[COPY:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] + ; DEFAULT-NEXT: [[COPY:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] ; DEFAULT-NEXT: undef [[DEF11:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 ; DEFAULT-NEXT: [[DEF11:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 ; DEFAULT-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 ; DEFAULT-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; DEFAULT-NEXT: [[DEF9:%[0-9]+]]:acc2048 = COPY [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm]] + ; DEFAULT-NEXT: [[DEF10:%[0-9]+]]:acc2048 = COPY [[COPY]] ; DEFAULT-NEXT: PseudoJ_jump_imm %bb.3 ; DEFAULT-NEXT: {{ $}} ; DEFAULT-NEXT: bb.3: @@ -118,10 +120,10 @@ body: | ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] ; BUDGET3-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] - ; BUDGET3-NEXT: undef [[DEF9:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET3-NEXT: [[DEF9:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET3-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET3-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 ; BUDGET3-NEXT: undef [[DEF10:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] ; BUDGET3-NEXT: [[DEF10:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] ; BUDGET3-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] @@ -130,6 +132,7 @@ body: | ; BUDGET3-NEXT: [[DEF11:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 ; BUDGET3-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 ; BUDGET3-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET3-NEXT: [[DEF9:%[0-9]+]]:acc2048 = COPY [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm]] ; BUDGET3-NEXT: PseudoJ_jump_imm %bb.3 ; BUDGET3-NEXT: {{ $}} ; BUDGET3-NEXT: bb.3: @@ -168,18 +171,21 @@ body: | ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF9]], [[DEF]], [[DEF1]], [[DEF2]] ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X1:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF10]], [[DEF]], [[DEF1]], [[DEF2]] ; BUDGET8-NEXT: dead [[VMAC_vmac_vmul_cm_core_X_X2:%[0-9]+]]:acc2048 = VMAC_vmac_vmul_cm_core_X_X [[DEF11]], [[DEF]], [[DEF1]], [[DEF2]] - ; BUDGET8-NEXT: undef [[DEF9:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: [[DEF9:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: [[DEF9:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: undef [[DEF10:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] - ; BUDGET8-NEXT: [[DEF10:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] - ; BUDGET8-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] - ; BUDGET8-NEXT: [[DEF10:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] - ; BUDGET8-NEXT: undef [[DEF11:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: [[DEF11:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 - ; BUDGET8-NEXT: [[DEF11:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: undef [[COPY:%[0-9]+]].sub_512_acc_lo:acc2048 = COPY [[DEF5]] + ; BUDGET8-NEXT: [[COPY:%[0-9]+]].sub_512_acc_hi:acc2048 = COPY [[DEF6]] + ; BUDGET8-NEXT: [[COPY:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048 = COPY [[DEF7]] + ; BUDGET8-NEXT: [[COPY:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048 = COPY [[DEF8]] + ; BUDGET8-NEXT: undef [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm1:%[0-9]+]].sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm1:%[0-9]+]].sub_512_acc_hi:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm1:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_lo:acc2048, [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm1:%[0-9]+]].sub_1024_acc_hi_then_sub_512_acc_hi:acc2048, dead [[DEF3:%[0-9]+]]:ep_as_32bit = VLDA_dmx_lda_bm_ld_pstm_nrm_imm [[DEF3]], 64 + ; BUDGET8-NEXT: [[DEF9:%[0-9]+]]:acc2048 = COPY [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm]] + ; BUDGET8-NEXT: [[DEF10:%[0-9]+]]:acc2048 = COPY [[COPY]] + ; BUDGET8-NEXT: [[DEF11:%[0-9]+]]:acc2048 = COPY [[VLDA_dmx_lda_bm_ld_pstm_nrm_imm1]] ; BUDGET8-NEXT: PseudoJ_jump_imm %bb.3 ; BUDGET8-NEXT: {{ $}} ; BUDGET8-NEXT: bb.3: @@ -215,27 +221,29 @@ body: | ; PHYS-NEXT: {{ $}} ; PHYS-NEXT: bb.2: ; PHYS-NEXT: successors: %bb.3(0x80000000) - ; PHYS-NEXT: liveins: $bmll1, $bmll2, $bmll3, $bmll4, $dm0:0x000000030000000C, $dm5:0x000000030000000C, $dm6:0x000000030000000C, $p0, $r8, $x0, $x2 + ; PHYS-NEXT: liveins: $bmll1, $bmll2, $bmll3, $bmll4, $dm0, $dm5, $dm6, $p0, $r8, $x0, $x2 ; PHYS-NEXT: {{ $}} ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, renamable $x0, renamable $x2, renamable $r8 ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm5, renamable $x0, renamable $x2, renamable $r8 ; PHYS-NEXT: dead renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm6, renamable $x0, renamable $x2, renamable $r8 - ; PHYS-NEXT: renamable $bmll0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmlh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmhl0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmhh0, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmll5 = COPY killed renamable $bmll1 - ; PHYS-NEXT: renamable $bmlh5 = COPY killed renamable $bmll2 - ; PHYS-NEXT: renamable $bmhl5 = COPY killed renamable $bmll3 - ; PHYS-NEXT: renamable $bmhh5 = COPY killed renamable $bmll4 - ; PHYS-NEXT: renamable $bmll6, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmlh6, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmhl6, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 - ; PHYS-NEXT: renamable $bmhh6, dead renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmll7, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh7, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl7, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh7, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh1 = COPY killed renamable $bmll2 + ; PHYS-NEXT: renamable $bmhl1 = COPY killed renamable $bmll3 + ; PHYS-NEXT: renamable $bmhh1 = COPY killed renamable $bmll4 + ; PHYS-NEXT: renamable $bmll2, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmlh2, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhl2, renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $bmhh2, dead renamable $p0 = VLDA_dmx_lda_bm_ld_pstm_nrm_imm killed renamable $p0, 64 + ; PHYS-NEXT: renamable $dm0 = COPY killed renamable $dm7 + ; PHYS-NEXT: renamable $dm5 = COPY killed renamable $dm1 + ; PHYS-NEXT: renamable $dm6 = COPY killed renamable $dm2 ; PHYS-NEXT: PseudoJ_jump_imm %bb.3 ; PHYS-NEXT: {{ $}} ; PHYS-NEXT: bb.3: - ; PHYS-NEXT: liveins: $dm0:0x000000030000000C, $dm5:0x000000030000000C, $dm6:0x000000030000000C, $r8, $x0, $x2 + ; PHYS-NEXT: liveins: $dm0, $dm5, $dm6, $r8, $x0, $x2 ; PHYS-NEXT: {{ $}} ; PHYS-NEXT: renamable $dm0 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm0, renamable $x0, renamable $x2, renamable $r8 ; PHYS-NEXT: renamable $dm1 = VMAC_vmac_vmul_cm_core_X_X killed renamable $dm5, renamable $x0, renamable $x2, renamable $r8 From fea4466c11965c7f0f7e903c3ba9a310170f7ab8 Mon Sep 17 00:00:00 2001 From: Fabian Stuckmann Date: Wed, 22 Jul 2026 12:48:04 +0200 Subject: [PATCH 5/5] [AIE2PS] Opt gemm_int8_psum_0 into the epilogue WAR rename --- .../AIE/aie2ps/end-to-end/gemm_int8_psum_0.ll | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/llvm/test/CodeGen/AIE/aie2ps/end-to-end/gemm_int8_psum_0.ll b/llvm/test/CodeGen/AIE/aie2ps/end-to-end/gemm_int8_psum_0.ll index 78dcc852242c..032bc587a84b 100644 --- a/llvm/test/CodeGen/AIE/aie2ps/end-to-end/gemm_int8_psum_0.ll +++ b/llvm/test/CodeGen/AIE/aie2ps/end-to-end/gemm_int8_psum_0.ll @@ -63,7 +63,7 @@ define dso_local void @gemm_int8_psum_0( ; REMARKS-NEXT: - Prologue: bb.1.steady.stage1.top ; REMARKS-NEXT: - PrologueBundles: '12' ; REMARKS-NEXT: - Epilogue: bb.3.steady.stage1.bottom.and.stage0.top -; REMARKS-NEXT: - EpilogueBundles: '22' +; REMARKS-NEXT: - EpilogueBundles: '19' ; REMARKS-NEXT: ... ; REMARKS: --- !Passed ; REMARKS-NEXT: Pass: pipeliner @@ -164,28 +164,25 @@ define dso_local void @gemm_int8_psum_0( ; CHECK-NEXT: vlda.3d x3, [p0], d0; nopb ; nops ; nopx ; vshuffle x1, x9, x0, r2; vmac dm2, dm2, x3, x10, r8 ; CHECK-NEXT: // %bb.3: // %steady.stage1.bottom.and.stage0.top ; CHECK-NEXT: // in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: nopa ; paddb.2d [p3], d3; nops ; nopx ; vshuffle x10, x1, x0, r4; vmac dm1, dm1, x5, x6, r8 -; CHECK-NEXT: nopa ; vldb.128 wl1, [p3, #16]; nops ; nopx ; vshuffle x8, x7, x0, r6; vmac dm0, dm0, x3, x6, r8 -; CHECK-NEXT: vldb.128 wl10, [p3, #0]; vshuffle x6, x8, x0, r16; vmac dm3, dm3, x5, x10, r8 -; CHECK-NEXT: vshuffle x1, x9, x0, r2; vmac dm2, dm2, x3, x10, r8 -; CHECK-NEXT: vldb x3, [p1], m4; vshuffle x10, x1, x0, r4; vmac dm1, dm1, x5, x6, r8 -; CHECK-NEXT: vldb.3d x5, [p1], d1; vshuffle x8, x7, x0, r6; vmac dm0, dm0, x3, x6, r8 -; CHECK-NEXT: vlda bmll3, [p4], #64; vldb x6, [p0], #64; vshuffle x6, x8, x0, r16; vmac dm3, dm3, x5, x10, r8 -; CHECK-NEXT: vlda bmlh3, [p4], #64; vldb.3d x8, [p0], d0; mov srssign0, r18; vmac dm2, dm2, x3, x10, r8 -; CHECK-NEXT: vlda bmhl3, [p4], #64; vsel.32 x2, x2, x1, r30; vmac dm1, dm1, x5, x6, r8 -; CHECK-NEXT: vlda bmhh3, [p4], #64; vsel.32 x4, x4, x10, r30; vmac dm0, dm0, x3, x6, r8 -; CHECK-NEXT: vlda bmll2, [p4], #64 -; CHECK-NEXT: vlda bmlh2, [p4], #64; vshuffle x10, x3, x0, r22 -; CHECK-NEXT: vlda bmhl2, [p4], #64; vst.srs.4x dm3, s0, srssign0, [p2], #64; vshuffle x1, x5, x0, r24 -; CHECK-NEXT: vlda bmhh2, [p4], #64; vst.srs.4x dm2, s0, srssign0, [p2], m5; vshuffle x10, x10, x0, r20 -; CHECK-NEXT: vlda bmll1, [p4], #64; vst.srs.4x dm1, s0, srssign0, [p2], #64; vshuffle x1, x1, x0, r26 -; CHECK-NEXT: vlda bmlh1, [p4], #64; vst.2d.srs.4x dm0, s0, srssign0, [p2], d2; movx srssign0, #0 -; CHECK-NEXT: vlda bmhl1, [p4], #64; jnzd r1, r1, p5 -; CHECK-NEXT: vlda bmhh1, [p4], #64 // Delay Slot 5 -; CHECK-NEXT: vlda bmll0, [p4], #64 // Delay Slot 4 -; CHECK-NEXT: vlda bmlh0, [p4], #64 // Delay Slot 3 -; CHECK-NEXT: vlda bmhl0, [p4], #64 // Delay Slot 2 -; CHECK-NEXT: vlda bmhh0, [p4], #64 // Delay Slot 1 +; CHECK-NEXT: vlda bmll4, [p4], #64; paddb.2d [p3], d3; nops ; nopx ; vshuffle x10, x1, x0, r4; vmac dm1, dm1, x5, x6, r8 +; CHECK-NEXT: vlda bmlh4, [p4], #64; vldb.128 wl1, [p3, #16]; nops ; nopx ; vshuffle x8, x7, x0, r6; vmac dm0, dm0, x3, x6, r8 +; CHECK-NEXT: vlda bmhl4, [p4], #64; vldb.128 wl10, [p3, #0]; nopx ; vshuffle x6, x8, x0, r16; vmac dm3, dm3, x5, x10, r8 +; CHECK-NEXT: vlda bmhh4, [p4], #64; vshuffle x1, x9, x0, r2; vmac dm2, dm2, x3, x10, r8 +; CHECK-NEXT: vlda bmll5, [p4], #64; vldb x3, [p1], m4; vshuffle x10, x1, x0, r4; vmac dm1, dm1, x5, x6, r8 +; CHECK-NEXT: vlda bmlh5, [p4], #64; vldb.3d x5, [p1], d1; vshuffle x8, x7, x0, r6; vmac dm0, dm0, x3, x6, r8 +; CHECK-NEXT: vlda bmhl5, [p4], #64; vldb x6, [p0], #64; vshuffle x6, x8, x0, r16; vmac dm3, dm3, x5, x10, r8 +; CHECK-NEXT: vlda bmhh5, [p4], #64; vldb.3d x8, [p0], d0; mov srssign0, r18; vmac dm2, dm2, x3, x10, r8 +; CHECK-NEXT: vlda bmll1, [p4], #64; vsel.32 x2, x2, x1, r30; vmac dm1, dm1, x5, x6, r8 +; CHECK-NEXT: vlda bmlh1, [p4], #64; vsel.32 x4, x4, x10, r30; vmac dm0, dm0, x3, x6, r8 +; CHECK-NEXT: vlda bmhl1, [p4], #64 +; CHECK-NEXT: vlda bmhh1, [p4], #64; vshuffle x10, x3, x0, r22 +; CHECK-NEXT: vlda bmll0, [p4], #64; vst.srs.4x dm3, s0, srssign0, [p2], #64; vshuffle x1, x5, x0, r24 +; CHECK-NEXT: vlda bmlh0, [p4], #64; vst.srs.4x dm2, s0, srssign0, [p2], m5; jnzd r1, r1, p5; vshuffle x10, x10, x0, r20 +; CHECK-NEXT: vlda bmhl0, [p4], #64; vst.srs.4x dm1, s0, srssign0, [p2], #64; vshuffle x1, x1, x0, r26 // Delay Slot 5 +; CHECK-NEXT: vlda bmhh0, [p4], #64; vst.2d.srs.4x dm0, s0, srssign0, [p2], d2; movx srssign0, #0; vmov cml3, cml4 // Delay Slot 4 +; CHECK-NEXT: vmov cmh3, cmh4 // Delay Slot 3 +; CHECK-NEXT: vmov cml2, cml5 // Delay Slot 2 +; CHECK-NEXT: vmov cmh2, cmh5 // Delay Slot 1 ; CHECK-NEXT: // %bb.4: // %lastiter.stage1.top ; CHECK-NEXT: vldb x3, [p1], m4; vmul dm4, x0, x4, r12 ; CHECK-NEXT: vlda.3d x1, [p1], d1 @@ -810,7 +807,7 @@ attributes #3 = { nocallback nofree nosync nounwind willreturn memory(inaccessib !201 = distinct !{!201, !"_ZN3aie6detail11load_vectorILj16EL15aie_dm_resource0EiEEDaPKT1_"} !202 = distinct !{!202, !203, !"_ZN3aie6load_vILj16EL15aie_dm_resource0ETkNS_21DecoratedElemBaseTypeEiEENS_6vectorIN22aie_dm_resource_removeIT1_E4typeEXT_EEEPKS4_: %agg.result"} !203 = distinct !{!203, !"_ZN3aie6load_vILj16EL15aie_dm_resource0ETkNS_21DecoratedElemBaseTypeEiEENS_6vectorIN22aie_dm_resource_removeIT1_E4typeEXT_EEEPKS4_"} -!204 = distinct !{!204, !205, !206} +!204 = distinct !{!204, !205, !206, !237} !205 = !{!"llvm.loop.mustprogress"} !206 = !{!"llvm.loop.itercount.range", i64 2} !207 = !{!208, !210, !212} @@ -843,3 +840,4 @@ attributes #3 = { nocallback nofree nosync nounwind willreturn memory(inaccessib !234 = distinct !{!234, !"_ZN3aie6load_vILj64EL15aie_dm_resource0ETkNS_21DecoratedElemBaseTypeEU3AS5aEENS_6vectorIN22aie_dm_resource_removeIT1_E4typeEXT_EEEPKS5_"} !235 = distinct !{!235, !205, !236} !236 = !{!"llvm.loop.itercount.range", i64 4} +!237 = !{!"llvm.loop.hint.aie-olp-war-rename", i64 1}