[AIEX][OLP] resolve WAR dependencies

llvm/lib/Target/AIE/AIE.h

@@ -60,6 +60,7 @@ MachineFunctionPass *createAIEEliminateDuplicatePHI();
 FunctionPass *createAIEOutlineMemoryGEP();
 FunctionPass *createAIESuperRegRewriter();
 FunctionPass *createAIEWawRegRewriter();
+FunctionPass *createAIEEpilogueRegRewriter();
 FunctionPass *createAIEUnallocatedSuperRegRewriter();
 FunctionPass *createAIESpillSlotOptimization();
 FunctionPass *createAIEPostSelectOptimize();
@@ -88,6 +89,8 @@ extern char &AIESuperRegRewriterID;
 void initializeAIESuperRegRewriterPass(PassRegistry &);
 extern char &AIEWawRegRewriterID;
 void initializeAIEWawRegRewriterPass(PassRegistry &);
+extern char &AIEEpilogueRegRewriterID;
+void initializeAIEEpilogueRegRewriterPass(PassRegistry &);
 extern char &AIEUnallocatedSuperRegRewriterID;
 void initializeAIEUnallocatedSuperRegRewriterPass(PassRegistry &);
 extern char &AIESpillSlotOptimizationID;

llvm/lib/Target/AIE/AIE2InstrInfo.cpp

@@ -488,85 +488,92 @@ void AIE2InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MBBI,
                                 const DebugLoc &DL, MCRegister DstReg,
                                 MCRegister SrcReg, bool KillSrc,
-                                bool RenamableDest, bool RenamableSrc) const {
-  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
-  const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+                                bool /* RenamableDest */,
+                                bool /* RenamableSrc */) const {
+  const TargetRegisterInfo &TRI =
+      *MBB.getParent()->getRegInfo().getTargetRegisterInfo();
+  CopyMaterializer M(*this, TRI, MBB, MBBI, DL);
+  if (!materializeCopy(M, DstReg, SrcReg, KillSrc))
+    llvm_unreachable("unhandled case in copyPhysReg");
+}
+
+bool AIE2InstrInfo::materializeCopy(CopyMaterializer &M, MCRegister DstReg,
+                                    MCRegister SrcReg, bool KillSrc) const {
+  const TargetRegisterInfo &TRI = M.getRegisterInfo();
 
   // TODO : add support for 128-bit mask register
   if (AIE2::mMvSclSrcRegClass.contains(SrcReg) &&
       AIE2::mMvSclDstRegClass.contains(DstReg)) {
     const unsigned MOVSclOpcode = getScalarMovOpcode(DstReg, SrcReg);
-    BuildMI(MBB, MBBI, DL, get(MOVSclOpcode), DstReg)
+    M.buildInstr(get(MOVSclOpcode), DstReg)
         .addReg(SrcReg, getKillRegState(KillSrc));
   } else if ((AIE2::eLRegClass.contains(SrcReg)) &&
              (AIE2::eLRegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::MOV_mv_scl),
-            TRI.getSubReg(DstReg, AIE2::sub_l_even))
+    M.buildInstr(get(AIE2::MOV_mv_scl), TRI.getSubReg(DstReg, AIE2::sub_l_even))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_l_even),
                 getKillRegState(KillSrc));
-    BuildMI(MBB, MBBI, DL, get(AIE2::MOV_mv_scl),
-            TRI.getSubReg(DstReg, AIE2::sub_l_odd))
+    M.buildInstr(get(AIE2::MOV_mv_scl), TRI.getSubReg(DstReg, AIE2::sub_l_odd))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_l_odd),
                 getKillRegState(KillSrc));
   } else if ((AIE2::eDRegClass.contains(SrcReg)) &&
              (AIE2::eDRegClass.contains(DstReg))) {
-    copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc);
+    if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc))
+      return false;
   } else if ((AIE2::eDSRegClass.contains(SrcReg)) &&
              (AIE2::eDSRegClass.contains(DstReg))) {
-    copyThroughSubRegs(MBB, MBBI, DL, DstReg, SrcReg, KillSrc);
+    if (!copyThroughSubRegs(M, DstReg, SrcReg, KillSrc))
+      return false;
   } else if ((AIE2::VEC128RegClass.contains(SrcReg) ||
               AIE2::VEC256RegClass.contains(SrcReg) ||
               AIE2::ACC256RegClass.contains(SrcReg)) &&
              (AIE2::VEC128RegClass.contains(DstReg) ||
               AIE2::VEC256RegClass.contains(DstReg) ||
               AIE2::ACC256RegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_w), DstReg)
+    M.buildInstr(get(AIE2::VMOV_mv_w), DstReg)
         .addReg(SrcReg, getKillRegState(KillSrc));
   } else if ((AIE2::VEC512RegClass.contains(SrcReg) ||
               AIE2::ACC512RegClass.contains(SrcReg)) &&
              (AIE2::VEC512RegClass.contains(DstReg) ||
               AIE2::ACC512RegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x), DstReg)
+    M.buildInstr(get(AIE2::VMOV_mv_x), DstReg)
         .addReg(SrcReg, getKillRegState(KillSrc));
   } else if ((AIE2::VEC1024RegClass.contains(SrcReg)) &&
              (AIE2::VEC1024RegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x),
-            TRI.getSubReg(DstReg, AIE2::sub_512_lo))
+    M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_lo))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_lo),
                 getKillRegState(KillSrc));
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x),
-            TRI.getSubReg(DstReg, AIE2::sub_512_hi))
+    M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_hi))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_hi),
                 getKillRegState(KillSrc));
   } else if ((AIE2::ACC1024RegClass.contains(SrcReg)) &&
              (AIE2::ACC1024RegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_cm), DstReg)
+    M.buildInstr(get(AIE2::VMOV_mv_cm), DstReg)
         .addReg(SrcReg, getKillRegState(KillSrc));
   } else if ((AIE2::VEC1024RegClass.contains(SrcReg) ||
               AIE2::ACC1024RegClass.contains(SrcReg)) &&
              (AIE2::VEC1024RegClass.contains(DstReg) ||
               AIE2::ACC1024RegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x),
-            TRI.getSubReg(DstReg, AIE2::sub_512_lo))
+    M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_lo))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_lo),
                 getKillRegState(KillSrc));
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x),
-            TRI.getSubReg(DstReg, AIE2::sub_512_hi))
+    M.buildInstr(get(AIE2::VMOV_mv_x), TRI.getSubReg(DstReg, AIE2::sub_512_hi))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_512_hi),
                 getKillRegState(KillSrc));
   } else if ((AIE2::SPARSEVEC640RegClass.contains(SrcReg)) &&
              (AIE2::SPARSEVEC640RegClass.contains(DstReg))) {
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_x),
-            TRI.getSubReg(DstReg, AIE2::sub_sparse_x))
+    M.buildInstr(get(AIE2::VMOV_mv_x),
+                 TRI.getSubReg(DstReg, AIE2::sub_sparse_x))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_sparse_x),
                 getKillRegState(KillSrc));
-    BuildMI(MBB, MBBI, DL, get(AIE2::VMOV_mv_w),
-            TRI.getSubReg(DstReg, AIE2::sub_sparse_q))
+    M.buildInstr(get(AIE2::VMOV_mv_w),
+                 TRI.getSubReg(DstReg, AIE2::sub_sparse_q))
         .addReg(TRI.getSubReg(SrcReg, AIE2::sub_sparse_q),
                 getKillRegState(KillSrc));
   } else {
-    llvm_unreachable("unhandled case in copyPhysReg");
+    return false;
   }
+
+  return true;
 }
 
 // Some AIE instructions like Load/Stores take compound register classes

llvm/lib/Target/AIE/AIE2InstrInfo.h

@@ -160,6 +160,11 @@ class AIE2InstrInfo : public AIE2GenInstrInfo {
                    bool KillSrc, bool RenamableDest = false,
                    bool RenamableSrc = false) const override;
 
+protected:
+  bool materializeCopy(CopyMaterializer &M, MCRegister DstReg,
+                       MCRegister SrcReg, bool KillSrc) const override;
+
+public:
   void storeRegToStackSlot(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MBBI, Register SrcReg,
                            bool IsKill, int FrameIndex,

llvm/lib/Target/AIE/AIE2TargetMachine.cpp

@@ -53,6 +53,7 @@ extern cl::opt<bool> EnableSuperRegSplitting;
 extern cl::opt<bool> AllocateMRegsFirst;
 extern cl::opt<bool> EnablePreMISchedCoalescer;
 extern cl::opt<bool> EnableWAWRegRewrite;
+extern cl::opt<bool> EnableEpilogueRegRewrite;
 extern cl::opt<bool> EnableAIEIfConversion;
 
 extern bool AIEDumpArtifacts;
@@ -193,10 +194,12 @@ bool AIE2PassConfig::addRegAssignAndRewriteOptimized() {
     addPass(createAIESuperRegRewriter());
   }
   addPass(createGreedyRegisterAllocator());
-  if (EnableWAWRegRewrite) {
+  if (EnableWAWRegRewrite)
     addPass(createAIEWawRegRewriter());
+  if (EnableEpilogueRegRewrite)
+    addPass(createAIEEpilogueRegRewriter());
+  if (EnableWAWRegRewrite || EnableEpilogueRegRewrite)
     addPass(createGreedyRegisterAllocator());
-  }
   addPass(createVirtRegRewriter());
 
   return true;

llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp

@@ -811,22 +811,71 @@ static void collectSubRegs(MCRegister Reg, SmallSet<MCRegister, 8> &SubRegs,
   }
 }
 
-void AIEBaseInstrInfo::copyThroughSubRegs(MachineBasicBlock &MBB,
-                                          MachineBasicBlock::iterator MBBI,
-                                          const DebugLoc &DL, MCRegister DstReg,
-                                          MCRegister SrcReg,
+bool AIEBaseInstrInfo::copyThroughSubRegs(CopyMaterializer &M,
+                                          MCRegister DstReg, MCRegister SrcReg,
                                           bool KillSrc) const {
-  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
-  const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+  const TargetRegisterInfo &TRI = M.getRegisterInfo();
 
   SmallSet<MCRegister, 8> SrcSubRegs;
   collectSubRegs(SrcReg, SrcSubRegs, TRI);
 
   for (MCRegister SrcSubReg : SrcSubRegs) {
     unsigned SubRegIdx = TRI.getSubRegIndex(SrcReg, SrcSubReg);
     MCRegister DstSubReg = TRI.getSubReg(DstReg, SubRegIdx);
-    copyPhysReg(MBB, MBBI, DL, DstSubReg, SrcSubReg, KillSrc);
+    if (!M.copy(DstSubReg, SrcSubReg, KillSrc))
+      return false;
   }
+  return true;
+}
+
+AIEBaseInstrInfo::CopyInstrBuilder &
+AIEBaseInstrInfo::CopyInstrBuilder::addReg(Register Reg, unsigned Flags) {
+  if (MI)
+    MachineInstrBuilder(*MF, MI).addReg(Reg, Flags);
+  return *this;
+}
+
+AIEBaseInstrInfo::CopyInstrBuilder &
+AIEBaseInstrInfo::CopyInstrBuilder::addImm(int64_t Imm) {
+  if (MI)
+    MachineInstrBuilder(*MF, MI).addImm(Imm);
+  return *this;
+}
+
+AIEBaseInstrInfo::CopyMaterializer::CopyMaterializer(
+    const AIEBaseInstrInfo &TII, const TargetRegisterInfo &TRI,
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    const DebugLoc &DL)
+    : TII(TII), TRI(TRI), InsertPt(InsertionPoint{MBB, MBBI, DL}) {}
+
+AIEBaseInstrInfo::CopyMaterializer::CopyMaterializer(
+    const AIEBaseInstrInfo &TII, const TargetRegisterInfo &TRI)
+    : TII(TII), TRI(TRI) {}
+
+AIEBaseInstrInfo::CopyInstrBuilder
+AIEBaseInstrInfo::CopyMaterializer::buildInstr(const MCInstrDesc &Desc,
+                                               Register DstReg) {
+  ++NumInstructions;
+  if (!InsertPt)
+    return CopyInstrBuilder();
+  return CopyInstrBuilder(
+      InsertPt->MBB.getParent(),
+      BuildMI(InsertPt->MBB, InsertPt->MBBI, InsertPt->DL, Desc, DstReg)
+          .getInstr());
+}
+
+bool AIEBaseInstrInfo::CopyMaterializer::copy(MCRegister DstReg,
+                                              MCRegister SrcReg, bool KillSrc) {
+  return TII.materializeCopy(*this, DstReg, SrcReg, KillSrc);
+}
+
+std::optional<unsigned>
+AIEBaseInstrInfo::getCopyCost(const TargetRegisterInfo &TRI, MCRegister DstReg,
+                              MCRegister SrcReg) const {
+  CopyMaterializer M(*this, TRI);
+  if (!M.copy(DstReg, SrcReg, false))
+    return std::nullopt;
+  return M.getNumInstructions();
 }
 
 static bool isPreRA(const MachineFunction &MF) {

llvm/lib/Target/AIE/AIEBaseInstrInfo.h

@@ -1023,6 +1023,54 @@ struct AIEBaseInstrInfo : public TargetInstrInfo {
   }
 
 public:
+  class CopyInstrBuilder {
+  public:
+    CopyInstrBuilder(MachineFunction *MF = nullptr, MachineInstr *MI = nullptr)
+        : MF(MF), MI(MI) {}
+
+    CopyInstrBuilder &addReg(Register Reg, unsigned Flags = 0);
+    CopyInstrBuilder &addImm(int64_t Imm);
+
+  private:
+    MachineFunction *MF;
+    MachineInstr *MI;
+  };
+
+  /// Materializes physical-register copies either into a machine basic block
+  /// or into an instruction count. Target copy lowering uses this common
+  /// interface so both operations exercise the same implementation.
+  class CopyMaterializer {
+  public:
+    CopyMaterializer(const AIEBaseInstrInfo &TII, const TargetRegisterInfo &TRI,
+                     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                     const DebugLoc &DL);
+    CopyMaterializer(const AIEBaseInstrInfo &TII,
+                     const TargetRegisterInfo &TRI);
+
+    CopyInstrBuilder buildInstr(const MCInstrDesc &Desc, Register DstReg);
+    bool copy(MCRegister DstReg, MCRegister SrcReg, bool KillSrc);
+    unsigned getNumInstructions() const { return NumInstructions; }
+    const TargetRegisterInfo &getRegisterInfo() const { return TRI; }
+
+  private:
+    struct InsertionPoint {
+      MachineBasicBlock &MBB;
+      MachineBasicBlock::iterator MBBI;
+      const DebugLoc &DL;
+    };
+
+    const AIEBaseInstrInfo &TII;
+    const TargetRegisterInfo &TRI;
+    std::optional<InsertionPoint> InsertPt;
+    unsigned NumInstructions = 0;
+  };
+
+  /// Return the number of instructions materialized for an exact physical
+  /// register copy, or std::nullopt if the copy is unsupported.
+  std::optional<unsigned> getCopyCost(const TargetRegisterInfo &TRI,
+                                      MCRegister DstReg,
+                                      MCRegister SrcReg) const;
+
   /// Expand a spill pseudo-instruction into actual target instructions. This
   /// will essentially split the register being handled into its sub-registers,
   /// until there is an actual instruction that can handle them.
@@ -1077,6 +1125,11 @@ struct AIEBaseInstrInfo : public TargetInstrInfo {
   };
 
 protected:
+  virtual bool materializeCopy(CopyMaterializer &M, MCRegister DstReg,
+                               MCRegister SrcReg, bool KillSrc) const {
+    return false;
+  };
+
   struct AIERegOffsetSpillInstrInfo {
     /// Opcode for spill using register offset.
     unsigned SpillOpCode;
@@ -1099,10 +1152,8 @@ struct AIEBaseInstrInfo : public TargetInstrInfo {
   }
 
   // Copy SrcReg to DstReg through their sub-registers.
-  void copyThroughSubRegs(MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
-                          MCRegister DstReg, MCRegister SrcReg,
-                          bool KillSrc) const;
+  bool copyThroughSubRegs(CopyMaterializer &M, MCRegister DstReg,
+                          MCRegister SrcReg, bool KillSrc) const;
 
 #if 0
   // TODO. I guess this should wait for Davy's PR to land

llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp

@@ -111,6 +111,11 @@ cl::opt<bool>
                         cl::desc("Enable the WAW Register Renaming in loops"),
                         cl::init(true), cl::Hidden);
 
+cl::opt<bool> EnableEpilogueRegRewrite(
+    "aie-enable-epilogue-reg-rewrite",
+    cl::desc("Enable epilogue register renaming after allocation"),
+    cl::init(true), cl::Hidden);
+
 cl::opt<bool>
     EnableSuperRegSplitting("aie-split-superregs", cl::Hidden, cl::init(true),
                             cl::desc("Enable splitting super-regs into their "
@@ -165,6 +170,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAIETarget() {
   initializeAIESuperRegRewriterPass(*PR);
   initializeAIEUnallocatedSuperRegRewriterPass(*PR);
   initializeAIEWawRegRewriterPass(*PR);
+  initializeAIEEpilogueRegRewriterPass(*PR);
   initializeAIEOutlineMemoryGEPPass(*PR);
   initializeAIEFinalizeBundlePass(*PR);
   initializeAIEMachineAlignmentPass(*PR);