From e126c8d4dda7404a21a56964f4715df5c1036ea2 Mon Sep 17 00:00:00 2001 From: Marc Ludevid Wulf Date: Wed, 8 Jul 2026 06:48:00 -0600 Subject: [PATCH 1/4] [AIE][NFC] Drop dead SWP epilogue trip-count plumbing --- llvm/lib/Target/AIE/AIEInterBlockScheduling.cpp | 12 ++++++------ llvm/lib/Target/AIE/AIEInterBlockScheduling.h | 14 +++++--------- llvm/lib/Target/AIE/AIEMachineScheduler.cpp | 6 +++--- llvm/lib/Target/AIE/AIEPostPipeliner.cpp | 5 ----- llvm/lib/Target/AIE/AIEPostPipeliner.h | 2 -- 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Target/AIE/AIEInterBlockScheduling.cpp b/llvm/lib/Target/AIE/AIEInterBlockScheduling.cpp index c3a939a8bfd0..86c78905d934 100644 --- a/llvm/lib/Target/AIE/AIEInterBlockScheduling.cpp +++ b/llvm/lib/Target/AIE/AIEInterBlockScheduling.cpp @@ -1571,20 +1571,20 @@ void BlockState::initInterBlock(const MachineSchedContext &Context, } } -std::optional -InterBlockScheduling::getSWPEpilogueContext(MachineBasicBlock *MBB) { +std::optional> +InterBlockScheduling::getSWPLoopBundlesForEpilogue( + MachineBasicBlock *Epilogue) { - BlockState &BS = getBlockState(MBB); + BlockState &BS = getBlockState(Epilogue); if (BS.Kind != BlockType::Epilogue) return std::nullopt; - BlockState &LoopBS = getBlockState(*MBB->pred_begin()); + BlockState &LoopBS = getBlockState(*Epilogue->pred_begin()); if (!LoopBS.isPipelined()) return std::nullopt; - return SWPEpilogueContext{LoopBS.getTop().Bundles, - LoopBS.getPostSWP().getFinalMinTripCount()}; + return LoopBS.getTop().Bundles; } } // namespace llvm::AIE diff --git a/llvm/lib/Target/AIE/AIEInterBlockScheduling.h b/llvm/lib/Target/AIE/AIEInterBlockScheduling.h index 6efcf74591dc..72b0d044342a 100644 --- a/llvm/lib/Target/AIE/AIEInterBlockScheduling.h +++ b/llvm/lib/Target/AIE/AIEInterBlockScheduling.h @@ -146,13 +146,6 @@ class Region { std::vector Bundles; }; -// Struct used do give a preceding execution context to an epilogue -// of a SWP loop. -struct SWPEpilogueContext { - ArrayRef Loop; - int LoopCount; -}; - class BlockState { /// This vector is created during the first fixpoint iteration, triggered /// by the enterRegion callback @@ -424,8 +417,11 @@ class InterBlockScheduling { const MachineSchedContext *getContext() const { return Context; } bool isGatheringPhase() const { return IsGatheringPhase; } - std::optional - getSWPEpilogueContext(MachineBasicBlock *MBB); + // Returns the scheduled bundles of the pipelined loop body preceding + // \p Epilogue. Returns nullopt if \p Epilogue is not the epilogue of a + // pipelined loop. + std::optional> + getSWPLoopBundlesForEpilogue(MachineBasicBlock *Epilogue); /// If \p LoopMBB is not the only Predecessor of \p CurrentMBB, create a /// dedicated Exit MBB by splitting the edge between LoopMBB and CurrentBB diff --git a/llvm/lib/Target/AIE/AIEMachineScheduler.cpp b/llvm/lib/Target/AIE/AIEMachineScheduler.cpp index 535ff94f8779..6a065cb003cd 100644 --- a/llvm/lib/Target/AIE/AIEMachineScheduler.cpp +++ b/llvm/lib/Target/AIE/AIEMachineScheduler.cpp @@ -414,9 +414,9 @@ void AIEPostRASchedStrategy::initializeTopScoreBoard() { if (!IsFirstRegion) return; - auto EpilogueContextOpt = InterBlock.getSWPEpilogueContext(CurMBB); + auto LoopBundlesOpt = InterBlock.getSWPLoopBundlesForEpilogue(CurMBB); - if (!EpilogueContextOpt) + if (!LoopBundlesOpt) return; AIEHazardRecognizer *TopHazardRec = getAIEHazardRecognizer(Top); @@ -425,7 +425,7 @@ void AIEPostRASchedStrategy::initializeTopScoreBoard() { }; const unsigned ConflictHorizon = TopHazardRec->getConflictHorizon(); - ArrayRef LoopBundles = EpilogueContextOpt->Loop; + ArrayRef LoopBundles = *LoopBundlesOpt; const unsigned LoopSize = LoopBundles.size(); // ceil(LoopSize / ConflictHorizon) diff --git a/llvm/lib/Target/AIE/AIEPostPipeliner.cpp b/llvm/lib/Target/AIE/AIEPostPipeliner.cpp index 7b4d604465ef..02a88ff0b289 100644 --- a/llvm/lib/Target/AIE/AIEPostPipeliner.cpp +++ b/llvm/lib/Target/AIE/AIEPostPipeliner.cpp @@ -1760,11 +1760,6 @@ void PostPipeliner::updateTripCount() const { TII->adjustTripCount(*TripCountDef, -Delta); } -int PostPipeliner::getFinalMinTripCount() const { - const int Delta = NStages - 1; - return MinTripCount - Delta; -} - void PostPipeliner::materializePipeline(PipelineScheduleVisitor &Visitor) { // A schedule NS=N, II=L is compatible with NS=1, II=N*L. // In general, we can take any divisor of N. diff --git a/llvm/lib/Target/AIE/AIEPostPipeliner.h b/llvm/lib/Target/AIE/AIEPostPipeliner.h index 2522b034909b..75d81e6ed723 100644 --- a/llvm/lib/Target/AIE/AIEPostPipeliner.h +++ b/llvm/lib/Target/AIE/AIEPostPipeliner.h @@ -376,8 +376,6 @@ class PostPipeliner { void materializePipeline(PipelineScheduleVisitor &Visitor); - int getFinalMinTripCount() const; - void dump() const; }; From 09069a4b56d765612242086a941f8fb71e9988d7 Mon Sep 17 00:00:00 2001 From: Marc Ludevid Wulf Date: Wed, 8 Jul 2026 08:55:02 -0600 Subject: [PATCH 2/4] [AIE2][AIE2P][AIE2PS] Add loop-version-threshold intrinsics and pseudos --- llvm/include/llvm/IR/IntrinsicsAIE2.td | 10 +++++++- llvm/include/llvm/IR/IntrinsicsAIE2P.td | 6 +++++ llvm/include/llvm/IR/IntrinsicsAIE2PS.td | 6 +++++ llvm/lib/Target/AIE/AIE2InstrInfo.cpp | 4 ++++ llvm/lib/Target/AIE/AIE2InstrInfo.h | 1 + .../Target/AIE/AIE2InstructionSelector.cpp | 2 ++ .../AIE/AIE2MultiSlotPseudoInstrInfo.td | 14 ++++++++++- llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp | 20 ++++++++++++++++ llvm/lib/Target/AIE/AIEBaseInstrInfo.h | 16 +++++++++++++ .../Target/AIE/AIEBaseInstructionSelector.cpp | 12 ++++++++++ .../Target/AIE/AIEBaseInstructionSelector.h | 2 ++ llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp | 4 ++++ llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h | 1 + .../AIE/aie2p/AIE2PInstructionSelector.cpp | 2 ++ .../aie2p/AIE2PMultiSlotPseudoInstrInfo.td | 10 +++++++- .../lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp | 4 ++++ llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h | 1 + .../AIE/aie2ps/AIE2PSInstructionSelector.cpp | 2 ++ .../aie2ps/AIE2PSMultiSlotPseudoInstrInfo.td | 8 +++++++ .../AIE/aie2/loop-version-threshold.ll | 23 +++++++++++++++++++ .../AIE/aie2p/loop-version-threshold.ll | 23 +++++++++++++++++++ .../AIE/aie2ps/loop-version-threshold.ll | 23 +++++++++++++++++++ 22 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/AIE/aie2/loop-version-threshold.ll create mode 100644 llvm/test/CodeGen/AIE/aie2p/loop-version-threshold.ll create mode 100644 llvm/test/CodeGen/AIE/aie2ps/loop-version-threshold.ll diff --git a/llvm/include/llvm/IR/IntrinsicsAIE2.td b/llvm/include/llvm/IR/IntrinsicsAIE2.td index 602b764ef9d9..c64d383e907e 100644 --- a/llvm/include/llvm/IR/IntrinsicsAIE2.td +++ b/llvm/include/llvm/IR/IntrinsicsAIE2.td @@ -4,7 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// (c) Copyright 2023-2025 Advanced Micro Devices, Inc. or its affiliates +// (c) Copyright 2023-2026 Advanced Micro Devices, Inc. or its affiliates // //===----------------------------------------------------------------------===// // @@ -967,4 +967,12 @@ def int_aie2_extract_I128_I512 : ClangBuiltin<"__builtin_aiev2_extract_I128_I512 // Set Sub-Vector def int_aie2_set_I512_I128 : ClangBuiltin<"__builtin_aiev2_set_I512_I128">, AIEV2Set_I512_I128; +// Loop-versioning trip-count guard threshold used to pick the pipelined or +// original copy of a versioned loop. The immediate is a placeholder, patched +// by the post-pipeliner once the required minimum trip count is known. +def int_aie2_loop_version_threshold + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], + [IntrNoMem, IntrHasSideEffects, IntrNoDuplicate, + ImmArg>]>; + } // TargetPrefix = "aie2" diff --git a/llvm/include/llvm/IR/IntrinsicsAIE2P.td b/llvm/include/llvm/IR/IntrinsicsAIE2P.td index 43c74f979a56..457fc404c1fe 100644 --- a/llvm/include/llvm/IR/IntrinsicsAIE2P.td +++ b/llvm/include/llvm/IR/IntrinsicsAIE2P.td @@ -748,4 +748,10 @@ def int_aie2p_fifo_ld_pop_3d_unaligned: AIE2PFIFO_LD_POPUnaligned3D; def int_aie2p_fifo_ld_pop_576_3d_bfp16: AIE2PFIFO_LD_POP_576_BFP163D; def int_aie2p_fifo_ld_pop_544_3d_bfp16: AIE2PFIFO_LD_POP_576_BFP163D; +// Loop-versioning trip-count guard threshold. See IntrinsicsAIE2.td. +def int_aie2p_loop_version_threshold + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], + [IntrNoMem, IntrHasSideEffects, IntrNoDuplicate, + ImmArg>]>; + } // TargetPrefix = "aie2p" diff --git a/llvm/include/llvm/IR/IntrinsicsAIE2PS.td b/llvm/include/llvm/IR/IntrinsicsAIE2PS.td index c2d3bbce300e..51dc0add66e8 100644 --- a/llvm/include/llvm/IR/IntrinsicsAIE2PS.td +++ b/llvm/include/llvm/IR/IntrinsicsAIE2PS.td @@ -1043,4 +1043,10 @@ def int_aie2ps_fifo_st_flush_2d_conv: AIE2PFIFO_ST_FLUSH_2D; def int_aie2ps_fifo_st_flush_3d: AIE2PFIFO_ST_FLUSH_3D; def int_aie2ps_fifo_st_flush_3d_conv: AIE2PFIFO_ST_FLUSH_3D; +// Loop-versioning trip-count guard threshold. See IntrinsicsAIE2.td. +def int_aie2ps_loop_version_threshold + : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], + [IntrNoMem, IntrHasSideEffects, IntrNoDuplicate, + ImmArg>]>; + } // TargetPrefix = "aie2ps" diff --git a/llvm/lib/Target/AIE/AIE2InstrInfo.cpp b/llvm/lib/Target/AIE/AIE2InstrInfo.cpp index 539e294775a7..c78dbf7a913f 100644 --- a/llvm/lib/Target/AIE/AIE2InstrInfo.cpp +++ b/llvm/lib/Target/AIE/AIE2InstrInfo.cpp @@ -1415,6 +1415,10 @@ AIE2InstrInfo::getZOLSupport() const { return Result; } +std::optional AIE2InstrInfo::getLoopVersionThresholdOpcode() const { + return AIE2::PseudoLoopVersionThreshold; +} + std::optional AIE2InstrInfo::getJNZDSupport() const { AIEBaseInstrInfo::JNZDSupport Result; diff --git a/llvm/lib/Target/AIE/AIE2InstrInfo.h b/llvm/lib/Target/AIE/AIE2InstrInfo.h index 90160c59016f..1a1604638cbc 100644 --- a/llvm/lib/Target/AIE/AIE2InstrInfo.h +++ b/llvm/lib/Target/AIE/AIE2InstrInfo.h @@ -106,6 +106,7 @@ class AIE2InstrInfo : public AIE2GenInstrInfo { virtual std::optional getZOLSupport() const override; virtual std::optional getJNZDSupport() const override; + std::optional getLoopVersionThresholdOpcode() const override; virtual std::optional getIfConvSupport() const override; virtual bool diff --git a/llvm/lib/Target/AIE/AIE2InstructionSelector.cpp b/llvm/lib/Target/AIE/AIE2InstructionSelector.cpp index 8c269b5b74e5..479d977a6b59 100644 --- a/llvm/lib/Target/AIE/AIE2InstructionSelector.cpp +++ b/llvm/lib/Target/AIE/AIE2InstructionSelector.cpp @@ -337,6 +337,8 @@ bool AIE2InstructionSelector::select(MachineInstr &I) { return selectStartLoop(I, MRI); case Intrinsic::set_loop_iterations: return selectSetLoopIterations(I, MRI, MIB); + case Intrinsic::aie2_loop_version_threshold: + return selectLoopVersionThreshold(I, MRI, MIB); case Intrinsic::aie2_scd_read_vec: case Intrinsic::aie2_scd_read_acc32: case Intrinsic::aie2_scd_expand_lo: diff --git a/llvm/lib/Target/AIE/AIE2MultiSlotPseudoInstrInfo.td b/llvm/lib/Target/AIE/AIE2MultiSlotPseudoInstrInfo.td index 5e87a2036589..cc4b386f98a2 100644 --- a/llvm/lib/Target/AIE/AIE2MultiSlotPseudoInstrInfo.td +++ b/llvm/lib/Target/AIE/AIE2MultiSlotPseudoInstrInfo.td @@ -4,7 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// (c) Copyright 2023-2024 Advanced Micro Devices, Inc. or its affiliates +// (c) Copyright 2023-2026 Advanced Micro Devices, Inc. or its affiliates // //===----------------------------------------------------------------------===// @@ -71,6 +71,18 @@ let isMoveImm = 1, isReMaterializable = 1, isAsCheapAsAMove = 1, Itinerary = II_ [MOVA_lda_cg, MOVXM_lng_cg]>; } +// Loop-versioning trip-count guard threshold, produced by the AIE loop +// versioning pass. The post-pipeliner patches its immediate operand with the +// required minimum trip count once a modulo schedule is found. +// Side-effecting and non-duplicable so MachineCSE never merges sibling guards' +// placeholders; dest is eR (the guard compare's class) so MOVA can be used. +let isMoveImm = 1, Itinerary = II_MOV, hasSideEffects = 1, isNotDuplicable = 1, + mayLoad = 0, mayStore = 0 in + def PseudoLoopVersionThreshold + : MultiSlot_Pseudo<(outs eR:$dst), (ins simm10:$i), + "pseudo_loop_version_threshold", "$dst, $i", + [MOVA_lda_cg, MOV_mv_cg, MOVXM_lng_cg]>; + // Pseudo VLD let hasSideEffects = false, mayLoad = true, mayStore = false in { let Itinerary = II_VLDA_W in { diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp b/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp index 914f38e7c619..cf18acb611c2 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp @@ -39,6 +39,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" #include #define DEBUG_TYPE "aie-codegen" @@ -424,6 +425,25 @@ void AIEBaseInstrInfo::adjustTripCount(MachineInstr &MI, int Adjustment) const { Imm.setImm(Imm.getImm() + Adjustment); } +bool AIEBaseInstrInfo::isLoopVersionThresholdDef(const MachineInstr &MI) const { + auto Opcode = getLoopVersionThresholdOpcode(); + return Opcode && MI.getOpcode() == *Opcode; +} + +void AIEBaseInstrInfo::setLoopVersionThreshold(MachineInstr &MI, + int MinTripCount) const { + assert(isLoopVersionThresholdDef(MI)); + // The threshold materializes into a narrow scalar-move immediate; enforce the + // narrowest field width (AIE2's simm10) across subtargets. A stage count is + // tiny in practice, so this only trips on a pathological schedule. Checked + // unconditionally, not just under assert: a truncated threshold would select + // the fast copy for trip counts too small for the schedule, a miscompile. + if (!isInt<10>(MinTripCount)) + report_fatal_error("AIE loop versioning: threshold does not fit the " + "scalar-move immediate"); + MI.getOperand(1).setImm(MinTripCount); +} + bool AIEBaseInstrInfo::isHardwareLoopStart(unsigned Opcode) const { const auto ZOLSupport = getZOLSupport(); return ZOLSupport && Opcode == ZOLSupport->LoopStartOpcode; diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h index 045e15ac9fb3..e7381ea28c9e 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h @@ -520,6 +520,22 @@ struct AIEBaseInstrInfo : public TargetInstrInfo { /// \pre isZOLTripCountDef(MI) virtual void adjustTripCount(MachineInstr &MI, int Update) const; + /// Opcode of this subtarget's loop-versioning threshold pseudo + /// (PseudoLoopVersionThreshold), or std::nullopt if unsupported. The pseudo + /// defines the threshold register (operand 0) and holds the patchable + /// threshold immediate (operand 1); see its def for the design rationale. + virtual std::optional getLoopVersionThresholdOpcode() const { + return std::nullopt; + } + + /// Check whether \p MI is the loop-versioning threshold move. + bool isLoopVersionThresholdDef(const MachineInstr &MI) const; + + /// Set the guard threshold immediate on the loop-versioning threshold + /// move \p MI to the (absolute) minimum trip count \p MinTripCount. + /// \pre isLoopVersionThresholdDef(MI) + void setLoopVersionThreshold(MachineInstr &MI, int MinTripCount) const; + /// Check whether this is a zero-overhead loop start block virtual bool isZeroOverheadLoopSetupInstr(const MachineInstr &) const; diff --git a/llvm/lib/Target/AIE/AIEBaseInstructionSelector.cpp b/llvm/lib/Target/AIE/AIEBaseInstructionSelector.cpp index b9957c9edf90..99642c0dee58 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstructionSelector.cpp +++ b/llvm/lib/Target/AIE/AIEBaseInstructionSelector.cpp @@ -142,6 +142,18 @@ bool AIEBaseInstructionSelector::selectSetLoopIterations( return constrainSelectedInstRegOperands(*LS, TII, TRI, RBI); } +bool AIEBaseInstructionSelector::selectLoopVersionThreshold( + MachineInstr &I, MachineRegisterInfo &MRI, MachineIRBuilder &MIB) { + auto Opcode = TII.getLoopVersionThresholdOpcode(); + assert(Opcode && "Loop versioning not supported on this subtarget"); + + const Register DstReg = I.getOperand(0).getReg(); + const int64_t Threshold = I.getOperand(2).getImm(); + auto Mov = MIB.buildInstr(*Opcode, {DstReg}, {}).addImm(Threshold); + I.eraseFromParent(); + return constrainSelectedInstRegOperands(*Mov, TII, TRI, RBI); +} + // Try to match BRCOND(Intrinsic::loop_decrement) bool AIEBaseInstructionSelector::selectBrCondLoopDecrement( MachineInstr &BrCond, MachineRegisterInfo &MRI) { diff --git a/llvm/lib/Target/AIE/AIEBaseInstructionSelector.h b/llvm/lib/Target/AIE/AIEBaseInstructionSelector.h index ef507b4bfdf9..ca7d2318b189 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstructionSelector.h +++ b/llvm/lib/Target/AIE/AIEBaseInstructionSelector.h @@ -155,6 +155,8 @@ class AIEBaseInstructionSelector : public InstructionSelector { MachineIRBuilder &MIB, MachineInstrBuilder &MI); bool selectSetLoopIterations(MachineInstr &I, MachineRegisterInfo &MRI, MachineIRBuilder &MIB); + bool selectLoopVersionThreshold(MachineInstr &I, MachineRegisterInfo &MRI, + MachineIRBuilder &MIB); bool selectBrCondLoopDecrement(MachineInstr &BrCond, MachineRegisterInfo &MRI); bool selectBrCondLoopDecrementReg(MachineInstr &BrCond, diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp index 330f14560986..4de2f4fe88f3 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.cpp @@ -1791,6 +1791,10 @@ AIE2PInstrInfo::getZOLSupport() const { return Result; } +std::optional AIE2PInstrInfo::getLoopVersionThresholdOpcode() const { + return AIE2P::PseudoLoopVersionThreshold; +} + std::optional AIE2PInstrInfo::getJNZDSupport() const { AIEBaseInstrInfo::JNZDSupport Result; diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h index 4337f2d9ee24..7b21c3ec97b9 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h @@ -116,6 +116,7 @@ class AIE2PInstrInfo : public AIE2PGenInstrInfo { virtual std::optional getZOLSupport() const override; virtual std::optional getJNZDSupport() const override; + std::optional getLoopVersionThresholdOpcode() const override; virtual std::optional getIfConvSupport() const override; bool isOffsetInImmediateRange(unsigned Opcode, unsigned LoadStoreSize, diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstructionSelector.cpp b/llvm/lib/Target/AIE/aie2p/AIE2PInstructionSelector.cpp index f6b802d3eb15..a4b6f83a9748 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstructionSelector.cpp +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstructionSelector.cpp @@ -424,6 +424,8 @@ bool AIE2PInstructionSelector::select(MachineInstr &I) { return selectVLD_FIFO_POP_BFP16_3D(I, MRI); case Intrinsic::set_loop_iterations: return selectSetLoopIterations(I, MRI, MIB); + case Intrinsic::aie2p_loop_version_threshold: + return selectLoopVersionThreshold(I, MRI, MIB); case Intrinsic::start_loop_iterations: return selectStartLoop(I, MRI); case Intrinsic::aie2p_acquire_ptr: diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PMultiSlotPseudoInstrInfo.td b/llvm/lib/Target/AIE/aie2p/AIE2PMultiSlotPseudoInstrInfo.td index ca4b06543b4e..2c36fa898536 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PMultiSlotPseudoInstrInfo.td +++ b/llvm/lib/Target/AIE/aie2p/AIE2PMultiSlotPseudoInstrInfo.td @@ -4,7 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// (c) Copyright 2024-2025 Advanced Micro Devices, Inc. or its affiliates +// (c) Copyright 2024-2026 Advanced Micro Devices, Inc. or its affiliates // //===---------------------------------------------------------------------===// @@ -56,6 +56,14 @@ let isMoveImm = 1, isReMaterializable = 1, isAsCheapAsAMove = 1, Itinerary = II_ "mov_scalar_imm11_pseudo ", "$dst, $i", [MOV_alu_mv_mv_mv_cg, MOVXM]>; } +// Loop-versioning trip-count guard threshold. See AIE2MultiSlotPseudoInstrInfo.td. +let isMoveImm = 1, Itinerary = II_MOVX_mvx_cr_r, hasSideEffects = 1, + isNotDuplicable = 1, mayLoad = 0, mayStore = 0 in + def PseudoLoopVersionThreshold + : MultiSlot_Pseudo<(outs eR:$dst), (ins c11s:$i), + "pseudo_loop_version_threshold", "$dst, $i", + [MOVA, MOV_alu_mv_mv_mv_cg, MOVXM]>; + // Pseudo Scalar MOV. The MOVS register set is a strict subset let Itinerary = II_MOVS, hasSideEffects = false, mayLoad = false, mayStore = false in { def MOV_scalar_pseudo : MultiSlot_Pseudo< (outs OP_mAguDst:$dst), (ins OP_mAguSrc:$src), diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp index 3c6cad68e01f..6d30555bdcea 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.cpp @@ -1381,6 +1381,10 @@ AIE2PSInstrInfo::getZOLSupport() const { return Result; } +std::optional AIE2PSInstrInfo::getLoopVersionThresholdOpcode() const { + return AIE2PS::PseudoLoopVersionThreshold; +} + std::optional AIE2PSInstrInfo::getJNZDSupport() const { AIEBaseInstrInfo::JNZDSupport Result; diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h index 99d263b87d10..5fc91899e95c 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstrInfo.h @@ -152,6 +152,7 @@ class AIE2PSInstrInfo : public AIE2PSGenInstrInfo { std::optional getZOLSupport() const override; std::optional getJNZDSupport() const override; + std::optional getLoopVersionThresholdOpcode() const override; std::optional getIfConvSupport() const override; // Implement MIR serialization of target flags diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstructionSelector.cpp b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstructionSelector.cpp index 3c2da87829b7..217a7c54aa79 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSInstructionSelector.cpp +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSInstructionSelector.cpp @@ -2032,6 +2032,8 @@ bool AIE2PSInstructionSelector::select(MachineInstr &I) { /*Has3DDimension=*/true); case Intrinsic::set_loop_iterations: return selectSetLoopIterations(I, MRI, MIB); + case Intrinsic::aie2ps_loop_version_threshold: + return selectLoopVersionThreshold(I, MRI, MIB); case Intrinsic::start_loop_iterations: return selectStartLoop(I, MRI); case Intrinsic::aie2ps_BFP768_BFP1536_ACC2048_bf_mul_conf: diff --git a/llvm/lib/Target/AIE/aie2ps/AIE2PSMultiSlotPseudoInstrInfo.td b/llvm/lib/Target/AIE/aie2ps/AIE2PSMultiSlotPseudoInstrInfo.td index cd69f23394c7..b8444820cfd8 100644 --- a/llvm/lib/Target/AIE/aie2ps/AIE2PSMultiSlotPseudoInstrInfo.td +++ b/llvm/lib/Target/AIE/aie2ps/AIE2PSMultiSlotPseudoInstrInfo.td @@ -56,6 +56,14 @@ let isMoveImm = 1, isReMaterializable = 1, isAsCheapAsAMove = 1, Itinerary = II_ "mov_scalar_imm11_pseudo ", "$dst, $i", [MOV_alu_mv_mv_mv_cg_or, MOVXM_lng_cg]>; } +// Loop-versioning trip-count guard threshold. See AIE2MultiSlotPseudoInstrInfo.td. +let isMoveImm = 1, Itinerary = II_MOVX_mvx_cr_r, hasSideEffects = 1, + isNotDuplicable = 1, mayLoad = 0, mayStore = 0 in + def PseudoLoopVersionThreshold + : MultiSlot_Pseudo<(outs eR:$dst), (ins c11s:$i), + "pseudo_loop_version_threshold", "$dst, $i", + [MOVA, MOV_alu_mv_mv_mv_cg_or, MOVXM_lng_cg]>; + // Pseudo Scalar MOV. The MOVS register set is a strict subset let Itinerary = II_MOVS, hasSideEffects = false, mayLoad = false, mayStore = false in { def MOV_scalar_pseudo : MultiSlot_Pseudo< (outs OP_mAguDst:$dst), (ins OP_mAguSrc:$src), diff --git a/llvm/test/CodeGen/AIE/aie2/loop-version-threshold.ll b/llvm/test/CodeGen/AIE/aie2/loop-version-threshold.ll new file mode 100644 index 000000000000..9d3a70143231 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2/loop-version-threshold.ll @@ -0,0 +1,23 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; RUN: llc -mtriple=aie2 -stop-after=instruction-select %s -o - | FileCheck %s --check-prefix=SELECT +; RUN: llc -mtriple=aie2 -stop-after=postmisched %s -o - | FileCheck %s --check-prefix=MOV + +; The intrinsic lowers to the multi-slot PseudoLoopVersionThreshold, which the +; post-RA scheduler materializes into a concrete scalar move. + +; SELECT-LABEL: name: threshold +; SELECT: PseudoLoopVersionThreshold 9 + +; MOV-LABEL: name: threshold +; MOV: MOVA_lda_cg 9 +define i32 @threshold() { +entry: + %t = call i32 @llvm.aie2.loop.version.threshold(i32 9) + ret i32 %t +} + +declare i32 @llvm.aie2.loop.version.threshold(i32 immarg) diff --git a/llvm/test/CodeGen/AIE/aie2p/loop-version-threshold.ll b/llvm/test/CodeGen/AIE/aie2p/loop-version-threshold.ll new file mode 100644 index 000000000000..e746b23e02dc --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/loop-version-threshold.ll @@ -0,0 +1,23 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; RUN: llc -mtriple=aie2p -stop-after=instruction-select %s -o - | FileCheck %s --check-prefix=SELECT +; RUN: llc -mtriple=aie2p -stop-after=postmisched %s -o - | FileCheck %s --check-prefix=MOV + +; The intrinsic lowers to the multi-slot PseudoLoopVersionThreshold, which the +; post-RA scheduler materializes into a concrete scalar move. + +; SELECT-LABEL: name: threshold +; SELECT: PseudoLoopVersionThreshold 9 + +; MOV-LABEL: name: threshold +; MOV: MOVA 9 +define i32 @threshold() { +entry: + %t = call i32 @llvm.aie2p.loop.version.threshold(i32 9) + ret i32 %t +} + +declare i32 @llvm.aie2p.loop.version.threshold(i32 immarg) diff --git a/llvm/test/CodeGen/AIE/aie2ps/loop-version-threshold.ll b/llvm/test/CodeGen/AIE/aie2ps/loop-version-threshold.ll new file mode 100644 index 000000000000..f17088ffcdab --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2ps/loop-version-threshold.ll @@ -0,0 +1,23 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; RUN: llc -mtriple=aie2ps -stop-after=instruction-select %s -o - | FileCheck %s --check-prefix=SELECT +; RUN: llc -mtriple=aie2ps -stop-after=postmisched %s -o - | FileCheck %s --check-prefix=MOV + +; The intrinsic lowers to the multi-slot PseudoLoopVersionThreshold, which the +; post-RA scheduler materializes into a concrete scalar move. + +; SELECT-LABEL: name: threshold +; SELECT: PseudoLoopVersionThreshold 9 + +; MOV-LABEL: name: threshold +; MOV: MOVA 9 +define i32 @threshold() { +entry: + %t = call i32 @llvm.aie2ps.loop.version.threshold(i32 9) + ret i32 %t +} + +declare i32 @llvm.aie2ps.loop.version.threshold(i32 immarg) From 038495968c3febec9713b904cde0c58309516431 Mon Sep 17 00:00:00 2001 From: Marc Ludevid Wulf Date: Wed, 8 Jul 2026 09:44:17 -0600 Subject: [PATCH 3/4] [AIE] Patch versioned-loop guard threshold in the post-pipeliner --- llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp | 3 +- llvm/lib/Target/AIE/AIEPostPipeliner.cpp | 160 ++++++++++++++- llvm/lib/Target/AIE/AIEPostPipeliner.h | 20 ++ llvm/lib/Target/AIE/Utils/AIELoopUtils.cpp | 5 + llvm/lib/Target/AIE/Utils/AIELoopUtils.h | 19 ++ .../AIE/aie2/postpipeliner-version-guard.mir | 187 ++++++++++++++++++ .../AIE/aie2p/postpipeliner-version-guard.mir | 185 +++++++++++++++++ .../aie2ps/postpipeliner-version-guard.mir | 185 +++++++++++++++++ 8 files changed, 760 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/AIE/aie2/postpipeliner-version-guard.mir create mode 100644 llvm/test/CodeGen/AIE/aie2p/postpipeliner-version-guard.mir create mode 100644 llvm/test/CodeGen/AIE/aie2ps/postpipeliner-version-guard.mir diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp b/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp index cf18acb611c2..ce5452668c67 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp @@ -437,7 +437,8 @@ void AIEBaseInstrInfo::setLoopVersionThreshold(MachineInstr &MI, // narrowest field width (AIE2's simm10) across subtargets. A stage count is // tiny in practice, so this only trips on a pathological schedule. Checked // unconditionally, not just under assert: a truncated threshold would select - // the fast copy for trip counts too small for the schedule, a miscompile. + // the high-trip-count copy for trip counts too small for the schedule, a + // miscompile. if (!isInt<10>(MinTripCount)) report_fatal_error("AIE loop versioning: threshold does not fit the " "scalar-move immediate"); diff --git a/llvm/lib/Target/AIE/AIEPostPipeliner.cpp b/llvm/lib/Target/AIE/AIEPostPipeliner.cpp index 02a88ff0b289..331e45a794e0 100644 --- a/llvm/lib/Target/AIE/AIEPostPipeliner.cpp +++ b/llvm/lib/Target/AIE/AIEPostPipeliner.cpp @@ -17,11 +17,15 @@ #include "AIESlotUtils.h" #include "Utils/AIELoopUtils.h" #include "Utils/AIEMachineInstrPrint.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" #include "llvm/CodeGen/ResourceScoreboard.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Transforms/Utils/LoopUtils.h" #include #include @@ -152,12 +156,20 @@ bool PostPipeliner::isPostPipelineCandidate(MachineBasicBlock &LoopBlock) { // count is checked before accepting the schedule. using namespace AIELoopUtils; auto ParsedMinTripCount = getMinTripCount(LoopBlock); - if (!ParsedMinTripCount) { + MinTripCount = ParsedMinTripCount.value_or(0); + + // A versioned loop only runs when its trip count clears the runtime guard, so + // any minimum is acceptable here; the guard threshold is patched with the + // required minimum after scheduling (see updateVersionGuard). + IsVersionedGuarded = getLoopVersioningHint(LoopBlock).has_value(); + + // The two checks below are the static min-trip-count gate. A versioned loop + // clears them via its runtime guard, so it exits early past them. + if (!IsVersionedGuarded && !ParsedMinTripCount) { LLVM_DEBUG(dbgs() << " PostPipeliner: No min tripcount\n"); return false; } - MinTripCount = *ParsedMinTripCount; - if (MinTripCount < 2) { + if (!IsVersionedGuarded && MinTripCount < 2) { LLVM_DEBUG(dbgs() << " PostPipeliner: min tripcount < 2\n"); return false; } @@ -1609,6 +1621,10 @@ bool PostPipeliner::schedule(ScheduleDAGMI &TheDAG, int InitiationInterval) { // Pipelining reduces the iteration count by NS - 1 // The result should be > 0, because ZOL doesn't support zero iterations. bool PostPipeliner::hasSufficientMinTripCount(int NS) const { + // A versioned loop's runtime guard makes any stage count safe (see + // updateVersionGuard). + if (IsVersionedGuarded) + return true; return MinTripCount - (NS - 1) > 0; } @@ -1760,6 +1776,143 @@ void PostPipeliner::updateTripCount() const { TII->adjustTripCount(*TripCountDef, -Delta); } +// Reaching definition of Reg seen from Use, scanning backwards into +// predecessors -- the threshold pseudo may sit in a dominating block. +static MachineInstr * +findReachingDef(MachineBasicBlock &BB, MachineBasicBlock::reverse_iterator Use, + Register Reg, const TargetRegisterInfo &TRI, + SmallPtrSetImpl &Visited) { + for (auto It = Use, End = BB.rend(); It != End; ++It) + if (It->definesRegister(Reg, &TRI)) + return &*It; + if (!Visited.insert(&BB).second) + return nullptr; + for (MachineBasicBlock *Pred : BB.predecessors()) + if (MachineInstr *Found = + findReachingDef(*Pred, Pred->rbegin(), Reg, TRI, Visited)) + return Found; + return nullptr; +} + +MachineBasicBlock *PostPipeliner::findVersionGuard() const { + // Structural invariant from AIEInnerLoopVersioning + canAccept: the preheader + // is dedicated and fallthrough, so it has a single predecessor, and that + // predecessor is the guard block branching to the low/high-trip-count copies + // (two successors). If either no longer holds, fail closed so + // updateVersionGuard bails rather than patching the wrong block. + if (Preheader->pred_size() != 1) + return nullptr; + MachineBasicBlock *Guard = *Preheader->pred_begin(); + if (Guard->succ_size() != 2) + return nullptr; + return Guard; +} + +// The register tested by the guard's conditional branch (the trip-count compare +// result), or an invalid register if none. The branch is the instruction that +// both targets a block and reads a register. +static Register findGuardConditionReg(MachineBasicBlock &Guard) { + Register CondReg; + for (MachineInstr &MI : Guard) { + bool TargetsBlock = false; + Register RegUse; + for (const MachineOperand &MO : MI.operands()) { + if (MO.isMBB()) + TargetsBlock = true; + else if (MO.isReg() && MO.isUse() && MO.getReg().isPhysical()) + RegUse = MO.getReg(); + } + if (TargetsBlock && RegUse.isValid()) + CondReg = RegUse; + } + return CondReg; +} + +MachineInstr * +PostPipeliner::findVersionThresholdPseudo(MachineInstr &Cmp, + const TargetRegisterInfo &TRI) const { + MachineInstr *Threshold = nullptr; + for (const MachineOperand &MO : Cmp.uses()) { + if (!MO.isReg() || !MO.getReg().isPhysical()) + continue; + SmallPtrSet Visited; + // Start scanning just above the compare: register allocation may reuse the + // compare's result register for one of its source operands (e.g. the + // threshold), so scanning from the compare itself would spuriously return + // the compare as its own operand's reaching def. + MachineInstr *Def = + findReachingDef(*Cmp.getParent(), std::next(Cmp.getReverseIterator()), + MO.getReg(), TRI, Visited); + if (Def && TII->isLoopVersionThresholdDef(*Def)) { + if (Threshold && Threshold != Def) + return nullptr; + Threshold = Def; + } + } + return Threshold; +} + +void PostPipeliner::updateVersionGuard() const { + if (!IsVersionedGuarded) + return; + + // Locate the threshold pseudo structurally (guard block -> conditional branch + // -> compare -> the pseudo feeding it), then patch its placeholder with the + // stage count. If any step fails to recognize the guard (e.g. a later MIR + // pass reshaped the region), bail without patching: the IR pass seeds the + // placeholder with the maximal threshold, so an un-patched guard keeps the + // loop on the low-trip-count copy. Diagnosable via debug output, never fatal. + MachineOptimizationRemarkEmitter ORE(*Preheader->getParent(), nullptr); + // A fall-through preheader may be empty, so pick a DebugLoc defensively. + const DebugLoc GuardLoc = + Preheader->empty() ? DebugLoc() : Preheader->front().getDebugLoc(); + auto bailWithMsg = [&](StringRef Msg) { + LLVM_DEBUG(dbgs() << "AIE loop versioning: " << Msg + << "; leaving guard unpatched (low-trip-count copy stays " + "live)\n"); + ORE.emit([&] { + return MachineOptimizationRemarkMissed( + DEBUG_TYPE, "VersionGuardUnpatched", GuardLoc, Preheader) + << "versioned loop was pipelined but its runtime guard could not " + "be patched (" + << ore::NV("Reason", Msg) + << "); execution stays on the un-pipelined copy"; + }); + }; + + MachineBasicBlock *Guard = findVersionGuard(); + if (!Guard) { + bailWithMsg("guard block not found"); + return; + } + const TargetRegisterInfo &TRI = + *Guard->getParent()->getSubtarget().getRegisterInfo(); + + Register CondReg = findGuardConditionReg(*Guard); + if (!CondReg.isValid()) { + bailWithMsg("guard conditional branch not found"); + return; + } + + SmallPtrSet Visited; + MachineInstr *Cmp = + findReachingDef(*Guard, Guard->rbegin(), CondReg, TRI, Visited); + if (!Cmp) { + bailWithMsg("guard compare not found"); + return; + } + + MachineInstr *Threshold = findVersionThresholdPseudo(*Cmp, TRI); + if (!Threshold) { + bailWithMsg("guard threshold pseudo not found"); + return; + } + + // Patch the minimum trip count required by this schedule: peeling NStages-1 + // stages needs at least NStages iterations for the pipelined loop to be safe. + TII->setLoopVersionThreshold(*Threshold, NStages); +} + void PostPipeliner::materializePipeline(PipelineScheduleVisitor &Visitor) { // A schedule NS=N, II=L is compatible with NS=1, II=N*L. // In general, we can take any divisor of N. @@ -1783,6 +1936,7 @@ void PostPipeliner::materializePipeline(PipelineScheduleVisitor &Visitor) { visitPipelineSchedule(Visitor); updateTripCount(); + updateVersionGuard(); } void NodeInfo::reset(bool FullReset) { diff --git a/llvm/lib/Target/AIE/AIEPostPipeliner.h b/llvm/lib/Target/AIE/AIEPostPipeliner.h index 75d81e6ed723..1df30b048c8f 100644 --- a/llvm/lib/Target/AIE/AIEPostPipeliner.h +++ b/llvm/lib/Target/AIE/AIEPostPipeliner.h @@ -246,6 +246,11 @@ class PostPipeliner { /// The minimum tripcount, read from the pragma, or from an LC initialization. int MinTripCount = 0; + /// Set when the loop is a runtime-guarded ("versioned") loop, in which case + /// the minimum trip-count requirement is lifted and the guard threshold is + /// patched with the required minimum after a schedule is found. + bool IsVersionedGuarded = false; + /// The II requested by a pragma. This will trigger expensive algorithms /// like solvers or exhaustive searches to be run if the heuristic methods /// don't find a solution. @@ -272,6 +277,17 @@ class PostPipeliner { /// can be trivially extracted into a pipelined loop. bool hasSufficientMinTripCount(int NS) const; + /// Find the guard block of a versioned loop: the preheader predecessor that + /// conditionally branches to the low- and high-trip-count copies. Null if not + /// found. + MachineBasicBlock *findVersionGuard() const; + + /// Find the loop-version-threshold pseudo feeding the guard compare \p Cmp, + /// via a reaching-def walk over the compare's operands. Null if there is no + /// unique such pseudo. + MachineInstr *findVersionThresholdPseudo(MachineInstr &Cmp, + const TargetRegisterInfo &TRI) const; + /// If the iteration count is too low, we may be able to peel off some /// side-effect-free initial stage and rotate the schedule accordingly. /// When the method returns true, this was successful, and all relevant @@ -374,6 +390,10 @@ class PostPipeliner { // Modify the tripcount to run StageCount-1 less iterations. void updateTripCount() const; + /// For a versioned loop, patch the guard's placeholder threshold pseudo with + /// the actual stage count. No-op when not versioned. + void updateVersionGuard() const; + void materializePipeline(PipelineScheduleVisitor &Visitor); void dump() const; diff --git a/llvm/lib/Target/AIE/Utils/AIELoopUtils.cpp b/llvm/lib/Target/AIE/Utils/AIELoopUtils.cpp index f517fe75e849..27a1d94c373d 100644 --- a/llvm/lib/Target/AIE/Utils/AIELoopUtils.cpp +++ b/llvm/lib/Target/AIE/Utils/AIELoopUtils.cpp @@ -63,6 +63,11 @@ bool isOuterLoopPipelined(const MachineBasicBlock &LoopLatch) { .has_value(); } +std::optional +getLoopVersioningHint(const MachineBasicBlock &LoopBlock) { + return getLoopHintInt(LoopBlock, LoopVersionedHintKey); +} + std::optional getPipelinerDisabled(const MachineBasicBlock &LoopBlock) { if (getLoopMetadata(getLoopID(LoopBlock), "llvm.loop.pipeline.disable")) return true; diff --git a/llvm/lib/Target/AIE/Utils/AIELoopUtils.h b/llvm/lib/Target/AIE/Utils/AIELoopUtils.h index 63d37f31da30..f1856ce125a2 100644 --- a/llvm/lib/Target/AIE/Utils/AIELoopUtils.h +++ b/llvm/lib/Target/AIE/Utils/AIELoopUtils.h @@ -40,6 +40,25 @@ constexpr StringLiteral OuterLoopPipelinedKey{ bool isOuterLoopPipelined(const MachineBasicBlock &LoopLatch); +/// Loop-hint key that requests AIE loop versioning. It is the user-facing hint +/// (set via pragma) and is consumed by the IR versioning pass, which strips it +/// from both copies once the loop has been processed. +constexpr StringLiteral LoopVersioningHintKey = + "llvm.loop.hint.aie-loop-versioning"; + +/// Loop-hint key that marks an already-versioned loop. It is set by the IR +/// versioning pass on the pipelined (high-trip-count) copy and consumed by the +/// post-pipeliner. Kept separate from LoopVersioningHintKey so a second run of +/// the versioning pass does not re-version an already-versioned loop. +constexpr StringLiteral LoopVersionedHintKey = + "llvm.loop.hint.aie-loop-versioned"; + +/// Read the LoopVersionedHintKey marker of a single-block loop. A present +/// (positive) value marks the pipelined (high-trip-count) copy of a versioned +/// loop. +std::optional +getLoopVersioningHint(const MachineBasicBlock &LoopBlock); + /// Returns true if this is a loop latch that has a pipeliner disable pragma, /// none otherwise. std::optional getPipelinerDisabled(const MachineBasicBlock &LoopBlock); diff --git a/llvm/test/CodeGen/AIE/aie2/postpipeliner-version-guard.mir b/llvm/test/CodeGen/AIE/aie2/postpipeliner-version-guard.mir new file mode 100644 index 000000000000..4c8ad2d4bb66 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2/postpipeliner-version-guard.mir @@ -0,0 +1,187 @@ +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +# +# RUN: llc -mtriple=aie2 -run-pass=postmisched -verify-machineinstrs %s -o - | FileCheck %s +# +# Start from an already-versioned loop: a guard block holding the placeholder +# PseudoLoopVersionThreshold (immediate -1) and the unsigned trip-count compare, +# a low-trip-count (verbatim) copy, and a high-trip-count (to-be-pipelined) copy +# carrying the aie-loop-versioned marker. Running only the post-pipeliner must +# accept the high-trip-count copy despite a small min trip count and patch the +# guard threshold with the schedule's stage count (2 here), materialized into a +# concrete scalar move. +# +# CHECK-LABEL: name: versioned +# The guard threshold placeholder is patched to the stage count and the compare +# tests the trip count against it. +# CHECK-LABEL: bb.0.loop.lver.guard: +# CHECK: MOVA_lda_cg 2 +# CHECK: LTU + +--- | + ; ModuleID = '/tmp/ver.ll' + source_filename = "/tmp/ver.ll" + target datalayout = "e-m:e-p:20:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-f32:32:32-i64:32-f64:32-a:0:32-n32" + target triple = "aie2" + + define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { + loop.lver.guard: + %smax = call i32 @llvm.smax.i32(i32 %n, i32 1) + %0 = zext nneg i32 %smax to i33 + %lver.threshold = call i32 @llvm.aie2.loop.version.threshold(i32 -1) + %lver.tc.i32 = trunc i33 %0 to i32 + %lver.low = icmp ult i32 %lver.tc.i32, %lver.threshold + br i1 %lver.low, label %loop.ph, label %loop.ph.lver.high + + loop.ph.lver.high: ; preds = %loop.lver.guard + call void @llvm.set.loop.iterations.i32(i32 %smax) + br label %loop.lver.high + + loop.lver.high: ; preds = %loop.lver.high, %loop.ph.lver.high + %i.lver.high = phi i32 [ 0, %loop.ph.lver.high ], [ %i.next.lver.high, %loop.lver.high ] + %pa.lver.high = getelementptr i32, ptr %a, i32 %i.lver.high + %x.lver.high = load i32, ptr %pa.lver.high, align 4 + %m1.lver.high = mul i32 %x.lver.high, %x.lver.high + %m2.lver.high = mul i32 %m1.lver.high, %x.lver.high + %m3.lver.high = add i32 %m2.lver.high, %m1.lver.high + %m4.lver.high = xor i32 %m3.lver.high, %x.lver.high + %pb.lver.high = getelementptr i32, ptr %b, i32 %i.lver.high + store i32 %m4.lver.high, ptr %pb.lver.high, align 4 + %i.next.lver.high = add i32 %i.lver.high, 1 + %1 = call i1 @llvm.loop.decrement.i32(i32 1) + br i1 %1, label %loop.lver.high, label %exit.loopexit, !llvm.loop !0 + + loop.ph: ; preds = %loop.lver.guard + call void @llvm.set.loop.iterations.i32(i32 %smax) + br label %loop + + loop: ; preds = %loop, %loop.ph + %i = phi i32 [ 0, %loop.ph ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %m1 = mul i32 %x, %x + %m2 = mul i32 %m1, %x + %m3 = add i32 %m2, %m1 + %m4 = xor i32 %m3, %x + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %m4, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %2 = call i1 @llvm.loop.decrement.i32(i32 1) + br i1 %2, label %loop, label %exit.loopexit1, !llvm.loop !1 + + exit.loopexit: ; preds = %loop.lver.high + br label %exit + + exit.loopexit1: ; preds = %loop + br label %exit + + exit: ; preds = %exit.loopexit1, %exit.loopexit + ret void + } + + ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) + declare i32 @llvm.smax.i32(i32, i32) #0 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i32 @llvm.aie2.loop.version.threshold(i32 immarg) #1 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare void @llvm.set.loop.iterations.i32(i32) #1 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i1 @llvm.loop.decrement.i32(i32) #1 + + attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + attributes #1 = { nocallback noduplicate nofree nosync nounwind willreturn } + + !0 = distinct !{!0, !2} + !1 = distinct !{!1} + !2 = !{!"llvm.loop.hint.aie-loop-versioned", i32 1} + +... +--- +name: versioned +alignment: 16 +legalized: true +regBankSelected: true +selected: true +tracksRegLiveness: true +liveins: [] +calleeSavedRegisters: [ '$lr', '$r16', '$r17', '$r18', '$r19', '$r20', + '$r21', '$r22', '$r23', '$p6', '$p7' ] +body: | + bb.0.loop.lver.guard: + successors: %bb.3(0x40000000), %bb.1(0x40000000) + liveins: $p0, $p1, $r0 + + renamable $r1 = MOV_RLC_imm10_pseudo 1 + renamable $r27 = LT renamable $r1, renamable $r0 + renamable $r1 = SELNEZ killed renamable $r0, killed renamable $r1, killed renamable $r27 + renamable $r0 = PseudoLoopVersionThreshold -1 + renamable $r2 = LTU renamable $r1, killed renamable $r0 + renamable $r0 = MOV_RLC_imm10_pseudo 0 + JZ killed renamable $r2, %bb.1 + DelayedSchedBarrier + + bb.3.loop.ph: + successors: %bb.4(0x80000000) + liveins: $p0, $p1, $r0, $r1 + + $lc = ADD_NC $r1, 0 + $ls = MOVXM_lng_cg %bb.4 + $le = MOVXM_lng_cg + renamable $r1 = MOV_RLC_imm10_pseudo 2 + + bb.4.loop: + successors: %bb.4(0x7c000000), %bb.6(0x04000000) + liveins: $p0, $p1, $r0, $r1 + + renamable $r2 = LSHL renamable $r0, renamable $r1 + $dj0 = MOV_mv_scl killed $r2 + renamable $r2 = LDA_dms_lda_idx renamable $p0, renamable $dj0 :: (load (s32) from %ir.pa) + renamable $r3 = MUL_mul_r_rr renamable $r2, renamable $r2 + renamable $r4 = MUL_mul_r_rr renamable $r3, renamable $r2 + renamable $r3 = ADD killed renamable $r4, killed renamable $r3, implicit-def dead $srcarry + renamable $r2 = XOR killed renamable $r3, killed renamable $r2 + ST_dms_sts_idx killed renamable $r2, renamable $p1, killed renamable $dj0 :: (store (s32) into %ir.pb) + renamable $r0 = ADD_add_r_ri killed renamable $r0, 1, implicit-def dead $srcarry + PseudoLoopEnd , %bb.4 + + bb.6.exit: + successors: %bb.5(0x80000000) + + J_jump_imm %bb.5 + DelayedSchedBarrier + + bb.1.loop.ph.lver.high: + successors: %bb.2(0x80000000) + liveins: $p0, $p1, $r0, $r1 + + $lc = ADD_NC $r1, 0 + $ls = MOVXM_lng_cg %bb.2 + $le = MOVXM_lng_cg + renamable $r1 = MOV_RLC_imm10_pseudo 2 + + bb.2.loop.lver.high: + successors: %bb.2(0x7c000000), %bb.5(0x04000000) + liveins: $p0, $p1, $r0, $r1 + + renamable $r2 = LSHL renamable $r0, renamable $r1 + $dj0 = MOV_mv_scl killed $r2 + renamable $r2 = LDA_dms_lda_idx renamable $p0, renamable $dj0 :: (load (s32) from %ir.pa.lver.high) + renamable $r3 = MUL_mul_r_rr renamable $r2, renamable $r2 + renamable $r4 = MUL_mul_r_rr renamable $r3, renamable $r2 + renamable $r3 = ADD killed renamable $r4, killed renamable $r3, implicit-def dead $srcarry + renamable $r2 = XOR killed renamable $r3, killed renamable $r2 + ST_dms_sts_idx killed renamable $r2, renamable $p1, killed renamable $dj0 :: (store (s32) into %ir.pb.lver.high) + renamable $r0 = ADD_add_r_ri killed renamable $r0, 1, implicit-def dead $srcarry + PseudoLoopEnd , %bb.2 + + bb.5.exit: + RET implicit $lr + DelayedSchedBarrier + +... diff --git a/llvm/test/CodeGen/AIE/aie2p/postpipeliner-version-guard.mir b/llvm/test/CodeGen/AIE/aie2p/postpipeliner-version-guard.mir new file mode 100644 index 000000000000..588c0211a07d --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/postpipeliner-version-guard.mir @@ -0,0 +1,185 @@ +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +# +# RUN: llc -mtriple=aie2p -run-pass=postmisched -verify-machineinstrs %s -o - | FileCheck %s +# +# Start from an already-versioned loop: a guard block holding the placeholder +# PseudoLoopVersionThreshold (immediate -1) and the unsigned trip-count compare, +# a low-trip-count (verbatim) copy, and a high-trip-count (to-be-pipelined) copy +# carrying the aie-loop-versioned marker. Running only the post-pipeliner must +# accept the high-trip-count copy despite a small min trip count and patch the +# guard threshold with the schedule's stage count (2 here), materialized into a +# concrete scalar move. +# +# CHECK-LABEL: name: versioned +# The guard threshold placeholder is patched to the stage count and the compare +# tests the trip count against it. +# CHECK-LABEL: bb.0.loop.lver.guard: +# CHECK: MOVA 2 +# CHECK: LTU + +--- | + ; ModuleID = '/tmp/ver.ll' + source_filename = "/tmp/ver.ll" + target datalayout = "e-m:e-p:20:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-f32:32:32-i64:32-f64:32-a:0:32-n32" + target triple = "aie2p" + + define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { + loop.lver.guard: + %smax = call i32 @llvm.smax.i32(i32 %n, i32 1) + %0 = zext nneg i32 %smax to i33 + %lver.threshold = call i32 @llvm.aie2p.loop.version.threshold(i32 -1) + %lver.tc.i32 = trunc i33 %0 to i32 + %lver.low = icmp ult i32 %lver.tc.i32, %lver.threshold + br i1 %lver.low, label %loop.ph, label %loop.ph.lver.high + + loop.ph.lver.high: ; preds = %loop.lver.guard + call void @llvm.set.loop.iterations.i32(i32 %smax) + br label %loop.lver.high + + loop.lver.high: ; preds = %loop.lver.high, %loop.ph.lver.high + %i.lver.high = phi i32 [ 0, %loop.ph.lver.high ], [ %i.next.lver.high, %loop.lver.high ] + %pa.lver.high = getelementptr i32, ptr %a, i32 %i.lver.high + %x.lver.high = load i32, ptr %pa.lver.high, align 4 + %m1.lver.high = mul i32 %x.lver.high, %x.lver.high + %m2.lver.high = mul i32 %m1.lver.high, %x.lver.high + %m3.lver.high = add i32 %m2.lver.high, %m1.lver.high + %m4.lver.high = xor i32 %m3.lver.high, %x.lver.high + %pb.lver.high = getelementptr i32, ptr %b, i32 %i.lver.high + store i32 %m4.lver.high, ptr %pb.lver.high, align 4 + %i.next.lver.high = add i32 %i.lver.high, 1 + %1 = call i1 @llvm.loop.decrement.i32(i32 1) + br i1 %1, label %loop.lver.high, label %exit.loopexit, !llvm.loop !0 + + loop.ph: ; preds = %loop.lver.guard + call void @llvm.set.loop.iterations.i32(i32 %smax) + br label %loop + + loop: ; preds = %loop, %loop.ph + %i = phi i32 [ 0, %loop.ph ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %m1 = mul i32 %x, %x + %m2 = mul i32 %m1, %x + %m3 = add i32 %m2, %m1 + %m4 = xor i32 %m3, %x + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %m4, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %2 = call i1 @llvm.loop.decrement.i32(i32 1) + br i1 %2, label %loop, label %exit.loopexit1, !llvm.loop !1 + + exit.loopexit: ; preds = %loop.lver.high + br label %exit + + exit.loopexit1: ; preds = %loop + br label %exit + + exit: ; preds = %exit.loopexit1, %exit.loopexit + ret void + } + + ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) + declare i32 @llvm.smax.i32(i32, i32) #0 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i32 @llvm.aie2p.loop.version.threshold(i32 immarg) #1 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare void @llvm.set.loop.iterations.i32(i32) #1 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i1 @llvm.loop.decrement.i32(i32) #1 + + attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + attributes #1 = { nocallback noduplicate nofree nosync nounwind willreturn } + + !0 = distinct !{!0, !2} + !1 = distinct !{!1} + !2 = !{!"llvm.loop.hint.aie-loop-versioned", i32 1} + +... +--- +name: versioned +alignment: 16 +legalized: true +regBankSelected: true +selected: true +tracksRegLiveness: true +liveins: [] +calleeSavedRegisters: [ '$lr', '$r8', '$r9', '$r10', '$r11', '$r12', '$r13', + '$r14', '$r15', '$p6', '$p7' ] +body: | + bb.0.loop.lver.guard: + successors: %bb.3(0x40000000), %bb.1(0x40000000) + liveins: $p0, $p1, $r0 + + renamable $r1 = MOV_RLC_imm11_pseudo 1 + renamable $r27 = LT renamable $r1, renamable $r0 + renamable $r1 = SEL_NEZ killed renamable $r0, killed renamable $r1, killed renamable $r27 + renamable $r0 = PseudoLoopVersionThreshold -1 + renamable $r2 = LTU renamable $r1, killed renamable $r0 + renamable $r0 = MOV_RLC_imm11_pseudo 0 + JZ killed renamable $r2, %bb.1 + DelayedSchedBarrier + + bb.3.loop.ph: + successors: %bb.4(0x80000000) + liveins: $p0, $p1, $r0, $r1 + + $lc = ADD_NC_mv_add_ri $r1, 0 + $ls = MOVXM %bb.4 + $le = MOVXM + renamable $r1 = MOV_RLC_imm11_pseudo 2 + + bb.4.loop: + successors: %bb.4(0x7c000000), %bb.6(0x04000000) + liveins: $p0, $p1, $r0, $r1 + + renamable $r2 = LSHL renamable $r0, renamable $r1 + $dj0 = MOV_scalar_pseudo killed $r2 + renamable $r2 = LDA_dms_lda_idx renamable $p0, renamable $dj0 :: (load (s32) from %ir.pa) + renamable $r3 = MUL renamable $r2, renamable $r2 + renamable $r3 = MAC killed renamable $r3, renamable $r3, renamable $r2 + renamable $r2 = XOR killed renamable $r3, killed renamable $r2 + ST_dms_sts_idx killed renamable $r2, renamable $p1, killed renamable $dj0 :: (store (s32) into %ir.pb) + renamable $r0 = ADD_add_r_ri killed renamable $r0, 1, implicit-def dead $srcarry + PseudoLoopEnd , %bb.4 + + bb.6.exit: + successors: %bb.5(0x80000000) + + J_lng %bb.5 + DelayedSchedBarrier + + bb.1.loop.ph.lver.high: + successors: %bb.2(0x80000000) + liveins: $p0, $p1, $r0, $r1 + + $lc = ADD_NC_mv_add_ri $r1, 0 + $ls = MOVXM %bb.2 + $le = MOVXM + renamable $r1 = MOV_RLC_imm11_pseudo 2 + + bb.2.loop.lver.high: + successors: %bb.2(0x7c000000), %bb.5(0x04000000) + liveins: $p0, $p1, $r0, $r1 + + renamable $r2 = LSHL renamable $r0, renamable $r1 + $dj0 = MOV_scalar_pseudo killed $r2 + renamable $r2 = LDA_dms_lda_idx renamable $p0, renamable $dj0 :: (load (s32) from %ir.pa.lver.high) + renamable $r3 = MUL renamable $r2, renamable $r2 + renamable $r3 = MAC killed renamable $r3, renamable $r3, renamable $r2 + renamable $r2 = XOR killed renamable $r3, killed renamable $r2 + ST_dms_sts_idx killed renamable $r2, renamable $p1, killed renamable $dj0 :: (store (s32) into %ir.pb.lver.high) + renamable $r0 = ADD_add_r_ri killed renamable $r0, 1, implicit-def dead $srcarry + PseudoLoopEnd , %bb.2 + + bb.5.exit: + RET implicit $lr + DelayedSchedBarrier + +... diff --git a/llvm/test/CodeGen/AIE/aie2ps/postpipeliner-version-guard.mir b/llvm/test/CodeGen/AIE/aie2ps/postpipeliner-version-guard.mir new file mode 100644 index 000000000000..2046ac3eb4be --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2ps/postpipeliner-version-guard.mir @@ -0,0 +1,185 @@ +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +# +# RUN: llc -mtriple=aie2ps -run-pass=postmisched -verify-machineinstrs %s -o - | FileCheck %s +# +# Start from an already-versioned loop: a guard block holding the placeholder +# PseudoLoopVersionThreshold (immediate -1) and the unsigned trip-count compare, +# a low-trip-count (verbatim) copy, and a high-trip-count (to-be-pipelined) copy +# carrying the aie-loop-versioned marker. Running only the post-pipeliner must +# accept the high-trip-count copy despite a small min trip count and patch the +# guard threshold with the schedule's stage count (2 here), materialized into a +# concrete scalar move. +# +# CHECK-LABEL: name: versioned +# The guard threshold placeholder is patched to the stage count and the compare +# tests the trip count against it. +# CHECK-LABEL: bb.0.loop.lver.guard: +# CHECK: MOVA 2 +# CHECK: LTU + +--- | + ; ModuleID = '/tmp/ver.ll' + source_filename = "/tmp/ver.ll" + target datalayout = "e-m:e-p:20:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-f32:32:32-i64:32-f64:32-a:0:32-n32" + target triple = "aie2ps" + + define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { + loop.lver.guard: + %smax = call i32 @llvm.smax.i32(i32 %n, i32 1) + %0 = zext nneg i32 %smax to i33 + %lver.threshold = call i32 @llvm.aie2ps.loop.version.threshold(i32 -1) + %lver.tc.i32 = trunc i33 %0 to i32 + %lver.low = icmp ult i32 %lver.tc.i32, %lver.threshold + br i1 %lver.low, label %loop.ph, label %loop.ph.lver.high + + loop.ph.lver.high: ; preds = %loop.lver.guard + call void @llvm.set.loop.iterations.i32(i32 %smax) + br label %loop.lver.high + + loop.lver.high: ; preds = %loop.lver.high, %loop.ph.lver.high + %i.lver.high = phi i32 [ 0, %loop.ph.lver.high ], [ %i.next.lver.high, %loop.lver.high ] + %pa.lver.high = getelementptr i32, ptr %a, i32 %i.lver.high + %x.lver.high = load i32, ptr %pa.lver.high, align 4 + %m1.lver.high = mul i32 %x.lver.high, %x.lver.high + %m2.lver.high = mul i32 %m1.lver.high, %x.lver.high + %m3.lver.high = add i32 %m2.lver.high, %m1.lver.high + %m4.lver.high = xor i32 %m3.lver.high, %x.lver.high + %pb.lver.high = getelementptr i32, ptr %b, i32 %i.lver.high + store i32 %m4.lver.high, ptr %pb.lver.high, align 4 + %i.next.lver.high = add i32 %i.lver.high, 1 + %1 = call i1 @llvm.loop.decrement.i32(i32 1) + br i1 %1, label %loop.lver.high, label %exit.loopexit, !llvm.loop !0 + + loop.ph: ; preds = %loop.lver.guard + call void @llvm.set.loop.iterations.i32(i32 %smax) + br label %loop + + loop: ; preds = %loop, %loop.ph + %i = phi i32 [ 0, %loop.ph ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %m1 = mul i32 %x, %x + %m2 = mul i32 %m1, %x + %m3 = add i32 %m2, %m1 + %m4 = xor i32 %m3, %x + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %m4, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %2 = call i1 @llvm.loop.decrement.i32(i32 1) + br i1 %2, label %loop, label %exit.loopexit1, !llvm.loop !1 + + exit.loopexit: ; preds = %loop.lver.high + br label %exit + + exit.loopexit1: ; preds = %loop + br label %exit + + exit: ; preds = %exit.loopexit1, %exit.loopexit + ret void + } + + ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) + declare i32 @llvm.smax.i32(i32, i32) #0 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i32 @llvm.aie2ps.loop.version.threshold(i32 immarg) #1 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare void @llvm.set.loop.iterations.i32(i32) #1 + + ; Function Attrs: nocallback noduplicate nofree nosync nounwind willreturn + declare i1 @llvm.loop.decrement.i32(i32) #1 + + attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + attributes #1 = { nocallback noduplicate nofree nosync nounwind willreturn } + + !0 = distinct !{!0, !2} + !1 = distinct !{!1} + !2 = !{!"llvm.loop.hint.aie-loop-versioned", i32 1} + +... +--- +name: versioned +alignment: 16 +legalized: true +regBankSelected: true +selected: true +tracksRegLiveness: true +liveins: [] +calleeSavedRegisters: [ '$lr', '$l4', '$l5', '$l6', '$l7', '$r8', '$r9', + '$r10', '$r11', '$r12', '$r13', '$r14', '$r15', + '$p6', '$p7' ] +body: | + bb.0.loop.lver.guard: + successors: %bb.3(0x40000000), %bb.1(0x40000000) + liveins: $p0, $p1, $r0 + + renamable $r2 = MOV_RLC_imm11_pseudo 1 + renamable $r2 = MAX killed renamable $r0, killed renamable $r2 + renamable $r0 = PseudoLoopVersionThreshold -1 + renamable $r4 = LTU renamable $r2, killed renamable $r0 + renamable $r0 = MOV_RLC_imm11_pseudo 0 + JZ killed renamable $r4, %bb.1 + DelayedSchedBarrier + + bb.3.loop.ph: + successors: %bb.4(0x80000000) + liveins: $p0, $p1, $r0, $r2 + + $lc = ADD_NC_add_lc_ri $r2, 0 + MOVXM_lng_cg_ls_abs %bb.4, implicit-def $ls + MOVXM_lng_cg_le_abs , implicit-def $le + renamable $r2 = MOV_RLC_imm11_pseudo 2 + + bb.4.loop: + successors: %bb.4(0x7c000000), %bb.6(0x04000000) + liveins: $p0, $p1, $r0, $r2 + + renamable $r4 = LSHL renamable $r0, renamable $r2 + $dj0 = MOV_scalar_pseudo killed $r4 + renamable $r4 = LDA_dms_lda_scalar_ld_idx renamable $p0, renamable $dj0 :: (load (s32) from %ir.pa) + renamable $r6 = MUL renamable $r4, renamable $r4 + renamable $r6 = MAC killed renamable $r6, renamable $r6, renamable $r4 + renamable $r4 = XOR killed renamable $r6, killed renamable $r4 + ST_dms_sts_scalar_st_idx killed renamable $r4, renamable $p1, killed renamable $dj0 :: (store (s32) into %ir.pb) + renamable $r0 = ADD_add_r_ri killed renamable $r0, 1, implicit-def dead $srcarry + PseudoLoopEnd , %bb.4 + + bb.6.exit: + successors: %bb.5(0x80000000) + + J_lng %bb.5 + DelayedSchedBarrier + + bb.1.loop.ph.lver.high: + successors: %bb.2(0x80000000) + liveins: $p0, $p1, $r0, $r2 + + $lc = ADD_NC_add_lc_ri $r2, 0 + MOVXM_lng_cg_ls_abs %bb.2, implicit-def $ls + MOVXM_lng_cg_le_abs , implicit-def $le + renamable $r2 = MOV_RLC_imm11_pseudo 2 + + bb.2.loop.lver.high: + successors: %bb.2(0x7c000000), %bb.5(0x04000000) + liveins: $p0, $p1, $r0, $r2 + + renamable $r4 = LSHL renamable $r0, renamable $r2 + $dj0 = MOV_scalar_pseudo killed $r4 + renamable $r4 = LDA_dms_lda_scalar_ld_idx renamable $p0, renamable $dj0 :: (load (s32) from %ir.pa.lver.high) + renamable $r6 = MUL renamable $r4, renamable $r4 + renamable $r6 = MAC killed renamable $r6, renamable $r6, renamable $r4 + renamable $r4 = XOR killed renamable $r6, killed renamable $r4 + ST_dms_sts_scalar_st_idx killed renamable $r4, renamable $p1, killed renamable $dj0 :: (store (s32) into %ir.pb.lver.high) + renamable $r0 = ADD_add_r_ri killed renamable $r0, 1, implicit-def dead $srcarry + PseudoLoopEnd , %bb.2 + + bb.5.exit: + RET implicit $lr + DelayedSchedBarrier + +... From 8cca6fea0324f478fdd92df31d46fe2f82881d1c Mon Sep 17 00:00:00 2001 From: Marc Ludevid Wulf Date: Wed, 8 Jul 2026 10:04:00 -0600 Subject: [PATCH 4/4] [AIE] Add inner-loop-versioning IR pass --- llvm/lib/Target/AIE/AIE.h | 7 + llvm/lib/Target/AIE/AIE2TargetMachine.cpp | 1 + llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp | 1 + .../lib/Target/AIE/AIEInnerLoopVersioning.cpp | 367 ++++++++++++++++++ llvm/lib/Target/AIE/CMakeLists.txt | 1 + llvm/lib/Target/AIE/Utils/AIEIRUtils.cpp | 37 ++ llvm/lib/Target/AIE/Utils/AIEIRUtils.h | 13 + .../aie2/inner-loop-versioning-siblings.ll | 53 +++ .../AIE/aie2/inner-loop-versioning-swp.ll | 58 +++ .../CodeGen/AIE/aie2/inner-loop-versioning.ll | 77 ++++ .../CodeGen/AIE/aie2/llc-pipeline-aie2.ll | 4 + .../aie2p/inner-loop-versioning-siblings.ll | 53 +++ .../AIE/aie2p/inner-loop-versioning-swp.ll | 58 +++ .../AIE/aie2p/inner-loop-versioning.ll | 77 ++++ .../CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll | 8 + .../aie2ps/inner-loop-versioning-siblings.ll | 53 +++ .../AIE/aie2ps/inner-loop-versioning-swp.ll | 58 +++ .../AIE/aie2ps/inner-loop-versioning.ll | 77 ++++ .../CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll | 8 + 19 files changed, 1011 insertions(+) create mode 100644 llvm/lib/Target/AIE/AIEInnerLoopVersioning.cpp create mode 100644 llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-siblings.ll create mode 100644 llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-swp.ll create mode 100644 llvm/test/CodeGen/AIE/aie2/inner-loop-versioning.ll create mode 100644 llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-siblings.ll create mode 100644 llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-swp.ll create mode 100644 llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning.ll create mode 100644 llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-siblings.ll create mode 100644 llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-swp.ll create mode 100644 llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning.ll diff --git a/llvm/lib/Target/AIE/AIE.h b/llvm/lib/Target/AIE/AIE.h index 5119c2e48702..f1f318781196 100644 --- a/llvm/lib/Target/AIE/AIE.h +++ b/llvm/lib/Target/AIE/AIE.h @@ -120,6 +120,13 @@ llvm::FunctionPass *createReservedRegsLICMPass(); extern char &AIEOuterLoopPipelinerID; void initializeAIEOuterLoopPipelinerPass(PassRegistry &); llvm::FunctionPass *createAIEOuterLoopPipelinerPass(); + +// Inner Loop Versioning (IR-level). Emits a runtime trip-count guard around a +// pipelined copy of a single-block inner loop whose minimum trip count is too +// small for the software pipeliner. See AIEInnerLoopVersioning.cpp. +extern char &AIEInnerLoopVersioningID; +void initializeAIEInnerLoopVersioningPass(PassRegistry &); +llvm::FunctionPass *createAIEInnerLoopVersioningPass(); } // namespace llvm #endif diff --git a/llvm/lib/Target/AIE/AIE2TargetMachine.cpp b/llvm/lib/Target/AIE/AIE2TargetMachine.cpp index 59e30d38d107..cd7afc516c6e 100644 --- a/llvm/lib/Target/AIE/AIE2TargetMachine.cpp +++ b/llvm/lib/Target/AIE/AIE2TargetMachine.cpp @@ -79,6 +79,7 @@ TargetPassConfig *AIE2TargetMachine::createPassConfig(PassManagerBase &PM) { bool AIE2PassConfig::addPreISel() { if (TM->getOptLevel() != CodeGenOptLevel::None) { + addPass(createAIEInnerLoopVersioningPass()); addPass(createHardwareLoopsLegacyPass()); addPass(createAIEOuterLoopPipelinerPass()); } diff --git a/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp b/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp index bdf4cef46f3a..4e05735fc0e9 100644 --- a/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp +++ b/llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp @@ -177,6 +177,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAIETarget() { initializeAIERegClassConstrainerPass(*PR); initializeReservedRegsLICMPass(*PR); initializeAIEOuterLoopPipelinerPass(*PR); + initializeAIEInnerLoopVersioningPass(*PR); initializeAIESpillSlotOptimizationPass(*PR); } diff --git a/llvm/lib/Target/AIE/AIEInnerLoopVersioning.cpp b/llvm/lib/Target/AIE/AIEInnerLoopVersioning.cpp new file mode 100644 index 000000000000..408a0af196c9 --- /dev/null +++ b/llvm/lib/Target/AIE/AIEInnerLoopVersioning.cpp @@ -0,0 +1,367 @@ +//===- AIEInnerLoopVersioning.cpp - Inner loop versioning for AIE ---------===// +// +// This file is licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +// +//===----------------------------------------------------------------------===// +// +// Inner loop versioning for AIE. +// +// A loop carrying the llvm.loop.hint.aie-loop-versioning hint is split into two +// copies guarded by a runtime trip-count check. Large trip counts run a +// specialized copy the postpipeliner may pipeline aggressively, while small +// trip counts stay correct on the original loop. The block structure below is +// what the postpipeliner's guard finder (findVersionGuard) relies on: a guard +// block with two successors, each a dedicated fallthrough preheader of its +// copy. (The block names are for humans; findVersionGuard matches on structure, +// not names.) +// +// [.lver.guard] if (trip < threshold) -> low else -> high +// / \ +// [.ph] [.ph.lver.high] +// | | +// [] [.lver.high] the high-trip-count copy is the clone; +// \ / it keeps the hint and gets pipelined. +// \ / the low-trip-count copy is the original +// \ / loop, kept verbatim as the fallback. +// [exit] +// +// The threshold is produced by a thin per-subtarget intrinsic that lowers to a +// non-CSE-able pseudo, giving the postpipeliner a stable handle to patch. After +// scheduling, the postpipeliner overwrites the placeholder with the required +// stage count, so the high-trip-count copy runs exactly when the trip count is +// large enough for its schedule. +// +// The placeholder is UINT32_MAX (-1): the guard compare is unsigned, so until +// the postpipeliner patches it, every trip count routes to the low-trip-count +// (verbatim, un-pipelined) copy. This fails safe -- if the guard is never +// patched (the high-trip-count copy is not pipelined, or updateVersionGuard +// bails on a reshaped region), the pipelined copy is simply never entered. +// +// Running before HardwareLoops lets both copies lower to ZOL uniformly. +// +//===----------------------------------------------------------------------===// + +#include "AIE.h" +#include "Utils/AIEIRUtils.h" +#include "Utils/AIELoopUtils.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/Debug.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/LoopSimplify.h" +#include "llvm/Transforms/Utils/LoopUtils.h" +#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" +#include "llvm/Transforms/Utils/ValueMapper.h" + +using namespace llvm; + +#define DEBUG_TYPE "aie-inner-loop-versioning" + +namespace { + +class AIEInnerLoopVersioning : public FunctionPass { +public: + static char ID; + AIEInnerLoopVersioning() : FunctionPass(ID) {} + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + // We deliberately do not force LoopSimplify/LCSSA: that canonicalization + // would perturb the IR seen by later passes (e.g. the outer-loop + // pipeliner) even for functions with no versioning hint. Instead we work + // with whatever form the loop is already in and bail on unsuitable loops. + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + FunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { return "AIE Inner Loop Versioning"; } +}; + +/// Versions a single loop: clone it into a high-trip-count copy, route a +/// runtime trip-count guard to either copy, and fix up metadata. One instance +/// is constructed per candidate loop; call tryVersionLoop() to run the +/// transform. +class AIELoopVersioner { +public: + AIELoopVersioner(Loop &L, DominatorTree &DT, LoopInfo &LI, + ScalarEvolution &SE) + : L(L), DT(DT), LI(LI), SE(SE) {} + + /// Version the loop. Returns true if it was versioned, false if versioning + /// bailed (in which case the loop is left unchanged). + bool tryVersionLoop(); + +private: + Loop &L; + DominatorTree &DT; + LoopInfo &LI; + ScalarEvolution &SE; + + /// Read-only profitability check: whether versioning this loop is worthwhile + /// and safe, based solely on its trip-count SCEV. Does not mutate the IR. + bool isProfitable() const; + /// Canonicalize the loop and check its structure supports versioning. Returns + /// the unique exit block on success, or nullptr if the loop is unsuitable. + BasicBlock *canonicalizeAndCheckStructure(); + /// Materialize the loop trip count at InsertPt, or nullptr if it cannot be + /// computed / expanded there. + Value *expandTripCount(Instruction *InsertPt) const; + /// Merge loop-defined values used after the loop across the two copies. + void addExitPHIs(Loop *ClonedLoop, BasicBlock *ExitBlock, + ValueToValueMapTy &VMap, + ArrayRef DefsUsedOutside) const; + /// Remove the versioning request hint so the postpipeliner leaves \p Loop + /// alone. + void stripVersioningHint(Loop &Loop) const; +}; + +/// True when \p L carries the iteration-count versioning request hint. +bool isIterCountVersioningEnabled(const Loop *L) { + std::optional Hint = AIELoopUtils::getLoopHintInt( + L->getLoopID(), AIELoopUtils::LoopVersioningHintKey); + return Hint && *Hint > 0; +} + +} // namespace + +char AIEInnerLoopVersioning::ID = 0; +char &llvm::AIEInnerLoopVersioningID = AIEInnerLoopVersioning::ID; + +INITIALIZE_PASS_BEGIN(AIEInnerLoopVersioning, DEBUG_TYPE, + "AIE Inner Loop Versioning", false, false) +INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) +INITIALIZE_PASS_END(AIEInnerLoopVersioning, DEBUG_TYPE, + "AIE Inner Loop Versioning", false, false) + +llvm::FunctionPass *llvm::createAIEInnerLoopVersioningPass() { + return new AIEInnerLoopVersioning(); +} + +bool AIEInnerLoopVersioning::runOnFunction(Function &F) { + if (skipFunction(F)) + return false; + + Triple TT(F.getParent()->getTargetTriple()); + Intrinsic::ID ThresholdIID = AIEIRUtils::getLoopVersionThresholdIntrinsic(TT); + if (ThresholdIID == Intrinsic::not_intrinsic) + return false; + + auto &LI = getAnalysis().getLoopInfo(); + auto &DT = getAnalysis().getDomTree(); + auto &SE = getAnalysis().getSE(); + + LLVM_DEBUG(dbgs() << "AIEInnerLoopVersioning: " << F.getName() << "\n"); + + // Collect enabled loops before mutating, so cloning does not disturb + // iteration. + SmallVector Candidates; + for (Loop *L : LI.getLoopsInPreorder()) + if (isIterCountVersioningEnabled(L)) + Candidates.push_back(L); + + bool Changed = false; + for (Loop *L : Candidates) + Changed |= AIELoopVersioner(*L, DT, LI, SE).tryVersionLoop(); + return Changed; +} + +Value *AIELoopVersioner::expandTripCount(Instruction *InsertPt) const { + // isProfitable() already guaranteed the backedge-taken count is computable + // and fits i32. + const SCEV *BEC = SE.getBackedgeTakenCount(&L); + // TripCount = BackedgeTakenCount + 1. + const SCEV *TC = SE.getTripCountFromExitCount(BEC); + SCEVExpander Exp(SE, InsertPt->getDataLayout(), "lver.tc"); + if (!Exp.isSafeToExpandAt(TC, InsertPt)) + return nullptr; + return Exp.expandCodeFor(TC, TC->getType(), InsertPt); +} + +bool AIELoopVersioner::isProfitable() const { + // The runtime guard compares the trip count in i32, and the pipelined + // high-trip-count copy can only become a zero-overhead loop if its trip count + // fits in a 32-bit register (see AIETTICommon::isHardwareLoopProfitable). + const SCEV *BEC = SE.getBackedgeTakenCount(&L); + if (isa(BEC)) + return false; + // If the exit count is wider than i32, versioning is both pointless (the copy + // will never pipeline) and unsafe (the later i32 truncation could drop high + // bits). Count-up i32 loops are unaffected: their exit count is i32 even + // though TripCountFromExitCount widens the "+1" to avoid overflow. + return SE.getTypeSizeInBits(BEC->getType()) <= 32; +} + +BasicBlock *AIELoopVersioner::canonicalizeAndCheckStructure() { + // We need simplify form for a unique preheader and a single merge point for + // the two copies' results. Rather than force global LoopSimplify/LCSSA (which + // would perturb non-hinted loops seen by later passes), canonicalize just + // this hinted candidate on demand. + simplifyLoop(&L, &DT, &LI, &SE, /*AC=*/nullptr, /*MSSAU=*/nullptr, + /*PreserveLCSSA=*/false); + formLCSSARecursively(L, DT, &LI, &SE); + if (!L.isLoopSimplifyForm()) { + LLVM_DEBUG(dbgs() << " Not in simplify form\n"); + return nullptr; + } + assert(L.isLCSSAForm(DT) && "loop must be in LCSSA form for exit PHIs"); + + BasicBlock *ExitBlock = L.getUniqueExitBlock(); + if (!ExitBlock || !L.getExitingBlock()) { + LLVM_DEBUG(dbgs() << " No unique exit / exiting block\n"); + return nullptr; + } + return ExitBlock; +} + +bool AIELoopVersioner::tryVersionLoop() { + // Read-only profitability gate first, so an unprofitable loop is rejected + // without any IR mutation. + if (!isProfitable()) { + LLVM_DEBUG(dbgs() << " Not profitable to version\n"); + stripVersioningHint(L); + return false; + } + + BasicBlock *ExitBlock = canonicalizeAndCheckStructure(); + if (!ExitBlock) { + // Versioning bailed but the request hint is still on the loop. Consume it + // (without adding the versioned marker) so a re-run does not retry this + // loop and the post-pipeliner does not look for a guard never built. + stripVersioningHint(L); + return false; + } + + BasicBlock *GuardBB = L.getLoopPreheader(); + BasicBlock *Header = L.getHeader(); + LLVMContext &Ctx = Header->getContext(); + Type *I32Ty = Type::getInt32Ty(Ctx); + + Value *TripCount = expandTripCount(GuardBB->getTerminator()); + if (!TripCount) { + LLVM_DEBUG(dbgs() << " Trip count not expandable\n"); + stripVersioningHint(L); + return false; + } + + LLVM_DEBUG(dbgs() << " Versioning loop "; Header->printAsOperand(dbgs()); + dbgs() << "\n"); + + // Build the runtime guard: take the low-trip-count copy when trip < + // threshold. The threshold is a placeholder from the thin intrinsic, patched + // later by the postpipeliner (see the file header for the fail-safe + // rationale). Seed it with -1 (UINT32_MAX): besides failing safe, -1 fits the + // narrow scalar-move immediate the pseudo materializes into, unlike a literal + // INT32_MAX, so the fallback move is emitted intact. + IRBuilder<> Builder(GuardBB->getTerminator()); + Triple TT(Header->getModule()->getTargetTriple()); + Intrinsic::ID ThresholdIID = AIEIRUtils::getLoopVersionThresholdIntrinsic(TT); + Value *Threshold = + Builder.CreateIntrinsic(ThresholdIID, {}, {Builder.getInt32(-1)}, + /*FMFSource=*/nullptr, "lver.threshold"); + if (TripCount->getType() != I32Ty) + TripCount = Builder.CreateZExtOrTrunc(TripCount, I32Ty, "lver.tc.i32"); + Value *TakeLow = Builder.CreateICmpULT(TripCount, Threshold, "lver.low"); + GuardBB->setName(Header->getName() + ".lver.guard"); + + SmallVector DefsUsedOutside = findDefsUsedOutsideOfLoop(&L); + + // Split off an empty preheader for the original (low-trip-count) copy, then + // clone the loop into the high-trip-count copy dominated by the guard block. + BasicBlock *LowPH = SplitBlock(GuardBB, GuardBB->getTerminator(), &DT, &LI, + nullptr, Header->getName() + ".ph"); + ValueToValueMapTy VMap; + SmallVector HighBlocks; + Loop *HighLoop = cloneLoopWithPreheader(LowPH, GuardBB, &L, VMap, + ".lver.high", &LI, &DT, HighBlocks); + remapInstructionsInBlocks(HighBlocks, VMap); + + // Replace the guard's fall-through with the trip-count branch: below the + // threshold run the original (low) copy, otherwise the pipelined high copy. + Instruction *OrigTerm = GuardBB->getTerminator(); + Builder.SetInsertPoint(OrigTerm); + Builder.CreateCondBr(TakeLow, LowPH, HighLoop->getLoopPreheader()); + OrigTerm->eraseFromParent(); + + DT.changeImmediateDominator(ExitBlock, GuardBB); + addExitPHIs(HighLoop, ExitBlock, VMap, DefsUsedOutside); + formDedicatedExitBlocks(HighLoop, &DT, &LI, nullptr, /*PreserveLCSSA=*/true); + formDedicatedExitBlocks(&L, &DT, &LI, nullptr, /*PreserveLCSSA=*/true); + + // Consume the request hint on both copies so a second run of the pass does + // not re-version either. Mark the high-trip-count copy as versioned (carrying + // the request's value) so the postpipeliner recognizes it as the pipelined + // copy; the low copy stays the verbatim un-pipelined fallback. + int64_t HintValue = *AIELoopUtils::getLoopHintInt( + L.getLoopID(), AIELoopUtils::LoopVersioningHintKey); + stripVersioningHint(L); + stripVersioningHint(*HighLoop); + addStringMetadataToLoop(HighLoop, AIELoopUtils::LoopVersionedHintKey.data(), + HintValue); + + // The runtime guard already guarantees the high copy's trip count, making its + // llvm.loop.itercount.range redundant. Drop it: a small minimum (e.g. 1) + // would otherwise make the hardware-loop profitability gate reject the ZOL + // the postpipeliner needs. + AIEIRUtils::dropLoopMetadata(*HighLoop, + StringRef("llvm.loop.itercount.range")); + + return true; +} + +void AIELoopVersioner::addExitPHIs( + Loop *ClonedLoop, BasicBlock *ExitBlock, ValueToValueMapTy &VMap, + ArrayRef DefsUsedOutside) const { + PHINode *PN = nullptr; + // Give every value used after the loop a merge PHI fed by the original copy. + for (Instruction *Inst : DefsUsedOutside) { + for (auto I = ExitBlock->begin(); (PN = dyn_cast(I)); ++I) + if (PN->getIncomingValue(0) == Inst) + break; + if (PN) + continue; + PN = PHINode::Create(Inst->getType(), 2, Inst->getName() + ".lver"); + PN->insertBefore(ExitBlock->begin()); + SmallVector UsersToUpdate; + for (User *U : Inst->users()) + if (!L.contains(cast(U)->getParent())) + UsersToUpdate.push_back(U); + for (User *U : UsersToUpdate) + U->replaceUsesOfWith(Inst, PN); + PN->addIncoming(Inst, L.getExitingBlock()); + } + // Add the matching incoming value from the cloned copy. Incoming 0 is the + // original copy's value: these are freshly formed LCSSA phis + // (formLCSSARecursively ran in canonicalizeAndCheckStructure), and every + // merge phi created above adds the original value as its sole incoming, so + // operand 0 is unambiguously the original edge. + for (auto I = ExitBlock->begin(); (PN = dyn_cast(I)); ++I) { + assert(PN->getIncomingBlock(0) == L.getExitingBlock() && + "exit PHI incoming 0 is expected to be the original copy's value"); + Value *OrigValue = PN->getIncomingValue(0); + auto Mapped = VMap.find(OrigValue); + Value *ClonedValue = + Mapped != VMap.end() ? cast(Mapped->second) : OrigValue; + PN->addIncoming(ClonedValue, ClonedLoop->getExitingBlock()); + } +} + +void AIELoopVersioner::stripVersioningHint(Loop &Loop) const { + AIEIRUtils::dropLoopMetadata(Loop, + StringRef(AIELoopUtils::LoopVersioningHintKey)); +} diff --git a/llvm/lib/Target/AIE/CMakeLists.txt b/llvm/lib/Target/AIE/CMakeLists.txt index 79265658531a..1be8a0fa4022 100644 --- a/llvm/lib/Target/AIE/CMakeLists.txt +++ b/llvm/lib/Target/AIE/CMakeLists.txt @@ -115,6 +115,7 @@ add_llvm_target(AIECodeGen AIEBaseHardwareLoops.cpp AIEBaseISelLowering.cpp AIEBaseInstrInfo.cpp + AIEInnerLoopVersioning.cpp AIEBaseRegisterInfo.cpp AIEBasePipelinerLoopInfo.cpp AIEBaseInstructionSelector.cpp diff --git a/llvm/lib/Target/AIE/Utils/AIEIRUtils.cpp b/llvm/lib/Target/AIE/Utils/AIEIRUtils.cpp index 8e4cef484c1c..ed2eaa45f3ea 100644 --- a/llvm/lib/Target/AIE/Utils/AIEIRUtils.cpp +++ b/llvm/lib/Target/AIE/Utils/AIEIRUtils.cpp @@ -9,12 +9,20 @@ //===----------------------------------------------------------------------===// #include "AIEIRUtils.h" +#include "AIELoopUtils.h" #include "llvm/ADT/APInt.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsAIE2.h" +#include "llvm/IR/IntrinsicsAIE2P.h" +#include "llvm/IR/IntrinsicsAIE2PS.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/Support/KnownBits.h" +#include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" #include "llvm/Transforms/Utils/Local.h" @@ -38,6 +46,35 @@ bool isHardwareLoopDecrement(const Instruction *I) { return getIntrinsicID(I) == Intrinsic::loop_decrement; } +Intrinsic::ID getLoopVersionThresholdIntrinsic(const Triple &TT) { + if (TT.isAIE2()) + return Intrinsic::aie2_loop_version_threshold; + if (TT.isAIE2P()) + return Intrinsic::aie2p_loop_version_threshold; + if (TT.isAIE2PS()) + return Intrinsic::aie2ps_loop_version_threshold; + return Intrinsic::not_intrinsic; +} + +void dropLoopMetadata(Loop &L, ArrayRef Keys) { + MDNode *LoopID = L.getLoopID(); + if (!LoopID) + return; + LLVMContext &Ctx = L.getHeader()->getContext(); + // Operand 0 of a loop id is a self-reference; reserve it with a placeholder. + SmallVector MDs(1, nullptr); + for (unsigned I = 1, E = LoopID->getNumOperands(); I < E; ++I) { + MDNode *Entry = cast(LoopID->getOperand(I)); + auto Key = AIELoopUtils::getMetadataKey(*Entry); + if (Key && llvm::is_contained(Keys, *Key)) + continue; + MDs.push_back(Entry); + } + MDNode *NewLoopID = MDNode::getDistinct(Ctx, MDs); + NewLoopID->replaceOperandWith(0, NewLoopID); + L.setLoopID(NewLoopID); +} + std::optional instCombineDemandedBits(InstCombiner &IC, IntrinsicInst &II, unsigned NumBits, diff --git a/llvm/lib/Target/AIE/Utils/AIEIRUtils.h b/llvm/lib/Target/AIE/Utils/AIEIRUtils.h index 28fb46b87ebb..71a5003c56fc 100644 --- a/llvm/lib/Target/AIE/Utils/AIEIRUtils.h +++ b/llvm/lib/Target/AIE/Utils/AIEIRUtils.h @@ -11,6 +11,9 @@ #ifndef LLVM_LIB_TARGET_AIE_UTILS_AIEIRUTILS_H #define LLVM_LIB_TARGET_AIE_UTILS_AIEIRUTILS_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Intrinsics.h" #include namespace llvm { @@ -18,6 +21,8 @@ class Instruction; class IntrinsicInst; class Type; class InstCombiner; +class Triple; +class Loop; } // namespace llvm namespace llvm::AIEIRUtils { @@ -49,6 +54,14 @@ std::optional instCombineDemandedBits(InstCombiner &IC, unsigned NumBits, unsigned Operand = 0); +/// Return the subtarget-specific loop-version-threshold intrinsic for \p TT, or +/// Intrinsic::not_intrinsic if the subtarget does not support loop versioning. +Intrinsic::ID getLoopVersionThresholdIntrinsic(const Triple &TT); + +/// Rebuild \p L's loop id, dropping every metadata entry whose string key is in +/// \p Keys. A no-op if the loop has no loop id. +void dropLoopMetadata(Loop &L, ArrayRef Keys); + } // namespace llvm::AIEIRUtils #endif // LLVM_LIB_TARGET_AIE_UTILS_AIEIRUTILS_H diff --git a/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-siblings.ll b/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-siblings.ll new file mode 100644 index 000000000000..87e463ff0ccf --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-siblings.ll @@ -0,0 +1,53 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; Two versioned loops in one function. Each guard's threshold pseudo is +; non-CSE-able, so the two placeholders never merge and the post-pipeliner +; patches each guard independently with its own stage count. Mis-identification +; would abort compilation (report_fatal_error), so reaching the end and emitting +; two guards is the check. +; +; RUN: llc -mtriple=aie2 -O2 -aie-force-postpipeliner %s -o - | FileCheck %s + +; CHECK-LABEL: two_versioned: +; Both loops are versioned: two guard blocks, each with its own patched +; threshold move and trip-count compare. +; CHECK-DAG: loop1.lver.guard +; CHECK-DAG: loop2.lver.guard +; CHECK-DAG: ltu r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}} + +define void @two_versioned(ptr noalias %a, ptr noalias %b, i32 %n, i32 %m) { +entry: + br label %loop1 +loop1: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop1 ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 7 + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c1 = icmp slt i32 %i.next, %n + br i1 %c1, label %loop1, label %mid, !llvm.loop !0 +mid: + br label %loop2 +loop2: + %j = phi i32 [ 0, %mid ], [ %j.next, %loop2 ] + %pb2 = getelementptr i32, ptr %b, i32 %j + %z = load i32, ptr %pb2, align 4 + %w = add i32 %z, 99 + %pa2 = getelementptr i32, ptr %a, i32 %j + store i32 %w, ptr %pa2, align 4 + %j.next = add i32 %j, 1 + %c2 = icmp slt i32 %j.next, %m + br i1 %c2, label %loop2, label %exit, !llvm.loop !2 +exit: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} +!2 = distinct !{!2, !1} diff --git a/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-swp.ll b/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-swp.ll new file mode 100644 index 000000000000..325d5260f213 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning-swp.ll @@ -0,0 +1,58 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; End-to-end: a versioned loop that the post-pipeliner can schedule gets its +; guard threshold patched with the required stage count (>= 2). The threshold is +; a dedicated side-effecting multi-slot pseudo (so MachineCSE never merges +; sibling guards' placeholders); the post-RA scheduler materializes it into a +; concrete scalar move. Without patching, the placeholder would remain 0. +; +; This also pins down the correctness coupling that has no other direct test: +; the guard threshold (NStages) and the high-trip-count copy's ZOL trip-count +; adjustment (-(NStages-1)) must agree. The guard guarantees the pipelined copy +; only runs when count >= NStages, so lc = count-(NStages-1) stays >= 1. Here +; NStages is 2: threshold #2, high-copy peel #-1, and the low copy keeps an +; unpeeled #0. +; +; RUN: llc -mtriple=aie2 -O2 -aie-force-postpipeliner %s -o - | FileCheck %s + +; CHECK-LABEL: versioned: +; The guard block holds the patched threshold (the stage count) and the unsigned +; trip-count compare selecting the pipelined vs fallback copy. The threshold is a +; small positive constant materialized into a scalar register (a mova, no +; slot-blocking movxm), and the compare tests the trip count against it. +; CHECK: mova [[THR:r[0-9]+]], #[[NSTAGES:[2-9][0-9]*]] +; CHECK: ltu r{{[0-9]+}}, r{{[0-9]+}}, [[THR]] +; The low-trip-count (fallback) copy runs the loop verbatim: its ZOL count is +; unpeeled. +; CHECK-LABEL: %loop.ph +; The high-trip-count (pipelined) copy peels NStages-1 stages from its ZOL +; count. For NStages == 2 that is an adjustment of -1. +; CHECK-LABEL: %loop.ph.lver.high +; CHECK: add.nc lc, r{{[0-9]+}}, #-1 + +define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %m1 = mul i32 %x, %x + %m2 = mul i32 %m1, %x + %m3 = add i32 %m2, %m1 + %m4 = xor i32 %m3, %x + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %m4, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} diff --git a/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning.ll b/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning.ll new file mode 100644 index 000000000000..042d8820a54f --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2/inner-loop-versioning.ll @@ -0,0 +1,77 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; IR-level loop-versioning transform: a loop carrying the versioning hint is +; split into a high-trip-count copy (the clone, keeps the hint, pipelined later) +; and a low-trip-count copy (the original, hint stripped), behind a runtime +; trip-count guard. The guard threshold is a placeholder produced by the thin +; intrinsic; the postpipeliner patches it. +; +; RUN: llc -mtriple=aie2 -stop-after=aie-inner-loop-versioning %s -o - \ +; RUN: | FileCheck %s + +; CHECK-LABEL: define void @versioned +; The guard: threshold from the intrinsic, unsigned trip-count compare. Below +; the threshold take the low-trip-count (original) copy, otherwise the cloned +; high-trip-count copy. +; CHECK: %[[THR:.*]] = call i32 @llvm.aie2.loop.version.threshold(i32 -1) +; CHECK: %[[LOW:.*]] = icmp ult i32 %{{.*}}, %[[THR]] +; CHECK: br i1 %[[LOW]], label %{{.*}}.ph, label %{{.*}}.lver.high +; The cloned high-trip-count copy carries the ".lver.high" suffix and is marked +; with the aie-loop-versioned hint. +; CHECK: loop.lver.high: +; CHECK: br i1 %{{.*}}, !llvm.loop [[HIGHMD:![0-9]+]] +; The original low-trip-count copy's loop id is rebuilt without the hint. +; CHECK: br i1 %{{.*}}, !llvm.loop [[LOWMD:![0-9]+]] + +define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 1234 + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +; A loop whose exit count is wider than i32 (i64 induction variable) cannot be +; expressed as a 32-bit zero-overhead loop, so the high-trip-count copy could +; never be pipelined. Versioning bails and leaves the loop untouched: no guard, +; no clone. +; CHECK-LABEL: define void @wide_trip_count +; CHECK-NOT: icmp ult +; CHECK-NOT: lver +define void @wide_trip_count(ptr noalias %a, ptr noalias %b, i64 %n) { +entry: + br label %loop +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i64 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 1234 + %pb = getelementptr i32, ptr %b, i64 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +; Metadata defs live at the module end, past both CHECK-LABEL barriers: the +; high-trip-count copy's loop id references the aie-loop-versioned marker. +; CHECK-DAG: [[HIGHMD]] = distinct !{[[HIGHMD]], [[MARK:![0-9]+]]} +; CHECK-DAG: [[MARK]] = !{!"llvm.loop.hint.aie-loop-versioned", i32 1} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} diff --git a/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll b/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll index f3c3cb8cd341..c29b17c9d0da 100644 --- a/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll +++ b/llvm/test/CodeGen/AIE/aie2/llc-pipeline-aie2.ll @@ -98,6 +98,10 @@ ; AIE-O123-NEXT: Dominator Tree Construction ; AIE-O123-NEXT: Natural Loop Information ; AIE-O123-NEXT: Scalar Evolution Analysis +; AIE-O123-NEXT: AIE Inner Loop Versioning +; AIE-O123-NEXT: Dominator Tree Construction +; AIE-O123-NEXT: Natural Loop Information +; AIE-O123-NEXT: Scalar Evolution Analysis ; AIE-O123-NEXT: Lazy Branch Probability Analysis ; AIE-O123-NEXT: Lazy Block Frequency Analysis ; AIE-O123-NEXT: Optimization Remark Emitter diff --git a/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-siblings.ll b/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-siblings.ll new file mode 100644 index 000000000000..749255f660ff --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-siblings.ll @@ -0,0 +1,53 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; Two versioned loops in one function. Each guard's threshold pseudo is +; non-CSE-able, so the two placeholders never merge and the post-pipeliner +; patches each guard independently with its own stage count. Mis-identification +; would abort compilation (report_fatal_error), so reaching the end and emitting +; two guards is the check. +; +; RUN: llc -mtriple=aie2p -O2 -aie-force-postpipeliner %s -o - | FileCheck %s + +; CHECK-LABEL: two_versioned: +; Both loops are versioned: two guard blocks, each with its own patched +; threshold move and trip-count compare. +; CHECK-DAG: loop1.lver.guard +; CHECK-DAG: loop2.lver.guard +; CHECK-DAG: ltu r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}} + +define void @two_versioned(ptr noalias %a, ptr noalias %b, i32 %n, i32 %m) { +entry: + br label %loop1 +loop1: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop1 ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 7 + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c1 = icmp slt i32 %i.next, %n + br i1 %c1, label %loop1, label %mid, !llvm.loop !0 +mid: + br label %loop2 +loop2: + %j = phi i32 [ 0, %mid ], [ %j.next, %loop2 ] + %pb2 = getelementptr i32, ptr %b, i32 %j + %z = load i32, ptr %pb2, align 4 + %w = add i32 %z, 99 + %pa2 = getelementptr i32, ptr %a, i32 %j + store i32 %w, ptr %pa2, align 4 + %j.next = add i32 %j, 1 + %c2 = icmp slt i32 %j.next, %m + br i1 %c2, label %loop2, label %exit, !llvm.loop !2 +exit: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} +!2 = distinct !{!2, !1} diff --git a/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-swp.ll b/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-swp.ll new file mode 100644 index 000000000000..9dd0a38565d7 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning-swp.ll @@ -0,0 +1,58 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; End-to-end: a versioned loop that the post-pipeliner can schedule gets its +; guard threshold patched with the required stage count (>= 2). The threshold is +; a dedicated side-effecting multi-slot pseudo (so MachineCSE never merges +; sibling guards' placeholders); the post-RA scheduler materializes it into a +; concrete scalar move. Without patching, the placeholder would remain 0. +; +; This also pins down the correctness coupling that has no other direct test: +; the guard threshold (NStages) and the high-trip-count copy's ZOL trip-count +; adjustment (-(NStages-1)) must agree. The guard guarantees the pipelined copy +; only runs when count >= NStages, so lc = count-(NStages-1) stays >= 1. Here +; NStages is 2: threshold #2, high-copy peel #-1, and the low copy keeps an +; unpeeled #0. +; +; RUN: llc -mtriple=aie2p -O2 -aie-force-postpipeliner %s -o - | FileCheck %s + +; CHECK-LABEL: versioned: +; The guard block holds the patched threshold (the stage count) and the unsigned +; trip-count compare selecting the pipelined vs fallback copy. The threshold is a +; small positive constant materialized into a scalar register (a mova, no +; slot-blocking movxm), and the compare tests the trip count against it. +; CHECK: mova [[THR:r[0-9]+]], #[[NSTAGES:[2-9][0-9]*]] +; CHECK: ltu r{{[0-9]+}}, r{{[0-9]+}}, [[THR]] +; The low-trip-count (fallback) copy runs the loop verbatim: its ZOL count is +; unpeeled. +; CHECK-LABEL: %loop.ph +; The high-trip-count (pipelined) copy peels NStages-1 stages from its ZOL +; count. For NStages == 2 that is an adjustment of -1. +; CHECK-LABEL: %loop.ph.lver.high +; CHECK: add.nc lc, r{{[0-9]+}}, #-1 + +define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %m1 = mul i32 %x, %x + %m2 = mul i32 %m1, %x + %m3 = add i32 %m2, %m1 + %m4 = xor i32 %m3, %x + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %m4, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} diff --git a/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning.ll b/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning.ll new file mode 100644 index 000000000000..c17df3c3b8ed --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/inner-loop-versioning.ll @@ -0,0 +1,77 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; IR-level loop-versioning transform: a loop carrying the versioning hint is +; split into a high-trip-count copy (the clone, keeps the hint, pipelined later) +; and a low-trip-count copy (the original, hint stripped), behind a runtime +; trip-count guard. The guard threshold is a placeholder produced by the thin +; intrinsic; the postpipeliner patches it. +; +; RUN: llc -mtriple=aie2p -stop-after=aie-inner-loop-versioning %s -o - \ +; RUN: | FileCheck %s + +; CHECK-LABEL: define void @versioned +; The guard: threshold from the intrinsic, unsigned trip-count compare. Below +; the threshold take the low-trip-count (original) copy, otherwise the cloned +; high-trip-count copy. +; CHECK: %[[THR:.*]] = call i32 @llvm.aie2p.loop.version.threshold(i32 -1) +; CHECK: %[[LOW:.*]] = icmp ult i32 %{{.*}}, %[[THR]] +; CHECK: br i1 %[[LOW]], label %{{.*}}.ph, label %{{.*}}.lver.high +; The cloned high-trip-count copy carries the ".lver.high" suffix and is marked +; with the aie-loop-versioned hint. +; CHECK: loop.lver.high: +; CHECK: br i1 %{{.*}}, !llvm.loop [[HIGHMD:![0-9]+]] +; The original low-trip-count copy's loop id is rebuilt without the hint. +; CHECK: br i1 %{{.*}}, !llvm.loop [[LOWMD:![0-9]+]] + +define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 1234 + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +; A loop whose exit count is wider than i32 (i64 induction variable) cannot be +; expressed as a 32-bit zero-overhead loop, so the high-trip-count copy could +; never be pipelined. Versioning bails and leaves the loop untouched: no guard, +; no clone. +; CHECK-LABEL: define void @wide_trip_count +; CHECK-NOT: icmp ult +; CHECK-NOT: lver +define void @wide_trip_count(ptr noalias %a, ptr noalias %b, i64 %n) { +entry: + br label %loop +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i64 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 1234 + %pb = getelementptr i32, ptr %b, i64 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +; Metadata defs live at the module end, past both CHECK-LABEL barriers: the +; high-trip-count copy's loop id references the aie-loop-versioned marker. +; CHECK-DAG: [[HIGHMD]] = distinct !{[[HIGHMD]], [[MARK:![0-9]+]]} +; CHECK-DAG: [[MARK]] = !{!"llvm.loop.hint.aie-loop-versioned", i32 1} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} diff --git a/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll b/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll index 08c3e0f15097..2183a7c581dc 100644 --- a/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll +++ b/llvm/test/CodeGen/AIE/aie2p/llc-pipeline-aie2p.ll @@ -158,6 +158,10 @@ ; AIE-O1-NEXT: Dominator Tree Construction ; AIE-O1-NEXT: Natural Loop Information ; AIE-O1-NEXT: Scalar Evolution Analysis +; AIE-O1-NEXT: AIE Inner Loop Versioning +; AIE-O1-NEXT: Dominator Tree Construction +; AIE-O1-NEXT: Natural Loop Information +; AIE-O1-NEXT: Scalar Evolution Analysis ; AIE-O1-NEXT: Lazy Branch Probability Analysis ; AIE-O1-NEXT: Lazy Block Frequency Analysis ; AIE-O1-NEXT: Optimization Remark Emitter @@ -382,6 +386,10 @@ ; AIE-O23-NEXT: Dominator Tree Construction ; AIE-O23-NEXT: Natural Loop Information ; AIE-O23-NEXT: Scalar Evolution Analysis +; AIE-O23-NEXT: AIE Inner Loop Versioning +; AIE-O23-NEXT: Dominator Tree Construction +; AIE-O23-NEXT: Natural Loop Information +; AIE-O23-NEXT: Scalar Evolution Analysis ; AIE-O23-NEXT: Lazy Branch Probability Analysis ; AIE-O23-NEXT: Lazy Block Frequency Analysis ; AIE-O23-NEXT: Optimization Remark Emitter diff --git a/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-siblings.ll b/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-siblings.ll new file mode 100644 index 000000000000..7b595f5c6977 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-siblings.ll @@ -0,0 +1,53 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; Two versioned loops in one function. Each guard's threshold pseudo is +; non-CSE-able, so the two placeholders never merge and the post-pipeliner +; patches each guard independently with its own stage count. Mis-identification +; would abort compilation (report_fatal_error), so reaching the end and emitting +; two guards is the check. +; +; RUN: llc -mtriple=aie2ps -O2 -aie-force-postpipeliner %s -o - | FileCheck %s + +; CHECK-LABEL: two_versioned: +; Both loops are versioned: two guard blocks, each with its own patched +; threshold move and trip-count compare. +; CHECK-DAG: loop1.lver.guard +; CHECK-DAG: loop2.lver.guard +; CHECK-DAG: ltu r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}} + +define void @two_versioned(ptr noalias %a, ptr noalias %b, i32 %n, i32 %m) { +entry: + br label %loop1 +loop1: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop1 ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 7 + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c1 = icmp slt i32 %i.next, %n + br i1 %c1, label %loop1, label %mid, !llvm.loop !0 +mid: + br label %loop2 +loop2: + %j = phi i32 [ 0, %mid ], [ %j.next, %loop2 ] + %pb2 = getelementptr i32, ptr %b, i32 %j + %z = load i32, ptr %pb2, align 4 + %w = add i32 %z, 99 + %pa2 = getelementptr i32, ptr %a, i32 %j + store i32 %w, ptr %pa2, align 4 + %j.next = add i32 %j, 1 + %c2 = icmp slt i32 %j.next, %m + br i1 %c2, label %loop2, label %exit, !llvm.loop !2 +exit: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} +!2 = distinct !{!2, !1} diff --git a/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-swp.ll b/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-swp.ll new file mode 100644 index 000000000000..a49ac900138e --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning-swp.ll @@ -0,0 +1,58 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; End-to-end: a versioned loop that the post-pipeliner can schedule gets its +; guard threshold patched with the required stage count (>= 2). The threshold is +; a dedicated side-effecting multi-slot pseudo (so MachineCSE never merges +; sibling guards' placeholders); the post-RA scheduler materializes it into a +; concrete scalar move. Without patching, the placeholder would remain 0. +; +; This also pins down the correctness coupling that has no other direct test: +; the guard threshold (NStages) and the high-trip-count copy's ZOL trip-count +; adjustment (-(NStages-1)) must agree. The guard guarantees the pipelined copy +; only runs when count >= NStages, so lc = count-(NStages-1) stays >= 1. Here +; NStages is 2: threshold #2, high-copy peel #-1, and the low copy keeps an +; unpeeled #0. +; +; RUN: llc -mtriple=aie2ps -O2 -aie-force-postpipeliner %s -o - | FileCheck %s + +; CHECK-LABEL: versioned: +; The guard block holds the patched threshold (the stage count) and the unsigned +; trip-count compare selecting the pipelined vs fallback copy. The threshold is a +; small positive constant materialized into a scalar register (a mova, no +; slot-blocking movxm), and the compare tests the trip count against it. +; CHECK: mova [[THR:r[0-9]+]], #[[NSTAGES:[2-9][0-9]*]] +; CHECK: ltu r{{[0-9]+}}, r{{[0-9]+}}, [[THR]] +; The low-trip-count (fallback) copy runs the loop verbatim: its ZOL count is +; unpeeled. +; CHECK-LABEL: %loop.ph +; The high-trip-count (pipelined) copy peels NStages-1 stages from its ZOL +; count. For NStages == 2 that is an adjustment of -1. +; CHECK-LABEL: %loop.ph.lver.high +; CHECK: add.nc lc, r{{[0-9]+}}, #-1 + +define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %m1 = mul i32 %x, %x + %m2 = mul i32 %m1, %x + %m3 = add i32 %m2, %m1 + %m4 = xor i32 %m3, %x + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %m4, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} diff --git a/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning.ll b/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning.ll new file mode 100644 index 000000000000..94b321cb4dec --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2ps/inner-loop-versioning.ll @@ -0,0 +1,77 @@ +; This file is licensed under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +; +; IR-level loop-versioning transform: a loop carrying the versioning hint is +; split into a high-trip-count copy (the clone, keeps the hint, pipelined later) +; and a low-trip-count copy (the original, hint stripped), behind a runtime +; trip-count guard. The guard threshold is a placeholder produced by the thin +; intrinsic; the postpipeliner patches it. +; +; RUN: llc -mtriple=aie2ps -stop-after=aie-inner-loop-versioning %s -o - \ +; RUN: | FileCheck %s + +; CHECK-LABEL: define void @versioned +; The guard: threshold from the intrinsic, unsigned trip-count compare. Below +; the threshold take the low-trip-count (original) copy, otherwise the cloned +; high-trip-count copy. +; CHECK: %[[THR:.*]] = call i32 @llvm.aie2ps.loop.version.threshold(i32 -1) +; CHECK: %[[LOW:.*]] = icmp ult i32 %{{.*}}, %[[THR]] +; CHECK: br i1 %[[LOW]], label %{{.*}}.ph, label %{{.*}}.lver.high +; The cloned high-trip-count copy carries the ".lver.high" suffix and is marked +; with the aie-loop-versioned hint. +; CHECK: loop.lver.high: +; CHECK: br i1 %{{.*}}, !llvm.loop [[HIGHMD:![0-9]+]] +; The original low-trip-count copy's loop id is rebuilt without the hint. +; CHECK: br i1 %{{.*}}, !llvm.loop [[LOWMD:![0-9]+]] + +define void @versioned(ptr noalias %a, ptr noalias %b, i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i32 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 1234 + %pb = getelementptr i32, ptr %b, i32 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +; A loop whose exit count is wider than i32 (i64 induction variable) cannot be +; expressed as a 32-bit zero-overhead loop, so the high-trip-count copy could +; never be pipelined. Versioning bails and leaves the loop untouched: no guard, +; no clone. +; CHECK-LABEL: define void @wide_trip_count +; CHECK-NOT: icmp ult +; CHECK-NOT: lver +define void @wide_trip_count(ptr noalias %a, ptr noalias %b, i64 %n) { +entry: + br label %loop +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %pa = getelementptr i32, ptr %a, i64 %i + %x = load i32, ptr %pa, align 4 + %y = mul i32 %x, 1234 + %pb = getelementptr i32, ptr %b, i64 %i + store i32 %y, ptr %pb, align 4 + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit, !llvm.loop !0 +exit: + ret void +} + +; Metadata defs live at the module end, past both CHECK-LABEL barriers: the +; high-trip-count copy's loop id references the aie-loop-versioned marker. +; CHECK-DAG: [[HIGHMD]] = distinct !{[[HIGHMD]], [[MARK:![0-9]+]]} +; CHECK-DAG: [[MARK]] = !{!"llvm.loop.hint.aie-loop-versioned", i32 1} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.hint.aie-loop-versioning", i64 1} diff --git a/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll b/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll index ecebdb214ad5..5d9b8e492fe6 100644 --- a/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll +++ b/llvm/test/CodeGen/AIE/aie2ps/llc-pipeline-aie2ps.ll @@ -161,6 +161,10 @@ ; AIE-O1-NEXT: Dominator Tree Construction ; AIE-O1-NEXT: Natural Loop Information ; AIE-O1-NEXT: Scalar Evolution Analysis +; AIE-O1-NEXT: AIE Inner Loop Versioning +; AIE-O1-NEXT: Dominator Tree Construction +; AIE-O1-NEXT: Natural Loop Information +; AIE-O1-NEXT: Scalar Evolution Analysis ; AIE-O1-NEXT: Lazy Branch Probability Analysis ; AIE-O1-NEXT: Lazy Block Frequency Analysis ; AIE-O1-NEXT: Optimization Remark Emitter @@ -388,6 +392,10 @@ ; AIE-O23-NEXT: Dominator Tree Construction ; AIE-O23-NEXT: Natural Loop Information ; AIE-O23-NEXT: Scalar Evolution Analysis +; AIE-O23-NEXT: AIE Inner Loop Versioning +; AIE-O23-NEXT: Dominator Tree Construction +; AIE-O23-NEXT: Natural Loop Information +; AIE-O23-NEXT: Scalar Evolution Analysis ; AIE-O23-NEXT: Lazy Branch Probability Analysis ; AIE-O23-NEXT: Lazy Block Frequency Analysis ; AIE-O23-NEXT: Optimization Remark Emitter