From 4e3c6bd7ffd56a6dbe537d1cefbd88e8cbfe34f9 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 15 Jul 2026 18:26:15 +0000 Subject: [PATCH 01/24] feat(tensilelite): add reusable ClusterLoad component and tri-state Multicast JIRA ID: AIHPBLAS-3929 The TDM multicast ("cluster load") mask machinery was duplicated inline across KernelWriter, KernelWriterAssembly, and Subtile/SubtileGREmit, and Multicast was implicitly coupled to ClusterDim != [1, 1]. That coupling made it impossible to compose barrier-only clustering and cooperative-load clustering independently, or to reuse the mask attach at a new load site. - New ClusterLoad component (Components/ClusterLoad.py, registered via Component.py + Components/__init__.py). ClusterLoadTDM centralizes the mask value computation (computeMasks), the MulticastMask* SGPR declare/undeclare (declareSgprs / undeclareSgprs), the topology decision (combined vs split A/B masks), and the descriptor attach at each load site (applyToDescriptor). It is a behavior-preserving extraction: every method emits byte-identical assembly to the previous inline code, and computeMasks receives the exact SGPR operands the caller already holds so register indices are unchanged. Selection is capability-based (HasTDM + TDMInst == 3), like TensorDataMoverLoad. - KernelWriter / KernelWriterAssembly / SubtileGREmit now route mask declare, undeclare, compute, and descriptor attach through ClusterLoadTDM.find(...). - Tri-state Multicast parameter (ValidParameters + GlobalParameters default -1): -1 = auto (legacy: ClusterDim != [1,1] implies Multicast, minus the StreamK cluster paths), 0 = force off, 1 = force on. Default -1 reproduces the historic derivation exactly, so every existing YAML (which omits Multicast) derives byte-identically. - Solution.py derives Multicast/ClusterBarrier from the tri-state, keying the legacy auto coupling and the ClusterBarrier gate on StreamK == 0. - Common/Utilities.clusterEnabled() helper for the ClusterDim != [1,1] test. - Unit: test_cluster_load_component.py (component find/declare/compute/attach), test_multicast_tristate.py (-1/0/1 derivation, legacy equivalence). Low. Behavior-preserving refactor; the tri-state defaults to -1 (legacy auto), so emitted assembly is unchanged for all existing configs. --- .../Tensile/Common/GlobalParameters.py | 3 + .../Tensile/Common/ValidParameters.py | 8 + .../tensilelite/Tensile/Component.py | 5 + .../Tensile/Components/ClusterLoad.py | 189 ++++++++++++ .../Components/Subtile/SubtileGREmit.py | 6 +- .../Tensile/Components/__init__.py | 1 + .../tensilelite/Tensile/KernelWriter.py | 32 +- .../Tensile/KernelWriterAssembly.py | 81 +---- .../Tensile/SolutionStructs/Naming.py | 5 + .../Tensile/SolutionStructs/Solution.py | 62 +++- .../SolutionArms/test_solution_arms_char.py | 13 +- .../__snapshots__/test_builders_char.ambr | 10 + .../unit/test_PrefetchAcrossPersistent.py | 5 + .../Tests/unit/test_cluster_load_component.py | 289 ++++++++++++++++++ .../unit/test_multicast_legacy_coercion.py | 229 ++++++++++++++ .../Tests/unit/test_multicast_tristate.py | 118 +++++++ 16 files changed, 950 insertions(+), 106 deletions(-) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_legacy_coercion.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py index e701a5695a6a..a1176fb5c593 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py @@ -612,6 +612,9 @@ # [1, 1] disables clustering. Non-[1, 1] enables Multicast so workgroups within # a cluster can share data loaded via TDM-multicast, reducing redundant global reads. {"ClusterDim": [[1, 1]]}, + # Multicast tri-state (see ValidParameters). Default -1 = legacy auto, so + # every existing YAML (which omits Multicast) derives identically to before. + {"Multicast": [-1]}, {"HalfPLR": [0]}, {"TDMIterateMode": [0]} ] diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index 123dc69a803c..95178b291e2d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -1117,6 +1117,14 @@ def makeValidMatrixInstructions(): # Cluster dimension. Clusters have up to 16 work-groups in a cluster, but each work-group in a # cluster runs on a separate WGP. "ClusterDim": validClusterDimensions, + # Multicast (cluster load) control. Decouples the TDM-multicast opt-in from + # ClusterDim so barrier-only clustering and cooperative-load clustering + # compose independently. Tri-state: + # -1 = auto (legacy: ClusterDim != [1,1] implies Multicast, except the + # StreamK cluster paths); reproduces historic behavior exactly. + # 0 = force multicast off. + # 1 = force multicast on (independent of the ClusterDim coupling). + "Multicast": [-1, 0, 1], # Enable PLR 0.5 to save vgprs # 0: Disabled # 1: Use PLR 0.5 for A diff --git a/projects/hipblaslt/tensilelite/Tensile/Component.py b/projects/hipblaslt/tensilelite/Tensile/Component.py index 9639eced4bef..46a27b893e88 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Component.py +++ b/projects/hipblaslt/tensilelite/Tensile/Component.py @@ -312,6 +312,11 @@ class GL2Prefetch(Component): GL2 Prefetch """ +class ClusterLoad(Component): + """ + Cluster (multicast) TDM load: multicast-mask compute + descriptor attach. + """ + # Importing here allows auto-registry of components in the Components directory. # Each file must be listed in __all__ in Components/__init__.py # "noqa" prevents linter from complaining here. diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py new file mode 100644 index 000000000000..b069c9ac0659 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -0,0 +1,189 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Cluster (multicast) TDM load component. + +Centralizes the multicast ("cluster load") mask machinery that was previously +duplicated across ``KernelWriter``/``KernelWriterAssembly``/``SubtileGREmit``: + + * the mask *value* computation (``computeMasks``), + * the ``MulticastMask*`` SGPR declare/undeclare (``declareSgprs`` / + ``undeclareSgprs``), + * the topology decision (``usesCombinedMask`` / ``maskSgprName``), and + * the descriptor attach at each load site (``applyToDescriptor``). + +This is a behavior-preserving extraction: every method emits byte-identical +assembly to the original inline code. ``computeMasks`` therefore receives the +exact SGPR operands the caller already holds (it does not re-allocate) so the +instruction stream and register indices are unchanged. + +Selection is capability-based (``HasTDM`` + ``TDMInst == 3``), identical to how +``TensorDataMoverLoad`` is found: ``ClusterLoad.find(writer)`` returns the TDM +impl on gfx1250 and ``None`` (fallback -> no multicast) elsewhere. +""" + +from ..Component import ClusterLoad +from ..Common import clusterEnabled +from typing import Mapping +from rocisa.code import Module, Label +from rocisa.container import sgpr +from rocisa.instruction import SLShiftLeftB32, SMulI32, SBitcmp1B32, SCBranchSCC1, SBranch + + +class ClusterLoadTDM(ClusterLoad): + asmCaps = {"HasTDM": True} + kernel = {"TDMInst": 3} + + def __call__(self, writer: "KernelWriterAssembly", kernel: Mapping): + # Abstract-satisfying no-op, mirrors TensorDataMoverLoad.__call__. + pass + + # -- topology decision --------------------------------------------------- + + def usesCombinedMask(self, kernel: Mapping) -> bool: + """Single-parity combined ``MulticastMask`` predicate. + + Subtile issues both A and B loads on every wave (no wave-parity load + split), so the single-parity ``MulticastMask`` is wrong there -- it + would OR one tensor's mask into both descriptors. Use the split A/B + masks in that case. This is the single source of truth for the + combined-vs-split decision used by declare/undeclare/computeMasks. + """ + tdmA: bool = kernel["enableTDMA"] + tdmB: bool = kernel["enableTDMB"] + return tdmA and tdmB and kernel["NumWaves"] > 1 and not kernel.get("UseSubtileImpl") + + def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, + waveSeparated: bool = False) -> str: + """Central multicast-mask SGPR name resolver. + + Reproduces the three prior naming rules exactly: + * wave-separated (non-subtile): the combined ``"MulticastMask"``; + * dense and subtile: the split ``f"MulticastMask{tc}"`` with any + ``MXS`` prefix stripped (dense passed ``MXSA``/``MXSB`` tensor + chars; subtile only ever passes ``A``/``B`` so the strip is a + no-op there). + """ + if waveSeparated and not subtile: + return "MulticastMask" + string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc + return f"MulticastMask{string}" + + def cooperativeThreadPartition(self, kernel: Mapping, tc: str) -> int: + """Number of cooperating workgroups for tensor ``tc``. + + ``ClusterDim[1]`` for A, ``ClusterDim[0]`` for B. Shared math with the + GL2 prefetch cooperative loads. + """ + subTc: str = tc[-1] + return kernel["ClusterDim"][1] if subTc == "A" else kernel["ClusterDim"][0] + + # -- SGPR declare / undeclare ------------------------------------------- + + def declareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> None: + """Allocate the ``MulticastMask*`` SGPRs (lift of KernelWriter).""" + if not kernel["Multicast"]: + return + tdmM: bool = kernel["enableTDMMetadata"] + if self.usesCombinedMask(kernel): + writer.defineSgpr("MulticastMask", 1) + else: + writer.defineSgpr("MulticastMaskA", 1) + writer.defineSgpr("MulticastMaskB", 1) + if tdmM: + writer.defineSgpr("MulticastMaskMetadata", 1) + + def undeclareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> Module: + """Free the ``MulticastMask*`` SGPRs (lift of KernelWriter).""" + mod = Module() + if not (kernel["Multicast"] and kernel["TDMInst"] != 0): + return mod + tdmM: bool = kernel["enableTDMMetadata"] + if self.usesCombinedMask(kernel): + mod.add(writer.undefineSgpr("MulticastMask")) + else: + mod.add(writer.undefineSgpr("MulticastMaskA")) + mod.add(writer.undefineSgpr("MulticastMaskB")) + if tdmM: + mod.add(writer.undefineSgpr("MulticastMaskMetadata")) + return mod + + # -- mask value computation --------------------------------------------- + + def computeMasks(self, writer: "KernelWriterAssembly", kernel: Mapping, *, + sgprWgX: int, sgprWgY: int, sgprNWgX: int, sTmp: int) -> Module: + """Compute the multicast mask value(s) into the ``MulticastMask*`` SGPRs. + + Verbatim lift of the ``defineAndResources`` mask compute. The caller + passes the exact SGPR operands it already holds (``sgprWgX`` = wg_x, + ``sgprWgY`` = wg_y, ``sgprNWgX`` = nwg_x, and ``sTmp`` whose ``+4`` slot + is scratch) so the emitted instructions and register indices are + byte-identical to the original inline code. + """ + mod = Module() + if not kernel["Multicast"]: + return mod + mod.addComment0("Calculate multicast mask") + + maskA = 1 + for idx in range(kernel["ClusterDim"][1]): + maskA |= (1 << (idx * kernel["ClusterDim"][0])) + + maskB = (1 << kernel["ClusterDim"][0]) - 1 + + if kernel["enableTDMMetadata"]: + if kernel["ProblemType"]["Sparse"] == 1: + mod.add(SLShiftLeftB32(dst=sgpr("MulticastMaskMetadata"), shiftHex=sgpr(sgprWgX), src=hex(maskA),\ + comment="Setting metadata mask (follows sparse A)")) + elif kernel["ProblemType"]["Sparse"] == 2: + mod.add(SMulI32(dst=sgpr(sTmp+4), src0=sgpr(sgprWgY), src1=sgpr(sgprNWgX),\ + comment="Shift factor: wg_y * nwg_x (metadata)")) + mod.add(SLShiftLeftB32(dst=sgpr("MulticastMaskMetadata"), shiftHex=sgpr(sTmp+4), src=hex(maskB),\ + comment="Setting metadata mask (follows sparse B)")) + + if self.usesCombinedMask(kernel): + setMulticastMaskLblOdd = Label(f"setMulticastMask_OddWave", "") + setMulticastMaskLblEven = Label(f"setMulticastMask_EvenWave", "") + setMulticastMaskLblEnd = Label(f"setMulticastMaskEnd", "") + + mod.add(SBitcmp1B32(sgpr("WaveIdx"), 0, "Check parity of wId")) + mod.add(SCBranchSCC1(setMulticastMaskLblOdd.getLabelName(), "Jump if wId is odd")) + + mod.add(setMulticastMaskLblEven) + mod.add(SLShiftLeftB32(dst=sgpr("MulticastMask"), shiftHex=sgpr(sgprWgX), src=hex(maskA),\ + comment="Setting maskA for even wave")) + mod.add(SBranch(setMulticastMaskLblEnd.getLabelName())) + mod.add(setMulticastMaskLblOdd) + mod.add(SMulI32(dst=sgpr(sgprWgY), src0=sgpr(sgprWgY), src1=sgpr(sgprNWgX),\ + comment="Shift factor: wg_y * nwg_x")) + mod.add(SLShiftLeftB32(dst=sgpr("MulticastMask"), shiftHex=sgpr(sgprWgY), src=hex(maskB),\ + comment="Setting maskB for odd wave")) + mod.add(setMulticastMaskLblEnd) + + else: + mod.add(SLShiftLeftB32(dst=sgpr("MulticastMaskA"), shiftHex=sgpr(sgprWgX), src=hex(maskA),\ + comment="Setting maskA")) + + mod.add(SMulI32(dst=sgpr(sgprWgY), src0=sgpr(sgprWgY), src1=sgpr(sgprNWgX),\ + comment="Shift factor: wg_y * nwg_x")) + mod.add(SLShiftLeftB32(dst=sgpr("MulticastMaskB"), shiftHex=sgpr(sgprWgY), src=hex(maskB),\ + comment="Setting maskB")) + return mod + + # -- descriptor attach --------------------------------------------------- + + def applyToDescriptor(self, writer: "KernelWriterAssembly", kernel: Mapping, + group1: int | str, tc: str, *, subtile: bool = False, + waveSeparated: bool = False) -> Module: + """OR the multicast mask into descriptor ``Group1[word0]``. + + Folds the ``kernel["Multicast"] and enableCluster`` gate, the mask-name + choice, and the ``SOrB32`` attach. Returns an empty ``Module`` when the + gate is not satisfied -- identical to today's skipped ``if``. + """ + from .TensorDataMover import TensorDataMoverLoad + mod = Module() + if kernel["Multicast"] and clusterEnabled(kernel["ClusterDim"]): + mask = self.maskSgprName(kernel, tc, subtile=subtile, waveSeparated=waveSeparated) + tdm = TensorDataMoverLoad.find(writer) + mod.add(tdm.setMulticastMask(group1, mask, writer)) + return mod diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py b/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py index f6c13ee53e84..f40919e0ac4a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py @@ -1109,8 +1109,10 @@ def sizeRefName(idx): # OR the per-tensor broadcast mask into the descriptor for TDM multicast. # Subtile loads both A and B on every wave, so it uses split masks # (MulticastMask{tc}), not the non-subtile single parity mask. - if kernel["Multicast"] and clusterEnabled(kernel["ClusterDim"]): - mod.add(comp.setMulticastMask(descSgprName(1), f"MulticastMask{tc}", writer)) + from ...Components.ClusterLoad import ClusterLoadTDM + clusterComp = ClusterLoadTDM.find(writer) + if clusterComp: + mod.add(clusterComp.applyToDescriptor(writer, kernel, descSgprName(1), tc, subtile=True)) with writer.allocTmpSgpr(1) as tmpSgprRes: waveOffsetSgprIdx = tmpSgprRes.idx diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/__init__.py b/projects/hipblaslt/tensilelite/Tensile/Components/__init__.py index 43b0cbbffe48..9ebd19ba097a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/__init__.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/__init__.py @@ -52,4 +52,5 @@ def use(): pass "LSU", "TensorDataMover", "GL2Prefetch", + "ClusterLoad", ] diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index 01aa1c7226df..1d3b7ca4ae77 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -48,6 +48,7 @@ from .Component import Component, LraTileProperties from .Components.Signature import UserArgumentsInfo from .Components.CustomSchedule import customMainLoopSchedule +from .Components.ClusterLoad import ClusterLoadTDM from .Components.StreamK import streamKVariantClass from .Components.Subtile.Kernel import * from .SolutionStructs import Solution, isPackedIndex @@ -2835,17 +2836,9 @@ def setupNewTile(self, kernel, tensorParametersA, tensorParametersB, isOptNLL=Fa module.add(self.graUnrollOffsets(kernel, tensorParametersB)) # Free sgpr that will not be used - if kernel["Multicast"] and kernel["TDMInst"] != 0: - tdmA: bool = kernel["enableTDMA"] - tdmB: bool = kernel["enableTDMB"] - tdmM: bool = kernel["enableTDMMetadata"] - if tdmA and tdmB and kernel["NumWaves"] > 1 and not kernel.get("UseSubtileImpl"): - module.add(self.undefineSgpr("MulticastMask")) - else: - module.add(self.undefineSgpr("MulticastMaskA")) - module.add(self.undefineSgpr("MulticastMaskB")) - if tdmM: - module.add(self.undefineSgpr("MulticastMaskMetadata")) + clusterComp = ClusterLoadTDM.find(self) + if clusterComp: + module.add(clusterComp.undeclareSgprs(self, kernel)) # tile edges if kernel["EdgeType"] == "ShiftPtr" and not tdmA and not tdmB: @@ -9218,20 +9211,9 @@ def vgprAllocationImplSubtile(): if kernel["enableTDMA"] or kernel["enableTDMB"]: self.defineSgpr("WaveIdx", 1) - if kernel["Multicast"]: - tdmA: bool = kernel["enableTDMA"] - tdmB: bool = kernel["enableTDMB"] - tdmM: bool = kernel["enableTDMMetadata"] - # Subtile issues both A and B loads on every wave (no wave-parity load - # split), so the single parity MulticastMask is wrong there -- it would - # OR one tensor's mask into both descriptors. Use the split A/B masks. - if tdmA and tdmB and kernel["NumWaves"] > 1 and not kernel.get("UseSubtileImpl"): - self.defineSgpr("MulticastMask", 1) - else: - self.defineSgpr("MulticastMaskA", 1) - self.defineSgpr("MulticastMaskB", 1) - if tdmM: - self.defineSgpr("MulticastMaskMetadata", 1) + clusterComp = ClusterLoadTDM.find(self) + if clusterComp: + clusterComp.declareSgprs(self, kernel) # SGPR above are user SGPR which are set by GPU hardware when the kernel is launched self.states.firstInitSgpr = self.sgprPool.size() diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py index 9b761f6369b9..68be1a5bd357 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py @@ -75,6 +75,7 @@ from .Component import Component, TensorDataMover, GL2Prefetch from .Components.TensorDataMover import TensorDataMoverLoad from .Components.GL2Prefetch import GL2PrefetchLoad +from .Components.ClusterLoad import ClusterLoadTDM from .Components.GlobalWriteBatch import GlobalWriteBatchWriter from .KernelWriterModules import * from .AsmMemoryHelpers import dsStore, dsLoad, _vgprOffset @@ -2640,55 +2641,15 @@ def defineAndResources(self, kernel, tPA, tPB, tPM): comment="WorkGroup2 = (cluster_z * nwg_z) + wg_z")) moduleRegInit.add(label_calculate_workgroup_done) - if kernel["Multicast"]: - moduleRegInit.addComment0("Calculate multicast mask") - sgprWgX = sTmp+1 - sgprWgY = sTmp+2 - sgprNWgX = sTmp+3 - - maskA = 1 - for idx in range(kernel["ClusterDim"][1]): - maskA |= (1 << (idx * kernel["ClusterDim"][0])) - - maskB = (1 << kernel["ClusterDim"][0]) - 1 - - if kernel["enableTDMMetadata"]: - if kernel["ProblemType"]["Sparse"] == 1: - moduleRegInit.add(SLShiftLeftB32(dst=sgpr("MulticastMaskMetadata"), shiftHex=sgpr(sgprWgX), src=hex(maskA),\ - comment="Setting metadata mask (follows sparse A)")) - elif kernel["ProblemType"]["Sparse"] == 2: - moduleRegInit.add(SMulI32(dst=sgpr(sTmp+4), src0=sgpr(sgprWgY), src1=sgpr(sgprNWgX),\ - comment="Shift factor: wg_y * nwg_x (metadata)")) - moduleRegInit.add(SLShiftLeftB32(dst=sgpr("MulticastMaskMetadata"), shiftHex=sgpr(sTmp+4), src=hex(maskB),\ - comment="Setting metadata mask (follows sparse B)")) - - if tdmA and tdmB and kernel["NumWaves"] > 1 and not kernel.get("UseSubtileImpl"): - setMulticastMaskLblOdd = Label(f"setMulticastMask_OddWave", "") - setMulticastMaskLblEven = Label(f"setMulticastMask_EvenWave", "") - setMulticastMaskLblEnd = Label(f"setMulticastMaskEnd", "") - - moduleRegInit.add(SBitcmp1B32(sgpr("WaveIdx"), 0, "Check parity of wId")) - moduleRegInit.add(SCBranchSCC1(setMulticastMaskLblOdd.getLabelName(), "Jump if wId is odd")) - - moduleRegInit.add(setMulticastMaskLblEven) - moduleRegInit.add(SLShiftLeftB32(dst=sgpr("MulticastMask"), shiftHex=sgpr(sgprWgX), src=hex(maskA),\ - comment="Setting maskA for even wave")) - moduleRegInit.add(SBranch(setMulticastMaskLblEnd.getLabelName())) - moduleRegInit.add(setMulticastMaskLblOdd) - moduleRegInit.add(SMulI32(dst=sgpr(sgprWgY), src0=sgpr(sgprWgY), src1=sgpr(sgprNWgX),\ - comment="Shift factor: wg_y * nwg_x")) - moduleRegInit.add(SLShiftLeftB32(dst=sgpr("MulticastMask"), shiftHex=sgpr(sgprWgY), src=hex(maskB),\ - comment="Setting maskB for odd wave")) - moduleRegInit.add(setMulticastMaskLblEnd) - - else: - moduleRegInit.add(SLShiftLeftB32(dst=sgpr("MulticastMaskA"), shiftHex=sgpr(sgprWgX), src=hex(maskA),\ - comment="Setting maskA")) - - moduleRegInit.add(SMulI32(dst=sgpr(sgprWgY), src0=sgpr(sgprWgY), src1=sgpr(sgprNWgX),\ - comment="Shift factor: wg_y * nwg_x")) - moduleRegInit.add(SLShiftLeftB32(dst=sgpr("MulticastMaskB"), shiftHex=sgpr(sgprWgY), src=hex(maskB),\ - comment="Setting maskB")) + # Guard the compute site like the apply sites: find() returns None + # unless TDMInst==3 + HasTDM match, so Multicast with TDMInst in {1,2} + # or a non-TDM arch would otherwise None-deref here. + clusterComp = ClusterLoadTDM.find(self) + if kernel["Multicast"] and clusterComp: + # Same SGPR operands allocated above (wg_x=sTmp+1, wg_y=sTmp+2, + # nwg_x=sTmp+3, scratch=sTmp+4) are passed through. + moduleRegInit.add(clusterComp.computeMasks( + self, kernel, sgprWgX=sTmp+1, sgprWgY=sTmp+2, sgprNWgX=sTmp+3, sTmp=sTmp)) # SrdD can be used as temp sgprs for a bit if self.states.doShadowInit and kernel["BufferStore"]: self.addSgprVarToPool("SrdD") @@ -18906,7 +18867,6 @@ def initTDMDescriptor(self, kernel: Mapping, tP: Mapping) -> Module: unrolledMajor = not tlu ti: int = tP["idx"] tileChar: str = tP["tileChar"] - enableCluster = clusterEnabled(kernel["ClusterDim"]) mod = Module(f"Init TDM Descriptor {tc}") def descSgprName(idx: int) -> str: @@ -18920,13 +18880,6 @@ def sizeRefName(idx: int) -> str: idxChar= INDEX_CHARS[idx] return f"Size{idxChar}" - def maskSgprName(tc: str) -> str: - if tc.startswith("MXS"): - string = tc.removeprefix("MXS") - else: - string = tc - return f"MulticastMask{string}" - dtype: DataType = kernel["ProblemType"][f"DataType{tc}"] mt: int = kernel[f"MacroTile{ti}"] du: int = kernel["DepthU"] @@ -18957,8 +18910,9 @@ def maskSgprName(tc: str) -> str: mod.add(comp.initOperands(descSgprName(0), descSgprName(1), group2Name, None)) mod.add(comp.setDataType(dtype, descSgprName(1), isMetadata)) mod.add(comp.setGlobalAddr(descSgprName(0), f"Address{tc}")) - if kernel["Multicast"] and enableCluster: - mod.add(comp.setMulticastMask(descSgprName(1), maskSgprName(tc), self)) + clusterComp = ClusterLoadTDM.find(self) + if clusterComp: + mod.add(clusterComp.applyToDescriptor(self, kernel, descSgprName(1), tc)) with self.allocTmpSgpr(2, tag="initTDMDescriptor_tmpSgprRes") as tmpSgprRes: waveOffsetSgprIdx: int = tmpSgprRes.idx @@ -19059,7 +19013,6 @@ def initTDMDescriptorWaveSeparatedImpl(self, kernel, tP, waveIdxSgpr: int | str unrolledMajor = not tlu ti: int = tP["idx"] tileChar: str = tP["tileChar"] - enableCluster = clusterEnabled(kernel["ClusterDim"]) mod = Module(f"Init TDM Descriptor {tc}") def descSgprName(idx: int) -> str: @@ -19073,9 +19026,6 @@ def sizeRefName(idx: int) -> str: idxChar= INDEX_CHARS[idx] return f"Size{idxChar}" - def maskSgprName(tc: str) -> str: - return f"MulticastMask" - dtype: DataType = kernel["ProblemType"][f"DataType{tc}"] mt: int = kernel[f"MacroTile{ti}"] du: int = kernel["DepthU"] @@ -19115,8 +19065,9 @@ def maskSgprName(tc: str) -> str: mod.add(comp.initOperands(descSgprName(0), descSgprName(1), group2Name, None)) mod.add(comp.setDataType(dtype, descSgprName(1), tc == "Metadata")) mod.add(comp.setGlobalAddr(descSgprName(0), f"Address{tc}")) - if kernel["Multicast"] and enableCluster: - mod.add(comp.setMulticastMask(descSgprName(1), maskSgprName(tc), self)) + clusterComp = ClusterLoadTDM.find(self) + if clusterComp: + mod.add(clusterComp.applyToDescriptor(self, kernel, descSgprName(1), tc, waveSeparated=True)) with self.allocTmpSgpr(2, tag="initTDMDescriptorWaveSeparatedImpl_tmpSgprRes") as tmpSgprRes: waveOffsetSgprIdx: int = tmpSgprRes.idx diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Naming.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Naming.py index 650e15ca75c9..a598d07357c5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Naming.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Naming.py @@ -213,6 +213,11 @@ def _getName(state, requiredParameters: frozenset, splitGSU: bool, ignoreInterna if state.get("LDSSegmentInterleave") == 1: requiredParametersTemp.add("LDSSegmentInterleave") + # Multicast is a tri-state derivation knob (-1 auto / 0 off / 1 on) that does + # not alter the emitted assembly, so it is intentionally excluded from the + # kernel name to keep names byte-identical to develop (no _M token). + requiredParametersTemp.discard("Multicast") + for key in sorted(requiredParametersTemp): if key not in state or key == "CustomKernelName": continue diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index d06d896dbfd2..aaa0ad75d08f 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -353,6 +353,21 @@ def validateParameterTypes(state, srcFile=""): return records +def coerceLegacyMulticastType(state) -> None: + """Normalize a legacy bool ``Multicast`` to the tri-state int, in place. + + Shipped library-logic artifacts serialize the derived value as a bool, but + ``Multicast`` is now an int ([-1, 0, 1]); bool and int are different msgpack + wire types, so a bool trips the create-library type gate (would ``std::bad_cast`` + at C++ deserialization). Coerce ``false -> 0`` / ``true -> 1`` (never -1 + "auto", to avoid switching a tuned kernel into legacy auto-coupling). Scoped to + the serialized-state loading path (the config/derivation path stays bool). + """ + mc = state.get("Multicast") + if type(mc) is bool: + state["Multicast"] = int(mc) + + def raiseIfTypeMismatches() -> None: """Raise when collected type mismatches exist. @@ -574,7 +589,17 @@ def __init__( self["AssignedProblemIndependentDerivedParameters"] = False if "AssignedDerivedParameters" not in self._state: self["AssignedDerivedParameters"] = False - + + # ``raiseProblemTypeOnTypeMismatch=False`` marks the paths that load a + # serialized / pre-derived solution state (library-logic artifacts, + # solution files, OriginalSolution). Those legacy artifacts carry a bool + # ``Multicast``; normalize it to the tri-state int so both the strict type + # gate and the emitted msgpack see an int. The config/derivation path + # (raiseProblemTypeOnTypeMismatch=True) is left untouched. + loadingSerializedState = not raiseProblemTypeOnTypeMismatch + if loadingSerializedState: + coerceLegacyMulticastType(self._state) + # Validate parameter types against the validParameters registry. # Catches bool-vs-int mismatches (YAML false vs 0) that would cause # std::bad_cast at C++ msgpack deserialization time. The mismatch @@ -602,6 +627,10 @@ def __init__( ) self._name = config["CustomKernelName"] if "CustomKernelName" in config and config["CustomKernelName"] else None + # Note: derivation now emits an int Multicast (0/1) on all paths, so no + # post-derivation coercion is needed. The pre-load coerceLegacyMulticastType + # above still guards shipped library YAMLs that serialize a bool Multicast. + # Only merge and report mismatches if there were no pre-existing mismatches # To avoid duplicates and noise from cascading issues. if not pre_records: @@ -1022,16 +1051,29 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, if state["DirectToVgprMXSA"] or state["DirectToVgprMXSB"]: reject(state, printRejectionReason, "UseSubtileImpl=1 PrefetchAcrossPersistent not supported with DirectToVgpr MX scale tensors") - state["Multicast"] = False state["ClusterBarrier"] = False - # Multicast uses a mask fixed to the physical cluster position, but Stream-K remaps - # each WG's tile per iteration, so the broadcast would target the wrong partner. - # Keep the cluster WG-id decode (gated on ClusterDim) but leave multicast off for Stream-K. - if state["ClusterDim"] != [1, 1] and state["StreamK"] == 0: - state["Multicast"] = True - # ClusterBarrier emits SCmp/branch on sgpr("WaveIdx"), which is only allocated when TDM is enabled. - if state["TDMInst"] != 0 and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): - state["ClusterBarrier"] = True + # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. + # Default -1 reproduces the historic ClusterDim-coupled derivation, so YAML + # that omits Multicast is byte-identical. + mc = state.get("Multicast", -1) + if mc == 1: + state["Multicast"] = 1 + elif mc == 0: + state["Multicast"] = 0 + else: # -1 auto (legacy) + # A legacy broadcast targets a fixed physical cluster position, which + # Stream-K tile remapping would send to the wrong partner, so auto-multicast + # is off for ALL Stream-K. Non-Stream-K clustered paths are unchanged, so + # this reproduces develop's exact behavior for every YAML that omits Multicast. + state["Multicast"] = int(state["ClusterDim"] != [1, 1] + and state["StreamK"] == 0) + # ClusterBarrier applies only to the non-Stream-K clustered (legacy/subtile) + # path; keyed off "StreamK == 0" so it excludes the StreamK cluster paths + # (which imply StreamK != 0). A forced-off Multicast also keeps it off. + if state["ClusterDim"] != [1, 1] and state["StreamK"] == 0 \ + and state["Multicast"] and state["TDMInst"] != 0 \ + and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): + state["ClusterBarrier"] = True # done state["AssignedProblemIndependentDerivedParameters"] = True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionArms/test_solution_arms_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionArms/test_solution_arms_char.py index 1ffd26f21e79..1ffb278e91b5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionArms/test_solution_arms_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionArms/test_solution_arms_char.py @@ -47,6 +47,11 @@ def _reset(state): """Clear derivation-done flags so static methods re-run from scratch.""" state["AssignedProblemIndependentDerivedParameters"] = False state["AssignedDerivedParameters"] = False + # Multicast is a tri-state derived param (-1 auto / 0 off / 1 on). The vendored + # fixture states are already fully derived, so Multicast carries a concrete + # 0/1; restore the -1 auto sentinel a fresh (pre-derivation) config would have, + # so re-derivation exercises the legacy ClusterDim-coupled auto path. + state["Multicast"] = -1 return state @@ -378,7 +383,7 @@ def test_piap_cluster_dim_non_default_sets_multicast(isa_info_map, hss_state): state["ClusterBarrier"] = False state = _reset(state) _piap(state, isa_info_map) - assert state.get("Multicast") is True + assert state.get("Multicast") == 1 def test_piap_cluster_dim_default_multicast_false(isa_info_map, hss_state): @@ -388,7 +393,7 @@ def test_piap_cluster_dim_default_multicast_false(isa_info_map, hss_state): state["ClusterBarrier"] = False state = _reset(state) _piap(state, isa_info_map) - assert state.get("Multicast") is False + assert state.get("Multicast") == 0 def test_piap_cluster_barrier_no_cluster_dim_cleared(isa_info_map, hss_state): @@ -408,7 +413,7 @@ def test_piap_cluster_barrier_no_cluster_dim_cleared(isa_info_map, hss_state): _piap(state, isa_info_map) assert state.get("Valid") is not False assert state.get("ClusterBarrier") is False - assert state.get("Multicast") is False + assert state.get("Multicast") == 0 def test_piap_cluster_barrier_no_tdm_cleared(isa_info_map, hss_state): @@ -427,7 +432,7 @@ def test_piap_cluster_barrier_no_tdm_cleared(isa_info_map, hss_state): _piap(state, isa_info_map) assert state.get("Valid") is not False assert state.get("ClusterBarrier") is False - assert state.get("Multicast") is True + assert state.get("Multicast") == 1 # --------------------------------------------------------------------------- diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr index 99ff757917be..0c2e30b5d753 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr @@ -1432,6 +1432,7 @@ 'MaxOccupancy', 'MbskPrefetchMethod', 'MinGRIncPerMfma', + 'Multicast', 'NoReject', 'NonTemporal', 'NonTemporalA', @@ -2174,6 +2175,15 @@ 'len': 10, 'type': 'list', }), + 'Multicast': dict({ + 'head': list([ + -1, + 0, + 1, + ]), + 'len': 3, + 'type': 'list', + }), 'NoReject': dict({ 'head': list([ False, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_PrefetchAcrossPersistent.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_PrefetchAcrossPersistent.py index a337cb36f015..e1c6f6987db3 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_PrefetchAcrossPersistent.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_PrefetchAcrossPersistent.py @@ -164,6 +164,11 @@ def __init__(self): memTokenLdsBuffer1=1, staggerUCode=False, unrollIdx=0, + # Capability/kernel state consumed by ClusterLoadTDM.find()'s + # PartialMatch (asmCaps HasTDM + kernel TDMInst==3), mirroring the + # real writer.states on a gfx1250 TDM path so the component matches. + asmCaps={"HasTDM": True}, + kernel={"TDMInst": 3}, ) self.do = {"executeToInitEnd": False} self.dontAppendCode = False diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py new file mode 100644 index 000000000000..a4370574cf6a --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Unit tests for the ClusterLoad (TDM multicast) component. +# +# Covers Tensile/Components/ClusterLoad.py: capability-based selection, the +# topology decision (usesCombinedMask / maskSgprName), the cooperative-thread +# partition, the SGPR declare/undeclare, and the emitted assembly of +# computeMasks / applyToDescriptor. These emit no GPU work themselves, so the +# asm string is the contract -- easy to break silently. +# +# Usage: +# pytest test_cluster_load_component.py -v +################################################################################ + +import os +import shutil +import sys + +import pytest +from types import SimpleNamespace + +pytestmark = pytest.mark.unit + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +TENSILE_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +sys.path.insert(0, TENSILE_ROOT) + +WAVESIZE_32 = 32 + + +def _init_rocisa_gfx1250(): + from rocisa import rocIsa + from Tensile.Common.Architectures import gfxToIsa + ri = rocIsa.getInstance() + isa = gfxToIsa("gfx1250") + asmpath = shutil.which('amdclang++') or '/usr/bin/amdclang++' + ri.init(isa, asmpath) + ri.setKernel(isa, WAVESIZE_32) + + +class _StubWriter: + """Minimal writer: capability map for find() + defineSgpr/undefineSgpr sinks.""" + + def __init__(self, has_tdm=True, tdm_inst=3): + self.states = SimpleNamespace( + asmCaps={"HasTDM": has_tdm}, + archCaps={}, + kernel={"TDMInst": tdm_inst}, + ) + self.defined = [] + self.undefined = [] + + def defineSgpr(self, name, numSgprs, align=1): + self.defined.append((name, numSgprs)) + + def undefineSgpr(self, name): + from rocisa.code import ValueSet + self.undefined.append(name) + return ValueSet(name="sgpr" + name, value="UNDEF", format=-1) + + +def _kernel(*, multicast=True, clusterDim=(2, 2), tdmA=True, tdmB=True, + numWaves=4, useSubtile=False, sparse=0, tdmMeta=False, tdmInst=3): + return { + "Multicast": multicast, + "ClusterDim": list(clusterDim), + "enableTDMA": tdmA, + "enableTDMB": tdmB, + "enableTDMMetadata": tdmMeta, + "NumWaves": numWaves, + "UseSubtileImpl": useSubtile, + "TDMInst": tdmInst, + "ProblemType": {"Sparse": sparse}, + } + + +# --- selection ------------------------------------------------------------- + +class TestFind: + # Mirrors the production call sites, which resolve the component via the + # concrete ClusterLoadTDM.find(writer) (capability-gated selection). + def test_find_returns_tdm_impl_on_gfx1250(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + comp = ClusterLoadTDM.find(_StubWriter(has_tdm=True, tdm_inst=3)) + assert isinstance(comp, ClusterLoadTDM) + + def test_find_returns_none_without_tdm(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + assert ClusterLoadTDM.find(_StubWriter(has_tdm=False, tdm_inst=0)) is None + + def test_find_returns_none_when_tdm_inst_not_3(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + assert ClusterLoadTDM.find(_StubWriter(has_tdm=True, tdm_inst=0)) is None + + +# --- topology decision ----------------------------------------------------- + +class TestUsesCombinedMask: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_combined_when_both_tdm_multiwave_non_subtile(self): + assert self._c().usesCombinedMask(_kernel(tdmA=True, tdmB=True, numWaves=4, useSubtile=False)) + + def test_split_when_subtile(self): + assert not self._c().usesCombinedMask(_kernel(useSubtile=True)) + + def test_split_when_single_wave(self): + assert not self._c().usesCombinedMask(_kernel(numWaves=1)) + + def test_split_when_single_tensor(self): + assert not self._c().usesCombinedMask(_kernel(tdmA=True, tdmB=False)) + + +class TestMaskSgprName: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_wave_separated_combined_name(self): + k = _kernel() + assert self._c().maskSgprName(k, "A", waveSeparated=True) == "MulticastMask" + assert self._c().maskSgprName(k, "B", waveSeparated=True) == "MulticastMask" + + def test_dense_split_names(self): + k = _kernel() + assert self._c().maskSgprName(k, "A") == "MulticastMaskA" + assert self._c().maskSgprName(k, "B") == "MulticastMaskB" + + def test_dense_strips_mxs_prefix(self): + k = _kernel() + assert self._c().maskSgprName(k, "MXSA") == "MulticastMaskA" + assert self._c().maskSgprName(k, "MXSB") == "MulticastMaskB" + + def test_metadata_name(self): + k = _kernel() + assert self._c().maskSgprName(k, "Metadata") == "MulticastMaskMetadata" + + def test_subtile_split_name(self): + k = _kernel(useSubtile=True) + assert self._c().maskSgprName(k, "A", subtile=True) == "MulticastMaskA" + assert self._c().maskSgprName(k, "B", subtile=True) == "MulticastMaskB" + + +class TestCooperativeThreadPartition: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_a_uses_clusterdim1_b_uses_clusterdim0(self): + k = _kernel(clusterDim=(4, 2)) + assert self._c().cooperativeThreadPartition(k, "A") == 2 + assert self._c().cooperativeThreadPartition(k, "B") == 4 + # MXS tensors resolve by their trailing tensor char. + assert self._c().cooperativeThreadPartition(k, "MXSA") == 2 + assert self._c().cooperativeThreadPartition(k, "MXSB") == 4 + + +# --- SGPR declare / undeclare ---------------------------------------------- + +class TestDeclareUndeclare: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_declare_combined(self): + w = _StubWriter() + self._c().declareSgprs(w, _kernel()) # combined + assert [n for n, _ in w.defined] == ["MulticastMask"] + + def test_declare_split(self): + w = _StubWriter() + self._c().declareSgprs(w, _kernel(useSubtile=True)) # split + assert [n for n, _ in w.defined] == ["MulticastMaskA", "MulticastMaskB"] + + def test_declare_metadata(self): + w = _StubWriter() + self._c().declareSgprs(w, _kernel(useSubtile=True, sparse=1, tdmMeta=True)) + assert w.defined[-1] == ("MulticastMaskMetadata", 1) + + def test_declare_noop_when_multicast_off(self): + w = _StubWriter() + self._c().declareSgprs(w, _kernel(multicast=False)) + assert w.defined == [] + + def test_undeclare_combined(self): + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel()) + assert w.undefined == ["MulticastMask"] + + def test_undeclare_split(self): + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel(useSubtile=True)) + assert w.undefined == ["MulticastMaskA", "MulticastMaskB"] + + def test_undeclare_noop_when_multicast_off(self): + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel(multicast=False)) + assert w.undefined == [] + + +# --- computeMasks emitted asm ---------------------------------------------- + +class TestComputeMasks: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_combined_parity_branch(self): + _init_rocisa_gfx1250() + # ClusterDim=[2,2] -> maskA = 1 | (1<<2) = 5 (0x5); maskB = (1<<2)-1 = 3 (0x3). + mod = self._c().computeMasks(_StubWriter(), _kernel(clusterDim=(2, 2)), + sgprWgX=61, sgprWgY=62, sgprNWgX=63, sTmp=60) + src = str(mod) + assert "Calculate multicast mask" in src + # Parity election on WaveIdx + even/odd label blocks. + assert "s_bitcmp1_b32 s[sgprWaveIdx], 0" in src + assert "setMulticastMask_OddWave" in src + assert "setMulticastMask_EvenWave" in src + # Combined mask target, both maskA (even) and maskB (odd) into MulticastMask. + assert "s_lshl_b32 s[sgprMulticastMask], 0x5, s61" in src + assert "s_lshl_b32 s[sgprMulticastMask], 0x3, s62" in src + # No split names on the combined path. + assert "MulticastMaskA" not in src + assert "MulticastMaskB" not in src + + def test_split_ab_branch(self): + _init_rocisa_gfx1250() + mod = self._c().computeMasks(_StubWriter(), _kernel(clusterDim=(2, 2), useSubtile=True), + sgprWgX=61, sgprWgY=62, sgprNWgX=63, sTmp=60) + src = str(mod) + assert "s_lshl_b32 s[sgprMulticastMaskA], 0x5, s61" in src + assert "s_lshl_b32 s[sgprMulticastMaskB], 0x3, s62" in src + # Split path has no wave-parity election. + assert "setMulticastMask_OddWave" not in src + + def test_noop_when_multicast_off(self): + _init_rocisa_gfx1250() + mod = self._c().computeMasks(_StubWriter(), _kernel(multicast=False), + sgprWgX=61, sgprWgY=62, sgprNWgX=63, sTmp=60) + assert str(mod).strip() == "" + + +# --- applyToDescriptor emitted asm ----------------------------------------- + +class TestApplyToDescriptor: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_dense_split_or(self): + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor(w, _kernel(), "tdmAGroup1", "A") + assert "s_or_b32 s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in str(mod) + + def test_wave_separated_combined_or(self): + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor(w, _kernel(), "tdmAGroup1", "A", waveSeparated=True) + assert "s_or_b32 s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMask]" in str(mod) + + def test_subtile_split_or(self): + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor(w, _kernel(useSubtile=True), "tdmBGroup1", "B", subtile=True) + assert "s_or_b32 s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in str(mod) + + def test_empty_when_multicast_off(self): + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor(w, _kernel(multicast=False), "tdmAGroup1", "A") + assert str(mod).strip() == "" + + def test_empty_when_cluster_disabled(self): + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor(w, _kernel(clusterDim=(1, 1)), "tdmAGroup1", "A") + assert str(mod).strip() == "" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_legacy_coercion.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_legacy_coercion.py new file mode 100644 index 000000000000..431b166163c0 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_legacy_coercion.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Regression tests for the legacy bool -> int Multicast coercion. +# +# Multicast became an int solution parameter (valid values [-1, 0, 1]: -1 auto, +# 0 off, 1 on). But 600+ shipped library-logic artifacts still serialize the +# pre-existing derived value as `Multicast: false` (BOOL). Because msgpack +# serializes bool and int as different wire types, a bool value trips the strict +# create-library type gate (raiseIfTypeMismatches) -> ConfigTypeError, and would +# cause std::bad_cast at runtime. +# +# Fix: on the serialized-state loading path (raiseProblemTypeOnTypeMismatch= +# False), coerce a bool Multicast to the tri-state int (false -> 0 "off", +# true -> 1 "on"; never -1 "auto"), so both the type gate and the emitted +# msgpack see an int. The config/derivation path now emits an int Multicast on +# all branches directly (no post-derivation coercion needed). +# +# Usage: +# pytest test_multicast_legacy_coercion.py -v +################################################################################ + +import copy +import importlib + +import pytest + +pytestmark = pytest.mark.unit + +from Tensile.SolutionStructs.Solution import ( + coerceLegacyMulticastType, + validateParameterTypes, +) + +S = importlib.import_module("Tensile.SolutionStructs.Solution") +Solution = S.Solution + + +# --- pure coercion helper -------------------------------------------------- + +class TestCoerceHelper: + def test_false_maps_to_int_zero_off(self): + state = {"Multicast": False} + coerceLegacyMulticastType(state) + assert state["Multicast"] == 0 + assert type(state["Multicast"]) is int + # Must be "off" (0), never "auto" (-1). + assert state["Multicast"] != -1 + + def test_true_maps_to_int_one_on(self): + state = {"Multicast": True} + coerceLegacyMulticastType(state) + assert state["Multicast"] == 1 + assert type(state["Multicast"]) is int + + def test_int_values_untouched(self): + for v in (-1, 0, 1): + state = {"Multicast": v} + coerceLegacyMulticastType(state) + assert state["Multicast"] == v + assert type(state["Multicast"]) is int + + def test_absent_key_is_noop(self): + state = {"SomethingElse": 5} + coerceLegacyMulticastType(state) + assert "Multicast" not in state + + +# --- coercion clears the strict type gate ---------------------------------- + +class TestGateIntegration: + def test_bool_multicast_flagged_before_coercion(self): + """A bool Multicast is what the create-library gate flags as + 'found bool ... expected int'.""" + records = validateParameterTypes({"Multicast": False}) + assert records, "expected a type-mismatch record for bool Multicast" + (name, actual, expected), _value, _src = records[0] + assert name == "Multicast" + assert actual == "bool" + assert "int" in expected + + def test_no_mismatch_after_coercion(self): + """After coercion the same state passes validateParameterTypes clean, + i.e. raiseIfTypeMismatches() would not fire.""" + for legacy in (False, True): + state = {"Multicast": legacy} + coerceLegacyMulticastType(state) + assert validateParameterTypes(state) == [] + + +# --- end-to-end through the real Solution constructor ---------------------- +# +# Mirrors the create-library / library-logic loading path +# (raiseProblemTypeOnTypeMismatch=False), which is where the CI build broke. +# Requires amdclang++ for the ISA-info map (same as the other Solution +# characterization tests); skipped gracefully if the toolchain is absent. + +@pytest.fixture(scope="module") +def _toolchain(): + from Tensile.Common.Architectures import gfxToIsa + from Tensile.Common.Capabilities import makeIsaInfoMap + from Tensile.Toolchain.Assembly import makeAssemblyToolchain + from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults + + try: + cxx = validateToolchain("amdclang++") + bundler = validateToolchain(ToolchainDefaults.OFFLOAD_BUNDLER) + except Exception as e: # pragma: no cover - environment dependent + pytest.skip(f"amdclang++ toolchain unavailable: {e}") + isa = gfxToIsa("gfx950") + iim = makeIsaInfoMap([isa], cxx) + assembler = makeAssemblyToolchain(cxx, bundler, "default").assembler + return iim, assembler + + +@pytest.fixture(scope="module") +def _gp_assigned(_toolchain): + from Tensile.Common.GlobalParameters import globalParameters, assignGlobalParameters + from Tensile.Common.ValidParameters import validParameters + + iim, _ = _toolchain + saved_gp = copy.deepcopy(dict(globalParameters)) + saved_vp = copy.deepcopy(dict(validParameters)) + assignGlobalParameters({}, iim) + yield + globalParameters.clear() + globalParameters.update(saved_gp) + validParameters.clear() + validParameters.update(saved_vp) + + +def _make_solution(iim, assembler, multicast, *, loadingSerializedState): + """Build & derive a minimal gfx950 MX FP8 solution carrying a bool Multicast. + + ``loadingSerializedState=True`` drives the library-logic/artifact path + (raiseProblemTypeOnTypeMismatch=False); False drives the config path. + """ + from Tensile.BenchmarkProblems import matrixInstructionToMIParameters + + isa = list(iim.keys())[0] + params = { + "ProblemType": { + "OperationType": "GEMM", + "DataType": "F8", + "DestDataType": "s", + "ComputeDataType": "s", + "HighPrecisionAccumulate": True, + "MXBlockA": 32, + "MXBlockB": 32, + "TransposeA": True, + "TransposeB": False, + "UseBeta": True, + "Batched": True, + }, + "ISA": isa, + "MatrixInstruction": [16, 16, 128, 1, 1, 2, 2, 2, 2], + "WorkGroup": [16, 16, 1], + "WavefrontSize": 64, + "DepthU": 256, + "KernelLanguage": "Assembly", + "PrefetchGlobalRead": 1, + "PrefetchLocalRead": 1, + "ScheduleIterAlg": 3, + "DirectToLds": 1, + "StaggerU": 0, + "StreamK": 3, + "UseSubtileImpl": True, + "LocalReadVectorWidth": -1, + "GlobalReadVectorWidthA": 16, + "GlobalReadVectorWidthB": 16, + "TransposeLDS": 1, + "LdsPadA": -1, + "LdsPadB": -1, + "LdsBlockSizePerPadA": -1, + "LdsBlockSizePerPadB": -1, + "1LDSBuffer": -1, + "VectorWidthA": -1, + "VectorWidthB": -1, + "StoreVectorWidth": -1, + "SourceSwap": True, + "ExpandPointerSwap": True, + "GlobalSplitU": 1, + "InnerUnroll": 1, + "DebugStreamK": 0, + # The legacy artifact wire type under test: a serialized bool. + "Multicast": multicast, + } + mi = params["MatrixInstruction"] + params.update(matrixInstructionToMIParameters( + mi, isa, params["WavefrontSize"], params["ProblemType"], params["WorkGroup"], iim)) + + return Solution( + params, False, False, False, assembler, iim, + raiseProblemTypeOnTypeMismatch=not loadingSerializedState, + ) + + +class TestSolutionConstructorPath: + @pytest.mark.parametrize("legacy,expected_int", [(False, 0), (True, 1)]) + def test_library_logic_path_coerces_to_int( + self, _toolchain, _gp_assigned, legacy, expected_int + ): + """On the serialized-state path a bool Multicast is coerced to the + tri-state int, and the state passes the strict type gate clean.""" + iim, assembler = _toolchain + sol = _make_solution(iim, assembler, legacy, loadingSerializedState=True) + mc = sol._state["Multicast"] + assert type(mc) is int, f"Multicast must be int on the load path, got {type(mc)}" + assert mc == expected_int + assert mc != -1 # coerced off/on, never auto + # The gate the create-library path runs must see no Multicast mismatch. + records = validateParameterTypes(sol._state) + assert all(rec[0][0] != "Multicast" for rec in records), records + + def test_config_path_derives_int(self, _toolchain, _gp_assigned): + """The config/derivation path now emits an int Multicast on all branches + (the post-derivation coercion was removed as redundant): a Multicast=False + input derives to int 0 (the mc==0 force-off branch), not a bool.""" + iim, assembler = _toolchain + sol = _make_solution(iim, assembler, False, loadingSerializedState=False) + mc = sol._state["Multicast"] + assert type(mc) is int, f"config path must derive int, got {type(mc)}" + assert mc == 0, mc + + +if __name__ == "__main__": + import sys + sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py new file mode 100644 index 000000000000..c0aa6119be23 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Unit tests for the decoupled tri-state Multicast solution parameter. +# +# Multicast used to be a derived-only state var, unconditionally forced on for +# ClusterDim != [1,1] (except StreamK cluster reduction). It is now an explicit +# tri-state control: +# -1 = auto (legacy coupling), 0 = force off, 1 = force on. +# Default -1 reproduces the historic derivation exactly. These tests pin: +# * registration (valid values + default), and +# * the derivation semantics (-1 legacy == old behavior; 0 off; 1 on), +# driven through the real config -> Solution derivation path. +# +# Usage: +# pytest test_multicast_tristate.py -v +################################################################################ + +import copy +import os +import sys + +import pytest + +pytestmark = pytest.mark.unit + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +TENSILE_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +sys.path.insert(0, TENSILE_ROOT) +sys.path.insert(0, os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen")) + +_DESIGNED = os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", + "_codegen", "data", "test_data", "_designed", "gfx1250") +_XCCREMAP = os.path.join(_DESIGNED, "xccremap.yaml") +_STREAMK_CLUSTER = os.path.join(_DESIGNED, "streamk_cluster.yaml") + + +# --- registration ---------------------------------------------------------- + +class TestRegistration: + def test_valid_values(self): + from Tensile.Common.ValidParameters import validParameters + assert validParameters["Multicast"] == [-1, 0, 1] + + def test_default_is_legacy_auto(self): + from Tensile.Common.GlobalParameters import defaultSolution + assert defaultSolution["Multicast"] == -1 + + +# --- derivation (through real config -> Solution) -------------------------- + +def _write_variant(tmp_path, base_yaml, name, *, multicast=None): + """Copy a designed config, optionally injecting a Multicast fork value.""" + from Tensile import LibraryIO + import yaml + + cfg = copy.deepcopy(LibraryIO.read(base_yaml)) + if multicast is not None: + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + fork.append({"Multicast": [multicast]}) + out = tmp_path / name + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + return str(out) + + +def _derive_states(cfg_path): + from config_harness import solutions_from_config + sols = solutions_from_config(cfg_path, arch="gfx1250", limit_solutions=8) + states = [] + for s in sols: + st = s._state if hasattr(s, "_state") else s + states.append(st) + return states + + +class TestDerivation: + def test_legacy_auto_cluster_on(self, tmp_path): + # -1 (omitted) + ClusterDim=[2,2], no StreamK reduction -> Multicast on. + cfg = _write_variant(tmp_path, _XCCREMAP, "legacy.yaml") + states = _derive_states(cfg) + assert states, "expected >=1 derived solution" + assert all(st["Multicast"] == 1 for st in states), ( + [st["Multicast"] for st in states]) + + def test_explicit_off(self, tmp_path): + # Multicast=0 forces off even with ClusterDim=[2,2]. + cfg = _write_variant(tmp_path, _XCCREMAP, "off.yaml", multicast=0) + states = _derive_states(cfg) + assert states, "expected >=1 derived solution" + assert all(st["Multicast"] == 0 for st in states), ( + [st["Multicast"] for st in states]) + # ClusterBarrier is gated on Multicast, so it must also be off. + assert all(st["ClusterBarrier"] is False for st in states) + + def test_explicit_on(self, tmp_path): + # Multicast=1 forces on. + cfg = _write_variant(tmp_path, _XCCREMAP, "on.yaml", multicast=1) + states = _derive_states(cfg) + assert states, "expected >=1 derived solution" + assert all(st["Multicast"] == 1 for st in states), ( + [st["Multicast"] for st in states]) + + def test_legacy_auto_streamk_reduction_off(self, tmp_path): + # -1 (omitted) + StreamKClusterReduction=1 -> Multicast stays off + # (the legacy StreamK cluster interaction is preserved). + cfg = _write_variant(tmp_path, _STREAMK_CLUSTER, "sk_legacy.yaml") + states = _derive_states(cfg) + assert states, "expected >=1 derived solution" + assert all(st["Multicast"] == 0 for st in states), ( + [st["Multicast"] for st in states]) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From 8cac9fe46650c2b3c52c2910de11e4c824776e41 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 20 Jul 2026 18:35:36 +0000 Subject: [PATCH 02/24] feat(tensilelite): add StreamK cooperative cluster loads (StreamKMulticast) Add the gfx1250 StreamK DP cooperative B-multicast fast path (StreamKMulticast), auto-derived from StreamK=3 + ClusterDim on top of the ClusterLoad component + tri-state Multicast foundation. - Collapse: StreamK==3 && ClusterDim!=[1,1] auto-enables StreamKMulticast (derived-only internal state; no YAML opt-in), forcing Multicast on and enabling the cluster-scope barrier handshake (ClusterBarrier). - StreamK.py: multicast mask predicate (clusterMulticastValid), DP->SK boundary mask clear, and a prologue wave-0 s_barrier_signal -3 that pairs the InsertClusterBarrierPass first-load wait (balanced signal/wait counts). - KernelWriter.py: postMainLoopBarrierCheckAndReset preserves cluster-scope -3 barriers. - ClusterLoad.py: re-add the split-mask hooks (usesCombinedMask / maskSgprName) for StreamKMulticast. - _validateStreamKMulticast: SK3 + [C,1] pow2 + gfx1250/HasTDM/TDMInst=3 + XCC=0; Multicast==1 force-on hard-guard. - Host: streamKMulticast field threaded through SizeMapping / C++ structs; grid multiple-of-C guard + getSKGridImpl ceil(tiles/C)*C override; SK4/SK5 + ClusterDim reject. Mutual-exclusion with the barrier-only cluster reduction is deferred to the following stacked PR. JIRA ID : AIHPBLAS-3929 --- ...er-load-component-and-streamk-multicast.md | 561 ++++++++++++++++++ .../Tensile/Common/GlobalParameters.py | 3 + .../Tensile/Common/ValidParameters.py | 10 + .../Tensile/Components/ClusterLoad.py | 17 + .../tensilelite/Tensile/Components/StreamK.py | 138 +++++ .../tensilelite/Tensile/Contractions.py | 2 + .../tensilelite/Tensile/KernelWriter.py | 8 +- .../Tensile/SolutionStructs/Solution.py | 174 +++++- ...mxf4_force_dp_only_cluster_multicast.yaml} | 2 +- .../core/sk_mxf4gemm_cluster_multicast.yaml | 141 +++++ .../gfx1250/core/sk_mxf4gemm_tdm_cluster.yaml | 101 ---- .../core/sk_mxf8gemm_cluster_multicast.yaml | 121 ++++ .../test_solution_class_char.ambr | 5 +- ...treamk_cluster_multicast_gfx1250_char.ambr | 21 + ...er.yaml => streamk_cluster_coop_load.yaml} | 16 +- .../gfx1250/streamk_cluster_multicast.yaml | 116 ++++ ..._streamk_cluster_coop_load_gfx1250_char.py | 104 ++++ .../test_streamk_cluster_gfx1250_char.py | 59 -- ..._streamk_cluster_multicast_gfx1250_char.py | 111 ++++ .../Tests/unit/test_multicast_tristate.py | 22 +- .../unit/test_streamk_cluster_sk45_reject.py | 96 +++ .../Tests/unit/test_streamk_multicast.py | 346 +++++++++++ .../include/Tensile/ContractionSolution.hpp | 1 + .../Serialization/ContractionSolution.hpp | 1 + .../tensilelite/src/ContractionSolution.cpp | 29 + 25 files changed, 2019 insertions(+), 186 deletions(-) create mode 100644 docs/design/cluster-load-component-and-streamk-multicast.md rename projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/{sk_mxf4_force_dp_only_cluster.yaml => sk_mxf4_force_dp_only_cluster_multicast.yaml} (98%) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml delete mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_tdm_cluster.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_gfx1250_char.ambr rename projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/{streamk_cluster.yaml => streamk_cluster_coop_load.yaml} (77%) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py delete mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_gfx1250_char.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_sk45_reject.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py diff --git a/docs/design/cluster-load-component-and-streamk-multicast.md b/docs/design/cluster-load-component-and-streamk-multicast.md new file mode 100644 index 000000000000..89e1b8c05cd7 --- /dev/null +++ b/docs/design/cluster-load-component-and-streamk-multicast.md @@ -0,0 +1,561 @@ + + +# Design: `ClusterLoad` Component + StreamK Cooperative Multicast Loads (gfx1250) + +**Target arch:** gfx1250 (ISA `(12,5,0)`, wave32, TDM: `MXLoadInst=TDM` → `TDMInst=3`). +**Scope:** Two coordinated pieces of work. + +1. **`ClusterLoad` component + behavior-preserving refactor.** Extract the multicast + ("cluster load") mask machinery — value computation, SGPR declare/undeclare, and the + three descriptor apply sites (dense, wave-separated, subtile) — into a single reusable + `ClusterLoad` tensilelite `Component`, and refactor the existing subtile Multicast path + onto it. This is a **pure refactor**: zero change to emitted assembly for existing + configs, gated by codegen snapshots. +2. **Decouple `Multicast` from `ClusterDim`** into an independent opt-in, so barrier-only + clustering (the separate cluster split-barrier reduction feature) and cooperative-load + clustering compose independently instead of `ClusterDim != [1,1]` unconditionally forcing + `Multicast=True`. +3. **StreamK cooperative cluster loads.** Build TDM multicast on top of `ClusterLoad` for + the StreamK **data-parallel (DP) region**, resolving the tension between the spatial + `[C0,C1]` cluster cooperative loads want and the `[C,1]` same-tile K-peer cluster the + shipped reduction wants. + +The separate barrier-only cluster reduction (`docs/design/streamk-wg-clusters.md`) is +orthogonal and tracked independently. + +Source-line anchors reference the code as it exists now and were verified against the tree. + +--- + +## 1. Background (verified anchors) + +### 1.1 What a "cluster load" is + +A cluster load is a normal TDM `tensor_load_to_lds` whose descriptor `Group1[word0]` has a +multicast-mask bit field OR'd in. The HW then broadcasts (multicasts) the loaded tile to +every workgroup in the cluster whose bit is set. Attachment is a single `SOrB32`: + +```511:514:projects/hipblaslt/tensilelite/Tensile/Components/TensorDataMover.py + def setMulticastMask(self, group1: int | str, mask: str, writer: "KernelWriterAssembly") -> Module: + mod = Module() + mod.add(SOrB32(sgpr(f"{group1}"), sgpr(f"{group1}"), sgpr(f"{mask}"))) + return mod +``` + +### 1.2 Mask value computation and the three topologies + +The mask *value* is computed once in `defineAndResources` +(`KernelWriterAssembly.py:2646-2694`), from the WG's position within the cluster and +`ClusterDim`: + +- `maskA = OR over idx in range(ClusterDim[1]) of (1 << (idx*ClusterDim[0]))`, then shifted + left by `wg_x`. Bit `wg_y*ClusterDim[0] + wg_x` = the cluster-linear index of the WG. + So `maskA` selects every WG sharing the same `wg_x` column (same M block) across all + `ClusterDim[1]` rows. +- `maskB = (1 << ClusterDim[0]) - 1`, then shifted left by `wg_y * ClusterDim[0]`. Selects + every WG in the same `wg_y` row (same N block). + +Three name topologies (this exact selection matrix must be preserved by the refactor): + +| Topology | Predicate | SGPR name(s) | +|---|---|---| +| Combined single-parity | `tdmA and tdmB and NumWaves>1 and not UseSubtileImpl` | `MulticastMask` (even wave = maskA, odd wave = maskB, chosen by `WaveIdx` parity) | +| Split A/B | otherwise (subtile, single-tensor, or `NumWaves==1`) | `MulticastMaskA`, `MulticastMaskB` | +| Metadata (sparse) | `enableTDMMetadata` | `MulticastMaskMetadata` (follows A for `Sparse==1`, B for `Sparse==2`) | + +Declare (`KernelWriter.py:9163-9176`) and undeclare (`KernelWriter.py:2838-2848`) mirror the +same predicate. + +### 1.3 The three descriptor apply sites (the duplication to unify) + +All three OR the mask into `Group1` under the same gate +`kernel["Multicast"] and enableCluster`, where `enableCluster = prod(ClusterDim) != 1`: + +- **Dense** `initTDMDescriptor` — `KernelWriterAssembly.py:18901-18902`; local + `maskSgprName(tc)` returns `MulticastMask{A|B}` (split). +- **Wave-separated** `initTDMDescriptorWaveSeparatedImpl` — `KernelWriterAssembly.py:19059-19060`; + local `maskSgprName` returns the combined `"MulticastMask"`. +- **Subtile** `initTDMDescriptorSubtile` — `Components/Subtile/SubtileGREmit.py:1108-1113`; + uses `MulticastMask{tc}` (split). + +`initTDMDescriptorSubtile` (`SubtileGREmit.py:1061-1155`) is a near-clone of the writer +`initTDMDescriptor` (`KernelWriterAssembly.py:18843-18994`); the mask attachment is the +piece we lift into `ClusterLoad`. The rest of the descriptor build (LDS offset, tensor +dims/strides, padding) stays where it is — `ClusterLoad` does **not** own descriptor-group +allocation or LDS offsets. + +### 1.4 Cooperative-thread partition (shared math) + +`numCooperativeWGs = ClusterDim[1] for A / ClusterDim[0] for B`, duplicated in +`Components/GL2Prefetch.py:26` and `:78`. `ClusterLoad` centralizes this as +`cooperativeThreadPartition`. + +### 1.5 Component framework + +- Auto-registration metaclass: `Component.py:113-124` (`__init__` sets `implementations` + and `setattr` on each base). `matches()` partial-matches `asmCaps`/`archCaps`/`kernel` + (`Component.py:132-144`); `find()` requires exactly one match, raises on >1, returns + `None` on 0 → fallback (`Component.py:167-177`). +- Categories (abstract, no `__call__`) declared near `Component.py:305-313` + (`TensorDataMover`, `GL2Prefetch`); `from .Components import *` at `:318`. +- `Components/__init__.py:28-55` `__all__` lists each component module. +- `TensorDataMoverLoad` shows the concrete pattern: `asmCaps = {"HasTDM": True}`, + `kernel = {"TDMInst": 3}`, method-bag returning rocisa `Module`s + (`TensorDataMover.py:13-15`). +- New-file SPDX header (`# Copyright Advanced Micro Devices, Inc., or its affiliates.` / + `# SPDX-License-Identifier: MIT`). + +### 1.6 `Multicast` / `ClusterDim` coupling (current) + +`Solution.py:1046-1064` forces `Multicast=True` for any `ClusterDim != [1,1]`, except it is +already suppressed for Stream-K. `Multicast` is **not** a user parameter in +`ValidParameters.py` (it is a derived state var). + +### 1.7 StreamK addressing (where multicast pays off) + +- `skTileIndex` (`StreamK.py:439`, math `469-483`) magic-divides `StreamKIter` into a tile + index and `StreamKLocalStart`/`StreamKLocalEnd` (iter offsets within the tile). +- `skIndexToWG` (`StreamK.py:487-503`) maps tile → `(WorkGroup0, WorkGroup1, WorkGroup2)` + with the linearization `tileID = WG2*(nWG0*nWG1) + WG1*nWG0 + WG0` — **WG0 (M) is + fastest**, so consecutive tiles are M-adjacent (same WG1/N block, consecutive WG0). +- The K-direction StreamK offset (`StreamKLocalStart*DepthU*strideL`) is added in + `computeLoadSrdCommon` (`StreamK.py:548`, `555-562`) and `graAddressesCommon` + (`StreamK.py:629`, `636-645`); both are `tc`-generic (shared A/B). **The `WorkGroup0*MT0` + / `WorkGroup1*MT1` tile-base term lives in the base KernelWriter GRA/SRD code, keyed on + the post-`skIndexToWG` `WorkGroup0/1`.** A/B thus bind to WG0/M and WG1/N respectively; + the K offset is identical for A and B. +- `graWorkGroup` (SK3 TwoTileDPFirst, `StreamK.py:2846`; DP shift `2921-2925`): a WG `w` + processes tiles `w, w+skGrid, w+2*skGrid, …`. In the **first DP round** consecutive WGs + map to consecutive tiles → **M-adjacent → share the same B (N-block) over full K**. +- `StreamKIdx = WorkGroup0` (`StreamK.py:2658-2663`), preceded by the `ttmp9` raw-wgid + workaround (`StreamK.py:2645-2656`), which is **already skipped under any cluster + (`ClusterDim != [1,1]`)** so the cluster-remapped `WorkGroup0` survives. +- DP/SK region split in `graWorkGroup` at `StreamK.py:2907-2932`; the "started & + finished the whole tile → regular store, no reduction" branch at `StreamK.py:922-933`. +- gfx1250 kernel-side WG remap: `WorkGroup0 = cluster_x*nwg_x + wg_x` + (`KernelWriterAssembly.py:2628-2632`), so cluster `c` owns a contiguous `WorkGroup0` + range `[c*C, c*C + C)`. + +### 1.8 Data-reuse fact (drives the whole StreamK design) + +Two DP WGs share **A** iff same WG0 (M) and overlapping K; share **B** iff same WG1 (N) and +overlapping K. DP tiles compute the whole tile over full K, so K always overlaps. + +- K-split peers of one tile (the shipped `[C,1]` reduction cluster): same WG0/WG1 but + **disjoint K** → **zero reuse**. Multicasting the reduction cluster multicasts nothing. +- Real reuse is **spatial** over a common full-K range: adjacent DP tiles. Consecutive-WG + clustering (`[C,1]`) gives M-adjacent tiles → **shared B**. (Tiles `nWG0` apart are + N-adjacent → shared A; not reachable by consecutive `[C,1]` clustering.) + +**Conclusion: in the StreamK DP region a `[C,1]` cluster multicasts B (not A).** This maps +exactly onto the existing mask math with `wg_y=0, C0=C, C1=1`: `maskB = (1< bool` | The single predicate `tdmA and tdmB and NumWaves>1 and not UseSubtileImpl`. Kills the 3 duplicated copies (`KernelWriter.py:9170`, `:2842`, `KernelWriterAssembly.py:2668`). | +| `maskSgprName` | `(kernel, tc, *, subtile=False) -> str` | Central name resolver: combined `"MulticastMask"` when `usesCombinedMask` and not `subtile`; else `f"MulticastMask{strip_MXS(tc)}"`; `"MulticastMaskMetadata"` for metadata. Subsumes the two local `maskSgprName` closures + the subtile literal. | +| `declareSgprs` | `(writer, kernel) -> None` | Moves `KernelWriter.py:9163-9176` (uses `writer.defineSgpr`). | +| `undeclareSgprs` | `(writer, kernel) -> Module` | Moves `KernelWriter.py:2838-2848` (uses `writer.undefineSgpr`). | +| `computeMasks` | `(writer, kernel, *, sgprWgX, sgprWgY, sgprNWgX, sTmp) -> Module` | Moves `KernelWriterAssembly.py:2646-2694` verbatim, including the wave-parity branch and the metadata cases. Takes the exact SGPR operands the caller already holds so emitted asm is byte-identical. | +| `applyToDescriptor` | `(writer, kernel, group1, tc, *, subtile=False) -> Module` | Folds the gate + name choice + the `SOrB32`. Returns an **empty `Module`** when `not (kernel["Multicast"] and enableCluster)` — identical to today's skipped `if`. Internally calls `TensorDataMover.setMulticastMask`. Replaces the 3 apply sites. | +| `cooperativeThreadPartition` | `(kernel, tc) -> int` | `ClusterDim[1] if tc-ends-A else ClusterDim[0]`. Shared with `GL2Prefetch` (`:26`, `:78`). | + +Rationale for passing SGPR indices into `computeMasks`: the current code emits into +`sTmp+1..4` allocated by the surrounding `defineAndResources`. To guarantee **zero asm +diff**, the component must not re-allocate; it receives the same operands and emits the same +instructions in the same order. This is a mechanical lift, not a rewrite. + +### 2.3 Behavior-preserving refactor wiring + +- `KernelWriter.py:9163-9176` → `ClusterLoad.find(self).declareSgprs(self, kernel)`. +- `KernelWriter.py:2838-2848` → `... .undeclareSgprs(self, kernel)`. +- `KernelWriterAssembly.py:2646-2694` → `... .computeMasks(self, kernel, sgprWgX=..., sgprWgY=..., sgprNWgX=..., sTmp=...)` (same operands as today). +- `KernelWriterAssembly.py:18901-18902` (dense) → `... .applyToDescriptor(self, kernel, descSgprName(1), tc)`. +- `KernelWriterAssembly.py:19059-19060` (wave-separated) → same call (component resolves to combined name via `usesCombinedMask`). +- `SubtileGREmit.py:1108-1113` (subtile) → `... .applyToDescriptor(writer, kernel, descSgprName(1), tc, subtile=True)`. + +Guard: `ClusterLoad.find` returns `None` when `HasTDM`/`TDMInst` don't match; callers keep a +`comp = ClusterLoad.find(self)` + `if comp:` fallback exactly like `TensorDataMoverLoad.find` +usage, so non-TDM archs are untouched. + +**Regression gate (must stay green, zero asm change):** +`test_r4_xccremap_char.py` (drives `ClusterDim=[2,2]` with both a `MIWaveGroup` prod==1 → +split `MulticastMaskA/B` kernel and a prod==4 → combined `MulticastMask` kernel) and +`test_streamk_cluster_coop_load_gfx1250_char.py`. Add a byte-exact golden snapshot of the +emitted mask/apply region for a `[2,2]` combined config and a subtile split config before +the refactor; diff to zero after. + +--- + +## 3. Decoupling `Multicast` from `ClusterDim` + +### 3.1 Mechanism + +Add an explicit tri-state solution parameter (`ValidParameters.py`, near `ClusterDim`): + +```python +# -1 = auto (legacy: ClusterDim!=[1,1] implies Multicast, except StreamK cluster paths) +# 0 = force multicast off +# 1 = force multicast on (independent of ClusterDim coupling) +"Multicast": [-1, 0, 1], +``` + +Default `-1` preserves **all** existing emitted asm. Rewrite `Solution.py:1046-1064`: + +```python +state["ClusterBarrier"] = False +mc = state.get("Multicast", -1) +if mc == 1: + state["Multicast"] = True +elif mc == 0: + state["Multicast"] = False +elif state.get("StreamKMulticast", 0): + state["Multicast"] = True # derived cooperative-load path +else: # auto (legacy behavior) + state["Multicast"] = (state["ClusterDim"] != [1, 1] + and state["StreamK"] == 0) +# ClusterBarrier decision (legacy/cluster-subtile path + the derived StreamKMulticast path) +if state["ClusterDim"] != [1, 1] and state["Multicast"] and state["TDMInst"] != 0 \ + and (state["StreamK"] == 0 or state.get("StreamKMulticast", 0)) \ + and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): + state["ClusterBarrier"] = True +``` + +Net effect for existing configs: `Multicast` default `-1` → identical derivation to today +(subtile/dense clustered configs keep `Multicast=True`; non-multicast Stream-K keeps it +`False`). New capability: `Multicast=1`/`0` explicit override, and `StreamKMulticast` (§4) +turns on multicast for its own path without relying on the coupling. + +This is the one step in the sequence that *can* legitimately change asm — but only for +configs that opt in. Keep existing subtile/multicast YAMLs on `Multicast=-1` so their +snapshots don't move. + +--- + +## 4. StreamK cooperative cluster loads (DP region) + +### 4.1 The spatial vs reduction tension, resolved + +- Cooperative loads need a **spatial** cluster over **distinct** tiles (`[C,1]` consecutive + WGs → M-adjacent → shared B). +- The shipped reduction needs a **`[C,1]` same-tile K-peer** cluster (host ties + `skGrid = C*tiles`, `skIndexToWG` collapses the C peers onto one tile). +- A WG's HW cluster membership is fixed at launch, so a single kernel cannot have the + cluster mean "spatial DP" and "K-peers" at once. + +**Resolution: mutual exclusion.** For v1, `StreamKMulticast` and the barrier-only cluster +reduction are mutually exclusive. When `StreamKMulticast` is on, `ClusterDim=[C,1]` is free +to denote the spatial cluster; the reduction is off, so no `[C,1]` is claimed for K-peers. **No +separate `SpatialClusterDim` is needed in v1** — the same `ClusterDim=[C,1]` is reused, its +meaning selected by which StreamK cluster mode is enabled. (A future 2-D `[C0,C1]` mode for +A-multicast or combined reduction+coop-load would introduce a distinct spatial dim; deferred.) + +### 4.2 MVP target: DP-only, B-multicast + +The multicast mask is computed **once** at kernel init and baked into the descriptor +`Group1[word0]`; it is applied to every mainloop TDM load. For correctness, the sharing +relationship it encodes must hold for **every** load the WG issues. Therefore v1 requires +that the WG never enters a partial (K-split) tile with a stale spatial mask: + +**v1 MVP = cooperative B-multicast for the DP region with `StreamKForceDPOnly=1`** (no SK +partial tiles → no reduction → the static spatial mask is valid for all issued loads), and +single DP round per cluster (see §4.4). + +Recommended smallest correct MVP: +- **`StreamKMulticast=1`** requires `StreamKForceDPOnly=1` and no barrier-only cluster reduction. +- Reuse `ClusterDim=[C,1]`, `C` a power of two in `2..16`. +- Multicast B across the cluster; A loaded per-WG. + +Deferred (see §7): mask-toggle at the DP→SK boundary to allow cooperative DP loads inside a +full SK3 kernel (SK tail reduces via the existing global-flag path); combined +reduction+coop-load; A-multicast / 2-D spatial cluster; multi-round with per-round validity. + +### 4.3 Mask derivation from post-`skIndexToWG` coords + +The existing `computeMasks` derives `wg_x`/`wg_y` from the raw HW cluster registers +(`ttmp*`). For StreamK the authoritative WG coordinates are the ones produced by +`skIndexToWG` (`StreamK.py:503`), not the raw HW position, because StreamK re-derives +`(WG0,WG1)` from `StreamKIdx`. For the DP `[C,1]` MVP the two coincide +(`wg_x = StreamKIdx & (C-1)`, since the reduction ttmp9 workaround is skipped and +`WorkGroup0 = cluster_x*C + wg_x`), so the MVP feeds `computeMasks` with +`sgprWgX = StreamKIdx & (C-1)`, `wg_y = 0`, `C0=C`, `C1=1`, yielding the B-broadcast mask +`(1< **Status (as shipped):** `StreamKMulticast` is **not** a public/benchmark parameter. +> It is a derived-only internal state key (like `ClusterBarrier`), enabled automatically +> when `ClusterDim != [1,1]` on SK3 — see §3.1 / the +> collapse in `assignProblemIndependentDerivedParameters`. It has no `ValidParameters.py` +> entry; `_validateStreamKMulticast` is the internal guard that hard-rejects a derived +> config it cannot satisfy. The `[0,1]` "param" framing below reflects the original plan. + +### 5.1 Parameters + +| Param | Values | Meaning | +|---|---|---| +| `Multicast` | `[-1,0,1]` | Decoupled multicast control (§3). Default `-1` = legacy auto. | + +`StreamKMulticast` is internal-derived (see status note above), not a user-settable param. +The separate barrier-only cluster reduction feature is orthogonal and unchanged. + +### 5.2 Validation rules (`Solution.py`, reject-with-reason pattern) + +The derived `StreamKMulticast` on-state is rejected (`_validateStreamKMulticast`) unless all hold: +- `StreamK == 3` (SK3; DP schedule + `skIndexToWG` assumptions). +- `StreamKForceDPOnly == 1` (v1: no partial tiles reach the static mask). +- not the barrier-only cluster reduction (mutually exclusive — cannot claim `[C,1]` two + ways; the collapse gives the reduction precedence, and the reject itself is added + alongside that feature). +- `ClusterDim == [C,1]`, `C` power of two, `2 <= C <= 16` (`ClusterDim[1] == 1`; StreamK + grid is effectively 1-D along x). +- gfx1250 `asmCaps["HasTDM"]` and `TDMInst != 0` (MX TDM loads; multicast is a TDM feature). +- `StreamKXCCMapping == 0` (XCC=3 overflows SGPRs; WGM/XCC remap is bypassed under + clustering anyway). + +Also set `Multicast=True` for this path via the decoupled derivation (§3.1) so +`ClusterLoad.applyToDescriptor` fires. + +### 5.3 Interaction notes + +- gfx1250 StreamK already uses TDM (`MXLoadInst=TDM` → `TDMInst=3`), so `HasTDM` is the + natural cap and no new transport is introduced. +- `StreamKAtomic` is incompatible (atomic path skips the workspace/tile structure the DP + schedule relies on); reject. + +--- + +## 6. Ordered implementation task breakdown + +Sequencing keeps the tree green: **pure extraction first (zero asm), then the decouple, +then the feature.** `[P]` marks tasks that can proceed in parallel within a group. + +### Group 0 — Snapshot baseline (do first, no source change) +- **T0.1** Add byte-exact golden snapshots for the mask compute+apply region: a `[2,2]` + combined-mask config, a `[2,2]` split-mask config, and a subtile split config. Anchor: + extend `test_r4_xccremap_char.py` and the subtile char config under + `Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/`. + +### Group 1 — `ClusterLoad` extraction (pure refactor, snapshot-gated) +- **T1.1** Declare `class ClusterLoad(Component)` at `Component.py:305-313`; add + `"ClusterLoad"` to `Components/__init__.py:__all__`. +- **T1.2** Create `Components/ClusterLoad.py` with `ClusterLoadTDM` (SPDX header; + `asmCaps={"HasTDM":True}`, `kernel={"TDMInst":3}`); implement `usesCombinedMask`, + `maskSgprName`, `cooperativeThreadPartition`. +- **T1.3** Move mask compute into `computeMasks`; wire `KernelWriterAssembly.py:2646-2694` + to call it with the existing SGPR operands. +- **T1.4 [P]** Move declare/undeclare into `declareSgprs`/`undeclareSgprs`; wire + `KernelWriter.py:9163-9176` and `:2838-2848`. +- **T1.5 [P]** Implement `applyToDescriptor`; wire the 3 apply sites + (`KernelWriterAssembly.py:18901-18902`, `:19059-19060`, `SubtileGREmit.py:1108-1113`). +- **T1.6** Point `GL2Prefetch.py:26,78` at `cooperativeThreadPartition` (optional but DRY). +- **Gate:** T0.1 snapshots + `test_r4_xccremap_char.py` + + `test_streamk_cluster_coop_load_gfx1250_char.py` all diff to zero. + +### Group 2 — Decouple `Multicast` (may change asm only for opt-in configs) +- **T2.1** Add `"Multicast": [-1,0,1]` to `ValidParameters.py`. +- **T2.2** Rewrite `Solution.py:1046-1064` per §3.1; keep existing YAMLs on `-1`. +- **Gate:** existing multicast/subtile/reduction snapshots unchanged; add a snapshot for an + explicit `Multicast=1`/`0` override. + +### Group 3 — StreamK cooperative DP multicast (feature) +- **T3.1** Derive `StreamKMulticast` internally (no `ValidParameters.py` entry — it is a + derived-only state key, auto-enabled on SK3 + `ClusterDim!=[1,1]` not reduction) + + `Solution.py` validation (§5.2) + decoupled `Multicast` enablement. +- **T3.2 [P]** Host: `sizeMapping.streamKMulticast` (`ContractionSolution.hpp:146`, + `Contractions.py:630,725`); `getSKGridImpl` grid = `ceil(tiles/C)*C` + (`ContractionSolution.cpp:4087-4090`/`:3214-3218` region); thread through launch (no + launch change — `:1783-1794` already cluster-aware). +- **T3.3** Kernel: at StreamK preLoop compute `wg_x = StreamKIdx & (C-1)` and the + `clusterMulticastValid` predicate (§4.4) from `NumWorkGroups0`/`totalTiles`; feed + `ClusterLoad.computeMasks` with `wg_y=0, C0=C, C1=1` and gate the mask OR on the predicate + (via `applyToDescriptor`). Anchors: `StreamK.py:2645-2663` (StreamKIdx/ttmp9), + `skIndexToWG` `:487-503`. +- **T3.4** Ensure A stays per-WG (maskA = self) and B multicast (maskB = all-C); verify the + `[C,1]` path selects split `MulticastMaskA/B` names. + +Dependencies: Group 1 → Group 2 → Group 3 (T3.1). T3.2 (host) is parallel to T3.3/T3.4 +(kernel) once T3.1 lands the param. + +--- + +## 7. Correctness, edge cases, risks + +| Risk | Why | De-risk | +|---|---|---| +| **Wrong-B multicast → silent wrong results** | Static mask applied to a load where the C WGs don't actually share B (SK partial tile, non-adjacent, partial cluster). | v1 requires `StreamKForceDPOnly` (no partial tiles) + single round + init-time `clusterMulticastValid` predicate that ORs the mask only when provably valid; else load normally. | +| **Idle WG in a partial last cluster** | `totalTiles % C != 0` leaves tail WGs with no tile; a multicast expecting them may hang/mis-deliver. | Predicate disables multicast for a not-fully-populated cluster. **OPEN Q**: exact HW behavior when a masked target WG is not at a matching TDM load (see §8). | +| **DP→SK boundary with a live spatial mask** | In a full SK3 kernel the mask would still be applied to SK-tile loads. | v1 forbids the SK region (`StreamKForceDPOnly`). Deferred: clear the mask bit (single `SAndB32` on `Group1[word0]`) at the boundary (`StreamK.py:2907-2932`) since DP is first and SK last. | +| **Refactor changes asm** | Component lift could reorder/realloc SGPRs. | `computeMasks` takes the exact existing SGPR operands; snapshot gate (Group 0) diffs to zero. | +| **`Multicast` decouple regresses existing configs** | Changing the coupling could flip `Multicast` for subtile/dense clustered kernels. | Default `-1` = identity derivation; existing YAMLs untouched; snapshot-gated. | +| **Mask from raw HW coords instead of skIndexToWG** | StreamK re-derives `(WG0,WG1)`. | For DP `[C,1]` the two coincide (`wg_x=StreamKIdx&(C-1)`); documented general rule for 2-D follow-ups. | +| **SGPR pressure** | gfx1250 StreamK SGPRs tight; XCC=3 overflows. | Reuse existing `MulticastMask*` SGPRs; predicate uses `allocTmpSgpr` scopes and bitwise ops (C power of two); forbid `StreamKXCCMapping=3`. | +| **A-multicast expectation** | Consecutive `[C,1]` cluster shares B, not A. | Documented; A stays per-WG in v1; A-multicast needs an N-strided/2-D cluster (deferred). | + +--- + +## 8. Test plan (behavior → test) — as shipped + +Mirrors existing patterns: CPU asm-string unit, syrupy snapshot characterization, GPU +roundtrip (run under the arch's simulator/hardware), and C++ client end-to-end. + +| Behavior | Test type | Where | +|---|---|---| +| **Refactor is byte-exact**: `computeMasks`/`applyToDescriptor`/declare/undeclare emit identical asm for `[2,2]` combined, `[2,2]` split, and subtile split configs | snapshot char (regression gate) | `test_r4_xccremap_char.py`, `test_streamk_cluster_coop_load_gfx1250_char.py` | +| `ClusterLoad.find` returns TDM impl on gfx1250, `None` (fallback) on non-TDM; `usesCombinedMask`/`maskSgprName` matrix (combined vs split vs metadata) | CPU unit | `Tests/unit/test_cluster_load_component.py` | +| `Multicast` tri-state: `-1` reproduces legacy derivation exactly; `0` forces off; `1` forces on independent of `ClusterDim` (derived value is an int on all paths) | CPU unit | `Tests/unit/test_multicast_tristate.py`, `Tests/unit/test_multicast_legacy_coercion.py` | +| `StreamKMulticast` validation matrix: accepted only for SK3 + `ClusterDim=[C,1]` (pow2 2..16) + gfx1250 `HasTDM`/`TDMInst=3` + XCC=0 + not `StreamKAtomic`; auto-enabled config with `Multicast=0` rejected; rejected otherwise | CPU unit | `Tests/unit/test_streamk_multicast.py` | +| Cluster config emits real gfx1250 asm (`err==0`); DP loads carry the B-broadcast mask (`MulticastMaskB` OR into `Group1`); A load carries self-only mask; predicate gate + DP→SK boundary clear present | snapshot char | `test_streamk_cluster_multicast_gfx1250_char.py` (designed config `_designed/gfx1250/streamk_cluster_multicast.yaml`, `__snapshots__/*.ambr`) | +| DP GEMM with `StreamKMulticast`, `C∈{2,4}` reduces correctly and does not hang; matches non-multicast reference | GPU roundtrip | `Tests/unit/test_streamk_cluster_reduction_gpu.py` and cluster char roundtrips (`@requires_gpu_gfx1250`; watchdog on hang) | +| Real multi-WG cluster StreamK DP GEMM end-to-end | C++ client | `Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml` (+ mxf4 sibling) | + +The gfx1250 GPU marker reuses `Tests/unit/gpu_test_helpers.py` +(`HAS_GFX1250`/`requires_gpu_gfx1250`, `TENSILE_GPU_TARGET=gfx1250`). + +--- + +## 9. Open questions (need a human decision at the design gate) + +1. **HW multicast semantics when a masked target WG is not executing a matching TDM load** + (idle tail WG, or a WG that finished its tiles): does the loader hang, drop the + broadcast, or corrupt? This determines whether the init-time predicate is sufficient or + whether the grid must be constrained to always-full clusters. (Mirrors the reduction + feature's deadlock concern.) +2. **Is `StreamKForceDPOnly`-scoped v1 valuable enough**, or should we prioritize the + DP→SK boundary mask-clear so cooperative loads apply inside a normal SK3 kernel (the + larger real-world win)? The boundary-clear is a modest add (§7) but widens the test + matrix. +3. **`Multicast` as tri-state `[-1,0,1]` vs a separate boolean** (`MulticastEnable`) — the + tri-state keeps one knob and a clean legacy default; a separate boolean is more explicit + but adds a second interacting param. Recommend tri-state; confirm. +4. **Grid rounding policy** for `StreamKMulticast`: `ceil(tiles/C)*C` (idle tail WGs) vs + requiring `totalTiles % C == 0` at selection time (fewer sizes, no idle WGs). Depends on + Q1. + +--- + +## 10. Summary of decisions + +- **`ClusterLoad` component**: new capability-selected category (`asmCaps HasTDM`, + `kernel TDMInst=3`) owning mask value (`computeMasks`), declare/undeclare, topology + decision (`usesCombinedMask`/`maskSgprName`), descriptor attach (`applyToDescriptor`, + gate folded in), and `cooperativeThreadPartition`. It does **not** own descriptor groups + or LDS offsets. The three apply sites and the subtile clone unify onto it; refactor is + byte-exact and snapshot-gated. +- **Decouple**: `Multicast` becomes a tri-state param (`-1` auto = legacy identity, `0` + off, `1` on); `Solution.py:1046-1064` derivation rewritten so `ClusterDim!=[1,1]` no + longer unconditionally forces multicast. +- **StreamK target = DP region, B-multicast** on a `[C,1]` consecutive-WG cluster + (M-adjacent tiles share the N-block). Spatial-vs-reduction tension resolved by **mutual + exclusion** (`StreamKMulticast` xor the barrier-only cluster reduction), reusing + `ClusterDim=[C,1]` with mode-selected meaning; no separate spatial dim in v1. +- **MVP** = `StreamKMulticast=1` requiring `StreamKForceDPOnly=1`, single DP round, init-time + `clusterMulticastValid` predicate gating the mask. Deferred: DP→SK boundary mask-clear, + multi-round, A-multicast / 2-D cluster, combined reduction+coop-load. +- **Enablement**: `Multicast` (`[-1,0,1]`) is the one public knob; `StreamKMulticast` is + internal-derived (auto-enabled on SK3 + `ClusterDim!=[1,1]`, not reduction), not a param. +- **Sequencing**: extraction (green) → decouple → feature. diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py index a1176fb5c593..1c045d9c8ecf 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py @@ -571,6 +571,9 @@ {"StreamKAtomic": [0]}, {"StreamKXCCMapping": [0]}, {"StreamKFixupTreeReduction": [0]}, + # NOTE: StreamKMulticast is a derived-only internal state key (like + # ClusterBarrier), auto-enabled by Solution.py for StreamK==3 + ClusterDim + # clusters; it is deliberately NOT a benchmark/default parameter here. {"DebugStreamK": [0]}, {"DebugPersistentKernelLoopForever": [False]}, {"ActivationFused": [True]}, diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index 95178b291e2d..2b953f96f5f1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -837,6 +837,16 @@ def makeValidMatrixInstructions(): # 0: use linear reduction # 1: use tree reduction "StreamKFixupTreeReduction": [0, 1], + # NOTE: StreamKMulticast (the gfx1250 StreamK DP cooperative cluster-load / + # TDM B-multicast fast path) is intentionally NOT a valid/benchmark + # parameter. It is a DERIVED-ONLY internal state key (see the ClusterBarrier + # precedent): Solution.assignProblemIndependentDerivedParameters auto-enables + # it for StreamK==3 + ClusterDim != [1,1] (the "bare StreamK cluster" + # collapse), and _validateStreamKMulticast then hard-rejects any cluster + # config that cannot satisfy its constraints. It must not be user/YAML- + # settable, so it has no entry here (checkParametersAreValid would otherwise + # accept it as a fork/constant param). It still serializes to C++ via + # Contractions.SizeMapping (streamKMulticast, read with d.get()). # Debug settings for stream-k kernels to disable parts of the kernel # Bit 0: Don't generate fixup code # Bit 1: Don't generate write to partials code diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index b069c9ac0659..82939a2e58fb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -48,6 +48,14 @@ def usesCombinedMask(self, kernel: Mapping) -> bool: masks in that case. This is the single source of truth for the combined-vs-split decision used by declare/undeclare/computeMasks. """ + # StreamK DP cooperative multicast needs the SPLIT A/B masks: A is + # loaded per-workgroup (MulticastMaskA = self bit) while B is broadcast + # across the [C,1] cluster (MulticastMaskB = all-C bits). The combined + # single-parity mask instead selects maskA on even waves / maskB on odd + # waves (for the wave-separated A-even/B-odd load split), which is wrong + # here, so force split whenever StreamKMulticast is on. Inert otherwise. + if kernel.get("StreamKMulticast", 0): + return False tdmA: bool = kernel["enableTDMA"] tdmB: bool = kernel["enableTDMB"] return tdmA and tdmB and kernel["NumWaves"] > 1 and not kernel.get("UseSubtileImpl") @@ -63,6 +71,15 @@ def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, chars; subtile only ever passes ``A``/``B`` so the strip is a no-op there). """ + # StreamK DP cooperative multicast always uses the split A/B masks + # (usesCombinedMask() returns False for it). The wave-separated dense + # apply site would otherwise resolve to the combined "MulticastMask" + # name, which is never declared on this path -> the B descriptor would + # OR an undefined SGPR. Force the split name so A binds MulticastMaskA + # (self) and B binds MulticastMaskB (broadcast). + if kernel.get("StreamKMulticast", 0): + string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc + return f"MulticastMask{string}" if waveSeparated and not subtile: return "MulticastMask" string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 7b5c93d7a346..f523ba9fc834 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -2484,6 +2484,125 @@ class StreamKTwoTileDPFirst(StreamK): requiresWorkspaceReductionStorePath = True supportsSubtileImpl = True + def streamKMulticastMaskPredicate(self, writer, kernel): + """Gate the DP B-multicast on the runtime clusterMulticastValid predicate. + + The B-broadcast mask value (MulticastMaskB = (1< share the same B), and + 2. the cluster is fully populated: clusterBase + C <= totalTiles, where + clusterBase = StreamKIdx & ~(C-1) and totalTiles = nWG0*nWG1 (so no + peer is an idle tail workgroup with no matching load). + + When either fails we rewrite MulticastMaskB to the self-only MulticastMaskA + so B is loaded per-workgroup (a normal load) -- correct for all sizes and, + critically, never leaving a masked target without a matching load (the + conservative full-cluster path chosen because the masked-idle-target HW + semantics could not be verified on silicon). Inert unless StreamKMulticast. + """ + module = Module("StreamK multicast mask predicate") + if not kernel.get("StreamKMulticast", 0): + return module + c = kernel["ClusterDim"][0] + module.addComment0("StreamKMulticast: gate B-broadcast on clusterMulticastValid") + skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) + mcInvalid = Label(writer.labels.getNameInc("SKMC_Invalid"), "") + mcEnd = Label(writer.labels.getNameInc("SKMC_End"), "") + with writer.allocTmpSgpr(3, tag="SKMulticastPredicate") as tRes: + t0 = tRes.idx + t1 = tRes.idx + 1 + t2 = tRes.idx + 2 + # cond1: nWG0 % C == 0 (C is a power of two) + module.add(SAndB32(dst=sgpr(t0), src0=sgpr("NumWorkGroups0"), src1=hex(c - 1), + comment="nWG0 %% C (C power of two)")) + module.add(SCmpEQU32(src0=sgpr(t0), src1=0, comment="nWG0 aligned to C?")) + module.add(SCBranchSCC0(labelName=mcInvalid.getLabelName(), + comment="unaligned M -> B not shared, load normally")) + # cond2: clusterBase + C <= totalTiles + sIdx = writer.acquireStreamKConstSgpr(kernel, "StreamKIdx") + if skConstsInVgprs: + module.add(VReadfirstlaneB32(dst=sgpr(sIdx), src=vgpr(writer.states.skConstVgprs["StreamKIdx"]))) + module.add(SAndB32(dst=sgpr(t2), src0=sgpr(sIdx), src1=hex((~(c - 1)) & 0xFFFFFFFF), + comment="clusterBase = StreamKIdx & ~(C-1)")) + writer.releaseStreamKConstSgpr(sIdx) + module.add(SAddU32(dst=sgpr(t2), src0=sgpr(t2), src1=hex(c), comment="clusterBase + C")) + module.add(SMulI32(dst=sgpr(t1), src0=sgpr("NumWorkGroups0"), src1=sgpr("NumWorkGroups1"), + comment="totalTiles = nWG0 * nWG1")) + module.add(SCmpLeU32(src0=sgpr(t2), src1=sgpr(t1), + comment="cluster fully populated? clusterBase+C <= totalTiles")) + module.add(SCBranchSCC1(labelName=mcEnd.getLabelName(), + comment="valid cluster -> keep B broadcast mask")) + module.add(mcInvalid) + module.add(SMovB32(dst=sgpr("MulticastMaskB"), src=sgpr("MulticastMaskA"), + comment="invalid cluster -> B loaded normally (self-only mask)")) + module.add(mcEnd) + return module + + def streamKMulticastBoundaryClear(self, writer, kernel): + """Clear the B-broadcast mask at the DP->SK boundary. + + The B multicast mask lives in the persistent ``MulticastMaskB`` SGPR and + is OR'd fresh into the B TDM descriptor by ``initTDMDescriptor`` + (``ClusterLoad.applyToDescriptor``) on *every* persistent-loop iteration. + Manipulating the descriptor word directly would be undone by the next + rebuild, so the correct lever is the mask SGPR itself. + + When a workgroup transitions out of the single DP round into SK + partial-tile work, its cluster peers no longer co-issue the identical + full-K B load, so the spatial broadcast is no longer valid. We rewrite + ``MulticastMaskB`` to the self-only ``MulticastMaskA`` bit; from here on + every descriptor rebuild ORs only the self bit -> normal per-workgroup B + load for the remaining SK iterations. Because DP is a single round first + and SK is last (no return to DP), this one-shot rewrite is sufficient. + Inert unless StreamKMulticast. + """ + module = Module("StreamK multicast DP->SK boundary clear") + if not kernel.get("StreamKMulticast", 0): + return module + module.addComment0("StreamKMulticast: clear B-broadcast mask at DP->SK boundary") + module.add(SMovB32(dst=sgpr("MulticastMaskB"), src=sgpr("MulticastMaskA"), + comment="DP->SK: drop B broadcast -> self-only (normal B load)")) + return module + + def streamKMulticastPrologueSignal(self, writer, kernel): + """Elect wave 0 to arrive at the cluster split barrier once per workgroup. + + The gfx1250 cluster-barrier pass emits a WAIT-only half + (``s_barrier_wait -3``) before the kernel's first ``tensor_load_to_lds`` + and expects a matching prologue ``s_barrier_signal -3`` to already + exist. On the StreamKMulticast path (GlobalSplitU == 0) the label that + would otherwise anchor that prologue arrive is never emitted, so this + helper supplies it: one wave per workgroup arrives (the remaining waves + branch over the signal), pairing the pass's first-load wait so the + cluster-scope signal/wait counts stay balanced. + + The arrive is unconditional on the StreamKMulticast path -- it is not + gated on the runtime clusterMulticastValid predicate -- so every cluster + peer participates in the barrier uniformly. Only the wave-0 election + gates it, matching the cluster reduce-signal idiom. Inert unless + StreamKMulticast. + """ + module = Module("StreamK multicast prologue signal") + if not kernel.get("StreamKMulticast", 0): + return module + assert writer.states.asmCaps.get("HasClusterBarrier", False), \ + "StreamKMulticast requires the HasClusterBarrier asm capability" + module.addComment0("StreamKMulticast: elect wave 0 to signal the cluster barrier (pairs first-load wait)") + skipSignal = Label(label=writer.labels.getNameInc("SKMC_SkipSignal"), comment="") + elect = writer.sgprPool.checkOut(1, "SKMulticastElect") + module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) + module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) + module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) + module.add(skipSignal) + writer.sgprPool.checkIn(elect) + return module + def preLoop(self, writer, kernel): module = Module("StreamK TwoTileDPFirst openLoop") skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) @@ -2506,6 +2625,17 @@ def preLoop(self, writer, kernel): module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), comment="Save original StreamK index")) + # StreamKMulticast: with StreamKIdx now known, gate the DP B-broadcast + # mask on the runtime clusterMulticastValid predicate (full + M-aligned + # cluster). No-op unless StreamKMulticast is enabled. + module.add(self.streamKMulticastMaskPredicate(writer, kernel)) + + # StreamKMulticast: arrive once per workgroup at the cluster split + # barrier here in the prologue, before the first tensor_load_to_lds, so + # it pairs the cluster-barrier pass's first-load wait. No-op unless + # StreamKMulticast is enabled. + module.add(self.streamKMulticastPrologueSignal(writer, kernel)) + if kernel["StreamKForceDPOnly"]: sIdx = writer.acquireStreamKConstSgpr(kernel, "StreamKIdx") sIpt = writer.acquireStreamKConstSgpr(kernel, "ItersPerTile") @@ -2811,6 +2941,14 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr(sTotalIters), comment="Make sure there's work to do")) module.add(writer.longBranchScc0(Label("KernelEnd", ""), posNeg=1)) + # DP->SK boundary: this WG just transitioned out of the DP region, so the + # spatial B-broadcast mask baked into the B descriptor is no longer valid + # (SK partial-tile peers do not co-issue the identical full-K B load). + # Clear it to a normal self-only B load for the remaining SK iterations. + # Reached only on the DP->SK switch path (the DP-continue / SK-continue + # cases branch to skUpdateDone above). No-op unless StreamKMulticast. + module.add(self.streamKMulticastBoundaryClear(writer, kernel)) + # If in SK, next iteration is sTmp+2 # Increment StreamK iteration module.add(skUpdateDone) diff --git a/projects/hipblaslt/tensilelite/Tensile/Contractions.py b/projects/hipblaslt/tensilelite/Tensile/Contractions.py index 645a5676c8a5..2bbf6ba5995f 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Contractions.py +++ b/projects/hipblaslt/tensilelite/Tensile/Contractions.py @@ -627,6 +627,7 @@ class SizeMapping: 'streamK', 'streamKForceDPOnly', 'streamKAtomic', + 'streamKMulticast', 'prefetchAcrossPersistent', 'sourceKernel', 'globalAccumulation', @@ -721,6 +722,7 @@ def convertSFCWGMListToHex(value): streamK = d['StreamK'] if 'StreamK' in d else 0, streamKForceDPOnly = d.get('StreamKForceDPOnly', 0), streamKAtomic = d['StreamKAtomic'] if 'StreamKAtomic' in d else 0, + streamKMulticast = d.get('StreamKMulticast', 0), prefetchAcrossPersistent = d.get('PrefetchAcrossPersistent', 0), magicDivAlg = d.get('MagicDivAlg', 1), sourceKernel = d['KernelLanguage'] == 'Source', diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index 1d3b7ca4ae77..5af51d241507 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -10569,7 +10569,13 @@ def postMainLoopBarrierCheckAndReset(self, kernel, rootModule): if isinstance(item, Module): modulesToScan.append(item) keptItems.append(item) - elif isinstance(item, SBarrier): + elif isinstance(item, SBarrier) and "-3" not in str(item).split("//", 1)[0]: + # Pass-2 rebuilds only workgroup-scope barriers from token-state + # transitions, so only those are cleared here. Cluster-scope split + # barriers (s_barrier_signal/wait -3), e.g. the StreamKMulticast + # prologue arrive, are placed deliberately by other components and + # carry no LDS token, so preserve them rather than dropping a half of + # a cluster handshake. removedCount += 1 continue else: diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index aaa0ad75d08f..3199e991d0db 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -231,6 +231,108 @@ def _validateStreamKForceDPOnly(state, printRejectionReason): return True +def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): + """Validate the gfx1250 StreamK DP cooperative cluster-load fast path. + + StreamKMulticast co-locates C consecutive StreamK DP workgroups in a 1-D + cluster (ClusterDim = [C, 1]); those M-adjacent tiles share the same B over + full K, so B is TDM-multicast to the cluster while A stays per-workgroup. + Solution-level requirements are rejected here at build time; the runtime + nWG0 % C "multiple-of-cluster-size" requirement is enforced by the + ClusterDimCheck predicate at selection time (not a silent fallback). + See docs/design/cluster-load-component-and-streamk-multicast.md. + + StreamKMulticast is auto-derived for StreamK=3 + ClusterDim != [1, 1] in + assignProblemIndependentDerivedParameters -- the bare index-only StreamK + cluster state was collapsed into this cooperative-load path. When such an + auto-derived config cannot meet the requirements below (e.g. TDMInst != 3), + the rejects here are the "reject an unusable cluster" behavior, not a + rejection of an explicit user opt-in. + """ + if not state.get("StreamKMulticast", 0): + return True + + # StreamKMulticast is auto-enabled by ClusterDim on SK3, but the multicast mask + # SGPRs are gated on the (derived) Multicast flag while the mask predicate / + # boundary-clear emitters are gated on StreamKMulticast. An effective + # Multicast=off (force-off 0) therefore leaves MulticastMaskA/B undeclared while + # still being referenced -> broken codegen. Reject rather than emit it. + if not state.get("Multicast", 0): + reject(state, printRejectionReason, + "StreamKMulticast (auto-enabled by ClusterDim on SK3) is incompatible " + "with Multicast=0; use Multicast=-1 or 1, or remove ClusterDim.") + return False + + # SK3 (StreamKTwoTileDPFirst) only: the DP schedule + skIndexToWG addressing + # the mask derivation relies on are SK3-specific. + if state["StreamK"] != 3: + reject(state, printRejectionReason, + "StreamKMulticast requires StreamK=3 (two-tile DP-first)") + return False + + # The atomic path skips the workspace/tile DP structure the cooperative loads + # rely on. + if state["StreamKAtomic"]: + reject(state, printRejectionReason, + "StreamKMulticast is not supported with StreamKAtomic") + return False + + # The DP cooperative multicast path currently supports single-buffered global + # prefetch only. + if state["PrefetchGlobalRead"] > 1: + reject(state, printRejectionReason, + "StreamKMulticast requires PrefetchGlobalRead <= 1") + return False + + # StreamKXCCMapping remap is bypassed under clustering and XCC=3 overflows the + # SGPR budget alongside the cluster coords; require the default (no remap). + if state["StreamKXCCMapping"] != 0: + reject(state, printRejectionReason, + "StreamKMulticast requires StreamKXCCMapping=0 (WGM/XCC remap is bypassed under clustering)") + return False + + # 1-D cluster [C, 1] with C a power of two in [2, 16]. ClusterDim[1] == 1 + # because the StreamK grid is effectively 1-D along x and consecutive-WG + # clustering is what produces M-adjacent (shared-B) DP tiles. + clusterDim = state["ClusterDim"] + c = clusterDim[0] + if clusterDim[1] != 1: + reject(state, printRejectionReason, + "StreamKMulticast requires ClusterDim = [C, 1] (got %s)" % clusterDim) + return False + if c < 2 or c > 16 or (c & (c - 1)) != 0: + reject(state, printRejectionReason, + "StreamKMulticast requires ClusterDim[0] a power of two in [2, 16] (got %d)" % c) + return False + + # gfx1250 with TDM multicast loads (multicast is a TDM feature). + isa = tuple(state["ISA"]) + if isa != (12, 5, 0): + reject(state, printRejectionReason, + "StreamKMulticast requires gfx1250 ISA (12, 5, 0)") + return False + if not isaInfoMap[isa].asmCaps.get("HasTDM", False): + reject(state, printRejectionReason, + "StreamKMulticast requires asmCap HasTDM") + return False + # The cluster-scope barrier handshake that keeps the C multicast peers in + # lockstep around each tensor_load_to_lds needs the HasClusterBarrier asm cap. + if not isaInfoMap[isa].asmCaps.get("HasClusterBarrier", False): + reject(state, printRejectionReason, + "StreamKMulticast requires asmCap HasClusterBarrier (cluster-scope " + "barrier handshake around the multicast loads)") + return False + # ClusterLoadTDM (the component that emits/applies the multicast masks) matches + # only TDMInst == 3, so TDMInst in {1, 2} would produce no multicast component + # and silently drop the masks. Require TDMInst == 3. + if state["TDMInst"] != 3: + reject(state, printRejectionReason, + "StreamKMulticast requires TDMInst == 3 (TDM multicast loads on A and B)") + return False + + return True + + # _getExpectedTypes / _expectedParamTypes / _skipTypeCheck were moved into # Tensile/Common/ValidParameters.py to keep the registry and its derived # type map co-located (and to keep the Common -> Solution import direction). @@ -1052,26 +1154,71 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, reject(state, printRejectionReason, "UseSubtileImpl=1 PrefetchAcrossPersistent not supported with DirectToVgpr MX scale tensors") state["ClusterBarrier"] = False + # StreamKMulticast is a DERIVED-ONLY internal state key (not a valid/benchmark + # parameter -- see ValidParameters.py). Seed it off here (mirroring the + # ClusterBarrier default above) so it is always present on state; the collapse + # below is the ONLY place it is turned on. + state["StreamKMulticast"] = 0 + # Collapse the bare StreamK cluster state: on StreamK=3 a non-[1,1] ClusterDim + # AUTO-ENABLES the DP cooperative B-multicast path. A StreamK cluster with no + # cooperative loads no longer exists -- "ClusterDim without cluster loads" is + # not a supported state. StreamKMulticast is derived-only (no user/YAML + # opt-in): this collapse is its sole enable site. + # + # If the resulting cooperative-load config cannot be satisfied (e.g. TDMInst!=3 + # or non-gfx1250), _validateStreamKMulticast (called later in + # assignDerivedParameters) rejects it at build time rather than silently + # degrading to a no-op cluster -- an unusable cluster is a hard reject. + # + # Scope note: the TDM B-multicast fast path is StreamK=3-only, so there is + # nothing to auto-enable for the dynamic (SK4) / hybrid (SK5) queue modes. + # SK4/SK5 with a non-[1,1] ClusterDim is rejected later in + # assignDerivedParameters (no cluster-load impl for those modes). + # ClusterDim is tested first so this (like the legacy Multicast auto branch + # below) never dereferences StreamK for the common non-clustered state, which + # some partial-state derivation call sites construct without a StreamK key. + if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3: + state["StreamKMulticast"] = 1 # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. # Default -1 reproduces the historic ClusterDim-coupled derivation, so YAML # that omits Multicast is byte-identical. mc = state.get("Multicast", -1) if mc == 1: + # Force-on requires a matching ClusterLoadTDM (TDMInst==3 on gfx1250 with + # HasTDM); without it the multicast masks are never emitted or applied, so + # reject rather than silently generate a degenerate kernel. + isa = tuple(state["ISA"]) + if state["TDMInst"] != 3 or isa != (12, 5, 0) \ + or not isaInfoMap[isa].asmCaps.get("HasTDM", False): + reject(state, printRejectionReason, + "Multicast=1 requires TDMInst=3 on gfx1250 (HasTDM); " + "no cluster-load multicast component matches otherwise") state["Multicast"] = 1 elif mc == 0: state["Multicast"] = 0 + elif state.get("StreamKMulticast", 0): + # StreamKMulticast (auto-derived above from StreamK=3 + ClusterDim) drives + # TDM B-multicast through the ClusterLoad component (its [C,1] cluster is + # spatial DP peers, not the legacy subtile coupling). The C co-resident + # peers must stay in lockstep around each multicast tensor_load_to_lds, so + # the cluster-scope barrier handshake is required (enabled below). + state["Multicast"] = 1 else: # -1 auto (legacy) # A legacy broadcast targets a fixed physical cluster position, which # Stream-K tile remapping would send to the wrong partner, so auto-multicast - # is off for ALL Stream-K. Non-Stream-K clustered paths are unchanged, so - # this reproduces develop's exact behavior for every YAML that omits Multicast. + # is off for ALL Stream-K here; the derived StreamKMulticast is the one + # exception (branch above). Non-Stream-K clustered paths are unchanged. state["Multicast"] = int(state["ClusterDim"] != [1, 1] and state["StreamK"] == 0) - # ClusterBarrier applies only to the non-Stream-K clustered (legacy/subtile) - # path; keyed off "StreamK == 0" so it excludes the StreamK cluster paths - # (which imply StreamK != 0). A forced-off Multicast also keeps it off. - if state["ClusterDim"] != [1, 1] and state["StreamK"] == 0 \ - and state["Multicast"] and state["TDMInst"] != 0 \ + # The cluster-scope barrier handshake (s_barrier_signal/wait -3, inserted by + # StinkyTofu's InsertClusterBarrierPass around each multicast tensor_load_to_lds) + # keeps the C cluster peers in lockstep on the multicast loads. It applies to + # both clustered multicast paths: the legacy/subtile clustered multicast + # (StreamK == 0) and the StreamK DP cooperative multicast (StreamKMulticast). + # A forced-off Multicast keeps it off. + if state["ClusterDim"] != [1, 1] and state["Multicast"] \ + and state["TDMInst"] != 0 \ + and (state["StreamK"] == 0 or state.get("StreamKMulticast", 0)) \ and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): state["ClusterBarrier"] = True @@ -1712,11 +1859,16 @@ def assignDerivedParameters( state["GlobalSplitUAlgorithm"] = "MultipleBuffer" # Set default Algorithm state["AdaptiveGemmGSUA"] = 0 # Disable AdaptiveGemmGSUA for Stream-K if state["ClusterDim"] != [1, 1]: - # Only SK3 (two-tile DP-first) is cluster-aware; SK4 (dynamic per-XCD - # work queues) and SK5 (hybrid) have no cluster WG-id decode support. + # WG-cluster support is StreamK==3-only. The [C,1] cluster feature is the + # SK3 DP cooperative B-multicast (StreamKMulticast, auto-derived above); + # there is no cluster-load implementation for the dynamic (SK4) or hybrid + # (SK5) work-queue modes, so a ClusterDim there would only decode a + # cluster WG-id that no feature consumes. Reject outright rather than emit + # an unusable cluster kernel. if state["StreamK"] in (4, 5): reject(state, printRejectionReason, - "Stream-K modes 4 and 5 do not support ClusterDim != [1, 1]") + "StreamK dynamic/hybrid (SK4/SK5) do not support ClusterDim " + "(cluster support is SK3-only)") # Stream-K launches a 1-D grid in X, so StreamKIdx = WorkGroup0 = cluster_x*nwg_x # + wg_x must stay a unique linear index. A Y-extent > 1 collides WorkGroup0 across # WGs that differ only in Y, so restrict clustering to the X dimension. @@ -1747,6 +1899,7 @@ def assignDerivedParameters( if not state["BufferStore"]: reject(state, printRejectionReason, "Stream-K requires BufferStore") _validateStreamKForceDPOnly(state, printRejectionReason) + _validateStreamKMulticast(state, printRejectionReason, isaInfoMap) if state["StreamKAtomic"] == 1: if state["StreamK"] == 4: reject(state, printRejectionReason, "Atomic Stream-K is not supported with dynamic work queue mode") @@ -1813,6 +1966,7 @@ def assignDerivedParameters( state["StreamKAtomic"] = 0 state["StreamKXCCMapping"] = 0 state["StreamKFixupTreeReduction"] = 0 + state["StreamKMulticast"] = 0 state["DebugStreamK"] = 0 state["PrefetchAcrossPersistent"] = 0 state["DebugPersistentKernelLoopForever"] = False diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml similarity index 98% rename from projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster.yaml rename to projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml index 38eca11ac228..08e3d69bc850 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml @@ -61,7 +61,7 @@ BenchmarkProblems: - LDSTrInst: [False] - PrefetchGlobalRead: [1] - PrefetchLocalRead: [1] - - ScheduleIterAlg: [3, 4] + - ScheduleIterAlg: [4] - SourceSwap: [False] - StoreRemapVectorWidth: [0] - GlobalSplitU: [0] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml new file mode 100644 index 000000000000..a6a229633c3d --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml @@ -0,0 +1,141 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Client-runnable StreamK DP cooperative B-multicast test for gfx1250, MX-FP4. +# +# F4 analog of the sibling core/sk_mxf8gemm_cluster_multicast.yaml (MX-FP8): same +# cluster/coverage structure, datatype-specific params differ. MX-F4 constraints +# (DataType F4, MXBlockA/B=32, TDMInst=3, MXLoadInst=TDM, +# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, DepthU=256, +# StreamKXCCMapping=0) with the DP cooperative B-multicast fast path +# (auto-derived from StreamK=3 + ClusterDim): +# +# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D spatial clusters, C in {2,4,8} +# +# StreamKMulticast is a derived-only internal state (no YAML opt-in): a StreamK=3 +# ClusterDim != [1,1] config auto-enables the DP cooperative B-multicast path. +# +# In the DP region the C consecutive workgroups of a [C,1] cluster process +# M-adjacent tiles that share the same B over full K, so B is TDM-multicast to +# the whole cluster while A is per-workgroup. The runtime clusterMulticastValid +# predicate enables the broadcast only on a fully-populated, M-aligned +# (nWG0 % C == 0) cluster; otherwise B loads normally. The host rounds the grid +# up to a multiple of C so every launched cluster is full. Each size below is +# chosen so nWG0 % C == 0 for the C it dispatches (a violation is a HARD REJECT +# via ClusterDimCheck at selection time, not a silent fallback); solution/size +# pairs whose nWG0 is not a multiple of the cluster C are simply rejected. +# +# Codegen coverage is broad: the MatrixInstruction list spans macro-tile-M in +# {32,64,128,256}, crossed with LDSTrInst {True,False}, at PrefetchGlobalRead 1. +# ScheduleIterAlg is pinned to 4 (the StinkyTofu-scheduled gfx1250 path) -- SIA3 is +# intentionally excluded so the tests exercise the real gfx1250 scheduler. The small +# sizes exercise the compact tiles at +# various nWG0; the two large [2048,...] sizes keep nWG0 % C == 0 for every tile +# (including MT-M=256) so the widest tile is covered on-cluster, and add a K-tail +# case (K % 256 != 0) for tail-loop B-multicast loads. +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch +GlobalParameters: + SyncsPerBenchmark: 0 + Architecture: gfx1250 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + ######################################## + # MX-FP4 (F4 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. + ######################################## + - + - # ProblemType: MX-F4 TN + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + Activation: True + SupportUserArgs: True + UseBias: 0 + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] # MT-M=32 + - [16, 16, 128, 1, 1, 2, 2, 2, 2] # MT-M=64 + - [16, 16, 128, 1, 1, 4, 4, 2, 2] # MT-M=128 + - [16, 16, 128, 1, 1, 8, 8, 2, 2] # MT-M=256 + - [16, 16, 128, 1, 1, 1, 1, 2, 1] # MT-M=32, WGN=1 + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [True, False] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - ClusterDim: [[2, 1], [4, 1], [8, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkForkParameters: + JoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 256] # nWG0=16 (MT-M=32) -> multicast ON (C=2,4,8) + - Exact: [256, 256, 1, 256] # nWG0=8 (MT-M=32) -> multicast ON (C=2,4,8) + - Exact: [128, 128, 1, 512] # nWG0=4 (MT-M=32) -> ON (C=2,4); K=512 + - Exact: [256, 256, 2, 256] # batched (batch=2); nWG0=8 (MT-M=32) -> ON (C=2,4,8) + - Exact: [256, 256, 1, 1920] # K-tail (K%256=128); nWG0=8 (MT-M=32) -> ON (C=2,4,8); tail-loop B-multicast loads + - Exact: [2048, 256, 1, 1024] # nWG0 in {64,32,16,8} across MT-M {32,64,128,256} -> ON (C=2,4,8) incl. widest tile + - Exact: [2048, 512, 1, 1056] # K-tail (K%256=32); nWG0 in {64,32,16,8} -> ON (C=2,4,8) incl. widest tile + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_tdm_cluster.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_tdm_cluster.yaml deleted file mode 100644 index f1b701f42855..000000000000 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_tdm_cluster.yaml +++ /dev/null @@ -1,101 +0,0 @@ -TestParameters: - marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch -GlobalParameters: - SyncsPerBenchmark: 0 - NumElementsToValidate: -1 - BoundsCheck: 0 - KernelTime: False - CpuThreads: 32 - DataInitTypeA: 3 - DataInitTypeB: 3 - DataInitTypeD: 2 - DataInitTypeMXSA: 3 - DataInitTypeMXSB: 3 - DataInitTypeAlpha: 1 - DataInitTypeBeta: 1 - PrintLevel: 2 - PrintSolutionRejectionReason: True - ForceGenerateKernel: True - -BenchmarkProblems: - ####################################### - # TN - StreamK=3, full tree-reduction (no StreamKForceDPOnly), 1-D ClusterDim - # range [2,1]/[4,1]/[8,1]. Same base sweep as sk_mxf4gemm_tdm.yaml with a 1-D - # ClusterDim range added: exercises the enableCluster guard in - # StreamKTwoTileDPFirst.preLoop on the general/split tile-splitting + fixup path - # (not just the DP-only grid-stride path) across cluster widths 2/4/8. - ####################################### - - - - # ProblemType, TN - OperationType: GEMM - DataType: F4 - DestDataType: s - ComputeDataType: s - HighPrecisionAccumulate: True - TransposeA: True - TransposeB: False - UseBeta: True - Batched: True - Activation: True - SupportUserArgs: True - UseBias: 0 - MXBlockA: 32 - MXBlockB: 32 - - - # BenchmarkProblemSizeGroup - Standard - InitialSolutionParameters: - BenchmarkCommonParameters: - - KernelLanguage: ["Assembly"] - ForkParameters: - - UseSgprForGRO: [0] - - ForceDisableShadowInit: [true] - - MatrixInstruction: - - [16, 16, 128, 1, 1, 1, 1, 2, 2] - - [16, 16, 128, 1, 1, 2, 2, 2, 2] - - [16, 16, 128, 1, 1, 4, 4, 2, 2] - - [16, 16, 128, 1, 1, 8, 8, 2, 2] - - DepthU: [256] - - ClusterLocalRead: [0] - - TransposeLDS: [-1] - - LdsPadA: [-1] - - LdsPadB: [-1] - - LdsBlockSizePerPadA: [-1] - - LdsBlockSizePerPadB: [-1] - - LdsPadMXSA: [-1] - - LdsPadMXSB: [-1] - - LdsBlockSizePerPadMXSA: [-1] - - LdsBlockSizePerPadMXSB: [-1] - - LdsPadMetadata: [0] - - LDSTrInst: [True, False] - - TDMInst: [3] - - PrefetchGlobalRead: [1, 2] - - PrefetchLocalRead: [1] - - ScheduleIterAlg: [0, 4] - - SourceSwap: [false] - - StoreRemapVectorWidth: [0] - - GlobalSplitU: [0] - - GlobalReadVectorWidthA: [-1] - - GlobalReadVectorWidthB: [-1] - - ExpandPointerSwap: [false] - - VectorWidthA: [-1] - - VectorWidthB: [-1] - - LocalReadVectorWidth: [-1] - - 1LDSBuffer: [0] - - DirectToVgprSparseMetadata: [false] - - StoreVectorWidth: [-1] - - StreamK: [3] - - StaggerU: [0] - - DirectToVgprA: [False] - - DirectToVgprB: [False] - - WavefrontSize: [32] - - WorkGroupMapping: [1] - - ClusterDim: [[2, 1], [4, 1], [8, 1]] - BenchmarkForkParameters: - JoinParameters: - BenchmarkJoinParameters: - BenchmarkFinalParameters: - - ProblemSizes: - - Exact: [2048, 256, 1, 1024] - - Exact: [2048, 512, 1, 1056] - - ActivationArgs: - - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml new file mode 100644 index 000000000000..b5e4ccfe4345 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml @@ -0,0 +1,121 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Client-runnable StreamK DP cooperative B-multicast test for gfx1250. +# +# MX-FP8 TN, StreamK=3 (TDMInst=3, MXLoadInst=TDM, +# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, StreamKXCCMapping=0) with the +# DP cooperative B-multicast fast path (auto-derived from StreamK=3 + ClusterDim): +# +# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D spatial clusters, C in {2,4,8} +# +# StreamKMulticast is a derived-only internal state (no YAML opt-in): a StreamK=3 +# ClusterDim != [1,1] config auto-enables the DP cooperative B-multicast path. +# +# In the DP region the C consecutive workgroups of a [C,1] cluster process +# M-adjacent tiles that share the same B over full K, so B is TDM-multicast to +# the whole cluster while A is per-workgroup. The runtime clusterMulticastValid +# predicate enables the broadcast only on a fully-populated, M-aligned +# (nWG0 % C == 0) cluster; otherwise B loads normally. The host rounds the grid +# up to a multiple of C so every launched cluster is full. Every size below keeps +# nWG0 % C == 0 for the C it dispatches (a violation is a HARD REJECT via +# ClusterDimCheck at selection time, not a silent fallback). +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch +GlobalParameters: + SyncsPerBenchmark: 0 + Architecture: gfx1250 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + ######################################## + # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. + ######################################## + - + - # ProblemType: MX-F8 TN + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - ClusterDim: [[2, 1], [4, 1], [8, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 256] # nWG0=16 -> multicast ON (C=2,4,8) + - Exact: [256, 256, 1, 256] # nWG0=8 -> multicast ON (C=2,4,8) + - Exact: [128, 128, 1, 512] # nWG0=4 -> ON (C=2,4); K=512 + - Exact: [256, 256, 2, 256] # batched (batch=2); nWG0=8 -> ON (C=2,4,8) + - Exact: [256, 256, 1, 1920] # K-tail (K%256=128); nWG0=8 -> ON (C=2,4,8); tail-loop B-multicast loads + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index 71229dcd381a..81019888e28a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -55,7 +55,7 @@ 'getitem_kernel_language': 'Assembly', 'iter_matches_keys': True, 'keys_is_list': True, - 'len': 337, + 'len': 338, }) # --- # name: test_solution_construction @@ -295,6 +295,7 @@ 'StreamKAtomic', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', + 'StreamKMulticast', 'StreamKXCCMapping', 'SubGroup0', 'SubGroup1', @@ -400,7 +401,7 @@ 'tailLoopOptB', ]), 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', - 'num_keys': 337, + 'num_keys': 338, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ 'DepthU': 32, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_gfx1250_char.ambr new file mode 100644 index 000000000000..5a24c0a83984 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_gfx1250_char.ambr @@ -0,0 +1,21 @@ +# serializer version: 1 +# name: test_streamk_cluster_multicast_gfx1250_golden + list([ + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg5n1_UAVQwNvJKjxzq3oOtWaugDzT0nK_8g8EArf7NFg=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgAuSSqpdzu7alrTkBrgFPyFkqyqpG9E_Ty8RQ1xyXrY8=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgchM4NognQc6wGYmCDZqnoSwpxn315nvCI1kv46p3XiY=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgiDkOmo2IiGAh_e-YoomKdpTWhfgU7cQn5f7fD7vSzWM=', + 'err': 0, + }), + ]) +# --- diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml similarity index 77% rename from projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster.yaml rename to projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml index 50a6abddc9fa..c5419f5949f7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml @@ -2,10 +2,16 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. # SPDX-License-Identifier: MIT ################################################################################ -# gfx1250 StreamK + ClusterDim=[2,1] codegen characterization config. -# Normal StreamK=3 (full tree-reduction). Exercises the enableCluster guard in -# StreamKTwoTileDPFirst.preLoop. Mirrors Tests/common/streamk/gfx1250/core/ -# sk_mxf4gemm_tdm.yaml with ClusterDim added. +# gfx1250 StreamK=3 + ClusterDim=[2,1] cooperative-load codegen characterization. +# +# StreamK=3 with ClusterDim != [1,1]. Under the +# "bare StreamK cluster" collapse this AUTO-ENABLES the DP cooperative B-multicast +# path (StreamKMulticast is a derived-only internal state; there is no bare +# index-only StreamK cluster anymore). So this config exercises BOTH the cluster +# WG-id decode / enableCluster guard in StreamKTwoTileDPFirst.preLoop AND the +# auto-derived multicast masks -- with MX-FP4 (C=2), complementing the MX-FP8 +# (C=4) sibling streamk_cluster_multicast.yaml. Mirrors +# Tests/common/streamk/gfx1250/core/sk_mxf4gemm_tdm.yaml with ClusterDim added. GlobalParameters: SyncsPerBenchmark: 0 MinimumRequiredVersion: 5.0.0 @@ -55,7 +61,7 @@ BenchmarkProblems: - LDSTrInst: [False] - PrefetchGlobalRead: [1] - PrefetchLocalRead: [1] - - ScheduleIterAlg: [3] + - ScheduleIterAlg: [4] - SourceSwap: [False] - StoreRemapVectorWidth: [0] - GlobalSplitU: [0] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml new file mode 100644 index 000000000000..265a43e3e17d --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml @@ -0,0 +1,116 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Designed StreamK + DP cooperative multicast config for gfx1250 codegen +# characterization. +# +# Mirrors the sibling _designed/gfx1250/streamk_cluster_coop_load.yaml (MX-FP8 TN, +# StreamK=3, gfx1250 MX constraints: TDMInst=3, MXLoadInst=TDM, +# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, ISA=(12,5,0), +# StreamKXCCMapping=0), but enables the DP cooperative B-multicast fast path. +# StreamKMulticast is a derived-only internal state (no YAML opt-in): StreamK=3 + +# ClusterDim != [1,1] auto-enables it, so this config only needs to set: +# +# - ClusterDim: [[4, 1]] -- 1-D spatial cluster, C=4 (power of two) +# +# Under this config StreamK.py must emit the split A/B multicast masks (A per-WG +# self bit, B broadcast across the [C,1] cluster) on the DP TDM loads, gated by +# the runtime clusterMulticastValid predicate, plus the DP->SK boundary mask +# clear. See docs/design/cluster-load-component-and-streamk-multicast.md. +# +# ClusterDim is pinned to the single C=4 here because the coupled unit test +# Tensile/Tests/unit/test_streamk_multicast.py derives its states from THIS +# config (test_accepted_baseline asserts ClusterDim==[4,1]; test_broadcast_mask +# _value asserts the maskB=(1< +# MT 32x32 (WaveTile 1x1, MIWaveGroup 2x2, 128 work-items) +# MT 64x64 (WaveTile 2x2, MIWaveGroup 2x2, 128 work-items) +# MT 128x128(WaveTile 4x4, MIWaveGroup 2x2, 128 work-items) +# MT 32x16 (WaveTile 1x1, MIWaveGroup 2x1, 64 work-items) -- WI variety +# - ClusterDim: C=4 +# => 4 x 1 = 4 fork permutations. +GlobalParameters: + SyncsPerBenchmark: 0 + MinimumRequiredVersion: 5.0.0 + NumElementsToValidate: 128 + DataInitTypeBeta: 0 + DataInitTypeAlpha: 1 + Device: 0 + +BenchmarkProblems: + ######################################## + # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. + ######################################## + - + - # ProblemType: MX-F8 TN + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - ClusterDim: [[4, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 256] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py new file mode 100644 index 000000000000..905c825d8acc --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py @@ -0,0 +1,104 @@ +################################################################################ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +################################################################################ +"""gfx1250 StreamK=3 + ClusterDim cooperative-load codegen characterization. + +A StreamK=3 kernel with ClusterDim != [1, 1] now +AUTO-ENABLES the DP cooperative B-multicast fast path (the "bare StreamK cluster" +state was collapsed; StreamKMulticast is a derived-only internal state with no +YAML opt-in). This test therefore verifies BOTH halves of that path from a config +that only sets ClusterDim: + + * the cluster WG-id decode (RemapWorkGroupDone) and the skipped ttmp9/ttmp7 + preLoop reread ("workaround" absent) under clustering -- defineAndResources + leaves the cluster-decoded rank in WorkGroup0/1/2, so the reread guard skips + it; and + * the auto-derived multicast masks: the split B-broadcast / A-self masks, the + runtime clusterMulticastValid predicate, and the DP->SK boundary clear. + +Uses MX-FP4 with ClusterDim=[2, 1] (C=2), complementing the MX-FP8 C=4 coverage +in the sibling test_streamk_cluster_multicast_gfx1250_char.py. +""" + +import os + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_cluster_coop_load.yaml", +) + + +def test_streamk_cluster_coop_load_gfx1250_emits_assembly(): + """StreamK=3 + ClusterDim=[2,1]: cluster decode present, preLoop reread + skipped, and the auto-derived DP cooperative B-multicast path emitted.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, f"Expected >=1 kernel, got {len(results)}" + assert all(err == 0 for (_b, _s, err) in results), ( + "All kernels must emit with err==0; " + + str([(b, e) for (b, _s, e) in results if e != 0]) + ) + for base, src, _err in results: + assert src and len(src.splitlines()) > 50, ( + f"Kernel {base!r} emitted suspiciously short source" + ) + assert ".amdgcn_target" in src, f"Kernel {base!r} missing .amdgcn_target" + assert "gfx1250" in src, f"Kernel {base!r} missing gfx1250 arch marker" + # Cluster WG-id decode arm. + assert "RemapWorkGroupDone" in src, ( + f"Kernel {base!r}: missing cluster WG-id decode ('RemapWorkGroupDone')" + ) + # preLoop ttmp reread must be skipped under clustering ("workaround" is + # unique to that reread block). + assert "workaround" not in src, ( + f"Kernel {base!r}: ttmp reread emitted under ClusterDim != [1, 1]" + ) + # Auto-derived DP cooperative B-multicast: B descriptor carries the + # broadcast mask, A the self-only mask (the split topology). + assert "s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in src, ( + f"Kernel {base!r}: missing auto-derived B-broadcast mask (StreamKMulticast)" + ) + assert "s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in src, ( + f"Kernel {base!r}: missing self-only mask on the A descriptor" + ) + # Runtime clusterMulticastValid predicate gates the broadcast. + assert "nWG0 aligned to C?" in src, ( + f"Kernel {base!r}: missing multicast M-alignment predicate" + ) + assert "cluster fully populated?" in src, ( + f"Kernel {base!r}: missing multicast cluster-population predicate" + ) + # DP->SK boundary clear drops the B broadcast for the SK region. + assert "DP->SK: drop B broadcast -> self-only" in src, ( + f"Kernel {base!r}: missing DP->SK boundary mask clear" + ) + # The multicast loads carry the cluster-scope barrier handshake + # (s_barrier_signal/wait -3) that keeps the C cluster peers in lockstep. + assert "s_barrier_signal -3" in src, ( + f"Kernel {base!r}: missing cluster-scope barrier signal (-3)" + ) + assert "s_barrier_wait -3" in src, ( + f"Kernel {base!r}: missing cluster-scope barrier wait (-3)" + ) + # The cluster-scope split barrier must be balanced: every arrive + # (s_barrier_signal -3) is matched by a completion (s_barrier_wait -3). + # The prologue wave-0 arrive pairs the pass's first-load wait; an + # imbalance (unpaired wait) deadlocks all cluster waves on HW. + n_signal = src.count("s_barrier_signal -3") + n_wait = src.count("s_barrier_wait -3") + assert n_signal == n_wait, ( + f"Kernel {base!r}: imbalanced cluster barrier: " + f"{n_signal} signal(-3) vs {n_wait} wait(-3)" + ) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_gfx1250_char.py deleted file mode 100644 index 36ebb6bcf094..000000000000 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_gfx1250_char.py +++ /dev/null @@ -1,59 +0,0 @@ -################################################################################ -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -################################################################################ -"""gfx1250 StreamK + ClusterDim codegen characterization. - -Checks that a StreamK kernel with ClusterDim != [1, 1] emits the cluster -WG-id decode (RemapWorkGroupDone) and skips the raw ttmp9/ttmp7 reread in the -StreamK preLoop. Under clustering, defineAndResources leaves the cluster-decoded -rank in WorkGroup0/1/2; the reread would overwrite it, so the enableCluster -guard skips it. - -Uses normal StreamK=3 with ClusterDim=[2, 1]; the guard applies equally to -StreamK modes 3, 4, and 5. -""" - -import os - -import pytest - -from config_harness import emit_kernels_from_config - -pytestmark = pytest.mark.unit - -_ARCH = "gfx1250" - -_CONFIG = os.path.join( - os.path.dirname(__file__), - "data", - "test_data", - "_designed", - "gfx1250", - "streamk_cluster.yaml", -) - - -def test_streamk_cluster_gfx1250_emits_assembly(): - """StreamK=3 + ClusterDim=[2,1]: cluster decode present, preLoop reread skipped.""" - results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) - assert len(results) >= 1, f"Expected >=1 kernel, got {len(results)}" - assert all(err == 0 for (_b, _s, err) in results), ( - "All kernels must emit with err==0; " - + str([(b, e) for (b, _s, e) in results if e != 0]) - ) - for base, src, _err in results: - assert src and len(src.splitlines()) > 50, ( - f"Kernel {base!r} emitted suspiciously short source" - ) - assert ".amdgcn_target" in src, f"Kernel {base!r} missing .amdgcn_target" - assert "gfx1250" in src, f"Kernel {base!r} missing gfx1250 arch marker" - # Cluster WG-id decode arm. - assert "RemapWorkGroupDone" in src, ( - f"Kernel {base!r}: missing cluster WG-id decode ('RemapWorkGroupDone')" - ) - # preLoop ttmp reread must be skipped under clustering ("workaround" is - # unique to that reread block). - assert "workaround" not in src, ( - f"Kernel {base!r}: ttmp reread emitted under ClusterDim != [1, 1]" - ) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py new file mode 100644 index 000000000000..6ee34d06fdc2 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py @@ -0,0 +1,111 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +"""StreamK DP cooperative B-multicast -- gfx1250 characterization (CPU-only). + +Exercises the StreamK DP cooperative-load fast path added to +``Tensile/Components/StreamK.py`` + ``Tensile/Components/ClusterLoad.py`` +(the derived StreamKMulticast on-state: ClusterDim=[C,1] +on SK3, which auto-derives StreamKMulticast=1). It drives the same config -> +Solutions -> emit path as the sibling +``test_streamk_cluster_coop_load_gfx1250_char.py``, but through the (explicitly +named) multicast designed config ``_designed/gfx1250/streamk_cluster_multicast.yaml``. + +Asserts: + * every kernel emits real gfx1250 assembly with ``err == 0``; + * the B TDM descriptor carries the split B-broadcast mask + (``MulticastMaskB`` OR'd into ``tdmBGroup1``) while the A descriptor carries + the self-only ``MulticastMaskA`` -- i.e. B is multicast, A is per-WG; + * the runtime ``clusterMulticastValid`` predicate gates the broadcast (M + alignment + fully-populated cluster, else self-only B); and + * the DP->SK boundary clear drops the B broadcast for the SK region; and + * an order-invariant ``{basename, err}`` syrupy snapshot is pinned. + +CPU-only: no GPU required. The emit harness instantiates rocisa and runs +Python+rocisa codegen without compiling or launching any GPU kernels. +""" + +import os + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_cluster_multicast.yaml", +) + + +def test_streamk_cluster_multicast_gfx1250_emits_assembly(): + """gfx1250 SK3 cluster config (StreamKMulticast auto-derived on) emits real + assembly, err==0, with the split B-broadcast mask, runtime predicate, and + DP->SK boundary clear.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + f"Expected all err==0, got: {[(b, e) for b, _s, e in results if e != 0]}" + ) + for base, src, _err in results: + assert src and len(src.splitlines()) > 50, ( + f"Kernel {base!r} emitted suspiciously short source" + ) + assert ".amdgcn_target" in src, f"Kernel {base!r} missing .amdgcn_target" + assert "gfx1250" in src, f"Kernel {base!r} missing gfx1250 target" + assert base.startswith("Cijk_"), f"Kernel {base!r} has unexpected prefix" + # Split A/B multicast: B broadcast, A self-only. + assert "s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in src, ( + f"Kernel {base!r} missing B-broadcast mask on the B descriptor" + ) + assert "s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in src, ( + f"Kernel {base!r} missing self-only mask on the A descriptor" + ) + # Runtime clusterMulticastValid predicate. + assert "nWG0 aligned to C?" in src, ( + f"Kernel {base!r} missing multicast M-alignment predicate" + ) + assert "cluster fully populated?" in src, ( + f"Kernel {base!r} missing multicast cluster-population predicate" + ) + # DP->SK boundary clear (inline comment survives canonicalization; the + # addComment0 banner does not). + assert "DP->SK: drop B broadcast -> self-only" in src, ( + f"Kernel {base!r} missing DP->SK boundary mask clear" + ) + # The multicast tensor_load_to_lds is wrapped by the cluster-scope barrier + # handshake (s_barrier_signal/wait -3) that keeps the C cluster peers in + # lockstep on the multicast loads. + assert "s_barrier_signal -3" in src, ( + f"Kernel {base!r} missing cluster-scope barrier signal (-3)" + ) + assert "s_barrier_wait -3" in src, ( + f"Kernel {base!r} missing cluster-scope barrier wait (-3)" + ) + # The cluster-scope split barrier must be balanced: every arrive + # (s_barrier_signal -3) is matched by a completion (s_barrier_wait -3). + # The prologue wave-0 arrive pairs the pass's first-load wait; an + # imbalance (unpaired wait) deadlocks all cluster waves on HW. + n_signal = src.count("s_barrier_signal -3") + n_wait = src.count("s_barrier_wait -3") + assert n_signal == n_wait, ( + f"Kernel {base!r} has imbalanced cluster barrier: " + f"{n_signal} signal(-3) vs {n_wait} wait(-3)" + ) + + +def test_streamk_cluster_multicast_gfx1250_golden(snapshot): + """Golden: order-invariant {basename, err} digest of the multicast emit.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + digest = sorted( + ({"basename": b, "err": e} for (b, _s, e) in results), + key=lambda d: d["basename"], + ) + assert digest == snapshot diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py index c0aa6119be23..c28c516f48d1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py @@ -5,7 +5,7 @@ # Unit tests for the decoupled tri-state Multicast solution parameter. # # Multicast used to be a derived-only state var, unconditionally forced on for -# ClusterDim != [1,1] (except StreamK cluster reduction). It is now an explicit +# ClusterDim != [1,1] (except Stream-K). It is now an explicit # tri-state control: # -1 = auto (legacy coupling), 0 = force off, 1 = force on. # Default -1 reproduces the historic derivation exactly. These tests pin: @@ -35,7 +35,7 @@ TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen", "data", "test_data", "_designed", "gfx1250") _XCCREMAP = os.path.join(_DESIGNED, "xccremap.yaml") -_STREAMK_CLUSTER = os.path.join(_DESIGNED, "streamk_cluster.yaml") +_STREAMK_CLUSTER = os.path.join(_DESIGNED, "streamk_cluster_coop_load.yaml") # --- registration ---------------------------------------------------------- @@ -104,14 +104,22 @@ def test_explicit_on(self, tmp_path): assert all(st["Multicast"] == 1 for st in states), ( [st["Multicast"] for st in states]) - def test_legacy_auto_streamk_reduction_off(self, tmp_path): - # -1 (omitted) + StreamKClusterReduction=1 -> Multicast stays off - # (the legacy StreamK cluster interaction is preserved). - cfg = _write_variant(tmp_path, _STREAMK_CLUSTER, "sk_legacy.yaml") + def test_streamk_cluster_auto_multicast(self, tmp_path): + # Collapse: -1 (omitted Multicast) + StreamK=3 + ClusterDim != [1,1] now + # AUTO-ENABLES the cooperative-load path. StreamKMulticast is derived to + # 1 and Multicast to True -- the bare index-only StreamK cluster state no + # longer exists. + cfg = _write_variant(tmp_path, _STREAMK_CLUSTER, "sk_auto_mc.yaml") states = _derive_states(cfg) assert states, "expected >=1 derived solution" - assert all(st["Multicast"] == 0 for st in states), ( + assert all(st["StreamKMulticast"] == 1 for st in states), ( + [st.get("StreamKMulticast") for st in states]) + assert all(st["Multicast"] == 1 for st in states), ( [st["Multicast"] for st in states]) + # The cooperative multicast now pairs the masks with the cluster-scope + # barrier handshake, so ClusterBarrier is derived on. + assert all(st["ClusterBarrier"] is True for st in states), ( + [st["ClusterBarrier"] for st in states]) if __name__ == "__main__": diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_sk45_reject.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_sk45_reject.py new file mode 100644 index 000000000000..72e98d5040d7 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_sk45_reject.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Unit test: StreamK dynamic (SK4) / hybrid (SK5) modes reject ClusterDim. +# +# WG-cluster support is StreamK==3-only (the [C,1] barrier reduction and the DP +# cooperative B-multicast are both SK3 features). There is no cluster-load / +# reduction implementation for the dynamic (SK4) or hybrid (SK5) work-queue +# modes, so Solution.assignDerivedParameters rejects StreamK in {4,5} with +# ClusterDim != [1,1] outright (rather than emitting an unusable cluster kernel +# whose decoded cluster WG-id no feature consumes). +# +# These tests drive the real config -> Solution derivation path and assert: +# * SK4/SK5 + ClusterDim != [1,1] -> 0 derived solutions (the reject fires); and +# * the SAME config with ClusterDim = [1,1] still derives solutions, proving +# the differentiator is the cluster (not some unrelated SK4/SK5 reject). +# +# Usage: +# pytest test_streamk_cluster_sk45_reject.py -v +################################################################################ + +import copy +import os +import sys + +import pytest + +pytestmark = pytest.mark.unit + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +TENSILE_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +sys.path.insert(0, TENSILE_ROOT) +sys.path.insert(0, os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen")) + +_DESIGNED = os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", + "_codegen", "data", "test_data", "_designed", "gfx1250") +# A known-good gfx1250 StreamK=3 + ClusterDim config; we only re-fork StreamK +# (and, for the control, ClusterDim) on top of it. +_BASE = os.path.join(_DESIGNED, "streamk_cluster_coop_load.yaml") + +_ARCH = "gfx1250" + + +def _write_variant(tmp_path, name, overrides): + """Copy _BASE, replacing/appending the given fork parameter values.""" + from Tensile import LibraryIO + import yaml + + cfg = copy.deepcopy(LibraryIO.read(_BASE)) + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + for key, val in overrides.items(): + replaced = False + for entry in fork: + if key in entry: + entry[key] = val + replaced = True + break + if not replaced: + fork.append({key: val}) + out = tmp_path / name + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + return str(out) + + +def _derive_states(cfg_path): + from config_harness import solutions_from_config + sols = solutions_from_config(cfg_path, arch=_ARCH, limit_solutions=8) + return [s._state if hasattr(s, "_state") else s for s in sols] + + +@pytest.mark.parametrize("sk", [4, 5]) +def test_sk45_cluster_rejected(tmp_path, sk): + """StreamK dynamic/hybrid + ClusterDim != [1,1] derives no solutions.""" + cfg = _write_variant(tmp_path, f"sk{sk}_cluster.yaml", {"StreamK": [sk]}) + assert _derive_states(cfg) == [], ( + f"StreamK={sk} with ClusterDim != [1,1] must be rejected " + "(cluster support is SK3-only)") + + +@pytest.mark.parametrize("sk", [4, 5]) +def test_sk45_without_cluster_still_valid(tmp_path, sk): + """Control: the same config with ClusterDim=[1,1] still derives solutions, + so the reject above is caused by the cluster, not an unrelated SK4/SK5 + constraint.""" + cfg = _write_variant(tmp_path, f"sk{sk}_nocluster.yaml", + {"StreamK": [sk], "ClusterDim": [[1, 1]]}) + assert _derive_states(cfg), ( + f"StreamK={sk} without ClusterDim should still derive solutions") + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py new file mode 100644 index 000000000000..d0c934c16b54 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Unit tests for the gfx1250 StreamK DP cooperative B-multicast fast path +# (StreamKMulticast). +# +# StreamKMulticast co-locates C consecutive StreamK DP workgroups in a 1-D +# workgroup cluster (ClusterDim = [C, 1]); in the DP region those C WGs process +# M-adjacent tiles that share the same B (N-block) over full K, so B is loaded +# once and TDM-multicast to the whole cluster while A stays per-WG. +# +# These tests pin (CPU-only, no GPU): +# * internalization (StreamKMulticast is derived-only: absent from +# validParameters/defaultSolution, never user-settable); +# * the validation matrix (accepted only for SK3 + ClusterDim=[C,1] pow2 2..16 +# + gfx1250 HasTDM/TDMInst + XCC=0 + not atomic; rejected else); +# * the emitted asm: DP loads carry the split B-broadcast mask +# (MulticastMaskB OR'd into the B descriptor Group1), A carries the self-only +# mask, the runtime clusterMulticastValid predicate is present, and the +# DP->SK boundary clear drops the B broadcast for the SK region. +# +# Usage: +# pytest test_streamk_multicast.py -v +################################################################################ + +import copy +import os +import sys + +import pytest + +pytestmark = pytest.mark.unit + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +TENSILE_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +sys.path.insert(0, TENSILE_ROOT) +sys.path.insert(0, os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen")) + +_DESIGNED = os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", + "_codegen", "data", "test_data", "_designed", "gfx1250") +_STREAMK_MULTICAST = os.path.join(_DESIGNED, "streamk_cluster_multicast.yaml") +_STREAMK_CLUSTER_BARE = os.path.join(_DESIGNED, "streamk_cluster_coop_load.yaml") + +_ARCH = "gfx1250" + + +# --- registration ---------------------------------------------------------- + +class TestRegistration: + def test_not_a_valid_parameter(self): + """StreamKMulticast is derived-only (ClusterBarrier precedent): it must + NOT be a user/benchmark-settable validParameter.""" + from Tensile.Common.ValidParameters import validParameters + assert "StreamKMulticast" not in validParameters + + def test_not_a_default_benchmark_parameter(self): + """Not in defaultSolution either -- it is seeded/derived on state only by + Solution.assignProblemIndependentDerivedParameters.""" + from Tensile.Common.GlobalParameters import defaultSolution + assert "StreamKMulticast" not in defaultSolution + + +# --- config -> Solution derivation helpers --------------------------------- + +def _write_variant(tmp_path, name, *, fork_overrides=None): + """Copy the designed multicast config, overriding fork param values. + + ``fork_overrides`` maps a fork parameter name to its single-element value + list; an existing fork entry is replaced, otherwise appended. + """ + from Tensile import LibraryIO + import yaml + + cfg = copy.deepcopy(LibraryIO.read(_STREAMK_MULTICAST)) + if fork_overrides: + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + for key, val in fork_overrides.items(): + replaced = False + for entry in fork: + if key in entry: + entry[key] = val + replaced = True + break + if not replaced: + fork.append({key: val}) + out = tmp_path / name + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + return str(out) + + +def _derive_states(cfg_path): + from config_harness import solutions_from_config + sols = solutions_from_config(cfg_path, arch=_ARCH, limit_solutions=8) + states = [] + for s in sols: + st = s._state if hasattr(s, "_state") else s + states.append(st) + return states + + +# --- validation matrix ----------------------------------------------------- + +class TestValidation: + def test_accepted_baseline(self, tmp_path): + """The designed SK3 cluster config (ClusterDim=[4,1]) derives valid + solutions with the internal StreamKMulticast auto-derived to 1 and + Multicast on.""" + cfg = _write_variant(tmp_path, "ok.yaml") + states = _derive_states(cfg) + assert states, "expected >=1 derived solution for the valid config" + for st in states: + assert st["StreamKMulticast"] == 1 + assert st["Multicast"] == 1, st["Multicast"] + assert st["ClusterDim"] == [4, 1] + # The cooperative multicast pairs the B-broadcast masks with the + # cluster-scope barrier handshake, so ClusterBarrier is derived on. + assert st["ClusterBarrier"] is True, st.get("ClusterBarrier") + + def test_auto_enable_from_bare_cluster(self, tmp_path): + """Collapse: a StreamK=3 + ClusterDim config that does NOT explicitly set + StreamKMulticast now auto-derives the cooperative-load path + (StreamKMulticast=1, Multicast=True). The bare index-only StreamK cluster + state has been removed.""" + from Tensile import LibraryIO + import yaml + cfg = copy.deepcopy(LibraryIO.read(_STREAMK_CLUSTER_BARE)) + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + # Guard the premise: the base bare-cluster config is opt-in-free. + assert not any("StreamKMulticast" in e for e in fork), \ + "base config unexpectedly sets StreamKMulticast" + out = tmp_path / "bare_cluster.yaml" + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + states = _derive_states(str(out)) + assert states, "expected the bare SK3 cluster config to derive solutions" + for st in states: + assert st["StreamKMulticast"] == 1, st.get("StreamKMulticast") + assert st["Multicast"] == 1, st["Multicast"] + + def test_reject_multicast_force_off(self, tmp_path): + """StreamKMulticast auto-enabled by ClusterDim on SK3 is incompatible with + an explicit Multicast=0 (force off): the mask SGPRs are gated on Multicast + while the predicate/boundary-clear emitters are gated on StreamKMulticast, + so Multicast=0 would reference undeclared MulticastMaskA/B. Reject.""" + cfg = _write_variant(tmp_path, "mc_off.yaml", + fork_overrides={"Multicast": [0]}) + assert _derive_states(cfg) == [] + + def test_control_multicast_auto_enabled(self, tmp_path): + """Control for test_reject_multicast_force_off: the same SK3 [C,1] cluster + config with Multicast=-1 (auto, the default) auto-enables StreamKMulticast + and derives valid solutions.""" + cfg = _write_variant(tmp_path, "mc_auto.yaml", + fork_overrides={"Multicast": [-1]}) + states = _derive_states(cfg) + assert states, "expected the Multicast=-1 control config to derive solutions" + for st in states: + assert st["StreamKMulticast"] == 1 + assert st["Multicast"] == 1, st["Multicast"] + + def test_reject_atomic(self, tmp_path): + cfg = _write_variant(tmp_path, "atomic.yaml", + fork_overrides={"StreamKAtomic": [1]}) + assert _derive_states(cfg) == [] + + def test_reject_pgr_gt1(self, tmp_path): + """The DP cooperative multicast path currently supports single-buffered + global prefetch only, so PrefetchGlobalRead > 1 is rejected while + PrefetchGlobalRead <= 1 stays accepted.""" + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + + class _Info: + asmCaps = {"HasTDM": True, "HasClusterBarrier": True} + isaInfoMap = {(12, 5, 0): _Info()} + + def _state(pgr): + return { + "StreamKMulticast": 1, + "Multicast": 1, + "StreamK": 3, + "StreamKAtomic": 0, + "StreamKXCCMapping": 0, + "ClusterDim": [4, 1], + "ISA": [12, 5, 0], + "TDMInst": 3, + "PrefetchGlobalRead": pgr, + } + + st = _state(2) + assert _validateStreamKMulticast(st, False, isaInfoMap) is False + assert st.get("Valid") is False + assert _validateStreamKMulticast(_state(1), False, isaInfoMap) is True + + cfg = _write_variant(tmp_path, "pgr2.yaml", + fork_overrides={"PrefetchGlobalRead": [2]}) + assert _derive_states(cfg) == [] + + def test_xcc_mapping_forced_to_zero(self, tmp_path): + """StreamKXCCMapping is coerced to 0 (not rejected) under StreamK+ClusterDim. + + The general Stream-K + ClusterDim reconciliation force-sets + StreamKXCCMapping = 0 (the WGM/XCC WorkGroup0 remap has no cluster + awareness) *before* _validateStreamKMulticast runs. That coerced value is + exactly what StreamKMulticast requires (XCC == 0), so the solution is + accepted with the remap disabled rather than rejected. Our + _validateStreamKMulticast XCC check remains as redundant safety.""" + cfg = _write_variant(tmp_path, "xcc.yaml", + fork_overrides={"StreamKXCCMapping": [3]}) + states = _derive_states(cfg) + assert states, "expected the XCC=3 config to be accepted with XCC coerced to 0" + for st in states: + assert st["StreamKMulticast"] == 1 + assert st["StreamKXCCMapping"] == 0, st["StreamKXCCMapping"] + + def test_reject_non_1d_cluster(self, tmp_path): + # ClusterDim = [2, 2] is not the [C, 1] spatial DP cluster. + cfg = _write_variant(tmp_path, "cd22.yaml", + fork_overrides={"ClusterDim": [[2, 2]]}) + assert _derive_states(cfg) == [] + + def test_reject_non_pow2_cluster(self, tmp_path): + cfg = _write_variant(tmp_path, "cd3.yaml", + fork_overrides={"ClusterDim": [[3, 1]]}) + assert _derive_states(cfg) == [] + + # NB: C > 16 is not an expressible ClusterDim (validParameters caps + # ClusterDim x at 16), so the "> 16" branch of the validator is defensive + # and unreachable through valid params -- no test drives it here. + + +class TestTDMInstValidation: + """The tightened TDMInst check: StreamKMulticast requires TDMInst == 3 (the + only TDMInst a ClusterLoadTDM component matches), so TDMInst in {1,2} is + rejected even on gfx1250 HasTDM -- otherwise the masks would silently drop.""" + + @staticmethod + def _state(tdminst, pgr=1): + return { + "StreamKMulticast": 1, + # StreamKMulticast on always co-derives Multicast on (real invariant). + "Multicast": 1, + "StreamK": 3, + "StreamKAtomic": 0, + "StreamKXCCMapping": 0, + "ClusterDim": [4, 1], + "ISA": [12, 5, 0], + "TDMInst": tdminst, + "PrefetchGlobalRead": pgr, + } + + @staticmethod + def _isa_map(has_tdm=True): + class _Info: + asmCaps = {"HasTDM": has_tdm, "HasClusterBarrier": True} + return {(12, 5, 0): _Info()} + + @pytest.mark.parametrize("tdminst", [1, 2]) + def test_reject_non_tdm3(self, tdminst): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + st = self._state(tdminst) + assert _validateStreamKMulticast(st, False, self._isa_map()) is False + assert st.get("Valid") is False + + def test_accept_tdm3(self): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + st = self._state(3) + assert _validateStreamKMulticast(st, False, self._isa_map()) is True + + +# --- emitted assembly ------------------------------------------------------ + +class TestEmit: + def _emit(self, cfg=_STREAMK_MULTICAST): + from config_harness import emit_kernels_from_config + return emit_kernels_from_config(cfg, limit=8, arch=_ARCH) + + def test_emits_assembly(self): + results = self._emit() + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + [(b, e) for b, _s, e in results if e != 0]) + for base, src, _err in results: + assert ".amdgcn_target" in src and "gfx1250" in src + assert base.startswith("Cijk_") + + def test_split_mask_bindings(self): + """A descriptors bind MulticastMaskA (self), B descriptors bind + MulticastMaskB (broadcast) -- the split topology, not the combined + MulticastMask (which would be an undeclared SGPR on this path).""" + _b, src, _e = self._emit()[0] + assert "s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in src, \ + "B descriptor must OR the B-broadcast mask (MulticastMaskB)" + assert "s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in src, \ + "A descriptor must OR the self-only mask (MulticastMaskA)" + # The combined single-parity name must not appear as a bare SGPR: only + # the split MaskA/MaskB (and optional Metadata) forms are declared. + for line in src.splitlines(): + if "sgprMulticastMask," in line: + pytest.fail("combined MulticastMask SGPR leaked into split path: " + + line.strip()) + + def test_broadcast_mask_value(self): + """maskB = (1< B loaded normally" in src, \ + "predicate fallback (self-only B) missing" + + def test_cluster_barrier_handshake(self): + """The multicast B-broadcast masks are paired with the cluster-scope + barrier handshake (s_barrier_signal/wait -3) around the multicast + tensor_load_to_lds, so ClusterBarrier is on and both barrier opcodes are + emitted.""" + _b, src, _e = self._emit()[0] + assert "s_barrier_signal -3" in src, \ + "missing cluster-scope barrier signal (-3) on the multicast path" + assert "s_barrier_wait -3" in src, \ + "missing cluster-scope barrier wait (-3) on the multicast path" + + def test_dp_to_sk_boundary_clear(self): + """At the DP->SK boundary the B broadcast is dropped to self-only so SK + partial-tile loads are normal per-WG loads.""" + _b, src, _e = self._emit()[0] + assert "DP->SK: drop B broadcast -> self-only" in src, \ + "boundary-clear rewrite of MulticastMaskB missing" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index e85995673b0f..0eaf7e959d08 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -143,6 +143,7 @@ namespace TensileLite int streamK = 0; int streamKForceDPOnly = 0; int streamKAtomic = 0; + int streamKMulticast = 0; int prefetchAcrossPersistent = 0; int persistentKernel = 0; bool persistentKernelAlongBatch = false; diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp index 9aaba8d9bd2f..ccec6cb76835 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp @@ -106,6 +106,7 @@ namespace TensileLite iot::mapOptional(io, "streamK", s.streamK); iot::mapOptional(io, "streamKForceDPOnly", s.streamKForceDPOnly); iot::mapOptional(io, "streamKAtomic", s.streamKAtomic); + iot::mapOptional(io, "streamKMulticast", s.streamKMulticast); iot::mapOptional(io, "prefetchAcrossPersistent", s.prefetchAcrossPersistent); iot::mapOptional(io, "persistentKernel", s.persistentKernel); iot::mapOptional(io, "persistentKernelAlongBatch", s.persistentKernelAlongBatch); diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 38f8b1df1100..7fa9f504c524 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -3190,6 +3190,20 @@ namespace TensileLite throw std::runtime_error("hipblasLT Error: Cannot use Parallel reduction with " "StreamK kernel with splitting factor < 2\n"); } + + // Keep the StreamK cluster launch grid a multiple of the cluster size C + // even if a workspace/DP fallback above changed sk.grid, so the clustered + // launch (gridDimX % C == 0) stays valid. In the common case sk.grid is + // already ceil(tiles/C)*C (multicast) -- a multiple of C -- so this is a + // no-op; when a fallback fired (e.g. sk.grid = tiles), per-tile + // correctness on any partially-filled cluster is handled by the kernel's + // clusterMulticastValid runtime guard + the normal (non-cooperative) load + // fallback. + if(sizeMapping.streamKMulticast && sizeMapping.clusterDim.x > 1) + { + size_t c = sizeMapping.clusterDim.x; + sk.grid = ((sk.grid + c - 1) / c) * c; + } } GSUSettings gsuSettings; @@ -4064,6 +4078,21 @@ namespace TensileLite } } + // StreamKMulticast (gfx1250): DP cooperative B-multicast. The multicast + // cluster is SPATIAL: the C peers of a cluster process C DISTINCT, + // M-adjacent tiles and share the B (N-block) tile. v1 is single-round, so + // the launched grid is `tiles` rounded UP to a multiple of C = clusterDim.x. + // Rounding up keeps every launched HW cluster full while giving each WG one + // tile in the single DP round; a trailing partial cluster (tiles % C != 0) + // has idle-but-present tail WGs whose cluster is disabled by the kernel's + // clusterMulticastValid runtime predicate (so a masked target is never left + // without a matching load). See docs/design/cluster-load-component-and-streamk-multicast.md. + if(self.sizeMapping.streamKMulticast && self.sizeMapping.clusterDim.x > 1) + { + size_t c = self.sizeMapping.clusterDim.x; + skGrid = ((tiles + c - 1) / c) * c; + } + return skGrid; } } // namespace From 6cf8d522caf8c852b254f0c861b1b782dc68e27c Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 20 Jul 2026 23:08:55 +0000 Subject: [PATCH 03/24] feat(tensilelite): add StreamK WG-cluster partial reduction for gfx1250 StreamK GEMM tiles whose partial results are reduced across workgroups (WGs) synchronize that reduction with a cross-CU global-workspace flag spin-wait. On gfx1250 the WGs that reduce one tile can instead be co-located into a hardware workgroup cluster and synchronized with an intra-cluster split barrier, removing the spin-wait from the fast path. Adds an opt-in solution parameter StreamKClusterReduction. When enabled, a StreamK tile's fixup peers are placed in a single 1-D workgroup cluster (ClusterDim = [C, 1]); the HW WG-id remap (WorkGroup0 = cluster_x*C + wg_x) makes each cluster own the contiguous StreamK index range [c*C, c*C+C), i.e. exactly one tile's peers. - Kernel (Components/StreamK.py): clusterReduceSignal / clusterReduceWait / clusterReduceIntraCheck helpers emit the wave-0-elected s_barrier_signal / s_barrier_wait handshake in the fixup epilogue. A uniform intra-cluster predicate selects the barrier fast path or the retained global-flag path so a cluster never mixes the two. The WG-id reread is skipped under clustering so StreamKIdx stays the global index. - Host (src/ContractionSolution.cpp, ContractionSolution.hpp + serialization): getSKGridImpl reduction override rounds the SK grid to C*tiles (fixed even split, SKItersPerWG = itersPerTile/C) and keeps the launch grid a multiple of C; the two-tile cluster-reduction kernarg block is emitted only when sk.grid == C*tiles; the field is threaded through SizeMapping / Contractions. - Predicate (ContractionProblemPredicates.hpp + serialization + Contractions.py): ClusterReductionIterCheck rejects at selection time any problem where itersPerTile is not a multiple of C (the split barrier would otherwise over-signal); the global-flag reduction is retained as the runtime fallback. - Solution.py: _validateStreamKClusterReduction gates the opt-in (SK3, non-atomic, non-DP-only, [C,1] power-of-two cluster, gfx1250 with HasClusterBarrier, TDMInst != 0). Multicast and the mainloop ClusterBarrier are kept off for a StreamK reduction cluster: its peers iterate different K-splits and never march the mainloop in lockstep. Multicast and cluster reduction are mutually exclusive: the StreamKMulticast auto-derivation is gated on NOT StreamKClusterReduction, and both validators reject the simultaneous combination with a neutral message. This stacks on the reusable ClusterLoad component + tri-state Multicast and the StreamK cooperative multicast; the mutual exclusion is intended to be unified by a later factored combined-cluster mode. Validation: CPU asm-string unit tests (test_streamk_cluster_reduction.py), snapshot codegen characterization (test_streamk_cluster_reduction_gfx1250_char), the ClusterReductionIterCheck predicate characterization and C++ gtests, plus opt-in-off client GEMM YAMLs. The feature is off by default and strictly opt-in. --- docs/design/streamk-wg-clusters.md | 507 ++++++++++++++++++ .../Tensile/Common/GlobalParameters.py | 1 + .../Tensile/Common/ValidParameters.py | 12 + .../tensilelite/Tensile/Components/StreamK.py | 150 ++++++ .../tensilelite/Tensile/Contractions.py | 14 + .../Tensile/SolutionStructs/Solution.py | 107 +++- .../core/sk_mxf4gemm_cluster_multicast.yaml | 5 +- .../core/sk_mxf4gemm_cluster_reduction.yaml | 124 +++++ .../core/sk_mxf8gemm_cluster_multicast.yaml | 5 +- .../core/sk_mxf8gemm_cluster_reduction.yaml | 117 ++++ .../Contractions/test_contractions_char.py | 41 ++ .../test_solution_class_char.ambr | 9 +- .../__snapshots__/test_builders_char.ambr | 9 + ...treamk_cluster_reduction_gfx1250_char.ambr | 37 ++ .../gfx1250/streamk_cluster_reduction.yaml | 112 ++++ ..._streamk_cluster_reduction_gfx1250_char.py | 82 +++ .../Tensile/Tests/unit/gpu_test_helpers.py | 21 + .../unit/test_streamk_cluster_reduction.py | 246 +++++++++ .../test_streamk_cluster_reduction_gpu.py | 309 +++++++++++ .../Tests/unit/test_streamk_multicast.py | 42 ++ .../Tensile/ContractionProblemPredicates.hpp | 64 +++ .../include/Tensile/ContractionSolution.hpp | 1 + .../Serialization/ContractionPredicates.hpp | 8 +- .../Serialization/ContractionSolution.hpp | 1 + .../tensilelite/src/ContractionSolution.cpp | 57 +- .../tensilelite/tests/Predicates_test.cpp | 108 ++++ 26 files changed, 2174 insertions(+), 15 deletions(-) create mode 100644 docs/design/streamk-wg-clusters.md create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction_gpu.py diff --git a/docs/design/streamk-wg-clusters.md b/docs/design/streamk-wg-clusters.md new file mode 100644 index 000000000000..9c6715ddde2c --- /dev/null +++ b/docs/design/streamk-wg-clusters.md @@ -0,0 +1,507 @@ + + +# Design: WG Clusters for StreamK GEMM (gfx1250) + +**Target arch:** gfx1250 (ISA `(12,5,0)`, wave32). +**Scope:** Use gfx1250 workgroup *clusters* to accelerate the StreamK partial-tile +reduction by replacing the cross-CU global-flag spin-wait with an intra-cluster +split barrier, while keeping a correct fallback for the general +(peers-span-multiple-clusters / DP / atomic) cases. The MVP implemented here is +StreamK variant 3 (two-tile), barrier-only, linear-reduction fast path, with a +fixed even split (one tile per cluster); the global-flag reduction is retained +as a runtime/compile fallback. Source-line anchors below reference the code as +implemented. + +--- + +## 1. Background + +### 1.1 Cluster hardware + existing (non-StreamK) support + +- **Split barrier objects** are addressed by negative id: `-1` = workgroup, + `-3` = cluster. The rocisa `SBarrier(separate, wait, clusterBarrier, comment)` + emitter lowers to id `-3` when `clusterBarrier=True`. Only one wave per WG + signals/arrives; any wave in the cluster may wait. +- **Solution derivation** (`Tensile/SolutionStructs/Solution.py:975-981`): + + ```975:981:projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py + state["Multicast"] = False + state["ClusterBarrier"] = False + if state["ClusterDim"] != [1, 1]: + state["Multicast"] = True + # ClusterBarrier emits SCmp/branch on sgpr("WaveIdx"), which is only allocated when TDM is enabled. + if state["TDMInst"] != 0 and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): + state["ClusterBarrier"] = True + ``` + + Note: `ClusterDim != [1,1]` unconditionally forces `Multicast=True`. This + coupling must be broken for a barrier-only StreamK path (see §5). +- **Constraints:** `maxWGsInCluster = 16`; `validClusterDimensions` is all + `[i,j]` with `i*j <= 16` (`Common/ValidParameters.py:74-80`, exported at + `:1107`). For `PrefetchGL2`/non-subtile, `ClusterDim` components must be + powers of two and `!= [1,1]` (`Solution.py:5296-5302`). +- **Host launch is already cluster-aware:** `src/hip/HipSolutionAdapter.cpp:573-604` + uses `hipDrvLaunchKernelEx` + `hipLaunchAttributeClusterDimension`, gated on + `HIP_HAS_CLUSTER_LAUNCH`, with `kernel.clusterDim` from + `ContractionSolution.hpp` / `Contractions.py`. The comment at `:586-589` + states explicitly: *"The grid dimension should be a multiple of cluster + size."* The launch sets `attribute[0].val.clusterDim.z = 1` — clusters are + effectively 2-D `[x,y]`. +- **Kernel-side WG-id derivation under clustering** + (`KernelWriterAssembly.py:2581-2644`). When `WorkGroupIdFromTTM` and + `enableCluster = (ClusterDim[0]*ClusterDim[1]) != 1`, and `cluster_id != 0`, + the WG ids are rebuilt from the cluster HW regs: + + ```2628:2632:projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py + moduleRegInit.add(SMulI32(dst=sgpr("WorkGroup0"), src0="ttmp9", src1=sgpr(sTmp+3),\ + comment="cluster_x * nwg_x")) + moduleRegInit.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), \ + src1=sgpr(sTmp+1), \ + comment="WorkGroup0 = (cluster_x * nwg_x) + wg_x")) + ``` + + i.e. **`WorkGroup0 = cluster_x * nwg_x + wg_x`**, where `nwg_x = ClusterDim[0]` + is the number of WGs per cluster along x, `cluster_x` is the cluster index, + `wg_x` is the WG's position within the cluster. **Key property: WGs of one + cluster occupy a *contiguous* `WorkGroup0` range `[cluster_x*C, cluster_x*C + C)` + with `C = ClusterDim[0]`.** (`ttmp9` here is the raw `wg_x`.) +- **WGM/XCC remap is bypassed under clustering:** + `Components/WorkGroupMappingAlgos.py:99-101` short-circuits `wgmXCC` when + `enableCluster`. + +### 1.2 StreamK internals (verified) + +- Variants dispatched by `kernel["StreamK"]` in + `Tensile/Components/StreamK.py`: `0` Off; `3` `StreamKTwoTileDPFirst` + (`:2479`); `4` `StreamKDynamic` (`:2896`, atomic work queue); `5` + `StreamKHybrid` (`:3301`). Table at `:4087-4089`. Orthogonal flags: + `StreamKAtomic`, `StreamKForceDPOnly`, `StreamKFixupTreeReduction`. +- **StreamKIdx == WorkGroup0** (`StreamK.py:2504`): + + ```2504:2505:projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py + module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), + comment="Save original StreamK index")) + ``` + + **Gotcha:** the SK3 `preLoop` re-reads a *raw* WG id from `ttmp9` + immediately before this, overwriting the cluster-remapped value: + + ```2495:2498:projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py + if writer.states.archCaps["WorkGroupIdFromTTM"]: + module.add(SMovB32(dst=sgpr("WorkGroup0"), src="ttmp9", comment="workaround")) + module.add(SAndB32(dst=sgpr("WorkGroup1"), src0=hex(0xFFFF), src1="ttmp7", comment="workaround")) + module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7", comment="workaround")) + ``` + + Under clustering `ttmp9 == wg_x` (position *within* the cluster), **not** the + global linear index. This workaround must be made cluster-aware (§2). +- `StreamKIter = StreamKIdx * SKItersPerWG (+ extras)` (tree-reduction init at + `:2621-2626`; parallel/DP paths at `:2513`, `:2569-2587`). So **consecutive + `StreamKIdx` map to consecutive global-iteration windows.** +- Tile mapping: `skTileIndex` (`:439`) magic-divides the global iter into a tile + index and derives `StreamKLocalStart`/`StreamKLocalEnd`; `skIndexToWG` (`:487`) + maps tile → `WorkGroup0/1/2`. **Owner** of a tile is the WG with + `StreamKLocalStart == 0` (it runs the reduction + final store). +- **Reduction (producer/consumer over global workspace + flag)** lives in the + epilogue, `storeBranchesCommon` (`:744`), in two topologies: + - *linear* (`:911-1035`): owner sets `sCtaIdx = StreamKIdx+1` (`:941`) and + walks **consecutive** peer indices `+1,+2,…`, per-peer spin-waiting on the + global flag: + + ```961:965:projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py + module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(sCtaIdx), shiftHex=log2(4), comment="flag offset based on CTA index")) + module.add(memOrder.readFlag(writer, dst=tmpSgpr+2, soffset=sgpr(tmpSgpr))) + if kernel["DebugStreamK"] & 2 == 0: + module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=1, comment="check if ready")) + module.add(SCBranchSCC0(labelName=skFixupLabel.getLabelName(), comment="if flag not set, wait and check again")) + ``` + + then `acquireFence`, a **workgroup** `SBarrier` (`:968`), wave-0 flag reset + (`:970-979`), and `fixupStep` (`:1010`, defined `:1680`) which loads peer + partials from the workspace and `VAddF32`-accumulates. + - *tree* (`StreamKFixupTreeReduction=1`, `:755-910`): same handshake with a + log-tree peer schedule (`sIdxOffset *= 2`). + - Non-owners write their partial to the global workspace slot via + `computeWorkspaceSrd` (`:1108`, slot `= AddressWS + StreamKIdx*(MT0*MT1*bpeCinternal)`) + and raise their flag via `setFlagValue` (`:1414`). +- **Memory ordering** (`StreamK.py:112-249`). gfx1250 uses + `StreamKMemoryOrderingDevScopeFences` (cap `HasInvWbDevFences`): release = + `s_waitcnt` + `global_wb scope:SCOPE_DEV`; acquire = `global_inv + scope:SCOPE_DEV`; the flag is read via **VMEM** (`readFlag`, `:236-246`) with + `s_wait_xcnt 0` before volatile VMEM (`preVolatileVmem`, `:144-153`, + `RequiresXCntForVolatileVMEM`). +- **Flags buffer:** `library/.../hipblaslt.cpp` allocates + zeros `d_Synchronizer` + and passes it as `AddressFlags`; `AddressFlags == 0` is the sentinel meaning + "parallel / post-kernel reduction" (checked at `StreamK.py:579`, `:2533`). +- **Host grid** forces `{sk.grid,1,1}` for StreamK + (`ContractionSolution.cpp:1758-1763`); the `enableCluster` branch immediately + after keeps y/z unflattened for cluster kernels: + + ```1758:1774:projects/hipblaslt/tensilelite/src/ContractionSolution.cpp + if(sizeMapping.streamK != 0) + { + rv.numWorkGroups.x = sk.grid; + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; + } + + bool enableCluster = (sizeMapping.clusterDim.x > 1 || sizeMapping.clusterDim.y > 1); + if(!enableCluster) + { + if(internalArgsSupport.version >= 1) + { + rv.numWorkGroups.x *= (rv.numWorkGroups.y * rv.numWorkGroups.z); + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; + } + } + ``` + + `sk.grid` is chosen by `getSKGridImpl` (`:3895-4045`); the SK3 per-WG + accounting (`skTiles`, `SKItersPerWG`, `extraIters`) is built at `:856-906` + and `:945-973`. There is **no** rounding of `sk.grid` to a cluster multiple + today. + +### 1.3 Cluster-barrier codegen paths that already exist + +- **Subtile Python path** `Tensile/Components/Subtile/ClusterBarrier.py`: + `subtileClusterBarrierSignal` (wave-0 election: `s_cmp_eq_u32 WaveIdx==0`, + `s_cbranch_scc0`, `SBarrier(True,False,True)` = `s_barrier_signal -3`), + `subtileClusterBarrierWait` (`SBarrier(True,True,True)` = `s_barrier_wait -3`), + and `insertClusterBarrier` which **splices around the mainloop's workgroup + barrier** and asserts `asmCaps["HasClusterBarrier"]`. +- **stinkytofu C++ pass** `shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp`, + run from `Gfx1250Backend.cpp` when `moduleOptions.ClusterBarrier`. Its anchors + are all **mainloop / load / tail-loop** points: `tensor_load_to_lds` (Rule 4), + `label_GSU_1` (Rule 1), `label_openLoopL` (Rule 3, currently disabled), and the + `/* Tail Loop */` TEXTBLOCK marker (Rule 5). **None of these fire in the + StreamK epilogue/store reduction path** — verified by reading the pass in full. + +--- + +## 2. WG-id / indexing reconciliation + +### 2.1 The core geometric fact + +With `ClusterDim = [C, 1]` (1-D cluster along x), the kernel-side remap gives +`WorkGroup0 = cluster_x * C + wg_x`. StreamK uses `StreamKIdx = WorkGroup0` and +`StreamKIter = StreamKIdx * SKItersPerWG (+extra)`, and the reduction walks +**consecutive** `StreamKIdx` (`sCtaIdx = StreamKIdx+1, +2, …`). Therefore: + +> **Cluster `c` owns exactly the contiguous StreamK index range +> `[c*C, c*C + C)`, which is also a contiguous block of global iteration +> windows.** A tile whose peer set is a contiguous run of `p` indices starting at +> its owner `k` is fully intra-cluster **iff** `k` and `k+p-1` fall in the same +> `[c*C, c*C+C)` block. + +This is the property the whole design leans on: no reshuffling is needed, the HW +remap already clusters consecutive StreamK indices. + +### 2.2 Chosen mapping + +- **`ClusterDim = [C, 1]`**, `C` a power of two, `2 <= C <= 16`, `ClusterDim[1] == 1`. + A 1-D cluster is required because the StreamK grid is 1-D `{skGrid,1,1}`; a + `ClusterDim[1] > 1` would demand `gridDimY % ClusterDim[1] == 0` while + `gridDimY == 1` (launch at `HipSolutionAdapter.cpp:589-600`), which cannot be + satisfied. +- **Fix the `ttmp9` workaround** (`StreamK.py:2495-2498`): under + `enableCluster`, `StreamKIdx` must be the *global* linear index + `cluster_x*C + wg_x`, not raw `ttmp9 (= wg_x)`. Two options: + 1. *(preferred)* When `enableCluster`, **skip** the `ttmp9` workaround and use + the already cluster-remapped `WorkGroup0` produced in + `KernelWriterAssembly.py:2588-2644`. + 2. Recompute `cluster_x` from `HW_REG_IB_STS2[6:4]` locally and form + `cluster_x*C + wg_x`. More SGPR traffic; only needed if the remapped + `WorkGroup0` is not live at `preLoop`. +- **Derive cluster-local coordinates** for barrier owner election and peer + bookkeeping (cheap, from values already computed): + - `wg_in_cluster = StreamKIdx & (C-1)` (C power of two). + - `cluster_base = StreamKIdx & ~(C-1)` (first StreamKIdx of this cluster). + - `cluster_last = cluster_base + C - 1`. +- **Grid sizing (host).** `getSKGridImpl` (`ContractionSolution.cpp:3895-4045`) + must round `sk.grid` **up to a multiple of `C`** when the StreamK-cluster path + is active, and that rounded value must flow into both `rv.numWorkGroups.x` + (`:1760`) and the `skGrid`/`SKItersPerWG`/`skTiles` kernel args (`:856-906`, + `:945-973`). The tree-fixup `< 2^24 / 2^16` guards at `:4031-4043` still apply + to the rounded grid. `HipSolutionAdapter.cpp:589` already passes + `numWorkGroups.x` as `gridDimX`, so a multiple-of-`C` grid satisfies the HW + requirement with no launch change. + +### 2.3 One-tile-per-cluster alignment (MVP partition) + +To make the intra-cluster fast path the *guaranteed* common case (and to sidestep +the deadlock hazard of §6), the MVP additionally **aligns each StreamK tile's +peer group to one cluster**: + +- Choose the per-WG split so **peers-per-tile `== C`** and **tiles align to + cluster boundaries**: `SKItersPerWG * C == itersPerTile` and + `skGrid == C * skTiles` (after rounding). Then cluster `c` ⟷ StreamK tile `c`, + and every WG in a cluster is a peer of the same tile. +- This is a constrained sub-mode of SK3 (fixed, even split — closest to the + existing "parallel reduction" `skSplit` path, `StreamK.py:2536-2591`, + `ContractionSolution.cpp:933-942`, where `skSplit = grid/tiles` and + `SKItersPerWG = itersPerTile/skSplit`). The `skSplit` model already produces a + **fixed, contiguous** peer group of size `skSplit` per tile — setting + `skSplit == C` makes each tile's peers exactly one cluster. +- The residual imbalance cases (last cluster partially filled, `extraIters` + giving big/little WGs) are handled by keeping the global-flag path as a runtime + fallback (§3.4). + +--- + +## 3. Reduction redesign + +### 3.1 Which handshake is replaced + +The expensive operation is the **cross-CU global-flag spin-wait** in +`storeBranchesCommon` (`:961-965` linear, `:872-877` tree) plus the device-scope +release/acquire fences and the wave-0 flag reset (`:970-979`). When the tile's +peers co-reside in a cluster, all of that is replaced by **one cluster split +barrier**: + +- **Non-owner (peer WG)** — after computing its partial: + 1. write partial to its global workspace slot (`computeWorkspaceSrd`, + `writePartials`) — *unchanged for v1*; + 2. `releaseFence` (relaxed scope, §3.3); + 3. wave-0 `s_barrier_signal -3`; + 4. exit (no flag store). +- **Owner WG** — after computing its own partial portion: + 1. `s_barrier_wait -3` **once** (not per-peer): the cluster split barrier + guarantees *every* WG in the cluster has signalled, i.e. all `C-1` peers + have published; + 2. `acquireFence` (relaxed scope, §3.3); + 3. run the existing `fixupStep`/`fixupBatch` loop over the `C-1` peer slots + (`StreamKIdx+1 … cluster_last`) to `VAddF32`-accumulate — *unchanged + accumulation math*; + 4. normal alpha/beta store. + +Because the barrier is a single all-cluster synchronization, the owner waits once +for the whole cluster instead of `C-1` per-peer flag polls, and **no global flag +store/reset/spin is executed at all** on the fast path. + +### 3.2 Where partials live — global WS for v1 + +- **v1:** keep partials in the **global workspace** (reuse `computeWorkspaceSrd` + slot layout and `fixupStep` verbatim). Only the *synchronization* changes + (barrier instead of flag). This minimizes the diff and reuses the audited + accumulation path. +- **v2 (future):** stage cluster-local partials in **LDS** (cluster WGs co-reside + on one shader engine and can share via LDS/TDM multicast), eliminating the WS + round-trip. Deferred: it changes the `fixupStep` addressing and interacts with + epilogue LDS usage/bias LDS barriers. + +### 3.3 Fences — can device scope relax to cluster scope? + +- The device-scope `global_wb`/`global_inv` (`StreamK.py:206-249`) exist because + producer and consumer may sit on **different CUs / L2 partitions**. Intra-cluster + peers co-schedule on **one shader engine**, so a narrower coherence scope is + correct for the fast path *iff* the arch exposes it. +- **v1 (safe):** keep `SCOPE_DEV` `global_wb`/`global_inv` around the WS + partials even on the fast path. This is strictly correct (superset ordering) + and lets us validate the barrier mechanics independently of fence tuning. The + win is removing the *spin-wait*, which dominates. +- **Future (opt):** if a shader-engine / cluster cache scope is available + (extend the `CacheScope` enum + a new arch cap, mirroring `HasInvWbDevFences`), + add a `StreamKMemoryOrderingClusterScopeFences` subclass selected only on the + intra-cluster fast path. Gate behind a cap so non-gfx1250 and the fallback + path are untouched. This depends on HW confirmation that a cluster/SE-scoped + `global_wb/global_inv` is semantically sufficient given the cluster + co-residency guarantee, so the MVP keeps `SCOPE_DEV`. + +### 3.4 Owner selection + correct fallback + +- **Owner within a cluster:** unchanged — the WG with `StreamKLocalStart == 0`. + Under one-tile-per-cluster alignment this is `wg_in_cluster == 0` + (`cluster_base`), which also naturally elects the flag-reset/store wave. +- **Fallback (peers span >1 cluster, DP tiles, atomic, tree-straddle):** keep the + existing global-flag reduction **compiled in** and select it at runtime. The + guard is computable in the epilogue: + + ``` + intra_cluster = (owner_idx == cluster_base) && (last_peer_idx <= cluster_last) + ``` + + where `last_peer_idx` is derived from the existing fixup bounds + (`sFixupEnd`/`sCtaIdx` logic, `:955`, `:1020-1032`). If `intra_cluster`, take + the cluster-barrier fast path; else fall through to the existing flag path + verbatim. Under the MVP alignment (§2.3) `intra_cluster` is true for all SK + tiles except the last partial cluster, so the fallback rarely runs but + guarantees correctness. +- **DP tiles** (`StreamKLocalStart==0 && finished`) never enter the reduction and + branch to the regular store (`:778`, `:932-933`), so they never *wait* on a + cluster barrier — but they must still **signal** if they share a cluster with + reducers (see §6 deadlock). MVP alignment avoids mixing DP and SK WGs in a + cluster, so this does not arise in v1. + +--- + +## 4. Where the cluster barrier is emitted + +**Decision: emit the cluster barrier inline in `StreamK.py`** (new small helpers +`clusterReduceSignal(writer, kernel)` / `clusterReduceWait(writer, kernel)`) +using the rocisa `SBarrier(separate=True, wait=…, clusterBarrier=True)` emitter +directly, reusing the **wave-0 election pattern** from +`Subtile/ClusterBarrier.py:subtileClusterBarrierSignal` (copy the 3-instruction +`s_cmp_eq_u32 WaveIdx,0 / s_cbranch_scc0 skip / s_barrier_signal -3 / skip:` +shape). Assert `asmCaps["HasClusterBarrier"]` exactly like +`insertClusterBarrier` (`ClusterBarrier.py:74-75`). + +**Rationale:** + +- **The stinkytofu `InsertClusterBarrierPass` will not fire here.** Its anchors + (`tensor_load_to_lds`, `label_GSU_1`, `label_openLoopL`, `Tail Loop`) are all + mainloop/prologue/tail markers; the StreamK reduction is in the epilogue + `storeBranchesCommon`. Adding a new anchor for the epilogue to that C++ pass + would duplicate StreamK's tile/peer bookkeeping in a place that has no access + to `StreamKLocalStart`/`sCtaIdx`. Rejected. +- **`Subtile/ClusterBarrier.py:insertClusterBarrier` is also mainloop-shaped** + (it splices around the *mainloop* workgroup barrier and hides the election + branch behind a WMMA). The StreamK reduction has no WMMA to hide behind and a + very different control-flow (spin loop, wave-0 flag reset). We *reuse the + primitive* (`SBarrier(...,clusterBarrier=True)` + wave-0 election) but not the + splice. This keeps the barrier co-located with the `StreamKLocalStart==0` owner + logic and the `intra_cluster` runtime guard, where the needed SGPRs are live. + +Concretely the emit sites in `storeBranchesCommon`: +- Replace the per-peer flag poll block (linear `:960-979`; tree `:868-889`) with: + `if intra_cluster: {owner: clusterReduceWait + acquireFence; skip flag reset}`. +- Add non-owner signal: at the end of `writePartials` + (`:1042`/`:2462`/`:2876`), after the partial store + `releaseFence`, emit + `clusterReduceSignal` on the fast path. + +--- + +## 5. Enablement / plumbing + +### 5.1 Solution parameter + +Add an explicit boolean solution parameter **`StreamKClusterReduction`** +(default `0`) rather than overloading `ClusterDim` alone, because clustering is +already overloaded to mean "Multicast on" (§1.1). Enabling it requires: + +- `StreamK == 3` (see §2.3; SK4/SK5 are out of MVP scope — dynamic/atomic peer + sets cannot be statically clustered). +- `ClusterDim == [C, 1]` with `C` a power of two, `2 <= C <= 16`. +- gfx1250 (`asmCaps["HasClusterBarrier"]`) and `archCaps["HasNewBarrier"]`. +- `TDMInst != 0` (so `WaveIdx` is allocated — the wave-0 election needs it; same + condition that gates `ClusterBarrier` at `Solution.py:980`). +- `not StreamKAtomic` and `not StreamKForceDPOnly` for the fast path (these skip + the reduction entirely — `storeBranchesCommon:748-749`). + +### 5.2 Decouple Multicast from ClusterDim + +`Solution.py:977-978` forces `Multicast=True` for any `ClusterDim != [1,1]`. +For a **barrier-only** StreamK cluster we do *not* want cooperative TDM loads in +the MVP. Change: when `StreamKClusterReduction` is on (and Multicast +is not independently requested), keep `Multicast=False` while still setting +`ClusterBarrier`-style capability for the epilogue emit. `Multicast` is now an +independent tri-state opt-in and cluster-cooperative loads (StreamK DP +B-multicast) have shipped as a sibling feature — see +`cluster-load-component-and-streamk-multicast.md`. + +### 5.3 Validation rules (SolutionStructs) + +- Reject `StreamKClusterReduction` unless the §5.1 predicate holds + (mirror the reject-with-reason pattern used throughout `Solution.py`). +- Reject `ClusterDim[1] != 1` when `StreamKClusterReduction` (1-D grid, §2.2). +- Interaction with existing gfx1250 StreamK constraints: gfx1250 StreamK today + requires MX data (`TDMInst=3`, MX TDM loads); `isStreamKConstantsToVgprEnabled` + is SK3-only; `StreamKXCCMapping=3` overflows SGPRs (use `0`). The new param + inherits all of these (SK3 + MX + XCC=0). + +### 5.4 Host plumbing + +- `getSKGridImpl` rounding to multiple of `C` (§2.2) — behind the same + size-mapping flag (thread `sizeMapping.streamKClusterReduction` + + `sizeMapping.clusterDim` through, both already partially present: + `clusterDim` at `ContractionSolution.cpp:1765,1776`). +- Kernel-arg accounting (`skGrid`, `SKItersPerWG`, `skTiles`) uses the rounded + grid and the fixed even split (`skSplit == C`, reuse the parallel-reduction + accounting at `:933-942`). + +--- + +## 6. Risks / gotchas and de-risking + +| Risk | Why | De-risk | +|---|---|---| +| **Deadlock if a cluster member never signals.** | `s_barrier_wait -3` blocks until *all* cluster WGs have `s_barrier_signal -3`'d. Any cluster WG that early-exits (e.g. `preLoop` `KernelEnd` branch `StreamK.py:2522`; a DP-only WG; a peer that took the "started & finished tile" store path `:778/:933`) never signals ⇒ owner hangs. | **MVP alignment (§2.3):** a cluster == one SK tile's peers, all of which run the reduction; DP WGs are in DP-only clusters that never wait. **Invariant to enforce in codegen:** *every* WG on the cluster fast path must signal on *every* exit path before leaving (including the "finished my slice" path). Add the signal at a single choke point in the epilogue guarded by `intra_cluster`, not scattered. Keep the global-flag fallback for any tile whose membership is not provably complete. | +| **Cluster size vs `fixup_peers` mismatch (big/little imbalance).** | SK3 `extraIters` gives the first `extraIters` WGs one extra iteration, so peers-per-tile can vary by ±1 and straddle a `C` boundary. | Use the **fixed even split** (`skSplit==C`, no extraIters) for the cluster mode; route the leftover/partial last cluster through the fallback via the `intra_cluster` runtime guard (§3.4). | +| **Peers span >1 cluster (general SK).** | General SK tiling does not align tiles to clusters. | Runtime `intra_cluster` guard selects the fast path only when provably intra-cluster; otherwise the existing global-flag path runs unchanged. | +| **SGPR pressure.** | gfx1250 SK SGPRs already tight; XCC=3 overflows. | Derive `wg_in_cluster`/`cluster_base` by cheap `AND`/`AND-NOT` on the existing `StreamKIdx` (C power of two) — no division, no new persistent SGPRs. Reuse `allocTmpSgpr` scopes. Forbid `StreamKXCCMapping=3` with the new param. | +| **`ttmp9` workaround corrupts index under clustering.** | `StreamK.py:2495-2498` reads raw `wg_x`. | §2.2 fix: use the cluster-remapped `WorkGroup0`. Covered by a codegen snapshot assert that `StreamKIdx` derivation contains the `cluster_x*C + wg_x` form. | +| **Fence relaxation incorrect.** | Cluster-scope `global_wb/global_inv` may not be coherent. | The MVP keeps `SCOPE_DEV` (correct superset). Relaxation is a gated future opt pending HW confirmation. | +| **Multicast side effects.** | `ClusterDim!=[1,1]` forces `Multicast=True`, changing load codegen and adding SGPR mask compute (`KernelWriterAssembly.py:2646-2655`). | Decouple (§5.2): barrier-only v1 keeps `Multicast=False`. | + +--- + +## 7. Test plan (behavior → test) — as shipped + +Mirrors existing patterns: CPU asm-string unit tests, syrupy snapshot +characterization, and a GPU roundtrip (run under the arch's simulator/hardware). + +| Behavior | Test type | Where | +|---|---|---| +| `clusterReduceSignal/Wait` emit exactly one wave-0 election branch + `-3` barrier ids; `HasClusterBarrier` asserted; reduction gate matrix (SK3/linear/non-atomic/non-DP only) | CPU asm-string unit | `Tensile/Tests/unit/test_streamk_cluster_reduction.py` | +| Cluster config emits real gfx1250 assembly (`err==0`); fast-path handshake present (`s_barrier_signal -3` peer arrive, `s_barrier_wait -3` owner wait); global-flag reduction retained as fallback; order-invariant golden | snapshot char | `Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py` (designed config `_designed/gfx1250/streamk_cluster_reduction.yaml`, golden `__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr`) | +| Store/release/signal/wait/acquire/accumulate sequence runs on gfx1250, reduces correctly, and does not deadlock, for `C` in {2,4} | GPU roundtrip | `Tensile/Tests/unit/test_streamk_cluster_reduction_gpu.py` (`@requires_gpu_gfx1250`; watchdog on hang) | +| Real multi-WG cluster StreamK GEMM end-to-end | C++ client | `Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml` (+ mxf4 sibling) | + +The cooperative-load / multicast siblings of these tests +(`test_streamk_cluster_coop_load_gfx1250_char.py`, +`test_streamk_cluster_multicast_gfx1250_char.py`, `test_streamk_multicast.py`, +`test_cluster_load_component.py`) cover the companion feature documented in +`cluster-load-component-and-streamk-multicast.md`. + +The gfx1250 GPU marker lives in `Tensile/Tests/unit/gpu_test_helpers.py` +(`HAS_GFX1250` + `requires_gpu_gfx1250`), independent of the existing gfx950 +`requires_gpu`; the target is driven via the `TENSILE_GPU_TARGET=gfx1250` +override. + +--- + +## 8. Follow-up work (out of MVP scope) + +- **Fence scope:** a cluster/shader-engine-scoped `global_wb`/`global_inv` + (§3.3) could replace `SCOPE_DEV` on the fast path once HW confirms it is + semantically sufficient given cluster co-residency, and once a `CacheScope` / + arch cap exposes it. +- **LDS partials (§3.2):** eliminate the global-WS round-trip by staging + cluster-local partials in LDS/TDM-multicast; this changes `fixupStep` + addressing and the epilogue LDS budget. +- **Broader partitions:** relax the fixed even split / one-tile-per-cluster + constraint (§2.3) toward general SK tiling with the runtime straddle guard, and + extend beyond SK3 where peer sets can be statically clustered. + +--- + +## 9. Summary of decisions + +- **Approach:** co-locate a StreamK tile's `fixup_peers` into one 1-D cluster + (`ClusterDim=[C,1]`), exploiting the HW remap `WorkGroup0 = cluster_x*C + wg_x` + which already clusters consecutive `StreamKIdx`. Replace the global-flag + spin-wait with a single cluster split barrier; keep the global-flag path as a + runtime-selected fallback. +- **Variant:** SK3 two-tile (MVP), fixed even split; SK4/SK5 out of scope + (dynamic/atomic peer sets can't be statically clustered). Barrier-only here; + cooperative-load multicast shipped separately (see + `cluster-load-component-and-streamk-multicast.md`). +- **Indexing:** `ClusterDim=[C,1]`, fix the `ttmp9` workaround to use the + cluster-remapped `WorkGroup0`, `cluster_base = StreamKIdx & ~(C-1)`; host + rounds `skGrid` to a multiple of `C`. +- **Reduction:** owner `s_barrier_wait -3` once + `acquireFence` + existing + `fixupStep`; peers write WS partial + `releaseFence` + wave-0 `s_barrier_signal -3`; + partials stay in global WS for v1; fences stay `SCOPE_DEV` for v1. +- **Barrier emission:** inline in `StreamK.py` epilogue via rocisa + `SBarrier(...,clusterBarrier=True)` + wave-0 election (reuse the + `Subtile/ClusterBarrier.py` primitive shape); **not** the stinkytofu pass / + subtile splice (their anchors don't reach the epilogue). +- **Enablement:** new `StreamKClusterReduction` solution param requiring SK3 + + `[C,1]` + gfx1250 `HasClusterBarrier` + `TDMInst!=0`; decouple Multicast. +- **Top risks:** cluster-barrier deadlock if any member fails to signal (⇒ + one-tile-per-cluster alignment + all-paths-signal invariant + fallback), SGPR + pressure (⇒ cheap bitwise cluster coords), big/little imbalance (⇒ fixed even + split + runtime guard). diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py index 1c045d9c8ecf..aae22c30fc1d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py @@ -571,6 +571,7 @@ {"StreamKAtomic": [0]}, {"StreamKXCCMapping": [0]}, {"StreamKFixupTreeReduction": [0]}, + {"StreamKClusterReduction": [0]}, # NOTE: StreamKMulticast is a derived-only internal state key (like # ClusterBarrier), auto-enabled by Solution.py for StreamK==3 + ClusterDim # clusters; it is deliberately NOT a benchmark/default parameter here. diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index 2b953f96f5f1..12ffd50a0baa 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -837,6 +837,18 @@ def makeValidMatrixInstructions(): # 0: use linear reduction # 1: use tree reduction "StreamKFixupTreeReduction": [0, 1], + # Enables the gfx1250 workgroup-cluster reduction fast path for StreamK. + # When enabled, a StreamK tile's fixup peers are co-located in a single 1-D + # workgroup cluster (ClusterDim = [C,1]) and the cross-CU global-flag + # spin-wait is replaced by an intra-cluster split barrier. Barrier-only in + # v1 (Multicast stays off); partials remain in the global workspace and the + # global-flag reduction is retained as a runtime/compile fallback. + # Requires StreamK == 3, ClusterDim == [C,1] with C a power of two in 2..16, + # gfx1250 (HasClusterBarrier) with TDMInst != 0, and NOT StreamKAtomic / + # NOT StreamKForceDPOnly. See docs/design/streamk-wg-clusters.md. + # 0: use the existing global-flag reduction + # 1: enable the cluster-barrier reduction fast path + "StreamKClusterReduction": [0, 1], # NOTE: StreamKMulticast (the gfx1250 StreamK DP cooperative cluster-load / # TDM B-multicast fast path) is intentionally NOT a valid/benchmark # parameter. It is a DERIVED-ONLY internal state key (see the ClusterBarrier diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index f523ba9fc834..2ca099919e01 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -954,8 +954,38 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele writer.releaseStreamKConstSgpr(sIpt) module.add(SSubU32(dst=sgpr(sFixupEnd), src0=sgpr("StreamKIterEnd"), src1=sgpr(tmpSgpr), comment="calc iterations completed by this WG")) + # --- Intra-cluster split-barrier fast path (StreamK v3) --- + # Deadlock invariant: this owner and every non-owner peer of the + # tile evaluate the SAME uniform intra-cluster predicate + # (clusterReduceIntraCheck), so the whole cluster commits to + # either the barrier path or the global-flag path together -- + # never a mix. On the fast path the owner arrives once and waits + # once for all C cluster members (peers signal after publishing + # their partials to the workspace), then reads the peer partials + # with the unchanged fixupStep loop; no per-peer flag + # read/spin/reset runs. + clusterFast = self._streamKClusterReductionEnabled(writer, kernel) + sClusterFast = None + skClusterSkipFlag = None + if clusterFast: + sClusterFast = writer.sgprPool.checkOut(1, "SKClusterFast") + skClusterSetupDone = Label(label=writer.labels.getNameInc("SK_ClusterSetupDone"), comment="") + skClusterSkipFlag = Label(label=writer.labels.getNameInc("SK_ClusterSkipFlag"), comment="") + module.add(self.clusterReduceIntraCheck(writer, kernel)) + module.add(SCSelectB32(dst=sgpr(sClusterFast), src0=1, src1=0, comment="latch intra-cluster verdict")) + module.add(SCmpEQU32(src0=sgpr(sClusterFast), src1=1, comment="intra-cluster fast path?")) + module.add(SCBranchSCC0(labelName=skClusterSetupDone.getLabelName(), comment="not intra-cluster: use global-flag reduction")) + module.add(self.clusterReduceSignal(writer, kernel)) # owner arrives at the cluster barrier + module.add(self.clusterReduceWait(writer, kernel)) # wait until all C cluster members arrived + module.add(memOrder.acquireFence(writer)) # once: observe peers' published partials + module.add(skClusterSetupDone) + module.add(skFixupLabel) + if clusterFast: + module.add(SCmpEQU32(src0=sgpr(sClusterFast), src1=1, comment="intra-cluster: peers already synced via cluster barrier")) + module.add(SCBranchSCC1(labelName=skClusterSkipFlag.getLabelName(), comment="skip per-peer global-flag handshake")) + # Check flag module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(sCtaIdx), shiftHex=log2(4), comment="flag offset based on CTA index")) module.add(memOrder.readFlag(writer, dst=tmpSgpr+2, soffset=sgpr(tmpSgpr))) @@ -977,6 +1007,10 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele module.add(VMovB32(dst=vgpr(tmpVgpr), src=0, comment="move 0 to tmpVgpr")) module.add(self.setFlagValue(writer, src=vgpr(tmpVgpr), soffset=sgpr(tmpSgpr), comment="reset flag")) module.add(skipFlagReset) + # Fast-path branch target: land here (after the flag handshake) + # so every runtime fixup iteration skips the global-flag block. + if clusterFast: + module.add(skClusterSkipFlag) writer.sgprPool.checkIn(tmpSgpr) fixupEdge = [False] # Test no edge variant @@ -1033,6 +1067,8 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele writer.sgprPool.checkIn(sFixupEnd) writer.sgprPool.checkIn(sCtaIdx) + if clusterFast: + writer.sgprPool.checkIn(sClusterFast) module.add(skStoreLabel) @@ -1350,6 +1386,24 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, module.add(memOrder.releaseFence(writer)) module.add(SBarrier(comment="store all data before setting flag")) + # Non-owner peer fast path: after publishing the partial and the + # release fence, arrive at the cluster split barrier INSTEAD OF + # raising the global completion flag. Deadlock invariant: the + # intra-cluster predicate is the identical uniform check the owner + # evaluates, so a cluster never mixes barrier signalers with flag + # setters, and this peer signals exactly once on this (its only) + # epilogue exit before branching to endLabel below. + clusterFast = self._streamKClusterReductionEnabled(writer, kernel) + skClusterSignalDone = None + if clusterFast: + skClusterUseFlag = Label(label=writer.labels.getNameInc("SK_ClusterUseFlag"), comment="") + skClusterSignalDone = Label(label=writer.labels.getNameInc("SK_ClusterSignalDone"), comment="") + module.add(self.clusterReduceIntraCheck(writer, kernel)) + module.add(SCBranchSCC0(labelName=skClusterUseFlag.getLabelName(), comment="not intra-cluster: fall back to global flag")) + module.add(self.clusterReduceSignal(writer, kernel)) + module.add(SBranch(labelName=skClusterSignalDone.getLabelName(), comment="peer arrived at cluster barrier: skip global-flag store")) + module.add(skClusterUseFlag) + if kernel["StreamK"] == 4: # TODO modularize this section into abstract function module.add(self.calculatePartialIdx(tmpSgpr)) @@ -1397,6 +1451,8 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, module.add(VMovB32(dst=vgpr(tmpVgpr), src=1, comment="move 1 to tmpVgpr")) module.add(self.setFlagValue(writer, src=vgpr(tmpVgpr), soffset=sgpr(tmpSgpr), comment="set flag")) module.add(skipFlagSet) + if clusterFast: + module.add(skClusterSignalDone) module.add(SWaitCnt(kmcnt=0, comment="wait for flag")) # TODO just for testing if "Deferred" in endLabel.getLabelName(): @@ -1461,6 +1517,100 @@ def getFlagValue(self, writer, dst, soffset, comment=""): return module + def _streamKClusterReductionEnabled(self, writer, kernel): + """Compile-time gate for the intra-cluster split-barrier reduction. + + Both the owner's cluster wait and the non-owner's cluster signal are + gated on this identical predicate (see clusterReduceIntraCheck for the + deadlock invariant it upholds). + + Restricted to StreamK variant 3 (StreamKTwoTileDPFirst), linear + reduction only (the tree schedule keeps the global-flag handshake), and + the non-atomic / non-DP-only reduction path. Requires the gfx1250 + cluster-barrier asm capability. + """ + return (bool(kernel.get("StreamKClusterReduction", False)) + and kernel["StreamK"] == 3 + and not kernel["StreamKFixupTreeReduction"] + and not kernel["StreamKAtomic"] + and not kernel["StreamKForceDPOnly"] + and writer.states.asmCaps.get("HasClusterBarrier", False)) + + def clusterReduceSignal(self, writer, kernel): + """Wave-0-elected cluster split-barrier arrive (``s_barrier_signal -3``). + + One wave per workgroup arrives at the cluster-scope split barrier; the + remaining waves branch over the signal. Wave election reuses the + Serial/readfirstlane idiom the StreamK flag path already uses (rather + than sgpr("WaveIdx"), which may be undefined in the epilogue), so the + helper is self-contained. + """ + assert writer.states.asmCaps.get("HasClusterBarrier", False), \ + "StreamK cluster reduction requires the HasClusterBarrier asm capability" + module = Module("StreamK cluster reduce signal") + skipSignal = Label(label=writer.labels.getNameInc("SK_ClusterSkipSignal"), comment="") + elect = writer.sgprPool.checkOut(1, "SKClusterElect") + module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) + module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) + module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) + module.add(skipSignal) + writer.sgprPool.checkIn(elect) + return module + + def clusterReduceWait(self, writer, kernel): + """Cluster split-barrier wait (``s_barrier_wait -3``). + + Emitted for every wave of the owner workgroup; unblocks only once every + workgroup in the cluster has arrived (i.e. all peers have published + their partials). Also serves as the intra-workgroup wave sync the + global-flag path got from its workgroup barrier. + """ + assert writer.states.asmCaps.get("HasClusterBarrier", False), \ + "StreamK cluster reduction requires the HasClusterBarrier asm capability" + module = Module("StreamK cluster reduce wait") + module.add(SBarrier(True, True, True, comment="cluster_barrier wait (all peers arrived)")) + return module + + def clusterReduceIntraCheck(self, writer, kernel): + """Emit the uniform intra-cluster predicate; leaves SCC=1 (fast path) + when this workgroup's cluster hosts a single, fully-populated StreamK + tile, SCC=0 (fall back to the global-flag path) otherwise. + + The predicate is a pure function of the cluster's top StreamK index and + the SK grid size (both uniform across a cluster), so every member of a + cluster -- owner and peers alike -- computes the identical verdict. This + is what makes the fast path deadlock-safe: a cluster is never split + between barrier signalers and flag setters. + + Contract (host accounting, StreamKClusterReduction): ClusterDim=[C,1] + with C a power of two; cluster c owns the contiguous StreamK index range + [c*C, c*C+C) which is exactly one tile's peer group (skSplit==C, fixed + even split); skGrid is rounded up to a multiple of C. Under that + contract cluster_last = StreamKIdx | (C-1) < skGrid holds for every SK + workgroup, so the fast path is taken; a partially-filled trailing + cluster (contract violated) fails the check and the whole cluster falls + back safely. + """ + module = Module("StreamK cluster intra-cluster check") + C = kernel["ClusterDim"][0] + skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) + sClusterLast = writer.sgprPool.checkOut(1, "SKClusterLast") + sIdx = writer.acquireStreamKConstSgpr(kernel, "StreamKIdx") + if skConstsInVgprs: + module.add(VReadfirstlaneB32(dst=sgpr(sIdx), src=vgpr(writer.states.skConstVgprs["StreamKIdx"]))) + # cluster_last = StreamKIdx | (C-1): the top StreamK index of this WG's + # cluster (C a power of two). Uniform across every WG in the cluster. + module.add(SOrB32(dst=sgpr(sClusterLast), src0=sgpr(sIdx), src1=hex(C - 1), comment="cluster_last = StreamKIdx | (C-1)")) + writer.releaseStreamKConstSgpr(sIdx) + sGrid = writer.acquireStreamKConstSgpr(kernel, "skGrid") + if skConstsInVgprs: + module.add(VReadfirstlaneB32(dst=sgpr(sGrid), src=vgpr(writer.states.skConstVgprs["skGrid"]))) + module.add(SCmpLtU32(src0=sgpr(sClusterLast), src1=sgpr(sGrid), comment="intra-cluster: cluster fully within SK grid?")) + writer.releaseStreamKConstSgpr(sGrid) + writer.sgprPool.checkIn(sClusterLast) + return module + def partialsWriteBatch(self, writer, kernel, ss, batchIdx, applyAlpha, beta, edge, gwvw, atomicW, \ batchElements, addrD, addrC, \ tmpVgpr, cvtVgprStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead): diff --git a/projects/hipblaslt/tensilelite/Tensile/Contractions.py b/projects/hipblaslt/tensilelite/Tensile/Contractions.py index 2bbf6ba5995f..3dd8a2d9fc92 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Contractions.py +++ b/projects/hipblaslt/tensilelite/Tensile/Contractions.py @@ -594,6 +594,18 @@ def CompoundPredicates(cls, state, problemType): valuepredicates.append(state["ClusterDim"][1]) rv += [cls('ClusterDimCheck', value=valuepredicates)] + # StreamK cluster-reduction split-barrier safety (gfx1250). The C = + # ClusterDim[0] peers split a tile's itersPerTile = ceil(K/DepthU) + # K-iterations and hand off through an intra-cluster split barrier; if + # itersPerTile % C != 0 (incl. C > itersPerTile) the split barrier + # over-signals -> hang. itersPerTile depends on runtime K, so this is a + # per-problem HARD REJECT (not a build-time reject, not a silent + # fallback). Only the K-split reduction path needs it; StreamKMulticast + # (no K-split) relies on ClusterDimCheck instead. + if state.get("StreamKClusterReduction", 0) and state["ClusterDim"][0] > 1: + rv += [cls('ClusterReductionIterCheck', + value=[state["DepthU"], state["ClusterDim"][0]])] + return rv @classmethod @@ -627,6 +639,7 @@ class SizeMapping: 'streamK', 'streamKForceDPOnly', 'streamKAtomic', + 'streamKClusterReduction', 'streamKMulticast', 'prefetchAcrossPersistent', 'sourceKernel', @@ -722,6 +735,7 @@ def convertSFCWGMListToHex(value): streamK = d['StreamK'] if 'StreamK' in d else 0, streamKForceDPOnly = d.get('StreamKForceDPOnly', 0), streamKAtomic = d['StreamKAtomic'] if 'StreamKAtomic' in d else 0, + streamKClusterReduction = d.get('StreamKClusterReduction', 0), streamKMulticast = d.get('StreamKMulticast', 0), prefetchAcrossPersistent = d.get('PrefetchAcrossPersistent', 0), magicDivAlg = d.get('MagicDivAlg', 1), diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 3199e991d0db..69bbc1e635bd 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -231,6 +231,96 @@ def _validateStreamKForceDPOnly(state, printRejectionReason): return True +def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): + """Validate the gfx1250 StreamK workgroup-cluster reduction fast path. + + StreamKClusterReduction co-locates a StreamK tile's fixup peers in a single + 1-D workgroup cluster (ClusterDim = [C, 1]) and replaces the cross-CU + global-flag spin-wait with an intra-cluster split barrier. It is an explicit, + opt-in fast path (barrier-only in v1); when its solution-level requirements + are not met the solution is rejected here at BUILD time with a clear message. + See docs/design/streamk-wg-clusters.md. + + Note on the "multiple of the cluster size" limitation: the reduction also + requires itersPerTile = ceil(K / DepthU) to be a multiple of ClusterDim[0]=C + (otherwise the split barrier over-signals and hangs). That is a HARD REJECT of + the cluster solution for non-conforming problems, but it CANNOT be enforced + here because itersPerTile depends on the runtime K, which is unknown at build + time. It is instead enforced as a per-problem selection reject by the + ClusterReductionIterCheck predicate (emitted from Contractions.py), whose + diagnostic names the limitation to the user. This is a deliberate reject, not + a silent fallback. + """ + if not state.get("StreamKClusterReduction", 0): + return True + + # Mutually exclusive with the cooperative-load multicast fast path: the two + # features want incompatible cluster semantics (K-split reduction peers vs. + # M-adjacent shared-B multicast peers) on the same 1-D cluster. + if state.get("StreamKMulticast", 0): + reject(state, printRejectionReason, + "StreamKMulticast and StreamKClusterReduction are mutually exclusive") + return False + + # SK3 (StreamKTwoTileDPFirst) only; SK4/SK5 dynamic/atomic peer sets can not + # be statically clustered. + if state["StreamK"] != 3: + reject(state, printRejectionReason, + "StreamKClusterReduction requires StreamK=3 (two-tile DP-first)") + return False + + # The fast path replaces the reduction handshake, which the atomic and + # DP-only paths skip entirely. + if state["StreamKAtomic"]: + reject(state, printRejectionReason, + "StreamKClusterReduction is not supported with StreamKAtomic") + return False + if state["StreamKForceDPOnly"]: + reject(state, printRejectionReason, + "StreamKClusterReduction is not supported with StreamKForceDPOnly") + return False + + # StreamKXCCMapping=3 overflows the SGPR budget alongside the cluster coords. + if state["StreamKXCCMapping"] == 3: + reject(state, printRejectionReason, + "StreamKClusterReduction is not supported with StreamKXCCMapping=3 (SGPR overflow)") + return False + + # 1-D cluster [C, 1] with C a power of two in [2, 16]. A ClusterDim[1] > 1 + # would require gridDimY % ClusterDim[1] == 0 while the StreamK grid is 1-D. + clusterDim = state["ClusterDim"] + c = clusterDim[0] + if clusterDim[1] != 1: + reject(state, printRejectionReason, + "StreamKClusterReduction requires ClusterDim = [C, 1] (got %s)" % clusterDim) + return False + if c < 2 or c > 16 or (c & (c - 1)) != 0: + reject(state, printRejectionReason, + "StreamKClusterReduction requires ClusterDim[0] a power of two in [2, 16] (got %d)" % c) + return False + + # gfx1250 with the cluster split barrier capability. + isa = tuple(state["ISA"]) + if isa != (12, 5, 0): + reject(state, printRejectionReason, + "StreamKClusterReduction requires gfx1250 ISA (12, 5, 0)") + return False + if not isaInfoMap[isa].asmCaps.get("HasClusterBarrier", False): + reject(state, printRejectionReason, + "StreamKClusterReduction requires asmCap HasClusterBarrier") + return False + + # The cluster split-barrier reduction is wired up only on the gfx1250 TDM + # cluster path (the cluster-coord SGPRs it relies on are allocated only when + # TDM is enabled, same condition that gates ClusterBarrier). + if state["TDMInst"] == 0: + reject(state, printRejectionReason, + "StreamKClusterReduction requires TDMInst != 0") + return False + + return True + + def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): """Validate the gfx1250 StreamK DP cooperative cluster-load fast path. @@ -252,6 +342,14 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): if not state.get("StreamKMulticast", 0): return True + # Mutually exclusive with the cluster split-barrier reduction fast path: the + # two features want incompatible cluster semantics on the same 1-D cluster and + # must never be enabled together. + if state.get("StreamKClusterReduction", 0): + reject(state, printRejectionReason, + "StreamKMulticast and StreamKClusterReduction are mutually exclusive") + return False + # StreamKMulticast is auto-enabled by ClusterDim on SK3, but the multicast mask # SGPRs are gated on the (derived) Multicast flag while the mask predicate / # boundary-clear emitters are gated on StreamKMulticast. An effective @@ -1177,7 +1275,12 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, # ClusterDim is tested first so this (like the legacy Multicast auto branch # below) never dereferences StreamK for the common non-clustered state, which # some partial-state derivation call sites construct without a StreamK key. - if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3: + # + # StreamKClusterReduction claims the [C,1] cluster for its own split-barrier + # K-split reduction (mutually exclusive with the cooperative-load multicast), + # so a reduction cluster must NOT auto-enable StreamKMulticast here. + if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3 \ + and not state.get("StreamKClusterReduction", 0): state["StreamKMulticast"] = 1 # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. # Default -1 reproduces the historic ClusterDim-coupled derivation, so YAML @@ -1899,6 +2002,7 @@ def assignDerivedParameters( if not state["BufferStore"]: reject(state, printRejectionReason, "Stream-K requires BufferStore") _validateStreamKForceDPOnly(state, printRejectionReason) + _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap) _validateStreamKMulticast(state, printRejectionReason, isaInfoMap) if state["StreamKAtomic"] == 1: if state["StreamK"] == 4: @@ -1966,6 +2070,7 @@ def assignDerivedParameters( state["StreamKAtomic"] = 0 state["StreamKXCCMapping"] = 0 state["StreamKFixupTreeReduction"] = 0 + state["StreamKClusterReduction"] = 0 state["StreamKMulticast"] = 0 state["DebugStreamK"] = 0 state["PrefetchAcrossPersistent"] = 0 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml index a6a229633c3d..3901b93a208e 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml @@ -11,9 +11,11 @@ # (auto-derived from StreamK=3 + ClusterDim): # # - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D spatial clusters, C in {2,4,8} +# - StreamKClusterReduction: [0] -- mutually exclusive with multicast # # StreamKMulticast is a derived-only internal state (no YAML opt-in): a StreamK=3 -# ClusterDim != [1,1] config auto-enables the DP cooperative B-multicast path. +# ClusterDim != [1,1] config that does not request StreamKClusterReduction +# auto-enables the DP cooperative B-multicast path. # # In the DP region the C consecutive workgroups of a [C,1] cluster process # M-adjacent tiles that share the same B over full K, so B is TDM-multicast to @@ -117,6 +119,7 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] + - StreamKClusterReduction: [0] - ClusterDim: [[2, 1], [4, 1], [8, 1]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml new file mode 100644 index 000000000000..e0335db630c9 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml @@ -0,0 +1,124 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Client-runnable StreamK workgroup-cluster-reduction test for gfx1250, MX-FP4. +# +# F4 analog of the sibling core/sk_mxf8gemm_cluster_reduction.yaml (MX-FP8): same +# cluster/coverage structure, datatype-specific params differ. MX-F4 constraints +# (DataType F4, MXBlockA/B=32, TDMInst=3, MXLoadInst=TDM, +# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, DepthU=256, +# StreamKXCCMapping=0) with the barrier-only cluster reduction fast path: +# +# - StreamKClusterReduction: [1] -- intra-cluster split-barrier reduce +# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D clusters, C in {2,4,8} +# +# Sizes force partial-tile reductions (K >> DepthU) and keep both nWG0 % C == 0 +# (cluster selection) and itersPerTile % C == 0 (balanced split barrier). When +# itersPerTile % C != 0 the split barrier over-signals -> hang, so the +# ClusterReductionIterCheck predicate HARD-REJECTS such problems at selection +# time (itersPerTile depends on runtime K, so it cannot be a build-time reject). +# A K-tail (K % DepthU != 0) is safe as long as itersPerTile % C == 0. +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch +GlobalParameters: + SyncsPerBenchmark: 0 + Architecture: gfx1250 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + ######################################## + # MX-FP4 (F4 in, float32 out) TN, Batched -- StreamK cluster reduction. + ######################################## + - + - # ProblemType: MX-F4 TN + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + Activation: True + SupportUserArgs: True + UseBias: 0 + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - StreamKClusterReduction: [1] + - ClusterDim: [[2, 1], [4, 1], [8, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [128, 128, 1, 1024] # MT32 nWG0=4 (C=2,4 admit; C=8 nWG0%8!=0); itersPerTile=4 -> C=2,4 partial-tile reduce + - Exact: [512, 512, 1, 2048] # MT32 nWG0=16 (C=2,4,8 admit); itersPerTile=8 -> C=2,4,8 partial-tile reduce (safe C=8) + - Exact: [256, 256, 1, 1920] # K-tail: itersPerTile=ceil(1920/256)=8 (8%2,4,8==0), K%256=128 tail -> tail loop inside cluster reduce (C=8 via MT32 nWG0=8) + - Exact: [256, 256, 2, 1024] # batched (batch=2); nWG0=8, itersPerTile=4 -> C=2,4 partial-tile reduce + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml index b5e4ccfe4345..9277a5b675d7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml @@ -8,9 +8,11 @@ # DP cooperative B-multicast fast path (auto-derived from StreamK=3 + ClusterDim): # # - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D spatial clusters, C in {2,4,8} +# - StreamKClusterReduction: [0] -- mutually exclusive with multicast # # StreamKMulticast is a derived-only internal state (no YAML opt-in): a StreamK=3 -# ClusterDim != [1,1] config auto-enables the DP cooperative B-multicast path. +# ClusterDim != [1,1] config that does not request StreamKClusterReduction +# auto-enables the DP cooperative B-multicast path. # # In the DP region the C consecutive workgroups of a [C,1] cluster process # M-adjacent tiles that share the same B over full K, so B is TDM-multicast to @@ -98,6 +100,7 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] + - StreamKClusterReduction: [0] - ClusterDim: [[2, 1], [4, 1], [8, 1]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml new file mode 100644 index 000000000000..c70fb19c2d4e --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml @@ -0,0 +1,117 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Client-runnable StreamK workgroup-cluster-reduction test for gfx1250. +# +# MX-FP8 TN, StreamK=3 (TDMInst=3, MXLoadInst=TDM, +# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, StreamKXCCMapping=0) with the +# barrier-only cluster reduction fast path: +# +# - StreamKClusterReduction: [1] -- intra-cluster split-barrier reduce +# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D clusters, C in {2,4,8} +# +# Sizes force partial-tile reductions (K >> DepthU) and keep both nWG0 % C == 0 +# (cluster selection) and itersPerTile % C == 0 (balanced split barrier). When +# itersPerTile % C != 0 the split barrier over-signals -> hang, so the +# ClusterReductionIterCheck predicate HARD-REJECTS such problems at selection +# time (itersPerTile depends on runtime K, so it cannot be a build-time reject). +# A K-tail (K % DepthU != 0) is safe as long as itersPerTile % C == 0. +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch +GlobalParameters: + SyncsPerBenchmark: 0 + Architecture: gfx1250 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + ######################################## + # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK cluster reduction. + ######################################## + - + - # ProblemType: MX-F8 TN + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - StreamKClusterReduction: [1] + - ClusterDim: [[2, 1], [4, 1], [8, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [128, 128, 1, 1024] # MT32 nWG0=4 (C=2,4 admit; C=8 nWG0%8!=0); itersPerTile=4 -> C=2,4 partial-tile reduce + - Exact: [512, 512, 1, 2048] # MT32 nWG0=16 (C=2,4,8 admit); itersPerTile=8 -> C=2,4,8 partial-tile reduce (safe C=8) + - Exact: [256, 256, 1, 1920] # K-tail: itersPerTile=ceil(1920/256)=8 (8%2,4,8==0), K%256=128 tail -> tail loop inside cluster reduce (C=8 via MT32 nWG0=8) + - Exact: [256, 256, 2, 1024] # batched (batch=2); nWG0=8, itersPerTile=4 -> C=2,4 partial-tile reduce + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py index edfc66b20721..531c3104d7b1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py @@ -97,3 +97,44 @@ def test_internal_args_support(solution_state): def test_problem_predicate_compound(problem_type, solution_state, snapshot): preds = C.ProblemPredicate.CompoundPredicates(solution_state, problem_type) assert {"count": len(preds), "tags": sorted({p.tag for p in preds})} == snapshot + + +# --- StreamK cluster-reduction split-barrier selection guard -- +# +# ClusterReductionIterCheck must be emitted only for StreamKClusterReduction +# solutions with a real cluster (ClusterDim[0] > 1); its value carries +# [DepthU, C] so the host predicate can reject problems whose +# itersPerTile = ceil(K/DepthU) is not a multiple of C (split-barrier +# over-signal). It must NOT be emitted for non-cluster or multicast solutions. + +def _preds_for(solution_state, problem_type, **overrides): + st = dict(solution_state) + st.update(overrides) + return C.ProblemPredicate.CompoundPredicates(st, problem_type) + + +def _cluster_iter_pred(preds): + return next((p for p in preds if p.tag == "ClusterReductionIterCheck"), None) + + +def test_cluster_reduction_iter_check_emitted(problem_type, solution_state): + preds = _preds_for(solution_state, problem_type, + StreamKClusterReduction=1, ClusterDim=[4, 1], DepthU=256) + p = _cluster_iter_pred(preds) + assert p is not None, "cluster-reduction solution must emit ClusterReductionIterCheck" + # value = [DepthU, C] so the host can compute itersPerTile % C. + assert p.value == [256, 4] + + +def test_cluster_reduction_iter_check_not_emitted_when_off(problem_type, solution_state): + # StreamKClusterReduction off -> no guard (the param is inert). + preds = _preds_for(solution_state, problem_type, + StreamKClusterReduction=0, ClusterDim=[4, 1], DepthU=256) + assert _cluster_iter_pred(preds) is None + + +def test_cluster_reduction_iter_check_not_emitted_without_cluster(problem_type, solution_state): + # Reduction requested but C == 1 (no real cluster) -> no guard. + preds = _preds_for(solution_state, problem_type, + StreamKClusterReduction=1, ClusterDim=[1, 1], DepthU=256) + assert _cluster_iter_pred(preds) is None diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index 81019888e28a..61914c301fa8 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -3,7 +3,7 @@ dict({ 'count': 1, 'names': list([ - 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKCR0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', ]), }) # --- @@ -55,7 +55,7 @@ 'getitem_kernel_language': 'Assembly', 'iter_matches_keys': True, 'keys_is_list': True, - 'len': 338, + 'len': 339, }) # --- # name: test_solution_construction @@ -293,6 +293,7 @@ 'StoreVectorWidth', 'StreamK', 'StreamKAtomic', + 'StreamKClusterReduction', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', 'StreamKMulticast', @@ -400,8 +401,8 @@ 'tailLoopOptA', 'tailLoopOptB', ]), - 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', - 'num_keys': 338, + 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKCR0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'num_keys': 339, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ 'DepthU': 32, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr index 0c2e30b5d753..b19f2c4e3506 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr @@ -1480,6 +1480,7 @@ 'StoreVectorWidth', 'StreamK', 'StreamKAtomic', + 'StreamKClusterReduction', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', 'StreamKXCCMapping', @@ -2580,6 +2581,14 @@ 'len': 2, 'type': 'list', }), + 'StreamKClusterReduction': dict({ + 'head': list([ + 0, + 1, + ]), + 'len': 2, + 'type': 'list', + }), 'StreamKFixupTreeReduction': dict({ 'head': list([ 0, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr new file mode 100644 index 000000000000..35e1f0afdea5 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr @@ -0,0 +1,37 @@ +# serializer version: 1 +# name: test_streamk_cluster_reduction_gfx1250_golden + list([ + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg5n1_UAVQwNvJKjxzq3oOtWaugDzT0nK_8g8EArf7NFg=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgAuSSqpdzu7alrTkBrgFPyFkqyqpG9E_Ty8RQ1xyXrY8=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgNYnvlbTuQlXSSxlSDUxJPqRmrqmX7nLul8abNN-bcIQ=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgWhFe1_TqD4ilHRCVYp87SZQz-26l15ntWpD7T0SbtLs=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgchM4NognQc6wGYmCDZqnoSwpxn315nvCI1kv46p3XiY=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgg-XLK9vb4JCnAz8i82mzuI2Kv0jBkS3o1PRsufEAP5k=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgiDkOmo2IiGAh_e-YoomKdpTWhfgU7cQn5f7fD7vSzWM=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgx11GWFngl0eXcIZTb0HUQb4t714WE_KGR2YLqVodM20=', + 'err': 0, + }), + ]) +# --- diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml new file mode 100644 index 000000000000..92d9d2b864a0 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml @@ -0,0 +1,112 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Designed StreamK + workgroup-cluster-reduction config for gfx1250 codegen +# characterization. +# +# Mirrors the sibling _designed/gfx1250/streamk.yaml (MX-FP8 TN, StreamK=3, +# gfx1250 MX constraints: TDMInst=3, MXLoadInst=TDM, +# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, ISA=(12,5,0), +# StreamKXCCMapping=0), but enables the workgroup-cluster reduction fast path: +# +# - StreamKClusterReduction: [1] -- opt-in barrier-only cluster reduction +# - ClusterDim: [[2, 1], [4, 1]] -- 1-D clusters, C in {2, 4} (power of two) +# +# Under this config StreamK.py must emit the intra-cluster split-barrier +# handshake on the fast path (s_barrier_signal -3 on the non-owner/peer arrive, +# s_barrier_wait -3 on the owner wait) while keeping the global-flag reduction +# compiled in as the runtime fallback. See docs/design/streamk-wg-clusters.md +# (sections 2.3, 3, 5). +# +# Coverage (mirrors the non-cluster MX-FP8 StreamK MT sweep in +# Tests/common/streamk/gfx1250/core/sk_mxf8gemm_tdm.yaml) across the cluster +# codegen surface -- MacroTile sizes x work-item (MIWaveGroup) sizes x C: +# - MatrixInstruction: 4 variants -> +# MT 32x32 (WaveTile 1x1, MIWaveGroup 2x2, 128 work-items) +# MT 64x64 (WaveTile 2x2, MIWaveGroup 2x2, 128 work-items) +# MT 128x128(WaveTile 4x4, MIWaveGroup 2x2, 128 work-items) +# MT 32x16 (WaveTile 1x1, MIWaveGroup 2x1, 64 work-items) -- WI variety +# - ClusterDim: C in {2, 4} +# => 4 x 2 = 8 fork permutations (the char harness emits limit=8; the client +# GEMM sibling sk_mxf8gemm_cluster_reduction.yaml adds C=8 + real problem-size +# sweeps validated on a gfx1250 simulation environment). +GlobalParameters: + SyncsPerBenchmark: 0 + MinimumRequiredVersion: 5.0.0 + NumElementsToValidate: 128 + DataInitTypeBeta: 0 + DataInitTypeAlpha: 1 + Device: 0 + +BenchmarkProblems: + ######################################## + # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK cluster reduction. + ######################################## + - + - # ProblemType: MX-F8 TN + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - StreamKClusterReduction: [1] + - ClusterDim: [[2, 1], [4, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [128, 128, 1, 256] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py new file mode 100644 index 000000000000..4afc18e968e9 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py @@ -0,0 +1,82 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +"""StreamK workgroup-cluster reduction -- gfx1250 characterization (CPU-only). + +Exercises the intra-cluster split-barrier reduction fast path added to +``Tensile/Components/StreamK.py`` (StreamKClusterReduction=1, ClusterDim=[C,1]). +It drives the same config -> Solutions -> emit path as the sibling +``test_r3_streamk_gfx1250_char.py``, but through the cluster-enabled designed +config ``_designed/gfx1250/streamk_cluster.yaml``. + +Asserts: + * every kernel emits real gfx1250 assembly with ``err == 0``; + * the intra-cluster handshake is present on the fast path + (``s_barrier_signal -3`` on the peer arrive, ``s_barrier_wait -3`` on the + owner wait) -- i.e. the split barrier really replaced the flag spin-wait; + * the global-flag reduction is still compiled in as the runtime fallback + (a per-CTA flag read via ``readFlagAbit`` / ``mubuf`` load survives), so the + param is additive rather than destructive; and + * an order-invariant ``{basename, err}`` syrupy snapshot is pinned. + +CPU-only: no GPU required. The emit harness instantiates rocisa and runs +Python+rocisa codegen without compiling or launching any GPU kernels. +""" + +import os + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_cluster_reduction.yaml", +) + + +def test_streamk_cluster_reduction_gfx1250_emits_assembly(): + """gfx1250 StreamK cluster-reduction config emits real assembly, err==0, + and contains the intra-cluster split-barrier handshake + retained fallback.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + f"Expected all err==0, got: {[(b, e) for b, _s, e in results if e != 0]}" + ) + for base, src, _err in results: + assert src and len(src.splitlines()) > 50, ( + f"Kernel {base!r} emitted suspiciously short source" + ) + assert ".amdgcn_target" in src, f"Kernel {base!r} missing .amdgcn_target" + assert "gfx1250" in src, f"Kernel {base!r} missing gfx1250 target" + assert base.startswith("Cijk_"), f"Kernel {base!r} has unexpected prefix" + # Fast-path cluster split barrier: peer arrive + owner wait (ids = -3). + assert "s_barrier_signal -3" in src, ( + f"Kernel {base!r} missing cluster barrier arrive (s_barrier_signal -3)" + ) + assert "s_barrier_wait -3" in src, ( + f"Kernel {base!r} missing cluster barrier wait (s_barrier_wait -3)" + ) + # Fallback still compiled in: the global-flag reduction path survives + # (per-CTA completion flag reset store to the synchronizer workspace). + assert "reset flag" in src or "set flag" in src, ( + f"Kernel {base!r} dropped the global-flag reduction fallback" + ) + + +def test_streamk_cluster_reduction_gfx1250_golden(snapshot): + """Golden: order-invariant {basename, err} digest of the cluster-reduction emit.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + digest = sorted( + ({"basename": b, "err": e} for (b, _s, e) in results), + key=lambda d: d["basename"], + ) + assert digest == snapshot diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/gpu_test_helpers.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/gpu_test_helpers.py index 8cb5fb1fcc6d..fa5722b60b67 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/gpu_test_helpers.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/gpu_test_helpers.py @@ -87,6 +87,7 @@ def _detect_gfx_target(): # ---- Constants ---- GFX_TARGET = _detect_gfx_target() HAS_GFX950 = GFX_TARGET == "gfx950" +HAS_GFX1250 = GFX_TARGET == "gfx1250" # GPU tests carry two composed marks so the tiering filters work correctly: # - `gpu` : selection marker so `pytest -m "not gpu"` (quick/standard tiers) # deselects these tests at collection time. @@ -112,6 +113,26 @@ def requires_gpu(func): for mark in GPU_MARKS: func = mark(func) return func + + +# gfx1250 gate for the StreamK workgroup-cluster reduction roundtrip. In some +# environments rocm_agent_enumerator reports the physical host arch, so +# GFX_TARGET can be driven via the TENSILE_GPU_TARGET=gfx1250 override (see +# _detect_gfx_target). Kept independent of the gfx950 gate above so both coexist. +GPU_GFX1250_MARKS = [ + pytest.mark.gpu, + pytest.mark.skipif( + hip is None or not HAS_GFX1250, + reason=f"requires hip module and gfx1250 (found hip={'yes' if hip else 'no'}, arch={GFX_TARGET})", + ), +] + + +def requires_gpu_gfx1250(func): + """gfx1250 variant of :func:`requires_gpu`; applies the ``gpu`` marker and skipif.""" + for mark in GPU_GFX1250_MARKS: + func = mark(func) + return func WAVESIZE = 64 NUM_WAVES = 4 NUM_THREADS = WAVESIZE * NUM_WAVES # 256 diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py new file mode 100644 index 000000000000..f4e05278c414 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Unit tests for the StreamK workgroup-cluster reduction fast path (gfx1250). +# +# Covers the cluster split-barrier primitives added to +# Tensile/Components/StreamK.py: +# - clusterReduceSignal : wave-0-elected s_barrier_signal -3 (peer arrive) +# - clusterReduceWait : s_barrier_wait -3 (owner wait) +# - clusterReduceIntraCheck : uniform intra-cluster predicate (no global flag) +# - _streamKClusterReductionEnabled : compile-time gate (param inert when off) +# +# These emit no GPU work by themselves, so the asm string + the gate matrix are +# the contract -- easy to break silently. Mirrors the sibling subtile test +# test_subtile_cluster_barrier.py (gfx1250 rocisa init, wave32, asm-string +# assertions, cap gating via pytest.raises). +# +# Usage: +# pytest test_streamk_cluster_reduction.py -v +################################################################################ + +import os +import shutil +import sys + +import pytest +from types import SimpleNamespace + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +TENSILE_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +sys.path.insert(0, TENSILE_ROOT) + +pytestmark = pytest.mark.unit + +WAVESIZE_32 = 32 + + +def _init_rocisa_gfx1250(): + from rocisa import rocIsa + from Tensile.Common.Architectures import gfxToIsa + ri = rocIsa.getInstance() + isa = gfxToIsa("gfx1250") + asmpath = shutil.which('amdclang++') or '/usr/bin/amdclang++' + ri.init(isa, asmpath) + ri.setKernel(isa, WAVESIZE_32) + + +def _streamk(): + """A StreamK component instance whose cluster helpers are self-contained. + + The helpers only call sibling methods on self and never touch component + construction state, so bypass __init__ with __new__. StreamKTwoTileDPFirst + is the concrete SK3 variant that owns the cluster reduction fast path. + """ + from Tensile.Components.StreamK import StreamKTwoTileDPFirst + return StreamKTwoTileDPFirst.__new__(StreamKTwoTileDPFirst) + + +def _make_writer(has_cluster_barrier=True): + """Minimal StreamK writer stub for the cluster reduction helpers. + + Provides the exact surface the helpers touch: + - sgprPool (real RegisterPool so checkOut returns concrete indices) + - labels.getNameInc unique-label factory + - states.asmCaps capability map + - the StreamK-constant SGPR acquire/release contract with + isStreamKConstantsToVgprEnabled == False (SGPR-resident constants, + the simplest deterministic path). + """ + from rocisa.register import RegisterPool + from rocisa.enum import RegisterType + + counters = {} + + def _getNameInc(base): + n = counters.get(base, 0) + counters[base] = n + 1 + return f"{base}_{n}" + + sgprPool = RegisterPool(0, RegisterType.Sgpr, + defaultPreventOverflow=False, printRP=False) + + writer = SimpleNamespace( + sgprPool=sgprPool, + labels=SimpleNamespace(getNameInc=_getNameInc), + states=SimpleNamespace( + asmCaps={"HasClusterBarrier": has_cluster_barrier}, + skConstVgprs={}, + ), + ) + # SGPR-resident SK constants: acquire returns the symbolic name, release is + # a no-op (mirrors KernelWriter.acquire/releaseStreamKConstSgpr when + # isStreamKConstantsToVgprEnabled is False). + writer.isStreamKConstantsToVgprEnabled = lambda kernel: False + writer.acquireStreamKConstSgpr = lambda kernel, name: name + writer.releaseStreamKConstSgpr = lambda nameOrIdx: None + return writer + + +def _valid_cluster_kernel(C=4): + """A kernel dict that satisfies _streamKClusterReductionEnabled.""" + return { + "StreamKClusterReduction": 1, + "StreamK": 3, + "StreamKFixupTreeReduction": 0, + "StreamKAtomic": 0, + "StreamKForceDPOnly": 0, + "ClusterDim": [C, 1], + } + + +class TestClusterReduceSignal: + """The non-owner/peer cluster arrive half (s_barrier_signal -3).""" + + def test_wave0_election_then_signal(self): + from rocisa.instruction import SCBranchSCC0, SCmpEQU32, VReadfirstlaneB32 + _init_rocisa_gfx1250() + w = _make_writer() + items = _streamk().clusterReduceSignal(w, _valid_cluster_kernel()).flatitems() + # exactly one wave-0 election branch, guarded by one readfirstlane + cmp + assert sum(isinstance(i, SCBranchSCC0) for i in items) == 1 + assert sum(isinstance(i, VReadfirstlaneB32) for i in items) == 1 + assert sum(isinstance(i, SCmpEQU32) for i in items) == 1 + + def test_emits_cluster_signal_id(self): + _init_rocisa_gfx1250() + out = str(_streamk().clusterReduceSignal(_make_writer(), _valid_cluster_kernel())) + assert "s_barrier_signal -3" in out + # the peer arrives at the cluster barrier; it must NOT wait here + assert "s_barrier_wait -3" not in out + + def test_election_ordered_before_signal(self): + _init_rocisa_gfx1250() + out = str(_streamk().clusterReduceSignal(_make_writer(), _valid_cluster_kernel())) + lines = [ln for ln in out.splitlines() if ln.strip()] + + def idx(substr): + return next(i for i, ln in enumerate(lines) if substr in ln) + + readfirstlane = idx("v_readfirstlane_b32") + cmp_ = idx("s_cmp_eq_u32") + branch = idx("s_cbranch_scc0") + signal = idx("s_barrier_signal -3") + # wave election (readfirstlane -> cmp -> branch) precedes the arrive + assert readfirstlane < cmp_ < branch < signal + + def test_no_global_flag_ops_on_fast_path(self): + """The barrier replaces the global-flag handshake: no VMEM flag read or + flag reset store is emitted on the cluster arrive.""" + _init_rocisa_gfx1250() + out = str(_streamk().clusterReduceSignal(_make_writer(), _valid_cluster_kernel())) + assert "buffer_load" not in out + assert "buffer_store" not in out + assert "flag" not in out.lower() + + +class TestClusterReduceWait: + """The owner cluster wait half (s_barrier_wait -3).""" + + def test_emits_cluster_wait_id(self): + _init_rocisa_gfx1250() + out = str(_streamk().clusterReduceWait(_make_writer(), _valid_cluster_kernel())) + assert "s_barrier_wait -3" in out + assert "s_barrier_signal -3" not in out + + def test_wait_has_no_election_branch(self): + """Every wave of the owner WG waits; there is no wave-0 election here.""" + from rocisa.instruction import SCBranchSCC0 + _init_rocisa_gfx1250() + items = _streamk().clusterReduceWait(_make_writer(), _valid_cluster_kernel()).flatitems() + assert sum(isinstance(i, SCBranchSCC0) for i in items) == 0 + + +class TestClusterReduceIntraCheck: + """The uniform intra-cluster predicate (owner and peers compute it alike).""" + + def test_uniform_predicate_shape(self): + """cluster_last = StreamKIdx | (C-1); compared < skGrid; no flag read.""" + from rocisa.instruction import SOrB32, SCmpLtU32 + _init_rocisa_gfx1250() + C = 4 + items = _streamk().clusterReduceIntraCheck( + _make_writer(), _valid_cluster_kernel(C)).flatitems() + assert sum(isinstance(i, SOrB32) for i in items) == 1 + assert sum(isinstance(i, SCmpLtU32) for i in items) == 1 + out = "\n".join(str(i) for i in items) + # the OR mask is C-1 (power-of-two cluster), i.e. 0x3 for C=4 + assert hex(C - 1) in out + # a pure index/grid predicate -- never a global-flag read + assert "buffer_load" not in out + assert "flag" not in out.lower() + + +class TestCapGating: + """HasClusterBarrier is a hard precondition for emitting the -3 barriers.""" + + def test_signal_requires_cluster_barrier_cap(self): + _init_rocisa_gfx1250() + w = _make_writer(has_cluster_barrier=False) + with pytest.raises(AssertionError): + _streamk().clusterReduceSignal(w, _valid_cluster_kernel()) + + def test_wait_requires_cluster_barrier_cap(self): + _init_rocisa_gfx1250() + w = _make_writer(has_cluster_barrier=False) + with pytest.raises(AssertionError): + _streamk().clusterReduceWait(w, _valid_cluster_kernel()) + + +class TestClusterReductionGate: + """_streamKClusterReductionEnabled: the fast path is taken only for the + valid gfx1250 SK3 linear-reduction combo, and the param is inert otherwise + (so the global-flag reduction stays selected as the fallback).""" + + def test_enabled_for_valid_combo(self): + _init_rocisa_gfx1250() + assert _streamk()._streamKClusterReductionEnabled( + _make_writer(), _valid_cluster_kernel()) is True + + def test_disabled_when_param_off(self): + _init_rocisa_gfx1250() + k = _valid_cluster_kernel() + k["StreamKClusterReduction"] = 0 + assert _streamk()._streamKClusterReductionEnabled(_make_writer(), k) is False + + @pytest.mark.parametrize("key,val", [ + ("StreamK", 5), + ("StreamKFixupTreeReduction", 1), + ("StreamKAtomic", 1), + ("StreamKForceDPOnly", 1), + ]) + def test_disabled_for_unsupported_mode(self, key, val): + _init_rocisa_gfx1250() + k = _valid_cluster_kernel() + k[key] = val + assert _streamk()._streamKClusterReductionEnabled(_make_writer(), k) is False + + def test_disabled_without_cluster_barrier_cap(self): + _init_rocisa_gfx1250() + w = _make_writer(has_cluster_barrier=False) + assert _streamk()._streamKClusterReductionEnabled(w, _valid_cluster_kernel()) is False + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction_gpu.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction_gpu.py new file mode 100644 index 000000000000..70fccd251d7b --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction_gpu.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# On-simulator (gfx1250) GPU test for the StreamK workgroup-cluster +# reduction fast-path handshake. +# +# Mirrors the structure of test_gr_lr_roundtrip.py (assemble a small gfx1250 +# kernel, run it on the device, compare to a numpy reference), but exercises the +# *reduction synchronization sequence* the StreamKClusterReduction fast path +# emits in Tensile/Components/StreamK.py, run on a gfx1250 simulation environment: +# +# peer : write partial -> global_wb scope:SCOPE_DEV (release) +# -> wave-0-elected s_barrier_signal -3 (arrive) +# owner: s_barrier_wait -3 (all cluster members arrived) +# -> global_inv scope:SCOPE_DEV (acquire) +# -> accumulate peers' partials (v_add_f32, as +# fixupStep does) +# +# The kernel is assembled with wavefront_size=None (gfx1250 is wave32 and its +# assembler rejects -mwavefrontsize32) and validated for numerical correctness +# of the accumulated reduction, for C in {2, 4}. A SIGALRM watchdog turns a +# barrier deadlock into a test failure instead of an indefinite hang. +# +# Two launch shapes are covered: +# * single workgroup (a degenerate cluster of one WG) -- proves the exact +# store/release/signal/wait/acquire/accumulate sequence runs on the +# simulation environment, computes the right reduction, and does not hang; and +# * C concurrent workgroups, each an independent cluster-of-one over its own +# partial slice -- proves the -3 signal/wait handshake replicated across +# many workgroups completes without deadlock and every workgroup's local +# reduction is correct. +# +# NOTE (scope / binding limitation): the *co-resident* multi-WG cluster launch +# the production host uses (hipDrvLaunchKernelEx + hipLaunchAttributeCluster- +# Dimension, see src/hip/HipSolutionAdapter.cpp) cannot be issued through the +# installed hip-python binding -- it exposes neither the cluster launch +# attribute id nor a clusterDim field on hipLaunchAttributeValue. So the true +# owner-waits-for-C-1-peers co-residency is not exercised here; that requires +# the C++ tensilelite-client cluster launch path. What is proven on-sim is that +# the emitted instruction/fence sequence assembles and executes on gfx1250 +# without hanging and with correct reduction arithmetic. +# +# Usage: +# source +# TENSILE_GPU_TARGET=gfx1250 pytest test_streamk_cluster_reduction_gpu.py -v +################################################################################ + +import os +import signal +import struct +import sys + +import pytest +import numpy as np + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPT_DIR) + +from gpu_test_helpers import ( # noqa: E402 + GFX_TARGET, + assemble_kernel, + run_on_gpu, + requires_gpu_gfx1250, +) + +pytestmark = pytest.mark.unit + +_RUN_TIMEOUT_S = 60 # per-run deadlock watchdog + +WAVESIZE_32 = 32 + + +def _init_rocisa_gfx1250(): + """Initialize the rocIsa singleton for gfx1250 (wave32) so the emitters + render the gfx1250 encodings (s_barrier_signal/-wait -3, v_add_nc_u32, + s_wait_loadcnt/storecnt, global_wb/global_inv scope:SCOPE_DEV).""" + import shutil + from rocisa import rocIsa + from Tensile.Common.Architectures import gfxToIsa + ri = rocIsa.getInstance() + isa = gfxToIsa("gfx1250") + asmpath = shutil.which("amdclang++") or "/usr/bin/amdclang++" + ri.init(isa, asmpath) + ri.setKernel(isa, WAVESIZE_32) + + +class _Timeout: + """SIGALRM watchdog: convert a barrier deadlock into a failure, not a hang.""" + + def __init__(self, seconds, what): + self.seconds = seconds + self.what = what + + def __enter__(self): + def _raise(signum, frame): + raise TimeoutError(f"{self.what} did not complete within {self.seconds}s " + "(possible cluster-barrier deadlock)") + self._old = signal.signal(signal.SIGALRM, _raise) + signal.alarm(self.seconds) + return self + + def __exit__(self, *exc): + signal.alarm(0) + signal.signal(signal.SIGALRM, self._old) + return False + + +def _emit_reduction_module(C): + """Emit the fast-path reduction body (rocisa) for a cluster of size C. + + Returns the assembly text for the kernel body. Uses the same rocisa + emitters and instruction shapes the StreamKClusterReduction fast path uses + (SBarrier cluster signal/wait, global_wb/global_inv SCOPE_DEV fences, + v_add_f32 accumulation), so the on-sim run exercises real gfx1250 encodings. + + Layout over the ``ws`` buffer (base 0, independent of workgroup id): lane + ``l`` (0 <= l < C) publishes partial ``l + 1`` into slot ``l``; after the + cluster handshake every lane accumulates all C slots and lane 0 stores the + sum into ``out[0]``. The published partials are idempotent (every workgroup + writes the identical value into each slot), so launching this kernel over a + grid of C workgroups is a benign race that still exercises the cross-WG + ``s_barrier_signal/-wait -3`` handshake and yields the same verifiable sum + regardless of which workgroup writes last -- deliberately avoiding a + dependence on the workgroup-id SGPR, whose value is unreliable on the + hand-rolled-kernel path. + """ + from rocisa.code import Module + from rocisa.container import vgpr, sgpr + from rocisa.enum import CacheScope + from rocisa.instruction import ( + SLoadB64, SWaitCnt, SCmpEQU32, SCBranchSCC0, + VLShiftLeftB32, VAddU32, VMovB32, VCvtU32toF32, VReadfirstlaneB32, + GlobalStoreB32, GlobalLoadB32, GlobalWb, GlobalInv, VAddF32, + SBarrier, + ) + from rocisa.code import Label + + _init_rocisa_gfx1250() + + m = Module("streamk cluster reduction on-sim body") + # --- kernarg load: s[4:5]=ws_ptr, s[6:7]=out_ptr --- + m.add(SLoadB64(dst=sgpr(4, 2), base=sgpr(0, 2), soffset=0x0, comment="ws_ptr")) + m.add(SLoadB64(dst=sgpr(6, 2), base=sgpr(0, 2), soffset=0x8, comment="out_ptr")) + m.add(SWaitCnt(kmcnt=0, comment="wait kernargs")) + + # --- peer publish: ws[tid] = float(tid + 1) --- + m.add(VLShiftLeftB32(dst=vgpr(1), shiftHex=2, src=vgpr(0), comment="tid*4 -> voffset")) + m.add(VAddU32(dst=vgpr(2), src0=1, src1=vgpr(0), comment="tid + 1 (u32)")) + m.add(VCvtU32toF32(dst=vgpr(2), src=vgpr(2), comment="partial = float(tid + 1)")) + m.add(GlobalStoreB32(vgpr(1), vgpr(2), sgpr(4, 2), comment="publish partial to ws")) + m.add(SWaitCnt(vscnt=0, comment="drain the partial store")) + # release: publish the partial before signalling (SCOPE_DEV, as the fast path keeps) + m.add(GlobalWb(CacheScope.SCOPE_DEV, comment="releaseFence: partial visible")) + + # --- wave-0-elected cluster arrive (s_barrier_signal -3) --- + skip = Label(label="skip_cluster_signal", comment="") + m.add(VReadfirstlaneB32(dst=sgpr(10), src=vgpr(0), comment="wave 0 signals the cluster")) + m.add(SCmpEQU32(src0=sgpr(10), src1=0, comment="check for wave 0")) + m.add(SCBranchSCC0(labelName=skip.getLabelName(), comment="only wave 0 signals")) + m.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) + m.add(skip) + + # --- owner cluster wait (s_barrier_wait -3) + acquire --- + m.add(SBarrier(True, True, True, comment="cluster_barrier wait (all peers arrived)")) + m.add(GlobalInv(CacheScope.SCOPE_DEV, comment="acquireFence: observe peers' partials")) + + # --- accumulate all C partials (fixupStep-style v_add_f32) --- + m.add(VMovB32(dst=vgpr(9), src=0, comment="accumulator = 0")) + for i in range(C): + m.add(VMovB32(dst=vgpr(3), src=i * 4, comment=f"slot {i} byte offset")) + m.add(GlobalLoadB32(vgpr(4), vgpr(3), sgpr(4, 2), comment=f"load peer partial {i}")) + m.add(SWaitCnt(vlcnt=0, comment="wait partial load")) + m.add(VAddF32(dst=vgpr(9), src0=vgpr(9), src1=vgpr(4), comment="accumulate")) + + # --- store the reduction result to out[0] (every lane writes the same sum) --- + m.add(VMovB32(dst=vgpr(5), src=0, comment="out[0] byte offset")) + m.add(GlobalStoreB32(vgpr(5), vgpr(9), sgpr(6, 2), comment="out[0] = sum")) + m.add(SWaitCnt(vscnt=0, comment="drain result store")) + return str(m) + + +def _build_kernel(C): + """Wrap the reduction body in a complete gfx1250 (wave32) AMDHSA kernel.""" + body = _emit_reduction_module(C) + return f"""\ +.amdgcn_target "amdgcn-amd-amdhsa--{GFX_TARGET}" +.text +.protected test_kernel +.globl test_kernel +.p2align 8 +.type test_kernel,@function +.section .rodata,#alloc +.p2align 6 +.amdhsa_kernel test_kernel + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_next_free_vgpr 16 + .amdhsa_next_free_sgpr 16 + .amdhsa_group_segment_fixed_size 0 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 +.end_amdhsa_kernel +.text +test_kernel: +{body} + s_endpgm + +.amdgpu_metadata +--- +amdhsa.version: + - 1 + - 2 +amdhsa.kernels: + - .name: test_kernel + .symbol: 'test_kernel.kd' + .language: OpenCL C + .language_version: + - 2 + - 0 + .args: + - .name: ws + .size: 8 + .offset: 0 + .value_kind: global_buffer + .value_type: f32 + .address_space: global + - .name: out + .size: 8 + .offset: 8 + .value_kind: global_buffer + .value_type: f32 + .address_space: global + .kernarg_segment_size: 16 + .kernarg_segment_align: 8 + .group_segment_fixed_size: 0 + .private_segment_fixed_size: 0 + .wavefront_size: 32 + .sgpr_count: 16 + .vgpr_count: 16 + .max_flat_workgroup_size: 256 +... +.end_amdgpu_metadata +""" + + +def _run_reduction(C, num_wgs, tmp_path, label): + """Assemble + run the cluster-reduction kernel; return the single-slot + reduction result as a numpy float32 scalar. + + ``num_wgs`` workgroups of C lanes each run the -3 handshake over the same + (base 0) ws region; the published partials are identical across workgroups, + so out[0] holds the reduction sum no matter which workgroup writes last.""" + asm = _build_kernel(C) + co_path = str(tmp_path / f"{label}.co") + with open(str(tmp_path / f"{label}.s"), "w") as f: + f.write(asm) + # gfx1250 is wave32: assemble with wavefront_size=None (no -mwavefrontsize32). + assemble_kernel(asm, co_path, wavefront_size=None) + + out_bytes = 4 + # A scratch ws buffer (input) holding the C partial slots, plus a single-slot + # output. run_on_gpu treats the first input as ws_ptr (s[4:5]) and the output + # buffer as out_ptr (s[6:7]). + ws_init = np.zeros(C, dtype=np.float32) + with _Timeout(_RUN_TIMEOUT_S, f"reduction C={C} num_wgs={num_wgs}"): + raw = run_on_gpu( + co_path, out_bytes, inputs=(ws_init,), scalars=(), + num_threads=C, grid=num_wgs, + ) + return np.array(struct.unpack("I", raw), dtype=np.uint32).view(np.float32)[0] + + +@requires_gpu_gfx1250 +class TestStreamKClusterReductionOnSim: + """gfx1250 execution of the cluster reduction fast-path handshake.""" + + @pytest.fixture(params=[2, 4], ids=lambda c: f"C{c}") + def cluster_size(self, request): + return request.param + + def test_single_workgroup_reduction(self, cluster_size, tmp_path): + """One workgroup: the store/release/signal/wait/acquire/accumulate + sequence runs, does not hang, and reduces to sum(l+1 for l in range(C)).""" + C = cluster_size + out = _run_reduction(C, num_wgs=1, tmp_path=tmp_path, label=f"skcls_1wg_C{C}") + expected = float(sum(l + 1 for l in range(C))) # C*(C+1)/2 + assert out == pytest.approx(expected), ( + f"C={C}: cluster reduction = {out}, expected {expected}" + ) + + def test_multi_workgroup_no_deadlock(self, cluster_size, tmp_path): + """C concurrent workgroups run the -3 signal/wait handshake: all + complete (no deadlock/hang under the watchdog) and the reduction is + numerically correct.""" + C = cluster_size + num_wgs = C + out = _run_reduction(C, num_wgs=num_wgs, tmp_path=tmp_path, + label=f"skcls_{num_wgs}wg_C{C}") + expected = float(sum(l + 1 for l in range(C))) + assert out == pytest.approx(expected), ( + f"C={C}, num_wgs={num_wgs}: reduction = {out}, expected {expected}" + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index d0c934c16b54..4d0b939d310d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -141,6 +141,48 @@ def test_auto_enable_from_bare_cluster(self, tmp_path): assert st["StreamKMulticast"] == 1, st.get("StreamKMulticast") assert st["Multicast"] == 1, st["Multicast"] + def test_reduction_keeps_cooperative_loads_off(self, tmp_path): + """Mutual exclusion (reduction wins): adding StreamKClusterReduction=1 to + an otherwise auto-multicast SK3 cluster turns the cooperative loads off + (StreamKMulticast stays 0, Multicast False) instead of auto-enabling.""" + from Tensile import LibraryIO + import yaml + cfg = copy.deepcopy(LibraryIO.read(_STREAMK_CLUSTER_BARE)) + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + fork.append({"StreamKClusterReduction": [1]}) + out = tmp_path / "bare_cluster_reduction.yaml" + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + states = _derive_states(str(out)) + assert states, "expected the SK3 reduction cluster config to derive solutions" + for st in states: + assert not st.get("StreamKMulticast", 0), st.get("StreamKMulticast") + assert st["Multicast"] == 0, st["Multicast"] + + def test_xor_streamk_cluster_reduction(self): + """The mutual-exclusion invariant is enforced at the validator: a state + that (hypothetically) has BOTH StreamKMulticast and StreamKClusterReduction + is rejected. Through config this is now unreachable -- the collapse gives + reduction precedence and leaves the derived StreamKMulticast off (see + test_reduction_keeps_cooperative_loads_off) -- but the validator keeps the + xor as a hard invariant the collapse depends on.""" + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + st = { + "StreamKMulticast": 1, + # StreamKMulticast on always co-derives Multicast on (real invariant). + "Multicast": 1, + "StreamK": 3, + "StreamKClusterReduction": 1, + "StreamKAtomic": 0, + "StreamKXCCMapping": 0, + "ClusterDim": [4, 1], + "ISA": [12, 5, 0], + "TDMInst": 3, + } + isa_map = {(12, 5, 0): type("_I", (), {"asmCaps": {"HasTDM": True, "HasClusterBarrier": True}})()} + assert _validateStreamKMulticast(st, False, isa_map) is False + assert st.get("Valid") is False + def test_reject_multicast_force_off(self, tmp_path): """StreamKMulticast auto-enabled by ClusterDim on SK3 is incompatible with an explicit Multicast=0 (force off): the mask SGPRs are gated on Multicast diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp index 31d28532f252..2e84292aa847 100755 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp @@ -3088,6 +3088,70 @@ namespace TensileLite clusterDim); } }; + + // StreamK cluster-reduction (gfx1250) split-barrier safety. The + // C = ClusterDim[0] peers split a tile's itersPerTile = ceil(K/DepthU) + // K-iterations and hand off through an intra-cluster split barrier; if + // itersPerTile % C != 0 (incl. C > itersPerTile) the arrival counts + // mismatch and the barrier over-signals -> hang. itersPerTile depends + // on runtime K, so this is a per-problem HARD REJECT (surfaced via + // debugEval; not build-time, not a silent fallback). Only the K-split + // reduction path emits it. value[0] = DepthU, value[1] = C. + struct ClusterReductionIterCheck + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + size_t index; + std::array value; + + ClusterReductionIterCheck() = default; + ClusterReductionIterCheck(std::array value) + : value(value) + { + } + + static std::string Type() + { + return "ClusterReductionIterCheck"; + } + + static size_t itersPerTile(ContractionProblemGemm const& problem, int depthU) + { + size_t k = 1; + for(size_t i = 0; i < problem.boundIndices().size(); ++i) + k *= problem.boundSize(i); + if(depthU <= 0) + return k; + return (k + static_cast(depthU) - 1) / static_cast(depthU); + } + + virtual bool operator()(ContractionProblemGemm const& problem) const override + { + int c = value[1]; + if(c <= 1) + return true; + return (itersPerTile(problem, value[0]) % static_cast(c)) == 0; + } + + virtual bool debugEval(ContractionProblemGemm const& problem, + std::ostream& stream) const override + { + int c = value[1]; + size_t ipt = itersPerTile(problem, value[0]); + return debugEvalCmp( + problem, + stream, + "StreamK cluster reduction requires iterations-per-tile ceil(K/DepthU)", + ipt, + "to be a multiple of ClusterDim[0]", + "C", + c); + } + }; } // namespace Contraction /** diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 0eaf7e959d08..b39857a3363a 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -143,6 +143,7 @@ namespace TensileLite int streamK = 0; int streamKForceDPOnly = 0; int streamKAtomic = 0; + int streamKClusterReduction = 0; int streamKMulticast = 0; int prefetchAcrossPersistent = 0; int persistentKernel = 0; diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionPredicates.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionPredicates.hpp index 4af7211d5ba7..4312ef77dbb4 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionPredicates.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionPredicates.hpp @@ -133,7 +133,8 @@ namespace TensileLite Base::template Pair(), Base::template Pair(), Base::template Pair(), - Base::template Pair()}); + Base::template Pair(), + Base::template Pair()}); auto gmap = Generic::GetSubclasses(); rv.insert(gmap.begin(), gmap.end()); @@ -596,6 +597,11 @@ namespace TensileLite : public AutoMappingTraits { }; + template + struct MappingTraits + : public AutoMappingTraits + { + }; } // namespace Serialization } // namespace TensileLite diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp index ccec6cb76835..8eecf3f6855c 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp @@ -106,6 +106,7 @@ namespace TensileLite iot::mapOptional(io, "streamK", s.streamK); iot::mapOptional(io, "streamKForceDPOnly", s.streamKForceDPOnly); iot::mapOptional(io, "streamKAtomic", s.streamKAtomic); + iot::mapOptional(io, "streamKClusterReduction", s.streamKClusterReduction); iot::mapOptional(io, "streamKMulticast", s.streamKMulticast); iot::mapOptional(io, "prefetchAcrossPersistent", s.prefetchAcrossPersistent); iot::mapOptional(io, "persistentKernel", s.persistentKernel); diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 7fa9f504c524..d314f18617fb 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -930,7 +930,32 @@ namespace TensileLite } // Stream-K 3 uses the two-tile ABI. - if(sk.reduction == origami::reduction_t::parallel) + // + // INVARIANT: the cluster-reduction kernarg path (fixed C-way + // split, one tile per cluster) is valid ONLY when the launch grid + // still matches the contract skGrid == C * tiles. getSKGridImpl + // sets that, but a workspace/DP fallback in solve() can re-round + // sk.grid to ceil(tiles/C)*C (a multiple of C, but not C*tiles). + // In that case skItersPerWG = itersPerTile/C no longer matches the + // grid, so DISABLE this path and fall through to the standard SK3 + // accounting (the kernel's intra_cluster runtime guard already + // routes partially-filled clusters to the global-flag reduction). + if(sizeMapping.streamKClusterReduction && sizeMapping.clusterDim.x > 1 + && sk.grid == static_cast(sizeMapping.clusterDim.x) * tiles) + { + // Fixed even split / one-tile-per-cluster: the C = clusterDim.x + // consecutive StreamKIdx of one cluster (its contiguous + // WorkGroup0 range [c*C, c*C + C)) are exactly the fixup peers of + // tile c; each peer runs itersPerTile/C iterations (skSplit == C) + // and skTiles == tiles (every tile split C ways). + uint32_t c = static_cast(sizeMapping.clusterDim.x); + uint32_t skItersPerWG = static_cast(itersPerTile) / c; + + args.template append("SKItersPerWG", skItersPerWG); + args.template append("skGrid", sk.grid); + args.template append("skTiles", static_cast(tiles)); + } + else if(sk.reduction == origami::reduction_t::parallel) { uint32_t skSplit = sk.grid / tiles; // skTiles is skSplit in parallel reduction path @@ -3194,12 +3219,13 @@ namespace TensileLite // Keep the StreamK cluster launch grid a multiple of the cluster size C // even if a workspace/DP fallback above changed sk.grid, so the clustered // launch (gridDimX % C == 0) stays valid. In the common case sk.grid is - // already ceil(tiles/C)*C (multicast) -- a multiple of C -- so this is a - // no-op; when a fallback fired (e.g. sk.grid = tiles), per-tile - // correctness on any partially-filled cluster is handled by the kernel's - // clusterMulticastValid runtime guard + the normal (non-cooperative) load - // fallback. - if(sizeMapping.streamKMulticast && sizeMapping.clusterDim.x > 1) + // already a multiple of C -- ceil(tiles/C)*C for multicast, C*tiles for + // cluster reduction -- so this is a no-op; when a fallback fired (e.g. + // sk.grid = tiles), per-tile correctness on any partially-filled cluster + // is handled by the kernel's runtime guard (clusterMulticastValid for + // multicast, intra_cluster for cluster reduction) + global-flag fallback. + if((sizeMapping.streamKMulticast || sizeMapping.streamKClusterReduction) + && sizeMapping.clusterDim.x > 1) { size_t c = sizeMapping.clusterDim.x; sk.grid = ((sk.grid + c - 1) / c) * c; @@ -4093,6 +4119,23 @@ namespace TensileLite skGrid = ((tiles + c - 1) / c) * c; } + // StreamKClusterReduction (gfx1250): fixed even split / one-tile-per-cluster + // (design docs/design/streamk-wg-clusters.md, section 2.3). Each of the + // `tiles` output tiles is owned by exactly one cluster of C = clusterDim.x + // peer WGs, so the launched grid is C * tiles. The HW WG-id remap gives + // WorkGroup0 = cluster_x*C + wg_x, so the C WGs of cluster c occupy the + // contiguous StreamK index range [c*C, c*C + C) -- exactly the consecutive + // fixup peers of tile c. C * tiles is inherently a multiple of C, satisfying + // the clustered-launch requirement that gridDimX be a multiple of the cluster + // size (see HipSolutionAdapter cluster launch). This override intentionally + // supersedes the grid-selection heuristics above so the alignment invariant + // always holds; residual/partial clusters fall back to the global-flag path + // via the kernel's intra_cluster runtime guard. + if(self.sizeMapping.streamKClusterReduction && self.sizeMapping.clusterDim.x > 1) + { + skGrid = static_cast(self.sizeMapping.clusterDim.x) * tiles; + } + return skGrid; } } // namespace diff --git a/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp b/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp index e31388ecf4c7..1776bb0ed12f 100644 --- a/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp @@ -127,3 +127,111 @@ TEST(Predicates, WorkgroupMappingXCCCheck_FallbackTreatsXCCAs1) problem.setParams().setFallbackStatus(true); EXPECT_TRUE((*pred)(problem)) << "With fallback status, effective XCC=1 so 38 % 1 == 0"; } + +// ---------------------------------------------------------------------------- +// ClusterReductionIterCheck: StreamK cluster-reduction split-barrier safety. +// The C cluster peers split a tile's itersPerTile = ceil(K / +// DepthU) K-iterations; the split barrier over-signals unless itersPerTile % C +// == 0. When itersPerTile % C != 0 the predicate is a HARD REJECT of the +// cluster-reduction solution (deliberate, user-visible via debugEval -- NOT a +// silent fallback): the "must be a multiple of the cluster size" limitation. +// K-tail (K % DepthU != 0) is safe on its own. value = {DepthU, C}. +// Problems set K via ContractionProblemGemm::GEMM(..., k, ...). +// ---------------------------------------------------------------------------- + +namespace +{ + TensileLite::ContractionProblemGemm gemmWithK(size_t k) + { + using namespace TensileLite; + // TN like the MX-FP8 cluster-reduction configs; only K matters here. + return ContractionProblemGemm::GEMM(true, false, 256, 256, k, 256, 256, 256, 1.0, false, 1); + } +} + +TEST(Predicates, ClusterReductionIterCheck_EvenSplit_NoTail_Passes) +{ + using namespace TensileLite; + // DepthU=256, C=4, K=2048 -> itersPerTile=8, 8 % 4 == 0 -> safe. + auto pred = std::make_shared( + std::array{256, 4}); + EXPECT_TRUE((*pred)(gemmWithK(2048))) << "itersPerTile=8, 8 % 4 == 0 should pass"; +} + +TEST(Predicates, ClusterReductionIterCheck_EvenSplit_WithKTail_Passes) +{ + using namespace TensileLite; + // DepthU=256, C=4, K=1920 -> itersPerTile=ceil(1920/256)=8, 8 % 4 == 0. + // K % 256 = 128 (K-tail) but that alone is SAFE (verified on gfx1250 sim). + auto pred = std::make_shared( + std::array{256, 4}); + EXPECT_TRUE((*pred)(gemmWithK(1920))) + << "itersPerTile=8, 8 % 4 == 0 must pass even with a K-tail"; +} + +TEST(Predicates, ClusterReductionIterCheck_CGreaterThanItersPerTile_Rejects) +{ + using namespace TensileLite; + // DepthU=256, C=8, K=1024 -> itersPerTile=4, 4 % 8 == 4 != 0 -> unsafe + // (C > itersPerTile). Would over-signal the split barrier -> HARD REJECT. + auto pred = std::make_shared( + std::array{256, 8}); + EXPECT_FALSE((*pred)(gemmWithK(1024))) + << "C=8 > itersPerTile=4 (4 % 8 != 0) must be rejected (not a fallback)"; +} + +TEST(Predicates, ClusterReductionIterCheck_UnevenSplit_NoTail_Rejects) +{ + using namespace TensileLite; + // DepthU=256, C=4, K=1536 (=6*256, no K-tail) -> itersPerTile=6, 6 % 4 == 2 + // != 0 -> unsafe even without a K-tail (verified on gfx1250 sim). + auto pred = std::make_shared( + std::array{256, 4}); + EXPECT_FALSE((*pred)(gemmWithK(1536))) + << "itersPerTile=6, 6 % 4 != 0 must be rejected (no K-tail involved)"; +} + +TEST(Predicates, ClusterReductionIterCheck_UnevenSplit_WithKTail_Rejects) +{ + using namespace TensileLite; + // DepthU=256, C=2, K=1056 -> itersPerTile=ceil(1056/256)=5, 5 % 2 == 1 != 0. + auto pred = std::make_shared( + std::array{256, 2}); + EXPECT_FALSE((*pred)(gemmWithK(1056))) << "itersPerTile=5, 5 % 2 != 0 must be rejected"; +} + +// Focused hard-reject test: the rejection is user-visible and its diagnostic +// message names the "multiple of the cluster size" limitation (debugEval output +// is what the solution selector surfaces on the DID_NOT_SATISFY_ASSERTS path). +TEST(Predicates, ClusterReductionIterCheck_RejectMessageNamesLimitation) +{ + using namespace TensileLite; + // DepthU=256, C=4, K=1024 -> itersPerTile=4, 4 % 4 == 0 -> the safe case + // (no message emitted for a passing predicate in non-verbose mode) is not + // what we assert here; instead take a non-conforming case: + // DepthU=256, C=4, K=1280 -> itersPerTile=5, 5 % 4 != 0 -> reject. + auto pred = std::make_shared( + std::array{256, 4}); + auto problem = gemmWithK(1280); + std::ostringstream oss; + bool rv = pred->debugEval(problem, oss); + EXPECT_FALSE(rv) << "itersPerTile=5, 5 % 4 != 0 must be rejected"; + const std::string msg = oss.str(); + EXPECT_NE(msg.find("iterations-per-tile ceil(K/DepthU)"), std::string::npos) + << "reject diagnostic must name iterations-per-tile; got: " << msg; + EXPECT_NE(msg.find("multiple of ClusterDim[0]"), std::string::npos) + << "reject diagnostic must name the multiple-of-cluster-size limitation; got: " << msg; + EXPECT_NE(msg.find("[!!]"), std::string::npos) + << "reject diagnostic must mark the predicate as failing; got: " << msg; +} + +TEST(Predicates, ClusterReductionIterCheck_NoCluster_AlwaysPasses) +{ + using namespace TensileLite; + // C <= 1 -> predicate inert (non-cluster / multicast never emits it, but + // guard defensively): any K passes. + auto pred = std::make_shared( + std::array{256, 1}); + EXPECT_TRUE((*pred)(gemmWithK(1056))) << "C=1 must always pass (guard inert)"; + EXPECT_TRUE((*pred)(gemmWithK(1024))) << "C=1 must always pass (guard inert)"; +} From 123fe0914376b307a6e159b403cdea52d511ddf4 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 21 Jul 2026 16:10:32 +0000 Subject: [PATCH 04/24] fix(tensilelite): consume StreamKMulticast prologue cluster arrive on the zero-iteration skip path The StreamKMulticast prologue emits a wave-0-elected cluster-scope arrive before the first cooperative-multicast load. Its only matching cluster-scope wait is the first-load wait, which sits after the last-iteration guard's long branch to PrefetchGlobalLastIterEnd. On the zero-full-iteration path (reachable when K is not a whole multiple of DepthU) that branch skips the first-load wait, leaving the prologue arrive unmatched and the cluster-scope barrier unbalanced on that edge. Add a matching cluster wait on the skip edge (guarded by StreamKMulticast) so the arrive is consumed on every control-flow path, preserving whole-cluster barrier symmetry. The wait branches over on scc0 (>=1 full iteration, where the first-load wait pairs the arrive) and executes an all-waves cluster wait on scc1 (zero iterations); it leaves scc intact so the subsequent long branch is unaffected. Update the gfx1250 cluster char-test balance checks: the skip-edge wait and the first-load wait are mutually exclusive at runtime but both emitted statically, so the static cluster-wait count is now exactly one greater than the signal count (one prologue arrive consumed by exactly one of the two waits). --- .../tensilelite/Tensile/Components/StreamK.py | 36 +++++++++++++++++++ .../Tensile/KernelWriterAssembly.py | 9 +++++ ..._streamk_cluster_coop_load_gfx1250_char.py | 20 +++++++---- ..._streamk_cluster_multicast_gfx1250_char.py | 20 +++++++---- 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 2ca099919e01..c0668ea32f91 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -2753,6 +2753,42 @@ def streamKMulticastPrologueSignal(self, writer, kernel): writer.sgprPool.checkIn(elect) return module + def streamKMulticastZeroIterClusterWait(self, writer, kernel): + """Consume the prologue cluster arrive on the zero-iteration skip path. + + The prologue arrive (``streamKMulticastPrologueSignal``) fires once per + cluster peer, uniformly, before the first cooperative-multicast load. + Its only matching cluster-scope ``s_barrier_wait -3`` is the pass's + first-load wait, which sits *after* the last-iteration guard + (``checkLastIter`` -> long-branch to ``PrefetchGlobalLastIterEnd``). On + the zero-full-iteration path (a participating workgroup that runs only + the tail/fixup, reachable when K is not a whole multiple of DepthU) that + long branch skips the first-load wait, leaving the arrive unmatched and + the cluster-scope barrier unbalanced on that edge. + + Emit the matching cluster wait on that skip edge so every cluster peer + executes exactly one arrive and exactly one wait on every control-flow + path out of the prologue. ``checkLastIter`` has set scc (scc1 == + numIterL == 0). Branch over the wait on scc0 (>=1 full iteration -> the + first-load wait pairs the arrive); on scc1 emit the all-waves cluster + wait (mirroring the all-waves first-load wait so the arrive is consumed + exactly once). The wait leaves scc intact, so the standard scc1 long + branch that follows still takes the skip edge. Inert unless + StreamKMulticast. + """ + module = Module("StreamK multicast zero-iteration cluster wait") + if not kernel.get("StreamKMulticast", 0): + return module + assert writer.states.asmCaps.get("HasClusterBarrier", False), \ + "StreamKMulticast requires the HasClusterBarrier asm capability" + module.addComment0("StreamKMulticast: zero-iteration skip path consumes the prologue cluster arrive (pairs prologue arrive)") + skipWait = Label(label=writer.labels.getNameInc("SKMC_SkipZeroIterClusterWait"), comment="") + module.add(SCBranchSCC0(labelName=skipWait.getLabelName(), + comment=">=1 full iteration: the first-load cluster wait pairs the arrive")) + module.add(SBarrier(True, True, True, comment="cluster_barrier wait")) + module.add(skipWait) + return module + def preLoop(self, writer, kernel): module = Module("StreamK TwoTileDPFirst openLoop") skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py index 68be1a5bd357..57810e3884dc 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py @@ -9694,6 +9694,15 @@ def openSumAtLeastUnroll(self, kernel, prefetch, isOptNLL, isNGLL=False, NLLinde if self.isPrefetchAcrossPersistentEnabled(kernel): module.add(SCMovB32(dst=sgpr("SkPrefetchPrimed"), src=0, comment="discard primed PAP group when current slice skips NLL")) + # StreamKMulticast: the long branch below skips the pass's first-load + # cluster wait on the zero-iteration path; emit the matching + # cluster-scope wait on that skip edge so the prologue cluster arrive + # is consumed on every control-flow path (whole-cluster barrier + # symmetry). scc (from checkLastIter) is preserved for the branch + # below. No-op unless StreamKMulticast. + if kernel.get("StreamKMulticast", 0): + skComponent = Component.StreamK.find(self) + module.add(skComponent.streamKMulticastZeroIterClusterWait(self, kernel)) # use positive offset only long jump with self.allocTmpSgpr(3, tag="openSumAtLeastUnroll_tmpSgprInfo") as tmpSgprInfo: module.add(self.longBranchScc1(lastIterEnd, posNeg=1, tmpSgprInfo=tmpSgprInfo)) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py index 905c825d8acc..cbfc0f6167f9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py @@ -92,13 +92,19 @@ def test_streamk_cluster_coop_load_gfx1250_emits_assembly(): assert "s_barrier_wait -3" in src, ( f"Kernel {base!r}: missing cluster-scope barrier wait (-3)" ) - # The cluster-scope split barrier must be balanced: every arrive - # (s_barrier_signal -3) is matched by a completion (s_barrier_wait -3). - # The prologue wave-0 arrive pairs the pass's first-load wait; an - # imbalance (unpaired wait) deadlocks all cluster waves on HW. + # Cluster-scope split-barrier balance. Every arrive + # (s_barrier_signal -3) must be consumed by a completion + # (s_barrier_wait -3) on every control-flow path. The prologue wave-0 + # arrive is consumed by exactly one of two mutually exclusive cluster + # waits: the last-iteration guard's zero-iteration skip-edge wait, or + # the first-load wait on the >=1-iteration fall-through. Both waits are + # emitted statically but only one executes on any given path, so the + # static wait count is exactly one greater than the signal count; every + # other arrive is a self-contained arrive/wait pair. Any other imbalance + # would leave a cluster wait unpaired and stall the cluster waves. n_signal = src.count("s_barrier_signal -3") n_wait = src.count("s_barrier_wait -3") - assert n_signal == n_wait, ( - f"Kernel {base!r}: imbalanced cluster barrier: " - f"{n_signal} signal(-3) vs {n_wait} wait(-3)" + assert n_wait == n_signal + 1, ( + f"Kernel {base!r}: unexpected cluster barrier balance: " + f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" ) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py index 6ee34d06fdc2..de80f48908df 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py @@ -89,15 +89,21 @@ def test_streamk_cluster_multicast_gfx1250_emits_assembly(): assert "s_barrier_wait -3" in src, ( f"Kernel {base!r} missing cluster-scope barrier wait (-3)" ) - # The cluster-scope split barrier must be balanced: every arrive - # (s_barrier_signal -3) is matched by a completion (s_barrier_wait -3). - # The prologue wave-0 arrive pairs the pass's first-load wait; an - # imbalance (unpaired wait) deadlocks all cluster waves on HW. + # Cluster-scope split-barrier balance. Every arrive + # (s_barrier_signal -3) must be consumed by a completion + # (s_barrier_wait -3) on every control-flow path. The prologue wave-0 + # arrive is consumed by exactly one of two mutually exclusive cluster + # waits: the last-iteration guard's zero-iteration skip-edge wait, or + # the first-load wait on the >=1-iteration fall-through. Both waits are + # emitted statically but only one executes on any given path, so the + # static wait count is exactly one greater than the signal count; every + # other arrive is a self-contained arrive/wait pair. Any other imbalance + # would leave a cluster wait unpaired and stall the cluster waves. n_signal = src.count("s_barrier_signal -3") n_wait = src.count("s_barrier_wait -3") - assert n_signal == n_wait, ( - f"Kernel {base!r} has imbalanced cluster barrier: " - f"{n_signal} signal(-3) vs {n_wait} wait(-3)" + assert n_wait == n_signal + 1, ( + f"Kernel {base!r} has unexpected cluster barrier balance: " + f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" ) From 9074f06778614c09aa6ad713fdfc772fe100df58 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 02:05:01 +0000 Subject: [PATCH 05/24] feat(gfx1250): enable + fix PrefetchGlobalRead=2 for StreamK cluster multicast - Option A prologue prefetch handshake bracketed with a cluster-scope barrier round (StreamK.py / KernelWriter.py StreamKMulticast plumb). - Remove the PGR<=1 enumeration guard so PGR2 multicast solutions build. - InsertClusterBarrierPass: drain the cooperative broadcast tensor group with both a pre-round drain and a producer-side drain (after the cooperative tensor_load group), gated on StreamKMulticast && PGR>=2. - PrefetchGlobalRead: [1,2] in the mxf4/mxf8 cluster-multicast configs plus a new PGR2 characterization test (+ snapshot + designed yaml). GPU-validated on gfx1250. JIRA ID : N/A --- .../tensilelite/Tensile/Components/StreamK.py | 37 ++++++ .../tensilelite/Tensile/KernelWriter.py | 15 +++ .../Tensile/SolutionStructs/Solution.py | 7 - .../core/sk_mxf4gemm_cluster_multicast.yaml | 4 +- .../core/sk_mxf8gemm_cluster_multicast.yaml | 8 +- ...k_cluster_multicast_pgr2_gfx1250_char.ambr | 21 +++ .../streamk_cluster_multicast_pgr2.yaml | 102 +++++++++++++++ ...amk_cluster_multicast_pgr2_gfx1250_char.py | 120 ++++++++++++++++++ .../Tests/unit/test_streamk_multicast.py | 18 +-- .../stinkytofu/bindings/python/Module.hpp | 1 + .../asm/InsertClusterBarrierPass.hpp | 5 +- .../src/pipeline/backend/Gfx1250Backend.cpp | 8 +- .../asm/InsertClusterBarrierPass.cpp | 71 ++++++++++- 13 files changed, 388 insertions(+), 29 deletions(-) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_pgr2_gfx1250_char.ambr create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index c0668ea32f91..5766863d3d9d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -2753,6 +2753,43 @@ def streamKMulticastPrologueSignal(self, writer, kernel): writer.sgprPool.checkIn(elect) return module + def streamKMulticastProloguePrefetchHandshake(self, writer, kernel): + """Bracket the prologue double-buffer prefetch multicast load with a + cluster-scope handshake. + + The double-buffer ("LDS1") prefetch load in the PrefetchGlobalRead >= 2 + prologue is emitted inside the single-iteration guard branch, so the + generic per-load bracketing does not reach it (the guard branch is a + segment boundary that the backward anchor scan stops at). On the + StreamKMulticast path every cooperative-multicast tensor_load_to_lds + must be immediately preceded by a cluster-scope arrive/wait handshake; + emit a self-contained one here (one wave arrives, all waves wait) so the + prefetch load is synchronized and the cluster-scope signal/wait counts + stay balanced. + + The guard branches on LoopCounterL, which is uniform across the + co-located cluster peers (they process M-adjacent tiles sharing the same + K), so all peers execute this handshake in lockstep. Only the wave-0 + election gates the arrive, matching the existing prologue-signal idiom. + Inert unless StreamKMulticast. + """ + module = Module("StreamK multicast prologue prefetch cluster handshake") + if not kernel.get("StreamKMulticast", 0): + return module + assert writer.states.asmCaps.get("HasClusterBarrier", False), \ + "StreamKMulticast requires the HasClusterBarrier asm capability" + module.addComment0("StreamKMulticast: bracket prologue double-buffer prefetch load with cluster handshake") + skipSignal = Label(label=writer.labels.getNameInc("SKMC_SkipPrefetchSignal"), comment="") + elect = writer.sgprPool.checkOut(1, "SKMulticastPrefetchElect") + module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) + module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) + module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) + module.add(skipSignal) + module.add(SBarrier(True, True, True, comment="cluster_barrier wait")) + writer.sgprPool.checkIn(elect) + return module + def streamKMulticastZeroIterClusterWait(self, writer, kernel): """Consume the prologue cluster arrive on the zero-iteration skip path. diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index 5af51d241507..10ae8c7c1a71 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -5466,6 +5466,15 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): if kernel["PrefetchGlobalRead"] >= 2: for idxPgr in range(1, kernel["PrefetchGlobalRead"]): module.add(self.openPrefetchGlobalRead2orMore(kernel, idxPgr)) + # StreamKMulticast: the cooperative-multicast loads emitted below for + # this prefetch stage sit inside the single-iteration guard branch, + # past the generic per-load cluster-barrier bracketing boundary. + # Bracket them with a self-contained cluster-scope handshake so every + # multicast load stays synchronized and signal/wait counts stay + # balanced. Gated on StreamKMulticast (only ever set on the StreamK=3 + # component), so the emitted code is unchanged for every other path. + if kernel.get("StreamKMulticast", 0): + module.add(skComponent.streamKMulticastProloguePrefetchHandshake(self, kernel)) # For UnrollLoopSwapGlobalReadOrder, we also need to swap ds write A/B order. # In scheduling, we always schedule lwa first then lwb second, # Putting lwb in lwa's code object can easily change the order. @@ -6673,6 +6682,12 @@ def _restoreNtabState(): # Cluster-barrier handshake insertion in Gfx1250Backend # (kernel-scope at every OptLevel when set). "ClusterBarrier": bool(kernel.get("ClusterBarrier", False)), + # StreamKMulticast gates the per-iteration cooperative-broadcast + # drain in InsertClusterBarrierPass Rule 4 (mainloop) mode c: with + # PGR>=2 an `s_wait_tensorcnt 0` is emitted before the cluster-scope + # `s_barrier_signal -3` arrive so the broadcast retires before peers + # re-enter the next round. Defaults off; no-op for every other kernel. + "StreamKMulticast": bool(kernel.get("StreamKMulticast", 0)), # PrefetchGlobalRead (PGR) passed to InsertClusterBarrierPass. # Gates Rule 3 (`LCL <= PGR` skip) and Rule 4 (`LCL == PGR+1` # skip in fresh-gate mode; inherits upstream `LCL == PGR` cmp diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 69bbc1e635bd..fe80bb46e136 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -375,13 +375,6 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): "StreamKMulticast is not supported with StreamKAtomic") return False - # The DP cooperative multicast path currently supports single-buffered global - # prefetch only. - if state["PrefetchGlobalRead"] > 1: - reject(state, printRejectionReason, - "StreamKMulticast requires PrefetchGlobalRead <= 1") - return False - # StreamKXCCMapping remap is bypassed under clustering and XCC=3 overflows the # SGPR budget alongside the cluster coords; require the default (no remap). if state["StreamKXCCMapping"] != 0: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml index 3901b93a208e..4c8020a73a89 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml @@ -28,7 +28,7 @@ # pairs whose nWG0 is not a multiple of the cluster C are simply rejected. # # Codegen coverage is broad: the MatrixInstruction list spans macro-tile-M in -# {32,64,128,256}, crossed with LDSTrInst {True,False}, at PrefetchGlobalRead 1. +# {32,64,128,256}, crossed with LDSTrInst {True,False}, PrefetchGlobalRead {1,2}. # ScheduleIterAlg is pinned to 4 (the StinkyTofu-scheduled gfx1250 path) -- SIA3 is # intentionally excluded so the tests exercise the real gfx1250 scheduler. The small # sizes exercise the compact tiles at @@ -103,7 +103,7 @@ BenchmarkProblems: - LDSTrInst: [True, False] - 1LDSBuffer: [0] - DirectToVgprSparseMetadata: [False] - - PrefetchGlobalRead: [1] + - PrefetchGlobalRead: [1, 2] - PrefetchLocalRead: [1] - ScheduleIterAlg: [4] - SourceSwap: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml index 9277a5b675d7..0d8afd112807 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml @@ -22,6 +22,12 @@ # up to a multiple of C so every launched cluster is full. Every size below keeps # nWG0 % C == 0 for the C it dispatches (a violation is a HARD REJECT via # ClusterDimCheck at selection time, not a silent fallback). +# +# Codegen coverage crosses macro-tile-M in {32,64,128} with PrefetchGlobalRead +# {1,2}. ScheduleIterAlg is pinned to 4 (the gfx1250 scheduler path). With +# PrefetchGlobalRead=2 the prologue double-buffers the first main-loop prefetch, +# so the sizes with K > DepthU (DepthU=256; e.g. K=512 and the K=1920 tail case) +# ensure that double-buffered prologue prefetch path is actually exercised. TestParameters: marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch GlobalParameters: @@ -84,7 +90,7 @@ BenchmarkProblems: - LdsBlockSizePerPadMXSB: [-1] - LdsPadMetadata: [0] - LDSTrInst: [False] - - PrefetchGlobalRead: [1] + - PrefetchGlobalRead: [1, 2] - PrefetchLocalRead: [1] - ScheduleIterAlg: [4] - SourceSwap: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_pgr2_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_pgr2_gfx1250_char.ambr new file mode 100644 index 000000000000..b63a612e4882 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_multicast_pgr2_gfx1250_char.ambr @@ -0,0 +1,21 @@ +# serializer version: 1 +# name: test_streamk_cluster_multicast_pgr2_gfx1250_golden + list([ + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg5tRpHXBjULkMIXSNVBHJHFmD1t3k0UwLm2yJZP4oexA=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgGNWGjVlzVS5wHYrKb7__g3mxK0KG3m4i23NRg-javuY=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgRlD1yh-2ZySyGsxGWXEw0e-4y6cBU5fJKfN-NANMhEA=', + 'err': 0, + }), + dict({ + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgSiq6ezRM6zYVfmOWJNJGQ9FXlCFA4RsbFcRyfLeirjs=', + 'err': 0, + }), + ]) +# --- diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml new file mode 100644 index 000000000000..f175132a4797 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml @@ -0,0 +1,102 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Designed StreamK + DP cooperative multicast config for gfx1250 codegen +# characterization -- double-buffered global prefetch (PrefetchGlobalRead=2). +# +# Companion to _designed/gfx1250/streamk_cluster_multicast.yaml. That config +# pins PrefetchGlobalRead=1 with K == DepthU (single main-loop iteration); this +# one sets PrefetchGlobalRead=2 with K > DepthU (>= 2 main-loop iterations) so +# the prologue emits the second, double-buffered ("LDS1") cooperative multicast +# prefetch load. That prefetch load is emitted inside the single-iteration guard +# branch, so it must be bracketed by its own cluster-scope split-barrier +# handshake (s_barrier_signal / s_barrier_wait -3) emitted by +# StreamK.streamKMulticastProloguePrefetchHandshake, keeping every cooperative +# multicast load synchronized and the cluster-scope signal/wait counts balanced. +# See docs/design/cluster-load-component-and-streamk-multicast.md (section 4.4). +# +# - ClusterDim: [[4, 1]] -- 1-D spatial cluster, C=4 (power of two) +# - PrefetchGlobalRead: [2] -- double-buffered prologue prefetch +# - ProblemSizes K = 512 > DepthU 256 -> >= 2 main-loop iterations +# +# Coverage mirrors the sibling multicast config's MatrixInstruction sweep +# (MacroTile-M in {32,64,128,256}), all at PrefetchGlobalRead=2 => 4 kernels. +GlobalParameters: + SyncsPerBenchmark: 0 + MinimumRequiredVersion: 5.0.0 + NumElementsToValidate: 128 + DataInitTypeBeta: 0 + DataInitTypeAlpha: 1 + Device: 0 + +BenchmarkProblems: + ######################################## + # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. + ######################################## + - + - # ProblemType: MX-F8 TN + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - # BenchmarkProblemSizeGroup + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - ClusterDim: [[4, 1]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py new file mode 100644 index 000000000000..19bdd856b789 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py @@ -0,0 +1,120 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +"""StreamK DP cooperative B-multicast, double-buffered prologue prefetch +(PrefetchGlobalRead=2) -- gfx1250 characterization (CPU-only). + +Companion to ``test_streamk_cluster_multicast_gfx1250_char.py`` (which pins +PrefetchGlobalRead=1). This exercises the ``PrefetchGlobalRead=2`` path with +K > DepthU, so the prologue emits the second, double-buffered ("LDS1") +cooperative multicast prefetch load. That prefetch load is emitted inside the +single-iteration guard branch, past the generic per-load cluster-barrier +bracketing boundary, so ``StreamK.streamKMulticastProloguePrefetchHandshake`` +must bracket it with a dedicated cluster-scope split-barrier handshake. + +Asserts, in addition to the sibling config's split-mask / predicate checks: + * the double-buffered prologue prefetch load is bracketed by a wave-0-elected + cluster-scope ``s_barrier_signal -3`` + ``s_barrier_wait -3`` handshake + (label ``SKMC_SkipPrefetchSignal``); and + * the cluster-scope split barrier stays balanced: every ``s_barrier_signal -3`` + is matched by an ``s_barrier_wait -3``. + +CPU-only: no GPU required. +""" + +import os + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_cluster_multicast_pgr2.yaml", +) + + +def _skip_prefetch_handshake_brackets_load(src): + """Return True iff the prologue prefetch (LDS1) multicast load is bracketed. + + The double-buffered prologue prefetch load sits in the ``skipPGR2`` guard + segment. The dedicated handshake elects wave 0 (branch to + ``SKMC_SkipPrefetchSignal``), signals ``-3``, then all waves wait ``-3`` + immediately before the LDS1 ``tensor_load_to_lds`` group. + """ + lines = src.splitlines() + for i, ln in enumerate(lines): + if "label_SKMC_SkipPrefetchSignal:" not in ln: + continue + # A cluster-scope wait must follow the skip label, before the LDS1 load. + window = lines[i : i + 6] + has_wait = any("s_barrier_wait -3" in w for w in window) + has_load = any("tensor_load_to_lds" in w for w in window) + # A wave-0 signal must precede the skip label. + pre = lines[max(0, i - 4) : i] + has_signal = any("s_barrier_signal -3" in p for p in pre) + if has_wait and has_load and has_signal: + return True + return False + + +def test_streamk_cluster_multicast_pgr2_gfx1250_emits_assembly(): + """gfx1250 SK3 cluster multicast config at PrefetchGlobalRead=2 (K>DepthU) + emits real assembly with err==0, and the double-buffered prologue prefetch + multicast load is bracketed by a balanced cluster-scope -3 handshake.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + f"Expected all err==0, got: {[(b, e) for b, _s, e in results if e != 0]}" + ) + for base, src, _err in results: + assert ".amdgcn_target" in src, f"Kernel {base!r} missing .amdgcn_target" + assert "gfx1250" in src, f"Kernel {base!r} missing gfx1250 target" + # Split A/B multicast: B broadcast, A self-only. + assert "s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in src, ( + f"Kernel {base!r} missing B-broadcast mask on the B descriptor" + ) + # The PGR>=2 prologue double-buffer prefetch region must exist for K>DepthU. + assert "skipPGR2" in src, ( + f"Kernel {base!r} missing the PGR2 prologue double-buffer region" + ) + # The double-buffered prologue prefetch multicast load must be bracketed + # by a dedicated cluster-scope handshake (the fix under test). + assert _skip_prefetch_handshake_brackets_load(src), ( + f"Kernel {base!r} PGR2 prologue prefetch load is NOT bracketed by a " + f"cluster-scope -3 handshake" + ) + # Cluster-scope split-barrier balance. The prologue wave-0 arrive + # (s_barrier_signal -3) is consumed by exactly one of two mutually + # exclusive cluster waits: the last-iteration guard's zero-iteration + # skip-edge wait, or the first-load wait on the >=1-iteration + # fall-through. Both are emitted statically but only one executes on any + # control-flow path, so the static wait count is exactly one greater + # than the signal count; every other arrive (including this config's + # dedicated prologue-prefetch handshake) is a self-contained arrive/wait + # pair. Any other imbalance would leave a cluster wait unpaired and + # stall the cluster waves. + n_signal = src.count("s_barrier_signal -3") + n_wait = src.count("s_barrier_wait -3") + assert n_wait == n_signal + 1, ( + f"Kernel {base!r} has unexpected cluster barrier balance: " + f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" + ) + + +def test_streamk_cluster_multicast_pgr2_gfx1250_golden(snapshot): + """Golden: order-invariant {basename, err} digest of the PGR2 multicast emit.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + digest = sorted( + ({"basename": b, "err": e} for (b, _s, e) in results), + key=lambda d: d["basename"], + ) + assert digest == snapshot diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index 4d0b939d310d..a1eede7632e9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -209,10 +209,11 @@ def test_reject_atomic(self, tmp_path): fork_overrides={"StreamKAtomic": [1]}) assert _derive_states(cfg) == [] - def test_reject_pgr_gt1(self, tmp_path): - """The DP cooperative multicast path currently supports single-buffered - global prefetch only, so PrefetchGlobalRead > 1 is rejected while - PrefetchGlobalRead <= 1 stays accepted.""" + def test_accept_pgr2(self, tmp_path): + """The DP cooperative multicast path supports double-buffered global + prefetch (PrefetchGlobalRead > 1): the prologue double-buffer prefetch + multicast load is bracketed by a cluster-scope handshake in codegen, so + both PrefetchGlobalRead 1 and 2 are accepted.""" from Tensile.SolutionStructs.Solution import _validateStreamKMulticast class _Info: @@ -232,14 +233,15 @@ def _state(pgr): "PrefetchGlobalRead": pgr, } - st = _state(2) - assert _validateStreamKMulticast(st, False, isaInfoMap) is False - assert st.get("Valid") is False + assert _validateStreamKMulticast(_state(2), False, isaInfoMap) is True assert _validateStreamKMulticast(_state(1), False, isaInfoMap) is True cfg = _write_variant(tmp_path, "pgr2.yaml", fork_overrides={"PrefetchGlobalRead": [2]}) - assert _derive_states(cfg) == [] + states = _derive_states(cfg) + assert states, "expected the PrefetchGlobalRead=2 multicast config to be accepted" + for st in states: + assert st["StreamKMulticast"] == 1 def test_xcc_mapping_forced_to_zero(self, tmp_path): """StreamKXCCMapping is coerced to 0 (not rejected) under StreamK+ClusterDim. diff --git a/shared/stinkytofu/include/stinkytofu/bindings/python/Module.hpp b/shared/stinkytofu/include/stinkytofu/bindings/python/Module.hpp index 74fe7d2d6f08..b453b734130e 100644 --- a/shared/stinkytofu/include/stinkytofu/bindings/python/Module.hpp +++ b/shared/stinkytofu/include/stinkytofu/bindings/python/Module.hpp @@ -75,6 +75,7 @@ X(EnableSwPrefetchInsertion, bool) \ X(SwPrefetchScratchSgpr, int) \ X(ClusterBarrier, bool) \ + X(StreamKMulticast, bool) \ X(PrefetchGlobalRead, int) \ X(PrefetchLocalRead, int) \ X(RemoveInstructions, std::string) \ diff --git a/shared/stinkytofu/include/stinkytofu/transforms/asm/InsertClusterBarrierPass.hpp b/shared/stinkytofu/include/stinkytofu/transforms/asm/InsertClusterBarrierPass.hpp index 4d1ad2e1f1db..3c66ee18a7b3 100644 --- a/shared/stinkytofu/include/stinkytofu/transforms/asm/InsertClusterBarrierPass.hpp +++ b/shared/stinkytofu/include/stinkytofu/transforms/asm/InsertClusterBarrierPass.hpp @@ -29,8 +29,7 @@ namespace stinkytofu { class Pass; -STINKYTOFU_EXPORT std::unique_ptr createInsertClusterBarrierPass(bool isKernelScope = true, - int pgrValue = 1, - int plrValue = 1); +STINKYTOFU_EXPORT std::unique_ptr createInsertClusterBarrierPass( + bool isKernelScope = true, int pgrValue = 1, int plrValue = 1, bool streamKMulticast = false); } // namespace stinkytofu diff --git a/shared/stinkytofu/src/pipeline/backend/Gfx1250Backend.cpp b/shared/stinkytofu/src/pipeline/backend/Gfx1250Backend.cpp index 79dc373392c8..bed154bc13b2 100644 --- a/shared/stinkytofu/src/pipeline/backend/Gfx1250Backend.cpp +++ b/shared/stinkytofu/src/pipeline/backend/Gfx1250Backend.cpp @@ -156,9 +156,11 @@ bool buildGfx1250Pipeline(PassManager& pm, StinkyAsmModule& module, const PassBu // the module opts in. Must precede InsertVgprMsbPass so the new // branches/labels are present when MSB configuration is materialized. if (moduleOptions.ClusterBarrier) { - pm.addPass(createInsertClusterBarrierPass(/*isKernelScope=*/true, - /*pgrValue=*/moduleOptions.PrefetchGlobalRead, - /*plrValue=*/moduleOptions.PrefetchLocalRead)); + pm.addPass(createInsertClusterBarrierPass( + /*isKernelScope=*/true, + /*pgrValue=*/moduleOptions.PrefetchGlobalRead, + /*plrValue=*/moduleOptions.PrefetchLocalRead, + /*streamKMulticast=*/moduleOptions.StreamKMulticast)); } // Build the CFG after the flat region splice-backs so RegionClonePass can match its diff --git a/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp b/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp index f0a0076cd440..cf772a075caf 100644 --- a/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp +++ b/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp @@ -610,6 +610,30 @@ void insertLoopCounterLGatedClusterBarrierWaitBefore(IRBase* anchor, AsmIRBuilde void insertClusterBarrierWaitBefore(IRBase* anchor, const char* comment, AsmIRBuilder& irBuilder, GfxArchID archId); +/// Emit `s_wait_tensorcnt 0` immediately before \p anchor, where \p anchor is +/// the instruction right after a cooperative `tensor_load_to_lds` group. Under +/// PGR>=2 the cluster/broadcast round runs at the TOP of the mainloop, so a +/// pre-round broadcast drain (before the wave-gated cluster-scope arrive) would +/// only retire the PREVIOUS iteration's load -- one iteration too late for the consumer +/// that reads the freshly cooperatively-loaded LDS half at the next loop head +/// (the producing wave is a peer wave, so the consumer's own tensor counter is +/// already zero and cannot order the peer's async write). Draining right after +/// the load issues makes the cooperative broadcast coherent before the back +/// edge, so the publishing workgroup barrier at the next loop head correctly +/// orders it for the consuming waves. Matches PGR1, which drains each load in +/// its own iteration. +void insertProducerTensorDrainBefore(IRBase* anchor, AsmIRBuilder& irBuilder, GfxArchID archId) { + const HwInstDesc* waitDesc = getMCIDByUOp(GFX::s_wait_tensorcnt, archId); + assert(waitDesc && "s_wait_tensorcnt opcode is not supported on this architecture"); + StinkyInstruction* w = irBuilder.create(waitDesc, anchor); + w->addSrcReg(StinkyRegister(0)); + SWaitTensorCntData d; + d.tlcnt = 0; + w->addModifier(d); + w->addModifier( + CommentData{"retire cooperative tensor_load_to_lds before back-edge (PGR>=2 coherence)"}); +} + /// Emit Rule 4's cluster-barrier handshake before `anchor` (the iterator /// position right after the load's anchoring `s_barrier_wait -1`). /// @@ -806,8 +830,12 @@ class InsertClusterBarrierPassImpl : public Pass { public: static char ID; - InsertClusterBarrierPassImpl(bool isKernelScope, int pgrValue, int plrValue) - : isKernelScope_(isKernelScope), pgrValue_(pgrValue), plrValue_(plrValue) {} + InsertClusterBarrierPassImpl(bool isKernelScope, int pgrValue, int plrValue, + bool streamKMulticast) + : isKernelScope_(isKernelScope), + pgrValue_(pgrValue), + plrValue_(plrValue), + streamKMulticast_(streamKMulticast) {} const char* getName() const override { return "Insert Cluster Barrier"; @@ -851,6 +879,10 @@ class InsertClusterBarrierPassImpl : public Pass { std::tuple> pending; std::unordered_set seenTriggers; + // Anchors (instruction right after each cooperative tensor_load + // group) for the mode-(c) producer-side drain; see + // insertProducerTensorDrainBefore. + std::vector producerDrainAnchors; auto segBegin = bb.begin(); for (auto it = bb.begin(); it != bb.end(); ++it) { @@ -886,6 +918,27 @@ class InsertClusterBarrierPassImpl : public Pass { sumLoopCounterLDecrementsBeforeInSegment(segBegin, trigger); pending.emplace_back(trigger, std::next(BasicBlock::iterator(trigger)), liveLclCmp, lclPreDecrement); + + // Mode (c) only: record the instruction right after this + // cooperative tensor_load group so a producer-side tensor + // drain can be planted there (retire the load before the back + // edge / next publishing barrier). Advance past any + // immediately-following tensor_load(s) so the drain covers the + // whole group (e.g. the A/B operand load plus its MX-scale + // load) rather than landing between them. + if (kRule4ForceUngatedSignalMode && streamKMulticast_ && pgrValue_ >= 2) { + auto postIt = std::next(it); + while (postIt != bb.end()) { + auto* pinst = dyn_cast(postIt.getNodePtr()); + if (pinst != nullptr && isTensorLoad(*pinst)) { + ++postIt; + continue; + } + break; + } + producerDrainAnchors.push_back( + (postIt != bb.end()) ? postIt.getNodePtr() : nullptr); + } } // Rule 1: signal-only handshake immediately AFTER each @@ -1101,6 +1154,12 @@ class InsertClusterBarrierPassImpl : public Pass { liveLclCmp, lclPreDecrement); (void)trigger; // queued for ordering only; insertion uses `anchor` } + // Mode (c): producer-side drain right after each cooperative + // tensor_load group (retire the async cooperative load before the + // back-edge so the next loop-head barrier publishes it coherently). + for (IRBase* postAnchor : producerDrainAnchors) { + insertProducerTensorDrainBefore(postAnchor, irBuilder, archId); + } for (IRBase* anchor : gsu1Anchors) { insertLoopCounterLGatedClusterBarrierSignalBefore( anchor, irBuilder, archId, @@ -1157,15 +1216,17 @@ class InsertClusterBarrierPassImpl : public Pass { const bool isKernelScope_; const int pgrValue_; const int plrValue_; + const bool streamKMulticast_; }; char InsertClusterBarrierPassImpl::ID = 0; } // namespace -std::unique_ptr createInsertClusterBarrierPass(bool isKernelScope, int pgrValue, - int plrValue) { - return std::make_unique(isKernelScope, pgrValue, plrValue); +std::unique_ptr createInsertClusterBarrierPass(bool isKernelScope, int pgrValue, int plrValue, + bool streamKMulticast) { + return std::make_unique(isKernelScope, pgrValue, plrValue, + streamKMulticast); } } // namespace stinkytofu From d9d066ddae22f8973112f9bee065b3c956bb8df4 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 18:06:51 +0000 Subject: [PATCH 06/24] chore(tensilelite): trim comment/YAML footprint for StreamK cluster reduction Condense verbose prose that duplicates the design docs (ClusterLoad docstrings, StreamK multicast/reduce cluster helpers, _validateStreamKMulticast / _validateStreamKClusterReduction) to concise summaries with a "See docs/design/..." pointer, and strip explanatory comments from the client/characterization test YAMLs (copyright/SPDX retained; parsed-YAML unchanged). No codegen change: char snapshots and targeted unit tests unchanged. --- .../Tensile/Components/ClusterLoad.py | 81 +++++--------- .../tensilelite/Tensile/Components/StreamK.py | 100 ++++++------------ .../Tensile/SolutionStructs/Solution.py | 28 ++--- ..._mxf4_force_dp_only_cluster_multicast.yaml | 9 +- .../core/sk_mxf4gemm_cluster_multicast.yaml | 68 +++--------- .../core/sk_mxf4gemm_cluster_reduction.yaml | 35 ++---- .../core/sk_mxf8gemm_cluster_multicast.yaml | 47 ++------ .../core/sk_mxf8gemm_cluster_reduction.yaml | 33 ++---- .../gfx1250/streamk_cluster_coop_load.yaml | 14 +-- .../gfx1250/streamk_cluster_multicast.yaml | 41 +------ .../streamk_cluster_multicast_pgr2.yaml | 27 +---- .../gfx1250/streamk_cluster_reduction.yaml | 36 +------ 12 files changed, 113 insertions(+), 406 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index 82939a2e58fb..b87e7cd46aab 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -2,23 +2,13 @@ # SPDX-License-Identifier: MIT """Cluster (multicast) TDM load component. -Centralizes the multicast ("cluster load") mask machinery that was previously -duplicated across ``KernelWriter``/``KernelWriterAssembly``/``SubtileGREmit``: - - * the mask *value* computation (``computeMasks``), - * the ``MulticastMask*`` SGPR declare/undeclare (``declareSgprs`` / - ``undeclareSgprs``), - * the topology decision (``usesCombinedMask`` / ``maskSgprName``), and - * the descriptor attach at each load site (``applyToDescriptor``). - -This is a behavior-preserving extraction: every method emits byte-identical -assembly to the original inline code. ``computeMasks`` therefore receives the -exact SGPR operands the caller already holds (it does not re-allocate) so the -instruction stream and register indices are unchanged. - -Selection is capability-based (``HasTDM`` + ``TDMInst == 3``), identical to how -``TensorDataMoverLoad`` is found: ``ClusterLoad.find(writer)`` returns the TDM -impl on gfx1250 and ``None`` (fallback -> no multicast) elsewhere. +Centralizes the multicast ("cluster load") mask machinery (value compute, +``MulticastMask*`` SGPR declare/undeclare, combined-vs-split topology decision, +and per-load-site descriptor attach) that was previously duplicated across +``KernelWriter``/``KernelWriterAssembly``/``SubtileGREmit``. Behavior-preserving: +every method emits byte-identical assembly, receiving the SGPR operands the +caller already holds rather than re-allocating. Capability-selected +(``HasTDM`` + ``TDMInst == 3``), like ``TensorDataMoverLoad``. """ from ..Component import ClusterLoad @@ -40,20 +30,14 @@ def __call__(self, writer: "KernelWriterAssembly", kernel: Mapping): # -- topology decision --------------------------------------------------- def usesCombinedMask(self, kernel: Mapping) -> bool: - """Single-parity combined ``MulticastMask`` predicate. + """True when the single-parity combined ``MulticastMask`` applies. - Subtile issues both A and B loads on every wave (no wave-parity load - split), so the single-parity ``MulticastMask`` is wrong there -- it - would OR one tensor's mask into both descriptors. Use the split A/B - masks in that case. This is the single source of truth for the - combined-vs-split decision used by declare/undeclare/computeMasks. + Single source of truth for the combined-vs-split decision. Subtile and + StreamK DP multicast both need the split A/B masks: subtile issues A and + B on every wave (no wave-parity split), and StreamK broadcasts only B + across the [C,1] cluster while A stays per-workgroup -- the combined + parity mask would be wrong for both. """ - # StreamK DP cooperative multicast needs the SPLIT A/B masks: A is - # loaded per-workgroup (MulticastMaskA = self bit) while B is broadcast - # across the [C,1] cluster (MulticastMaskB = all-C bits). The combined - # single-parity mask instead selects maskA on even waves / maskB on odd - # waves (for the wave-separated A-even/B-odd load split), which is wrong - # here, so force split whenever StreamKMulticast is on. Inert otherwise. if kernel.get("StreamKMulticast", 0): return False tdmA: bool = kernel["enableTDMA"] @@ -62,21 +46,13 @@ def usesCombinedMask(self, kernel: Mapping) -> bool: def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, waveSeparated: bool = False) -> str: - """Central multicast-mask SGPR name resolver. - - Reproduces the three prior naming rules exactly: - * wave-separated (non-subtile): the combined ``"MulticastMask"``; - * dense and subtile: the split ``f"MulticastMask{tc}"`` with any - ``MXS`` prefix stripped (dense passed ``MXSA``/``MXSB`` tensor - chars; subtile only ever passes ``A``/``B`` so the strip is a - no-op there). + """Resolve the multicast-mask SGPR name. + + Wave-separated (non-subtile) uses the combined ``"MulticastMask"``; + dense/subtile and StreamK multicast use the split ``f"MulticastMask{tc}"`` + (any ``MXS`` prefix stripped). StreamK forces the split name so B never + resolves to the never-declared combined SGPR. """ - # StreamK DP cooperative multicast always uses the split A/B masks - # (usesCombinedMask() returns False for it). The wave-separated dense - # apply site would otherwise resolve to the combined "MulticastMask" - # name, which is never declared on this path -> the B descriptor would - # OR an undefined SGPR. Force the split name so A binds MulticastMaskA - # (self) and B binds MulticastMaskB (broadcast). if kernel.get("StreamKMulticast", 0): string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc return f"MulticastMask{string}" @@ -86,11 +62,7 @@ def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, return f"MulticastMask{string}" def cooperativeThreadPartition(self, kernel: Mapping, tc: str) -> int: - """Number of cooperating workgroups for tensor ``tc``. - - ``ClusterDim[1]`` for A, ``ClusterDim[0]`` for B. Shared math with the - GL2 prefetch cooperative loads. - """ + """Cooperating-workgroup count for ``tc``: ClusterDim[1] (A) / [0] (B).""" subTc: str = tc[-1] return kernel["ClusterDim"][1] if subTc == "A" else kernel["ClusterDim"][0] @@ -130,11 +102,9 @@ def computeMasks(self, writer: "KernelWriterAssembly", kernel: Mapping, *, sgprWgX: int, sgprWgY: int, sgprNWgX: int, sTmp: int) -> Module: """Compute the multicast mask value(s) into the ``MulticastMask*`` SGPRs. - Verbatim lift of the ``defineAndResources`` mask compute. The caller - passes the exact SGPR operands it already holds (``sgprWgX`` = wg_x, - ``sgprWgY`` = wg_y, ``sgprNWgX`` = nwg_x, and ``sTmp`` whose ``+4`` slot - is scratch) so the emitted instructions and register indices are - byte-identical to the original inline code. + Verbatim lift of the ``defineAndResources`` mask compute; the caller + passes the operands it already holds (``sgprWgX``/``sgprWgY``/``sgprNWgX`` + and ``sTmp`` whose ``+4`` slot is scratch) so the output is byte-identical. """ mod = Module() if not kernel["Multicast"]: @@ -193,9 +163,8 @@ def applyToDescriptor(self, writer: "KernelWriterAssembly", kernel: Mapping, waveSeparated: bool = False) -> Module: """OR the multicast mask into descriptor ``Group1[word0]``. - Folds the ``kernel["Multicast"] and enableCluster`` gate, the mask-name - choice, and the ``SOrB32`` attach. Returns an empty ``Module`` when the - gate is not satisfied -- identical to today's skipped ``if``. + Folds the ``Multicast and enableCluster`` gate, mask-name choice, and the + ``SOrB32`` attach; returns an empty ``Module`` when the gate is not met. """ from .TensorDataMover import TensorDataMoverLoad mod = Module() diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 5766863d3d9d..9061d4940fb0 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -1573,24 +1573,14 @@ def clusterReduceWait(self, writer, kernel): return module def clusterReduceIntraCheck(self, writer, kernel): - """Emit the uniform intra-cluster predicate; leaves SCC=1 (fast path) - when this workgroup's cluster hosts a single, fully-populated StreamK - tile, SCC=0 (fall back to the global-flag path) otherwise. - - The predicate is a pure function of the cluster's top StreamK index and - the SK grid size (both uniform across a cluster), so every member of a - cluster -- owner and peers alike -- computes the identical verdict. This - is what makes the fast path deadlock-safe: a cluster is never split - between barrier signalers and flag setters. - - Contract (host accounting, StreamKClusterReduction): ClusterDim=[C,1] - with C a power of two; cluster c owns the contiguous StreamK index range - [c*C, c*C+C) which is exactly one tile's peer group (skSplit==C, fixed - even split); skGrid is rounded up to a multiple of C. Under that - contract cluster_last = StreamKIdx | (C-1) < skGrid holds for every SK - workgroup, so the fast path is taken; a partially-filled trailing - cluster (contract violated) fails the check and the whole cluster falls - back safely. + """Emit the uniform intra-cluster predicate: SCC=1 (fast path) when this + cluster hosts a single fully-populated StreamK tile, SCC=0 (fall back to + the global-flag path) otherwise. + + A pure function of the cluster's top StreamK index and the SK grid size + (both cluster-uniform), so every peer computes the identical verdict -- + the cluster is never split between barrier signalers and flag setters + (deadlock-safe). See docs/design/streamk-wg-clusters.md. """ module = Module("StreamK cluster intra-cluster check") C = kernel["ClusterDim"][0] @@ -2722,20 +2712,12 @@ def streamKMulticastBoundaryClear(self, writer, kernel): def streamKMulticastPrologueSignal(self, writer, kernel): """Elect wave 0 to arrive at the cluster split barrier once per workgroup. - The gfx1250 cluster-barrier pass emits a WAIT-only half - (``s_barrier_wait -3``) before the kernel's first ``tensor_load_to_lds`` - and expects a matching prologue ``s_barrier_signal -3`` to already - exist. On the StreamKMulticast path (GlobalSplitU == 0) the label that - would otherwise anchor that prologue arrive is never emitted, so this - helper supplies it: one wave per workgroup arrives (the remaining waves - branch over the signal), pairing the pass's first-load wait so the - cluster-scope signal/wait counts stay balanced. - - The arrive is unconditional on the StreamKMulticast path -- it is not - gated on the runtime clusterMulticastValid predicate -- so every cluster - peer participates in the barrier uniformly. Only the wave-0 election - gates it, matching the cluster reduce-signal idiom. Inert unless - StreamKMulticast. + Supplies the prologue ``s_barrier_signal -3`` that the gfx1250 + cluster-barrier pass's first-load wait expects but that is otherwise + never anchored on the StreamKMulticast path (GlobalSplitU == 0). One + wave per workgroup arrives (others branch over it), uniformly across + peers, keeping cluster-scope signal/wait counts balanced. Inert unless + StreamKMulticast. See docs/design/cluster-load-component-and-streamk-multicast.md. """ module = Module("StreamK multicast prologue signal") if not kernel.get("StreamKMulticast", 0): @@ -2754,24 +2736,15 @@ def streamKMulticastPrologueSignal(self, writer, kernel): return module def streamKMulticastProloguePrefetchHandshake(self, writer, kernel): - """Bracket the prologue double-buffer prefetch multicast load with a - cluster-scope handshake. - - The double-buffer ("LDS1") prefetch load in the PrefetchGlobalRead >= 2 - prologue is emitted inside the single-iteration guard branch, so the - generic per-load bracketing does not reach it (the guard branch is a - segment boundary that the backward anchor scan stops at). On the - StreamKMulticast path every cooperative-multicast tensor_load_to_lds - must be immediately preceded by a cluster-scope arrive/wait handshake; - emit a self-contained one here (one wave arrives, all waves wait) so the - prefetch load is synchronized and the cluster-scope signal/wait counts - stay balanced. - - The guard branches on LoopCounterL, which is uniform across the - co-located cluster peers (they process M-adjacent tiles sharing the same - K), so all peers execute this handshake in lockstep. Only the wave-0 - election gates the arrive, matching the existing prologue-signal idiom. - Inert unless StreamKMulticast. + """Bracket the PGR>=2 prologue double-buffer prefetch multicast load with + a self-contained cluster-scope arrive/wait handshake. + + That "LDS1" prefetch load sits inside the single-iteration guard branch, + which the generic per-load bracketing's backward anchor scan stops at, so + it needs its own handshake (one wave arrives, all waves wait). The guard + branches on LoopCounterL, uniform across co-located peers, so peers run it + in lockstep. Inert unless StreamKMulticast. + See docs/design/cluster-load-component-and-streamk-multicast.md. """ module = Module("StreamK multicast prologue prefetch cluster handshake") if not kernel.get("StreamKMulticast", 0): @@ -2793,25 +2766,14 @@ def streamKMulticastProloguePrefetchHandshake(self, writer, kernel): def streamKMulticastZeroIterClusterWait(self, writer, kernel): """Consume the prologue cluster arrive on the zero-iteration skip path. - The prologue arrive (``streamKMulticastPrologueSignal``) fires once per - cluster peer, uniformly, before the first cooperative-multicast load. - Its only matching cluster-scope ``s_barrier_wait -3`` is the pass's - first-load wait, which sits *after* the last-iteration guard - (``checkLastIter`` -> long-branch to ``PrefetchGlobalLastIterEnd``). On - the zero-full-iteration path (a participating workgroup that runs only - the tail/fixup, reachable when K is not a whole multiple of DepthU) that - long branch skips the first-load wait, leaving the arrive unmatched and - the cluster-scope barrier unbalanced on that edge. - - Emit the matching cluster wait on that skip edge so every cluster peer - executes exactly one arrive and exactly one wait on every control-flow - path out of the prologue. ``checkLastIter`` has set scc (scc1 == - numIterL == 0). Branch over the wait on scc0 (>=1 full iteration -> the - first-load wait pairs the arrive); on scc1 emit the all-waves cluster - wait (mirroring the all-waves first-load wait so the arrive is consumed - exactly once). The wait leaves scc intact, so the standard scc1 long - branch that follows still takes the skip edge. Inert unless - StreamKMulticast. + The prologue arrive's only matching wait is the pass's first-load wait, + which sits after the last-iteration guard; on the zero-full-iteration + path (K not a whole multiple of DepthU) that guard skips the wait, + leaving the arrive unbalanced. Emit the matching all-waves wait on the + skip edge (scc1 == numIterL == 0; branch over it on scc0) so every peer + does exactly one arrive + one wait on every path. The wait leaves scc + intact for the following long branch. Inert unless StreamKMulticast. + See docs/design/cluster-load-component-and-streamk-multicast.md. """ module = Module("StreamK multicast zero-iteration cluster wait") if not kernel.get("StreamKMulticast", 0): diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index fe80bb46e136..bb1a9bbb1194 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -237,19 +237,11 @@ def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): StreamKClusterReduction co-locates a StreamK tile's fixup peers in a single 1-D workgroup cluster (ClusterDim = [C, 1]) and replaces the cross-CU global-flag spin-wait with an intra-cluster split barrier. It is an explicit, - opt-in fast path (barrier-only in v1); when its solution-level requirements - are not met the solution is rejected here at BUILD time with a clear message. - See docs/design/streamk-wg-clusters.md. - - Note on the "multiple of the cluster size" limitation: the reduction also - requires itersPerTile = ceil(K / DepthU) to be a multiple of ClusterDim[0]=C - (otherwise the split barrier over-signals and hangs). That is a HARD REJECT of - the cluster solution for non-conforming problems, but it CANNOT be enforced - here because itersPerTile depends on the runtime K, which is unknown at build - time. It is instead enforced as a per-problem selection reject by the - ClusterReductionIterCheck predicate (emitted from Contractions.py), whose - diagnostic names the limitation to the user. This is a deliberate reject, not - a silent fallback. + opt-in fast path (barrier-only in v1); solution-level requirements are rejected + here at build time. The runtime requirement itersPerTile = ceil(K/DepthU) % C + == 0 (unknown at build time, else the split barrier over-signals) is instead a + per-problem selection reject via the ClusterReductionIterCheck predicate + (Contractions.py). See docs/design/streamk-wg-clusters.md. """ if not state.get("StreamKClusterReduction", 0): return True @@ -330,14 +322,10 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): Solution-level requirements are rejected here at build time; the runtime nWG0 % C "multiple-of-cluster-size" requirement is enforced by the ClusterDimCheck predicate at selection time (not a silent fallback). + Auto-derived for StreamK=3 + ClusterDim != [1, 1] (the bare index-only cluster + state collapsed into this path), so the rejects below reject an unusable + cluster rather than an explicit opt-in. See docs/design/cluster-load-component-and-streamk-multicast.md. - - StreamKMulticast is auto-derived for StreamK=3 + ClusterDim != [1, 1] in - assignProblemIndependentDerivedParameters -- the bare index-only StreamK - cluster state was collapsed into this cooperative-load path. When such an - auto-derived config cannot meet the requirements below (e.g. TDMInst != 3), - the rejects here are the "reject an unusable cluster" behavior, not a - rejection of an explicit user opt-in. """ if not state.get("StreamKMulticast", 0): return True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml index 08e3d69bc850..1b4d8ddf2bce 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml @@ -1,5 +1,5 @@ TestParameters: - marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] GlobalParameters: SyncsPerBenchmark: 0 @@ -19,8 +19,8 @@ GlobalParameters: ForceGenerateKernel: True BenchmarkProblems: - - # gfx1250 MXFP4, StreamK force DP-only persistent mode, 1-D ClusterDim range [2,1]/[4,1]/[8,1] - - # ProblemType, TN + - + - OperationType: GEMM DataType: F4 DestDataType: s @@ -36,8 +36,7 @@ BenchmarkProblems: MXBlockA: 32 MXBlockB: 32 - - # Persistent DP-only StreamK, 1-D ClusterDim range [2,1]/[4,1]/[8,1]; exercises the - # enableCluster guard in StreamKTwoTileDPFirst.preLoop. + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml index 4c8020a73a89..ad3918297cd2 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml @@ -1,42 +1,7 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -# -# Client-runnable StreamK DP cooperative B-multicast test for gfx1250, MX-FP4. -# -# F4 analog of the sibling core/sk_mxf8gemm_cluster_multicast.yaml (MX-FP8): same -# cluster/coverage structure, datatype-specific params differ. MX-F4 constraints -# (DataType F4, MXBlockA/B=32, TDMInst=3, MXLoadInst=TDM, -# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, DepthU=256, -# StreamKXCCMapping=0) with the DP cooperative B-multicast fast path -# (auto-derived from StreamK=3 + ClusterDim): -# -# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D spatial clusters, C in {2,4,8} -# - StreamKClusterReduction: [0] -- mutually exclusive with multicast -# -# StreamKMulticast is a derived-only internal state (no YAML opt-in): a StreamK=3 -# ClusterDim != [1,1] config that does not request StreamKClusterReduction -# auto-enables the DP cooperative B-multicast path. -# -# In the DP region the C consecutive workgroups of a [C,1] cluster process -# M-adjacent tiles that share the same B over full K, so B is TDM-multicast to -# the whole cluster while A is per-workgroup. The runtime clusterMulticastValid -# predicate enables the broadcast only on a fully-populated, M-aligned -# (nWG0 % C == 0) cluster; otherwise B loads normally. The host rounds the grid -# up to a multiple of C so every launched cluster is full. Each size below is -# chosen so nWG0 % C == 0 for the C it dispatches (a violation is a HARD REJECT -# via ClusterDimCheck at selection time, not a silent fallback); solution/size -# pairs whose nWG0 is not a multiple of the cluster C are simply rejected. -# -# Codegen coverage is broad: the MatrixInstruction list spans macro-tile-M in -# {32,64,128,256}, crossed with LDSTrInst {True,False}, PrefetchGlobalRead {1,2}. -# ScheduleIterAlg is pinned to 4 (the StinkyTofu-scheduled gfx1250 path) -- SIA3 is -# intentionally excluded so the tests exercise the real gfx1250 scheduler. The small -# sizes exercise the compact tiles at -# various nWG0; the two large [2048,...] sizes keep nWG0 % C == 0 for every tile -# (including MT-M=256) so the widest tile is covered on-cluster, and add a K-tail -# case (K % 256 != 0) for tail-loop B-multicast loads. TestParameters: - marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] GlobalParameters: SyncsPerBenchmark: 0 Architecture: gfx1250 @@ -56,11 +21,8 @@ GlobalParameters: ForceGenerateKernel: True BenchmarkProblems: - ######################################## - # MX-FP4 (F4 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. - ######################################## - - - # ProblemType: MX-F4 TN + - OperationType: GEMM DataType: F4 DestDataType: s @@ -75,7 +37,7 @@ BenchmarkProblems: UseBias: 0 MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] @@ -83,11 +45,11 @@ BenchmarkProblems: - UseSgprForGRO: [0] - ForceDisableShadowInit: [True] - MatrixInstruction: - - [16, 16, 128, 1, 1, 1, 1, 2, 2] # MT-M=32 - - [16, 16, 128, 1, 1, 2, 2, 2, 2] # MT-M=64 - - [16, 16, 128, 1, 1, 4, 4, 2, 2] # MT-M=128 - - [16, 16, 128, 1, 1, 8, 8, 2, 2] # MT-M=256 - - [16, 16, 128, 1, 1, 1, 1, 2, 1] # MT-M=32, WGN=1 + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] + - [16, 16, 128, 1, 1, 4, 4, 2, 2] + - [16, 16, 128, 1, 1, 8, 8, 2, 2] + - [16, 16, 128, 1, 1, 1, 1, 2, 1] - DepthU: [256] - ClusterLocalRead: [0] - TransposeLDS: [-1] @@ -133,12 +95,12 @@ BenchmarkProblems: JoinParameters: BenchmarkFinalParameters: - ProblemSizes: - - Exact: [512, 512, 1, 256] # nWG0=16 (MT-M=32) -> multicast ON (C=2,4,8) - - Exact: [256, 256, 1, 256] # nWG0=8 (MT-M=32) -> multicast ON (C=2,4,8) - - Exact: [128, 128, 1, 512] # nWG0=4 (MT-M=32) -> ON (C=2,4); K=512 - - Exact: [256, 256, 2, 256] # batched (batch=2); nWG0=8 (MT-M=32) -> ON (C=2,4,8) - - Exact: [256, 256, 1, 1920] # K-tail (K%256=128); nWG0=8 (MT-M=32) -> ON (C=2,4,8); tail-loop B-multicast loads - - Exact: [2048, 256, 1, 1024] # nWG0 in {64,32,16,8} across MT-M {32,64,128,256} -> ON (C=2,4,8) incl. widest tile - - Exact: [2048, 512, 1, 1056] # K-tail (K%256=32); nWG0 in {64,32,16,8} -> ON (C=2,4,8) incl. widest tile + - Exact: [512, 512, 1, 256] + - Exact: [256, 256, 1, 256] + - Exact: [128, 128, 1, 512] + - Exact: [256, 256, 2, 256] + - Exact: [256, 256, 1, 1920] + - Exact: [2048, 256, 1, 1024] + - Exact: [2048, 512, 1, 1056] - ActivationArgs: - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml index e0335db630c9..f3ed1ca43d27 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml @@ -1,25 +1,7 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -# -# Client-runnable StreamK workgroup-cluster-reduction test for gfx1250, MX-FP4. -# -# F4 analog of the sibling core/sk_mxf8gemm_cluster_reduction.yaml (MX-FP8): same -# cluster/coverage structure, datatype-specific params differ. MX-F4 constraints -# (DataType F4, MXBlockA/B=32, TDMInst=3, MXLoadInst=TDM, -# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, DepthU=256, -# StreamKXCCMapping=0) with the barrier-only cluster reduction fast path: -# -# - StreamKClusterReduction: [1] -- intra-cluster split-barrier reduce -# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D clusters, C in {2,4,8} -# -# Sizes force partial-tile reductions (K >> DepthU) and keep both nWG0 % C == 0 -# (cluster selection) and itersPerTile % C == 0 (balanced split barrier). When -# itersPerTile % C != 0 the split barrier over-signals -> hang, so the -# ClusterReductionIterCheck predicate HARD-REJECTS such problems at selection -# time (itersPerTile depends on runtime K, so it cannot be a build-time reject). -# A K-tail (K % DepthU != 0) is safe as long as itersPerTile % C == 0. TestParameters: - marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] GlobalParameters: SyncsPerBenchmark: 0 Architecture: gfx1250 @@ -39,11 +21,8 @@ GlobalParameters: ForceGenerateKernel: True BenchmarkProblems: - ######################################## - # MX-FP4 (F4 in, float32 out) TN, Batched -- StreamK cluster reduction. - ######################################## - - - # ProblemType: MX-F4 TN + - OperationType: GEMM DataType: F4 DestDataType: s @@ -58,7 +37,7 @@ BenchmarkProblems: UseBias: 0 MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] @@ -116,9 +95,9 @@ BenchmarkProblems: BenchmarkJoinParameters: BenchmarkFinalParameters: - ProblemSizes: - - Exact: [128, 128, 1, 1024] # MT32 nWG0=4 (C=2,4 admit; C=8 nWG0%8!=0); itersPerTile=4 -> C=2,4 partial-tile reduce - - Exact: [512, 512, 1, 2048] # MT32 nWG0=16 (C=2,4,8 admit); itersPerTile=8 -> C=2,4,8 partial-tile reduce (safe C=8) - - Exact: [256, 256, 1, 1920] # K-tail: itersPerTile=ceil(1920/256)=8 (8%2,4,8==0), K%256=128 tail -> tail loop inside cluster reduce (C=8 via MT32 nWG0=8) - - Exact: [256, 256, 2, 1024] # batched (batch=2); nWG0=8, itersPerTile=4 -> C=2,4 partial-tile reduce + - Exact: [128, 128, 1, 1024] + - Exact: [512, 512, 1, 2048] + - Exact: [256, 256, 1, 1920] + - Exact: [256, 256, 2, 1024] - ActivationArgs: - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml index 0d8afd112807..171b27702c95 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml @@ -1,35 +1,7 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -# -# Client-runnable StreamK DP cooperative B-multicast test for gfx1250. -# -# MX-FP8 TN, StreamK=3 (TDMInst=3, MXLoadInst=TDM, -# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, StreamKXCCMapping=0) with the -# DP cooperative B-multicast fast path (auto-derived from StreamK=3 + ClusterDim): -# -# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D spatial clusters, C in {2,4,8} -# - StreamKClusterReduction: [0] -- mutually exclusive with multicast -# -# StreamKMulticast is a derived-only internal state (no YAML opt-in): a StreamK=3 -# ClusterDim != [1,1] config that does not request StreamKClusterReduction -# auto-enables the DP cooperative B-multicast path. -# -# In the DP region the C consecutive workgroups of a [C,1] cluster process -# M-adjacent tiles that share the same B over full K, so B is TDM-multicast to -# the whole cluster while A is per-workgroup. The runtime clusterMulticastValid -# predicate enables the broadcast only on a fully-populated, M-aligned -# (nWG0 % C == 0) cluster; otherwise B loads normally. The host rounds the grid -# up to a multiple of C so every launched cluster is full. Every size below keeps -# nWG0 % C == 0 for the C it dispatches (a violation is a HARD REJECT via -# ClusterDimCheck at selection time, not a silent fallback). -# -# Codegen coverage crosses macro-tile-M in {32,64,128} with PrefetchGlobalRead -# {1,2}. ScheduleIterAlg is pinned to 4 (the gfx1250 scheduler path). With -# PrefetchGlobalRead=2 the prologue double-buffers the first main-loop prefetch, -# so the sizes with K > DepthU (DepthU=256; e.g. K=512 and the K=1920 tail case) -# ensure that double-buffered prologue prefetch path is actually exercised. TestParameters: - marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] GlobalParameters: SyncsPerBenchmark: 0 Architecture: gfx1250 @@ -49,11 +21,8 @@ GlobalParameters: ForceGenerateKernel: True BenchmarkProblems: - ######################################## - # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. - ######################################## - - - # ProblemType: MX-F8 TN + - OperationType: GEMM DataType: F8 DestDataType: s @@ -65,7 +34,7 @@ BenchmarkProblems: Batched: True MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] @@ -121,10 +90,10 @@ BenchmarkProblems: BenchmarkJoinParameters: BenchmarkFinalParameters: - ProblemSizes: - - Exact: [512, 512, 1, 256] # nWG0=16 -> multicast ON (C=2,4,8) - - Exact: [256, 256, 1, 256] # nWG0=8 -> multicast ON (C=2,4,8) - - Exact: [128, 128, 1, 512] # nWG0=4 -> ON (C=2,4); K=512 - - Exact: [256, 256, 2, 256] # batched (batch=2); nWG0=8 -> ON (C=2,4,8) - - Exact: [256, 256, 1, 1920] # K-tail (K%256=128); nWG0=8 -> ON (C=2,4,8); tail-loop B-multicast loads + - Exact: [512, 512, 1, 256] + - Exact: [256, 256, 1, 256] + - Exact: [128, 128, 1, 512] + - Exact: [256, 256, 2, 256] + - Exact: [256, 256, 1, 1920] - ActivationArgs: - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml index c70fb19c2d4e..6b45c50c3fca 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml @@ -1,23 +1,7 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -# -# Client-runnable StreamK workgroup-cluster-reduction test for gfx1250. -# -# MX-FP8 TN, StreamK=3 (TDMInst=3, MXLoadInst=TDM, -# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, StreamKXCCMapping=0) with the -# barrier-only cluster reduction fast path: -# -# - StreamKClusterReduction: [1] -- intra-cluster split-barrier reduce -# - ClusterDim: [[2,1],[4,1],[8,1]] -- 1-D clusters, C in {2,4,8} -# -# Sizes force partial-tile reductions (K >> DepthU) and keep both nWG0 % C == 0 -# (cluster selection) and itersPerTile % C == 0 (balanced split barrier). When -# itersPerTile % C != 0 the split barrier over-signals -> hang, so the -# ClusterReductionIterCheck predicate HARD-REJECTS such problems at selection -# time (itersPerTile depends on runtime K, so it cannot be a build-time reject). -# A K-tail (K % DepthU != 0) is safe as long as itersPerTile % C == 0. TestParameters: - marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] GlobalParameters: SyncsPerBenchmark: 0 Architecture: gfx1250 @@ -37,11 +21,8 @@ GlobalParameters: ForceGenerateKernel: True BenchmarkProblems: - ######################################## - # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK cluster reduction. - ######################################## - - - # ProblemType: MX-F8 TN + - OperationType: GEMM DataType: F8 DestDataType: s @@ -53,7 +34,7 @@ BenchmarkProblems: Batched: True MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] @@ -109,9 +90,9 @@ BenchmarkProblems: BenchmarkJoinParameters: BenchmarkFinalParameters: - ProblemSizes: - - Exact: [128, 128, 1, 1024] # MT32 nWG0=4 (C=2,4 admit; C=8 nWG0%8!=0); itersPerTile=4 -> C=2,4 partial-tile reduce - - Exact: [512, 512, 1, 2048] # MT32 nWG0=16 (C=2,4,8 admit); itersPerTile=8 -> C=2,4,8 partial-tile reduce (safe C=8) - - Exact: [256, 256, 1, 1920] # K-tail: itersPerTile=ceil(1920/256)=8 (8%2,4,8==0), K%256=128 tail -> tail loop inside cluster reduce (C=8 via MT32 nWG0=8) - - Exact: [256, 256, 2, 1024] # batched (batch=2); nWG0=8, itersPerTile=4 -> C=2,4 partial-tile reduce + - Exact: [128, 128, 1, 1024] + - Exact: [512, 512, 1, 2048] + - Exact: [256, 256, 1, 1920] + - Exact: [256, 256, 2, 1024] - ActivationArgs: - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml index c5419f5949f7..0b757d2d3f1b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_coop_load.yaml @@ -2,16 +2,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. # SPDX-License-Identifier: MIT ################################################################################ -# gfx1250 StreamK=3 + ClusterDim=[2,1] cooperative-load codegen characterization. -# -# StreamK=3 with ClusterDim != [1,1]. Under the -# "bare StreamK cluster" collapse this AUTO-ENABLES the DP cooperative B-multicast -# path (StreamKMulticast is a derived-only internal state; there is no bare -# index-only StreamK cluster anymore). So this config exercises BOTH the cluster -# WG-id decode / enableCluster guard in StreamKTwoTileDPFirst.preLoop AND the -# auto-derived multicast masks -- with MX-FP4 (C=2), complementing the MX-FP8 -# (C=4) sibling streamk_cluster_multicast.yaml. Mirrors -# Tests/common/streamk/gfx1250/core/sk_mxf4gemm_tdm.yaml with ClusterDim added. GlobalParameters: SyncsPerBenchmark: 0 MinimumRequiredVersion: 5.0.0 @@ -22,7 +12,7 @@ GlobalParameters: BenchmarkProblems: - - - # ProblemType, TN, MX-FP4 + - OperationType: GEMM DataType: F4 DestDataType: s @@ -37,7 +27,7 @@ BenchmarkProblems: UseBias: 0 MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup: StreamK=3, ClusterDim=[2,1] + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml index 265a43e3e17d..4d2560a57cbb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast.yaml @@ -1,40 +1,6 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT ################################################################################ -# Designed StreamK + DP cooperative multicast config for gfx1250 codegen -# characterization. -# -# Mirrors the sibling _designed/gfx1250/streamk_cluster_coop_load.yaml (MX-FP8 TN, -# StreamK=3, gfx1250 MX constraints: TDMInst=3, MXLoadInst=TDM, -# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, ISA=(12,5,0), -# StreamKXCCMapping=0), but enables the DP cooperative B-multicast fast path. -# StreamKMulticast is a derived-only internal state (no YAML opt-in): StreamK=3 + -# ClusterDim != [1,1] auto-enables it, so this config only needs to set: -# -# - ClusterDim: [[4, 1]] -- 1-D spatial cluster, C=4 (power of two) -# -# Under this config StreamK.py must emit the split A/B multicast masks (A per-WG -# self bit, B broadcast across the [C,1] cluster) on the DP TDM loads, gated by -# the runtime clusterMulticastValid predicate, plus the DP->SK boundary mask -# clear. See docs/design/cluster-load-component-and-streamk-multicast.md. -# -# ClusterDim is pinned to the single C=4 here because the coupled unit test -# Tensile/Tests/unit/test_streamk_multicast.py derives its states from THIS -# config (test_accepted_baseline asserts ClusterDim==[4,1]; test_broadcast_mask -# _value asserts the maskB=(1< -# MT 32x32 (WaveTile 1x1, MIWaveGroup 2x2, 128 work-items) -# MT 64x64 (WaveTile 2x2, MIWaveGroup 2x2, 128 work-items) -# MT 128x128(WaveTile 4x4, MIWaveGroup 2x2, 128 work-items) -# MT 32x16 (WaveTile 1x1, MIWaveGroup 2x1, 64 work-items) -- WI variety -# - ClusterDim: C=4 -# => 4 x 1 = 4 fork permutations. GlobalParameters: SyncsPerBenchmark: 0 MinimumRequiredVersion: 5.0.0 @@ -44,11 +10,8 @@ GlobalParameters: Device: 0 BenchmarkProblems: - ######################################## - # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. - ######################################## - - - # ProblemType: MX-F8 TN + - OperationType: GEMM DataType: F8 DestDataType: s @@ -60,7 +23,7 @@ BenchmarkProblems: Batched: True MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml index f175132a4797..f51da5ac1c19 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_multicast_pgr2.yaml @@ -1,26 +1,6 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT ################################################################################ -# Designed StreamK + DP cooperative multicast config for gfx1250 codegen -# characterization -- double-buffered global prefetch (PrefetchGlobalRead=2). -# -# Companion to _designed/gfx1250/streamk_cluster_multicast.yaml. That config -# pins PrefetchGlobalRead=1 with K == DepthU (single main-loop iteration); this -# one sets PrefetchGlobalRead=2 with K > DepthU (>= 2 main-loop iterations) so -# the prologue emits the second, double-buffered ("LDS1") cooperative multicast -# prefetch load. That prefetch load is emitted inside the single-iteration guard -# branch, so it must be bracketed by its own cluster-scope split-barrier -# handshake (s_barrier_signal / s_barrier_wait -3) emitted by -# StreamK.streamKMulticastProloguePrefetchHandshake, keeping every cooperative -# multicast load synchronized and the cluster-scope signal/wait counts balanced. -# See docs/design/cluster-load-component-and-streamk-multicast.md (section 4.4). -# -# - ClusterDim: [[4, 1]] -- 1-D spatial cluster, C=4 (power of two) -# - PrefetchGlobalRead: [2] -- double-buffered prologue prefetch -# - ProblemSizes K = 512 > DepthU 256 -> >= 2 main-loop iterations -# -# Coverage mirrors the sibling multicast config's MatrixInstruction sweep -# (MacroTile-M in {32,64,128,256}), all at PrefetchGlobalRead=2 => 4 kernels. GlobalParameters: SyncsPerBenchmark: 0 MinimumRequiredVersion: 5.0.0 @@ -30,11 +10,8 @@ GlobalParameters: Device: 0 BenchmarkProblems: - ######################################## - # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK DP cooperative multicast. - ######################################## - - - # ProblemType: MX-F8 TN + - OperationType: GEMM DataType: F8 DestDataType: s @@ -46,7 +23,7 @@ BenchmarkProblems: Batched: True MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml index 92d9d2b864a0..2e14c01da049 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml @@ -1,35 +1,6 @@ # Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT ################################################################################ -# Designed StreamK + workgroup-cluster-reduction config for gfx1250 codegen -# characterization. -# -# Mirrors the sibling _designed/gfx1250/streamk.yaml (MX-FP8 TN, StreamK=3, -# gfx1250 MX constraints: TDMInst=3, MXLoadInst=TDM, -# MXScaleFormat=InMemorySwizzle, WavefrontSize=32, ISA=(12,5,0), -# StreamKXCCMapping=0), but enables the workgroup-cluster reduction fast path: -# -# - StreamKClusterReduction: [1] -- opt-in barrier-only cluster reduction -# - ClusterDim: [[2, 1], [4, 1]] -- 1-D clusters, C in {2, 4} (power of two) -# -# Under this config StreamK.py must emit the intra-cluster split-barrier -# handshake on the fast path (s_barrier_signal -3 on the non-owner/peer arrive, -# s_barrier_wait -3 on the owner wait) while keeping the global-flag reduction -# compiled in as the runtime fallback. See docs/design/streamk-wg-clusters.md -# (sections 2.3, 3, 5). -# -# Coverage (mirrors the non-cluster MX-FP8 StreamK MT sweep in -# Tests/common/streamk/gfx1250/core/sk_mxf8gemm_tdm.yaml) across the cluster -# codegen surface -- MacroTile sizes x work-item (MIWaveGroup) sizes x C: -# - MatrixInstruction: 4 variants -> -# MT 32x32 (WaveTile 1x1, MIWaveGroup 2x2, 128 work-items) -# MT 64x64 (WaveTile 2x2, MIWaveGroup 2x2, 128 work-items) -# MT 128x128(WaveTile 4x4, MIWaveGroup 2x2, 128 work-items) -# MT 32x16 (WaveTile 1x1, MIWaveGroup 2x1, 64 work-items) -- WI variety -# - ClusterDim: C in {2, 4} -# => 4 x 2 = 8 fork permutations (the char harness emits limit=8; the client -# GEMM sibling sk_mxf8gemm_cluster_reduction.yaml adds C=8 + real problem-size -# sweeps validated on a gfx1250 simulation environment). GlobalParameters: SyncsPerBenchmark: 0 MinimumRequiredVersion: 5.0.0 @@ -39,11 +10,8 @@ GlobalParameters: Device: 0 BenchmarkProblems: - ######################################## - # MX-FP8 (F8 in, float32 out) TN, Batched -- StreamK cluster reduction. - ######################################## - - - # ProblemType: MX-F8 TN + - OperationType: GEMM DataType: F8 DestDataType: s @@ -55,7 +23,7 @@ BenchmarkProblems: Batched: True MXBlockA: 32 MXBlockB: 32 - - # BenchmarkProblemSizeGroup + - InitialSolutionParameters: BenchmarkCommonParameters: - KernelLanguage: ["Assembly"] From 022b142814b35ba35c162d4907d48ca96053dda2 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 19:19:07 +0000 Subject: [PATCH 07/24] test(tensilelite): add PGR2 coverage to StreamK cluster reduction + force-DP-only client tests Extend PrefetchGlobalRead to [1, 2] in the MX-FP4/FP8 cluster-reduction client configs and sk_mxf4_force_dp_only_cluster_multicast.yaml so both PGR1 and PGR2 are exercised on HW. No validator guard blocks PGR2; solutions enumerate err==0 (reduction: 12 PGR1 + 12 PGR2 each; force-DP-only: 3 + 3). --- .../gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml | 2 +- .../streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml | 2 +- .../streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml index 1b4d8ddf2bce..2cd34939272e 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml @@ -58,7 +58,7 @@ BenchmarkProblems: - LdsBlockSizePerPadMXSB: [-1] - LdsPadMetadata: [0] - LDSTrInst: [False] - - PrefetchGlobalRead: [1] + - PrefetchGlobalRead: [1, 2] - PrefetchLocalRead: [1] - ScheduleIterAlg: [4] - SourceSwap: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml index f3ed1ca43d27..e62b5b720e74 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml @@ -64,7 +64,7 @@ BenchmarkProblems: - LDSTrInst: [False] - 1LDSBuffer: [0] - DirectToVgprSparseMetadata: [False] - - PrefetchGlobalRead: [1] + - PrefetchGlobalRead: [1, 2] - PrefetchLocalRead: [1] - ScheduleIterAlg: [4] - SourceSwap: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml index 6b45c50c3fca..ae4cb8aec1ae 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml @@ -59,7 +59,7 @@ BenchmarkProblems: - LdsBlockSizePerPadMXSB: [-1] - LdsPadMetadata: [0] - LDSTrInst: [False] - - PrefetchGlobalRead: [1] + - PrefetchGlobalRead: [1, 2] - PrefetchLocalRead: [1] - ScheduleIterAlg: [4] - SourceSwap: [False] From 3c33a702851b91cf0a2fc33b6e04cda82b8fe247 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 20:17:55 +0000 Subject: [PATCH 08/24] test(tensilelite): redesign StreamK cluster-reduction client configs for zero-skip coverage The mxf4/mxf8 cluster-reduction client YAMLs paired big tiles (MT64x64, MT128x128) and high cluster factors (C up to 8) with low-M / low-K sizes, so 32 of 96 solution x size runs were rejected at selection time by ClusterDimCheck (ceil(M/MT0) % C != 0) or ClusterReductionIterCheck (ceil(K/DepthU) % C != 0), surfacing as DID_NOT_SATISFY_ASSERTS with no real coverage. Redesign both configs so every (tile x C x size) combo runs: - Drop MT128x128 (MT128 x C8 needs M a multiple of 1024 and is cluster-resource heavy); keep MT32x16 / MT32x32 / MT64x64. - Sizes with M and N multiples of 512 and K a multiple of 2048 so itersPerTile = ceil(K/256) in {8,16} is divisible by C in {2,4,8} and ceil(M/MT0) is divisible by C for every kept tile: Exact [512,512,1,2048], [1024,512,1,2048], [512,512,1,4096], [512,1024,2,2048]. - Retain C in {2,4,8}, PrefetchGlobalRead [1,2], a multi-iter K (4096) and a batched size ([512,1024,2,2048]). Result: 0 DID_NOT_SATISFY_ASSERTS (72 runs each, was 32/96), verified by enumerating solutions via the char config harness and evaluating both host predicates per solution x size, and end-to-end on the FFM client. --- .../gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml | 9 ++++----- .../gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml index e62b5b720e74..1055162a3793 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml @@ -46,9 +46,8 @@ BenchmarkProblems: - ForceDisableShadowInit: [True] - MatrixInstruction: - [16, 16, 128, 1, 1, 1, 1, 2, 2] - - [16, 16, 128, 1, 1, 2, 2, 2, 2] - - [16, 16, 128, 1, 1, 4, 4, 2, 2] - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] - DepthU: [256] - ClusterLocalRead: [0] - TransposeLDS: [-1] @@ -95,9 +94,9 @@ BenchmarkProblems: BenchmarkJoinParameters: BenchmarkFinalParameters: - ProblemSizes: - - Exact: [128, 128, 1, 1024] - Exact: [512, 512, 1, 2048] - - Exact: [256, 256, 1, 1920] - - Exact: [256, 256, 2, 1024] + - Exact: [1024, 512, 1, 2048] + - Exact: [512, 512, 1, 4096] + - Exact: [512, 1024, 2, 2048] - ActivationArgs: - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml index ae4cb8aec1ae..ccd17226ba7b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml @@ -43,9 +43,8 @@ BenchmarkProblems: - ForceDisableShadowInit: [True] - MatrixInstruction: - [16, 16, 128, 1, 1, 1, 1, 2, 2] - - [16, 16, 128, 1, 1, 2, 2, 2, 2] - - [16, 16, 128, 1, 1, 4, 4, 2, 2] - [16, 16, 128, 1, 1, 1, 1, 2, 1] + - [16, 16, 128, 1, 1, 2, 2, 2, 2] - DepthU: [256] - ClusterLocalRead: [0] - TransposeLDS: [-1] @@ -90,9 +89,9 @@ BenchmarkProblems: BenchmarkJoinParameters: BenchmarkFinalParameters: - ProblemSizes: - - Exact: [128, 128, 1, 1024] - Exact: [512, 512, 1, 2048] - - Exact: [256, 256, 1, 1920] - - Exact: [256, 256, 2, 1024] + - Exact: [1024, 512, 1, 2048] + - Exact: [512, 512, 1, 4096] + - Exact: [512, 1024, 2, 2048] - ActivationArgs: - [Enum: none] From 37f8c08317629159f0ed96fa4d231f0bf07e8316 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 20:27:45 +0000 Subject: [PATCH 09/24] test(tensilelite): cover ClusterLoad sparse-metadata + StreamK cluster validator reject branches Add unit coverage for previously-untested branches (codecov gaps): ClusterLoad computeMasks sparse-metadata (Sparse==1/2) + undeclareSgprs metadata; and the direct reject branches of _validateStreamKMulticast and _validateStreamKClusterReduction that are unreachable through config derivation (StreamK!=3, atomic, force-DP-only, XCC=3, non-[C,1]/non-pow2 cluster, non-gfx1250 ISA, missing HasTDM/HasClusterBarrier, TDMInst==0, multicast/reduction combo). --- .../Tests/unit/test_cluster_load_component.py | 31 +++++++++ .../unit/test_streamk_cluster_reduction.py | 63 +++++++++++++++++++ .../Tests/unit/test_streamk_multicast.py | 42 +++++++++++++ 3 files changed, 136 insertions(+) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py index a4370574cf6a..fa3e04190b41 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py @@ -198,6 +198,14 @@ def test_undeclare_split(self): self._c().undeclareSgprs(w, _kernel(useSubtile=True)) assert w.undefined == ["MulticastMaskA", "MulticastMaskB"] + def test_undeclare_metadata(self): + # The metadata SGPR is freed alongside the split A/B masks on the sparse + # TDM path (enableTDMMetadata). + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel(useSubtile=True, sparse=1, tdmMeta=True)) + assert w.undefined == ["MulticastMaskA", "MulticastMaskB", "MulticastMaskMetadata"] + def test_undeclare_noop_when_multicast_off(self): _init_rocisa_gfx1250() w = _StubWriter() @@ -246,6 +254,29 @@ def test_noop_when_multicast_off(self): sgprWgX=61, sgprWgY=62, sgprNWgX=63, sTmp=60) assert str(mod).strip() == "" + def test_metadata_mask_sparse_a(self): + # Sparse==1: the metadata mask follows sparse A -- shift maskA (0x5 for + # ClusterDim=[2,2]) by wg_x into MulticastMaskMetadata. + _init_rocisa_gfx1250() + mod = self._c().computeMasks( + _StubWriter(), _kernel(clusterDim=(2, 2), sparse=1, tdmMeta=True), + sgprWgX=61, sgprWgY=62, sgprNWgX=63, sTmp=60) + src = str(mod) + assert "Setting metadata mask (follows sparse A)" in src + assert "s_lshl_b32 s[sgprMulticastMaskMetadata], 0x5, s61" in src + + def test_metadata_mask_sparse_b(self): + # Sparse==2: the metadata mask follows sparse B -- shift maskB (0x3) by + # (wg_y * nwg_x) computed into the sTmp+4 scratch slot. + _init_rocisa_gfx1250() + mod = self._c().computeMasks( + _StubWriter(), _kernel(clusterDim=(2, 2), sparse=2, tdmMeta=True), + sgprWgX=61, sgprWgY=62, sgprNWgX=63, sTmp=60) + src = str(mod) + assert "Shift factor: wg_y * nwg_x (metadata)" in src + assert "Setting metadata mask (follows sparse B)" in src + assert "s_lshl_b32 s[sgprMulticastMaskMetadata], 0x3, s64" in src + # --- applyToDescriptor emitted asm ----------------------------------------- diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py index f4e05278c414..28134d7d39ac 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py @@ -242,5 +242,68 @@ def test_disabled_without_cluster_barrier_cap(self): assert _streamk()._streamKClusterReductionEnabled(w, _valid_cluster_kernel()) is False +class TestReductionValidation: + """Direct _validateStreamKClusterReduction reject-branch coverage. Several + branches are unreachable through config derivation, so drive them with a + hand-built state (mirrors test_streamk_multicast's direct validator tests).""" + + @staticmethod + def _state(**overrides): + st = { + "StreamKClusterReduction": 1, "StreamKMulticast": 0, "StreamK": 3, + "StreamKAtomic": 0, "StreamKForceDPOnly": 0, "StreamKXCCMapping": 0, + "ClusterDim": [4, 1], "ISA": [12, 5, 0], "TDMInst": 3, + } + st.update(overrides) + return st + + @staticmethod + def _isa_map(has_cluster_barrier=True): + class _Info: + asmCaps = {"HasClusterBarrier": has_cluster_barrier} + return {(12, 5, 0): _Info()} + + def _validate(self, st, isa=None): + from Tensile.SolutionStructs.Solution import _validateStreamKClusterReduction + return _validateStreamKClusterReduction(st, False, isa if isa is not None else self._isa_map()) + + def test_accept_baseline(self): + assert self._validate(self._state()) is True + + def test_noop_when_param_off(self): + assert self._validate(self._state(StreamKClusterReduction=0)) is True + + def test_reject_multicast_combo(self): + # #9611 keeps multicast and cluster reduction mutually exclusive. + assert self._validate(self._state(StreamKMulticast=1)) is False + + def test_reject_streamk_not_3(self): + assert self._validate(self._state(StreamK=4)) is False + + def test_reject_atomic(self): + assert self._validate(self._state(StreamKAtomic=1)) is False + + def test_reject_force_dp_only(self): + assert self._validate(self._state(StreamKForceDPOnly=1)) is False + + def test_reject_xcc3(self): + assert self._validate(self._state(StreamKXCCMapping=3)) is False + + def test_reject_non_1d_cluster(self): + assert self._validate(self._state(ClusterDim=[2, 2])) is False + + def test_reject_non_pow2_cluster(self): + assert self._validate(self._state(ClusterDim=[3, 1])) is False + + def test_reject_non_gfx1250(self): + assert self._validate(self._state(ISA=[9, 4, 2])) is False + + def test_reject_missing_cluster_barrier(self): + assert self._validate(self._state(), self._isa_map(has_cluster_barrier=False)) is False + + def test_reject_tdminst_zero(self): + assert self._validate(self._state(TDMInst=0)) is False + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index a1eede7632e9..7b7e9ebcaf1b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -275,6 +275,48 @@ def test_reject_non_pow2_cluster(self, tmp_path): # ClusterDim x at 16), so the "> 16" branch of the validator is defensive # and unreachable through valid params -- no test drives it here. + # --- direct _validateStreamKMulticast reject branches ------------------ + @staticmethod + def _direct_state(**overrides): + st = { + "StreamKMulticast": 1, "Multicast": 1, "StreamK": 3, + "StreamKAtomic": 0, "StreamKXCCMapping": 0, "ClusterDim": [4, 1], + "ISA": [12, 5, 0], "TDMInst": 3, "PrefetchGlobalRead": 1, + } + st.update(overrides) + return st + + @staticmethod + def _isa_map(has_tdm=True, has_cluster_barrier=True): + class _Info: + asmCaps = {"HasTDM": has_tdm, "HasClusterBarrier": has_cluster_barrier} + return {(12, 5, 0): _Info()} + + def test_reject_streamk_not_3_direct(self): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + assert _validateStreamKMulticast( + self._direct_state(StreamK=4), False, self._isa_map()) is False + + def test_reject_xcc_mapping_direct(self): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + assert _validateStreamKMulticast( + self._direct_state(StreamKXCCMapping=3), False, self._isa_map()) is False + + def test_reject_non_gfx1250_isa(self): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + assert _validateStreamKMulticast( + self._direct_state(ISA=[9, 4, 2]), False, self._isa_map()) is False + + def test_reject_missing_hastdm(self): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + assert _validateStreamKMulticast( + self._direct_state(), False, self._isa_map(has_tdm=False)) is False + + def test_reject_missing_hasclusterbarrier(self): + from Tensile.SolutionStructs.Solution import _validateStreamKMulticast + assert _validateStreamKMulticast( + self._direct_state(), False, self._isa_map(has_cluster_barrier=False)) is False + class TestTDMInstValidation: """The tightened TDMInst check: StreamKMulticast requires TDMInst == 3 (the From 63580ae8ed079f80430803a2fe37f36dcf7dcea9 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 20:43:00 +0000 Subject: [PATCH 10/24] fix(tensilelite): escalate StreamK cluster-reduction handshake fences to SCOPE_SYS The gfx1250 cluster-reduction split-barrier handshake published peer partials with a SCOPE_DEV global_wb and read them after a SCOPE_DEV global_inv. The split barrier orders execution but carries no memory semantics, so on gfx1250's partitioned L2 a SCOPE_DEV writeback was not guaranteed visible to a peer reading on a different partition -> a rare (~0.1%, one boundary cluster) miscompute at the largest grid (512,512,1,2048) on PGR1 (PGR2 masked it via prefetch timing). Escalate only the cluster-reduction release (peer partial writeback) and the owner's post-barrier acquire invalidate to SCOPE_SYS via a new optional `scope` arg on releaseFence/acquireFence; the global-flag fallback and all non-cluster StreamK keep SCOPE_DEV. InsertClusterBarrierPass.cpp (blob cf772a075c) untouched. Confirmed on real gfx1250: MXFP4 and MXFP8 cluster reduction pass (the PGR1 (512,512,1,2048) cases go FAILED->PASSED, no regression). Reduction char + unit tests green (30 passed). --- .../tensilelite/Tensile/Components/StreamK.py | 69 ++++++++++++++----- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 9061d4940fb0..365c63c4ae97 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -153,13 +153,26 @@ def preVolatileVmem(self, writer, comment="") -> Module: return module @abc.abstractmethod - def releaseFence(self, writer) -> Module: - """Memory fence ordering prior partial-tile stores before the flag store.""" + def releaseFence(self, writer, scope=None) -> Module: + """Memory fence ordering prior partial-tile stores before the flag store. + + `scope` optionally overrides the cache scope of the writeback (arches + with explicit fences only); None keeps the arch default. The cluster + split-barrier reduction passes CacheScope.SCOPE_SYS so the peer's + partials are pushed past gfx1250's partitioned L2 to the + system-coherent point the owner reads from after the barrier. + """ pass @abc.abstractmethod - def acquireFence(self, writer) -> Module: - """Memory fence after observing the flag and before reading partials.""" + def acquireFence(self, writer, scope=None) -> Module: + """Memory fence after observing the flag and before reading partials. + + `scope` optionally overrides the cache scope of the invalidate (arches + with explicit fences only); None keeps the arch default. Paired with + the release-side scope so a cross-partition release/acquire is + symmetric. + """ pass @abc.abstractmethod @@ -180,12 +193,14 @@ class StreamKMemoryOrderingDefault(StreamKMemoryOrdering): """ archCaps = {"HasInvWbDevFences": False} - def releaseFence(self, writer) -> Module: + def releaseFence(self, writer, scope=None) -> Module: + # `scope` is meaningful only for arches with explicit global_wb/inv + # fences; the default path has no cache-scoped op to retarget. module = Module("StreamK release fence (default)") module.add(SWaitCnt(vscnt=0, comment="wait for data store")) return module - def acquireFence(self, writer) -> Module: + def acquireFence(self, writer, scope=None) -> Module: return Module("StreamK acquire fence (default, no-op)") def readFlag(self, writer, dst, soffset) -> Module: @@ -212,24 +227,30 @@ class StreamKMemoryOrderingDevScopeFences(StreamKMemoryOrdering): """ archCaps = {"HasInvWbDevFences": True} - def releaseFence(self, writer) -> Module: + def releaseFence(self, writer, scope=None) -> Module: + # Default to SCOPE_DEV; callers may escalate (e.g. SCOPE_SYS for the + # cluster split-barrier reduction, whose peers can straddle a gfx1250 + # L2 partition boundary and therefore need a system-coherent push). + wbScope = scope if scope is not None else CacheScope.SCOPE_DEV module = Module("StreamK release fence (dev-scope)") module.add(SWaitCnt(vlcnt=0, comment="release: drain in-flight loads before global_wb")) module.add(SWaitCnt(vscnt=0, comment="wait for data store")) - module.add(GlobalWb(scope=CacheScope.SCOPE_DEV, + module.add(GlobalWb(scope=wbScope, comment="release: writeback partials to L2-coherent point")) module.add(SWaitCnt(vlcnt=0, vscnt=0, comment="release: wait for global_wb")) return module - def acquireFence(self, writer) -> Module: - # Drop stale dev-scope cache lines so the next dependent read (the flag - # word in getFlagValue, or the partials after the flag is observed) is - # re-fetched from the L2-coherent point. + def acquireFence(self, writer, scope=None) -> Module: + # Drop stale cache lines so the next dependent read (the flag word in + # getFlagValue, or the partials after the flag is observed) is + # re-fetched from the coherent point. Scope must match the paired + # release: SCOPE_DEV by default, SCOPE_SYS for the cluster reduction. + invScope = scope if scope is not None else CacheScope.SCOPE_DEV module = Module("StreamK acquire fence (dev-scope)") - module.add(GlobalInv(scope=CacheScope.SCOPE_DEV, - comment="acquire: invalidate before dependent dev-scope read")) + module.add(GlobalInv(scope=invScope, + comment="acquire: invalidate before dependent read")) module.add(SWaitCnt(vlcnt=0, comment="acquire: wait for global_inv")) return module @@ -977,7 +998,14 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele module.add(SCBranchSCC0(labelName=skClusterSetupDone.getLabelName(), comment="not intra-cluster: use global-flag reduction")) module.add(self.clusterReduceSignal(writer, kernel)) # owner arrives at the cluster barrier module.add(self.clusterReduceWait(writer, kernel)) # wait until all C cluster members arrived - module.add(memOrder.acquireFence(writer)) # once: observe peers' published partials + # System-scope acquire: the split barrier orders execution + # but not memory. On gfx1250's partitioned L2 a SCOPE_DEV + # invalidate is not guaranteed to re-fetch a peer partial + # written back on a different partition, so escalate the + # cluster-reduction acquire (paired with the SCOPE_SYS + # release below) to the system-coherent point. Held fix for + # the PGR1 boundary-cluster race; see red-pgr1-fix notes. + module.add(memOrder.acquireFence(writer, scope=CacheScope.SCOPE_SYS)) # once: observe peers' published partials module.add(skClusterSetupDone) module.add(skFixupLabel) @@ -1383,7 +1411,16 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, # kStr += PreLoopVmcntCaseStr # Set flag - module.add(memOrder.releaseFence(writer)) + # For cluster-reduction kernels the peer publishes its partial and + # then arrives at the split barrier; escalate the release writeback + # to SCOPE_SYS so the partial is globally visible past gfx1250's + # partitioned L2 before the owner's paired SCOPE_SYS acquire reads + # it. Non-cluster StreamK keeps the SCOPE_DEV default. Held fix for + # the PGR1 boundary-cluster race; see red-pgr1-fix notes. + releaseScope = (CacheScope.SCOPE_SYS + if self._streamKClusterReductionEnabled(writer, kernel) + else None) + module.add(memOrder.releaseFence(writer, scope=releaseScope)) module.add(SBarrier(comment="store all data before setting flag")) # Non-owner peer fast path: after publishing the partial and the From 5e0a4b27ba4b55506048c747dd64c2d9ced8d346 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 22 Jul 2026 21:04:01 +0000 Subject: [PATCH 11/24] style(stinkytofu): satisfy clang-format in cluster-barrier producer-drain Reflow the producerDrainAnchors.push_back ternary to the repo clang-format style (Google/IndentWidth 4/ColumnLimit 100). Whitespace only; no behavior change. --- .../src/transforms/asm/InsertClusterBarrierPass.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp b/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp index cf772a075caf..23e6e6e2d6bd 100644 --- a/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp +++ b/shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp @@ -936,8 +936,8 @@ class InsertClusterBarrierPassImpl : public Pass { } break; } - producerDrainAnchors.push_back( - (postIt != bb.end()) ? postIt.getNodePtr() : nullptr); + producerDrainAnchors.push_back((postIt != bb.end()) ? postIt.getNodePtr() + : nullptr); } } From 4953abbb80d3cfa2e6bfa150794dafb7ce823118 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Thu, 23 Jul 2026 17:43:12 +0000 Subject: [PATCH 12/24] refactor(tensilelite): make StreamK cluster reduction param-free (ClusterDim-driven) Derive the StreamK workgroup-cluster roles purely from the cluster shape ClusterDim = [Cs, Ck] on StreamK=3 instead of a user opt-in: * StreamKMulticast iff Cs = ClusterDim[0] > 1 ([C,1] = pure multicast) * StreamKClusterReduction iff Ck = ClusterDim[1] > 1 ([1,C] = pure reduction) Remove the StreamKClusterReduction user/benchmark parameter (now a derived-only internal state key, mirroring StreamKMulticast). Pure reduction migrates from the 1-D [C,1] cluster to a genuine 2-D [1,C] cluster: the launch grid becomes [skGrid/Ck, Ck, 1] and the kernel folds the cluster Y rank into the linear index (StreamKIdx = WorkGroup0*Ck + WorkGroup1), reusing the HW-validated 2-D mechanics. The whole-cluster size C = Cs*Ck now drives the split-barrier span, the ClusterReductionIterCheck predicate (Ck = ClusterDim[1]), the ClusterDimCheck N-tile divisor (pinned to 1 for genuine 2-D clusters), and the host grid launch. Pure multicast [C,1] is preserved byte-identically (Ck==1: no 2-D fold, 1-D launch, C = ClusterDim[0]); the SCOPE_SYS reduction fence is unchanged. Migrate reduction client + designed configs [C,1]+StreamKClusterReduction -> [1,C]; drop the now-invalid StreamKClusterReduction knob from multicast configs. Regenerate reduction/ValidParameters/SolutionClass goldens (param removal + [C,1]->[1,C] expression); multicast + StreamK codegen goldens stay byte-identical. --- .../Tensile/Common/GlobalParameters.py | 9 +- .../tensilelite/Tensile/Common/Utilities.py | 22 ++++ .../Tensile/Common/ValidParameters.py | 34 ++--- .../tensilelite/Tensile/Components/StreamK.py | 24 +++- .../tensilelite/Tensile/Contractions.py | 30 +++-- .../Tensile/SolutionStructs/Solution.py | 120 ++++++++++-------- .../core/sk_mxf4gemm_cluster_multicast.yaml | 3 +- .../core/sk_mxf4gemm_cluster_reduction.yaml | 5 +- .../core/sk_mxf8gemm_cluster_multicast.yaml | 3 +- .../core/sk_mxf8gemm_cluster_reduction.yaml | 5 +- .../Contractions/test_contractions_char.py | 20 +-- .../test_solution_class_char.ambr | 4 +- .../__snapshots__/test_builders_char.ambr | 9 -- ...treamk_cluster_reduction_gfx1250_char.ambr | 16 +-- .../gfx1250/streamk_cluster_reduction.yaml | 5 +- .../unit/test_streamk_cluster_reduction.py | 19 ++- .../Tests/unit/test_streamk_multicast.py | 39 ++++-- .../tensilelite/src/ContractionSolution.cpp | 81 ++++++++---- 18 files changed, 286 insertions(+), 162 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py index aae22c30fc1d..6696f1ba4178 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py @@ -571,10 +571,11 @@ {"StreamKAtomic": [0]}, {"StreamKXCCMapping": [0]}, {"StreamKFixupTreeReduction": [0]}, - {"StreamKClusterReduction": [0]}, - # NOTE: StreamKMulticast is a derived-only internal state key (like - # ClusterBarrier), auto-enabled by Solution.py for StreamK==3 + ClusterDim - # clusters; it is deliberately NOT a benchmark/default parameter here. + # NOTE: StreamKMulticast and StreamKClusterReduction are derived-only internal + # state keys (like ClusterBarrier), auto-enabled by Solution.py purely from the + # cluster shape ClusterDim = [Cs, Ck] for StreamK==3 (StreamKMulticast iff Cs>1, + # StreamKClusterReduction iff Ck>1); they are deliberately NOT benchmark/default + # parameters here. See docs/design/streamk-wg-clusters.md. {"DebugStreamK": [0]}, {"DebugPersistentKernelLoopForever": [False]}, {"ActivationFused": [True]}, diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py index eeb01004ce37..070e749c9dba 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py @@ -365,6 +365,28 @@ def clusterEnabled(clusterDim): """True when a workgroup cluster is requested (ClusterDim [x, y] is not [1, 1]).""" return (clusterDim[0] * clusterDim[1]) != 1 +def streamKClusterFactors(d): + """Return (Cs, Ck, C, is2D) for a StreamK workgroup cluster (ClusterDim-driven). + + The StreamK cluster is fully described by ClusterDim = [Cs, Ck]; there are no + user factoring/reduction knobs. Cs = ClusterDim[0] is the spatial B-multicast + axis (X), Ck = ClusterDim[1] is the K-split reduction axis (Y), and the total + cluster is C = Cs * Ck. ``is2D`` is True exactly when Ck > 1, i.e. when the + launch grid must be genuinely 2-D ([skGrid/Ck, Ck, 1]) and the linear StreamK + index folds the cluster Y rank in (StreamKIdx = WorkGroup0*Ck + WorkGroup1). + + Config expressions: + * [C, 1] -> Cs=C, Ck=1 : pure multicast (1-D launch, byte-identical) + * [1, C] -> Cs=1, Ck=C : pure reduction (2-D launch) + * [Cs,Ck]-> both > 1 : factored (2-D launch; factored branch only) + + ``d`` may be a kernel or a solution ``state`` dict; both expose "ClusterDim". + See docs/design/streamk-wg-clusters.md. + """ + cd = d["ClusterDim"] + cs, ck = cd[0], cd[1] + return cs, ck, cs * ck, (ck > 1) + def log2(x): return int(log(x, 2) + 0.5) diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index 12ffd50a0baa..f1a91e11aea7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -837,28 +837,20 @@ def makeValidMatrixInstructions(): # 0: use linear reduction # 1: use tree reduction "StreamKFixupTreeReduction": [0, 1], - # Enables the gfx1250 workgroup-cluster reduction fast path for StreamK. - # When enabled, a StreamK tile's fixup peers are co-located in a single 1-D - # workgroup cluster (ClusterDim = [C,1]) and the cross-CU global-flag - # spin-wait is replaced by an intra-cluster split barrier. Barrier-only in - # v1 (Multicast stays off); partials remain in the global workspace and the - # global-flag reduction is retained as a runtime/compile fallback. - # Requires StreamK == 3, ClusterDim == [C,1] with C a power of two in 2..16, - # gfx1250 (HasClusterBarrier) with TDMInst != 0, and NOT StreamKAtomic / - # NOT StreamKForceDPOnly. See docs/design/streamk-wg-clusters.md. - # 0: use the existing global-flag reduction - # 1: enable the cluster-barrier reduction fast path - "StreamKClusterReduction": [0, 1], - # NOTE: StreamKMulticast (the gfx1250 StreamK DP cooperative cluster-load / - # TDM B-multicast fast path) is intentionally NOT a valid/benchmark - # parameter. It is a DERIVED-ONLY internal state key (see the ClusterBarrier + # NOTE: StreamKMulticast and StreamKClusterReduction (the gfx1250 StreamK DP + # cooperative cluster-load / TDM B-multicast fast path and the cluster split- + # barrier K-reduction fast path) are intentionally NOT valid/benchmark + # parameters. They are DERIVED-ONLY internal state keys (see the ClusterBarrier # precedent): Solution.assignProblemIndependentDerivedParameters auto-enables - # it for StreamK==3 + ClusterDim != [1,1] (the "bare StreamK cluster" - # collapse), and _validateStreamKMulticast then hard-rejects any cluster - # config that cannot satisfy its constraints. It must not be user/YAML- - # settable, so it has no entry here (checkParametersAreValid would otherwise - # accept it as a fork/constant param). It still serializes to C++ via - # Contractions.SizeMapping (streamKMulticast, read with d.get()). + # them for StreamK==3 purely from the cluster shape ClusterDim = [Cs, Ck] + # (StreamKMulticast iff Cs = ClusterDim[0] > 1, StreamKClusterReduction iff + # Ck = ClusterDim[1] > 1) -- so [C,1] = pure multicast, [1,C] = pure reduction. + # _validateStreamKMulticast / _validateStreamKClusterReduction then hard-reject + # any cluster config that cannot satisfy their constraints. They must not be + # user/YAML-settable, so they have no entry here (checkParametersAreValid would + # otherwise accept them as a fork/constant param). They still serialize to C++ + # via Contractions.SizeMapping (streamKMulticast / streamKClusterReduction, read + # with d.get()). See docs/design/streamk-wg-clusters.md. # Debug settings for stream-k kernels to disable parts of the kernel # Bit 0: Don't generate fixup code # Bit 1: Don't generate write to partials code diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 365c63c4ae97..d1572a4286e5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -36,7 +36,7 @@ from .Subtile.SubtileLREmit import localReadResetOffsetsSubtile -from ..Common import print2, ceilDivide, log2, clusterEnabled +from ..Common import print2, ceilDivide, log2, clusterEnabled, streamKClusterFactors from ..Component import Component from ..AsmStoreState import StoreState, VectorDataTypes from ..AsmAddressCalculation import AddrCalculation @@ -1620,7 +1620,10 @@ def clusterReduceIntraCheck(self, writer, kernel): (deadlock-safe). See docs/design/streamk-wg-clusters.md. """ module = Module("StreamK cluster intra-cluster check") - C = kernel["ClusterDim"][0] + # Whole-cluster size C spans every co-resident peer WG. 1-D pure-multicast + # [C,1]: C = Cs = ClusterDim[0] (byte-identical). 2-D pure-reduction [1,C]: + # C = Ck = ClusterDim[1]. In general C = Cs*Ck. + _, _, C, _ = streamKClusterFactors(kernel) skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) sClusterLast = writer.sgprPool.checkOut(1, "SKClusterLast") sIdx = writer.acquireStreamKConstSgpr(kernel, "StreamKIdx") @@ -2840,6 +2843,23 @@ def preLoop(self, writer, kernel): module.add(SAndB32(dst=sgpr("WorkGroup1"), src0=hex(0xFFFF), src1="ttmp7", comment="workaround")) module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7", comment="workaround")) + # Genuine 2-D StreamK cluster (Ck = ClusterDim[1] > 1, e.g. pure reduction + # [1, C]): the K-split reduction axis is the HW cluster Y rank (WorkGroup1 + # in [0, Ck)). The launch grid is 2-D [skGrid/Ck, Ck, 1], so fold the Y rank + # into the linear StreamK index: + # StreamKIdx = WorkGroup0*Ck + WorkGroup1 (k = WorkGroup1 fastest) + # This keeps StreamKIdx a dense unique index whose (s,k) decode matches the + # 1-D [C,1] scheme (StreamKIdx & (Ck-1) = wg_y = k, StreamKIdx & (C-1) = the + # within-cluster rank). The fold is written into WorkGroup0 so the SMov/VMov + # save below (unchanged) still copies the final index; the 1-D Ck==1 path is + # byte-identical (no fold emitted). See docs/design/streamk-wg-clusters.md. + _, ck2d, _, is2d = streamKClusterFactors(kernel) + if is2d: + module.add(SMulI32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=hex(ck2d), + comment="2-D cluster: WorkGroup0 * Ck")) + module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr("WorkGroup1"), + comment="2-D cluster: StreamKIdx = WorkGroup0*Ck + WorkGroup1 (K/Y rank)")) + if skConstsInVgprs: module.add(VMovB32(dst=vgpr(self._skv(writer, "StreamKIdx")), src=sgpr("WorkGroup0"), comment="Save original StreamK index to VGPR")) diff --git a/projects/hipblaslt/tensilelite/Tensile/Contractions.py b/projects/hipblaslt/tensilelite/Tensile/Contractions.py index 3dd8a2d9fc92..39a8d7bb7a9b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Contractions.py +++ b/projects/hipblaslt/tensilelite/Tensile/Contractions.py @@ -590,21 +590,33 @@ def CompoundPredicates(cls, state, problemType): valuepredicates.append(state["MacroTile0"]) valuepredicates.append(state["MacroTile1"]) valuepredicates.append(state["GlobalSplitU"]) + # value[3] is the M-adjacency (shared-B) alignment axis Cs = ClusterDim[0]. + # Pure multicast [C,1]: Cs = C (byte-identical to historic value). Pure + # reduction [1,C]: Cs = 1 (no M-alignment constraint). valuepredicates.append(state["ClusterDim"][0]) - valuepredicates.append(state["ClusterDim"][1]) + # value[4] is the N-tile divisor. For a genuine 2-D StreamK cluster + # (Ck = ClusterDim[1] > 1, e.g. pure reduction [1,C]) the Y-extent is the + # K-split reduction / index-generation axis, NOT an N-tiling axis, so it + # must NOT constrain the N-tile grid -> pin to 1. 1-D StreamK ([C,1]) and + # dense (non-StreamK) clusters keep ClusterDim[1] (byte-identical). + if state.get("StreamK", 0) == 3 and state["ClusterDim"][1] > 1: + valuepredicates.append(1) + else: + valuepredicates.append(state["ClusterDim"][1]) rv += [cls('ClusterDimCheck', value=valuepredicates)] - # StreamK cluster-reduction split-barrier safety (gfx1250). The C = - # ClusterDim[0] peers split a tile's itersPerTile = ceil(K/DepthU) + # StreamK cluster-reduction split-barrier safety (gfx1250). The Ck = + # ClusterDim[1] reduction peers split a tile's itersPerTile = ceil(K/DepthU) # K-iterations and hand off through an intra-cluster split barrier; if - # itersPerTile % C != 0 (incl. C > itersPerTile) the split barrier + # itersPerTile % Ck != 0 (incl. Ck > itersPerTile) the split barrier # over-signals -> hang. itersPerTile depends on runtime K, so this is a - # per-problem HARD REJECT (not a build-time reject, not a silent - # fallback). Only the K-split reduction path needs it; StreamKMulticast - # (no K-split) relies on ClusterDimCheck instead. - if state.get("StreamKClusterReduction", 0) and state["ClusterDim"][0] > 1: + # per-problem HARD REJECT (not a build-time reject, not a silent fallback). + # Reduction is derived from the cluster shape: pure reduction is [1, C] + # (Ck = ClusterDim[1] = C > 1). Pure multicast (no K-split) relies on + # ClusterDimCheck instead. + if state.get("StreamKClusterReduction", 0) and state["ClusterDim"][1] > 1: rv += [cls('ClusterReductionIterCheck', - value=[state["DepthU"], state["ClusterDim"][0]])] + value=[state["DepthU"], state["ClusterDim"][1]])] return rv diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index bb1a9bbb1194..2cbf7cff7e92 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -235,23 +235,26 @@ def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): """Validate the gfx1250 StreamK workgroup-cluster reduction fast path. StreamKClusterReduction co-locates a StreamK tile's fixup peers in a single - 1-D workgroup cluster (ClusterDim = [C, 1]) and replaces the cross-CU - global-flag spin-wait with an intra-cluster split barrier. It is an explicit, - opt-in fast path (barrier-only in v1); solution-level requirements are rejected - here at build time. The runtime requirement itersPerTile = ceil(K/DepthU) % C - == 0 (unknown at build time, else the split barrier over-signals) is instead a - per-problem selection reject via the ClusterReductionIterCheck predicate - (Contractions.py). See docs/design/streamk-wg-clusters.md. + workgroup cluster and replaces the cross-CU global-flag spin-wait with an + intra-cluster split barrier. It is DERIVED (not a user opt-in) from the cluster + shape ClusterDim = [1, C] (Ck = ClusterDim[1] > 1); solution-level requirements + are rejected here at build time. The runtime requirement itersPerTile = + ceil(K/DepthU) % C == 0 (unknown at build time, else the split barrier + over-signals) is instead a per-problem selection reject via the + ClusterReductionIterCheck predicate (Contractions.py). + See docs/design/streamk-wg-clusters.md. """ if not state.get("StreamKClusterReduction", 0): return True - # Mutually exclusive with the cooperative-load multicast fast path: the two - # features want incompatible cluster semantics (K-split reduction peers vs. - # M-adjacent shared-B multicast peers) on the same 1-D cluster. + # Pure reduction ([1, C]) has Cs = ClusterDim[0] = 1, so StreamKMulticast is + # off. Factored ([Cs,Ck] with BOTH axes > 1) would turn both on, but that is + # rejected in the SK cluster guard on this branch (it lives on the factored + # branch). Defensive: reject the combination here too. if state.get("StreamKMulticast", 0): reject(state, printRejectionReason, - "StreamKMulticast and StreamKClusterReduction are mutually exclusive") + "StreamKMulticast (Cs>1) + StreamKClusterReduction (Ck>1) is the " + "factored cluster; not supported on this branch") return False # SK3 (StreamKTwoTileDPFirst) only; SK4/SK5 dynamic/atomic peer sets can not @@ -278,17 +281,19 @@ def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): "StreamKClusterReduction is not supported with StreamKXCCMapping=3 (SGPR overflow)") return False - # 1-D cluster [C, 1] with C a power of two in [2, 16]. A ClusterDim[1] > 1 - # would require gridDimY % ClusterDim[1] == 0 while the StreamK grid is 1-D. + # Pure reduction is expressed as ClusterDim = [1, C] (Cs = ClusterDim[0] = 1, + # Ck = ClusterDim[1] = C the K-split peer count), a genuine 2-D cluster whose + # launch grid is [skGrid/C, C, 1] (so gridDimY % C == 0 holds). C must be a + # power of two in [2, 16]. clusterDim = state["ClusterDim"] - c = clusterDim[0] - if clusterDim[1] != 1: + c = clusterDim[1] + if clusterDim[0] != 1: reject(state, printRejectionReason, - "StreamKClusterReduction requires ClusterDim = [C, 1] (got %s)" % clusterDim) + "StreamKClusterReduction requires ClusterDim = [1, C] (got %s)" % clusterDim) return False if c < 2 or c > 16 or (c & (c - 1)) != 0: reject(state, printRejectionReason, - "StreamKClusterReduction requires ClusterDim[0] a power of two in [2, 16] (got %d)" % c) + "StreamKClusterReduction requires ClusterDim[1] a power of two in [2, 16] (got %d)" % c) return False # gfx1250 with the cluster split barrier capability. @@ -330,12 +335,15 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): if not state.get("StreamKMulticast", 0): return True - # Mutually exclusive with the cluster split-barrier reduction fast path: the - # two features want incompatible cluster semantics on the same 1-D cluster and - # must never be enabled together. + # Multicast (Cs>1) and reduction (Ck>1) both on is the FACTORED cluster + # [Cs,Ck] with both axes > 1 -- that lives on the streamk-factored-cluster-mode + # branch and is rejected in the SK cluster guard on this branch. Defensive + # reject here too: on this (multicast/reduction-only) branch the two derived + # roles are mutually exclusive (pure multicast [C,1] xor pure reduction [1,C]). if state.get("StreamKClusterReduction", 0): reject(state, printRejectionReason, - "StreamKMulticast and StreamKClusterReduction are mutually exclusive") + "StreamKMulticast (Cs>1) + StreamKClusterReduction (Ck>1) is the " + "factored cluster; not supported on this branch") return False # StreamKMulticast is auto-enabled by ClusterDim on SK3, but the multicast mask @@ -1238,31 +1246,35 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, # ClusterBarrier default above) so it is always present on state; the collapse # below is the ONLY place it is turned on. state["StreamKMulticast"] = 0 - # Collapse the bare StreamK cluster state: on StreamK=3 a non-[1,1] ClusterDim - # AUTO-ENABLES the DP cooperative B-multicast path. A StreamK cluster with no - # cooperative loads no longer exists -- "ClusterDim without cluster loads" is - # not a supported state. StreamKMulticast is derived-only (no user/YAML - # opt-in): this collapse is its sole enable site. - # - # If the resulting cooperative-load config cannot be satisfied (e.g. TDMInst!=3 - # or non-gfx1250), _validateStreamKMulticast (called later in - # assignDerivedParameters) rejects it at build time rather than silently - # degrading to a no-op cluster -- an unusable cluster is a hard reject. - # - # Scope note: the TDM B-multicast fast path is StreamK=3-only, so there is - # nothing to auto-enable for the dynamic (SK4) / hybrid (SK5) queue modes. - # SK4/SK5 with a non-[1,1] ClusterDim is rejected later in - # assignDerivedParameters (no cluster-load impl for those modes). - # ClusterDim is tested first so this (like the legacy Multicast auto branch - # below) never dereferences StreamK for the common non-clustered state, which - # some partial-state derivation call sites construct without a StreamK key. + # StreamKClusterReduction is a DERIVED-ONLY internal state key (like + # StreamKMulticast): reduction is no longer a user opt-in, it is expressed + # purely by the cluster shape ClusterDim = [1, C] (Ck = ClusterDim[1] > 1). + # Seed it off here (mirroring StreamKMulticast); the collapse below is its + # sole enable site. + state["StreamKClusterReduction"] = 0 + # ClusterDim-driven StreamK cluster derivation (fully param-free). On StreamK=3 + # a non-[1,1] ClusterDim = [Cs, Ck] AUTO-ENABLES the cooperative cluster path; + # both derived booleans fall out of the cluster shape: + # * Cs = ClusterDim[0] spatial B-multicast peers -> StreamKMulticast iff Cs>1 + # * Ck = ClusterDim[1] K-split reduction peers -> StreamKClusterReduction iff Ck>1 + # Config expressions: [C,1] = pure multicast, [1,C] = pure reduction. + # Factored (both axes > 1) is the streamk-factored-cluster-mode branch and is + # rejected here (see the SK cluster guard in assignDerivedParameters). # - # StreamKClusterReduction claims the [C,1] cluster for its own split-barrier - # K-split reduction (mutually exclusive with the cooperative-load multicast), - # so a reduction cluster must NOT auto-enable StreamKMulticast here. - if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3 \ - and not state.get("StreamKClusterReduction", 0): - state["StreamKMulticast"] = 1 + # A StreamK cluster with no cooperative role no longer exists -- "ClusterDim + # without cluster loads" is not a supported state. If the resulting config + # cannot be satisfied (e.g. TDMInst!=3 or non-gfx1250), _validateStreamKMulticast + # / _validateStreamKClusterReduction reject it at build time rather than + # silently degrading. The TDM/cluster fast paths are StreamK=3-only; SK4/SK5 + # with a non-[1,1] ClusterDim is rejected later in assignDerivedParameters. + # ClusterDim is tested first so this never dereferences StreamK for the common + # non-clustered state, which some partial-state derivation call sites construct + # without a StreamK key. See docs/design/streamk-wg-clusters.md. + if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3: + cs = state["ClusterDim"][0] + ck = state["ClusterDim"][1] + state["StreamKMulticast"] = 1 if cs > 1 else 0 + state["StreamKClusterReduction"] = 1 if ck > 1 else 0 # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. # Default -1 reproduces the historic ClusterDim-coupled derivation, so YAML # that omits Multicast is byte-identical. @@ -1953,12 +1965,20 @@ def assignDerivedParameters( reject(state, printRejectionReason, "StreamK dynamic/hybrid (SK4/SK5) do not support ClusterDim " "(cluster support is SK3-only)") - # Stream-K launches a 1-D grid in X, so StreamKIdx = WorkGroup0 = cluster_x*nwg_x - # + wg_x must stay a unique linear index. A Y-extent > 1 collides WorkGroup0 across - # WGs that differ only in Y, so restrict clustering to the X dimension. - if state["ClusterDim"][1] != 1: + # ClusterDim-driven cluster shape (SK3). A genuine 2-D cluster ([Cs,Ck] + # with Ck=ClusterDim[1]>1) launches a 2-D grid [skGrid/Ck, Ck, 1] and folds + # the Y rank into the index (StreamKIdx = WorkGroup0*Ck + WorkGroup1, see + # StreamK.preLoop), so a Y-extent > 1 no longer collides WorkGroup0. On this + # (reduction) branch only the pure-reduction shape [1, C] is supported; + # factored [Cs,Ck] with BOTH axes > 1 (spatial multicast AND K-split in one + # cluster) lives on the streamk-factored-cluster-mode branch -- reject it + # here. Pure multicast stays [C, 1] (Ck==1, 1-D launch, byte-identical). + if state["ClusterDim"][1] > 1 and state["ClusterDim"][0] > 1: reject(state, printRejectionReason, - "Stream-K + ClusterDim requires ClusterDim Y-extent == 1") + "Factored StreamK cluster [Cs,Ck] with both axes > 1 is not " + "supported on this branch; use ClusterDim=[C,1] (multicast) or " + "[1,C] (reduction), or the streamk-factored-cluster-mode branch " + "(got %s)" % state["ClusterDim"]) # StreamKXCCMapping remaps WorkGroup0 with no cluster awareness; disable it. state["StreamKXCCMapping"] = 0 if not state["EnableMatrixInstruction"]: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml index ad3918297cd2..5c262606f067 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml @@ -81,7 +81,8 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] - - StreamKClusterReduction: [0] + # Pure multicast is derived from ClusterDim = [C, 1] (Cs = ClusterDim[0] > 1); + # StreamKClusterReduction is no longer a user param (derived Ck==1 -> off). - ClusterDim: [[2, 1], [4, 1], [8, 1]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml index 1055162a3793..263ce5e09567 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml @@ -79,8 +79,9 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] - - StreamKClusterReduction: [1] - - ClusterDim: [[2, 1], [4, 1], [8, 1]] + # Pure reduction is now derived from the cluster shape ClusterDim = [1, C] + # (Ck = ClusterDim[1] > 1); StreamKClusterReduction is no longer a user param. + - ClusterDim: [[1, 2], [1, 4], [1, 8]] - StaggerU: [0] - DirectToVgprA: [False] - DirectToVgprB: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml index 171b27702c95..7c492df9bf37 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml @@ -75,7 +75,8 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] - - StreamKClusterReduction: [0] + # Pure multicast is derived from ClusterDim = [C, 1] (Cs = ClusterDim[0] > 1); + # StreamKClusterReduction is no longer a user param (derived Ck==1 -> off). - ClusterDim: [[2, 1], [4, 1], [8, 1]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml index ccd17226ba7b..f8bb72d0c2a2 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml @@ -74,8 +74,9 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] - - StreamKClusterReduction: [1] - - ClusterDim: [[2, 1], [4, 1], [8, 1]] + # Pure reduction is now derived from the cluster shape ClusterDim = [1, C] + # (Ck = ClusterDim[1] > 1); StreamKClusterReduction is no longer a user param. + - ClusterDim: [[1, 2], [1, 4], [1, 8]] - StaggerU: [0] - DirectToVgprA: [False] - DirectToVgprB: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py index 531c3104d7b1..760b7d7bf6ec 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/Contractions/test_contractions_char.py @@ -102,10 +102,11 @@ def test_problem_predicate_compound(problem_type, solution_state, snapshot): # --- StreamK cluster-reduction split-barrier selection guard -- # # ClusterReductionIterCheck must be emitted only for StreamKClusterReduction -# solutions with a real cluster (ClusterDim[0] > 1); its value carries -# [DepthU, C] so the host predicate can reject problems whose -# itersPerTile = ceil(K/DepthU) is not a multiple of C (split-barrier -# over-signal). It must NOT be emitted for non-cluster or multicast solutions. +# solutions with a real reduction cluster (pure reduction [1, C], so +# Ck = ClusterDim[1] > 1); its value carries [DepthU, Ck] so the host predicate +# can reject problems whose itersPerTile = ceil(K/DepthU) is not a multiple of +# Ck (split-barrier over-signal). It must NOT be emitted for non-cluster or +# multicast ([C,1]) solutions. def _preds_for(solution_state, problem_type, **overrides): st = dict(solution_state) @@ -118,23 +119,24 @@ def _cluster_iter_pred(preds): def test_cluster_reduction_iter_check_emitted(problem_type, solution_state): + # Pure reduction is ClusterDim = [1, C] (Ck = ClusterDim[1] = C). preds = _preds_for(solution_state, problem_type, - StreamKClusterReduction=1, ClusterDim=[4, 1], DepthU=256) + StreamKClusterReduction=1, ClusterDim=[1, 4], DepthU=256) p = _cluster_iter_pred(preds) assert p is not None, "cluster-reduction solution must emit ClusterReductionIterCheck" - # value = [DepthU, C] so the host can compute itersPerTile % C. + # value = [DepthU, Ck] so the host can compute itersPerTile % Ck. assert p.value == [256, 4] def test_cluster_reduction_iter_check_not_emitted_when_off(problem_type, solution_state): - # StreamKClusterReduction off -> no guard (the param is inert). + # StreamKClusterReduction off -> no guard (derived flag off). preds = _preds_for(solution_state, problem_type, - StreamKClusterReduction=0, ClusterDim=[4, 1], DepthU=256) + StreamKClusterReduction=0, ClusterDim=[1, 4], DepthU=256) assert _cluster_iter_pred(preds) is None def test_cluster_reduction_iter_check_not_emitted_without_cluster(problem_type, solution_state): - # Reduction requested but C == 1 (no real cluster) -> no guard. + # Reduction requested but Ck == 1 (no real reduction cluster) -> no guard. preds = _preds_for(solution_state, problem_type, StreamKClusterReduction=1, ClusterDim=[1, 1], DepthU=256) assert _cluster_iter_pred(preds) is None diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index 61914c301fa8..d38c09b4a0e0 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -3,7 +3,7 @@ dict({ 'count': 1, 'names': list([ - 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKCR0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', ]), }) # --- @@ -401,7 +401,7 @@ 'tailLoopOptA', 'tailLoopOptB', ]), - 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKCR0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', + 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', 'num_keys': 339, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr index b19f2c4e3506..0c2e30b5d753 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/ValidParameters/__snapshots__/test_builders_char.ambr @@ -1480,7 +1480,6 @@ 'StoreVectorWidth', 'StreamK', 'StreamKAtomic', - 'StreamKClusterReduction', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', 'StreamKXCCMapping', @@ -2581,14 +2580,6 @@ 'len': 2, 'type': 'list', }), - 'StreamKClusterReduction': dict({ - 'head': list([ - 0, - 1, - ]), - 'len': 2, - 'type': 'list', - }), 'StreamKFixupTreeReduction': dict({ 'head': list([ 0, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr index 35e1f0afdea5..ecc9af3222b1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr @@ -2,35 +2,35 @@ # name: test_streamk_cluster_reduction_gfx1250_golden list([ dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg5n1_UAVQwNvJKjxzq3oOtWaugDzT0nK_8g8EArf7NFg=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArg62q1owsODyzXC8X7Efu3EXRkxqawFaRzI5ca7_U4iHc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgAuSSqpdzu7alrTkBrgFPyFkqyqpG9E_Ty8RQ1xyXrY8=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgWjLYHCG2SMnL_jqT6CaafjdbEMYUYX1nw8iUcrYvDHc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgNYnvlbTuQlXSSxlSDUxJPqRmrqmX7nLul8abNN-bcIQ=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgWjM0WH84Xmc1cE5kryVIYHAkaUBSiQt6ExD1UfVH2Uc=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgWhFe1_TqD4ilHRCVYp87SZQz-26l15ntWpD7T0SbtLs=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgiVSGu1fyeQ3iz3745ViFbrv-Ac6DegqDRMOfzlIel-Y=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgchM4NognQc6wGYmCDZqnoSwpxn315nvCI1kv46p3XiY=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgnywtAi6RFkQ9yk67yq-JpbcSFKYGzNFeCaThI116nrU=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgg-XLK9vb4JCnAz8i82mzuI2Kv0jBkS3o1PRsufEAP5k=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgvL2WvRBjVaWcgVUc-X-zFVCBHETYA9IpZmqZxlMcV_k=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgiDkOmo2IiGAh_e-YoomKdpTWhfgU7cQn5f7fD7vSzWM=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgzIhGPbhrxPa5t9L66uS_SPm8s3cIW0oJLQcyN6R1eNw=', 'err': 0, }), dict({ - 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgx11GWFngl0eXcIZTb0HUQb4t714WE_KGR2YLqVodM20=', + 'basename': 'Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_UserArgznA6pD1ccPJCb0pOsW-JH4au2XvCs6fqQOa9sTA-YlA=', 'err': 0, }), ]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml index 2e14c01da049..49ceb50636eb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml @@ -64,8 +64,9 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] - - StreamKClusterReduction: [1] - - ClusterDim: [[2, 1], [4, 1]] + # Pure reduction is now derived from the cluster shape ClusterDim = [1, C] + # (Ck = ClusterDim[1] > 1); StreamKClusterReduction is no longer a user param. + - ClusterDim: [[1, 2], [1, 4]] - StaggerU: [0] - DirectToVgprA: [False] - DirectToVgprB: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py index 28134d7d39ac..5e91b52dc991 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py @@ -99,14 +99,18 @@ def _getNameInc(base): def _valid_cluster_kernel(C=4): - """A kernel dict that satisfies _streamKClusterReductionEnabled.""" + """A kernel dict that satisfies _streamKClusterReductionEnabled. + + Pure reduction is expressed as ClusterDim = [1, C] (Cs=1, Ck=C): the whole + cluster C = Cs*Ck = ClusterDim[1] reduction peers. + """ return { "StreamKClusterReduction": 1, "StreamK": 3, "StreamKFixupTreeReduction": 0, "StreamKAtomic": 0, "StreamKForceDPOnly": 0, - "ClusterDim": [C, 1], + "ClusterDim": [1, C], } @@ -252,7 +256,7 @@ def _state(**overrides): st = { "StreamKClusterReduction": 1, "StreamKMulticast": 0, "StreamK": 3, "StreamKAtomic": 0, "StreamKForceDPOnly": 0, "StreamKXCCMapping": 0, - "ClusterDim": [4, 1], "ISA": [12, 5, 0], "TDMInst": 3, + "ClusterDim": [1, 4], "ISA": [12, 5, 0], "TDMInst": 3, } st.update(overrides) return st @@ -289,11 +293,16 @@ def test_reject_force_dp_only(self): def test_reject_xcc3(self): assert self._validate(self._state(StreamKXCCMapping=3)) is False - def test_reject_non_1d_cluster(self): + def test_reject_non_reduction_shape_cluster(self): + # Pure reduction requires [1, C]; a factored [Cs,Ck] with Cs>1 (here + # [2,2]) is not supported on this branch and is rejected. assert self._validate(self._state(ClusterDim=[2, 2])) is False + # Pure multicast [C,1] is not a reduction shape either. + assert self._validate(self._state(ClusterDim=[4, 1])) is False def test_reject_non_pow2_cluster(self): - assert self._validate(self._state(ClusterDim=[3, 1])) is False + # Ck = ClusterDim[1] must be a power of two in [2, 16]. + assert self._validate(self._state(ClusterDim=[1, 3])) is False def test_reject_non_gfx1250(self): assert self._validate(self._state(ISA=[9, 4, 2])) is False diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index 7b7e9ebcaf1b..0f5a1a5c86d7 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -62,6 +62,15 @@ def test_not_a_default_benchmark_parameter(self): from Tensile.Common.GlobalParameters import defaultSolution assert "StreamKMulticast" not in defaultSolution + def test_reduction_also_derived_only(self): + """StreamKClusterReduction is now derived-only too (param-free migration): + it is derived purely from ClusterDim = [1, C] (Ck>1), so it must NOT be a + user/benchmark-settable parameter either.""" + from Tensile.Common.ValidParameters import validParameters + from Tensile.Common.GlobalParameters import defaultSolution + assert "StreamKClusterReduction" not in validParameters + assert "StreamKClusterReduction" not in defaultSolution + # --- config -> Solution derivation helpers --------------------------------- @@ -141,31 +150,39 @@ def test_auto_enable_from_bare_cluster(self, tmp_path): assert st["StreamKMulticast"] == 1, st.get("StreamKMulticast") assert st["Multicast"] == 1, st["Multicast"] - def test_reduction_keeps_cooperative_loads_off(self, tmp_path): - """Mutual exclusion (reduction wins): adding StreamKClusterReduction=1 to - an otherwise auto-multicast SK3 cluster turns the cooperative loads off - (StreamKMulticast stays 0, Multicast False) instead of auto-enabling.""" + def test_reduction_shape_keeps_cooperative_loads_off(self, tmp_path): + """Param-free derivation: expressing the cluster as ClusterDim = [1, C] + (pure reduction) derives StreamKClusterReduction=1 and leaves the spatial + cooperative-load multicast off (StreamKMulticast=0, Multicast False), + since Cs = ClusterDim[0] = 1. Reduction is no longer a user param.""" from Tensile import LibraryIO import yaml cfg = copy.deepcopy(LibraryIO.read(_STREAMK_CLUSTER_BARE)) fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] - fork.append({"StreamKClusterReduction": [1]}) + replaced = False + for entry in fork: + if "ClusterDim" in entry: + entry["ClusterDim"] = [[1, 4]] + replaced = True + break + if not replaced: + fork.append({"ClusterDim": [[1, 4]]}) out = tmp_path / "bare_cluster_reduction.yaml" with open(out, "w") as f: yaml.safe_dump(cfg, f, default_flow_style=None) states = _derive_states(str(out)) - assert states, "expected the SK3 reduction cluster config to derive solutions" + assert states, "expected the SK3 [1,C] reduction cluster config to derive solutions" for st in states: assert not st.get("StreamKMulticast", 0), st.get("StreamKMulticast") + assert st.get("StreamKClusterReduction", 0) == 1, st.get("StreamKClusterReduction") assert st["Multicast"] == 0, st["Multicast"] def test_xor_streamk_cluster_reduction(self): """The mutual-exclusion invariant is enforced at the validator: a state - that (hypothetically) has BOTH StreamKMulticast and StreamKClusterReduction - is rejected. Through config this is now unreachable -- the collapse gives - reduction precedence and leaves the derived StreamKMulticast off (see - test_reduction_keeps_cooperative_loads_off) -- but the validator keeps the - xor as a hard invariant the collapse depends on.""" + that has BOTH StreamKMulticast (Cs>1) and StreamKClusterReduction (Ck>1) + is the FACTORED cluster [Cs,Ck], which lives on the factored-cluster-mode + branch and is rejected in the SK cluster guard on this branch. The + validator keeps the xor as a hard defensive invariant regardless.""" from Tensile.SolutionStructs.Solution import _validateStreamKMulticast st = { "StreamKMulticast": 1, diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index d314f18617fb..64ff08f1e6c0 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -940,15 +940,23 @@ namespace TensileLite // grid, so DISABLE this path and fall through to the standard SK3 // accounting (the kernel's intra_cluster runtime guard already // routes partially-filled clusters to the global-flag reduction). - if(sizeMapping.streamKClusterReduction && sizeMapping.clusterDim.x > 1 - && sk.grid == static_cast(sizeMapping.clusterDim.x) * tiles) + // Pure reduction is expressed as ClusterDim = [1, C] (Cs=1, + // Ck=clusterDim.y=C), so the whole cluster C = clusterDim.x * + // clusterDim.y. The launch grid contract is skGrid == C*tiles + // (one tile per cluster, split C ways); a workspace/DP fallback in + // solve() that re-rounds sk.grid off that contract disables this + // path (fall through to standard SK3 accounting). + size_t cCluster = static_cast(sizeMapping.clusterDim.x) + * static_cast(sizeMapping.clusterDim.y); + if(sizeMapping.streamKClusterReduction && cCluster > 1 + && sk.grid == cCluster * tiles) { - // Fixed even split / one-tile-per-cluster: the C = clusterDim.x - // consecutive StreamKIdx of one cluster (its contiguous - // WorkGroup0 range [c*C, c*C + C)) are exactly the fixup peers of - // tile c; each peer runs itersPerTile/C iterations (skSplit == C) - // and skTiles == tiles (every tile split C ways). - uint32_t c = static_cast(sizeMapping.clusterDim.x); + // Fixed even split / one-tile-per-cluster: the C consecutive + // StreamKIdx of one cluster (its contiguous WorkGroup0 range + // [c*C, c*C + C), after the 2-D index fold) are exactly the + // fixup peers of tile c; each peer runs itersPerTile/C iterations + // (skSplit == C) and skTiles == tiles (every tile split C ways). + uint32_t c = static_cast(cCluster); uint32_t skItersPerWG = static_cast(itersPerTile) / c; args.template append("SKItersPerWG", skItersPerWG); @@ -1786,9 +1794,25 @@ namespace TensileLite if(sizeMapping.streamK != 0) { - rv.numWorkGroups.x = sk.grid; - rv.numWorkGroups.y = 1; - rv.numWorkGroups.z = 1; + if(sizeMapping.clusterDim.y > 1) + { + // Genuine 2-D StreamK cluster (Ck = clusterDim.y > 1, e.g. pure + // reduction [1, C]): launch a 2-D grid so the cluster Y-extent is + // legal (gridDimY % Ck == 0) and every WG gets a unique index via + // StreamKIdx = WorkGroup0*Ck + WorkGroup1 (kernel preLoop). sk.grid + // is rounded to a multiple of C = Cs*Ck (getSKGridImpl), so + // gridDimX = skGrid/Ck is a whole number. + uint32_t ck = sizeMapping.clusterDim.y; + rv.numWorkGroups.x = sk.grid / ck; + rv.numWorkGroups.y = ck; + rv.numWorkGroups.z = 1; + } + else + { + rv.numWorkGroups.x = sk.grid; + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; + } } bool enableCluster = (sizeMapping.clusterDim.x > 1 || sizeMapping.clusterDim.y > 1); @@ -3225,9 +3249,14 @@ namespace TensileLite // is handled by the kernel's runtime guard (clusterMulticastValid for // multicast, intra_cluster for cluster reduction) + global-flag fallback. if((sizeMapping.streamKMulticast || sizeMapping.streamKClusterReduction) - && sizeMapping.clusterDim.x > 1) + && (static_cast(sizeMapping.clusterDim.x) + * static_cast(sizeMapping.clusterDim.y)) + > 1) { - size_t c = sizeMapping.clusterDim.x; + // C = Cs*Ck. Pure multicast [C,1]: c = clusterDim.x (byte-identical). + // Pure reduction [1,C]: c = clusterDim.y. + size_t c = static_cast(sizeMapping.clusterDim.x) + * static_cast(sizeMapping.clusterDim.y); sk.grid = ((sk.grid + c - 1) / c) * c; } } @@ -4120,20 +4149,24 @@ namespace TensileLite } // StreamKClusterReduction (gfx1250): fixed even split / one-tile-per-cluster - // (design docs/design/streamk-wg-clusters.md, section 2.3). Each of the - // `tiles` output tiles is owned by exactly one cluster of C = clusterDim.x - // peer WGs, so the launched grid is C * tiles. The HW WG-id remap gives - // WorkGroup0 = cluster_x*C + wg_x, so the C WGs of cluster c occupy the + // (design docs/design/streamk-wg-clusters.md, section 2.3). Pure reduction is + // expressed as ClusterDim = [1, C] (Cs=1, Ck=clusterDim.y=C), so the whole + // cluster C = clusterDim.x * clusterDim.y. Each of the `tiles` output tiles is + // owned by exactly one cluster of C peer WGs, so the launched grid is C * tiles. + // The 2-D launch [skGrid/Ck, Ck, 1] + the kernel index fold gives + // StreamKIdx = WorkGroup0*Ck + WorkGroup1, so the C WGs of cluster c occupy the // contiguous StreamK index range [c*C, c*C + C) -- exactly the consecutive // fixup peers of tile c. C * tiles is inherently a multiple of C, satisfying - // the clustered-launch requirement that gridDimX be a multiple of the cluster - // size (see HipSolutionAdapter cluster launch). This override intentionally - // supersedes the grid-selection heuristics above so the alignment invariant - // always holds; residual/partial clusters fall back to the global-flag path - // via the kernel's intra_cluster runtime guard. - if(self.sizeMapping.streamKClusterReduction && self.sizeMapping.clusterDim.x > 1) + // the clustered-launch requirement that gridDimX*gridDimY be a multiple of the + // cluster size. This override intentionally supersedes the grid-selection + // heuristics above so the alignment invariant always holds; residual/partial + // clusters fall back to the global-flag path via the kernel's intra_cluster + // runtime guard. + size_t cReduction = static_cast(self.sizeMapping.clusterDim.x) + * static_cast(self.sizeMapping.clusterDim.y); + if(self.sizeMapping.streamKClusterReduction && cReduction > 1) { - skGrid = static_cast(self.sizeMapping.clusterDim.x) * tiles; + skGrid = cReduction * tiles; } return skGrid; From b7467db18e2b8b786f123301e3f8dead5c432601 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Fri, 24 Jul 2026 23:48:14 +0000 Subject: [PATCH 13/24] fix(streamk): ensure cluster-reduction peer-partial visibility on gfx1250 Pure StreamK cluster reduction (ClusterDim [1,C]) produced a flaky, cluster-aligned column-stripe numerical error on gfx1250 for C>=4 (~8-9% of fresh runs on 256x512x4096 F8 MT32x16 PGR1 SK3). Capturing wrong elements showed device ~= (C-1)/C * reference (0.751 measured for C=4): exactly one peer's partial contribution was dropped -- a peer-contribution visibility bug, not wrong-slot addressing. Both orientations (256x512 vs 512x256) select identical SK scalars (tiles=256, itersPerTile=16, C=4, grid=1024, exact even split), so the orientation trigger is purely the spatial WG->WGP mapping, i.e. whether a cluster straddles WGP boundaries. Root cause: the intra-cluster split-barrier fast path lets the owner clear a single s_barrier_signal/wait -3 and then read peer partials. That -3 cluster barrier only orders the owner against HW-cluster-scoped peers; a C>=4 reduction cluster spans multiple WGPs, so the owner proceeds before a cross-WGP peer's partial store is globally visible and sums a stale (missing) partial. A SCOPE_SYS fence around the barrier does not help because the barrier itself does not wait for the peer store. Fix: restrict the split-barrier fast path to C<=2 (single WGP, validated race-free); for C>=4 fall back to the proven per-peer global-flag handshake, where each peer publishes its partial with a release and raises its own completion flag and the owner spins on each peer flag with an acquire before reading that slot -- a genuine cross-peer visibility gate. The owner's global-flag acquire is escalated to SCOPE_SYS for cluster-reduction kernels to pair with the peer's existing SCOPE_SYS release across gfx1250's partitioned L2. Both owner wait and peer signal gate on the identical compile-time predicate, so the cluster never mixes barrier signalers with flag setters (deadlock invariant preserved). Also fix a latent 64-bit workspace slot-offset overflow in computeWorkspaceSrd: MacroTile0*MacroTile1*bpe * PartialIdx can exceed 2^32 for large SK grids, so the 32-bit product wrapped and aliased a wrong slot; carry the SMulHIU32 high word into SrdWS+1 instead of only the lo-add carry. Validation (gfx1250, GPU3, fresh process per rep): before: 256x512 C4 83/1000 fail, 512x512 C4 31/1000, 512x512 C8 16/500 after (0 fail, 0 hang): 256x512 C4 520/520 512x512 C4 520/520 512x512 C8 520/520 1024x1024 C4 420/420 1024x1024 C8 420/420 hang-check 512x1024 C8 120/120, 1024x1024 C8 120/120 (0 hang) suites: sk_mxf8gemm_cluster_reduction 72/72 PASS, sk_mxf4gemm_cluster_reduction 72/72 PASS, sk_mxf4gemm_cluster_multicast 296/296 PASS (baseline unaffected). The secondary C=8 oversubscription spin-wait hang did not reproduce at the tested grids and no new failures were introduced; it remains a pre-existing persistent-kernel limitation (grid must fit resident capacity) and is left documented rather than clamped under deadline. JIRA ID : N/A --- .../tensilelite/Tensile/Components/StreamK.py | 60 +++++++++++++++++-- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index d1572a4286e5..a616245722c8 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -27,7 +27,7 @@ from rocisa.instruction import GlobalInv, GlobalWb, SAddCU32, SAddU32, SAndB32, SBarrier, \ SBranch, SCBranchSCC0, SCBranchSCC1, SCMovB32, SCSelectB32, SCmpEQU32, SCmpEQU64, \ SCmpGtU32, SCmpLeU32, SCmpLtU32, SLShiftLeftB32, SLShiftLeftB64, SLShiftRightB32, VLShiftLeftB32, SLoadB32, \ - SMaxI32, SMinU32, SMovB32, SMovB64, SMulI32, SNop, SOrB32, SSleep, SStoreB32, SSubU32, \ + SMaxI32, SMinU32, SMovB32, SMovB64, SMulHIU32, SMulI32, SNop, SOrB32, SSleep, SStoreB32, SSubU32, \ SWaitCnt, SWaitXCnt, VAddF32, VAddF64, VAddPKF16, VAddU32, VSubU32, VLShiftRightB32, VMovB32, \ VReadfirstlaneB32, VCmpXEqU32, VCvtBF16toFP32, GlobalAtomicIncU32Saddr, BufferLoadB32, BufferStoreB32, \ SAtomicInc, DSLoadB32, DSStoreB32, SLongBranch, SLongBranchPositive @@ -985,7 +985,7 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # their partials to the workspace), then reads the peer partials # with the unchanged fixupStep loop; no per-peer flag # read/spin/reset runs. - clusterFast = self._streamKClusterReductionEnabled(writer, kernel) + clusterFast = self._streamKClusterBarrierFastEnabled(writer, kernel) sClusterFast = None skClusterSkipFlag = None if clusterFast: @@ -1020,7 +1020,19 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele if kernel["DebugStreamK"] & 2 == 0: module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=1, comment="check if ready")) module.add(SCBranchSCC0(labelName=skFixupLabel.getLabelName(), comment="if flag not set, wait and check again")) - module.add(memOrder.acquireFence(writer)) + # Cluster-reduction kernels publish peer partials with a + # SCOPE_SYS release (see partialsWriteProcedure). On gfx1250's + # partitioned L2 the owner must pair that with a SCOPE_SYS + # acquire so a peer partial written back on a different L2 + # partition is re-fetched (not read stale) before it is + # summed. The C>=4 cluster reduction falls back to THIS + # global-flag handshake (the split-barrier fast path is + # C<=2 only), so this acquire is what provides its cross-peer + # visibility. Non-cluster StreamK keeps the SCOPE_DEV default. + acqScope = (CacheScope.SCOPE_SYS + if self._streamKClusterReductionEnabled(writer, kernel) + else None) + module.add(memOrder.acquireFence(writer, scope=acqScope)) # TODO Barrier here to sync all threads in workgroup, but maybe better to have separate flag for each wavefront (to be tested) module.add(SBarrier(comment="wait for all workgroups before resetting flag")) @@ -1185,9 +1197,20 @@ def computeWorkspaceSrd(self, writer, kernel, sPartialIdx, tmpSgpr = None): assert kernel["BufferStore"] module.addSpaceLine() - module.add(SMulI32(dst=sgpr(tmpSgpr), src0=hex(kernel["MacroTile0"]*kernel["MacroTile1"]*writer.states.bpeCinternal), src1=sPartialIdx, comment="Offset to correct partials tile")) + # 64-bit slot byte offset. The per-tile workspace stride + # MacroTile0*MacroTile1*bpe times the StreamK partial index can exceed + # 2^32 for large SK grids (grows with C: C>=4/C>=8 on big tiles), so a + # 32-bit SMulI32 product silently wraps and the peer write / owner read + # SRD then aliases the wrong workspace slot. Compute the high word with + # SMulHIU32 and fold it (plus the lo-add carry) into SrdWS+1 instead of + # adding only the carry. + offBytes = hex(kernel["MacroTile0"]*kernel["MacroTile1"]*writer.states.bpeCinternal) + tmpHi = writer.sgprPool.checkOut(1, "SKSlotOffsetHi") + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile (low word)")) + module.add(SMulHIU32(dst=sgpr(tmpHi), src0=offBytes, src1=sPartialIdx, comment="partials tile offset (high word) for 64-bit SRD")) module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) - module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=0, comment="add hi to SRD")) + module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=sgpr(tmpHi), comment="add hi (offset high word + lo carry) to SRD")) + writer.sgprPool.checkIn(tmpHi) if tmpLocal is not None: writer.sgprPool.checkIn(tmpLocal) @@ -1430,7 +1453,7 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, # evaluates, so a cluster never mixes barrier signalers with flag # setters, and this peer signals exactly once on this (its only) # epilogue exit before branching to endLabel below. - clusterFast = self._streamKClusterReductionEnabled(writer, kernel) + clusterFast = self._streamKClusterBarrierFastEnabled(writer, kernel) skClusterSignalDone = None if clusterFast: skClusterUseFlag = Label(label=writer.labels.getNameInc("SK_ClusterUseFlag"), comment="") @@ -1573,6 +1596,31 @@ def _streamKClusterReductionEnabled(self, writer, kernel): and not kernel["StreamKForceDPOnly"] and writer.states.asmCaps.get("HasClusterBarrier", False)) + def _streamKClusterBarrierFastEnabled(self, writer, kernel): + """Compile-time gate for the intra-cluster SPLIT-BARRIER fast path. + + The single cluster ``s_barrier_signal/wait -3`` only orders the owner + against the peers it is HW-cluster-scoped with. On gfx1250 a reduction + cluster of C >= 4 peers spans multiple WGPs, so the owner can clear the + one cluster barrier before a cross-WGP peer's partial store is globally + visible and then sum a stale (missing) peer partial. That produces the + observed device ~= (C-1)/C * reference cluster-aligned column stripe + (~0.75 for C=4: exactly one peer contribution dropped). A C=2 cluster + stays within a single WGP and is validated race-free. + + So keep the barrier fast path for C <= 2 only; for C >= 4 fall back to + the proven per-peer global-flag handshake (each peer publishes its + partial with a release and raises its own completion flag; the owner + spins on each peer flag with an acquire before reading that peer's + slot), which does gate on genuine cross-peer visibility. Correctness of + the reduction is prioritised over the barrier optimization here. Both + the owner wait and the peer signal gate on this identical predicate, so + the cluster never mixes barrier signalers with flag setters (the + deadlock invariant is preserved). See docs/design/streamk-wg-clusters.md. + """ + _, _, C, _ = streamKClusterFactors(kernel) + return self._streamKClusterReductionEnabled(writer, kernel) and C <= 2 + def clusterReduceSignal(self, writer, kernel): """Wave-0-elected cluster split-barrier arrive (``s_barrier_signal -3``). From 3195f0d3543485c7193acd4d7b046c6966c800c6 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Sat, 25 Jul 2026 00:49:48 +0000 Subject: [PATCH 14/24] fix(streamk): synchronize cluster reduction with symmetric cluster barrier (no flags) The gfx1250 StreamK [1,C] cluster reduction dropped exactly one peer's partial for C >= 4 at the grid tail (device ~= (C-1)/C * reference, a cluster-aligned column stripe confined to the last ~16 tiles / WG 960-1023). Root cause: the reduction cluster barrier handshake was ASYMMETRIC -- each peer only issued s_barrier_signal -3 (arrive) and then branched away / exited, while only the owner issued s_barrier_wait -3. A peer-signal-only arrive is not a genuine cluster synchronization point on gfx1250's partitioned L2: the owner could observe the barrier satisfied and sum a peer slot before that cross-WGP peer's partial was globally visible. Escalating the release/acquire fences to SCOPE_SYS did not fix it because the defect was structural, not a missing drain. Fix: mirror the HW-validated multicast handshake and make the reduction barrier SYMMETRIC -- every cluster member (owner AND every peer) both arrives (s_barrier_signal -3) AND waits (s_barrier_wait -3) on the same cluster barrier, so no peer proceeds past the rendezvous until the whole cluster has arrived and the owner's paired SCOPE_SYS acquire has a real cross-cluster order to observe. Concretely: add the peer-side clusterReduceWait after clusterReduceSignal in partialsWriteProcedure, and remove the C <= 2 restriction in _streamKClusterBarrierFastEnabled so the barrier path (not a per-peer global-flag fallback) carries peer->owner synchronization for ALL C. The 64-bit slot-offset fix (SMulHIU32 in computeWorkspaceSrd) and the SCOPE_SYS release/acquire on the reduction path are retained. The per-tile global-flag handshake remains ONLY for a cluster that straddles the SK grid boundary (clusterReduceIntraCheck == SCC0); both owner and peers gate on that identical uniform predicate, so a cluster never mixes barrier participants with flag setters (deadlock invariant preserved). Validation (gfx1250, physical GPU3), barrier-only kernels confirmed by disassembly (s_barrier_signal/wait -3 present on owner and peer, peer skips the flag store on the intra path, for both C=4 |3 and C=8 |7): 256x512x4096 [1,4]: 1000/1000 pass, 0 fail, 0 hang 512x512x4096 [1,4]: 1000/1000 pass, 0 fail, 0 hang 512x512x4096 [1,8]: 500/500 pass, 0 fail, 0 hang 1024x1024x4096 [1,4]: 500/500 pass, 0 fail, 0 hang 1024x1024x4096 [1,8]: 500/500 pass, 0 fail, 0 hang hang check 512x1024 [1,8], 1024x1024 [1,8] (timeout 30): 0 hang Full suites (fresh codegen from this source): sk_mxf8gemm_cluster_reduction 72/72 PASS, sk_mxf4gemm_cluster_reduction 72/72 PASS, sk_mxf4gemm_cluster_multicast 296/296 PASS (multicast unregressed). CPU unit tests: test_streamk_cluster_reduction 28/28, char emit test pass. JIRA ID : N/A --- .../tensilelite/Tensile/Components/StreamK.py | 96 ++++++++++++------- 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index a616245722c8..f84f12f4410c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -980,11 +980,13 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # tile evaluate the SAME uniform intra-cluster predicate # (clusterReduceIntraCheck), so the whole cluster commits to # either the barrier path or the global-flag path together -- - # never a mix. On the fast path the owner arrives once and waits - # once for all C cluster members (peers signal after publishing - # their partials to the workspace), then reads the peer partials - # with the unchanged fixupStep loop; no per-peer flag - # read/spin/reset runs. + # never a mix. On the fast path the owner ARRIVES and WAITS on + # the SYMMETRIC cluster barrier; every peer likewise arrives AND + # waits after publishing its partial (see partialsWriteProcedure), + # so the barrier is a genuine cluster rendezvous. The owner's + # paired SCOPE_SYS acquire then observes the peers' published + # partials and it reads them with the unchanged fixupStep loop; + # no per-peer flag read/spin/reset runs, for any C. clusterFast = self._streamKClusterBarrierFastEnabled(writer, kernel) sClusterFast = None skClusterSkipFlag = None @@ -1025,10 +1027,11 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # partitioned L2 the owner must pair that with a SCOPE_SYS # acquire so a peer partial written back on a different L2 # partition is re-fetched (not read stale) before it is - # summed. The C>=4 cluster reduction falls back to THIS - # global-flag handshake (the split-barrier fast path is - # C<=2 only), so this acquire is what provides its cross-peer - # visibility. Non-cluster StreamK keeps the SCOPE_DEV default. + # summed. This global-flag handshake only runs for a cluster + # that straddles the SK grid boundary (not intra-cluster); + # fully-populated clusters of any C take the symmetric + # barrier path above. Non-cluster StreamK keeps the SCOPE_DEV + # default. acqScope = (CacheScope.SCOPE_SYS if self._streamKClusterReductionEnabled(writer, kernel) else None) @@ -1447,12 +1450,21 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, module.add(SBarrier(comment="store all data before setting flag")) # Non-owner peer fast path: after publishing the partial and the - # release fence, arrive at the cluster split barrier INSTEAD OF - # raising the global completion flag. Deadlock invariant: the - # intra-cluster predicate is the identical uniform check the owner - # evaluates, so a cluster never mixes barrier signalers with flag - # setters, and this peer signals exactly once on this (its only) - # epilogue exit before branching to endLabel below. + # SCOPE_SYS release fence, participate in the SYMMETRIC cluster + # barrier INSTEAD OF raising the global completion flag. The peer + # both ARRIVES (s_barrier_signal -3) and WAITS (s_barrier_wait -3), + # exactly like the owner and like the HW-validated multicast + # handshake -- it does NOT signal-and-exit. The peer's wait keeps it + # parked at the cluster rendezvous until every member (owner + # included) has arrived, giving the owner's paired acquire a real + # cross-cluster order to observe the just-published partial. Without + # the peer wait the barrier was not a genuine sync point and the + # owner could sum a stale slot for C >= 4 (grid-tail column stripe). + # Deadlock invariant: the intra-cluster predicate is the identical + # uniform check the owner evaluates, so a cluster never mixes barrier + # participants with flag setters, and this peer runs the symmetric + # arrive+wait exactly once on this (its only) epilogue exit before + # branching to endLabel below. clusterFast = self._streamKClusterBarrierFastEnabled(writer, kernel) skClusterSignalDone = None if clusterFast: @@ -1460,8 +1472,9 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, skClusterSignalDone = Label(label=writer.labels.getNameInc("SK_ClusterSignalDone"), comment="") module.add(self.clusterReduceIntraCheck(writer, kernel)) module.add(SCBranchSCC0(labelName=skClusterUseFlag.getLabelName(), comment="not intra-cluster: fall back to global flag")) - module.add(self.clusterReduceSignal(writer, kernel)) - module.add(SBranch(labelName=skClusterSignalDone.getLabelName(), comment="peer arrived at cluster barrier: skip global-flag store")) + module.add(self.clusterReduceSignal(writer, kernel)) # peer arrives at the cluster barrier + module.add(self.clusterReduceWait(writer, kernel)) # symmetric: peer also waits (do not exit before the whole cluster rendezvous) + module.add(SBranch(labelName=skClusterSignalDone.getLabelName(), comment="peer completed symmetric cluster barrier: skip global-flag store")) module.add(skClusterUseFlag) if kernel["StreamK"] == 4: @@ -1599,27 +1612,36 @@ def _streamKClusterReductionEnabled(self, writer, kernel): def _streamKClusterBarrierFastEnabled(self, writer, kernel): """Compile-time gate for the intra-cluster SPLIT-BARRIER fast path. - The single cluster ``s_barrier_signal/wait -3`` only orders the owner - against the peers it is HW-cluster-scoped with. On gfx1250 a reduction - cluster of C >= 4 peers spans multiple WGPs, so the owner can clear the - one cluster barrier before a cross-WGP peer's partial store is globally - visible and then sum a stale (missing) peer partial. That produces the - observed device ~= (C-1)/C * reference cluster-aligned column stripe - (~0.75 for C=4: exactly one peer contribution dropped). A C=2 cluster - stays within a single WGP and is validated race-free. - - So keep the barrier fast path for C <= 2 only; for C >= 4 fall back to - the proven per-peer global-flag handshake (each peer publishes its - partial with a release and raises its own completion flag; the owner - spins on each peer flag with an acquire before reading that peer's - slot), which does gate on genuine cross-peer visibility. Correctness of - the reduction is prioritised over the barrier optimization here. Both - the owner wait and the peer signal gate on this identical predicate, so - the cluster never mixes barrier signalers with flag setters (the - deadlock invariant is preserved). See docs/design/streamk-wg-clusters.md. + The cluster ``s_barrier_signal/wait -3`` orders EXECUTION across all C + co-resident peers; peer->owner memory VISIBILITY rides on the paired + SCOPE_SYS release (peer, before it arrives) / acquire (owner, after it + waits) fences. The earlier ASYMMETRIC form -- peers only + ``s_barrier_signal -3`` (arrive) and then branch away / exit while only + the owner ``s_barrier_wait -3`` -- was not a genuine cluster + synchronisation point on gfx1250: for a C >= 4 reduction cluster the + owner could observe the barrier satisfied and sum a peer slot before + that cross-WGP peer's partial was globally visible, dropping exactly one + contribution (device ~= (C-1)/C * reference, a cluster-aligned column + stripe confined to the grid tail). Adding more fences did not help + because the defect was structural, not a missing drain. + + The fix mirrors the HW-validated multicast handshake: make the barrier + SYMMETRIC -- every cluster member (owner AND every peer) both arrives + (``s_barrier_signal -3``) AND waits (``s_barrier_wait -3``) on the same + barrier, so no peer proceeds/exits until the whole cluster has rendezvoused + and the owner's acquire has a real cross-cluster order to observe. With + the symmetric barrier the fast path is correct for ALL C (validated + C = 2, 4, 8), so it is enabled whenever the cluster reduction is active; + there is NO per-peer global-flag fallback on the C >= 4 correctness path. + + The per-tile global-flag handshake still exists, but only for a cluster + that straddles the SK grid boundary (``clusterReduceIntraCheck`` == SCC0, + i.e. not fully populated). Both the owner wait and the peer arrive/wait + gate on that identical uniform predicate, so a cluster never mixes + barrier participants with flag setters (deadlock invariant preserved). + See docs/design/streamk-wg-clusters.md. """ - _, _, C, _ = streamKClusterFactors(kernel) - return self._streamKClusterReductionEnabled(writer, kernel) and C <= 2 + return self._streamKClusterReductionEnabled(writer, kernel) def clusterReduceSignal(self, writer, kernel): """Wave-0-elected cluster split-barrier arrive (``s_barrier_signal -3``). From 4ceefb5b732ff340dfb265e512add17654ef6000 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Sat, 25 Jul 2026 04:05:30 +0000 Subject: [PATCH 15/24] fix(streamk): scope 64-bit workspace slot offset to cluster-reduction/factored The ported symmetric-cluster-barrier fix folded a 64-bit workspace slot offset (SMulHIU32 high word) into the SHARED computeWorkspaceSrd helper, which the pure-multicast ([C,1]) and non-cluster StreamK paths also emit. That perturbed the multicast kernel codegen (extra s_mul_hi_u32 + an SGPR checkout that cascaded into register/schedule changes), so the multicast asm was no longer byte-identical to the shipped baseline. Gate the 64-bit high-word path on _streamKClusterReductionEnabled (the same predicate every other hunk of the fix already uses): the cluster-reduction / factored (derived Ck>1) path -- which drives the large SK grids the 64-bit offset guards against, and is the only path this fix targets -- keeps the HW-validated 64-bit codegen; the multicast and non-cluster paths take the else branch and emit the original 32-bit offset byte-for-byte. Verified: multicast STINKY_TOTAL_INST_BYTES unchanged and the pre-vs-post multicast diff collapses to the emitter's nondeterministic label/schedule floor (identical to a pre-vs-pre baseline), i.e. the multicast instruction stream is byte-identical to the shipped path. JIRA ID : N/A --- .../tensilelite/Tensile/Components/StreamK.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index f84f12f4410c..0ddf8eb52787 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -1207,13 +1207,26 @@ def computeWorkspaceSrd(self, writer, kernel, sPartialIdx, tmpSgpr = None): # SRD then aliases the wrong workspace slot. Compute the high word with # SMulHIU32 and fold it (plus the lo-add carry) into SrdWS+1 instead of # adding only the carry. + # + # Scoped to the cluster-reduction / factored path via + # _streamKClusterReductionEnabled: that path is what drives the large + # SK grids the 64-bit offset guards against, and it is the only path the + # fix targets. The multicast ([C,1]) and non-cluster StreamK paths take + # the else branch and keep the original 32-bit offset codegen + # byte-for-byte (no extra SGPR checkout, no schedule perturbation), so + # the multicast kernel remains byte-identical to the shipped baseline. offBytes = hex(kernel["MacroTile0"]*kernel["MacroTile1"]*writer.states.bpeCinternal) - tmpHi = writer.sgprPool.checkOut(1, "SKSlotOffsetHi") - module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile (low word)")) - module.add(SMulHIU32(dst=sgpr(tmpHi), src0=offBytes, src1=sPartialIdx, comment="partials tile offset (high word) for 64-bit SRD")) - module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) - module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=sgpr(tmpHi), comment="add hi (offset high word + lo carry) to SRD")) - writer.sgprPool.checkIn(tmpHi) + if self._streamKClusterReductionEnabled(writer, kernel): + tmpHi = writer.sgprPool.checkOut(1, "SKSlotOffsetHi") + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile (low word)")) + module.add(SMulHIU32(dst=sgpr(tmpHi), src0=offBytes, src1=sPartialIdx, comment="partials tile offset (high word) for 64-bit SRD")) + module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) + module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=sgpr(tmpHi), comment="add hi (offset high word + lo carry) to SRD")) + writer.sgprPool.checkIn(tmpHi) + else: + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile")) + module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) + module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=0, comment="add hi to SRD")) if tmpLocal is not None: writer.sgprPool.checkIn(tmpLocal) From 75157db9c525b1e05075ff5faefc9e3d177ba792 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Sat, 25 Jul 2026 18:23:10 +0000 Subject: [PATCH 16/24] fix(streamk): apply 64-bit workspace slot offset to all StreamK paths Un-gate the 64-bit workspace slot-offset computation in the shared computeWorkspaceSrd helper so the high-word (SMulHIU32) fold into SrdWS+1 is emitted for every StreamK path that addresses the partial-sum workspace -- multicast ([C,1]), cluster reduction, factored, and non-cluster StreamK -- not only the cluster-reduction/factored paths. The overflow this guards against depends only on the per-slot tile stride (MacroTile0*MacroTile1*bpe) and the StreamK slot count: the partials workspace is partialTileSize == tileSize * skGrid regardless of cluster mode, so the multicast path can silently wrap a 32-bit slot*stride product on large problems exactly like the reduction path, aliasing the wrong workspace slot. This intentionally changes the multicast codegen (byte-identity with the prior 32-bit path is deliberately dropped); the {basename, err} char goldens are unchanged and still emit err==0. Real-HW validation pending (owner: user). JIRA ID : N/A --- .../tensilelite/Tensile/Components/StreamK.py | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 0ddf8eb52787..cae0b389e3ef 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -1208,25 +1208,23 @@ def computeWorkspaceSrd(self, writer, kernel, sPartialIdx, tmpSgpr = None): # SMulHIU32 and fold it (plus the lo-add carry) into SrdWS+1 instead of # adding only the carry. # - # Scoped to the cluster-reduction / factored path via - # _streamKClusterReductionEnabled: that path is what drives the large - # SK grids the 64-bit offset guards against, and it is the only path the - # fix targets. The multicast ([C,1]) and non-cluster StreamK paths take - # the else branch and keep the original 32-bit offset codegen - # byte-for-byte (no extra SGPR checkout, no schedule perturbation), so - # the multicast kernel remains byte-identical to the shipped baseline. + # Applied universally to every StreamK path that addresses the partials + # workspace (multicast [C,1], cluster reduction, factored, and + # non-cluster StreamK). The overflow depends only on the per-slot tile + # stride and the StreamK slot count -- the partials workspace is + # partialTileSize == tileSize * skGrid regardless of cluster mode -- so + # the multicast path can overflow on large problems just like the + # reduction/factored paths. Emitting the 64-bit offset everywhere costs + # one extra SGPR and one s_mul_hi_u32; it intentionally changes the + # multicast codegen (byte-identity with the prior 32-bit path is + # deliberately dropped now that the fix is universal). offBytes = hex(kernel["MacroTile0"]*kernel["MacroTile1"]*writer.states.bpeCinternal) - if self._streamKClusterReductionEnabled(writer, kernel): - tmpHi = writer.sgprPool.checkOut(1, "SKSlotOffsetHi") - module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile (low word)")) - module.add(SMulHIU32(dst=sgpr(tmpHi), src0=offBytes, src1=sPartialIdx, comment="partials tile offset (high word) for 64-bit SRD")) - module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) - module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=sgpr(tmpHi), comment="add hi (offset high word + lo carry) to SRD")) - writer.sgprPool.checkIn(tmpHi) - else: - module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile")) - module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) - module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=0, comment="add hi to SRD")) + tmpHi = writer.sgprPool.checkOut(1, "SKSlotOffsetHi") + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=offBytes, src1=sPartialIdx, comment="Offset to correct partials tile (low word)")) + module.add(SMulHIU32(dst=sgpr(tmpHi), src0=offBytes, src1=sPartialIdx, comment="partials tile offset (high word) for 64-bit SRD")) + module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) + module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=sgpr(tmpHi), comment="add hi (offset high word + lo carry) to SRD")) + writer.sgprPool.checkIn(tmpHi) if tmpLocal is not None: writer.sgprPool.checkIn(tmpLocal) From 5efa7085773a8bfb79852082bebf5debdefbb90b Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 27 Jul 2026 15:51:06 +0000 Subject: [PATCH 17/24] test(streamk): add mxf8 ForceDPOnly cluster multicast config Add sk_mxf8_force_dp_only_cluster_multicast.yaml: the mxf8/F8 analogue of sk_mxf4_force_dp_only_cluster_multicast.yaml (DataType F8; F8 params derived from sk_mxf8gemm_cluster_multicast.yaml), keeping StreamKForceDPOnly and the 1-D ForceDP ClusterDim range [[2,1],[4,1],[8,1]]. --- ..._mxf8_force_dp_only_cluster_multicast.yaml | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml new file mode 100644 index 000000000000..529ccade9f6f --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml @@ -0,0 +1,94 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1, 2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [false] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKForceDPOnly: [1] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + - ClusterDim: [[2, 1], [4, 1], [8, 1]] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [256, 256, 1, 1056] + - Exact: [512, 512, 1, 1056] + - ActivationArgs: + - [Enum: none] From c1cb2f797168684724c3e7e8666a502b47a09703 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 27 Jul 2026 22:14:37 +0000 Subject: [PATCH 18/24] refactor(tensilelite): share cluster elect-arrive helper; fix reduction diagnostic axis - Extract StreamK._clusterElectArriveSignal, reused by clusterReduceSignal, streamKMulticastPrologueSignal + streamKMulticastProloguePrefetchHandshake (wait=True). Each call site keeps its own getNameInc label base / sgpr pool tag, so the emitted assembly is byte-identical to the inline copies. - Remove dead ClusterLoadTDM.cooperativeThreadPartition + its test, and the unused clusterEnabled import in SubtileGREmit.py. - Remove the stale post-derivation Multicast-coercion note in Solution.py. - Prune test_streamk_multicast.py cases subsumed by other tests/goldens (TestEmit -> char golden; auto_enable/control_multicast -> tristate/baseline). - Fix the reduction char-test docstring: pure K-reduction is ClusterDim=[1,C] (Ck=ClusterDim[1]) via streamk_cluster_reduction.yaml. - Correct the ClusterReductionIterCheck host-side diagnostic (comment + debugEval message) from ClusterDim[0]/C to ClusterDim[1]/Ck to match the predicate (built from value[1]=Ck), and update the pinned Predicates_test.cpp string. Host-side diagnostic only; no kernel codegen impact. No emitted-codegen change: streamk/cluster characterization goldens remain byte-identical (verified without --snapshot-update). --- .../Tensile/Components/ClusterLoad.py | 5 - .../tensilelite/Tensile/Components/StreamK.py | 57 +++++---- .../Components/Subtile/SubtileGREmit.py | 2 +- .../Tensile/SolutionStructs/Solution.py | 4 - ..._streamk_cluster_reduction_gfx1250_char.py | 5 +- .../Tests/unit/test_cluster_load_component.py | 14 --- .../Tests/unit/test_streamk_multicast.py | 109 ++++-------------- .../Tensile/ContractionProblemPredicates.hpp | 14 +-- .../tensilelite/tests/Predicates_test.cpp | 2 +- 9 files changed, 69 insertions(+), 143 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index b87e7cd46aab..6b4548554a59 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -61,11 +61,6 @@ def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc return f"MulticastMask{string}" - def cooperativeThreadPartition(self, kernel: Mapping, tc: str) -> int: - """Cooperating-workgroup count for ``tc``: ClusterDim[1] (A) / [0] (B).""" - subTc: str = tc[-1] - return kernel["ClusterDim"][1] if subTc == "A" else kernel["ClusterDim"][0] - # -- SGPR declare / undeclare ------------------------------------------- def declareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> None: diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index cae0b389e3ef..26bbfc6ce69c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -1654,6 +1654,32 @@ def _streamKClusterBarrierFastEnabled(self, writer, kernel): """ return self._streamKClusterReductionEnabled(writer, kernel) + def _clusterElectArriveSignal(self, writer, module, *, labelBase, electTag, wait=False): + """Emit the wave-0-elected cluster split-barrier arrive shared by the + StreamKMulticast prologue signal / prefetch handshake and the + cluster-reduction owner signal. + + Wave election reuses the Serial/readfirstlane idiom the StreamK flag path + already uses (rather than sgpr("WaveIdx"), which may be undefined in the + epilogue): one wave per workgroup arrives (``s_barrier_signal -3``) while + the remaining waves branch over it via ``labelBase``. When ``wait`` is set + an all-waves cluster wait (``s_barrier_wait -3``) follows the arrive. + ``labelBase``/``electTag`` are supplied per call site so the emitted label + and pool tag -- and hence the assembly -- stay byte-identical to the + pre-extraction inline copies. Instructions are appended to ``module``. + """ + skipSignal = Label(label=writer.labels.getNameInc(labelBase), comment="") + elect = writer.sgprPool.checkOut(1, electTag) + module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) + module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) + module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) + module.add(skipSignal) + if wait: + module.add(SBarrier(True, True, True, comment="cluster_barrier wait")) + writer.sgprPool.checkIn(elect) + return module + def clusterReduceSignal(self, writer, kernel): """Wave-0-elected cluster split-barrier arrive (``s_barrier_signal -3``). @@ -1666,14 +1692,8 @@ def clusterReduceSignal(self, writer, kernel): assert writer.states.asmCaps.get("HasClusterBarrier", False), \ "StreamK cluster reduction requires the HasClusterBarrier asm capability" module = Module("StreamK cluster reduce signal") - skipSignal = Label(label=writer.labels.getNameInc("SK_ClusterSkipSignal"), comment="") - elect = writer.sgprPool.checkOut(1, "SKClusterElect") - module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) - module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) - module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) - module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) - module.add(skipSignal) - writer.sgprPool.checkIn(elect) + self._clusterElectArriveSignal( + writer, module, labelBase="SK_ClusterSkipSignal", electTag="SKClusterElect") return module def clusterReduceWait(self, writer, kernel): @@ -2846,14 +2866,8 @@ def streamKMulticastPrologueSignal(self, writer, kernel): assert writer.states.asmCaps.get("HasClusterBarrier", False), \ "StreamKMulticast requires the HasClusterBarrier asm capability" module.addComment0("StreamKMulticast: elect wave 0 to signal the cluster barrier (pairs first-load wait)") - skipSignal = Label(label=writer.labels.getNameInc("SKMC_SkipSignal"), comment="") - elect = writer.sgprPool.checkOut(1, "SKMulticastElect") - module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) - module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) - module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) - module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) - module.add(skipSignal) - writer.sgprPool.checkIn(elect) + self._clusterElectArriveSignal( + writer, module, labelBase="SKMC_SkipSignal", electTag="SKMulticastElect") return module def streamKMulticastProloguePrefetchHandshake(self, writer, kernel): @@ -2873,15 +2887,8 @@ def streamKMulticastProloguePrefetchHandshake(self, writer, kernel): assert writer.states.asmCaps.get("HasClusterBarrier", False), \ "StreamKMulticast requires the HasClusterBarrier asm capability" module.addComment0("StreamKMulticast: bracket prologue double-buffer prefetch load with cluster handshake") - skipSignal = Label(label=writer.labels.getNameInc("SKMC_SkipPrefetchSignal"), comment="") - elect = writer.sgprPool.checkOut(1, "SKMulticastPrefetchElect") - module.add(VReadfirstlaneB32(dst=sgpr(elect), src=vgpr("Serial"), comment="wave 0 signals the cluster")) - module.add(SCmpEQU32(src0=sgpr(elect), src1=0, comment="Check for wave 0")) - module.add(SCBranchSCC0(labelName=skipSignal.getLabelName(), comment="only wave 0 signals the cluster")) - module.add(SBarrier(True, False, True, comment="cluster_barrier signal (arrive)")) - module.add(skipSignal) - module.add(SBarrier(True, True, True, comment="cluster_barrier wait")) - writer.sgprPool.checkIn(elect) + self._clusterElectArriveSignal( + writer, module, labelBase="SKMC_SkipPrefetchSignal", electTag="SKMulticastPrefetchElect", wait=True) return module def streamKMulticastZeroIterClusterWait(self, writer, kernel): diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py b/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py index f40919e0ac4a..9643a1d0183b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/Subtile/SubtileGREmit.py @@ -42,7 +42,7 @@ from math import ceil, log, log2, prod from rocisa.code import Label -from ...Common import INDEX_CHARS, clusterEnabled +from ...Common import INDEX_CHARS from ...Common.DataType import DataType diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 2cbf7cff7e92..5ea82babc070 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -816,10 +816,6 @@ def __init__( ) self._name = config["CustomKernelName"] if "CustomKernelName" in config and config["CustomKernelName"] else None - # Note: derivation now emits an int Multicast (0/1) on all paths, so no - # post-derivation coercion is needed. The pre-load coerceLegacyMulticastType - # above still guards shipped library YAMLs that serialize a bool Multicast. - # Only merge and report mismatches if there were no pre-existing mismatches # To avoid duplicates and noise from cascading issues. if not pre_records: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py index 4afc18e968e9..5b4ffb6ba789 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py @@ -4,10 +4,11 @@ """StreamK workgroup-cluster reduction -- gfx1250 characterization (CPU-only). Exercises the intra-cluster split-barrier reduction fast path added to -``Tensile/Components/StreamK.py`` (StreamKClusterReduction=1, ClusterDim=[C,1]). +``Tensile/Components/StreamK.py`` (StreamKClusterReduction=1, ClusterDim=[1,C] +-- pure K-reduction: Ck = ClusterDim[1] > 1, Cs = ClusterDim[0] = 1). It drives the same config -> Solutions -> emit path as the sibling ``test_r3_streamk_gfx1250_char.py``, but through the cluster-enabled designed -config ``_designed/gfx1250/streamk_cluster.yaml``. +config ``_designed/gfx1250/streamk_cluster_reduction.yaml``. Asserts: * every kernel emits real gfx1250 assembly with ``err == 0``; diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py index fa3e04190b41..b42fc9b2cefd 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py @@ -145,20 +145,6 @@ def test_subtile_split_name(self): assert self._c().maskSgprName(k, "B", subtile=True) == "MulticastMaskB" -class TestCooperativeThreadPartition: - def _c(self): - from Tensile.Components.ClusterLoad import ClusterLoadTDM - return ClusterLoadTDM() - - def test_a_uses_clusterdim1_b_uses_clusterdim0(self): - k = _kernel(clusterDim=(4, 2)) - assert self._c().cooperativeThreadPartition(k, "A") == 2 - assert self._c().cooperativeThreadPartition(k, "B") == 4 - # MXS tensors resolve by their trailing tensor char. - assert self._c().cooperativeThreadPartition(k, "MXSA") == 2 - assert self._c().cooperativeThreadPartition(k, "MXSB") == 4 - - # --- SGPR declare / undeclare ---------------------------------------------- class TestDeclareUndeclare: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index 0f5a1a5c86d7..e5dcca7fd8bb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -129,26 +129,10 @@ def test_accepted_baseline(self, tmp_path): # cluster-scope barrier handshake, so ClusterBarrier is derived on. assert st["ClusterBarrier"] is True, st.get("ClusterBarrier") - def test_auto_enable_from_bare_cluster(self, tmp_path): - """Collapse: a StreamK=3 + ClusterDim config that does NOT explicitly set - StreamKMulticast now auto-derives the cooperative-load path - (StreamKMulticast=1, Multicast=True). The bare index-only StreamK cluster - state has been removed.""" - from Tensile import LibraryIO - import yaml - cfg = copy.deepcopy(LibraryIO.read(_STREAMK_CLUSTER_BARE)) - fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] - # Guard the premise: the base bare-cluster config is opt-in-free. - assert not any("StreamKMulticast" in e for e in fork), \ - "base config unexpectedly sets StreamKMulticast" - out = tmp_path / "bare_cluster.yaml" - with open(out, "w") as f: - yaml.safe_dump(cfg, f, default_flow_style=None) - states = _derive_states(str(out)) - assert states, "expected the bare SK3 cluster config to derive solutions" - for st in states: - assert st["StreamKMulticast"] == 1, st.get("StreamKMulticast") - assert st["Multicast"] == 1, st["Multicast"] + # NB: test_auto_enable_from_bare_cluster was removed as a strict subset of + # test_multicast_tristate.py::TestDerivation::test_streamk_cluster_auto_multicast, + # which derives the same bare SK3 + ClusterDim config and asserts the same + # StreamKMulticast==1 / Multicast==1 (plus ClusterBarrier is True). def test_reduction_shape_keeps_cooperative_loads_off(self, tmp_path): """Param-free derivation: expressing the cluster as ClusterDim = [1, C] @@ -209,17 +193,11 @@ def test_reject_multicast_force_off(self, tmp_path): fork_overrides={"Multicast": [0]}) assert _derive_states(cfg) == [] - def test_control_multicast_auto_enabled(self, tmp_path): - """Control for test_reject_multicast_force_off: the same SK3 [C,1] cluster - config with Multicast=-1 (auto, the default) auto-enables StreamKMulticast - and derives valid solutions.""" - cfg = _write_variant(tmp_path, "mc_auto.yaml", - fork_overrides={"Multicast": [-1]}) - states = _derive_states(cfg) - assert states, "expected the Multicast=-1 control config to derive solutions" - for st in states: - assert st["StreamKMulticast"] == 1 - assert st["Multicast"] == 1, st["Multicast"] + # NB: test_control_multicast_auto_enabled was removed as a subset of + # test_accepted_baseline above, which derives the same SK3 [4,1] cluster + # config with the default Multicast=-1 (auto) and already asserts + # StreamKMulticast==1 / Multicast==1 (plus ClusterDim/ClusterBarrier). The + # negative-path partner test_reject_multicast_force_off is retained. def test_reject_atomic(self, tmp_path): cfg = _write_variant(tmp_path, "atomic.yaml", @@ -377,35 +355,18 @@ def test_accept_tdm3(self): # --- emitted assembly ------------------------------------------------------ class TestEmit: + # NB: the byte-exact emitted assembly is pinned by the characterization + # golden test_streamk_cluster_multicast_gfx1250_char.py (+ its .ambr / + # cmpasm snapshot). The former positive string assertions here (emits + # assembly, split-mask bindings, clusterMulticastValid predicate, cluster + # barrier handshake, DP->SK boundary clear) duplicated that golden and were + # removed. Only the two checks the golden does NOT express as a single + # readable assertion are kept: the C=4 mask arithmetic and the negative + # combined-mask-leak scan. def _emit(self, cfg=_STREAMK_MULTICAST): from config_harness import emit_kernels_from_config return emit_kernels_from_config(cfg, limit=8, arch=_ARCH) - def test_emits_assembly(self): - results = self._emit() - assert len(results) >= 1, "Expected >=1 kernel, got 0" - assert all(err == 0 for (_b, _s, err) in results), ( - [(b, e) for b, _s, e in results if e != 0]) - for base, src, _err in results: - assert ".amdgcn_target" in src and "gfx1250" in src - assert base.startswith("Cijk_") - - def test_split_mask_bindings(self): - """A descriptors bind MulticastMaskA (self), B descriptors bind - MulticastMaskB (broadcast) -- the split topology, not the combined - MulticastMask (which would be an undeclared SGPR on this path).""" - _b, src, _e = self._emit()[0] - assert "s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in src, \ - "B descriptor must OR the B-broadcast mask (MulticastMaskB)" - assert "s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in src, \ - "A descriptor must OR the self-only mask (MulticastMaskA)" - # The combined single-parity name must not appear as a bare SGPR: only - # the split MaskA/MaskB (and optional Metadata) forms are declared. - for line in src.splitlines(): - if "sgprMulticastMask," in line: - pytest.fail("combined MulticastMask SGPR leaked into split path: " - + line.strip()) - def test_broadcast_mask_value(self): """maskB = (1< B loaded normally" in src, \ - "predicate fallback (self-only B) missing" - - def test_cluster_barrier_handshake(self): - """The multicast B-broadcast masks are paired with the cluster-scope - barrier handshake (s_barrier_signal/wait -3) around the multicast - tensor_load_to_lds, so ClusterBarrier is on and both barrier opcodes are - emitted.""" - _b, src, _e = self._emit()[0] - assert "s_barrier_signal -3" in src, \ - "missing cluster-scope barrier signal (-3) on the multicast path" - assert "s_barrier_wait -3" in src, \ - "missing cluster-scope barrier wait (-3) on the multicast path" - - def test_dp_to_sk_boundary_clear(self): - """At the DP->SK boundary the B broadcast is dropped to self-only so SK - partial-tile loads are normal per-WG loads.""" - _b, src, _e = self._emit()[0] - assert "DP->SK: drop B broadcast -> self-only" in src, \ - "boundary-clear rewrite of MulticastMaskB missing" + for line in src.splitlines(): + if "sgprMulticastMask," in line: + pytest.fail("combined MulticastMask SGPR leaked into split path: " + + line.strip()) if __name__ == "__main__": diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp index 2e84292aa847..e07f9d19668b 100755 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionProblemPredicates.hpp @@ -3090,13 +3090,13 @@ namespace TensileLite }; // StreamK cluster-reduction (gfx1250) split-barrier safety. The - // C = ClusterDim[0] peers split a tile's itersPerTile = ceil(K/DepthU) + // Ck = ClusterDim[1] peers split a tile's itersPerTile = ceil(K/DepthU) // K-iterations and hand off through an intra-cluster split barrier; if - // itersPerTile % C != 0 (incl. C > itersPerTile) the arrival counts + // itersPerTile % Ck != 0 (incl. Ck > itersPerTile) the arrival counts // mismatch and the barrier over-signals -> hang. itersPerTile depends // on runtime K, so this is a per-problem HARD REJECT (surfaced via // debugEval; not build-time, not a silent fallback). Only the K-split - // reduction path emits it. value[0] = DepthU, value[1] = C. + // reduction path emits it. value[0] = DepthU, value[1] = Ck. struct ClusterReductionIterCheck : public Predicate_CRTP { @@ -3140,16 +3140,16 @@ namespace TensileLite virtual bool debugEval(ContractionProblemGemm const& problem, std::ostream& stream) const override { - int c = value[1]; + int ck = value[1]; size_t ipt = itersPerTile(problem, value[0]); return debugEvalCmp( problem, stream, "StreamK cluster reduction requires iterations-per-tile ceil(K/DepthU)", ipt, - "to be a multiple of ClusterDim[0]", - "C", - c); + "to be a multiple of ClusterDim[1]", + "Ck", + ck); } }; } // namespace Contraction diff --git a/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp b/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp index 1776bb0ed12f..5e20cfb39ffd 100644 --- a/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/Predicates_test.cpp @@ -219,7 +219,7 @@ TEST(Predicates, ClusterReductionIterCheck_RejectMessageNamesLimitation) const std::string msg = oss.str(); EXPECT_NE(msg.find("iterations-per-tile ceil(K/DepthU)"), std::string::npos) << "reject diagnostic must name iterations-per-tile; got: " << msg; - EXPECT_NE(msg.find("multiple of ClusterDim[0]"), std::string::npos) + EXPECT_NE(msg.find("multiple of ClusterDim[1]"), std::string::npos) << "reject diagnostic must name the multiple-of-cluster-size limitation; got: " << msg; EXPECT_NE(msg.find("[!!]"), std::string::npos) << "reject diagnostic must mark the predicate as failing; got: " << msg; From 3ce79e4f5f1dd3bfbe9b91061a21bfeb16972838 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 28 Jul 2026 15:16:28 +0000 Subject: [PATCH 19/24] docs(streamk): scrub session artifacts from comments and design docs Rewrite user-facing StreamK cluster validator reject messages to state the supported configuration instead of git branch names, and remove branch-name references from the surrounding comments and the ClusterDim shape-decoder docstring. Trim bug-fix narrative from the cluster split-barrier gate docstring down to the durable symmetric-barrier / SCOPE_SYS visibility / deadlock invariant, and drop the "Held fix"/"red-pgr1" dangling references while keeping the cross-L2 memory-ordering rationale. Convert before/after narration to present tense, drop "byte-identical"/"HW-validated"/"lift" annotations (keeping the durable fact each was attached to), and delete removed-test tombstones. Rewrite both cluster design docs as concise as-shipped notes (reduction shape [1,C], derived Ck, SCOPE_SYS symmetric barrier, boundary-clear present). No emitted-codegen change: cluster/multicast/reduction char goldens are byte-identical without --snapshot-update; CPU unit suite green. --- ...er-load-component-and-streamk-multicast.md | 609 ++++-------------- docs/design/streamk-wg-clusters.md | 602 ++++------------- .../tensilelite/Tensile/Common/Utilities.py | 4 +- .../Tensile/Components/ClusterLoad.py | 21 +- .../tensilelite/Tensile/Components/StreamK.py | 45 +- .../tensilelite/Tensile/Contractions.py | 6 +- .../Tensile/SolutionStructs/Solution.py | 47 +- .../core/sk_mxf4gemm_cluster_multicast.yaml | 2 +- .../core/sk_mxf4gemm_cluster_reduction.yaml | 2 +- .../core/sk_mxf8gemm_cluster_multicast.yaml | 2 +- .../core/sk_mxf8gemm_cluster_reduction.yaml | 4 +- .../gfx1250/streamk_cluster_reduction.yaml | 2 +- .../Tests/unit/test_multicast_tristate.py | 9 +- .../unit/test_streamk_cluster_reduction.py | 2 +- .../Tests/unit/test_streamk_multicast.py | 19 +- 15 files changed, 326 insertions(+), 1050 deletions(-) diff --git a/docs/design/cluster-load-component-and-streamk-multicast.md b/docs/design/cluster-load-component-and-streamk-multicast.md index 89e1b8c05cd7..03b7d6b0032a 100644 --- a/docs/design/cluster-load-component-and-streamk-multicast.md +++ b/docs/design/cluster-load-component-and-streamk-multicast.md @@ -5,60 +5,42 @@ SPDX-License-Identifier: MIT # Design: `ClusterLoad` Component + StreamK Cooperative Multicast Loads (gfx1250) -**Target arch:** gfx1250 (ISA `(12,5,0)`, wave32, TDM: `MXLoadInst=TDM` → `TDMInst=3`). -**Scope:** Two coordinated pieces of work. - -1. **`ClusterLoad` component + behavior-preserving refactor.** Extract the multicast - ("cluster load") mask machinery — value computation, SGPR declare/undeclare, and the - three descriptor apply sites (dense, wave-separated, subtile) — into a single reusable - `ClusterLoad` tensilelite `Component`, and refactor the existing subtile Multicast path - onto it. This is a **pure refactor**: zero change to emitted assembly for existing - configs, gated by codegen snapshots. -2. **Decouple `Multicast` from `ClusterDim`** into an independent opt-in, so barrier-only - clustering (the separate cluster split-barrier reduction feature) and cooperative-load - clustering compose independently instead of `ClusterDim != [1,1]` unconditionally forcing - `Multicast=True`. -3. **StreamK cooperative cluster loads.** Build TDM multicast on top of `ClusterLoad` for - the StreamK **data-parallel (DP) region**, resolving the tension between the spatial - `[C0,C1]` cluster cooperative loads want and the `[C,1]` same-tile K-peer cluster the - shipped reduction wants. - -The separate barrier-only cluster reduction (`docs/design/streamk-wg-clusters.md`) is -orthogonal and tracked independently. - -Source-line anchors reference the code as it exists now and were verified against the tree. +**Target arch:** gfx1250 (ISA `(12,5,0)`, wave32, TDM: `MXLoadInst=TDM` -> `TDMInst=3`). + +This note describes the shipped design of two coordinated pieces: + +1. the reusable `ClusterLoad` component that owns the multicast ("cluster load") + mask machinery, and +2. StreamK cooperative cluster loads, which multicast the shared B operand across + a 1-D `[C,1]` StreamK data-parallel (DP) cluster. + +The barrier-only StreamK cluster reduction is described in +`docs/design/streamk-wg-clusters.md`. --- -## 1. Background (verified anchors) +## 1. Background ### 1.1 What a "cluster load" is -A cluster load is a normal TDM `tensor_load_to_lds` whose descriptor `Group1[word0]` has a -multicast-mask bit field OR'd in. The HW then broadcasts (multicasts) the loaded tile to -every workgroup in the cluster whose bit is set. Attachment is a single `SOrB32`: - -```511:514:projects/hipblaslt/tensilelite/Tensile/Components/TensorDataMover.py - def setMulticastMask(self, group1: int | str, mask: str, writer: "KernelWriterAssembly") -> Module: - mod = Module() - mod.add(SOrB32(sgpr(f"{group1}"), sgpr(f"{group1}"), sgpr(f"{mask}"))) - return mod -``` +A cluster load is a normal TDM `tensor_load_to_lds` whose descriptor +`Group1[word0]` has a multicast-mask bit field OR'd in. The HW then broadcasts +the loaded tile to every workgroup in the cluster whose bit is set. Attachment is +a single `SOrB32` (`Components/TensorDataMover.py`, `setMulticastMask`). ### 1.2 Mask value computation and the three topologies -The mask *value* is computed once in `defineAndResources` -(`KernelWriterAssembly.py:2646-2694`), from the WG's position within the cluster and +The mask *value* is a function of the WG's position within the cluster and `ClusterDim`: -- `maskA = OR over idx in range(ClusterDim[1]) of (1 << (idx*ClusterDim[0]))`, then shifted - left by `wg_x`. Bit `wg_y*ClusterDim[0] + wg_x` = the cluster-linear index of the WG. - So `maskA` selects every WG sharing the same `wg_x` column (same M block) across all - `ClusterDim[1]` rows. -- `maskB = (1 << ClusterDim[0]) - 1`, then shifted left by `wg_y * ClusterDim[0]`. Selects - every WG in the same `wg_y` row (same N block). +- `maskA = OR over idx in range(ClusterDim[1]) of (1 << (idx*ClusterDim[0]))`, + shifted left by `wg_x`. Bit `wg_y*ClusterDim[0] + wg_x` is the cluster-linear + index of the WG, so `maskA` selects every WG sharing the same `wg_x` column + (same M block) across all `ClusterDim[1]` rows. +- `maskB = (1 << ClusterDim[0]) - 1`, shifted left by `wg_y * ClusterDim[0]`. + Selects every WG in the same `wg_y` row (same N block). -Three name topologies (this exact selection matrix must be preserved by the refactor): +Three name topologies: | Topology | Predicate | SGPR name(s) | |---|---|---| @@ -66,496 +48,137 @@ Three name topologies (this exact selection matrix must be preserved by the refa | Split A/B | otherwise (subtile, single-tensor, or `NumWaves==1`) | `MulticastMaskA`, `MulticastMaskB` | | Metadata (sparse) | `enableTDMMetadata` | `MulticastMaskMetadata` (follows A for `Sparse==1`, B for `Sparse==2`) | -Declare (`KernelWriter.py:9163-9176`) and undeclare (`KernelWriter.py:2838-2848`) mirror the -same predicate. - -### 1.3 The three descriptor apply sites (the duplication to unify) - -All three OR the mask into `Group1` under the same gate -`kernel["Multicast"] and enableCluster`, where `enableCluster = prod(ClusterDim) != 1`: - -- **Dense** `initTDMDescriptor` — `KernelWriterAssembly.py:18901-18902`; local - `maskSgprName(tc)` returns `MulticastMask{A|B}` (split). -- **Wave-separated** `initTDMDescriptorWaveSeparatedImpl` — `KernelWriterAssembly.py:19059-19060`; - local `maskSgprName` returns the combined `"MulticastMask"`. -- **Subtile** `initTDMDescriptorSubtile` — `Components/Subtile/SubtileGREmit.py:1108-1113`; - uses `MulticastMask{tc}` (split). - -`initTDMDescriptorSubtile` (`SubtileGREmit.py:1061-1155`) is a near-clone of the writer -`initTDMDescriptor` (`KernelWriterAssembly.py:18843-18994`); the mask attachment is the -piece we lift into `ClusterLoad`. The rest of the descriptor build (LDS offset, tensor -dims/strides, padding) stays where it is — `ClusterLoad` does **not** own descriptor-group -allocation or LDS offsets. - -### 1.4 Cooperative-thread partition (shared math) - -`numCooperativeWGs = ClusterDim[1] for A / ClusterDim[0] for B`, duplicated in -`Components/GL2Prefetch.py:26` and `:78`. `ClusterLoad` centralizes this as -`cooperativeThreadPartition`. - -### 1.5 Component framework - -- Auto-registration metaclass: `Component.py:113-124` (`__init__` sets `implementations` - and `setattr` on each base). `matches()` partial-matches `asmCaps`/`archCaps`/`kernel` - (`Component.py:132-144`); `find()` requires exactly one match, raises on >1, returns - `None` on 0 → fallback (`Component.py:167-177`). -- Categories (abstract, no `__call__`) declared near `Component.py:305-313` - (`TensorDataMover`, `GL2Prefetch`); `from .Components import *` at `:318`. -- `Components/__init__.py:28-55` `__all__` lists each component module. -- `TensorDataMoverLoad` shows the concrete pattern: `asmCaps = {"HasTDM": True}`, - `kernel = {"TDMInst": 3}`, method-bag returning rocisa `Module`s - (`TensorDataMover.py:13-15`). -- New-file SPDX header (`# Copyright Advanced Micro Devices, Inc., or its affiliates.` / - `# SPDX-License-Identifier: MIT`). - -### 1.6 `Multicast` / `ClusterDim` coupling (current) - -`Solution.py:1046-1064` forces `Multicast=True` for any `ClusterDim != [1,1]`, except it is -already suppressed for Stream-K. `Multicast` is **not** a user parameter in -`ValidParameters.py` (it is a derived state var). - -### 1.7 StreamK addressing (where multicast pays off) - -- `skTileIndex` (`StreamK.py:439`, math `469-483`) magic-divides `StreamKIter` into a tile - index and `StreamKLocalStart`/`StreamKLocalEnd` (iter offsets within the tile). -- `skIndexToWG` (`StreamK.py:487-503`) maps tile → `(WorkGroup0, WorkGroup1, WorkGroup2)` - with the linearization `tileID = WG2*(nWG0*nWG1) + WG1*nWG0 + WG0` — **WG0 (M) is - fastest**, so consecutive tiles are M-adjacent (same WG1/N block, consecutive WG0). -- The K-direction StreamK offset (`StreamKLocalStart*DepthU*strideL`) is added in - `computeLoadSrdCommon` (`StreamK.py:548`, `555-562`) and `graAddressesCommon` - (`StreamK.py:629`, `636-645`); both are `tc`-generic (shared A/B). **The `WorkGroup0*MT0` - / `WorkGroup1*MT1` tile-base term lives in the base KernelWriter GRA/SRD code, keyed on - the post-`skIndexToWG` `WorkGroup0/1`.** A/B thus bind to WG0/M and WG1/N respectively; - the K offset is identical for A and B. -- `graWorkGroup` (SK3 TwoTileDPFirst, `StreamK.py:2846`; DP shift `2921-2925`): a WG `w` - processes tiles `w, w+skGrid, w+2*skGrid, …`. In the **first DP round** consecutive WGs - map to consecutive tiles → **M-adjacent → share the same B (N-block) over full K**. -- `StreamKIdx = WorkGroup0` (`StreamK.py:2658-2663`), preceded by the `ttmp9` raw-wgid - workaround (`StreamK.py:2645-2656`), which is **already skipped under any cluster - (`ClusterDim != [1,1]`)** so the cluster-remapped `WorkGroup0` survives. -- DP/SK region split in `graWorkGroup` at `StreamK.py:2907-2932`; the "started & - finished the whole tile → regular store, no reduction" branch at `StreamK.py:922-933`. -- gfx1250 kernel-side WG remap: `WorkGroup0 = cluster_x*nwg_x + wg_x` - (`KernelWriterAssembly.py:2628-2632`), so cluster `c` owns a contiguous `WorkGroup0` - range `[c*C, c*C + C)`. - -### 1.8 Data-reuse fact (drives the whole StreamK design) - -Two DP WGs share **A** iff same WG0 (M) and overlapping K; share **B** iff same WG1 (N) and -overlapping K. DP tiles compute the whole tile over full K, so K always overlaps. - -- K-split peers of one tile (the shipped `[C,1]` reduction cluster): same WG0/WG1 but - **disjoint K** → **zero reuse**. Multicasting the reduction cluster multicasts nothing. -- Real reuse is **spatial** over a common full-K range: adjacent DP tiles. Consecutive-WG - clustering (`[C,1]`) gives M-adjacent tiles → **shared B**. (Tiles `nWG0` apart are - N-adjacent → shared A; not reachable by consecutive `[C,1]` clustering.) - -**Conclusion: in the StreamK DP region a `[C,1]` cluster multicasts B (not A).** This maps -exactly onto the existing mask math with `wg_y=0, C0=C, C1=1`: `maskB = (1< **M-adjacent -> share the same B (N-block) over full K**. +- gfx1250 kernel-side WG remap `WorkGroup0 = cluster_x*nwg_x + wg_x`, so cluster + `c` owns a contiguous `WorkGroup0` range `[c*C, c*C + C)`. -### 2.1 Registration +### 1.4 Data-reuse fact (drives the StreamK design) -- New category in `Component.py` next to `TensorDataMover`/`GL2Prefetch` (~`305-313`): +Two DP WGs share **A** iff same WG0 (M) and overlapping K; share **B** iff same +WG1 (N) and overlapping K. DP tiles compute the whole tile over full K, so K +always overlaps. -```python -class ClusterLoad(Component): - """ - Cluster (multicast) TDM load: multicast-mask compute + descriptor attach. - """ -``` +- K-split peers of one tile (the reduction cluster `[1,C]`): same tile but + **disjoint K** -> **zero reuse**. Multicasting a reduction cluster multicasts + nothing. +- Real reuse is **spatial** over a common full-K range: adjacent DP tiles. + Consecutive-WG clustering (`[C,1]`) gives M-adjacent tiles -> **shared B**. -- New concrete module `Tensile/Components/ClusterLoad.py`: +**Conclusion: in the StreamK DP region a `[C,1]` cluster multicasts B (not A).** +This maps onto the mask math with `wg_y=0, C0=C, C1=1`: `maskB = (1< A per-WG). -```python -# Copyright Advanced Micro Devices, Inc., or its affiliates. -# SPDX-License-Identifier: MIT -from ..Component import ClusterLoad -... -class ClusterLoadTDM(ClusterLoad): - asmCaps = {"HasTDM": True} - kernel = {"TDMInst": 3} - def __call__(self, writer, kernel): # abstract-satisfying no-op, mirrors TensorDataMoverLoad - pass -``` +--- -- Add `"ClusterLoad"` to `Components/__init__.py:__all__` (`:28-55`). +## 2. The `ClusterLoad` component -Selection is capability-based (`HasTDM` + `TDMInst=3`), identical to how -`TensorDataMoverLoad` is found, so `ClusterLoad.find(writer)` returns the TDM impl on -gfx1250 and `None` (fallback → no multicast) elsewhere. +`ClusterLoad` is a capability-selected tensilelite `Component` +(`asmCaps={"HasTDM":True}`, `kernel={"TDMInst":3}`, like `TensorDataMoverLoad`), +so `ClusterLoad.find(writer)` returns the TDM implementation on gfx1250 and +`None` (fallback -> no multicast) elsewhere. -### 2.2 API (method bag; all returning rocisa `Module` or plain values) +It **owns** the multicast mask machinery and nothing else (it does not own +descriptor-group SGPRs, LDS offsets, or the `tensor_load_to_lds` itself): -`ClusterLoad` owns **mask value + declare + attach + topology decision + cooperative -partition**. It does **not** own descriptor-group SGPRs, LDS offsets, or the -`tensor_load_to_lds` itself. +| Method | Responsibility | +|---|---| +| `usesCombinedMask(kernel)` | Single source of truth for the combined-vs-split decision (`tdmA and tdmB and NumWaves>1 and not UseSubtileImpl`; StreamK multicast always uses split A/B). | +| `maskSgprName(kernel, tc, ...)` | Central name resolver: combined `"MulticastMask"` when wave-separated and not subtile; else `f"MulticastMask{strip_MXS(tc)}"`. | +| `declareSgprs` / `undeclareSgprs` | Allocate / free the `MulticastMask*` SGPRs. | +| `computeMasks(writer, kernel, *, sgprWgX, sgprWgY, sgprNWgX, sTmp)` | Compute the mask value(s). The caller passes the SGPR operands it already holds (`sTmp+4` slot is scratch) rather than re-allocating. | +| `applyToDescriptor(...)` | Fold the gate + name choice + the `SOrB32`; returns an empty `Module` when multicast is inactive. | -| Method | Signature | Responsibility | -|---|---|---| -| `usesCombinedMask` | `(kernel) -> bool` | The single predicate `tdmA and tdmB and NumWaves>1 and not UseSubtileImpl`. Kills the 3 duplicated copies (`KernelWriter.py:9170`, `:2842`, `KernelWriterAssembly.py:2668`). | -| `maskSgprName` | `(kernel, tc, *, subtile=False) -> str` | Central name resolver: combined `"MulticastMask"` when `usesCombinedMask` and not `subtile`; else `f"MulticastMask{strip_MXS(tc)}"`; `"MulticastMaskMetadata"` for metadata. Subsumes the two local `maskSgprName` closures + the subtile literal. | -| `declareSgprs` | `(writer, kernel) -> None` | Moves `KernelWriter.py:9163-9176` (uses `writer.defineSgpr`). | -| `undeclareSgprs` | `(writer, kernel) -> Module` | Moves `KernelWriter.py:2838-2848` (uses `writer.undefineSgpr`). | -| `computeMasks` | `(writer, kernel, *, sgprWgX, sgprWgY, sgprNWgX, sTmp) -> Module` | Moves `KernelWriterAssembly.py:2646-2694` verbatim, including the wave-parity branch and the metadata cases. Takes the exact SGPR operands the caller already holds so emitted asm is byte-identical. | -| `applyToDescriptor` | `(writer, kernel, group1, tc, *, subtile=False) -> Module` | Folds the gate + name choice + the `SOrB32`. Returns an **empty `Module`** when `not (kernel["Multicast"] and enableCluster)` — identical to today's skipped `if`. Internally calls `TensorDataMover.setMulticastMask`. Replaces the 3 apply sites. | -| `cooperativeThreadPartition` | `(kernel, tc) -> int` | `ClusterDim[1] if tc-ends-A else ClusterDim[0]`. Shared with `GL2Prefetch` (`:26`, `:78`). | - -Rationale for passing SGPR indices into `computeMasks`: the current code emits into -`sTmp+1..4` allocated by the surrounding `defineAndResources`. To guarantee **zero asm -diff**, the component must not re-allocate; it receives the same operands and emits the same -instructions in the same order. This is a mechanical lift, not a rewrite. - -### 2.3 Behavior-preserving refactor wiring - -- `KernelWriter.py:9163-9176` → `ClusterLoad.find(self).declareSgprs(self, kernel)`. -- `KernelWriter.py:2838-2848` → `... .undeclareSgprs(self, kernel)`. -- `KernelWriterAssembly.py:2646-2694` → `... .computeMasks(self, kernel, sgprWgX=..., sgprWgY=..., sgprNWgX=..., sTmp=...)` (same operands as today). -- `KernelWriterAssembly.py:18901-18902` (dense) → `... .applyToDescriptor(self, kernel, descSgprName(1), tc)`. -- `KernelWriterAssembly.py:19059-19060` (wave-separated) → same call (component resolves to combined name via `usesCombinedMask`). -- `SubtileGREmit.py:1108-1113` (subtile) → `... .applyToDescriptor(writer, kernel, descSgprName(1), tc, subtile=True)`. - -Guard: `ClusterLoad.find` returns `None` when `HasTDM`/`TDMInst` don't match; callers keep a -`comp = ClusterLoad.find(self)` + `if comp:` fallback exactly like `TensorDataMoverLoad.find` -usage, so non-TDM archs are untouched. - -**Regression gate (must stay green, zero asm change):** -`test_r4_xccremap_char.py` (drives `ClusterDim=[2,2]` with both a `MIWaveGroup` prod==1 → -split `MulticastMaskA/B` kernel and a prod==4 → combined `MulticastMask` kernel) and -`test_streamk_cluster_coop_load_gfx1250_char.py`. Add a byte-exact golden snapshot of the -emitted mask/apply region for a `[2,2]` combined config and a subtile split config before -the refactor; diff to zero after. +The SGPR-operand contract on `computeMasks`/`applyToDescriptor` (caller supplies +the operands) is deliberate: the component never re-allocates the mask SGPRs. --- ## 3. Decoupling `Multicast` from `ClusterDim` -### 3.1 Mechanism - -Add an explicit tri-state solution parameter (`ValidParameters.py`, near `ClusterDim`): +`Multicast` is an explicit tri-state solution parameter: ```python -# -1 = auto (legacy: ClusterDim!=[1,1] implies Multicast, except StreamK cluster paths) +# -1 = auto (ClusterDim!=[1,1] implies Multicast, except StreamK cluster paths) # 0 = force multicast off # 1 = force multicast on (independent of ClusterDim coupling) "Multicast": [-1, 0, 1], ``` -Default `-1` preserves **all** existing emitted asm. Rewrite `Solution.py:1046-1064`: - -```python -state["ClusterBarrier"] = False -mc = state.get("Multicast", -1) -if mc == 1: - state["Multicast"] = True -elif mc == 0: - state["Multicast"] = False -elif state.get("StreamKMulticast", 0): - state["Multicast"] = True # derived cooperative-load path -else: # auto (legacy behavior) - state["Multicast"] = (state["ClusterDim"] != [1, 1] - and state["StreamK"] == 0) -# ClusterBarrier decision (legacy/cluster-subtile path + the derived StreamKMulticast path) -if state["ClusterDim"] != [1, 1] and state["Multicast"] and state["TDMInst"] != 0 \ - and (state["StreamK"] == 0 or state.get("StreamKMulticast", 0)) \ - and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): - state["ClusterBarrier"] = True -``` - -Net effect for existing configs: `Multicast` default `-1` → identical derivation to today -(subtile/dense clustered configs keep `Multicast=True`; non-multicast Stream-K keeps it -`False`). New capability: `Multicast=1`/`0` explicit override, and `StreamKMulticast` (§4) -turns on multicast for its own path without relying on the coupling. - -This is the one step in the sequence that *can* legitimately change asm — but only for -configs that opt in. Keep existing subtile/multicast YAMLs on `Multicast=-1` so their -snapshots don't move. +Default `-1` reproduces the ClusterDim-coupled derivation, so YAML that omits +`Multicast` is unchanged. `Multicast=1`/`0` are explicit overrides, and the +derived `StreamKMulticast` path (§4) sets `Multicast=True` for itself without +relying on the coupling. --- ## 4. StreamK cooperative cluster loads (DP region) -### 4.1 The spatial vs reduction tension, resolved - -- Cooperative loads need a **spatial** cluster over **distinct** tiles (`[C,1]` consecutive - WGs → M-adjacent → shared B). -- The shipped reduction needs a **`[C,1]` same-tile K-peer** cluster (host ties - `skGrid = C*tiles`, `skIndexToWG` collapses the C peers onto one tile). -- A WG's HW cluster membership is fixed at launch, so a single kernel cannot have the - cluster mean "spatial DP" and "K-peers" at once. - -**Resolution: mutual exclusion.** For v1, `StreamKMulticast` and the barrier-only cluster -reduction are mutually exclusive. When `StreamKMulticast` is on, `ClusterDim=[C,1]` is free -to denote the spatial cluster; the reduction is off, so no `[C,1]` is claimed for K-peers. **No -separate `SpatialClusterDim` is needed in v1** — the same `ClusterDim=[C,1]` is reused, its -meaning selected by which StreamK cluster mode is enabled. (A future 2-D `[C0,C1]` mode for -A-multicast or combined reduction+coop-load would introduce a distinct spatial dim; deferred.) - -### 4.2 MVP target: DP-only, B-multicast - -The multicast mask is computed **once** at kernel init and baked into the descriptor -`Group1[word0]`; it is applied to every mainloop TDM load. For correctness, the sharing -relationship it encodes must hold for **every** load the WG issues. Therefore v1 requires -that the WG never enters a partial (K-split) tile with a stale spatial mask: - -**v1 MVP = cooperative B-multicast for the DP region with `StreamKForceDPOnly=1`** (no SK -partial tiles → no reduction → the static spatial mask is valid for all issued loads), and -single DP round per cluster (see §4.4). - -Recommended smallest correct MVP: -- **`StreamKMulticast=1`** requires `StreamKForceDPOnly=1` and no barrier-only cluster reduction. -- Reuse `ClusterDim=[C,1]`, `C` a power of two in `2..16`. -- Multicast B across the cluster; A loaded per-WG. - -Deferred (see §7): mask-toggle at the DP→SK boundary to allow cooperative DP loads inside a -full SK3 kernel (SK tail reduces via the existing global-flag path); combined -reduction+coop-load; A-multicast / 2-D spatial cluster; multi-round with per-round validity. - -### 4.3 Mask derivation from post-`skIndexToWG` coords - -The existing `computeMasks` derives `wg_x`/`wg_y` from the raw HW cluster registers -(`ttmp*`). For StreamK the authoritative WG coordinates are the ones produced by -`skIndexToWG` (`StreamK.py:503`), not the raw HW position, because StreamK re-derives -`(WG0,WG1)` from `StreamKIdx`. For the DP `[C,1]` MVP the two coincide -(`wg_x = StreamKIdx & (C-1)`, since the reduction ttmp9 workaround is skipped and -`WorkGroup0 = cluster_x*C + wg_x`), so the MVP feeds `computeMasks` with -`sgprWgX = StreamKIdx & (C-1)`, `wg_y = 0`, `C0=C`, `C1=1`, yielding the B-broadcast mask -`(1< 1 (pure multicast `[C,1]`, 1-D launch), +- the cluster reduction iff Ck > 1 (pure reduction `[1,C]`, 2-D launch). -> **Status (as shipped):** `StreamKMulticast` is **not** a public/benchmark parameter. -> It is a derived-only internal state key (like `ClusterBarrier`), enabled automatically -> when `ClusterDim != [1,1]` on SK3 — see §3.1 / the -> collapse in `assignProblemIndependentDerivedParameters`. It has no `ValidParameters.py` -> entry; `_validateStreamKMulticast` is the internal guard that hard-rejects a derived -> config it cannot satisfy. The `[0,1]` "param" framing below reflects the original plan. +`StreamKMulticast` is a **derived-only** internal state key (like `ClusterBarrier`); +it has no `ValidParameters.py` entry. `_validateStreamKMulticast` hard-rejects a +derived config it cannot satisfy (SK3, `ClusterDim=[C,1]` with C a power of two in +`2..16`, gfx1250 `HasTDM`/`TDMInst=3`, `StreamKXCCMapping != 3`, not +`StreamKAtomic`). -### 5.1 Parameters +### 4.2 Mask derivation and validity -| Param | Values | Meaning | -|---|---|---| -| `Multicast` | `[-1,0,1]` | Decoupled multicast control (§3). Default `-1` = legacy auto. | - -`StreamKMulticast` is internal-derived (see status note above), not a user-settable param. -The separate barrier-only cluster reduction feature is orthogonal and unchanged. - -### 5.2 Validation rules (`Solution.py`, reject-with-reason pattern) - -The derived `StreamKMulticast` on-state is rejected (`_validateStreamKMulticast`) unless all hold: -- `StreamK == 3` (SK3; DP schedule + `skIndexToWG` assumptions). -- `StreamKForceDPOnly == 1` (v1: no partial tiles reach the static mask). -- not the barrier-only cluster reduction (mutually exclusive — cannot claim `[C,1]` two - ways; the collapse gives the reduction precedence, and the reject itself is added - alongside that feature). -- `ClusterDim == [C,1]`, `C` power of two, `2 <= C <= 16` (`ClusterDim[1] == 1`; StreamK - grid is effectively 1-D along x). -- gfx1250 `asmCaps["HasTDM"]` and `TDMInst != 0` (MX TDM loads; multicast is a TDM feature). -- `StreamKXCCMapping == 0` (XCC=3 overflows SGPRs; WGM/XCC remap is bypassed under - clustering anyway). - -Also set `Multicast=True` for this path via the decoupled derivation (§3.1) so -`ClusterLoad.applyToDescriptor` fires. - -### 5.3 Interaction notes - -- gfx1250 StreamK already uses TDM (`MXLoadInst=TDM` → `TDMInst=3`), so `HasTDM` is the - natural cap and no new transport is introduced. -- `StreamKAtomic` is incompatible (atomic path skips the workspace/tile structure the DP - schedule relies on); reject. - ---- - -## 6. Ordered implementation task breakdown - -Sequencing keeps the tree green: **pure extraction first (zero asm), then the decouple, -then the feature.** `[P]` marks tasks that can proceed in parallel within a group. - -### Group 0 — Snapshot baseline (do first, no source change) -- **T0.1** Add byte-exact golden snapshots for the mask compute+apply region: a `[2,2]` - combined-mask config, a `[2,2]` split-mask config, and a subtile split config. Anchor: - extend `test_r4_xccremap_char.py` and the subtile char config under - `Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/`. - -### Group 1 — `ClusterLoad` extraction (pure refactor, snapshot-gated) -- **T1.1** Declare `class ClusterLoad(Component)` at `Component.py:305-313`; add - `"ClusterLoad"` to `Components/__init__.py:__all__`. -- **T1.2** Create `Components/ClusterLoad.py` with `ClusterLoadTDM` (SPDX header; - `asmCaps={"HasTDM":True}`, `kernel={"TDMInst":3}`); implement `usesCombinedMask`, - `maskSgprName`, `cooperativeThreadPartition`. -- **T1.3** Move mask compute into `computeMasks`; wire `KernelWriterAssembly.py:2646-2694` - to call it with the existing SGPR operands. -- **T1.4 [P]** Move declare/undeclare into `declareSgprs`/`undeclareSgprs`; wire - `KernelWriter.py:9163-9176` and `:2838-2848`. -- **T1.5 [P]** Implement `applyToDescriptor`; wire the 3 apply sites - (`KernelWriterAssembly.py:18901-18902`, `:19059-19060`, `SubtileGREmit.py:1108-1113`). -- **T1.6** Point `GL2Prefetch.py:26,78` at `cooperativeThreadPartition` (optional but DRY). -- **Gate:** T0.1 snapshots + `test_r4_xccremap_char.py` + - `test_streamk_cluster_coop_load_gfx1250_char.py` all diff to zero. - -### Group 2 — Decouple `Multicast` (may change asm only for opt-in configs) -- **T2.1** Add `"Multicast": [-1,0,1]` to `ValidParameters.py`. -- **T2.2** Rewrite `Solution.py:1046-1064` per §3.1; keep existing YAMLs on `-1`. -- **Gate:** existing multicast/subtile/reduction snapshots unchanged; add a snapshot for an - explicit `Multicast=1`/`0` override. - -### Group 3 — StreamK cooperative DP multicast (feature) -- **T3.1** Derive `StreamKMulticast` internally (no `ValidParameters.py` entry — it is a - derived-only state key, auto-enabled on SK3 + `ClusterDim!=[1,1]` not reduction) + - `Solution.py` validation (§5.2) + decoupled `Multicast` enablement. -- **T3.2 [P]** Host: `sizeMapping.streamKMulticast` (`ContractionSolution.hpp:146`, - `Contractions.py:630,725`); `getSKGridImpl` grid = `ceil(tiles/C)*C` - (`ContractionSolution.cpp:4087-4090`/`:3214-3218` region); thread through launch (no - launch change — `:1783-1794` already cluster-aware). -- **T3.3** Kernel: at StreamK preLoop compute `wg_x = StreamKIdx & (C-1)` and the - `clusterMulticastValid` predicate (§4.4) from `NumWorkGroups0`/`totalTiles`; feed - `ClusterLoad.computeMasks` with `wg_y=0, C0=C, C1=1` and gate the mask OR on the predicate - (via `applyToDescriptor`). Anchors: `StreamK.py:2645-2663` (StreamKIdx/ttmp9), - `skIndexToWG` `:487-503`. -- **T3.4** Ensure A stays per-WG (maskA = self) and B multicast (maskB = all-C); verify the - `[C,1]` path selects split `MulticastMaskA/B` names. - -Dependencies: Group 1 → Group 2 → Group 3 (T3.1). T3.2 (host) is parallel to T3.3/T3.4 -(kernel) once T3.1 lands the param. +For the DP `[C,1]` cluster the authoritative coordinates are the ones produced by +`skIndexToWG`; for this shape they coincide with the raw cluster position +(`wg_x = StreamKIdx & (C-1)`, `wg_y = 0`), yielding the B-broadcast mask +`(1< SK boundary clear -| Behavior | Test type | Where | -|---|---|---| -| **Refactor is byte-exact**: `computeMasks`/`applyToDescriptor`/declare/undeclare emit identical asm for `[2,2]` combined, `[2,2]` split, and subtile split configs | snapshot char (regression gate) | `test_r4_xccremap_char.py`, `test_streamk_cluster_coop_load_gfx1250_char.py` | -| `ClusterLoad.find` returns TDM impl on gfx1250, `None` (fallback) on non-TDM; `usesCombinedMask`/`maskSgprName` matrix (combined vs split vs metadata) | CPU unit | `Tests/unit/test_cluster_load_component.py` | -| `Multicast` tri-state: `-1` reproduces legacy derivation exactly; `0` forces off; `1` forces on independent of `ClusterDim` (derived value is an int on all paths) | CPU unit | `Tests/unit/test_multicast_tristate.py`, `Tests/unit/test_multicast_legacy_coercion.py` | -| `StreamKMulticast` validation matrix: accepted only for SK3 + `ClusterDim=[C,1]` (pow2 2..16) + gfx1250 `HasTDM`/`TDMInst=3` + XCC=0 + not `StreamKAtomic`; auto-enabled config with `Multicast=0` rejected; rejected otherwise | CPU unit | `Tests/unit/test_streamk_multicast.py` | -| Cluster config emits real gfx1250 asm (`err==0`); DP loads carry the B-broadcast mask (`MulticastMaskB` OR into `Group1`); A load carries self-only mask; predicate gate + DP→SK boundary clear present | snapshot char | `test_streamk_cluster_multicast_gfx1250_char.py` (designed config `_designed/gfx1250/streamk_cluster_multicast.yaml`, `__snapshots__/*.ambr`) | -| DP GEMM with `StreamKMulticast`, `C∈{2,4}` reduces correctly and does not hang; matches non-multicast reference | GPU roundtrip | `Tests/unit/test_streamk_cluster_reduction_gpu.py` and cluster char roundtrips (`@requires_gpu_gfx1250`; watchdog on hang) | -| Real multi-WG cluster StreamK DP GEMM end-to-end | C++ client | `Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml` (+ mxf4 sibling) | +In a full SK3 kernel the WG transitions from DP tiles (where the spatial mask is +valid) into SK partial tiles (where it is not). The multicast mask bit is cleared +at the DP -> SK boundary so the SK-region loads do not carry a stale spatial mask. +This boundary clear is emitted as part of the shipped path. -The gfx1250 GPU marker reuses `Tests/unit/gpu_test_helpers.py` -(`HAS_GFX1250`/`requires_gpu_gfx1250`, `TENSILE_GPU_TARGET=gfx1250`). - ---- +### 4.4 Host plumbing -## 9. Open questions (need a human decision at the design gate) - -1. **HW multicast semantics when a masked target WG is not executing a matching TDM load** - (idle tail WG, or a WG that finished its tiles): does the loader hang, drop the - broadcast, or corrupt? This determines whether the init-time predicate is sufficient or - whether the grid must be constrained to always-full clusters. (Mirrors the reduction - feature's deadlock concern.) -2. **Is `StreamKForceDPOnly`-scoped v1 valuable enough**, or should we prioritize the - DP→SK boundary mask-clear so cooperative loads apply inside a normal SK3 kernel (the - larger real-world win)? The boundary-clear is a modest add (§7) but widens the test - matrix. -3. **`Multicast` as tri-state `[-1,0,1]` vs a separate boolean** (`MulticastEnable`) — the - tri-state keeps one knob and a clean legacy default; a separate boolean is more explicit - but adds a second interacting param. Recommend tri-state; confirm. -4. **Grid rounding policy** for `StreamKMulticast`: `ceil(tiles/C)*C` (idle tail WGs) vs - requiring `totalTiles % C == 0` at selection time (fewer sizes, no idle WGs). Depends on - Q1. +- `sizeMapping.streamKMulticast` mirrors the reduction size-mapping. +- `getSKGridImpl` rounds `skGrid` up to a multiple of `C`; the tail WGs of a + not-fully-populated cluster are disabled by the validity predicate. +- The `enableCluster` launch branch already keeps the grid cluster-friendly and + sets `rv.clusterDim`; no launch change is required. --- -## 10. Summary of decisions - -- **`ClusterLoad` component**: new capability-selected category (`asmCaps HasTDM`, - `kernel TDMInst=3`) owning mask value (`computeMasks`), declare/undeclare, topology - decision (`usesCombinedMask`/`maskSgprName`), descriptor attach (`applyToDescriptor`, - gate folded in), and `cooperativeThreadPartition`. It does **not** own descriptor groups - or LDS offsets. The three apply sites and the subtile clone unify onto it; refactor is - byte-exact and snapshot-gated. -- **Decouple**: `Multicast` becomes a tri-state param (`-1` auto = legacy identity, `0` - off, `1` on); `Solution.py:1046-1064` derivation rewritten so `ClusterDim!=[1,1]` no - longer unconditionally forces multicast. -- **StreamK target = DP region, B-multicast** on a `[C,1]` consecutive-WG cluster - (M-adjacent tiles share the N-block). Spatial-vs-reduction tension resolved by **mutual - exclusion** (`StreamKMulticast` xor the barrier-only cluster reduction), reusing - `ClusterDim=[C,1]` with mode-selected meaning; no separate spatial dim in v1. -- **MVP** = `StreamKMulticast=1` requiring `StreamKForceDPOnly=1`, single DP round, init-time - `clusterMulticastValid` predicate gating the mask. Deferred: DP→SK boundary mask-clear, - multi-round, A-multicast / 2-D cluster, combined reduction+coop-load. -- **Enablement**: `Multicast` (`[-1,0,1]`) is the one public knob; `StreamKMulticast` is - internal-derived (auto-enabled on SK3 + `ClusterDim!=[1,1]`, not reduction), not a param. -- **Sequencing**: extraction (green) → decouple → feature. +## 5. Enablement summary + +- **`ClusterLoad`**: capability-selected component owning mask value + (`computeMasks`), declare/undeclare, topology decision + (`usesCombinedMask`/`maskSgprName`), and descriptor attach + (`applyToDescriptor`). It does not own descriptor groups or LDS offsets. +- **`Multicast`**: tri-state (`-1` auto = legacy identity, `0` off, `1` on); the + `ClusterDim!=[1,1]` coupling no longer unconditionally forces multicast. +- **StreamK cooperative loads**: DP-region B-multicast on a `[C,1]` + consecutive-WG cluster (M-adjacent tiles share the N-block). `StreamKMulticast` + is derived from the cluster shape, gated by the `ClusterDimCheck` validity + predicate, with the DP -> SK boundary mask clear present. diff --git a/docs/design/streamk-wg-clusters.md b/docs/design/streamk-wg-clusters.md index 9c6715ddde2c..e8f71efede08 100644 --- a/docs/design/streamk-wg-clusters.md +++ b/docs/design/streamk-wg-clusters.md @@ -6,502 +6,184 @@ SPDX-License-Identifier: MIT # Design: WG Clusters for StreamK GEMM (gfx1250) **Target arch:** gfx1250 (ISA `(12,5,0)`, wave32). -**Scope:** Use gfx1250 workgroup *clusters* to accelerate the StreamK partial-tile -reduction by replacing the cross-CU global-flag spin-wait with an intra-cluster -split barrier, while keeping a correct fallback for the general -(peers-span-multiple-clusters / DP / atomic) cases. The MVP implemented here is -StreamK variant 3 (two-tile), barrier-only, linear-reduction fast path, with a -fixed even split (one tile per cluster); the global-flag reduction is retained -as a runtime/compile fallback. Source-line anchors below reference the code as -implemented. + +The StreamK partial-tile reduction co-locates a tile's fixup peers in a single +gfx1250 workgroup cluster and replaces the cross-CU global-flag spin-wait with an +intra-cluster split barrier. The global-flag reduction is retained as a runtime +fallback for the general case (peers span more than one cluster). This applies to +StreamK variant 3 (two-tile, DP-first); SK4/SK5 dynamic/atomic peer sets cannot +be statically clustered. --- ## 1. Background -### 1.1 Cluster hardware + existing (non-StreamK) support +### 1.1 Cluster hardware - **Split barrier objects** are addressed by negative id: `-1` = workgroup, `-3` = cluster. The rocisa `SBarrier(separate, wait, clusterBarrier, comment)` - emitter lowers to id `-3` when `clusterBarrier=True`. Only one wave per WG + emitter lowers to id `-3` when `clusterBarrier=True`. One wave per WG signals/arrives; any wave in the cluster may wait. -- **Solution derivation** (`Tensile/SolutionStructs/Solution.py:975-981`): - - ```975:981:projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py - state["Multicast"] = False - state["ClusterBarrier"] = False - if state["ClusterDim"] != [1, 1]: - state["Multicast"] = True - # ClusterBarrier emits SCmp/branch on sgpr("WaveIdx"), which is only allocated when TDM is enabled. - if state["TDMInst"] != 0 and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): - state["ClusterBarrier"] = True - ``` - - Note: `ClusterDim != [1,1]` unconditionally forces `Multicast=True`. This - coupling must be broken for a barrier-only StreamK path (see §5). - **Constraints:** `maxWGsInCluster = 16`; `validClusterDimensions` is all - `[i,j]` with `i*j <= 16` (`Common/ValidParameters.py:74-80`, exported at - `:1107`). For `PrefetchGL2`/non-subtile, `ClusterDim` components must be - powers of two and `!= [1,1]` (`Solution.py:5296-5302`). -- **Host launch is already cluster-aware:** `src/hip/HipSolutionAdapter.cpp:573-604` - uses `hipDrvLaunchKernelEx` + `hipLaunchAttributeClusterDimension`, gated on + `[i,j]` with `i*j <= 16` (`Common/ValidParameters.py`). `ClusterDim` components + must be powers of two. +- **Host launch is cluster-aware:** `HipSolutionAdapter.cpp` uses + `hipDrvLaunchKernelEx` + `hipLaunchAttributeClusterDimension`, gated on `HIP_HAS_CLUSTER_LAUNCH`, with `kernel.clusterDim` from - `ContractionSolution.hpp` / `Contractions.py`. The comment at `:586-589` - states explicitly: *"The grid dimension should be a multiple of cluster - size."* The launch sets `attribute[0].val.clusterDim.z = 1` — clusters are - effectively 2-D `[x,y]`. -- **Kernel-side WG-id derivation under clustering** - (`KernelWriterAssembly.py:2581-2644`). When `WorkGroupIdFromTTM` and - `enableCluster = (ClusterDim[0]*ClusterDim[1]) != 1`, and `cluster_id != 0`, - the WG ids are rebuilt from the cluster HW regs: - - ```2628:2632:projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py - moduleRegInit.add(SMulI32(dst=sgpr("WorkGroup0"), src0="ttmp9", src1=sgpr(sTmp+3),\ - comment="cluster_x * nwg_x")) - moduleRegInit.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), \ - src1=sgpr(sTmp+1), \ - comment="WorkGroup0 = (cluster_x * nwg_x) + wg_x")) - ``` - - i.e. **`WorkGroup0 = cluster_x * nwg_x + wg_x`**, where `nwg_x = ClusterDim[0]` - is the number of WGs per cluster along x, `cluster_x` is the cluster index, - `wg_x` is the WG's position within the cluster. **Key property: WGs of one - cluster occupy a *contiguous* `WorkGroup0` range `[cluster_x*C, cluster_x*C + C)` - with `C = ClusterDim[0]`.** (`ttmp9` here is the raw `wg_x`.) -- **WGM/XCC remap is bypassed under clustering:** - `Components/WorkGroupMappingAlgos.py:99-101` short-circuits `wgmXCC` when - `enableCluster`. - -### 1.2 StreamK internals (verified) - -- Variants dispatched by `kernel["StreamK"]` in - `Tensile/Components/StreamK.py`: `0` Off; `3` `StreamKTwoTileDPFirst` - (`:2479`); `4` `StreamKDynamic` (`:2896`, atomic work queue); `5` - `StreamKHybrid` (`:3301`). Table at `:4087-4089`. Orthogonal flags: - `StreamKAtomic`, `StreamKForceDPOnly`, `StreamKFixupTreeReduction`. -- **StreamKIdx == WorkGroup0** (`StreamK.py:2504`): - - ```2504:2505:projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py - module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), - comment="Save original StreamK index")) - ``` - - **Gotcha:** the SK3 `preLoop` re-reads a *raw* WG id from `ttmp9` - immediately before this, overwriting the cluster-remapped value: - - ```2495:2498:projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py - if writer.states.archCaps["WorkGroupIdFromTTM"]: - module.add(SMovB32(dst=sgpr("WorkGroup0"), src="ttmp9", comment="workaround")) - module.add(SAndB32(dst=sgpr("WorkGroup1"), src0=hex(0xFFFF), src1="ttmp7", comment="workaround")) - module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7", comment="workaround")) - ``` - - Under clustering `ttmp9 == wg_x` (position *within* the cluster), **not** the - global linear index. This workaround must be made cluster-aware (§2). -- `StreamKIter = StreamKIdx * SKItersPerWG (+ extras)` (tree-reduction init at - `:2621-2626`; parallel/DP paths at `:2513`, `:2569-2587`). So **consecutive - `StreamKIdx` map to consecutive global-iteration windows.** -- Tile mapping: `skTileIndex` (`:439`) magic-divides the global iter into a tile - index and derives `StreamKLocalStart`/`StreamKLocalEnd`; `skIndexToWG` (`:487`) - maps tile → `WorkGroup0/1/2`. **Owner** of a tile is the WG with + `ContractionSolution` / `Contractions.py`. The grid dimension must be a + multiple of the cluster size. +- **Kernel-side WG-id remap under clustering:** `WorkGroup0 = cluster_x*nwg_x + + wg_x`, so the WGs of one cluster occupy a *contiguous* `WorkGroup0` range + `[cluster_x*C, cluster_x*C + C)`. WGM/XCC remap is bypassed under clustering. + +### 1.2 StreamK internals + +- Variant 3 `StreamKTwoTileDPFirst` is the clustered path. `StreamKIdx` is derived + from the cluster-remapped `WorkGroup0` (the raw-`ttmp9` workaround is skipped + under `enableCluster`, so `StreamKIdx` is the global linear index, not the + within-cluster `wg_x`). +- `StreamKIter = StreamKIdx * SKItersPerWG (+ extras)`, so consecutive + `StreamKIdx` map to consecutive global-iteration windows. +- `skTileIndex` magic-divides the global iter into a tile index and derives + `StreamKLocalStart`/`StreamKLocalEnd`; `skIndexToWG` maps tile -> + `WorkGroup0/1/2`. The **owner** of a tile is the WG with `StreamKLocalStart == 0` (it runs the reduction + final store). -- **Reduction (producer/consumer over global workspace + flag)** lives in the - epilogue, `storeBranchesCommon` (`:744`), in two topologies: - - *linear* (`:911-1035`): owner sets `sCtaIdx = StreamKIdx+1` (`:941`) and - walks **consecutive** peer indices `+1,+2,…`, per-peer spin-waiting on the - global flag: - - ```961:965:projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py - module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(sCtaIdx), shiftHex=log2(4), comment="flag offset based on CTA index")) - module.add(memOrder.readFlag(writer, dst=tmpSgpr+2, soffset=sgpr(tmpSgpr))) - if kernel["DebugStreamK"] & 2 == 0: - module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=1, comment="check if ready")) - module.add(SCBranchSCC0(labelName=skFixupLabel.getLabelName(), comment="if flag not set, wait and check again")) - ``` - - then `acquireFence`, a **workgroup** `SBarrier` (`:968`), wave-0 flag reset - (`:970-979`), and `fixupStep` (`:1010`, defined `:1680`) which loads peer - partials from the workspace and `VAddF32`-accumulates. - - *tree* (`StreamKFixupTreeReduction=1`, `:755-910`): same handshake with a - log-tree peer schedule (`sIdxOffset *= 2`). - - Non-owners write their partial to the global workspace slot via - `computeWorkspaceSrd` (`:1108`, slot `= AddressWS + StreamKIdx*(MT0*MT1*bpeCinternal)`) - and raise their flag via `setFlagValue` (`:1414`). -- **Memory ordering** (`StreamK.py:112-249`). gfx1250 uses - `StreamKMemoryOrderingDevScopeFences` (cap `HasInvWbDevFences`): release = - `s_waitcnt` + `global_wb scope:SCOPE_DEV`; acquire = `global_inv - scope:SCOPE_DEV`; the flag is read via **VMEM** (`readFlag`, `:236-246`) with - `s_wait_xcnt 0` before volatile VMEM (`preVolatileVmem`, `:144-153`, - `RequiresXCntForVolatileVMEM`). -- **Flags buffer:** `library/.../hipblaslt.cpp` allocates + zeros `d_Synchronizer` - and passes it as `AddressFlags`; `AddressFlags == 0` is the sentinel meaning - "parallel / post-kernel reduction" (checked at `StreamK.py:579`, `:2533`). -- **Host grid** forces `{sk.grid,1,1}` for StreamK - (`ContractionSolution.cpp:1758-1763`); the `enableCluster` branch immediately - after keeps y/z unflattened for cluster kernels: - - ```1758:1774:projects/hipblaslt/tensilelite/src/ContractionSolution.cpp - if(sizeMapping.streamK != 0) - { - rv.numWorkGroups.x = sk.grid; - rv.numWorkGroups.y = 1; - rv.numWorkGroups.z = 1; - } - - bool enableCluster = (sizeMapping.clusterDim.x > 1 || sizeMapping.clusterDim.y > 1); - if(!enableCluster) - { - if(internalArgsSupport.version >= 1) - { - rv.numWorkGroups.x *= (rv.numWorkGroups.y * rv.numWorkGroups.z); - rv.numWorkGroups.y = 1; - rv.numWorkGroups.z = 1; - } - } - ``` - - `sk.grid` is chosen by `getSKGridImpl` (`:3895-4045`); the SK3 per-WG - accounting (`skTiles`, `SKItersPerWG`, `extraIters`) is built at `:856-906` - and `:945-973`. There is **no** rounding of `sk.grid` to a cluster multiple - today. - -### 1.3 Cluster-barrier codegen paths that already exist - -- **Subtile Python path** `Tensile/Components/Subtile/ClusterBarrier.py`: - `subtileClusterBarrierSignal` (wave-0 election: `s_cmp_eq_u32 WaveIdx==0`, - `s_cbranch_scc0`, `SBarrier(True,False,True)` = `s_barrier_signal -3`), - `subtileClusterBarrierWait` (`SBarrier(True,True,True)` = `s_barrier_wait -3`), - and `insertClusterBarrier` which **splices around the mainloop's workgroup - barrier** and asserts `asmCaps["HasClusterBarrier"]`. -- **stinkytofu C++ pass** `shared/stinkytofu/src/transforms/asm/InsertClusterBarrierPass.cpp`, - run from `Gfx1250Backend.cpp` when `moduleOptions.ClusterBarrier`. Its anchors - are all **mainloop / load / tail-loop** points: `tensor_load_to_lds` (Rule 4), - `label_GSU_1` (Rule 1), `label_openLoopL` (Rule 3, currently disabled), and the - `/* Tail Loop */` TEXTBLOCK marker (Rule 5). **None of these fire in the - StreamK epilogue/store reduction path** — verified by reading the pass in full. +- Non-owners write their partial to the global workspace slot + (`AddressWS + StreamKIdx*(MT0*MT1*bpeCinternal)`); the owner accumulates peer + partials with `fixupStep` (`VAddF32`). +- **Memory ordering:** gfx1250 uses `StreamKMemoryOrderingDevScopeFences` (cap + `HasInvWbDevFences`); the flag is read via VMEM with `s_wait_xcnt 0` before + volatile VMEM. `AddressFlags == 0` is the sentinel for the parallel / + post-kernel reduction. --- -## 2. WG-id / indexing reconciliation - -### 2.1 The core geometric fact - -With `ClusterDim = [C, 1]` (1-D cluster along x), the kernel-side remap gives -`WorkGroup0 = cluster_x * C + wg_x`. StreamK uses `StreamKIdx = WorkGroup0` and -`StreamKIter = StreamKIdx * SKItersPerWG (+extra)`, and the reduction walks -**consecutive** `StreamKIdx` (`sCtaIdx = StreamKIdx+1, +2, …`). Therefore: - -> **Cluster `c` owns exactly the contiguous StreamK index range -> `[c*C, c*C + C)`, which is also a contiguous block of global iteration -> windows.** A tile whose peer set is a contiguous run of `p` indices starting at -> its owner `k` is fully intra-cluster **iff** `k` and `k+p-1` fall in the same -> `[c*C, c*C+C)` block. - -This is the property the whole design leans on: no reshuffling is needed, the HW -remap already clusters consecutive StreamK indices. - -### 2.2 Chosen mapping - -- **`ClusterDim = [C, 1]`**, `C` a power of two, `2 <= C <= 16`, `ClusterDim[1] == 1`. - A 1-D cluster is required because the StreamK grid is 1-D `{skGrid,1,1}`; a - `ClusterDim[1] > 1` would demand `gridDimY % ClusterDim[1] == 0` while - `gridDimY == 1` (launch at `HipSolutionAdapter.cpp:589-600`), which cannot be - satisfied. -- **Fix the `ttmp9` workaround** (`StreamK.py:2495-2498`): under - `enableCluster`, `StreamKIdx` must be the *global* linear index - `cluster_x*C + wg_x`, not raw `ttmp9 (= wg_x)`. Two options: - 1. *(preferred)* When `enableCluster`, **skip** the `ttmp9` workaround and use - the already cluster-remapped `WorkGroup0` produced in - `KernelWriterAssembly.py:2588-2644`. - 2. Recompute `cluster_x` from `HW_REG_IB_STS2[6:4]` locally and form - `cluster_x*C + wg_x`. More SGPR traffic; only needed if the remapped - `WorkGroup0` is not live at `preLoop`. -- **Derive cluster-local coordinates** for barrier owner election and peer - bookkeeping (cheap, from values already computed): - - `wg_in_cluster = StreamKIdx & (C-1)` (C power of two). - - `cluster_base = StreamKIdx & ~(C-1)` (first StreamKIdx of this cluster). - - `cluster_last = cluster_base + C - 1`. -- **Grid sizing (host).** `getSKGridImpl` (`ContractionSolution.cpp:3895-4045`) - must round `sk.grid` **up to a multiple of `C`** when the StreamK-cluster path - is active, and that rounded value must flow into both `rv.numWorkGroups.x` - (`:1760`) and the `skGrid`/`SKItersPerWG`/`skTiles` kernel args (`:856-906`, - `:945-973`). The tree-fixup `< 2^24 / 2^16` guards at `:4031-4043` still apply - to the rounded grid. `HipSolutionAdapter.cpp:589` already passes - `numWorkGroups.x` as `gridDimX`, so a multiple-of-`C` grid satisfies the HW - requirement with no launch change. - -### 2.3 One-tile-per-cluster alignment (MVP partition) - -To make the intra-cluster fast path the *guaranteed* common case (and to sidestep -the deadlock hazard of §6), the MVP additionally **aligns each StreamK tile's -peer group to one cluster**: - -- Choose the per-WG split so **peers-per-tile `== C`** and **tiles align to - cluster boundaries**: `SKItersPerWG * C == itersPerTile` and - `skGrid == C * skTiles` (after rounding). Then cluster `c` ⟷ StreamK tile `c`, - and every WG in a cluster is a peer of the same tile. -- This is a constrained sub-mode of SK3 (fixed, even split — closest to the - existing "parallel reduction" `skSplit` path, `StreamK.py:2536-2591`, - `ContractionSolution.cpp:933-942`, where `skSplit = grid/tiles` and - `SKItersPerWG = itersPerTile/skSplit`). The `skSplit` model already produces a - **fixed, contiguous** peer group of size `skSplit` per tile — setting - `skSplit == C` makes each tile's peers exactly one cluster. -- The residual imbalance cases (last cluster partially filled, `extraIters` - giving big/little WGs) are handled by keeping the global-flag path as a runtime - fallback (§3.4). +## 2. Indexing and grid ---- +### 2.1 Reduction shape and geometry -## 3. Reduction redesign - -### 3.1 Which handshake is replaced - -The expensive operation is the **cross-CU global-flag spin-wait** in -`storeBranchesCommon` (`:961-965` linear, `:872-877` tree) plus the device-scope -release/acquire fences and the wave-0 flag reset (`:970-979`). When the tile's -peers co-reside in a cluster, all of that is replaced by **one cluster split -barrier**: - -- **Non-owner (peer WG)** — after computing its partial: - 1. write partial to its global workspace slot (`computeWorkspaceSrd`, - `writePartials`) — *unchanged for v1*; - 2. `releaseFence` (relaxed scope, §3.3); - 3. wave-0 `s_barrier_signal -3`; - 4. exit (no flag store). -- **Owner WG** — after computing its own partial portion: - 1. `s_barrier_wait -3` **once** (not per-peer): the cluster split barrier - guarantees *every* WG in the cluster has signalled, i.e. all `C-1` peers - have published; - 2. `acquireFence` (relaxed scope, §3.3); - 3. run the existing `fixupStep`/`fixupBatch` loop over the `C-1` peer slots - (`StreamKIdx+1 … cluster_last`) to `VAddF32`-accumulate — *unchanged - accumulation math*; - 4. normal alpha/beta store. - -Because the barrier is a single all-cluster synchronization, the owner waits once -for the whole cluster instead of `C-1` per-peer flag polls, and **no global flag -store/reset/spin is executed at all** on the fast path. - -### 3.2 Where partials live — global WS for v1 - -- **v1:** keep partials in the **global workspace** (reuse `computeWorkspaceSrd` - slot layout and `fixupStep` verbatim). Only the *synchronization* changes - (barrier instead of flag). This minimizes the diff and reuses the audited - accumulation path. -- **v2 (future):** stage cluster-local partials in **LDS** (cluster WGs co-reside - on one shader engine and can share via LDS/TDM multicast), eliminating the WS - round-trip. Deferred: it changes the `fixupStep` addressing and interacts with - epilogue LDS usage/bias LDS barriers. - -### 3.3 Fences — can device scope relax to cluster scope? - -- The device-scope `global_wb`/`global_inv` (`StreamK.py:206-249`) exist because - producer and consumer may sit on **different CUs / L2 partitions**. Intra-cluster - peers co-schedule on **one shader engine**, so a narrower coherence scope is - correct for the fast path *iff* the arch exposes it. -- **v1 (safe):** keep `SCOPE_DEV` `global_wb`/`global_inv` around the WS - partials even on the fast path. This is strictly correct (superset ordering) - and lets us validate the barrier mechanics independently of fence tuning. The - win is removing the *spin-wait*, which dominates. -- **Future (opt):** if a shader-engine / cluster cache scope is available - (extend the `CacheScope` enum + a new arch cap, mirroring `HasInvWbDevFences`), - add a `StreamKMemoryOrderingClusterScopeFences` subclass selected only on the - intra-cluster fast path. Gate behind a cap so non-gfx1250 and the fallback - path are untouched. This depends on HW confirmation that a cluster/SE-scoped - `global_wb/global_inv` is semantically sufficient given the cluster - co-residency guarantee, so the MVP keeps `SCOPE_DEV`. - -### 3.4 Owner selection + correct fallback - -- **Owner within a cluster:** unchanged — the WG with `StreamKLocalStart == 0`. - Under one-tile-per-cluster alignment this is `wg_in_cluster == 0` - (`cluster_base`), which also naturally elects the flag-reset/store wave. -- **Fallback (peers span >1 cluster, DP tiles, atomic, tree-straddle):** keep the - existing global-flag reduction **compiled in** and select it at runtime. The - guard is computable in the epilogue: - - ``` - intra_cluster = (owner_idx == cluster_base) && (last_peer_idx <= cluster_last) - ``` - - where `last_peer_idx` is derived from the existing fixup bounds - (`sFixupEnd`/`sCtaIdx` logic, `:955`, `:1020-1032`). If `intra_cluster`, take - the cluster-barrier fast path; else fall through to the existing flag path - verbatim. Under the MVP alignment (§2.3) `intra_cluster` is true for all SK - tiles except the last partial cluster, so the fallback rarely runs but - guarantees correctness. -- **DP tiles** (`StreamKLocalStart==0 && finished`) never enter the reduction and - branch to the regular store (`:778`, `:932-933`), so they never *wait* on a - cluster barrier — but they must still **signal** if they share a cluster with - reducers (see §6 deadlock). MVP alignment avoids mixing DP and SK WGs in a - cluster, so this does not arise in v1. +Pure reduction is expressed as `ClusterDim = [1, C]` (Cs = ClusterDim[0] = 1, +Ck = ClusterDim[1] = C the K-split peer count), a genuine 2-D cluster whose launch +grid is `[skGrid/C, C, 1]` (so `gridDimY % C == 0` holds). The Y rank is folded +into the linear StreamK index at preLoop: ---- +``` +StreamKIdx = WorkGroup0*Ck + WorkGroup1 (k = WorkGroup1 fastest) +``` -## 4. Where the cluster barrier is emitted +This keeps `StreamKIdx` a dense unique index; the 1-D `Ck==1` path emits no fold. +C must be a power of two in `2..16`. -**Decision: emit the cluster barrier inline in `StreamK.py`** (new small helpers -`clusterReduceSignal(writer, kernel)` / `clusterReduceWait(writer, kernel)`) -using the rocisa `SBarrier(separate=True, wait=…, clusterBarrier=True)` emitter -directly, reusing the **wave-0 election pattern** from -`Subtile/ClusterBarrier.py:subtileClusterBarrierSignal` (copy the 3-instruction -`s_cmp_eq_u32 WaveIdx,0 / s_cbranch_scc0 skip / s_barrier_signal -3 / skip:` -shape). Assert `asmCaps["HasClusterBarrier"]` exactly like -`insertClusterBarrier` (`ClusterBarrier.py:74-75`). - -**Rationale:** - -- **The stinkytofu `InsertClusterBarrierPass` will not fire here.** Its anchors - (`tensor_load_to_lds`, `label_GSU_1`, `label_openLoopL`, `Tail Loop`) are all - mainloop/prologue/tail markers; the StreamK reduction is in the epilogue - `storeBranchesCommon`. Adding a new anchor for the epilogue to that C++ pass - would duplicate StreamK's tile/peer bookkeeping in a place that has no access - to `StreamKLocalStart`/`sCtaIdx`. Rejected. -- **`Subtile/ClusterBarrier.py:insertClusterBarrier` is also mainloop-shaped** - (it splices around the *mainloop* workgroup barrier and hides the election - branch behind a WMMA). The StreamK reduction has no WMMA to hide behind and a - very different control-flow (spin loop, wave-0 flag reset). We *reuse the - primitive* (`SBarrier(...,clusterBarrier=True)` + wave-0 election) but not the - splice. This keeps the barrier co-located with the `StreamKLocalStart==0` owner - logic and the `intra_cluster` runtime guard, where the needed SGPRs are live. - -Concretely the emit sites in `storeBranchesCommon`: -- Replace the per-peer flag poll block (linear `:960-979`; tree `:868-889`) with: - `if intra_cluster: {owner: clusterReduceWait + acquireFence; skip flag reset}`. -- Add non-owner signal: at the end of `writePartials` - (`:1042`/`:2462`/`:2876`), after the partial store + `releaseFence`, emit - `clusterReduceSignal` on the fast path. +Reduction (Ck) is **derived** from the cluster shape, not a user parameter: there +is no `StreamKClusterReduction` / `StreamKClusterKSplit` opt-in. The internal +derived key falls out of `ClusterDim[1] > 1`. ---- +### 2.2 Cluster ownership property + +Because the HW remap already clusters consecutive `StreamKIdx`, cluster `c` owns a +contiguous `StreamKIdx` range, which is also a contiguous block of global +iteration windows. A tile whose peer set is a contiguous run is fully +intra-cluster iff its first and last peer indices fall in the same cluster block. +No reshuffling is needed. -## 5. Enablement / plumbing - -### 5.1 Solution parameter - -Add an explicit boolean solution parameter **`StreamKClusterReduction`** -(default `0`) rather than overloading `ClusterDim` alone, because clustering is -already overloaded to mean "Multicast on" (§1.1). Enabling it requires: - -- `StreamK == 3` (see §2.3; SK4/SK5 are out of MVP scope — dynamic/atomic peer - sets cannot be statically clustered). -- `ClusterDim == [C, 1]` with `C` a power of two, `2 <= C <= 16`. -- gfx1250 (`asmCaps["HasClusterBarrier"]`) and `archCaps["HasNewBarrier"]`. -- `TDMInst != 0` (so `WaveIdx` is allocated — the wave-0 election needs it; same - condition that gates `ClusterBarrier` at `Solution.py:980`). -- `not StreamKAtomic` and `not StreamKForceDPOnly` for the fast path (these skip - the reduction entirely — `storeBranchesCommon:748-749`). - -### 5.2 Decouple Multicast from ClusterDim - -`Solution.py:977-978` forces `Multicast=True` for any `ClusterDim != [1,1]`. -For a **barrier-only** StreamK cluster we do *not* want cooperative TDM loads in -the MVP. Change: when `StreamKClusterReduction` is on (and Multicast -is not independently requested), keep `Multicast=False` while still setting -`ClusterBarrier`-style capability for the epilogue emit. `Multicast` is now an -independent tri-state opt-in and cluster-cooperative loads (StreamK DP -B-multicast) have shipped as a sibling feature — see -`cluster-load-component-and-streamk-multicast.md`. - -### 5.3 Validation rules (SolutionStructs) - -- Reject `StreamKClusterReduction` unless the §5.1 predicate holds - (mirror the reject-with-reason pattern used throughout `Solution.py`). -- Reject `ClusterDim[1] != 1` when `StreamKClusterReduction` (1-D grid, §2.2). -- Interaction with existing gfx1250 StreamK constraints: gfx1250 StreamK today - requires MX data (`TDMInst=3`, MX TDM loads); `isStreamKConstantsToVgprEnabled` - is SK3-only; `StreamKXCCMapping=3` overflows SGPRs (use `0`). The new param - inherits all of these (SK3 + MX + XCC=0). - -### 5.4 Host plumbing - -- `getSKGridImpl` rounding to multiple of `C` (§2.2) — behind the same - size-mapping flag (thread `sizeMapping.streamKClusterReduction` + - `sizeMapping.clusterDim` through, both already partially present: - `clusterDim` at `ContractionSolution.cpp:1765,1776`). -- Kernel-arg accounting (`skGrid`, `SKItersPerWG`, `skTiles`) uses the rounded - grid and the fixed even split (`skSplit == C`, reuse the parallel-reduction - accounting at `:933-942`). +### 2.3 Grid sizing + +`getSKGridImpl` rounds `skGrid` to a multiple of `C` when the cluster path is +active, and that rounded value flows into `rv.numWorkGroups` and the +`skGrid`/`SKItersPerWG`/`skTiles` kernel args. The workspace-overflow guards +(tree-fixup `< 2^24 / 2^16`) apply to the rounded grid. The reduction uses a fixed +even split (`skSplit == C`), so a tile's peers are exactly one cluster. --- -## 6. Risks / gotchas and de-risking +## 3. Reduction handshake -| Risk | Why | De-risk | -|---|---|---| -| **Deadlock if a cluster member never signals.** | `s_barrier_wait -3` blocks until *all* cluster WGs have `s_barrier_signal -3`'d. Any cluster WG that early-exits (e.g. `preLoop` `KernelEnd` branch `StreamK.py:2522`; a DP-only WG; a peer that took the "started & finished tile" store path `:778/:933`) never signals ⇒ owner hangs. | **MVP alignment (§2.3):** a cluster == one SK tile's peers, all of which run the reduction; DP WGs are in DP-only clusters that never wait. **Invariant to enforce in codegen:** *every* WG on the cluster fast path must signal on *every* exit path before leaving (including the "finished my slice" path). Add the signal at a single choke point in the epilogue guarded by `intra_cluster`, not scattered. Keep the global-flag fallback for any tile whose membership is not provably complete. | -| **Cluster size vs `fixup_peers` mismatch (big/little imbalance).** | SK3 `extraIters` gives the first `extraIters` WGs one extra iteration, so peers-per-tile can vary by ±1 and straddle a `C` boundary. | Use the **fixed even split** (`skSplit==C`, no extraIters) for the cluster mode; route the leftover/partial last cluster through the fallback via the `intra_cluster` runtime guard (§3.4). | -| **Peers span >1 cluster (general SK).** | General SK tiling does not align tiles to clusters. | Runtime `intra_cluster` guard selects the fast path only when provably intra-cluster; otherwise the existing global-flag path runs unchanged. | -| **SGPR pressure.** | gfx1250 SK SGPRs already tight; XCC=3 overflows. | Derive `wg_in_cluster`/`cluster_base` by cheap `AND`/`AND-NOT` on the existing `StreamKIdx` (C power of two) — no division, no new persistent SGPRs. Reuse `allocTmpSgpr` scopes. Forbid `StreamKXCCMapping=3` with the new param. | -| **`ttmp9` workaround corrupts index under clustering.** | `StreamK.py:2495-2498` reads raw `wg_x`. | §2.2 fix: use the cluster-remapped `WorkGroup0`. Covered by a codegen snapshot assert that `StreamKIdx` derivation contains the `cluster_x*C + wg_x` form. | -| **Fence relaxation incorrect.** | Cluster-scope `global_wb/global_inv` may not be coherent. | The MVP keeps `SCOPE_DEV` (correct superset). Relaxation is a gated future opt pending HW confirmation. | -| **Multicast side effects.** | `ClusterDim!=[1,1]` forces `Multicast=True`, changing load codegen and adding SGPR mask compute (`KernelWriterAssembly.py:2646-2655`). | Decouple (§5.2): barrier-only v1 keeps `Multicast=False`. | +### 3.1 What is replaced ---- +When a tile's peers co-reside in a cluster, the cross-CU global-flag spin-wait +(plus the per-peer flag reset) is replaced by **one cluster split barrier**: -## 7. Test plan (behavior → test) — as shipped +- **Peer WG**, after computing its partial: write the partial to its global + workspace slot; `releaseFence`; wave-0 `s_barrier_signal -3`; **and** + `s_barrier_wait -3`; then exit (no flag store). +- **Owner WG**, after its own partial: `s_barrier_signal -3` + `s_barrier_wait -3` + once (not per-peer); `acquireFence`; run the existing `fixupStep` over the peer + slots; normal alpha/beta store. -Mirrors existing patterns: CPU asm-string unit tests, syrupy snapshot -characterization, and a GPU roundtrip (run under the arch's simulator/hardware). +### 3.2 Symmetric-barrier invariant -| Behavior | Test type | Where | -|---|---|---| -| `clusterReduceSignal/Wait` emit exactly one wave-0 election branch + `-3` barrier ids; `HasClusterBarrier` asserted; reduction gate matrix (SK3/linear/non-atomic/non-DP only) | CPU asm-string unit | `Tensile/Tests/unit/test_streamk_cluster_reduction.py` | -| Cluster config emits real gfx1250 assembly (`err==0`); fast-path handshake present (`s_barrier_signal -3` peer arrive, `s_barrier_wait -3` owner wait); global-flag reduction retained as fallback; order-invariant golden | snapshot char | `Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_reduction_gfx1250_char.py` (designed config `_designed/gfx1250/streamk_cluster_reduction.yaml`, golden `__snapshots__/test_streamk_cluster_reduction_gfx1250_char.ambr`) | -| Store/release/signal/wait/acquire/accumulate sequence runs on gfx1250, reduces correctly, and does not deadlock, for `C` in {2,4} | GPU roundtrip | `Tensile/Tests/unit/test_streamk_cluster_reduction_gpu.py` (`@requires_gpu_gfx1250`; watchdog on hang) | -| Real multi-WG cluster StreamK GEMM end-to-end | C++ client | `Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml` (+ mxf4 sibling) | +The split barrier must be **SYMMETRIC**: every cluster member (owner and every +peer) both arrives (`s_barrier_signal -3`) and waits (`s_barrier_wait -3`) on the +same barrier. An arrive-and-exit peer is not a genuine synchronisation point -- +the owner could sum a peer slot before that cross-WGP peer's partial is globally +visible across gfx1250's partitioned L2, dropping ~(C-1)/C of the result for +C >= 4 (a cluster-aligned column stripe confined to the grid tail). + +### 3.3 Memory visibility (SCOPE_SYS) + +The cluster split barrier orders **execution** across the C co-resident peers, but +not memory. On gfx1250's partitioned L2 a `SCOPE_DEV` invalidate is not guaranteed +to re-fetch a peer partial written back on a different partition. The +cluster-reduction handshake therefore uses paired **SCOPE_SYS** fences: the peer's +partial writeback release is escalated to `SCOPE_SYS` (globally visible past the +partitioned L2), and the owner's acquire is a paired `SCOPE_SYS` fence, so the +owner observes the just-published partials. Non-cluster StreamK keeps the +`SCOPE_DEV` default. + +### 3.4 Fallback and deadlock invariant + +The per-tile global-flag handshake still exists, but only for a cluster that +straddles the SK grid boundary (not fully populated). The intra-cluster predicate +(`clusterReduceIntraCheck`) is SCC=1 when the cluster hosts a single fully +populated StreamK tile, SCC=0 otherwise; it is a pure function of the cluster's +top StreamK index and the SK grid size (both cluster-uniform), so every peer +computes the identical verdict. Both the owner wait and the peer arrive/wait gate +on that identical uniform predicate, so a cluster is never split between barrier +participants and flag setters (**deadlock invariant**). DP tiles are in DP-only +clusters that never wait on the reduction barrier. + +--- + +## 4. Where the cluster barrier is emitted -The cooperative-load / multicast siblings of these tests -(`test_streamk_cluster_coop_load_gfx1250_char.py`, -`test_streamk_cluster_multicast_gfx1250_char.py`, `test_streamk_multicast.py`, -`test_cluster_load_component.py`) cover the companion feature documented in -`cluster-load-component-and-streamk-multicast.md`. +The cluster barrier is emitted inline in `StreamK.py` (`clusterReduceSignal` / +`clusterReduceWait`) using the rocisa `SBarrier(..., clusterBarrier=True)` emitter +with a wave-0 election (`s_cmp_eq_u32 WaveIdx,0` / `s_cbranch_scc0` / +`s_barrier_signal -3`), asserting `asmCaps["HasClusterBarrier"]`. -The gfx1250 GPU marker lives in `Tensile/Tests/unit/gpu_test_helpers.py` -(`HAS_GFX1250` + `requires_gpu_gfx1250`), independent of the existing gfx950 -`requires_gpu`; the target is driven via the `TENSILE_GPU_TARGET=gfx1250` -override. +The stinkytofu `InsertClusterBarrierPass` does not fire here: its anchors +(`tensor_load_to_lds`, `label_GSU_1`, `Tail Loop`) are mainloop/prologue/tail +markers, whereas the StreamK reduction is in the epilogue `storeBranchesCommon`. +Emitting inline keeps the barrier co-located with the owner logic and the +intra-cluster runtime guard, where the needed SGPRs are live. --- -## 8. Follow-up work (out of MVP scope) +## 5. Enablement -- **Fence scope:** a cluster/shader-engine-scoped `global_wb`/`global_inv` - (§3.3) could replace `SCOPE_DEV` on the fast path once HW confirms it is - semantically sufficient given cluster co-residency, and once a `CacheScope` / - arch cap exposes it. -- **LDS partials (§3.2):** eliminate the global-WS round-trip by staging - cluster-local partials in LDS/TDM-multicast; this changes `fixupStep` - addressing and the epilogue LDS budget. -- **Broader partitions:** relax the fixed even split / one-tile-per-cluster - constraint (§2.3) toward general SK tiling with the runtime straddle guard, and - extend beyond SK3 where peer sets can be statically clustered. +- Reduction is derived from `ClusterDim = [1, C]` (Ck > 1); there is no user + opt-in parameter. Requirements (SK3; `ClusterDim` power-of-two `2..16`; gfx1250 + `HasClusterBarrier`; `TDMInst != 0`; not `StreamKAtomic`; not + `StreamKForceDPOnly`; `StreamKXCCMapping != 3`) are rejected at build time by + `_validateStreamKClusterReduction`. +- The runtime `itersPerTile % C == 0` requirement (unknown at build time) is a + per-problem selection reject via the `ClusterReductionIterCheck` predicate. +- `Multicast` is decoupled from `ClusterDim` (tri-state); the barrier-only + reduction path keeps `Multicast` off. Cooperative-load multicast ships as a + sibling feature (see `cluster-load-component-and-streamk-multicast.md`). --- -## 9. Summary of decisions - -- **Approach:** co-locate a StreamK tile's `fixup_peers` into one 1-D cluster - (`ClusterDim=[C,1]`), exploiting the HW remap `WorkGroup0 = cluster_x*C + wg_x` - which already clusters consecutive `StreamKIdx`. Replace the global-flag - spin-wait with a single cluster split barrier; keep the global-flag path as a - runtime-selected fallback. -- **Variant:** SK3 two-tile (MVP), fixed even split; SK4/SK5 out of scope - (dynamic/atomic peer sets can't be statically clustered). Barrier-only here; - cooperative-load multicast shipped separately (see - `cluster-load-component-and-streamk-multicast.md`). -- **Indexing:** `ClusterDim=[C,1]`, fix the `ttmp9` workaround to use the - cluster-remapped `WorkGroup0`, `cluster_base = StreamKIdx & ~(C-1)`; host - rounds `skGrid` to a multiple of `C`. -- **Reduction:** owner `s_barrier_wait -3` once + `acquireFence` + existing - `fixupStep`; peers write WS partial + `releaseFence` + wave-0 `s_barrier_signal -3`; - partials stay in global WS for v1; fences stay `SCOPE_DEV` for v1. -- **Barrier emission:** inline in `StreamK.py` epilogue via rocisa - `SBarrier(...,clusterBarrier=True)` + wave-0 election (reuse the - `Subtile/ClusterBarrier.py` primitive shape); **not** the stinkytofu pass / - subtile splice (their anchors don't reach the epilogue). -- **Enablement:** new `StreamKClusterReduction` solution param requiring SK3 + - `[C,1]` + gfx1250 `HasClusterBarrier` + `TDMInst!=0`; decouple Multicast. -- **Top risks:** cluster-barrier deadlock if any member fails to signal (⇒ - one-tile-per-cluster alignment + all-paths-signal invariant + fallback), SGPR - pressure (⇒ cheap bitwise cluster coords), big/little imbalance (⇒ fixed even - split + runtime guard). +## 6. Summary + +- Co-locate a StreamK tile's fixup peers into one cluster (`ClusterDim=[1,C]`, + reduction along Ck), exploiting the HW remap that already clusters consecutive + `StreamKIdx`. Replace the global-flag spin-wait with a single cluster split + barrier; keep the global-flag path as a runtime-selected fallback for a + boundary-straddling cluster. +- The barrier is symmetric (every member arrives and waits); peer -> owner + visibility rides on paired `SCOPE_SYS` release/acquire fences. +- Reduction/Ck is derived from the cluster shape (no user parameter); the host + rounds `skGrid` to a multiple of `C` with a fixed even split. +- Deadlock invariant: owner and peers gate on the identical uniform intra-cluster + predicate, so a cluster never mixes barrier participants with flag setters. diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py index 070e749c9dba..76545d6d75db 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py @@ -376,9 +376,9 @@ def streamKClusterFactors(d): index folds the cluster Y rank in (StreamKIdx = WorkGroup0*Ck + WorkGroup1). Config expressions: - * [C, 1] -> Cs=C, Ck=1 : pure multicast (1-D launch, byte-identical) + * [C, 1] -> Cs=C, Ck=1 : pure multicast (1-D launch) * [1, C] -> Cs=1, Ck=C : pure reduction (2-D launch) - * [Cs,Ck]-> both > 1 : factored (2-D launch; factored branch only) + * [Cs,Ck]-> both > 1 : factored (2-D launch; not supported) ``d`` may be a kernel or a solution ``state`` dict; both expose "ClusterDim". See docs/design/streamk-wg-clusters.md. diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index 6b4548554a59..9264de1b8018 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -2,13 +2,11 @@ # SPDX-License-Identifier: MIT """Cluster (multicast) TDM load component. -Centralizes the multicast ("cluster load") mask machinery (value compute, -``MulticastMask*`` SGPR declare/undeclare, combined-vs-split topology decision, -and per-load-site descriptor attach) that was previously duplicated across -``KernelWriter``/``KernelWriterAssembly``/``SubtileGREmit``. Behavior-preserving: -every method emits byte-identical assembly, receiving the SGPR operands the -caller already holds rather than re-allocating. Capability-selected -(``HasTDM`` + ``TDMInst == 3``), like ``TensorDataMoverLoad``. +Owns the multicast ("cluster load") mask machinery: mask value compute, the +``MulticastMask*`` SGPR declare/undeclare, the combined-vs-split topology +decision, and the per-load-site descriptor attach. Each method receives the +SGPR operands the caller already holds rather than re-allocating them. +Capability-selected (``HasTDM`` + ``TDMInst == 3``), like ``TensorDataMoverLoad``. """ from ..Component import ClusterLoad @@ -64,7 +62,7 @@ def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, # -- SGPR declare / undeclare ------------------------------------------- def declareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> None: - """Allocate the ``MulticastMask*`` SGPRs (lift of KernelWriter).""" + """Allocate the ``MulticastMask*`` SGPRs.""" if not kernel["Multicast"]: return tdmM: bool = kernel["enableTDMMetadata"] @@ -77,7 +75,7 @@ def declareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> None: writer.defineSgpr("MulticastMaskMetadata", 1) def undeclareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> Module: - """Free the ``MulticastMask*`` SGPRs (lift of KernelWriter).""" + """Free the ``MulticastMask*`` SGPRs.""" mod = Module() if not (kernel["Multicast"] and kernel["TDMInst"] != 0): return mod @@ -97,9 +95,8 @@ def computeMasks(self, writer: "KernelWriterAssembly", kernel: Mapping, *, sgprWgX: int, sgprWgY: int, sgprNWgX: int, sTmp: int) -> Module: """Compute the multicast mask value(s) into the ``MulticastMask*`` SGPRs. - Verbatim lift of the ``defineAndResources`` mask compute; the caller - passes the operands it already holds (``sgprWgX``/``sgprWgY``/``sgprNWgX`` - and ``sTmp`` whose ``+4`` slot is scratch) so the output is byte-identical. + The caller passes the operands it already holds (``sgprWgX``/``sgprWgY``/ + ``sgprNWgX`` and ``sTmp`` whose ``+4`` slot is scratch). """ mod = Module() if not kernel["Multicast"]: diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 26bbfc6ce69c..8e01dff3aaa1 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -1005,8 +1005,7 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # invalidate is not guaranteed to re-fetch a peer partial # written back on a different partition, so escalate the # cluster-reduction acquire (paired with the SCOPE_SYS - # release below) to the system-coherent point. Held fix for - # the PGR1 boundary-cluster race; see red-pgr1-fix notes. + # release below) to the system-coherent point. module.add(memOrder.acquireFence(writer, scope=CacheScope.SCOPE_SYS)) # once: observe peers' published partials module.add(skClusterSetupDone) @@ -1452,8 +1451,7 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, # then arrives at the split barrier; escalate the release writeback # to SCOPE_SYS so the partial is globally visible past gfx1250's # partitioned L2 before the owner's paired SCOPE_SYS acquire reads - # it. Non-cluster StreamK keeps the SCOPE_DEV default. Held fix for - # the PGR1 boundary-cluster race; see red-pgr1-fix notes. + # it. Non-cluster StreamK keeps the SCOPE_DEV default. releaseScope = (CacheScope.SCOPE_SYS if self._streamKClusterReductionEnabled(writer, kernel) else None) @@ -1464,7 +1462,7 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, # SCOPE_SYS release fence, participate in the SYMMETRIC cluster # barrier INSTEAD OF raising the global completion flag. The peer # both ARRIVES (s_barrier_signal -3) and WAITS (s_barrier_wait -3), - # exactly like the owner and like the HW-validated multicast + # exactly like the owner and like the multicast # handshake -- it does NOT signal-and-exit. The peer's wait keeps it # parked at the cluster rendezvous until every member (owner # included) has arrived, giving the owner's paired acquire a real @@ -1626,24 +1624,15 @@ def _streamKClusterBarrierFastEnabled(self, writer, kernel): The cluster ``s_barrier_signal/wait -3`` orders EXECUTION across all C co-resident peers; peer->owner memory VISIBILITY rides on the paired SCOPE_SYS release (peer, before it arrives) / acquire (owner, after it - waits) fences. The earlier ASYMMETRIC form -- peers only - ``s_barrier_signal -3`` (arrive) and then branch away / exit while only - the owner ``s_barrier_wait -3`` -- was not a genuine cluster - synchronisation point on gfx1250: for a C >= 4 reduction cluster the - owner could observe the barrier satisfied and sum a peer slot before - that cross-WGP peer's partial was globally visible, dropping exactly one - contribution (device ~= (C-1)/C * reference, a cluster-aligned column - stripe confined to the grid tail). Adding more fences did not help - because the defect was structural, not a missing drain. - - The fix mirrors the HW-validated multicast handshake: make the barrier - SYMMETRIC -- every cluster member (owner AND every peer) both arrives - (``s_barrier_signal -3``) AND waits (``s_barrier_wait -3``) on the same - barrier, so no peer proceeds/exits until the whole cluster has rendezvoused - and the owner's acquire has a real cross-cluster order to observe. With - the symmetric barrier the fast path is correct for ALL C (validated - C = 2, 4, 8), so it is enabled whenever the cluster reduction is active; - there is NO per-peer global-flag fallback on the C >= 4 correctness path. + waits) fences. + + The split barrier must be SYMMETRIC: every cluster member (owner AND + every peer) both arrives (``s_barrier_signal -3``) AND waits + (``s_barrier_wait -3``) on the same barrier. An arrive-and-exit peer is + not a genuine synchronisation point -- the owner could sum a peer slot + before that cross-WGP peer's partial is globally visible across the + partitioned L2, dropping ~(C-1)/C for C >= 4 (a cluster-aligned column + stripe confined to the grid tail). The per-tile global-flag handshake still exists, but only for a cluster that straddles the SK grid boundary (``clusterReduceIntraCheck`` == SCC0, @@ -1665,8 +1654,8 @@ def _clusterElectArriveSignal(self, writer, module, *, labelBase, electTag, wait the remaining waves branch over it via ``labelBase``. When ``wait`` is set an all-waves cluster wait (``s_barrier_wait -3``) follows the arrive. ``labelBase``/``electTag`` are supplied per call site so the emitted label - and pool tag -- and hence the assembly -- stay byte-identical to the - pre-extraction inline copies. Instructions are appended to ``module``. + and pool tag are distinct per call site. Instructions are appended to + ``module``. """ skipSignal = Label(label=writer.labels.getNameInc(labelBase), comment="") elect = writer.sgprPool.checkOut(1, electTag) @@ -1722,7 +1711,7 @@ def clusterReduceIntraCheck(self, writer, kernel): """ module = Module("StreamK cluster intra-cluster check") # Whole-cluster size C spans every co-resident peer WG. 1-D pure-multicast - # [C,1]: C = Cs = ClusterDim[0] (byte-identical). 2-D pure-reduction [1,C]: + # [C,1]: C = Cs = ClusterDim[0]. 2-D pure-reduction [1,C]: # C = Ck = ClusterDim[1]. In general C = Cs*Ck. _, _, C, _ = streamKClusterFactors(kernel) skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) @@ -2939,8 +2928,8 @@ def preLoop(self, writer, kernel): # This keeps StreamKIdx a dense unique index whose (s,k) decode matches the # 1-D [C,1] scheme (StreamKIdx & (Ck-1) = wg_y = k, StreamKIdx & (C-1) = the # within-cluster rank). The fold is written into WorkGroup0 so the SMov/VMov - # save below (unchanged) still copies the final index; the 1-D Ck==1 path is - # byte-identical (no fold emitted). See docs/design/streamk-wg-clusters.md. + # save below (unchanged) still copies the final index; the 1-D Ck==1 path + # emits no fold. See docs/design/streamk-wg-clusters.md. _, ck2d, _, is2d = streamKClusterFactors(kernel) if is2d: module.add(SMulI32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=hex(ck2d), diff --git a/projects/hipblaslt/tensilelite/Tensile/Contractions.py b/projects/hipblaslt/tensilelite/Tensile/Contractions.py index 39a8d7bb7a9b..0c163bb7f8b0 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Contractions.py +++ b/projects/hipblaslt/tensilelite/Tensile/Contractions.py @@ -591,14 +591,14 @@ def CompoundPredicates(cls, state, problemType): valuepredicates.append(state["MacroTile1"]) valuepredicates.append(state["GlobalSplitU"]) # value[3] is the M-adjacency (shared-B) alignment axis Cs = ClusterDim[0]. - # Pure multicast [C,1]: Cs = C (byte-identical to historic value). Pure - # reduction [1,C]: Cs = 1 (no M-alignment constraint). + # Pure multicast [C,1]: Cs = C. Pure reduction [1,C]: Cs = 1 (no + # M-alignment constraint). valuepredicates.append(state["ClusterDim"][0]) # value[4] is the N-tile divisor. For a genuine 2-D StreamK cluster # (Ck = ClusterDim[1] > 1, e.g. pure reduction [1,C]) the Y-extent is the # K-split reduction / index-generation axis, NOT an N-tiling axis, so it # must NOT constrain the N-tile grid -> pin to 1. 1-D StreamK ([C,1]) and - # dense (non-StreamK) clusters keep ClusterDim[1] (byte-identical). + # dense (non-StreamK) clusters keep ClusterDim[1]. if state.get("StreamK", 0) == 3 and state["ClusterDim"][1] > 1: valuepredicates.append(1) else: diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 5ea82babc070..dacafd8ec865 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -248,13 +248,12 @@ def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): return True # Pure reduction ([1, C]) has Cs = ClusterDim[0] = 1, so StreamKMulticast is - # off. Factored ([Cs,Ck] with BOTH axes > 1) would turn both on, but that is - # rejected in the SK cluster guard on this branch (it lives on the factored - # branch). Defensive: reject the combination here too. + # off. Factored ([Cs,Ck] with BOTH axes > 1) would turn both on, which is + # rejected in the SK cluster guard. Defensive: reject the combination here too. if state.get("StreamKMulticast", 0): reject(state, printRejectionReason, - "StreamKMulticast (Cs>1) + StreamKClusterReduction (Ck>1) is the " - "factored cluster; not supported on this branch") + "Factored StreamK cluster [Cs,Ck] with both axes > 1 is not " + "supported; use ClusterDim=[C,1] (multicast) or [1,C] (reduction)") return False # SK3 (StreamKTwoTileDPFirst) only; SK4/SK5 dynamic/atomic peer sets can not @@ -336,14 +335,13 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): return True # Multicast (Cs>1) and reduction (Ck>1) both on is the FACTORED cluster - # [Cs,Ck] with both axes > 1 -- that lives on the streamk-factored-cluster-mode - # branch and is rejected in the SK cluster guard on this branch. Defensive - # reject here too: on this (multicast/reduction-only) branch the two derived - # roles are mutually exclusive (pure multicast [C,1] xor pure reduction [1,C]). + # [Cs,Ck] with both axes > 1, which is rejected in the SK cluster guard. + # Defensive reject here too: the two derived roles are mutually exclusive + # (pure multicast [C,1] xor pure reduction [1,C]). if state.get("StreamKClusterReduction", 0): reject(state, printRejectionReason, - "StreamKMulticast (Cs>1) + StreamKClusterReduction (Ck>1) is the " - "factored cluster; not supported on this branch") + "Factored StreamK cluster [Cs,Ck] with both axes > 1 is not " + "supported; use ClusterDim=[C,1] (multicast) or [1,C] (reduction)") return False # StreamKMulticast is auto-enabled by ClusterDim on SK3, but the multicast mask @@ -1243,7 +1241,7 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, # below is the ONLY place it is turned on. state["StreamKMulticast"] = 0 # StreamKClusterReduction is a DERIVED-ONLY internal state key (like - # StreamKMulticast): reduction is no longer a user opt-in, it is expressed + # StreamKMulticast): reduction is not a user opt-in -- it is expressed # purely by the cluster shape ClusterDim = [1, C] (Ck = ClusterDim[1] > 1). # Seed it off here (mirroring StreamKMulticast); the collapse below is its # sole enable site. @@ -1254,10 +1252,10 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, # * Cs = ClusterDim[0] spatial B-multicast peers -> StreamKMulticast iff Cs>1 # * Ck = ClusterDim[1] K-split reduction peers -> StreamKClusterReduction iff Ck>1 # Config expressions: [C,1] = pure multicast, [1,C] = pure reduction. - # Factored (both axes > 1) is the streamk-factored-cluster-mode branch and is - # rejected here (see the SK cluster guard in assignDerivedParameters). + # Factored (both axes > 1) is not supported and is rejected here (see the + # SK cluster guard in assignDerivedParameters). # - # A StreamK cluster with no cooperative role no longer exists -- "ClusterDim + # A StreamK cluster always carries a cooperative role -- "ClusterDim # without cluster loads" is not a supported state. If the resulting config # cannot be satisfied (e.g. TDMInst!=3 or non-gfx1250), _validateStreamKMulticast # / _validateStreamKClusterReduction reject it at build time rather than @@ -1272,8 +1270,8 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, state["StreamKMulticast"] = 1 if cs > 1 else 0 state["StreamKClusterReduction"] = 1 if ck > 1 else 0 # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. - # Default -1 reproduces the historic ClusterDim-coupled derivation, so YAML - # that omits Multicast is byte-identical. + # Default -1 reproduces the ClusterDim-coupled derivation, so YAML that omits + # Multicast is unchanged. mc = state.get("Multicast", -1) if mc == 1: # Force-on requires a matching ClusterLoadTDM (TDMInst==3 on gfx1250 with @@ -1964,17 +1962,16 @@ def assignDerivedParameters( # ClusterDim-driven cluster shape (SK3). A genuine 2-D cluster ([Cs,Ck] # with Ck=ClusterDim[1]>1) launches a 2-D grid [skGrid/Ck, Ck, 1] and folds # the Y rank into the index (StreamKIdx = WorkGroup0*Ck + WorkGroup1, see - # StreamK.preLoop), so a Y-extent > 1 no longer collides WorkGroup0. On this - # (reduction) branch only the pure-reduction shape [1, C] is supported; - # factored [Cs,Ck] with BOTH axes > 1 (spatial multicast AND K-split in one - # cluster) lives on the streamk-factored-cluster-mode branch -- reject it - # here. Pure multicast stays [C, 1] (Ck==1, 1-D launch, byte-identical). + # StreamK.preLoop), so a Y-extent > 1 no longer collides WorkGroup0. Only + # the pure-reduction shape [1, C] is supported here; factored [Cs,Ck] with + # BOTH axes > 1 (spatial multicast AND K-split in one cluster) is not + # supported -- reject it here. Pure multicast stays [C, 1] (Ck==1, 1-D + # launch). if state["ClusterDim"][1] > 1 and state["ClusterDim"][0] > 1: reject(state, printRejectionReason, "Factored StreamK cluster [Cs,Ck] with both axes > 1 is not " - "supported on this branch; use ClusterDim=[C,1] (multicast) or " - "[1,C] (reduction), or the streamk-factored-cluster-mode branch " - "(got %s)" % state["ClusterDim"]) + "supported; use ClusterDim=[C,1] (multicast) or [1,C] " + "(reduction) (got %s)" % state["ClusterDim"]) # StreamKXCCMapping remaps WorkGroup0 with no cluster awareness; disable it. state["StreamKXCCMapping"] = 0 if not state["EnableMatrixInstruction"]: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml index 5c262606f067..7bb9a4037fc8 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_multicast.yaml @@ -82,7 +82,7 @@ BenchmarkProblems: - StreamKAtomic: [0] - StreamKXCCMapping: [0] # Pure multicast is derived from ClusterDim = [C, 1] (Cs = ClusterDim[0] > 1); - # StreamKClusterReduction is no longer a user param (derived Ck==1 -> off). + # reduction is derived from the shape (here Ck==1 -> off), not a user param. - ClusterDim: [[2, 1], [4, 1], [8, 1]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml index 263ce5e09567..a0109ded6125 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4gemm_cluster_reduction.yaml @@ -80,7 +80,7 @@ BenchmarkProblems: - StreamKAtomic: [0] - StreamKXCCMapping: [0] # Pure reduction is now derived from the cluster shape ClusterDim = [1, C] - # (Ck = ClusterDim[1] > 1); StreamKClusterReduction is no longer a user param. + # (Ck = ClusterDim[1] > 1); reduction is derived from the shape, not a user param. - ClusterDim: [[1, 2], [1, 4], [1, 8]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml index 7c492df9bf37..e3f10b3c80ef 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_multicast.yaml @@ -76,7 +76,7 @@ BenchmarkProblems: - StreamKAtomic: [0] - StreamKXCCMapping: [0] # Pure multicast is derived from ClusterDim = [C, 1] (Cs = ClusterDim[0] > 1); - # StreamKClusterReduction is no longer a user param (derived Ck==1 -> off). + # reduction is derived from the shape (here Ck==1 -> off), not a user param. - ClusterDim: [[2, 1], [4, 1], [8, 1]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml index f8bb72d0c2a2..56083a08c1ae 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8gemm_cluster_reduction.yaml @@ -74,8 +74,8 @@ BenchmarkProblems: - StreamK: [3] - StreamKAtomic: [0] - StreamKXCCMapping: [0] - # Pure reduction is now derived from the cluster shape ClusterDim = [1, C] - # (Ck = ClusterDim[1] > 1); StreamKClusterReduction is no longer a user param. + # Pure reduction is derived from the cluster shape ClusterDim = [1, C] + # (Ck = ClusterDim[1] > 1); reduction is derived from the shape, not a user param. - ClusterDim: [[1, 2], [1, 4], [1, 8]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml index 49ceb50636eb..c8fc505dd65d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_cluster_reduction.yaml @@ -65,7 +65,7 @@ BenchmarkProblems: - StreamKAtomic: [0] - StreamKXCCMapping: [0] # Pure reduction is now derived from the cluster shape ClusterDim = [1, C] - # (Ck = ClusterDim[1] > 1); StreamKClusterReduction is no longer a user param. + # (Ck = ClusterDim[1] > 1); reduction is derived from the shape, not a user param. - ClusterDim: [[1, 2], [1, 4]] - StaggerU: [0] - DirectToVgprA: [False] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py index c28c516f48d1..c2ab672a867b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py @@ -4,11 +4,10 @@ ################################################################################ # Unit tests for the decoupled tri-state Multicast solution parameter. # -# Multicast used to be a derived-only state var, unconditionally forced on for -# ClusterDim != [1,1] (except Stream-K). It is now an explicit -# tri-state control: -# -1 = auto (legacy coupling), 0 = force off, 1 = force on. -# Default -1 reproduces the historic derivation exactly. These tests pin: +# Multicast is an explicit tri-state control (not a derived-only state var): +# -1 = auto (couples to ClusterDim != [1,1], except Stream-K), +# 0 = force off, 1 = force on. +# Default -1 reproduces the auto derivation exactly. These tests pin: # * registration (valid values + default), and # * the derivation semantics (-1 legacy == old behavior; 0 off; 1 on), # driven through the real config -> Solution derivation path. diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py index 5e91b52dc991..2d0bdee83a17 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py @@ -295,7 +295,7 @@ def test_reject_xcc3(self): def test_reject_non_reduction_shape_cluster(self): # Pure reduction requires [1, C]; a factored [Cs,Ck] with Cs>1 (here - # [2,2]) is not supported on this branch and is rejected. + # [2,2]) is not supported and is rejected. assert self._validate(self._state(ClusterDim=[2, 2])) is False # Pure multicast [C,1] is not a reduction shape either. assert self._validate(self._state(ClusterDim=[4, 1])) is False diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index e5dcca7fd8bb..518b2ad307e5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -129,16 +129,11 @@ def test_accepted_baseline(self, tmp_path): # cluster-scope barrier handshake, so ClusterBarrier is derived on. assert st["ClusterBarrier"] is True, st.get("ClusterBarrier") - # NB: test_auto_enable_from_bare_cluster was removed as a strict subset of - # test_multicast_tristate.py::TestDerivation::test_streamk_cluster_auto_multicast, - # which derives the same bare SK3 + ClusterDim config and asserts the same - # StreamKMulticast==1 / Multicast==1 (plus ClusterBarrier is True). - def test_reduction_shape_keeps_cooperative_loads_off(self, tmp_path): """Param-free derivation: expressing the cluster as ClusterDim = [1, C] (pure reduction) derives StreamKClusterReduction=1 and leaves the spatial cooperative-load multicast off (StreamKMulticast=0, Multicast False), - since Cs = ClusterDim[0] = 1. Reduction is no longer a user param.""" + since Cs = ClusterDim[0] = 1. Reduction is derived from ClusterDim[1] (Ck).""" from Tensile import LibraryIO import yaml cfg = copy.deepcopy(LibraryIO.read(_STREAMK_CLUSTER_BARE)) @@ -164,9 +159,9 @@ def test_reduction_shape_keeps_cooperative_loads_off(self, tmp_path): def test_xor_streamk_cluster_reduction(self): """The mutual-exclusion invariant is enforced at the validator: a state that has BOTH StreamKMulticast (Cs>1) and StreamKClusterReduction (Ck>1) - is the FACTORED cluster [Cs,Ck], which lives on the factored-cluster-mode - branch and is rejected in the SK cluster guard on this branch. The - validator keeps the xor as a hard defensive invariant regardless.""" + is the FACTORED cluster [Cs,Ck], which is not supported and is rejected + in the SK cluster guard. The validator keeps the xor as a hard defensive + invariant regardless.""" from Tensile.SolutionStructs.Solution import _validateStreamKMulticast st = { "StreamKMulticast": 1, @@ -193,12 +188,6 @@ def test_reject_multicast_force_off(self, tmp_path): fork_overrides={"Multicast": [0]}) assert _derive_states(cfg) == [] - # NB: test_control_multicast_auto_enabled was removed as a subset of - # test_accepted_baseline above, which derives the same SK3 [4,1] cluster - # config with the default Multicast=-1 (auto) and already asserts - # StreamKMulticast==1 / Multicast==1 (plus ClusterDim/ClusterBarrier). The - # negative-path partner test_reject_multicast_force_off is retained. - def test_reject_atomic(self, tmp_path): cfg = _write_variant(tmp_path, "atomic.yaml", fork_overrides={"StreamKAtomic": [1]}) From 9dbbbb876fea350d7adcbb72b6b27806cb3ad13b Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 28 Jul 2026 19:08:07 +0000 Subject: [PATCH 20/24] fix(tensilelite): enable ClusterBarrier for any active cluster Decouple the cluster-scope barrier handshake from Multicast: a ClusterBarrier is now derived for ANY active cluster (ClusterDim != [1, 1] with TDM live and HasClusterBarrier), independent of the derived Multicast flag (review request, jichangjichang, applied consistently across the stacked StreamK-cluster PRs). In particular a pure reduction cluster ([1, C], StreamKClusterReduction) derives Multicast=0 yet still needs the cluster barrier around its cooperative loads. This IS a codegen change on this branch: the reduction configs (ClusterDim=[1,2] and [1,4], StreamK=3, gfx1250) now emit the InsertClusterBarrierPass split barrier (s_barrier_signal/wait -3) in the cluster mainloop -> requires gfx1250 HW re-validation. Update test_multicast_tristate::test_explicit_off to the new semantics. --- .../Tensile/SolutionStructs/Solution.py | 19 +++++++++++-------- .../Tests/unit/test_multicast_tristate.py | 8 ++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 8278b104e411..268377f397ef 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1322,14 +1322,17 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, state["Multicast"] = int(state["ClusterDim"] != [1, 1] and state["StreamK"] == 0) # The cluster-scope barrier handshake (s_barrier_signal/wait -3, inserted by - # StinkyTofu's InsertClusterBarrierPass around each multicast tensor_load_to_lds) - # keeps the C cluster peers in lockstep on the multicast loads. It applies to - # both clustered multicast paths: the legacy/subtile clustered multicast - # (StreamK == 0) and the StreamK DP cooperative multicast (StreamKMulticast). - # A forced-off Multicast keeps it off. - if state["ClusterDim"] != [1, 1] and state["Multicast"] \ - and state["TDMInst"] != 0 \ - and (state["StreamK"] == 0 or state.get("StreamKMulticast", 0)) \ + # StinkyTofu's InsertClusterBarrierPass) keeps the cluster peers in lockstep. + # ANY active cluster (ClusterDim != [1, 1]) needs it, independent of Multicast + # or the StreamK mode: every cluster role -- multicast, partial reduction, + # factored and dual-2D -- relies on its co-resident peers staying synchronized, + # not just the B-multicast path. In particular a pure reduction cluster ([1, C], + # StreamKClusterReduction) derives Multicast=0 yet still needs the barrier. + # Still gated on TDM being live (TDMInst != 0 -- an active cluster loads + # cooperatively via tensor_load_to_lds; without TDM there is nothing to + # synchronize) and on the ISA providing the cluster-barrier instruction + # (HasClusterBarrier). + if state["ClusterDim"] != [1, 1] and state["TDMInst"] != 0 \ and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): state["ClusterBarrier"] = True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py index c2ab672a867b..989805898028 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py @@ -92,8 +92,12 @@ def test_explicit_off(self, tmp_path): assert states, "expected >=1 derived solution" assert all(st["Multicast"] == 0 for st in states), ( [st["Multicast"] for st in states]) - # ClusterBarrier is gated on Multicast, so it must also be off. - assert all(st["ClusterBarrier"] is False for st in states) + # ClusterBarrier is decoupled from Multicast: any active cluster + # (ClusterDim != [1, 1] with TDM live) keeps the cluster-scope barrier even + # with the B-multicast forced off -- the [2, 2] cluster peers still load + # cooperatively and must stay in lockstep. + assert all(st["ClusterBarrier"] is True for st in states), ( + [st["ClusterBarrier"] for st in states]) def test_explicit_on(self, tmp_path): # Multicast=1 forces on. From 3acfe3dc01d253f2fa8f105a6e6cf56abda0645f Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 28 Jul 2026 22:31:05 +0000 Subject: [PATCH 21/24] fix(tensilelite): gate ClusterBarrier on Cs>1 (exclude pure-reduction) The decoupled ClusterBarrier enabled the mainloop cluster split-barrier on ANY active cluster, including pure-reduction [1,C] (Cs=1). That path has no cooperative multicast tensor_load_to_lds to bracket, so the InsertClusterBarrier mainloop group emits an unmatched prologue s_barrier_wait -3 (Member N / Signal 0) that never completes -> FFM cluster deadlock (SIGABRT), and would hang real HW. The reduction is already synchronized by its own StreamK reduction -3 barriers. Gate the decouple on Cs = ClusterDim[0] > 1 so the mainloop barrier is emitted only where cooperative multicast loads exist ([C,1], [C,C], factored, dual-2D); pure-reduction [1,C] keeps ClusterBarrier False (pre-decouple, HW-green codegen). Spatial clusters (including Multicast forced off) are unaffected. Validated: FFM reduction [1,2] and [1,4] PASS (no deadlock); full CPU unit + characterization suite green; no golden movement. --- .../Tensile/SolutionStructs/Solution.py | 24 +++++----- .../Tests/unit/test_multicast_tristate.py | 45 +++++++++++++++++-- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 268377f397ef..0e6ca290153b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -1322,17 +1322,19 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, state["Multicast"] = int(state["ClusterDim"] != [1, 1] and state["StreamK"] == 0) # The cluster-scope barrier handshake (s_barrier_signal/wait -3, inserted by - # StinkyTofu's InsertClusterBarrierPass) keeps the cluster peers in lockstep. - # ANY active cluster (ClusterDim != [1, 1]) needs it, independent of Multicast - # or the StreamK mode: every cluster role -- multicast, partial reduction, - # factored and dual-2D -- relies on its co-resident peers staying synchronized, - # not just the B-multicast path. In particular a pure reduction cluster ([1, C], - # StreamKClusterReduction) derives Multicast=0 yet still needs the barrier. - # Still gated on TDM being live (TDMInst != 0 -- an active cluster loads - # cooperatively via tensor_load_to_lds; without TDM there is nothing to - # synchronize) and on the ISA providing the cluster-barrier instruction - # (HasClusterBarrier). - if state["ClusterDim"] != [1, 1] and state["TDMInst"] != 0 \ + # StinkyTofu's InsertClusterBarrierPass) brackets the cooperative multicast + # tensor_load_to_lds groups, keeping the Cs spatial peers in lockstep. It is + # meaningful only when there ARE cooperative multicast loads to bracket, i.e. + # when Cs = ClusterDim[0] > 1 (spatial multicast peers: [C,1], [C,C], factored, + # dual-2D). A pure-reduction cluster ([1, C], Cs=1, StreamKClusterReduction) has + # NO cooperative multicast loads and is synchronized entirely by its own StreamK + # reduction -3 barriers; enabling the mainloop cluster barrier there emits an + # unmatched prologue s_barrier_wait -3 (Member N / Signal 0) that never completes + # -> cluster deadlock. So gate on Cs>1, in addition to an active cluster, TDM + # live (TDMInst != 0 -- cooperative tensor_load_to_lds), and the ISA providing + # the cluster-barrier instruction (HasClusterBarrier). + if state["ClusterDim"] != [1, 1] and state["ClusterDim"][0] > 1 \ + and state["TDMInst"] != 0 \ and isaInfoMap[state["ISA"]].asmCaps.get("HasClusterBarrier", False): state["ClusterBarrier"] = True diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py index 989805898028..bae078c206f8 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py @@ -92,10 +92,12 @@ def test_explicit_off(self, tmp_path): assert states, "expected >=1 derived solution" assert all(st["Multicast"] == 0 for st in states), ( [st["Multicast"] for st in states]) - # ClusterBarrier is decoupled from Multicast: any active cluster - # (ClusterDim != [1, 1] with TDM live) keeps the cluster-scope barrier even - # with the B-multicast forced off -- the [2, 2] cluster peers still load - # cooperatively and must stay in lockstep. + # ClusterBarrier is gated on Cs = ClusterDim[0] > 1 (spatial multicast peers + # with cooperative tensor_load_to_lds to bracket), independent of the + # B-multicast tri-state. The [2, 2] cluster here has Cs=2 > 1, so its peers + # still load cooperatively and keep the cluster-scope barrier even with the + # B-multicast forced off. (A pure-reduction [1, C] cluster has Cs=1 and would + # NOT get the barrier -- see test_pure_reduction_cluster_barrier_off.) assert all(st["ClusterBarrier"] is True for st in states), ( [st["ClusterBarrier"] for st in states]) @@ -124,6 +126,41 @@ def test_streamk_cluster_auto_multicast(self, tmp_path): assert all(st["ClusterBarrier"] is True for st in states), ( [st["ClusterBarrier"] for st in states]) + def test_pure_reduction_cluster_barrier_off(self, tmp_path): + # Pure-reduction cluster: ClusterDim = [1, C] (Cs=1, Ck=C). With Cs=1 there + # are NO cooperative multicast loads to bracket, so the mainloop + # ClusterBarrier is gated OFF (Cs>1 gate) -- the reduction is synchronized + # by its own StreamK reduction -3 barriers. Enabling the mainloop barrier + # here would emit an unmatched prologue s_barrier_wait -3 (Member N / + # Signal 0) that never completes => cluster deadlock (observed in FFM). + from Tensile import LibraryIO + import yaml + cfg = copy.deepcopy(LibraryIO.read(_STREAMK_CLUSTER)) + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + for entry in fork: + if "ClusterDim" in entry: + entry["ClusterDim"] = [[1, 4]] + break + else: + fork.append({"ClusterDim": [[1, 4]]}) + out = tmp_path / "pure_reduction.yaml" + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + states = _derive_states(str(out)) + if not states: + # Branches without the StreamKClusterReduction fast path (e.g. the + # multicast-only branch) reject the pure-reduction [1, C] shape, so + # there is no Cs=1 active cluster to gate -- the Cs>1 gate is a no-op + # there. Nothing to pin on those branches. + pytest.skip("branch does not derive [1, C] pure-reduction solutions") + assert all(st["Multicast"] == 0 for st in states), ( + [st["Multicast"] for st in states]) + assert all(st.get("StreamKClusterReduction", 0) == 1 for st in states), ( + [st.get("StreamKClusterReduction") for st in states]) + # Cs=1 => no cooperative loads => cluster barrier stays OFF. + assert all(st["ClusterBarrier"] is False for st in states), ( + [st["ClusterBarrier"] for st in states]) + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) From 195b31fa02cacef82719fd95283cad980aae715a Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 29 Jul 2026 03:21:46 +0000 Subject: [PATCH 22/24] fix(tensilelite): keep StreamK multicast B mask live across PAP refresh PrefetchAcrossPersistent re-applies the TDM multicast mask on every persistent-loop iteration, but the mask SGPRs were freed in the prologue. Keeping the reuse valid pushed the StreamK [C,1] + StreamKForceDPOnly=0 kernel to 107 SGPRs (> the 106 budget), so KernelWriterAssembly replaced the body with an s_endpgm overflow stub and the output tensor D was left entirely unwritten -- exactly the PAP=1 + cluster + FDPO=0 failure. Keep the [C,1] broadcast mask (MulticastMaskB) live under PAP and free the self-only A mask (ClusterDim[1]==1 -> maskA==1, per-workgroup) whose re-application is a no-op, bringing the kernel back to 104 SGPRs. 2-D clusters (ClusterDim[1]>1, real A multicast) keep both masks live; PAP=0 and non-StreamK-multicast paths are byte-identical. --- .../Tensile/Components/ClusterLoad.py | 38 +++++++++++- .../Tests/unit/test_cluster_load_component.py | 60 ++++++++++++++++++- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index 9264de1b8018..4752d3844932 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -74,17 +74,45 @@ def declareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> None: if tdmM: writer.defineSgpr("MulticastMaskMetadata", 1) + def papRefreshesMask(self, kernel: Mapping) -> bool: + """True when PrefetchAcrossPersistent re-applies the mask after prologue. + + PAP re-emits the TDM descriptor setup (``applyToDescriptor``) on every + persistent-loop iteration, so the StreamK multicast mask SGPR must stay + live past the prologue -- freeing it makes those reuses reference an + undeclared SGPR (``expected absolute expression`` at assembly time). + """ + return bool(kernel.get("PrefetchAcrossPersistent") and kernel.get("StreamKMulticast", 0)) + + def papDropsSelfOnlyMaskA(self, kernel: Mapping) -> bool: + """True when the PAP-live A mask can be freed because it is self-only. + + StreamK keeps A per-workgroup on a 1-D ``[C,1]`` cluster + (``ClusterDim[1]==1`` -> ``maskA==1`` -> ``1< kernel replaced by an + ``s_endpgm`` stub -> output tensor left unwritten). On a 2-D cluster + (``ClusterDim[1]>1``) A is a real multicast and must stay live. + """ + return self.papRefreshesMask(kernel) and kernel["ClusterDim"][1] == 1 + def undeclareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> Module: """Free the ``MulticastMask*`` SGPRs.""" mod = Module() if not (kernel["Multicast"] and kernel["TDMInst"] != 0): return mod tdmM: bool = kernel["enableTDMMetadata"] + refresh: bool = self.papRefreshesMask(kernel) + dropMaskA: bool = self.papDropsSelfOnlyMaskA(kernel) if self.usesCombinedMask(kernel): mod.add(writer.undefineSgpr("MulticastMask")) else: - mod.add(writer.undefineSgpr("MulticastMaskA")) - mod.add(writer.undefineSgpr("MulticastMaskB")) + # Under PAP the A mask stays live unless it is self-only (freed then). + if not refresh or dropMaskA: + mod.add(writer.undefineSgpr("MulticastMaskA")) + # Under PAP the B broadcast mask is re-applied every iteration: keep live. + if not refresh: + mod.add(writer.undefineSgpr("MulticastMaskB")) if tdmM: mod.add(writer.undefineSgpr("MulticastMaskMetadata")) return mod @@ -162,6 +190,12 @@ def applyToDescriptor(self, writer: "KernelWriterAssembly", kernel: Mapping, mod = Module() if kernel["Multicast"] and clusterEnabled(kernel["ClusterDim"]): mask = self.maskSgprName(kernel, tc, subtile=subtile, waveSeparated=waveSeparated) + # Under PAP the self-only A-side mask SGPR is freed (see + # papDropsSelfOnlyMaskA): the StreamK [C,1] A mask carries no multicast + # peers, so re-applying it is a no-op. Skip it so the freed SGPR is not + # referenced across the persistent-loop refresh. + if self.papDropsSelfOnlyMaskA(kernel) and mask == "MulticastMaskA": + return mod tdm = TensorDataMoverLoad.find(writer) mod.add(tdm.setMulticastMask(group1, mask, writer)) return mod diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py index b42fc9b2cefd..df37f6acdde9 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py @@ -62,7 +62,8 @@ def undefineSgpr(self, name): def _kernel(*, multicast=True, clusterDim=(2, 2), tdmA=True, tdmB=True, - numWaves=4, useSubtile=False, sparse=0, tdmMeta=False, tdmInst=3): + numWaves=4, useSubtile=False, sparse=0, tdmMeta=False, tdmInst=3, + pap=False, streamKMulticast=False): return { "Multicast": multicast, "ClusterDim": list(clusterDim), @@ -73,6 +74,8 @@ def _kernel(*, multicast=True, clusterDim=(2, 2), tdmA=True, tdmB=True, "UseSubtileImpl": useSubtile, "TDMInst": tdmInst, "ProblemType": {"Sparse": sparse}, + "PrefetchAcrossPersistent": pap, + "StreamKMulticast": streamKMulticast, } @@ -198,6 +201,34 @@ def test_undeclare_noop_when_multicast_off(self): self._c().undeclareSgprs(w, _kernel(multicast=False)) assert w.undefined == [] + def test_undeclare_keeps_maskB_live_frees_selfonly_maskA_under_pap(self): + # PAP re-applies the [C,1] broadcast mask (MulticastMaskB) on every + # persistent-loop TDM refresh, so it must stay live past the prologue; + # freeing it makes those reuses reference an undeclared SGPR (assembly + # failure). The self-only A mask (ClusterDim[1]==1) is still freed so the + # kernel stays within the 106-SGPR budget (the PAP+cluster+FDPO=0 overflow + # bug: sgprs=107 -> s_endpgm stub -> output unwritten). + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 1))) + assert w.undefined == ["MulticastMaskA"] + + def test_undeclare_keeps_both_live_under_pap_2d_cluster(self): + # On a 2-D cluster (ClusterDim[1]>1) A is a real multicast, not self-only, + # so both masks stay live across the PAP refresh (neither is freed). + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 2))) + assert w.undefined == [] + + def test_undeclare_frees_both_without_pap(self): + # Same StreamK multicast kernel but PAP off: no persistent refresh, so both + # masks are freed in the prologue (byte-identical to the pre-fix behavior). + _init_rocisa_gfx1250() + w = _StubWriter() + self._c().undeclareSgprs(w, _kernel(streamKMulticast=True, pap=False, clusterDim=(2, 1))) + assert w.undefined == ["MulticastMaskA", "MulticastMaskB"] + # --- computeMasks emitted asm ---------------------------------------------- @@ -301,6 +332,33 @@ def test_empty_when_cluster_disabled(self): mod = self._c().applyToDescriptor(w, _kernel(clusterDim=(1, 1)), "tdmAGroup1", "A") assert str(mod).strip() == "" + def test_pap_streamk_skips_selfonly_maskA(self): + # Under PAP+StreamK multicast the A mask SGPR is freed (kept out of the + # SGPR budget); its value is self-only (A stays per-workgroup, no [C,1] + # broadcast) so re-applying it is a no-op. applyToDescriptor must emit + # nothing for the A side rather than reference the freed SGPR. + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor( + w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 1)), "tdmAGroup1", "A") + assert str(mod).strip() == "" + + def test_pap_streamk_still_applies_maskB(self): + # The B broadcast mask is still applied on every refresh (it stays live). + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor( + w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 1)), "tdmBGroup1", "B") + assert "s_or_b32 s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in str(mod) + + def test_no_pap_streamk_still_applies_maskA(self): + # Without PAP the A mask remains live and is applied as before. + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor( + w, _kernel(streamKMulticast=True, pap=False, clusterDim=(2, 1)), "tdmAGroup1", "A") + assert "s_or_b32 s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in str(mod) + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) From 4a7c30bb0581b5b474ec016cd470abb04cab4ff1 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 29 Jul 2026 16:45:57 +0000 Subject: [PATCH 23/24] refactor(tensilelite): derive StreamK B-multicast from ClusterDim, drop StreamKMulticast key The StreamK=3 DP cooperative B-multicast fast path was tracked as a derived-only internal state key (StreamKMulticast) plus a serialized C++ SizeMapping member. Both are redundant with ClusterDim, which is already the source of truth and is serialized to the library. Replace every read with a single derived helper streamKMulticast(kernel) => kernel.get("StreamK",0)==3 and ClusterDim[0]>1 (Common/Utilities.py), and remove the derivation writes in Solution.py. _validateStreamKMulticast and the reduction validator's factored-cluster xor now trigger off the helper. StreamKClusterReduction is deliberately kept exactly as-is (its own derived state key, serialized C++ member, and the standalone reduction grid read). C++: delete SizeMapping::streamKMulticast + its mapOptional (clean removal) and rewrite the two grid guards to the byte-exact equivalent (streamK == 3 && ... clusterDim), leaving the streamKClusterReduction terms intact. Emitted assembly is byte-identical across the gfx1250 StreamK cluster config matrix (multicast + reduction) except the durable comment rename ("StreamKMulticast:" -> "cluster B-multicast:", not snapshotted). The only golden movement is SolutionClass .ambr dropping StreamKMulticast from the state-key list (num_keys 341 -> 340). --- .../tensilelite/Tensile/Common/Utilities.py | 17 ++++++++++ .../Tensile/Common/ValidParameters.py | 27 +++++++-------- .../Tensile/Components/ClusterLoad.py | 8 ++--- .../tensilelite/Tensile/Components/StreamK.py | 28 +++++++-------- .../tensilelite/Tensile/Contractions.py | 2 -- .../tensilelite/Tensile/KernelWriter.py | 6 ++-- .../Tensile/KernelWriterAssembly.py | 4 +-- .../Tensile/SolutionStructs/Solution.py | 30 +++++++--------- .../test_solution_class_char.ambr | 5 ++- .../Tests/unit/test_cluster_load_component.py | 6 +++- .../Tests/unit/test_multicast_tristate.py | 5 +-- .../unit/test_streamk_cluster_reduction.py | 8 +++-- .../Tests/unit/test_streamk_multicast.py | 34 ++++++++++++------- .../include/Tensile/ContractionSolution.hpp | 1 - .../Serialization/ContractionSolution.hpp | 1 - .../tensilelite/src/ContractionSolution.cpp | 4 +-- 16 files changed, 103 insertions(+), 83 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py index 76545d6d75db..d535a0eac25d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py @@ -365,6 +365,23 @@ def clusterEnabled(clusterDim): """True when a workgroup cluster is requested (ClusterDim [x, y] is not [1, 1]).""" return (clusterDim[0] * clusterDim[1]) != 1 +def streamKMulticast(d): + """True when the StreamK=3 DP cooperative B-multicast fast path is active. + + Single source of truth derived from ClusterDim: on StreamK=3 a spatial + cluster (ClusterDim[0] = Cs > 1, i.e. Cs peers sharing B across M-adjacent + DP tiles) IS the cooperative B-multicast path. This replaces the former + derived/serialized ``StreamKMulticast`` state key -- ClusterDim is now the + only source of truth, so there is nothing extra to store or serialize. + StreamKClusterReduction (the Ck axis) is kept as its own derived key. + + ``d`` may be a kernel or a solution ``state`` dict; both expose "StreamK" + and "ClusterDim". Uses ``.get`` for partial-state derivation call sites that + construct a dict without a StreamK / ClusterDim key. + See docs/design/streamk-wg-clusters.md. + """ + return d.get("StreamK", 0) == 3 and d.get("ClusterDim", [1, 1])[0] > 1 + def streamKClusterFactors(d): """Return (Cs, Ck, C, is2D) for a StreamK workgroup cluster (ClusterDim-driven). diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index 92b347f0b72d..96ee4741cb30 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -845,20 +845,19 @@ def makeValidMatrixInstructions(): # 0: use linear reduction # 1: use tree reduction "StreamKFixupTreeReduction": [0, 1], - # NOTE: StreamKMulticast and StreamKClusterReduction (the gfx1250 StreamK DP - # cooperative cluster-load / TDM B-multicast fast path and the cluster split- - # barrier K-reduction fast path) are intentionally NOT valid/benchmark - # parameters. They are DERIVED-ONLY internal state keys (see the ClusterBarrier - # precedent): Solution.assignProblemIndependentDerivedParameters auto-enables - # them for StreamK==3 purely from the cluster shape ClusterDim = [Cs, Ck] - # (StreamKMulticast iff Cs = ClusterDim[0] > 1, StreamKClusterReduction iff - # Ck = ClusterDim[1] > 1) -- so [C,1] = pure multicast, [1,C] = pure reduction. - # _validateStreamKMulticast / _validateStreamKClusterReduction then hard-reject - # any cluster config that cannot satisfy their constraints. They must not be - # user/YAML-settable, so they have no entry here (checkParametersAreValid would - # otherwise accept them as a fork/constant param). They still serialize to C++ - # via Contractions.SizeMapping (streamKMulticast / streamKClusterReduction, read - # with d.get()). See docs/design/streamk-wg-clusters.md. + # NOTE: the gfx1250 StreamK DP cooperative cluster-load / TDM B-multicast fast + # path and the cluster split-barrier K-reduction fast path are intentionally + # NOT valid/benchmark parameters and are auto-derived for StreamK==3 purely + # from the cluster shape ClusterDim = [Cs, Ck] -- [C,1] = pure multicast, + # [1,C] = pure reduction. The B-multicast path is no longer a state key at all: + # it is derived on demand via the streamKMulticast(state) helper (StreamK==3 and + # ClusterDim[0] = Cs > 1). StreamKClusterReduction remains a DERIVED-ONLY + # internal state key (iff Ck = ClusterDim[1] > 1). _validateStreamKMulticast / + # _validateStreamKClusterReduction hard-reject any cluster config that cannot + # satisfy their constraints. Neither is user/YAML-settable (no entry here). + # ClusterDim is the single source of truth for the multicast axis and is what + # serializes to C++ (SizeMapping.clusterDim); only StreamKClusterReduction still + # serializes via Contractions.SizeMapping. See docs/design/streamk-wg-clusters.md. # Debug settings for stream-k kernels to disable parts of the kernel # Bit 0: Don't generate fixup code # Bit 1: Don't generate write to partials code diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index 4752d3844932..42e1cea8743b 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -10,7 +10,7 @@ """ from ..Component import ClusterLoad -from ..Common import clusterEnabled +from ..Common import clusterEnabled, streamKMulticast from typing import Mapping from rocisa.code import Module, Label from rocisa.container import sgpr @@ -36,7 +36,7 @@ def usesCombinedMask(self, kernel: Mapping) -> bool: across the [C,1] cluster while A stays per-workgroup -- the combined parity mask would be wrong for both. """ - if kernel.get("StreamKMulticast", 0): + if streamKMulticast(kernel): return False tdmA: bool = kernel["enableTDMA"] tdmB: bool = kernel["enableTDMB"] @@ -51,7 +51,7 @@ def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, (any ``MXS`` prefix stripped). StreamK forces the split name so B never resolves to the never-declared combined SGPR. """ - if kernel.get("StreamKMulticast", 0): + if streamKMulticast(kernel): string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc return f"MulticastMask{string}" if waveSeparated and not subtile: @@ -82,7 +82,7 @@ def papRefreshesMask(self, kernel: Mapping) -> bool: live past the prologue -- freeing it makes those reuses reference an undeclared SGPR (``expected absolute expression`` at assembly time). """ - return bool(kernel.get("PrefetchAcrossPersistent") and kernel.get("StreamKMulticast", 0)) + return bool(kernel.get("PrefetchAcrossPersistent") and streamKMulticast(kernel)) def papDropsSelfOnlyMaskA(self, kernel: Mapping) -> bool: """True when the PAP-live A mask can be freed because it is self-only. diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index 016c35e6fdb5..ba560d522fdb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -36,7 +36,7 @@ from .Subtile.SubtileLREmit import localReadResetOffsetsSubtile -from ..Common import print2, ceilDivide, log2, clusterEnabled, streamKClusterFactors +from ..Common import print2, ceilDivide, log2, clusterEnabled, streamKMulticast, streamKClusterFactors from ..Component import Component from ..AsmStoreState import StoreState, VectorDataTypes from ..AsmAddressCalculation import AddrCalculation @@ -2915,10 +2915,10 @@ def streamKMulticastMaskPredicate(self, writer, kernel): semantics could not be verified on silicon). Inert unless StreamKMulticast. """ module = Module("StreamK multicast mask predicate") - if not kernel.get("StreamKMulticast", 0): + if not streamKMulticast(kernel): return module c = kernel["ClusterDim"][0] - module.addComment0("StreamKMulticast: gate B-broadcast on clusterMulticastValid") + module.addComment0("cluster B-multicast: gate B-broadcast on clusterMulticastValid") skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) mcInvalid = Label(writer.labels.getNameInc("SKMC_Invalid"), "") mcEnd = Label(writer.labels.getNameInc("SKMC_End"), "") @@ -2971,9 +2971,9 @@ def streamKMulticastBoundaryClear(self, writer, kernel): Inert unless StreamKMulticast. """ module = Module("StreamK multicast DP->SK boundary clear") - if not kernel.get("StreamKMulticast", 0): + if not streamKMulticast(kernel): return module - module.addComment0("StreamKMulticast: clear B-broadcast mask at DP->SK boundary") + module.addComment0("cluster B-multicast: clear B-broadcast mask at DP->SK boundary") module.add(SMovB32(dst=sgpr("MulticastMaskB"), src=sgpr("MulticastMaskA"), comment="DP->SK: drop B broadcast -> self-only (normal B load)")) return module @@ -2989,11 +2989,11 @@ def streamKMulticastPrologueSignal(self, writer, kernel): StreamKMulticast. See docs/design/cluster-load-component-and-streamk-multicast.md. """ module = Module("StreamK multicast prologue signal") - if not kernel.get("StreamKMulticast", 0): + if not streamKMulticast(kernel): return module assert writer.states.asmCaps.get("HasClusterBarrier", False), \ - "StreamKMulticast requires the HasClusterBarrier asm capability" - module.addComment0("StreamKMulticast: elect wave 0 to signal the cluster barrier (pairs first-load wait)") + "cluster B-multicast requires the HasClusterBarrier asm capability" + module.addComment0("cluster B-multicast: elect wave 0 to signal the cluster barrier (pairs first-load wait)") self._clusterElectArriveSignal( writer, module, labelBase="SKMC_SkipSignal", electTag="SKMulticastElect") return module @@ -3010,11 +3010,11 @@ def streamKMulticastProloguePrefetchHandshake(self, writer, kernel): See docs/design/cluster-load-component-and-streamk-multicast.md. """ module = Module("StreamK multicast prologue prefetch cluster handshake") - if not kernel.get("StreamKMulticast", 0): + if not streamKMulticast(kernel): return module assert writer.states.asmCaps.get("HasClusterBarrier", False), \ - "StreamKMulticast requires the HasClusterBarrier asm capability" - module.addComment0("StreamKMulticast: bracket prologue double-buffer prefetch load with cluster handshake") + "cluster B-multicast requires the HasClusterBarrier asm capability" + module.addComment0("cluster B-multicast: bracket prologue double-buffer prefetch load with cluster handshake") self._clusterElectArriveSignal( writer, module, labelBase="SKMC_SkipPrefetchSignal", electTag="SKMulticastPrefetchElect", wait=True) return module @@ -3032,11 +3032,11 @@ def streamKMulticastZeroIterClusterWait(self, writer, kernel): See docs/design/cluster-load-component-and-streamk-multicast.md. """ module = Module("StreamK multicast zero-iteration cluster wait") - if not kernel.get("StreamKMulticast", 0): + if not streamKMulticast(kernel): return module assert writer.states.asmCaps.get("HasClusterBarrier", False), \ - "StreamKMulticast requires the HasClusterBarrier asm capability" - module.addComment0("StreamKMulticast: zero-iteration skip path consumes the prologue cluster arrive (pairs prologue arrive)") + "cluster B-multicast requires the HasClusterBarrier asm capability" + module.addComment0("cluster B-multicast: zero-iteration skip path consumes the prologue cluster arrive (pairs prologue arrive)") skipWait = Label(label=writer.labels.getNameInc("SKMC_SkipZeroIterClusterWait"), comment="") module.add(SCBranchSCC0(labelName=skipWait.getLabelName(), comment=">=1 full iteration: the first-load cluster wait pairs the arrive")) diff --git a/projects/hipblaslt/tensilelite/Tensile/Contractions.py b/projects/hipblaslt/tensilelite/Tensile/Contractions.py index 8d2773e5a71d..f7648edc0cce 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Contractions.py +++ b/projects/hipblaslt/tensilelite/Tensile/Contractions.py @@ -667,7 +667,6 @@ class SizeMapping: 'streamKForceDPOnly', 'streamKAtomic', 'streamKClusterReduction', - 'streamKMulticast', 'prefetchAcrossPersistent', 'sourceKernel', 'globalAccumulation', @@ -763,7 +762,6 @@ def convertSFCWGMListToHex(value): streamKForceDPOnly = d.get('StreamKForceDPOnly', 0), streamKAtomic = d['StreamKAtomic'] if 'StreamKAtomic' in d else 0, streamKClusterReduction = d.get('StreamKClusterReduction', 0), - streamKMulticast = d.get('StreamKMulticast', 0), prefetchAcrossPersistent = d.get('PrefetchAcrossPersistent', 0), magicDivAlg = d.get('MagicDivAlg', 1), sourceKernel = d['KernelLanguage'] == 'Source', diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index 714ad32c581b..d21f1596c030 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -56,7 +56,7 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule from .Common import printWarning, roundUp, print2, DebugConfig, DataDirection, \ - INDEX_CHARS, IsaVersion, log2 + INDEX_CHARS, IsaVersion, log2, streamKMulticast from .Common.GlobalParameters import globalParameters from Tensile.SolutionStructs.Naming import getKernelNameMin from Tensile.Toolchain.Component import Assembler @@ -5486,7 +5486,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): # multicast load stays synchronized and signal/wait counts stay # balanced. Gated on StreamKMulticast (only ever set on the StreamK=3 # component), so the emitted code is unchanged for every other path. - if kernel.get("StreamKMulticast", 0): + if streamKMulticast(kernel): module.add(skComponent.streamKMulticastProloguePrefetchHandshake(self, kernel)) # For UnrollLoopSwapGlobalReadOrder, we also need to swap ds write A/B order. # In scheduling, we always schedule lwa first then lwb second, @@ -6700,7 +6700,7 @@ def _restoreNtabState(): # PGR>=2 an `s_wait_tensorcnt 0` is emitted before the cluster-scope # `s_barrier_signal -3` arrive so the broadcast retires before peers # re-enter the next round. Defaults off; no-op for every other kernel. - "StreamKMulticast": bool(kernel.get("StreamKMulticast", 0)), + "StreamKMulticast": bool(streamKMulticast(kernel)), # PrefetchGlobalRead (PGR) passed to InsertClusterBarrierPass. # Gates Rule 3 (`LCL <= PGR` skip) and Rule 4 (`LCL == PGR+1` # skip in fresh-gate mode; inherits upstream `LCL == PGR` cmp diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py index 538736260cbc..833e6d0d9a97 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py @@ -83,7 +83,7 @@ from .AsmStoreState import StoreState, VectorDataTypes from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from .Common import roundUp, log2, ceilDivide, choose_multiplier, wmmaV3InputVgprLayout, clusterEnabled +from .Common import roundUp, log2, ceilDivide, choose_multiplier, wmmaV3InputVgprLayout, clusterEnabled, streamKMulticast from .OccupancyMeasure import compute_occupancy_from_asm_source, _arch_caps_for_kernel from rocisa.instruction import ECvtF16toF32, ECvtF32toF16, ECvtPkFP8toF32 from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, isSubtileMultiDU @@ -9713,7 +9713,7 @@ def openSumAtLeastUnroll(self, kernel, prefetch, isOptNLL, isNGLL=False, NLLinde # is consumed on every control-flow path (whole-cluster barrier # symmetry). scc (from checkLastIter) is preserved for the branch # below. No-op unless StreamKMulticast. - if kernel.get("StreamKMulticast", 0): + if streamKMulticast(kernel): skComponent = Component.StreamK.find(self) module.add(skComponent.streamKMulticastZeroIterClusterWait(self, kernel)) # use positive offset only long jump diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index 0e6ca290153b..a206e465b3ba 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -36,7 +36,7 @@ from Tensile.Common import assignParameterWithDefault, IsaInfo, \ print2, printExit, printWarning, \ roundUp, INDEX_CHARS, IsaVersion, SemanticVersion, \ - roundUpToNearestMultiple, effectiveMatrixInstMN + roundUpToNearestMultiple, effectiveMatrixInstMN, streamKMulticast from Tensile.Common.DataType import DataType from Tensile.Common.TypeValidationErrors import ConfigTypeError from Tensile.SolutionStructs.LdsPadding import get_fp4_mt_config, get_fp8_mt_config, get_mxs_mt_config, \ @@ -250,7 +250,7 @@ def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): # Pure reduction ([1, C]) has Cs = ClusterDim[0] = 1, so StreamKMulticast is # off. Factored ([Cs,Ck] with BOTH axes > 1) would turn both on, which is # rejected in the SK cluster guard. Defensive: reject the combination here too. - if state.get("StreamKMulticast", 0): + if streamKMulticast(state): reject(state, printRejectionReason, "Factored StreamK cluster [Cs,Ck] with both axes > 1 is not " "supported; use ClusterDim=[C,1] (multicast) or [1,C] (reduction)") @@ -331,7 +331,7 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): cluster rather than an explicit opt-in. See docs/design/cluster-load-component-and-streamk-multicast.md. """ - if not state.get("StreamKMulticast", 0): + if not streamKMulticast(state): return True # Multicast (Cs>1) and reduction (Ck>1) both on is the FACTORED cluster @@ -1256,16 +1256,13 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, reject(state, printRejectionReason, "UseSubtileImpl=1 PrefetchAcrossPersistent not supported with DirectToVgpr MX scale tensors") state["ClusterBarrier"] = False - # StreamKMulticast is a DERIVED-ONLY internal state key (not a valid/benchmark - # parameter -- see ValidParameters.py). Seed it off here (mirroring the - # ClusterBarrier default above) so it is always present on state; the collapse - # below is the ONLY place it is turned on. - state["StreamKMulticast"] = 0 - # StreamKClusterReduction is a DERIVED-ONLY internal state key (like - # StreamKMulticast): reduction is not a user opt-in -- it is expressed - # purely by the cluster shape ClusterDim = [1, C] (Ck = ClusterDim[1] > 1). - # Seed it off here (mirroring StreamKMulticast); the collapse below is its - # sole enable site. + # The StreamK=3 DP cooperative B-multicast fast path is no longer stored as a + # separate derived state key: it is derived on demand from ClusterDim via the + # streamKMulticast(state) helper (StreamK==3 and ClusterDim[0] = Cs > 1). + # StreamKClusterReduction is a DERIVED-ONLY internal state key: reduction is + # not a user opt-in -- it is expressed purely by the cluster shape + # ClusterDim = [1, C] (Ck = ClusterDim[1] > 1). Seed it off here (mirroring the + # ClusterBarrier default above); the collapse below is its sole enable site. state["StreamKClusterReduction"] = 0 # ClusterDim-driven StreamK cluster derivation (fully param-free). On StreamK=3 # a non-[1,1] ClusterDim = [Cs, Ck] AUTO-ENABLES the cooperative cluster path; @@ -1286,9 +1283,7 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, # non-clustered state, which some partial-state derivation call sites construct # without a StreamK key. See docs/design/streamk-wg-clusters.md. if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3: - cs = state["ClusterDim"][0] ck = state["ClusterDim"][1] - state["StreamKMulticast"] = 1 if cs > 1 else 0 state["StreamKClusterReduction"] = 1 if ck > 1 else 0 # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. # Default -1 reproduces the ClusterDim-coupled derivation, so YAML that omits @@ -1307,8 +1302,8 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, state["Multicast"] = 1 elif mc == 0: state["Multicast"] = 0 - elif state.get("StreamKMulticast", 0): - # StreamKMulticast (auto-derived above from StreamK=3 + ClusterDim) drives + elif streamKMulticast(state): + # StreamKMulticast (auto-derived from StreamK=3 + ClusterDim) drives # TDM B-multicast through the ClusterLoad component (its [C,1] cluster is # spatial DP peers, not the legacy subtile coupling). The C co-resident # peers must stay in lockstep around each multicast tensor_load_to_lds, so @@ -2130,7 +2125,6 @@ def assignDerivedParameters( state["StreamKXCCMapping"] = 0 state["StreamKFixupTreeReduction"] = 0 state["StreamKClusterReduction"] = 0 - state["StreamKMulticast"] = 0 state["DebugStreamK"] = 0 state["PrefetchAcrossPersistent"] = 0 state["DebugPersistentKernelLoopForever"] = False diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr index b5a170090c7f..07f5e745dbc4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/SolutionClass/__snapshots__/test_solution_class_char.ambr @@ -55,7 +55,7 @@ 'getitem_kernel_language': 'Assembly', 'iter_matches_keys': True, 'keys_is_list': True, - 'len': 341, + 'len': 340, }) # --- # name: test_solution_construction @@ -297,7 +297,6 @@ 'StreamKClusterReduction', 'StreamKFixupTreeReduction', 'StreamKForceDPOnly', - 'StreamKMulticast', 'StreamKWorkStealing', 'StreamKXCCMapping', 'SubGroup0', @@ -404,7 +403,7 @@ 'tailLoopOptB', ]), 'name': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x32_MI16x16x1_SN_LDSB0_AA0_AFC1_AF1_AG0_AGGSUA0_AGNTAB0_AAIGTEn1_AAILTEn1_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DPKLF0_DSK0_DU32_DTL0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA1_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_IA0_KLA_LDSSI0_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA4_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW4_LRVWA4_LRVWB4_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MDA2_MI16_16_16_1_MLDS65536_MO40_MPM0_MGRIPM1_NR0_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS2_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA1_SFCWGM1_1_1_1_SGROB0_SGR1_SIA3_SLW1_SS1_SU32_SUM1_SUS256_SPO1_SRVW0_SSO0_SVW1_SK0_SKA0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TT1_1_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UCMLS0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WSK0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1_WGR0', - 'num_keys': 341, + 'num_keys': 340, 'problem_type': 'Cijk_Alik_Bjlk_HSS_BH_Bias_HA_S_SAV_UserArgs', 'stable': dict({ 'DepthU': 32, diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py index df37f6acdde9..43c2e6531419 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py @@ -75,7 +75,11 @@ def _kernel(*, multicast=True, clusterDim=(2, 2), tdmA=True, tdmB=True, "TDMInst": tdmInst, "ProblemType": {"Sparse": sparse}, "PrefetchAcrossPersistent": pap, - "StreamKMulticast": streamKMulticast, + # The StreamK=3 DP cooperative B-multicast path is now derived from + # StreamK==3 + ClusterDim[0] > 1 (streamKMulticast helper), not a stored + # state key. Drive it here by setting StreamK (the streamKMulticast=True + # cases all use a ClusterDim with Cs = clusterDim[0] > 1). + "StreamK": 3 if streamKMulticast else 0, } diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py index bae078c206f8..82e494bd6534 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_multicast_tristate.py @@ -117,8 +117,9 @@ def test_streamk_cluster_auto_multicast(self, tmp_path): cfg = _write_variant(tmp_path, _STREAMK_CLUSTER, "sk_auto_mc.yaml") states = _derive_states(cfg) assert states, "expected >=1 derived solution" - assert all(st["StreamKMulticast"] == 1 for st in states), ( - [st.get("StreamKMulticast") for st in states]) + from Tensile.Common import streamKMulticast + assert all(streamKMulticast(st) for st in states), ( + [(st.get("StreamK"), st.get("ClusterDim")) for st in states]) assert all(st["Multicast"] == 1 for st in states), ( [st["Multicast"] for st in states]) # The cooperative multicast now pairs the masks with the cluster-scope diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py index 2d0bdee83a17..4fb57906bc7f 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_cluster_reduction.py @@ -254,7 +254,7 @@ class TestReductionValidation: @staticmethod def _state(**overrides): st = { - "StreamKClusterReduction": 1, "StreamKMulticast": 0, "StreamK": 3, + "StreamKClusterReduction": 1, "StreamK": 3, "StreamKAtomic": 0, "StreamKForceDPOnly": 0, "StreamKXCCMapping": 0, "ClusterDim": [1, 4], "ISA": [12, 5, 0], "TDMInst": 3, } @@ -278,8 +278,10 @@ def test_noop_when_param_off(self): assert self._validate(self._state(StreamKClusterReduction=0)) is True def test_reject_multicast_combo(self): - # #9611 keeps multicast and cluster reduction mutually exclusive. - assert self._validate(self._state(StreamKMulticast=1)) is False + # #9611 keeps multicast and cluster reduction mutually exclusive. The + # multicast role is derived from ClusterDim[0] (Cs), so a factored + # [Cs,Ck] with both axes > 1 turns both on -> the combo reject fires. + assert self._validate(self._state(ClusterDim=[2, 4])) is False def test_reject_streamk_not_3(self): assert self._validate(self._state(StreamK=4)) is False diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py index 518b2ad307e5..f0bd6c4b0418 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_multicast.py @@ -38,6 +38,10 @@ sys.path.insert(0, os.path.join( TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen")) +# The StreamK=3 DP cooperative B-multicast path is no longer a stored state key; +# it is derived from ClusterDim (StreamK==3 and ClusterDim[0] > 1) via this helper. +from Tensile.Common import streamKMulticast + _DESIGNED = os.path.join( TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen", "data", "test_data", "_designed", "gfx1250") @@ -122,7 +126,7 @@ def test_accepted_baseline(self, tmp_path): states = _derive_states(cfg) assert states, "expected >=1 derived solution for the valid config" for st in states: - assert st["StreamKMulticast"] == 1 + assert streamKMulticast(st) assert st["Multicast"] == 1, st["Multicast"] assert st["ClusterDim"] == [4, 1] # The cooperative multicast pairs the B-broadcast masks with the @@ -152,7 +156,7 @@ def test_reduction_shape_keeps_cooperative_loads_off(self, tmp_path): states = _derive_states(str(out)) assert states, "expected the SK3 [1,C] reduction cluster config to derive solutions" for st in states: - assert not st.get("StreamKMulticast", 0), st.get("StreamKMulticast") + assert not streamKMulticast(st), (st.get("StreamK"), st.get("ClusterDim")) assert st.get("StreamKClusterReduction", 0) == 1, st.get("StreamKClusterReduction") assert st["Multicast"] == 0, st["Multicast"] @@ -164,8 +168,8 @@ def test_xor_streamk_cluster_reduction(self): invariant regardless.""" from Tensile.SolutionStructs.Solution import _validateStreamKMulticast st = { - "StreamKMulticast": 1, - # StreamKMulticast on always co-derives Multicast on (real invariant). + # ClusterDim=[4,1] -> streamKMulticast helper True (Cs>1); the forced + # StreamKClusterReduction=1 makes this the (unsupported) factored state. "Multicast": 1, "StreamK": 3, "StreamKClusterReduction": 1, @@ -206,7 +210,6 @@ class _Info: def _state(pgr): return { - "StreamKMulticast": 1, "Multicast": 1, "StreamK": 3, "StreamKAtomic": 0, @@ -225,7 +228,7 @@ def _state(pgr): states = _derive_states(cfg) assert states, "expected the PrefetchGlobalRead=2 multicast config to be accepted" for st in states: - assert st["StreamKMulticast"] == 1 + assert streamKMulticast(st) def test_xcc_mapping_forced_to_zero(self, tmp_path): """StreamKXCCMapping is coerced to 0 (not rejected) under StreamK+ClusterDim. @@ -241,7 +244,7 @@ def test_xcc_mapping_forced_to_zero(self, tmp_path): states = _derive_states(cfg) assert states, "expected the XCC=3 config to be accepted with XCC coerced to 0" for st in states: - assert st["StreamKMulticast"] == 1 + assert streamKMulticast(st) assert st["StreamKXCCMapping"] == 0, st["StreamKXCCMapping"] def test_reject_non_1d_cluster(self, tmp_path): @@ -263,7 +266,7 @@ def test_reject_non_pow2_cluster(self, tmp_path): @staticmethod def _direct_state(**overrides): st = { - "StreamKMulticast": 1, "Multicast": 1, "StreamK": 3, + "Multicast": 1, "StreamK": 3, "StreamKAtomic": 0, "StreamKXCCMapping": 0, "ClusterDim": [4, 1], "ISA": [12, 5, 0], "TDMInst": 3, "PrefetchGlobalRead": 1, } @@ -276,10 +279,16 @@ class _Info: asmCaps = {"HasTDM": has_tdm, "HasClusterBarrier": has_cluster_barrier} return {(12, 5, 0): _Info()} - def test_reject_streamk_not_3_direct(self): + def test_streamk_not_3_is_not_multicast_path(self): + # The multicast fast path is now DERIVED (StreamK==3 and ClusterDim[0] > 1), + # so a non-SK3 state is simply not the multicast path: the helper is False + # and _validateStreamKMulticast is a no-op (returns True) there. SK4/SK5 + + # ClusterDim is rejected by the general Stream-K reconciliation (cluster + # support is SK3-only), not by this validator. from Tensile.SolutionStructs.Solution import _validateStreamKMulticast - assert _validateStreamKMulticast( - self._direct_state(StreamK=4), False, self._isa_map()) is False + st = self._direct_state(StreamK=4) + assert streamKMulticast(st) is False + assert _validateStreamKMulticast(st, False, self._isa_map()) is True def test_reject_xcc_mapping_direct(self): from Tensile.SolutionStructs.Solution import _validateStreamKMulticast @@ -310,8 +319,7 @@ class TestTDMInstValidation: @staticmethod def _state(tdminst, pgr=1): return { - "StreamKMulticast": 1, - # StreamKMulticast on always co-derives Multicast on (real invariant). + # The derived multicast path co-derives Multicast on (real invariant). "Multicast": 1, "StreamK": 3, "StreamKAtomic": 0, diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 47b012f57dbe..810cb9c87203 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -144,7 +144,6 @@ namespace TensileLite int streamKForceDPOnly = 0; int streamKAtomic = 0; int streamKClusterReduction = 0; - int streamKMulticast = 0; int prefetchAcrossPersistent = 0; int persistentKernel = 0; bool persistentKernelAlongBatch = false; diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp index f4bfe0bc3dd4..2f3b71984fba 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp @@ -107,7 +107,6 @@ namespace TensileLite iot::mapOptional(io, "streamKForceDPOnly", s.streamKForceDPOnly); iot::mapOptional(io, "streamKAtomic", s.streamKAtomic); iot::mapOptional(io, "streamKClusterReduction", s.streamKClusterReduction); - iot::mapOptional(io, "streamKMulticast", s.streamKMulticast); iot::mapOptional(io, "prefetchAcrossPersistent", s.prefetchAcrossPersistent); iot::mapOptional(io, "persistentKernel", s.persistentKernel); iot::mapOptional(io, "persistentKernelAlongBatch", s.persistentKernelAlongBatch); diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 634b01c09319..a1f0e4b4b34f 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -3454,7 +3454,7 @@ namespace TensileLite // sk.grid = tiles), per-tile correctness on any partially-filled cluster // is handled by the kernel's runtime guard (clusterMulticastValid for // multicast, intra_cluster for cluster reduction) + global-flag fallback. - if((sizeMapping.streamKMulticast || sizeMapping.streamKClusterReduction) + if(sizeMapping.streamK == 3 && (static_cast(sizeMapping.clusterDim.x) * static_cast(sizeMapping.clusterDim.y)) > 1) @@ -4395,7 +4395,7 @@ namespace TensileLite // has idle-but-present tail WGs whose cluster is disabled by the kernel's // clusterMulticastValid runtime predicate (so a masked target is never left // without a matching load). See docs/design/cluster-load-component-and-streamk-multicast.md. - if(self.sizeMapping.streamKMulticast && self.sizeMapping.clusterDim.x > 1) + if(self.sizeMapping.streamK == 3 && self.sizeMapping.clusterDim.x > 1) { size_t c = self.sizeMapping.clusterDim.x; skGrid = ((tiles + c - 1) / c) * c; From 382f940ca4115b421a2fa499ff1281aeb0b99fec Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Wed, 29 Jul 2026 18:13:44 +0000 Subject: [PATCH 24/24] feat(streamk): add 2-D dual-operand multicast on top of cluster-reduction Port the full 2-D dual-multicast feature (both cluster axes broadcast A+B) onto the cluster-reduction branch so it stays a cumulative SUPERSET of the cluster-multicast branch (= 1-D [C,1] + 2-D dual-multicast + 1-D [1,C] reduction). Cherry-picked from the cluster-multicast port with the reduction machinery kept intact; the two 2-D shapes are made mutually exclusive: - Common: streamKForceDP2DMulticast / streamKDual2DMulticast detectors + StreamKDualMulticast knob (kept alongside streamKClusterFactors and the reduction derivation). - Solution derivation: StreamKClusterReduction is now derived only for a K-split cluster (Ck>1 AND NOT dual-2D); a 2-D dual cluster keeps reduction OFF so _validateStreamKMulticast accepts the [Cs,Ck] shape instead of rejecting it as factored. - SK cluster guard: [1,C] reduction and dual-2D ([Cs,Ck]+dual flag) both allowed; factored (both>1, no flag) still rejected. - StreamK.preLoop: dual-2D uses the M-fastest DP fold; the K-split reduction [1,C] keeps the WorkGroup0*Ck + WorkGroup1 fold (if/elif, mutually exclusive). - Contractions ClusterDimCheck: dual-2D keeps Ck as the N-tile divisor (value[4]=Ck); K-split reduction still pins value[4]=1. - C++ (getSKGridImpl / generateSingleCall): dual grid geometry (ForceDPOnly-2D tiles, standard-dual reshape) added ahead of the reduction grid override; streamKDualMulticast serialized field. - Configs/tests: dual-2D configs (FDPO=0/1), the 2-D+PAP home, designed char configs + _codegen char tests, and unit coverage. Reduction and 1-D multicast goldens unchanged (additive-only); the dual-2D nodes are new. No behavior change on the [1,C]/[C,1] paths. --- .../Tensile/Common/GlobalParameters.py | 1 + .../tensilelite/Tensile/Common/Utilities.py | 38 +++ .../Tensile/Common/ValidParameters.py | 9 + .../Tensile/Components/ClusterLoad.py | 5 + .../tensilelite/Tensile/Components/StreamK.py | 108 ++++++++- .../tensilelite/Tensile/Contractions.py | 20 +- .../Tensile/SolutionStructs/Solution.py | 93 +++++-- .../core/sk_mxf4_2d_dual_multicast.yaml | 152 ++++++++++++ ..._mxf4_force_dp_only_cluster_multicast.yaml | 10 +- .../core/sk_mxf4_force_dp_only_pap.yaml | 111 +++++++++ .../core/sk_mxf8_2d_dual_multicast.yaml | 150 ++++++++++++ .../gfx1250/core/sk_mxf8_force_dp_only.yaml | 94 ++++++++ ..._mxf8_force_dp_only_cluster_multicast.yaml | 10 +- .../core/sk_mxf8_force_dp_only_pap.yaml | 109 +++++++++ ...t_streamk_forcedp_2d_pap_gfx1250_char.ambr | 9 + .../_codegen/config_harness.py | 21 ++ .../gfx1250/streamk_dual_2d_multicast.yaml | 105 ++++++++ .../_designed/gfx1250/streamk_forcedp_2d.yaml | 99 ++++++++ .../gfx1250/streamk_forcedp_2d_pap.yaml | 92 +++++++ ..._streamk_cluster_coop_load_gfx1250_char.py | 20 +- ..._streamk_cluster_multicast_gfx1250_char.py | 20 +- ...amk_cluster_multicast_pgr2_gfx1250_char.py | 20 +- ..._streamk_dual_2d_multicast_gfx1250_char.py | 150 ++++++++++++ .../test_streamk_forcedp_2d_gfx1250_char.py | 117 +++++++++ ...est_streamk_forcedp_2d_pap_gfx1250_char.py | 105 ++++++++ .../Tests/unit/test_cluster_load_component.py | 48 +++- .../unit/test_streamk_dual_2d_multicast.py | 228 ++++++++++++++++++ .../include/Tensile/ContractionSolution.hpp | 4 + .../Serialization/ContractionSolution.hpp | 1 + .../tensilelite/src/ContractionSolution.cpp | 132 ++++++++-- 30 files changed, 1959 insertions(+), 122 deletions(-) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_2d_dual_multicast.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_pap.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_2d_dual_multicast.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_pap.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_forcedp_2d_pap_gfx1250_char.ambr create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_dual_2d_multicast.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d_pap.yaml create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_dual_2d_multicast_gfx1250_char.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_gfx1250_char.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_pap_gfx1250_char.py create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_dual_2d_multicast.py diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py index ebb48932eaee..a04da4ba5294 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/GlobalParameters.py @@ -571,6 +571,7 @@ {"MIArchVgpr": [False]}, {"StreamK": [0]}, {"StreamKForceDPOnly": [0]}, + {"StreamKDualMulticast": [0]}, {"StreamKAtomic": [0]}, {"StreamKWorkStealing": [0]}, {"StreamKXCCMapping": [0]}, diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py index d535a0eac25d..1c2843a9de01 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/Utilities.py @@ -404,6 +404,44 @@ def streamKClusterFactors(d): cs, ck = cd[0], cd[1] return cs, ck, cs * ck, (ck > 1) +def streamKForceDP2DMulticast(d): + """True for the ForceDPOnly 2-D dual-operand multicast cluster. + + This is a StreamK==3 ``StreamKForceDPOnly`` (dense data-parallel, no K-split + reduction) kernel given a GENUINE 2-D cluster ClusterDim = [Cs, Ck] with BOTH + axes > 1. Unlike the factored K-split cluster (where Ck is a reduction axis), + here the Ck (Y) axis maps to N-ADJACENT output tiles so the Y-peers reuse the + A operand (A-multicast), while the Cs (X) peers reuse B on M-adjacent tiles + exactly as in the shipped 1-D [C,1] ForceDPOnly multicast. Both operands are + multicast via the DENSE ClusterLoad 2-D masks. + + Detected purely structurally (ForceDPOnly + ClusterDim[0]>1 + ClusterDim[1]>1). + ``d`` may be a kernel or a solution ``state`` dict; both expose + "StreamKForceDPOnly" and "ClusterDim". + See docs/design/streamk-wg-clusters.md. + """ + return bool(d.get("StreamKForceDPOnly", 0)) \ + and d["ClusterDim"][0] > 1 and d["ClusterDim"][1] > 1 + +def streamKDual2DMulticast(d): + """True for a 2-D DUAL-operand multicast cluster (generalized detector). + + Covers BOTH the ForceDPOnly 2-D dual-operand multicast (``StreamKForceDPOnly`` + + 2-D cluster) and the STANDARD two-tile StreamK path + (``StreamKForceDPOnly == 0``) opted in via ``StreamKDualMulticast``. In both + the DP (full-tile) round does the 2-D dual multicast (Cs/X peers share B on + M-adjacent tiles, Ck/Y peers share A on N-adjacent tiles); the standard path's + SK (partial-tile) round reduces 1-D via the workspace as today. + + Both cases require a GENUINE 2-D cluster ClusterDim = [Cs, Ck] (both axes > 1) + where Ck (Y) is an N-tiling / A-multicast axis, NOT a K-split reduction axis. + ``d`` may be a kernel or a solution ``state`` dict. + See docs/design/streamk-wg-clusters.md. + """ + if not (d["ClusterDim"][0] > 1 and d["ClusterDim"][1] > 1): + return False + return bool(d.get("StreamKForceDPOnly", 0)) or bool(d.get("StreamKDualMulticast", 0)) + def log2(x): return int(log(x, 2) + 0.5) diff --git a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py index 96ee4741cb30..c26122a0da8d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py +++ b/projects/hipblaslt/tensilelite/Tensile/Common/ValidParameters.py @@ -828,6 +828,15 @@ def makeValidMatrixInstructions(): # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step # 1: uses atomics to accumulate partial tiles "StreamKAtomic": [0, 1], + # Opt-in for 2-D DUAL-operand multicast on the STANDARD two-tile StreamK + # path (StreamKForceDPOnly=0). On a genuine 2-D cluster ClusterDim=[Cs,Ck] + # (both > 1) the DP (full-tile) round does 2-D dual multicast (Cs/X peers + # share B on M-adjacent tiles, Ck/Y peers share A on N-adjacent tiles) and + # the SK (partial-tile) round reduces 1-D via the workspace as today. Ck is + # an N-tiling / A-multicast axis, NOT a K-split reduction axis. ForceDPOnly-2D + # does not need it (that shape is unambiguous). Limited to batch==1. See + # streamKDual2DMulticast and docs/design/streamk-wg-clusters.md. + "StreamKDualMulticast": [0, 1], # Codegen-time toggle for single-hop next-neighbor work stealing in the # dynamic-queue StreamK fetch (SK4 / SK5-dynamic). Queue count = # archCaps['NumXCD'] (8 on gfx942/gfx950). When a workgroup's home queue diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py index 42e1cea8743b..df5384907991 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/ClusterLoad.py @@ -59,6 +59,11 @@ def maskSgprName(self, kernel: Mapping, tc: str, *, subtile: bool = False, string = tc.removeprefix("MXS") if tc.startswith("MXS") else tc return f"MulticastMask{string}" + def cooperativeThreadPartition(self, kernel: Mapping, tc: str) -> int: + """Cooperating-workgroup count for ``tc``: ClusterDim[1] (A) / [0] (B).""" + subTc: str = tc[-1] + return kernel["ClusterDim"][1] if subTc == "A" else kernel["ClusterDim"][0] + # -- SGPR declare / undeclare ------------------------------------------- def declareSgprs(self, writer: "KernelWriter", kernel: Mapping) -> None: diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index ba560d522fdb..ff88a2b291f4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -36,7 +36,7 @@ from .Subtile.SubtileLREmit import localReadResetOffsetsSubtile -from ..Common import print2, ceilDivide, log2, clusterEnabled, streamKMulticast, streamKClusterFactors +from ..Common import print2, ceilDivide, log2, clusterEnabled, streamKMulticast, streamKClusterFactors, streamKDual2DMulticast from ..Component import Component from ..AsmStoreState import StoreState, VectorDataTypes from ..AsmAddressCalculation import AddrCalculation @@ -2917,6 +2917,18 @@ def streamKMulticastMaskPredicate(self, writer, kernel): module = Module("StreamK multicast mask predicate") if not streamKMulticast(kernel): return module + if streamKDual2DMulticast(kernel): + # 2-D DUAL-multicast (ForceDPOnly 2-D dual multicast AND the standard + # StreamKDualMulticast path): the dense 2-D masks emitted at kernel init + # (ClusterLoad.computeMasks, aPeers=Ck) are already correct for BOTH + # operands (A along Ck/Y N-adjacent peers, B along Cs/X M-adjacent peers) + # using the raw HW wg_x/wg_y within-cluster ranks. ClusterDimCheck + # guarantees full/aligned DP clusters (nWG0%Cs==0 && gridDimY%Ck==0), so + # the DP round needs no preLoop overwrite or self-mask fallback. (On the + # standard path the masks are later dropped to self-only at the DP->SK + # boundary -- see streamKMulticastBoundaryClear.) + module.addComment0("StreamKDual2DMulticast: keep dense 2-D masks (A & B); no preLoop overwrite") + return module c = kernel["ClusterDim"][0] module.addComment0("cluster B-multicast: gate B-broadcast on clusterMulticastValid") skConstsInVgprs = writer.isStreamKConstantsToVgprEnabled(kernel) @@ -2973,6 +2985,24 @@ def streamKMulticastBoundaryClear(self, writer, kernel): module = Module("StreamK multicast DP->SK boundary clear") if not streamKMulticast(kernel): return module + if streamKDual2DMulticast(kernel): + # 2-D DUAL multicast (standard StreamKForceDPOnly=0 path): the + # DP round multicasts BOTH operands via the dense kernel-init masks (A + # along the Ck/Y peers, B along the Cs/X peers). At the DP->SK boundary + # the SK partial-tile peers no longer co-issue the identical full-K load + # for EITHER operand, so BOTH masks must drop to self-only (unlike the + # 1-D path, where only B was ever broadcast). The single self bit is + # (MulticastMaskA & MulticastMaskB): in a 2-D cluster the A-peer set + # (same X, all Y) and the B-peer set (same Y, all X) intersect at + # exactly this WG's own lane. Set both masks to it so each operand loads + # per-workgroup for the remaining SK iterations. (ForceDPOnly-2D never + # reaches here -- it has no SK round.) + module.addComment0("StreamKDual2DMulticast: clear BOTH A & B broadcast masks at DP->SK boundary") + module.add(SAndB32(dst=sgpr("MulticastMaskA"), src0=sgpr("MulticastMaskA"), src1=sgpr("MulticastMaskB"), + comment="StreamKDual2DMulticast: clear BOTH A & B broadcast masks at DP->SK boundary: self = maskA & maskB (2-D cluster A/B peer sets meet at self)")) + module.add(SMovB32(dst=sgpr("MulticastMaskB"), src=sgpr("MulticastMaskA"), + comment="DP->SK: drop A & B broadcast -> self-only (normal loads)")) + return module module.addComment0("cluster B-multicast: clear B-broadcast mask at DP->SK boundary") module.add(SMovB32(dst=sgpr("MulticastMaskB"), src=sgpr("MulticastMaskA"), comment="DP->SK: drop B broadcast -> self-only (normal B load)")) @@ -3059,18 +3089,45 @@ def preLoop(self, writer, kernel): module.add(SAndB32(dst=sgpr("WorkGroup1"), src0=hex(0xFFFF), src1="ttmp7", comment="workaround")) module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7", comment="workaround")) - # Genuine 2-D StreamK cluster (Ck = ClusterDim[1] > 1, e.g. pure reduction - # [1, C]): the K-split reduction axis is the HW cluster Y rank (WorkGroup1 - # in [0, Ck)). The launch grid is 2-D [skGrid/Ck, Ck, 1], so fold the Y rank - # into the linear StreamK index: - # StreamKIdx = WorkGroup0*Ck + WorkGroup1 (k = WorkGroup1 fastest) - # This keeps StreamKIdx a dense unique index whose (s,k) decode matches the - # 1-D [C,1] scheme (StreamKIdx & (Ck-1) = wg_y = k, StreamKIdx & (C-1) = the - # within-cluster rank). The fold is written into WorkGroup0 so the SMov/VMov - # save below (unchanged) still copies the final index; the 1-D Ck==1 path - # emits no fold. See docs/design/streamk-wg-clusters.md. + # Genuine 2-D StreamK cluster fold. Two mutually-exclusive shapes share the + # 2-D launch grid but fold the HW workgroup coords into StreamKIdx + # DIFFERENTLY: + # + # (a) 2-D DUAL-multicast (ForceDPOnly 2-D dual multicast AND the standard + # StreamKDualMulticast path): Ck (Y) is an N-TILING / A-multicast axis, + # NOT a reduction axis. Fold the genuine 2-D (+batch) coords into the + # linear DP tile index the DP decode expects (M-fastest, matching + # skIndexToWG): + # StreamKIdx = WorkGroup2*(nWG0*nWG1) + WorkGroup1*nWG0 + WorkGroup0 + # so X-peers land on M-adjacent tiles (reuse B) and Y-peers on + # N-adjacent tiles (reuse A). ClusterDimCheck guarantees nWG0 % Cs == 0 + # and nWG1 % Ck == 0 so every HW cluster is a full 2-D tile block. + # + # (b) K-split reduction cluster (Ck = ClusterDim[1] > 1 WITHOUT a dual flag, + # e.g. pure reduction [1, C]): the K-split reduction axis is the HW + # cluster Y rank (WorkGroup1 in [0, Ck)); the grid is [skGrid/Ck, Ck, 1]: + # StreamKIdx = WorkGroup0*Ck + WorkGroup1 (k = WorkGroup1 fastest) + # keeping the dense (s,k) decode of the 1-D [C,1] scheme. + # + # The fold is written into WorkGroup0 so the SMov/VMov save below (unchanged) + # still copies the final index; the 1-D [C,1] path emits no fold. + # See docs/design/streamk-wg-clusters.md. _, ck2d, _, is2d = streamKClusterFactors(kernel) - if is2d: + if streamKDual2DMulticast(kernel): + with writer.allocTmpSgpr(2, tag="ForceDP2DFold") as tRes: + t0 = tRes.idx + t1 = tRes.idx + 1 + module.add(SMulI32(dst=sgpr(t0), src0=sgpr("WorkGroup1"), src1=sgpr("NumWorkGroups0"), + comment="2-D DP: WorkGroup1 * nWG0 (N-tile row)")) + module.add(SMulI32(dst=sgpr(t1), src0=sgpr("NumWorkGroups0"), src1=sgpr("NumWorkGroups1"), + comment="2-D DP: nWG0 * nWG1 (tiles per batch)")) + module.add(SMulI32(dst=sgpr(t1), src0=sgpr(t1), src1=sgpr("WorkGroup2"), + comment="2-D DP: batch * (nWG0*nWG1)")) + module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(t0), + comment="2-D DP: + WorkGroup1*nWG0")) + module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(t1), + comment="2-D DP: StreamKIdx = batch*(nWG0*nWG1) + N*nWG0 + M")) + elif is2d: module.add(SMulI32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=hex(ck2d), comment="2-D cluster: WorkGroup0 * Ck")) module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr("WorkGroup1"), @@ -3092,7 +3149,18 @@ def preLoop(self, writer, kernel): # barrier here in the prologue, before the first tensor_load_to_lds, so # it pairs the cluster-barrier pass's first-load wait. No-op unless # StreamKMulticast is enabled. - module.add(self.streamKMulticastPrologueSignal(writer, kernel)) + # + # EXCEPTION -- standard dual-2D (StreamKForceDPOnly==0 + + # StreamKDualMulticast): the SK partial round RE-ENTERS the persistent loop, + # so the main-loop first-load `s_barrier_wait -3` executes ONCE PER PASS while + # this pre-loop arrive fires only ONCE -> arrivals(1) < waits(N) -> the SK + # round's cluster barrier deadlocks (HW-observed hang). For that path the + # arrive is instead emitted PER PERSISTENT PASS in graWorkGroup (after the + # alpha/start-tile gate, on the same main-loop path as the wait) so every + # pass is balanced. ForceDPOnly-2D (single pass, no SK round) and 1-D + # multicast keep the pre-loop arrive. + if not (streamKDual2DMulticast(kernel) and not kernel["StreamKForceDPOnly"]): + module.add(self.streamKMulticastPrologueSignal(writer, kernel)) if kernel["StreamKForceDPOnly"]: sIdx = writer.acquireStreamKConstSgpr(kernel, "StreamKIdx") @@ -3432,6 +3500,20 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): writer.releaseStreamKConstSgpr(sIpt) module.add(alphaLabel) + # Standard dual-2D (StreamKForceDPOnly==0 + StreamKDualMulticast): + # emit the cluster prologue arrive PER PERSISTENT PASS here, on the main-loop + # path (past the alpha / start-tile gate that reaches alphaLabel), so it pairs + # THIS pass's first-load `s_barrier_wait -3`. The pre-loop arrive is skipped + # for this path (see preLoop), so every DP and SK pass has exactly one arrive + # + one wait -> the split cluster barrier stays balanced when the SK partial + # round re-enters the persistent loop (fixes the [2,2] SK-round deadlock). The + # SK reduction/fixup itself uses the global-flag/workspace handshake, not the + # cluster -3 barrier. Assumes cluster peers make matching passes (uniform DP+SK + # share, true for the [2,2] case with alpha!=0); non-uniform/zero-share + # divergence is not handled. No-op for every other path. + if streamKDual2DMulticast(kernel) and not kernel["StreamKForceDPOnly"]: + module.add(self.streamKMulticastPrologueSignal(writer, kernel)) + writer.sgprPool.checkIn(sTmp) return module diff --git a/projects/hipblaslt/tensilelite/Tensile/Contractions.py b/projects/hipblaslt/tensilelite/Tensile/Contractions.py index f7648edc0cce..073160af0c79 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Contractions.py +++ b/projects/hipblaslt/tensilelite/Tensile/Contractions.py @@ -27,7 +27,7 @@ from .Activation import ActivationType from . import Hardware from . import Properties -from Tensile.Common import state, state_key_ordering, IsaInfo +from Tensile.Common import state, state_key_ordering, IsaInfo, streamKDual2DMulticast from Tensile.Common.Architectures import gfxToIsa from Tensile.Common.DataType import DataType from Tensile.Common.GlobalParameters import internalParameters @@ -609,12 +609,16 @@ def CompoundPredicates(cls, state, problemType): # Pure multicast [C,1]: Cs = C. Pure reduction [1,C]: Cs = 1 (no # M-alignment constraint). valuepredicates.append(state["ClusterDim"][0]) - # value[4] is the N-tile divisor. For a genuine 2-D StreamK cluster - # (Ck = ClusterDim[1] > 1, e.g. pure reduction [1,C]) the Y-extent is the - # K-split reduction / index-generation axis, NOT an N-tiling axis, so it - # must NOT constrain the N-tile grid -> pin to 1. 1-D StreamK ([C,1]) and - # dense (non-StreamK) clusters keep ClusterDim[1]. - if state.get("StreamK", 0) == 3 and state["ClusterDim"][1] > 1: + # value[4] is the N-tile divisor. For a genuine K-SPLIT reduction StreamK + # cluster (Ck = ClusterDim[1] > 1 WITHOUT a dual flag, e.g. pure reduction + # [1,C]) the Y-extent is the reduction / index-generation axis, NOT an + # N-tiling axis, so it must NOT constrain the N-tile grid -> pin to 1. + # For a 2-D DUAL-multicast cluster (ForceDPOnly-2D or StreamKDualMulticast) + # Ck IS a spatial N-tiling / A-multicast axis, so it must remain the N-tile + # divisor (nWG1 % Ck == 0). 1-D StreamK ([C,1]) and dense (non-StreamK) + # clusters keep ClusterDim[1]. + if state.get("StreamK", 0) == 3 and state["ClusterDim"][1] > 1 \ + and not streamKDual2DMulticast(state): valuepredicates.append(1) else: valuepredicates.append(state["ClusterDim"][1]) @@ -667,6 +671,7 @@ class SizeMapping: 'streamKForceDPOnly', 'streamKAtomic', 'streamKClusterReduction', + 'streamKDualMulticast', 'prefetchAcrossPersistent', 'sourceKernel', 'globalAccumulation', @@ -762,6 +767,7 @@ def convertSFCWGMListToHex(value): streamKForceDPOnly = d.get('StreamKForceDPOnly', 0), streamKAtomic = d['StreamKAtomic'] if 'StreamKAtomic' in d else 0, streamKClusterReduction = d.get('StreamKClusterReduction', 0), + streamKDualMulticast = d.get('StreamKDualMulticast', 0), prefetchAcrossPersistent = d.get('PrefetchAcrossPersistent', 0), magicDivAlg = d.get('MagicDivAlg', 1), sourceKernel = d['KernelLanguage'] == 'Source', diff --git a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py index a206e465b3ba..196bc03d39cb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/projects/hipblaslt/tensilelite/Tensile/SolutionStructs/Solution.py @@ -36,7 +36,8 @@ from Tensile.Common import assignParameterWithDefault, IsaInfo, \ print2, printExit, printWarning, \ roundUp, INDEX_CHARS, IsaVersion, SemanticVersion, \ - roundUpToNearestMultiple, effectiveMatrixInstMN, streamKMulticast + roundUpToNearestMultiple, effectiveMatrixInstMN, streamKMulticast, \ + streamKDual2DMulticast from Tensile.Common.DataType import DataType from Tensile.Common.TypeValidationErrors import ConfigTypeError from Tensile.SolutionStructs.LdsPadding import get_fp4_mt_config, get_fp8_mt_config, get_mxs_mt_config, \ @@ -317,6 +318,21 @@ def _validateStreamKClusterReduction(state, printRejectionReason, isaInfoMap): return True +def _isPow2(n): + """True when ``n`` is a positive power of two.""" + return n >= 1 and (n & (n - 1)) == 0 + +def _validateStreamK2DClusterShape(cs, ck): + """Shape check for a StreamK cluster [Cs, Ck]. + + Cs and Ck must each be powers of two and the total cluster C = Cs*Ck must lie + in the supported [2, 16] range (matches the 1-D [C,1] limit). The mask bit-math + assumes powers of two, so a non-pow2 factoring is rejected. For the 1-D + [C,1] path (Ck==1) this reduces exactly to the historic pow2/range check. + """ + return _isPow2(cs) and _isPow2(ck) and 2 <= cs * ck <= 16 + + def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): """Validate the gfx1250 StreamK DP cooperative cluster-load fast path. @@ -376,19 +392,31 @@ def _validateStreamKMulticast(state, printRejectionReason, isaInfoMap): "StreamKMulticast requires StreamKXCCMapping=0 (WGM/XCC remap is bypassed under clustering)") return False - # 1-D cluster [C, 1] with C a power of two in [2, 16]. ClusterDim[1] == 1 - # because the StreamK grid is effectively 1-D along x and consecutive-WG - # clustering is what produces M-adjacent (shared-B) DP tiles. + # Cluster shape: 1-D pure multicast [C, 1] (Cs=C, Ck=1) OR a genuine 2-D + # DUAL-operand multicast cluster [Cs, Ck] (both > 1, ForceDPOnly-2D or + # StreamKDualMulticast) where Ck is an N-tiling / A-multicast axis. Both are + # validated by the shared shape helper (Cs, Ck each a power of two with + # C = Cs*Ck in [2, 16]); for the 1-D case that reduces exactly to the historic + # pow2/range check. A 2-D cluster WITHOUT a dual flag (factored K-split) is not + # supported on this PR and is rejected as a non-[C,1] shape below. clusterDim = state["ClusterDim"] - c = clusterDim[0] - if clusterDim[1] != 1: - reject(state, printRejectionReason, - "StreamKMulticast requires ClusterDim = [C, 1] (got %s)" % clusterDim) - return False - if c < 2 or c > 16 or (c & (c - 1)) != 0: - reject(state, printRejectionReason, - "StreamKMulticast requires ClusterDim[0] a power of two in [2, 16] (got %d)" % c) - return False + cs, ck = clusterDim[0], clusterDim[1] + if streamKDual2DMulticast(state): + if not _validateStreamK2DClusterShape(cs, ck): + reject(state, printRejectionReason, + "StreamK 2-D dual multicast requires Cs=ClusterDim[0] and " + "Ck=ClusterDim[1] each a power of two with C=Cs*Ck in [2, 16] " + "(got %s)" % clusterDim) + return False + else: + if ck != 1: + reject(state, printRejectionReason, + "StreamKMulticast requires ClusterDim = [C, 1] (got %s)" % clusterDim) + return False + if cs < 2 or cs > 16 or (cs & (cs - 1)) != 0: + reject(state, printRejectionReason, + "StreamKMulticast requires ClusterDim[0] a power of two in [2, 16] (got %d)" % cs) + return False # gfx1250 with TDM multicast loads (multicast is a TDM feature). isa = tuple(state["ISA"]) @@ -1284,7 +1312,15 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, # without a StreamK key. See docs/design/streamk-wg-clusters.md. if state["ClusterDim"] != [1, 1] and state.get("StreamK", 0) == 3: ck = state["ClusterDim"][1] - state["StreamKClusterReduction"] = 1 if ck > 1 else 0 + # Ck = ClusterDim[1] > 1 is the K-split REDUCTION axis only for the pure + # reduction [1,C] shape. For a 2-D DUAL-operand multicast cluster + # (ForceDPOnly-2D or StreamKDualMulticast, both axes > 1) Ck is instead an + # N-tiling / A-multicast axis: both operands broadcast and there is no + # workspace K-split, so reduction must NOT be derived (the M-fastest DP fold + # in StreamK.preLoop replaces the K-split fold). Keeping this off is what lets + # _validateStreamKMulticast accept the dual 2-D shape instead of rejecting it + # as factored. + state["StreamKClusterReduction"] = 1 if (ck > 1 and not streamKDual2DMulticast(state)) else 0 # Multicast tri-state (see ValidParameters): -1 auto (legacy), 0 off, 1 on. # Default -1 reproduces the ClusterDim-coupled derivation, so YAML that omits # Multicast is unchanged. @@ -1980,19 +2016,26 @@ def assignDerivedParameters( reject(state, printRejectionReason, "StreamK dynamic/hybrid (SK4/SK5) do not support ClusterDim " "(cluster support is SK3-only)") - # ClusterDim-driven cluster shape (SK3). A genuine 2-D cluster ([Cs,Ck] - # with Ck=ClusterDim[1]>1) launches a 2-D grid [skGrid/Ck, Ck, 1] and folds - # the Y rank into the index (StreamKIdx = WorkGroup0*Ck + WorkGroup1, see - # StreamK.preLoop), so a Y-extent > 1 no longer collides WorkGroup0. Only - # the pure-reduction shape [1, C] is supported here; factored [Cs,Ck] with - # BOTH axes > 1 (spatial multicast AND K-split in one cluster) is not - # supported -- reject it here. Pure multicast stays [C, 1] (Ck==1, 1-D - # launch). - if state["ClusterDim"][1] > 1 and state["ClusterDim"][0] > 1: + # ClusterDim-driven cluster shape (SK3). Legal shapes on this branch: + # [C,1] 1-D pure B-multicast (Ck==1, 1-D launch). + # [1,C] pure K-split reduction (Cs==1, genuine 2-D launch; + # StreamK.preLoop folds StreamKIdx = WorkGroup0*Ck + + # WorkGroup1) -- so a Y-extent > 1 no longer collides + # WorkGroup0. + # [Cs,Ck] both>1 2-D DUAL-multicast, ONLY with the dual opt-in + # (StreamKForceDPOnly-2D or StreamKDualMulticast): genuine + # 2-D launch, M-fastest fold, both operands broadcast. + # The factored [Cs,Ck] (both axes > 1 as spatial multicast AND K-split in + # one cluster) is NOT supported here (it lives in the factored PR) -- reject + # a both-axes cluster that is not the dual-multicast shape. + if state["ClusterDim"][1] > 1 and state["ClusterDim"][0] > 1 \ + and not streamKDual2DMulticast(state): reject(state, printRejectionReason, "Factored StreamK cluster [Cs,Ck] with both axes > 1 is not " - "supported; use ClusterDim=[C,1] (multicast) or [1,C] " - "(reduction) (got %s)" % state["ClusterDim"]) + "supported unless it is 2-D dual multicast (StreamKForceDPOnly " + "or StreamKDualMulticast); use ClusterDim=[C,1] (multicast), " + "[1,C] (reduction), or set the dual flag (got %s)" + % state["ClusterDim"]) # StreamKXCCMapping remaps WorkGroup0 with no cluster awareness; disable it. state["StreamKXCCMapping"] = 0 if not state["EnableMatrixInstruction"]: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_2d_dual_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_2d_dual_multicast.yaml new file mode 100644 index 000000000000..93475c88a34d --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_2d_dual_multicast.yaml @@ -0,0 +1,152 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# STANDARD StreamK (StreamKForceDPOnly=0) 2-D DUAL-multicast (gfx1250, HW-runnable). +# +# A genuine 2-D HW cluster ClusterDim = [Cs, Ck] = [2, 2] on a StreamK==3 +# TWO-TILE (DP-first) kernel -- StreamKForceDPOnly=0, so there is a REAL SK +# partial-tile round in addition to the DP full-tile round. Selected via the +# StreamKDualMulticast=1 opt-in (mutually exclusive with the factored [Cs,Ck] +# K-reduction path, which leaves the flag 0). +# +# Temporal reuse of ONE physical cluster: +# * DP (full-tile) round: 2-D DUAL multicast of BOTH operands -- +# Cs = ClusterDim[0] = 2 : X-peers on M-ADJACENT output tiles reuse B; +# Ck = ClusterDim[1] = 2 : Y-peers on N-ADJACENT output tiles reuse A +# (Ck is an N-tiling / A-multicast axis, NOT a K-split reduction axis). +# * DP->SK boundary: BOTH masks drop to self-only +# (streamKMulticastBoundaryClear). +# * SK (partial-tile) round: 1-D workspace reduction exactly as today. +# +# Run on a gfx1250 device (functional-sim is a regression check only). +# +# Sizes are cluster-aligned. MT = 32x32 (MatrixInstruction [16,16,128,1,1,1,1,2,2]): +# M=256 -> nWG0=8 (8 % Cs=2 == 0); N=192 -> nWG1=6 (6 % Ck=2 == 0); tiles=48. +# +# *** IMPORTANT -- forcing a LIVE SK partial round (env vars, NOT a config knob) *** +# The DP grid is CU-budget-clamped: getSKGridImpl uses origami select_grid_size +# (TENSILE_STREAMK_DYNAMIC_GRID default 6) or, failing that, skGrid=cuCount +# (ContractionSolution.cpp getSKGridImpl). The dual-2D reshape then pins gridX=nWG0 +# and clamps gridY UP to floor(nWG1/Ck)*Ck == nWG1 (ClusterDimCheck requires +# nWG1 % Ck == 0). So on a large-CU gfx1250 the budget >= tiles => gridY rounds to +# nWG1=6 => skGrid = nWG0*nWG1 = 48 = tiles => tiles % skGrid == 0 => DP-ONLY (the +# SK round is NOT exercised). Problem size / MacroTile CANNOT force the SK round +# device-independently for this reason. +# +# To DETERMINISTICALLY exercise BOTH the DP dual-multicast AND a live SK reduction +# on ANY gfx1250 CU count, set these host env vars when running: +# TENSILE_STREAMK_FIXED_GRID=32 # -> reshape gridY=ceil(32/8)=4 (mult of Ck), +# # skGrid = nWG0*gridY = 8*4 = 32 < tiles=48 +# TENSILE_STREAMK_FULL_TILES=0 # -> skTiles = tiles % skGrid = 16 (else the +# # default FULL_TILES=1 makes skTiles=48 = +# # ALL-SK, dpTiles=0, no DP multicast) +# Result: skGrid=32, 32 DP full tiles (the [nWG0=8 x gridY=4] block, dual-multicast) +# + 16 SK partial tiles (the remaining 2 N-rows, workspace-reduced; itersPerTile=4, +# SKItersPerWG=16*4/32=2). Any TENSILE_STREAMK_FIXED_GRID in [17,32] reshapes to the +# same gridY=4/skGrid=32; 32 is canonical (== the final skGrid, no rounding). +# The [512,512,...] size is a fully-DP-divisible companion (pure DP dual-multicast). +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + Activation: True + SupportUserArgs: True + UseBias: 0 + MXBlockA: 32 + MXBlockB: 32 + + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [false] + - StoreVectorWidth: [-1] + - StreamK: [3] + # STANDARD two-tile schedule (NOT ForceDPOnly) + dual-2D opt-in. + - StreamKForceDPOnly: [0] + - StreamKDualMulticast: [1] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + # DP round: Cs=2 (B, M-adjacent) x Ck=2 (A, N-adjacent). Ck is an + # N-tiling / A-multicast axis, NOT a K-split reduction axis. + - ClusterDim: [[2, 2]] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + # M=256->nWG0=8, N=192->nWG1=6 (both cluster-aligned); skGrid=32, + # tiles=48 -> 16-tile SK partial round: exercises DP dual-multicast + # AND SK reduction. + - Exact: [256, 192, 1, 1024] + # Fully DP-divisible companion (nWG0=nWG1=16): pure DP dual-multicast. + - Exact: [512, 512, 1, 1056] + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml index 2cd34939272e..ac383f2fe22f 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml @@ -83,7 +83,15 @@ BenchmarkProblems: - TDMInst: [3] - MXLoadInst: ["TDM"] - MXScaleFormat: ["InMemorySwizzle"] - - ClusterDim: [[2, 1], [4, 1], [8, 1]] + # ForceDPOnly cluster multicast sweep, 1-D AND 2-D in one file: + # [C,1] = 1-D B-multicast (Cs=C X-peers on M-adjacent tiles reuse B); + # [2,2] = 2-D DUAL-multicast (Cs=2 reuse B on M-adjacent tiles, Ck=2 + # Y-peers reuse A on N-adjacent tiles). Ck here is an N-tiling + # axis, NOT a K-split reduction. + # ProblemSizes below are chosen so nWG0=ceil(M/MT0) is divisible by 8 + # (covers Cs in {2,4,8}) and nWG1=ceil(N/MT1) is even (covers Ck=2), so + # EVERY swept ClusterDim enumerates with ZERO DID_NOT_SATISFY_ASSERTS. + - ClusterDim: [[2, 1], [4, 1], [8, 1], [2, 2]] BenchmarkForkParameters: JoinParameters: BenchmarkJoinParameters: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_pap.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_pap.yaml new file mode 100644 index 000000000000..81eb606f914f --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_pap.yaml @@ -0,0 +1,111 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# ForceDPOnly 2-D DUAL-operand multicast + PrefetchAcrossPersistent, mxf4. +# +# PAP companion to sk_mxf4_force_dp_only_cluster_multicast.yaml (the [2,2] row). +# A genuine 2-D cluster ClusterDim=[Cs,Ck]=[2,2] where BOTH operands multicast +# (Cs X-peers reuse B on M-adjacent tiles; Ck Y-peers reuse A on N-adjacent +# tiles -- an A-multicast, NOT a K-split reduction). Because A IS a real +# multicast (aPeers=Ck>1) the guard refinement does NOT free maskA: BOTH masks +# stay live across the PAP refresh. Already fit the SGPR budget (FDPO=1); +# included to lock in the dual-2D + PAP path alongside the factored one. +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + Activation: True + SupportUserArgs: True + UseBias: 0 + MXBlockA: 32 + MXBlockB: 32 + + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1, 2] + - PrefetchLocalRead: [1] + - PrefetchAcrossPersistent: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [false] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKForceDPOnly: [1] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + # 2-D DUAL-multicast [2,2]: Cs=2 reuse B (M-adjacent), Ck=2 reuse A + # (N-adjacent). Both masks stay live under PAP. Zero-skip: nWG0=ceil(M/MT0) + # divisible by 2 and nWG1=ceil(N/MT1) even. + - ClusterDim: [[2, 2]] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [256, 256, 1, 1056] + - Exact: [512, 512, 1, 1056] + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_2d_dual_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_2d_dual_multicast.yaml new file mode 100644 index 000000000000..4024e0a5c8f5 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_2d_dual_multicast.yaml @@ -0,0 +1,150 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# STANDARD StreamK (StreamKForceDPOnly=0) 2-D DUAL-multicast (gfx1250, HW-runnable). +# MXFP8 analogue of sk_mxf4_2d_dual_multicast.yaml (identical schedule; DataType F8). +# +# A genuine 2-D HW cluster ClusterDim = [Cs, Ck] = [2, 2] on a StreamK==3 +# TWO-TILE (DP-first) kernel -- StreamKForceDPOnly=0, so there is a REAL SK +# partial-tile round in addition to the DP full-tile round. Selected via the +# StreamKDualMulticast=1 opt-in (mutually exclusive with the factored [Cs,Ck] +# K-reduction path, which leaves the flag 0). +# +# Temporal reuse of ONE physical cluster: +# * DP (full-tile) round: 2-D DUAL multicast of BOTH operands -- +# Cs = ClusterDim[0] = 2 : X-peers on M-ADJACENT output tiles reuse B; +# Ck = ClusterDim[1] = 2 : Y-peers on N-ADJACENT output tiles reuse A +# (Ck is an N-tiling / A-multicast axis, NOT a K-split reduction axis). +# * DP->SK boundary: BOTH masks drop to self-only +# (streamKMulticastBoundaryClear). +# * SK (partial-tile) round: 1-D workspace reduction exactly as today. +# +# Run on a gfx1250 device (functional-sim is a regression check only). +# +# Sizes are cluster-aligned. MT = 32x32 (MatrixInstruction [16,16,128,1,1,1,1,2,2]): +# M=256 -> nWG0=8 (8 % Cs=2 == 0); N=192 -> nWG1=6 (6 % Ck=2 == 0); tiles=48. +# +# *** IMPORTANT -- forcing a LIVE SK partial round (env vars, NOT a config knob) *** +# The DP grid is CU-budget-clamped: getSKGridImpl uses origami select_grid_size +# (TENSILE_STREAMK_DYNAMIC_GRID default 6) or, failing that, skGrid=cuCount +# (ContractionSolution.cpp getSKGridImpl). The dual-2D reshape then pins gridX=nWG0 +# and clamps gridY UP to floor(nWG1/Ck)*Ck == nWG1 (ClusterDimCheck requires +# nWG1 % Ck == 0). So on a large-CU gfx1250 the budget >= tiles => gridY rounds to +# nWG1=6 => skGrid = nWG0*nWG1 = 48 = tiles => tiles % skGrid == 0 => DP-ONLY (the +# SK round is NOT exercised). Problem size / MacroTile CANNOT force the SK round +# device-independently for this reason. +# +# To DETERMINISTICALLY exercise BOTH the DP dual-multicast AND a live SK reduction +# on ANY gfx1250 CU count, set these host env vars when running: +# TENSILE_STREAMK_FIXED_GRID=32 # -> reshape gridY=ceil(32/8)=4 (mult of Ck), +# # skGrid = nWG0*gridY = 8*4 = 32 < tiles=48 +# TENSILE_STREAMK_FULL_TILES=0 # -> skTiles = tiles % skGrid = 16 (else the +# # default FULL_TILES=1 makes skTiles=48 = +# # ALL-SK, dpTiles=0, no DP multicast) +# Result: skGrid=32, 32 DP full tiles (the [nWG0=8 x gridY=4] block, dual-multicast) +# + 16 SK partial tiles (the remaining 2 N-rows, workspace-reduced; itersPerTile=4, +# SKItersPerWG=16*4/32=2). Any TENSILE_STREAMK_FIXED_GRID in [17,32] reshapes to the +# same gridY=4/skGrid=32; 32 is canonical (== the final skGrid, no rounding). +# The [512,512,...] size is a fully-DP-divisible companion (pure DP dual-multicast). +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [false] + - StoreVectorWidth: [-1] + - StreamK: [3] + # STANDARD two-tile schedule (NOT ForceDPOnly) + dual-2D opt-in. + - StreamKForceDPOnly: [0] + - StreamKDualMulticast: [1] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + # DP round: Cs=2 (B, M-adjacent) x Ck=2 (A, N-adjacent). Ck is an + # N-tiling / A-multicast axis, NOT a K-split reduction axis. + - ClusterDim: [[2, 2]] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + # M=256->nWG0=8, N=192->nWG1=6 (both cluster-aligned); skGrid=32, + # tiles=48 -> 16-tile SK partial round: exercises DP dual-multicast + # AND SK reduction. + - Exact: [256, 192, 1, 1024] + # Fully DP-divisible companion (nWG0=nWG1=16): pure DP dual-multicast. + - Exact: [512, 512, 1, 1056] + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only.yaml new file mode 100644 index 000000000000..9ab67972784b --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only.yaml @@ -0,0 +1,94 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# MXFP8 analogue of sk_mxf4_force_dp_only.yaml (identical schedule; DataType F8). +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + +GlobalParameters: + SyncsPerBenchmark: 0 + Architecture: gfx1250 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + - # gfx1250 MXFP8, StreamK force DP-only persistent mode + - # ProblemType, TN + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + + - # Force StreamK to use persistent DP only, with no SK region + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3, 4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [false] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKForceDPOnly: [1] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [258, 258, 1, 1056] + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml index 529ccade9f6f..496d332b69d4 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_cluster_multicast.yaml @@ -82,7 +82,15 @@ BenchmarkProblems: - TDMInst: [3] - MXLoadInst: ["TDM"] - MXScaleFormat: ["InMemorySwizzle"] - - ClusterDim: [[2, 1], [4, 1], [8, 1]] + # ForceDPOnly cluster multicast sweep, 1-D AND 2-D in one file: + # [C,1] = 1-D B-multicast (Cs=C X-peers on M-adjacent tiles reuse B); + # [2,2] = 2-D DUAL-multicast (Cs=2 reuse B on M-adjacent tiles, Ck=2 + # Y-peers reuse A on N-adjacent tiles). Ck here is an N-tiling + # axis, NOT a K-split reduction. + # ProblemSizes below are chosen so nWG0=ceil(M/MT0) is divisible by 8 + # (covers Cs in {2,4,8}) and nWG1=ceil(N/MT1) is even (covers Ck=2), so + # EVERY swept ClusterDim enumerates with ZERO DID_NOT_SATISFY_ASSERTS. + - ClusterDim: [[2, 1], [4, 1], [8, 1], [2, 2]] BenchmarkForkParameters: JoinParameters: BenchmarkJoinParameters: diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_pap.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_pap.yaml new file mode 100644 index 000000000000..30e61f0003f4 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/common/streamk/gfx1250/core/sk_mxf8_force_dp_only_pap.yaml @@ -0,0 +1,109 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# ForceDPOnly 2-D DUAL-operand multicast + PrefetchAcrossPersistent, mxf8. +# +# PAP companion to sk_mxf8_force_dp_only_cluster_multicast.yaml (the [2,2] row). +# Genuine 2-D cluster ClusterDim=[Cs,Ck]=[2,2], BOTH operands multicast (Ck is +# an A-multicast N-tiling axis, NOT a K-split reduction). A is a real multicast +# (aPeers=Ck>1) so the guard refinement does NOT free maskA -- both masks stay +# live under PAP. Already fit the SGPR budget (FDPO=1). +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx90a, skip-gfx940, skip-gfx941, skip-gfx942, skip-gfx950, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: -1 + BoundsCheck: 0 + KernelTime: False + CpuThreads: 32 + DataInitTypeA: 3 + DataInitTypeB: 3 + DataInitTypeD: 2 + DataInitTypeMXSA: 3 + DataInitTypeMXSB: 3 + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + PrintLevel: 2 + PrintSolutionRejectionReason: True + ForceGenerateKernel: True + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F8 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + Activation: True + SupportUserArgs: True + UseBias: 0 + MXBlockA: 32 + MXBlockB: 32 + + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1, 2] + - PrefetchLocalRead: [1] + - PrefetchAcrossPersistent: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - 1LDSBuffer: [0] + - DirectToVgprSparseMetadata: [false] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKForceDPOnly: [1] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + # 2-D DUAL-multicast [2,2]: Cs=2 reuse B (M-adjacent), Ck=2 reuse A + # (N-adjacent). Both masks stay live under PAP. Zero-skip: nWG0=ceil(M/MT0) + # divisible by 2 and nWG1=ceil(N/MT1) even. + - ClusterDim: [[2, 2]] + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [256, 256, 1, 1056] + - Exact: [512, 512, 1, 1056] + - ActivationArgs: + - [Enum: none] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_forcedp_2d_pap_gfx1250_char.ambr b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_forcedp_2d_pap_gfx1250_char.ambr new file mode 100644 index 000000000000..4604fde45761 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/__snapshots__/test_streamk_forcedp_2d_pap_gfx1250_char.ambr @@ -0,0 +1,9 @@ +# serializer version: 1 +# name: test_streamk_forcedp_2d_pap_gfx1250_golden + list([ + dict({ + 'basename': 'Cijk_Alik_Bljk_F4SS_MXAE8B32_MXBE8B32_BH_UserArgC-SsVRMvMU24VAm3_KICvTjxEgYgV8zjUsf22eFkVD0=', + 'err': 0, + }), + ]) +# --- diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/config_harness.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/config_harness.py index c03c4896fc0b..692c6644078e 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/config_harness.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/config_harness.py @@ -233,6 +233,27 @@ def _emit_one(kwa, kernel, splitGSU, canonical): return base, src, res.err +def assert_cluster_barrier_balanced(src, base): + """Cluster-scope split-barrier balance check shared by the gfx1250 StreamK + cluster char tests. Every arrive (``s_barrier_signal -3``) must be consumed by + a completion (``s_barrier_wait -3``) on every control-flow path. The prologue + wave-0 arrive is consumed by exactly one of two mutually exclusive cluster + waits: the last-iteration guard's zero-iteration skip-edge wait, or the + first-load wait on the >=1-iteration fall-through. Both waits are emitted + statically but only one executes on any given path, so the static wait count + is exactly one greater than the signal count; every other arrive (including a + config's dedicated prologue-prefetch handshake) is a self-contained arrive/wait + pair. Any other imbalance would leave a cluster wait unpaired and stall the + cluster waves. + """ + n_signal = src.count("s_barrier_signal -3") + n_wait = src.count("s_barrier_wait -3") + assert n_wait == n_signal + 1, ( + f"Kernel {base!r}: unexpected cluster barrier balance: " + f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" + ) + + # --- in-file smoke runner --------------------------------------------------- # # NOT a pytest test (no ``test_`` prefix; guarded under __main__). Drives the diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_dual_2d_multicast.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_dual_2d_multicast.yaml new file mode 100644 index 000000000000..721fabb805b9 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_dual_2d_multicast.yaml @@ -0,0 +1,105 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# STANDARD StreamK (StreamKForceDPOnly=0) 2-D DUAL-multicast (gfx1250). +# +# A genuine 2-D HW cluster ClusterDim = [Cs, Ck] = [2, 2] on a StreamK==3 +# TWO-TILE (DP-first) kernel -- i.e. StreamKForceDPOnly=0, so there is a real SK +# partial-tile round in addition to the DP full-tile round. The DP round does the +# 2-D dual multicast (BOTH operands TDM-multicast): +# * Cs = ClusterDim[0] = 2 : X-peers on M-ADJACENT output tiles reuse B; +# * Ck = ClusterDim[1] = 2 : Y-peers on N-ADJACENT output tiles reuse A. +# The SK (partial-tile) round reduces 1-D via the workspace exactly as today; at +# the DP->SK boundary BOTH masks drop to self-only (streamKMulticastBoundaryClear). +# This is temporal reuse of ONE physical cluster: 2-D mask grouping in DP, 1-D +# reduction grouping in SK. It is selected via StreamKDualMulticast=1 (mutually +# exclusive with the factored [Cs,Ck] K-reduction path, which leaves the flag 0). +# +# CPU-only characterization: no GPU required. This validates codegen (dense dual +# masks on the DP round + 2-D tile fold + DP->SK dual-mask clear + intact SK +# partial-tile workspace addressing + balanced cluster barrier). The on-device +# correctness check is the user's HW run of +# Tests/common/streamk/gfx1250/core/sk_mxf4_2d_dual_multicast.yaml. +GlobalParameters: + SyncsPerBenchmark: 0 + MinimumRequiredVersion: 5.0.0 + NumElementsToValidate: 128 + DataInitTypeBeta: 0 + DataInitTypeAlpha: 1 + Device: 0 + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + # STANDARD two-tile schedule (NOT ForceDPOnly) + dual-2D opt-in. + - StreamKForceDPOnly: [0] + - StreamKDualMulticast: [1] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + # Cs=2 (B, M-adjacent) x Ck=2 (A, N-adjacent). Ck is an N-tiling axis, + # NOT a K-split reduction axis (StreamKClusterReduction NOT derived). + - ClusterDim: [[2, 2]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + # M=256 -> nWG0=8 (8 % Cs=2 == 0); N=192 -> nWG1=6 (6 % Ck=2 == 0). + # getSKGridImpl picks gridY=4 (largest mult of Ck < nWG1) -> skGrid=32, + # tiles=48, tiles % skGrid = 16 != 0 -> a genuine SK partial round of + # 16 tiles split 2-way (48/16), each SK WG itersPerTile/2 iters. K=1024 + # -> itersPerTile=4 (even) so the SK round has extraIters==0 (balanced). + - Exact: [256, 192, 1, 1024] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d.yaml new file mode 100644 index 000000000000..82ef199d2670 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d.yaml @@ -0,0 +1,99 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# ForceDPOnly 2-D dual-operand multicast (gfx1250). +# +# A genuine 2-D HW cluster ClusterDim = [Cs, Ck] = [2, 2] on a StreamK==3 +# StreamKForceDPOnly (dense data-parallel, NO K-split reduction) kernel where +# BOTH operands are TDM-multicast: +# * Cs = ClusterDim[0] = 2 : X-peers on M-ADJACENT output tiles reuse B +# (exactly as the shipped 1-D [C,1] ForceDPOnly multicast); +# * Ck = ClusterDim[1] = 2 : Y-peers on N-ADJACENT output tiles reuse A. +# Ck here is an N-tiling / A-multicast axis, NOT a K-split reduction axis, so +# StreamKClusterReduction is deliberately NOT derived (see Solution.py collapse). +# The dense ClusterLoad 2-D masks (aPeers=Ck) are reused verbatim for A and B, +# and the ForceDPOnly DP tile decode is folded to +# StreamKIdx = batch*(nWG0*nWG1) + WorkGroup1*nWG0 + WorkGroup0 +# so X-peers land M-adjacent and Y-peers land N-adjacent (StreamK.preLoop). +# +# CPU-only characterization: no GPU required. This validates codegen (masks + +# 2-D tile decode + balanced cluster barrier); the definitive correctness check +# is the user's HW run of the [2, 2] ClusterDim entry in +# Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml +# (the 1-D and 2-D ForceDPOnly multicast sweeps now share one config). +GlobalParameters: + SyncsPerBenchmark: 0 + MinimumRequiredVersion: 5.0.0 + NumElementsToValidate: 128 + DataInitTypeBeta: 0 + DataInitTypeAlpha: 1 + Device: 0 + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKForceDPOnly: [1] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + # ForceDPOnly 2-D DUAL-multicast probe: Cs=2 (B, M-adjacent) x Ck=2 + # (A, N-adjacent). Ck is an N-tiling axis, NOT a K-split. + - ClusterDim: [[2, 2]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [256, 256, 1, 256] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d_pap.yaml b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d_pap.yaml new file mode 100644 index 000000000000..d69eac7f83a9 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx1250/streamk_forcedp_2d_pap.yaml @@ -0,0 +1,92 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# PrefetchAcrossPersistent=1 variant of streamk_forcedp_2d.yaml. +# +# ForceDPOnly 2-D DUAL-operand multicast cluster ClusterDim = [Cs, Ck] = [2, 2] +# (Cs = X-peers reuse B on M-adjacent tiles; Ck = Y-peers reuse A on N-adjacent +# tiles -- a genuine A-multicast, NOT a K-split reduction), now with PAP on. +# +# Because A IS a real multicast here (aPeers = Ck > 1), the guard refinement +# does NOT free maskA: BOTH multicast masks stay live across the PAP refresh. +# This dual-2D + PAP + FDPO=1 config already fit the SGPR budget before the +# refinement, so it must remain a real kernel (err==0) with both masks -- the +# counterpart to the factored (FDPO=0, maskA freed) config. It also guards +# against the refinement accidentally dropping a real A-multicast. +GlobalParameters: + SyncsPerBenchmark: 0 + MinimumRequiredVersion: 5.0.0 + NumElementsToValidate: 128 + DataInitTypeBeta: 0 + DataInitTypeAlpha: 1 + Device: 0 + +BenchmarkProblems: + - + - + OperationType: GEMM + DataType: F4 + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + MXBlockA: 32 + MXBlockB: 32 + - + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + ForkParameters: + - UseSgprForGRO: [0] + - ForceDisableShadowInit: [True] + - MatrixInstruction: + - [16, 16, 128, 1, 1, 1, 1, 2, 2] + - DepthU: [256] + - ClusterLocalRead: [0] + - TransposeLDS: [-1] + - LdsPadA: [-1] + - LdsPadB: [-1] + - LdsBlockSizePerPadA: [-1] + - LdsBlockSizePerPadB: [-1] + - LdsPadMXSA: [-1] + - LdsPadMXSB: [-1] + - LdsBlockSizePerPadMXSA: [-1] + - LdsBlockSizePerPadMXSB: [-1] + - LdsPadMetadata: [0] + - LDSTrInst: [False] + - PrefetchGlobalRead: [1] + - PrefetchLocalRead: [1] + - PrefetchAcrossPersistent: [1] + - ScheduleIterAlg: [4] + - SourceSwap: [False] + - StoreRemapVectorWidth: [0] + - GlobalSplitU: [0] + - GlobalReadVectorWidthA: [-1] + - GlobalReadVectorWidthB: [-1] + - ExpandPointerSwap: [False] + - VectorWidthA: [-1] + - VectorWidthB: [-1] + - LocalReadVectorWidth: [-1] + - StoreVectorWidth: [-1] + - StreamK: [3] + - StreamKForceDPOnly: [1] + - StreamKAtomic: [0] + - StreamKXCCMapping: [0] + # ForceDPOnly 2-D dual multicast: Cs=2 (B, M-adjacent) x Ck=2 (A, + # N-adjacent). Both operands multicast -> both masks stay live under PAP. + - ClusterDim: [[2, 2]] + - StaggerU: [0] + - DirectToVgprA: [False] + - DirectToVgprB: [False] + - WavefrontSize: [32] + - WorkGroupMapping: [1] + - TDMInst: [3] + - MXLoadInst: ["TDM"] + - MXScaleFormat: ["InMemorySwizzle"] + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [256, 256, 1, 256] diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py index cbfc0f6167f9..01c2039a93bb 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_coop_load_gfx1250_char.py @@ -25,7 +25,7 @@ import pytest -from config_harness import emit_kernels_from_config +from config_harness import assert_cluster_barrier_balanced, emit_kernels_from_config pytestmark = pytest.mark.unit @@ -92,19 +92,5 @@ def test_streamk_cluster_coop_load_gfx1250_emits_assembly(): assert "s_barrier_wait -3" in src, ( f"Kernel {base!r}: missing cluster-scope barrier wait (-3)" ) - # Cluster-scope split-barrier balance. Every arrive - # (s_barrier_signal -3) must be consumed by a completion - # (s_barrier_wait -3) on every control-flow path. The prologue wave-0 - # arrive is consumed by exactly one of two mutually exclusive cluster - # waits: the last-iteration guard's zero-iteration skip-edge wait, or - # the first-load wait on the >=1-iteration fall-through. Both waits are - # emitted statically but only one executes on any given path, so the - # static wait count is exactly one greater than the signal count; every - # other arrive is a self-contained arrive/wait pair. Any other imbalance - # would leave a cluster wait unpaired and stall the cluster waves. - n_signal = src.count("s_barrier_signal -3") - n_wait = src.count("s_barrier_wait -3") - assert n_wait == n_signal + 1, ( - f"Kernel {base!r}: unexpected cluster barrier balance: " - f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" - ) + # Cluster-scope split-barrier balance (shared check). + assert_cluster_barrier_balanced(src, base) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py index de80f48908df..3b63eec3bc2d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_gfx1250_char.py @@ -29,7 +29,7 @@ import pytest -from config_harness import emit_kernels_from_config +from config_harness import assert_cluster_barrier_balanced, emit_kernels_from_config pytestmark = pytest.mark.unit @@ -89,22 +89,8 @@ def test_streamk_cluster_multicast_gfx1250_emits_assembly(): assert "s_barrier_wait -3" in src, ( f"Kernel {base!r} missing cluster-scope barrier wait (-3)" ) - # Cluster-scope split-barrier balance. Every arrive - # (s_barrier_signal -3) must be consumed by a completion - # (s_barrier_wait -3) on every control-flow path. The prologue wave-0 - # arrive is consumed by exactly one of two mutually exclusive cluster - # waits: the last-iteration guard's zero-iteration skip-edge wait, or - # the first-load wait on the >=1-iteration fall-through. Both waits are - # emitted statically but only one executes on any given path, so the - # static wait count is exactly one greater than the signal count; every - # other arrive is a self-contained arrive/wait pair. Any other imbalance - # would leave a cluster wait unpaired and stall the cluster waves. - n_signal = src.count("s_barrier_signal -3") - n_wait = src.count("s_barrier_wait -3") - assert n_wait == n_signal + 1, ( - f"Kernel {base!r} has unexpected cluster barrier balance: " - f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" - ) + # Cluster-scope split-barrier balance (shared check). + assert_cluster_barrier_balanced(src, base) def test_streamk_cluster_multicast_gfx1250_golden(snapshot): diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py index 19bdd856b789..52c2c2604354 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_cluster_multicast_pgr2_gfx1250_char.py @@ -26,7 +26,7 @@ import pytest -from config_harness import emit_kernels_from_config +from config_harness import assert_cluster_barrier_balanced, emit_kernels_from_config pytestmark = pytest.mark.unit @@ -92,22 +92,8 @@ def test_streamk_cluster_multicast_pgr2_gfx1250_emits_assembly(): f"Kernel {base!r} PGR2 prologue prefetch load is NOT bracketed by a " f"cluster-scope -3 handshake" ) - # Cluster-scope split-barrier balance. The prologue wave-0 arrive - # (s_barrier_signal -3) is consumed by exactly one of two mutually - # exclusive cluster waits: the last-iteration guard's zero-iteration - # skip-edge wait, or the first-load wait on the >=1-iteration - # fall-through. Both are emitted statically but only one executes on any - # control-flow path, so the static wait count is exactly one greater - # than the signal count; every other arrive (including this config's - # dedicated prologue-prefetch handshake) is a self-contained arrive/wait - # pair. Any other imbalance would leave a cluster wait unpaired and - # stall the cluster waves. - n_signal = src.count("s_barrier_signal -3") - n_wait = src.count("s_barrier_wait -3") - assert n_wait == n_signal + 1, ( - f"Kernel {base!r} has unexpected cluster barrier balance: " - f"{n_signal} signal(-3) vs {n_wait} wait(-3) (expected wait == signal + 1)" - ) + # Cluster-scope split-barrier balance (shared check). + assert_cluster_barrier_balanced(src, base) def test_streamk_cluster_multicast_pgr2_gfx1250_golden(snapshot): diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_dual_2d_multicast_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_dual_2d_multicast_gfx1250_char.py new file mode 100644 index 000000000000..eb7c6be2be8a --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_dual_2d_multicast_gfx1250_char.py @@ -0,0 +1,150 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +"""STANDARD StreamK 2-D DUAL-multicast (StreamKDualMulticast) -- gfx1250 characterization. + +A [2,2] cluster on the STANDARD two-tile StreamK path +(``StreamKForceDPOnly=0``): a genuine 2-D cluster ClusterDim=[Cs,Ck]=[2,2] where + + * Cs = ClusterDim[0] = 2 : X-peers on M-ADJACENT tiles reuse B; and + * Ck = ClusterDim[1] = 2 : Y-peers on N-ADJACENT tiles reuse A + +on the DP (full-tile) round, while the SK (partial-tile) round reduces 1-D via +the workspace exactly as today. It is opted in via ``StreamKDualMulticast=1`` +(mutually exclusive with the factored [Cs,Ck] K-reduction path). + +Unlike the ForceDPOnly 2-D dual multicast (``streamk_forcedp_2d``) this kernel HAS a real +SK round, so it additionally asserts: + * the DP->SK boundary drops BOTH masks to self-only (not just B); and + * the SK partial-tile workspace/reduction machinery is intact. + +Asserts (see ``_designed/gfx1250/streamk_dual_2d_multicast.yaml``): + * err == 0, real gfx1250 assembly; + * DP round binds BOTH MulticastMaskA (0x5) and MulticastMaskB (0x3) -- A is + genuinely multicast -- via the dense 2-D masks (keep-dense short-circuit); + * the 2-D DP tile fold StreamKIdx = batch*(nWG0*nWG1)+N*nWG0+M is present; + * the DP->SK boundary clears BOTH masks (self = maskA & maskB); + * the SK partial round + cluster split-barrier arrive/wait are present; and + * the factored K-split decode/shift are ABSENT (Ck is a spatial N-axis). + +CPU-only. The on-device correctness check is the user's HW run of +``Tests/common/streamk/gfx1250/core/sk_mxf4_2d_dual_multicast.yaml``. +""" + +import os + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_dual_2d_multicast.yaml", +) + + +def test_streamk_dual_2d_multicast_gfx1250_emits_assembly(): + """gfx1250 standard-StreamK [2,2] dual-2D emits real assembly, err==0, + with dual DP masks (A on Ck/Y, B on Cs/X), the 2-D DP fold, a DP->SK BOTH-mask + clear, an intact SK partial round, and NO factored K-split decode.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + f"Expected all err==0, got: {[(b, e) for b, _s, e in results if e != 0]}" + ) + for base, src, _err in results: + assert src and len(src.splitlines()) > 50, ( + f"Kernel {base!r} emitted suspiciously short source" + ) + assert ".amdgcn_target" in src and "gfx1250" in src, ( + f"Kernel {base!r} missing gfx1250 target" + ) + assert base.startswith("Cijk_"), f"Kernel {base!r} has unexpected prefix" + + # --- (a) DP round binds BOTH operands' masks with the dense 2-D values --- + # maskA=0x5 (bits {0,2}) multicasts A across the Ck=2 Y-peers; maskB=0x3 + # (bits {0,1}) multicasts B across the Cs=2 X-peers. + assert "s[sgprMulticastMaskA], 0x5" in src, ( + f"Kernel {base!r} DP round does not set maskA=0x5 (A NOT multicast!)" + ) + assert "s[sgprMulticastMaskB], 0x3" in src, ( + f"Kernel {base!r} DP round does not set maskB=0x3 (B NOT multicast!)" + ) + assert "keep dense 2-D masks" in src, ( + f"Kernel {base!r} missing the dense-mask short-circuit note" + ) + assert "sgprMulticastMaskA" in src and "sgprMulticastMaskB" in src, ( + f"Kernel {base!r} never binds both multicast masks to TDM descriptors" + ) + + # --- (b) 2-D DP tile fold: StreamKIdx = batch*(nWG0*nWG1) + N*nWG0 + M --- + assert "2-D DP: WorkGroup1 * nWG0 (N-tile row)" in src, ( + f"Kernel {base!r} missing the N-tile-row fold (WorkGroup1*nWG0)" + ) + assert "2-D DP: StreamKIdx = batch*(nWG0*nWG1) + N*nWG0 + M" in src, ( + f"Kernel {base!r} missing the linear 2-D DP StreamKIdx fold" + ) + + # --- (c) DP->SK boundary clears BOTH masks (dual mode), self=maskA&maskB --- + assert "clear BOTH A & B broadcast masks at DP->SK boundary" in src, ( + f"Kernel {base!r} missing the dual DP->SK BOTH-mask clear" + ) + assert "self = maskA & maskB" in src, ( + f"Kernel {base!r} missing the self = maskA & maskB reduction bit" + ) + assert "drop A & B broadcast -> self-only" in src, ( + f"Kernel {base!r} missing the A&B self-only drop" + ) + + # --- (d) a real SK partial round exists (this is NOT ForceDPOnly) --- + # DP-first grid-stride shift + the SK-section offset are only emitted on + # the two-tile (StreamKForceDPOnly=0) schedule. + assert "DP iterations shift" in src, ( + f"Kernel {base!r} missing the DP grid-stride shift (no DP round?)" + ) + assert "Offset to start of SK section" in src, ( + f"Kernel {base!r} missing the SK-section offset (no SK partial round?)" + ) + + # --- (e) cluster split-barrier present (prologue arrive + wait phases) --- + assert src.count("s_barrier_signal -3") >= 1, ( + f"Kernel {base!r} missing cluster barrier arrive (s_barrier_signal -3)" + ) + assert src.count("s_barrier_wait -3") >= 1, ( + f"Kernel {base!r} missing cluster barrier wait (s_barrier_wait -3)" + ) + assert "cluster_barrier signal (arrive)" in src, ( + f"Kernel {base!r} missing the prologue cluster arrive" + ) + + # --- (f) SK-round deadlock fix: the cluster prologue arrive is emitted + # PER PERSISTENT PASS (INSIDE the persistent loop), not once before it. + # The multicast split barrier is 1 arrive + 1 first-load wait PER PASS; the + # SK partial round re-enters the persistent loop, so a single pre-loop arrive + # would leave the SK pass's wait unpaired -> cluster -3 barrier deadlock + # (HW-observed hang). The arrive must sit after label_PersistentLoopStart so + # every DP and SK pass balances its own first-load/zero-iter wait. + loop_top = src.find("label_PersistentLoopStart:") + assert loop_top != -1, f"Kernel {base!r} has no persistent loop" + arrive_pos = src.find("cluster_barrier signal (arrive)") + assert arrive_pos > loop_top, ( + f"Kernel {base!r} emits the cluster prologue arrive BEFORE the persistent " + f"loop (once) -- the SK partial round's first-load wait would be unpaired " + f"(the [2,2] SK-round deadlock). It must be per-pass (inside the loop)." + ) + + # --- factored K-split decode/shift MUST be ABSENT (Ck is a spatial axis) --- + assert "k = StreamKIdx & (Ck-1)" not in src, ( + f"Kernel {base!r} wrongly emitted the factored K-slice decode" + ) + assert "MulticastMaskB = maskB_base << k" not in src, ( + f"Kernel {base!r} wrongly emitted the factored K-split maskB shift" + ) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_gfx1250_char.py new file mode 100644 index 000000000000..2c3c4a7e0420 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_gfx1250_char.py @@ -0,0 +1,117 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +"""ForceDPOnly 2-D dual-operand multicast -- gfx1250 characterization (CPU-only). + +A StreamK==3 ``StreamKForceDPOnly`` (dense data-parallel, NO +K-split reduction) kernel given a GENUINE 2-D cluster ClusterDim=[Cs,Ck]=[2,2] +in which BOTH operands are TDM-multicast: + + * Cs = ClusterDim[0] = 2 : X-peers on M-ADJACENT output tiles reuse B + (exactly as the shipped 1-D [C,1] ForceDPOnly multicast); and + * Ck = ClusterDim[1] = 2 : Y-peers on N-ADJACENT output tiles reuse A. + +Unlike the factored cluster (``streamk_factored_cluster``) Ck here is an +N-tiling / A-multicast axis, NOT a K-split reduction axis, so the factored +"k = StreamKIdx & (Ck-1)" decode and the "MulticastMaskB = maskB_base << k" +shift MUST be ABSENT. + +Asserts (see ``_designed/gfx1250/streamk_forcedp_2d.yaml``): + * the kernel emits real gfx1250 assembly with ``err == 0``; + * the 2-D DP tile decode folds the genuine 2-D (+batch) HW workgroup coords + into the linear ForceDPOnly index + StreamKIdx = batch*(nWG0*nWG1) + WorkGroup1*nWG0 + WorkGroup0 + (M-fastest) so X-peers land M-adjacent (reuse B) and Y-peers N-adjacent + (reuse A); + * the DENSE ClusterLoad 2-D masks are reused verbatim for BOTH operands and + the runtime preLoop overwrite is short-circuited (keep-dense-masks note); + * both MulticastMaskA and MulticastMaskB are bound onto their TDM descriptors; + * the cluster split-barrier (``s_barrier_signal/-wait -3``) is present and the + prologue/epilogue arrive+wait phases co-exist; and + * the factored K-split decode/shift are ABSENT (this is a spatial N-axis). + +CPU-only: no GPU required. The on-device correctness check is the user's HW run +of the ``[2, 2]`` ClusterDim entry in +``Tests/common/streamk/gfx1250/core/sk_mxf4_force_dp_only_cluster_multicast.yaml`` +(the 1-D and 2-D ForceDPOnly multicast sweeps now share one config; functional-sim +is a regression check only). +""" + +import os + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_forcedp_2d.yaml", +) + + +def test_streamk_forcedp_2d_gfx1250_emits_assembly(): + """gfx1250 ForceDPOnly [2,2] emits real assembly, err==0, with the + 2-D DP tile decode + DENSE dual-multicast masks (A on Ck/Y, B on Cs/X) and + NO factored K-split decode.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + f"Expected all err==0, got: {[(b, e) for b, _s, e in results if e != 0]}" + ) + for base, src, _err in results: + assert src and len(src.splitlines()) > 50, ( + f"Kernel {base!r} emitted suspiciously short source" + ) + assert ".amdgcn_target" in src and "gfx1250" in src, ( + f"Kernel {base!r} missing gfx1250 target" + ) + assert base.startswith("Cijk_"), f"Kernel {base!r} has unexpected prefix" + + # --- 2-D DP tile decode: StreamKIdx = batch*(nWG0*nWG1) + N*nWG0 + M --- + assert "2-D DP: WorkGroup1 * nWG0 (N-tile row)" in src, ( + f"Kernel {base!r} missing the N-tile-row fold (WorkGroup1*nWG0)" + ) + assert "2-D DP: StreamKIdx = batch*(nWG0*nWG1) + N*nWG0 + M" in src, ( + f"Kernel {base!r} missing the linear 2-D DP StreamKIdx fold" + ) + + # --- dense 2-D masks reused for BOTH operands; no preLoop overwrite --- + assert "keep dense 2-D masks" in src, ( + f"Kernel {base!r} missing the dense-mask short-circuit note" + ) + assert "sgprMulticastMaskA" in src, ( + f"Kernel {base!r} never binds MulticastMaskA (A is NOT multicast!)" + ) + assert "sgprMulticastMaskB" in src, ( + f"Kernel {base!r} never binds MulticastMaskB to a B descriptor" + ) + + # --- cluster split-barrier present + prologue/epilogue phases co-exist --- + assert src.count("s_barrier_signal -3") >= 1, ( + f"Kernel {base!r} missing cluster barrier arrive (s_barrier_signal -3)" + ) + assert src.count("s_barrier_wait -3") >= 1, ( + f"Kernel {base!r} missing cluster barrier wait (s_barrier_wait -3)" + ) + assert "cluster_barrier signal (arrive)" in src, ( + f"Kernel {base!r} missing the prologue cluster arrive" + ) + assert "cluster_barrier wait" in src, ( + f"Kernel {base!r} missing a cluster barrier wait phase" + ) + + # --- factored K-split decode/shift MUST be ABSENT (Ck is a spatial axis) --- + assert "k = StreamKIdx & (Ck-1)" not in src, ( + f"Kernel {base!r} wrongly emitted the factored K-slice decode" + ) + assert "MulticastMaskB = maskB_base << k" not in src, ( + f"Kernel {base!r} wrongly emitted the factored K-split maskB shift" + ) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_pap_gfx1250_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_pap_gfx1250_char.py new file mode 100644 index 000000000000..41383aaaf518 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/_codegen/test_streamk_forcedp_2d_pap_gfx1250_char.py @@ -0,0 +1,105 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +"""ForceDPOnly 2-D dual-operand multicast + PrefetchAcrossPersistent -- gfx1250 +characterization (CPU-only). + +PAP companion to ``test_streamk_forcedp_2d_gfx1250_char.py``. A genuine 2-D +cluster ClusterDim=[Cs,Ck]=[2,2] where BOTH operands are multicast (Cs X-peers +reuse B on M-adjacent tiles; Ck Y-peers reuse A on N-adjacent tiles -- an +A-multicast, NOT a K-split reduction), with PrefetchAcrossPersistent on. + +Because A IS a real multicast (aPeers = Ck > 1), the +``ClusterLoad.papDropsSelfOnlyMaskA`` refinement does NOT free maskA here: BOTH +multicast masks must stay live across the PAP refresh. This is the counterpart +to the factored (FDPO=0, self-only maskA FREED) PAP config and guards against +the refinement accidentally dropping a real A-multicast. It already fit the SGPR +budget before the refinement, so it must remain a real kernel (err==0). + +Asserts: + * every kernel emits real gfx1250 assembly with ``err == 0``; + * BOTH multicast masks stay live and are applied (MulticastMaskA to the A + descriptor, MulticastMaskB to the B descriptor); + * the cluster-scope barrier handshake is present; and + * the kernel fits the 106-SGPR budget. + +CPU-only: no GPU required. Real-HW / FFM validation of the dual-2D + PAP point +is part of the residual re-validation scope. +""" + +import os +import re + +import pytest + +from config_harness import emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_ARCH = "gfx1250" + +_CONFIG = os.path.join( + os.path.dirname(__file__), + "data", + "test_data", + "_designed", + "gfx1250", + "streamk_forcedp_2d_pap.yaml", +) + +_SGPR_BUDGET = 106 +_NEXT_FREE_SGPR = re.compile(r"\.amdhsa_next_free_sgpr\s+(\d+)") + + +def _max_sgpr(src): + return max((int(m.group(1)) for m in _NEXT_FREE_SGPR.finditer(src)), default=0) + + +def test_streamk_forcedp_2d_pap_gfx1250_keeps_both_masks_live(): + """Dual-2D (FDPO=1) + PAP emits real assembly (err==0) and keeps BOTH multicast + masks live -- the real A-multicast is NOT freed by the guard refinement.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + assert len(results) >= 1, "Expected >=1 kernel, got 0" + assert all(err == 0 for (_b, _s, err) in results), ( + f"Expected all err==0, got: {[(b, e) for b, _s, e in results if e != 0]}" + ) + for base, src, _err in results: + assert src and len(src.splitlines()) > 50, ( + f"Kernel {base!r} emitted suspiciously short source" + ) + assert ".amdgcn_target" in src and "gfx1250" in src, ( + f"Kernel {base!r} missing gfx1250 target" + ) + assert base.startswith("Cijk_"), f"Kernel {base!r} has unexpected prefix" + + # Both masks stay live and are applied (dual-operand multicast under PAP). + assert "s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in src, ( + f"Kernel {base!r} dropped the real A-multicast mask (MulticastMaskA) -- " + "the guard refinement must NOT free A on a dual-2D cluster" + ) + assert "s[sgprtdmBGroup1], s[sgprtdmBGroup1], s[sgprMulticastMaskB]" in src, ( + f"Kernel {base!r} dropped the B-multicast mask (MulticastMaskB)" + ) + + # Cluster-scope barrier handshake present. + assert src.count("s_barrier_signal -3") >= 1, ( + f"Kernel {base!r} missing cluster barrier arrive (s_barrier_signal -3)" + ) + assert src.count("s_barrier_wait -3") >= 1, ( + f"Kernel {base!r} missing cluster barrier wait (s_barrier_wait -3)" + ) + + sgprs = _max_sgpr(src) + assert 0 < sgprs <= _SGPR_BUDGET, ( + f"Kernel {base!r} uses {sgprs} SGPRs, exceeds the {_SGPR_BUDGET} budget" + ) + + +def test_streamk_forcedp_2d_pap_gfx1250_golden(snapshot): + """Golden: order-invariant {basename, err} digest of the dual-2D PAP emit.""" + results = emit_kernels_from_config(_CONFIG, limit=8, arch=_ARCH) + digest = sorted( + ({"basename": b, "err": e} for (b, _s, e) in results), + key=lambda d: d["basename"], + ) + assert digest == snapshot diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py index 43c2e6531419..ece08e53c579 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_cluster_load_component.py @@ -63,7 +63,12 @@ def undefineSgpr(self, name): def _kernel(*, multicast=True, clusterDim=(2, 2), tdmA=True, tdmB=True, numWaves=4, useSubtile=False, sparse=0, tdmMeta=False, tdmInst=3, - pap=False, streamKMulticast=False): + pap=False, streamKMulticast=False, forceDPOnly=0, dualMulticast=0): + # The StreamK=3 DP cooperative B-multicast path is no longer a stored state + # key: the component derives it from StreamK==3 + ClusterDim[0] > 1 via the + # streamKMulticast(kernel) helper. Drive it here by setting StreamK (the + # streamKMulticast=True cases all use a ClusterDim with Cs = clusterDim[0] > 1). + # StreamKForceDPOnly / StreamKDualMulticast select the 2-D dual-multicast path. return { "Multicast": multicast, "ClusterDim": list(clusterDim), @@ -80,6 +85,8 @@ def _kernel(*, multicast=True, clusterDim=(2, 2), tdmA=True, tdmB=True, # state key. Drive it here by setting StreamK (the streamKMulticast=True # cases all use a ClusterDim with Cs = clusterDim[0] > 1). "StreamK": 3 if streamKMulticast else 0, + "StreamKForceDPOnly": forceDPOnly, + "StreamKDualMulticast": dualMulticast, } @@ -152,6 +159,20 @@ def test_subtile_split_name(self): assert self._c().maskSgprName(k, "B", subtile=True) == "MulticastMaskB" +class TestCooperativeThreadPartition: + def _c(self): + from Tensile.Components.ClusterLoad import ClusterLoadTDM + return ClusterLoadTDM() + + def test_a_uses_clusterdim1_b_uses_clusterdim0(self): + k = _kernel(clusterDim=(4, 2)) + assert self._c().cooperativeThreadPartition(k, "A") == 2 + assert self._c().cooperativeThreadPartition(k, "B") == 4 + # MXS tensors resolve by their trailing tensor char. + assert self._c().cooperativeThreadPartition(k, "MXSA") == 2 + assert self._c().cooperativeThreadPartition(k, "MXSB") == 4 + + # --- SGPR declare / undeclare ---------------------------------------------- class TestDeclareUndeclare: @@ -217,13 +238,20 @@ def test_undeclare_keeps_maskB_live_frees_selfonly_maskA_under_pap(self): self._c().undeclareSgprs(w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 1))) assert w.undefined == ["MulticastMaskA"] - def test_undeclare_keeps_both_live_under_pap_2d_cluster(self): - # On a 2-D cluster (ClusterDim[1]>1) A is a real multicast, not self-only, - # so both masks stay live across the PAP refresh (neither is freed). + def test_undeclare_keeps_both_live_under_pap_dual_2d_cluster(self): + # On a DUAL-2D cluster (ForceDPOnly 2-D or StreamKDualMulticast) the Ck peers + # reuse A on N-adjacent tiles, so A is a REAL multicast (aPeers>1), not + # self-only. Both masks must stay live across the PAP refresh (neither freed). _init_rocisa_gfx1250() w = _StubWriter() - self._c().undeclareSgprs(w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 2))) + self._c().undeclareSgprs( + w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 2), forceDPOnly=1)) assert w.undefined == [] + # StreamKDualMulticast (standard two-tile) is the other dual-2D entry. + w2 = _StubWriter() + self._c().undeclareSgprs( + w2, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 2), dualMulticast=1)) + assert w2.undefined == [] def test_undeclare_frees_both_without_pap(self): # Same StreamK multicast kernel but PAP off: no persistent refresh, so both @@ -363,6 +391,16 @@ def test_no_pap_streamk_still_applies_maskA(self): w, _kernel(streamKMulticast=True, pap=False, clusterDim=(2, 1)), "tdmAGroup1", "A") assert "s_or_b32 s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in str(mod) + def test_pap_dual_2d_still_applies_maskA(self): + # Dual-2D (ForceDPOnly 2-D): A IS a real multicast across Ck peers, so it + # stays live and is applied on every refresh. + _init_rocisa_gfx1250() + w = _StubWriter() + mod = self._c().applyToDescriptor( + w, _kernel(streamKMulticast=True, pap=True, clusterDim=(2, 2), forceDPOnly=1), + "tdmAGroup1", "A") + assert "s_or_b32 s[sgprtdmAGroup1], s[sgprtdmAGroup1], s[sgprMulticastMaskA]" in str(mod) + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_dual_2d_multicast.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_dual_2d_multicast.py new file mode 100644 index 000000000000..26cafe034df9 --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_dual_2d_multicast.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +################################################################################ +# Focused unit tests for the standard two-tile StreamK (StreamKForceDPOnly=0) +# path with StreamKDualMulticast=1: 2-D DUAL-operand multicast (gfx1250). +# +# On a genuine 2-D cluster ClusterDim = [Cs, Ck] (both > 1) a StreamK==3 +# StreamKForceDPOnly=0 config is FACTORED by default (Ck is the K-split reduction +# axis -> StreamKClusterReduction). Setting StreamKDualMulticast=1 selects the +# dual-2D multicast interpretation INSTEAD: the DP (full-tile) round does 2-D +# dual multicast (Cs/X peers share B on M-adjacent tiles, Ck/Y peers share A on +# N-adjacent tiles) and the SK (partial-tile) round reduces 1-D via the workspace +# as today -- so StreamKClusterReduction is NOT derived (Ck is an N-tiling / +# A-multicast axis). +# +# These pin (CPU-only, no GPU): +# * streamKDual2DMulticast -- the generalized structural detector (True for +# ForceDPOnly-2D AND the standard StreamKDualMulticast opt-in; False for the +# factored/reduction/1-D shapes); +# * the derivation gating vs the factored path (same ClusterDim=[2,2], SK3, +# ForceDPOnly=0: flag on -> multicast + NO reduction; flag off -> factored, +# i.e. multicast + reduction), proving mutual exclusion; +# * the selection predicates (ClusterDimCheck keeps Ck as the N-tile divisor, +# value[4]=Ck, unlike the factored path which pins it to 1; and +# ClusterReductionIterCheck is ABSENT because reduction is not derived); and +# * the dense 2-D dual-mask math (maskA=0x5, maskB=0x3) and the DP->SK +# boundary self bit (self = maskA & maskB). +# +# Usage: +# pytest test_streamk_dual_2d_multicast.py -v +################################################################################ + +import copy +import os +import sys + +import pytest + +pytestmark = pytest.mark.unit + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +TENSILE_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +sys.path.insert(0, TENSILE_ROOT) +sys.path.insert(0, os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", "_codegen")) + +_DESIGNED = os.path.join( + TENSILE_ROOT, "Tensile", "Tests", "unit", "characterization", + "_codegen", "data", "test_data", "_designed", "gfx1250") +_DUAL2D = os.path.join(_DESIGNED, "streamk_dual_2d_multicast.yaml") + +_ARCH = "gfx1250" + + +# --- helpers --------------------------------------------------------------- + +def _write_variant(tmp_path, base_cfg, name, *, fork_overrides=None): + from Tensile import LibraryIO + import yaml + + cfg = copy.deepcopy(LibraryIO.read(base_cfg)) + if fork_overrides: + fork = cfg["BenchmarkProblems"][0][1]["ForkParameters"] + for key, val in fork_overrides.items(): + replaced = False + for entry in fork: + if key in entry: + entry[key] = val + replaced = True + break + if not replaced: + fork.append({key: val}) + out = tmp_path / name + with open(out, "w") as f: + yaml.safe_dump(cfg, f, default_flow_style=None) + return str(out) + + +def _derive_states(cfg_path): + from config_harness import solutions_from_config + sols = solutions_from_config(cfg_path, arch=_ARCH, limit_solutions=8) + return [s._state if hasattr(s, "_state") else s for s in sols] + + +def _compound_preds(state): + import Tensile.Contractions as C + from Tensile.Contractions import ProblemType + pt = ProblemType.FromOriginalState(state["ProblemType"]) + return C.ProblemPredicate.CompoundPredicates(state, pt) + + +def _pred(preds, tag): + return next((p for p in preds if p.tag == tag), None) + + +def _k(**kw): + d = {"StreamKForceDPOnly": 0, "StreamKDualMulticast": 0, "ClusterDim": [1, 1]} + d.update(kw) + return d + + +# --- detector -------------------------------------------------------------- + +class TestDetector: + def test_true_for_standard_dual2d_optin(self): + from Tensile.Common import streamKDual2DMulticast + assert streamKDual2DMulticast( + _k(StreamKForceDPOnly=0, StreamKDualMulticast=1, ClusterDim=[2, 2])) is True + + def test_true_for_forcedp_2d(self): + """Generalizes the ForceDPOnly-2D detector: ForceDPOnly + both>1 stays True + (no opt-in flag needed -- that shape is unambiguous).""" + from Tensile.Common import streamKDual2DMulticast + assert streamKDual2DMulticast( + _k(StreamKForceDPOnly=1, ClusterDim=[2, 2])) is True + + @pytest.mark.parametrize("state", [ + _k(StreamKDualMulticast=1, ClusterDim=[4, 1]), # opt-in but 1-D (Ck==1): not 2-D + _k(StreamKDualMulticast=1, ClusterDim=[1, 4]), # opt-in but Cs==1: no B axis + _k(StreamKForceDPOnly=0, StreamKDualMulticast=0, ClusterDim=[2, 2]), # FACTORED (no flag) + _k(StreamKForceDPOnly=1, ClusterDim=[8, 1]), # 1-D ForceDPOnly multicast + _k(ClusterDim=[1, 1]), # no cluster + ]) + def test_false_for_non_dual2d(self, state): + from Tensile.Common import streamKDual2DMulticast + assert streamKDual2DMulticast(state) is False + + def test_factored_stays_forcedp_detector_false(self): + """The ForceDPOnly-specific detector must remain False for the standard + opt-in (it is unchanged for existing configs).""" + from Tensile.Common import streamKForceDP2DMulticast + assert streamKForceDP2DMulticast( + _k(StreamKForceDPOnly=0, StreamKDualMulticast=1, ClusterDim=[2, 2])) is False + + +# --- derivation & gating vs factored --------------------------------------- + +class TestDerivation: + def test_dual2d_derives_multicast_without_reduction(self): + """StreamKDualMulticast=1 on [2,2]/SK3/ForceDPOnly=0: B-multicast on, + StreamKClusterReduction OFF (Ck is an N-tiling / A-multicast axis).""" + from Tensile.Common import streamKMulticast + states = _derive_states(_DUAL2D) + assert states + for st in states: + assert st["ClusterDim"] == [2, 2] + assert st["StreamKForceDPOnly"] == 0 + assert st["StreamKDualMulticast"] == 1 + assert streamKMulticast(st) + assert st.get("StreamKClusterReduction", 0) == 0 + assert st["Multicast"] == 1 + assert st["ClusterBarrier"] is True + + def test_same_config_without_flag_is_rejected(self, tmp_path): + """MUTUAL EXCLUSION on the multicast PR: the identical config with + StreamKDualMulticast=0 is NOT a dual-2D cluster. On this PR (1-D multicast + + 2-D dual-multicast only, no reduction/factored) a 2-D cluster [Cs,Ck] + both>1 without a dual flag is the factored (K-split) shape, which is not + supported here and is REJECTED at build time -- so no solution derives. + The opt-in flag is the sole discriminator between dual-2D and the + (rejected-here) factored interpretation.""" + cfg = _write_variant(tmp_path, _DUAL2D, "factored_no_flag.yaml", + fork_overrides={"StreamKDualMulticast": [0]}) + states = _derive_states(cfg) + assert not states, ( + "2-D [Cs,Ck] without a dual flag (factored K-split) must be rejected on " + "the multicast PR; got %d derived solution(s)" % len(states)) + + +# --- selection predicates -------------------------------------------------- + +class TestPredicates: + def test_cluster_dim_check_keeps_ck_as_n_divisor(self): + """For dual-2D the Ck axis is a spatial N-tiling axis, so ClusterDimCheck + keeps ClusterDim[1] as the N-tile divisor (value[4]=Ck=2) -- unlike the + factored path, which pins value[4]=1. value[3] is Cs=2 (M-adjacency).""" + states = _derive_states(_DUAL2D) + assert states + st = states[0] + preds = _compound_preds(st) + p = _pred(preds, "ClusterDimCheck") + assert p is not None + assert p.value[3] == st["ClusterDim"][0] == 2, p.value + assert p.value[4] == st["ClusterDim"][1] == 2, p.value + + def test_no_cluster_reduction_iter_check(self): + """ClusterReductionIterCheck is emitted only when StreamKClusterReduction + is derived; dual-2D does not derive it, so the predicate is ABSENT (no + itersPerTile % Ck constraint -- Ck is not a K-split).""" + states = _derive_states(_DUAL2D) + assert states + st = states[0] + preds = _compound_preds(st) + assert _pred(preds, "ClusterReductionIterCheck") is None + + +# --- dense 2-D dual-mask math (mirrors ClusterLoad.computeMasks) ------------ + +def _dense_masks(cs, ck): + maskA = 1 + for idx in range(ck): # dual-2D: aPeers = ClusterDim[1] = Ck (A IS multicast) + maskA |= (1 << (idx * cs)) + maskB = (1 << cs) - 1 + return maskA, maskB + + +class TestMasks: + def test_dense_masks_for_2x2(self): + maskA, maskB = _dense_masks(cs=2, ck=2) + assert maskA == 0x5, hex(maskA) # A across Ck=2 Y-peers, stride Cs=2 -> {0,2} + assert maskB == 0x3, hex(maskB) # B across Cs=2 X-peers -> {0,1} + + def test_dp_to_sk_self_bit_is_intersection(self): + """The DP->SK boundary clear sets both masks to the single self bit + self = maskA & maskB. In a 2-D cluster the A-peer set and B-peer set + intersect at exactly this WG's own lane (rank 0 here).""" + maskA, maskB = _dense_masks(cs=2, ck=2) + assert (maskA & maskB) == 0x1 + assert bin(maskA).count("1") == 2 # Ck A-peers + assert bin(maskB).count("1") == 2 # Cs B-peers + + def test_1d_forcedp_multicast_keeps_a_self_only(self): + """The shipped 1-D [C,1] ForceDPOnly multicast (Ck==1) yields a self-only + A mask -- the dual-2D A-multicast is strictly the Ck>1 addition.""" + maskA, maskB = _dense_masks(cs=8, ck=1) + assert maskA == 0x1 # self only + assert maskB == (1 << 8) - 1 # B across all 8 X-peers diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 810cb9c87203..ac1c69a88aff 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -144,6 +144,10 @@ namespace TensileLite int streamKForceDPOnly = 0; int streamKAtomic = 0; int streamKClusterReduction = 0; + // STANDARD two-tile StreamK (streamKForceDPOnly==0) opt-in to + // 2-D dual-operand multicast on the DP round (SK round reduces 1-D). See + // getSKGridImpl / generateSingleCall dual-2D branches. + int streamKDualMulticast = 0; int prefetchAcrossPersistent = 0; int persistentKernel = 0; bool persistentKernelAlongBatch = false; diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp index 2f3b71984fba..9f71e78f7c58 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Serialization/ContractionSolution.hpp @@ -107,6 +107,7 @@ namespace TensileLite iot::mapOptional(io, "streamKForceDPOnly", s.streamKForceDPOnly); iot::mapOptional(io, "streamKAtomic", s.streamKAtomic); iot::mapOptional(io, "streamKClusterReduction", s.streamKClusterReduction); + iot::mapOptional(io, "streamKDualMulticast", s.streamKDualMulticast); iot::mapOptional(io, "prefetchAcrossPersistent", s.prefetchAcrossPersistent); iot::mapOptional(io, "persistentKernel", s.persistentKernel); iot::mapOptional(io, "persistentKernelAlongBatch", s.persistentKernelAlongBatch); diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index a1f0e4b4b34f..9ccad75eb251 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -1910,14 +1910,46 @@ namespace TensileLite if(sizeMapping.streamK != 0) { - if(sizeMapping.clusterDim.y > 1) + if(sizeMapping.streamKForceDPOnly != 0 && sizeMapping.clusterDim.y > 1) { - // Genuine 2-D StreamK cluster (Ck = clusterDim.y > 1, e.g. pure - // reduction [1, C]): launch a 2-D grid so the cluster Y-extent is - // legal (gridDimY % Ck == 0) and every WG gets a unique index via - // StreamKIdx = WorkGroup0*Ck + WorkGroup1 (kernel preLoop). sk.grid - // is rounded to a multiple of C = Cs*Ck (getSKGridImpl), so - // gridDimX = skGrid/Ck is a whole number. + // ForceDPOnly 2-D dual-operand multicast: a genuine 2-D HW + // cluster ClusterDim=[Cs,Ck] where Cs (x) B-multicast peers are + // M-adjacent tiles and Ck (y) A-multicast peers are N-ADJACENT tiles. + // Launch a grid that spans the FULL M x N tile space -- gridX = nWG0 + // (M-tiles), gridY = nWG1 (N-tiles) -- so Y-neighbors are N-adjacent + // output tiles. The kernel folds StreamKIdx = batch*(nWG0*nWG1) + + // WorkGroup1*nWG0 + WorkGroup0 (StreamK.preLoop) so each WG owns one + // tile. ClusterDimCheck guarantees nWG0 % Cs == 0 and nWG1 % Ck == 0, + // so gridDimX % Cs == 0 and gridDimY % Ck == 0 (legal cluster launch) + // and every HW cluster is a full 2-D tile block. sk.grid == tiles == + // nWG0*nWG1 here (getSKGrid). See docs/design/streamk-wg-clusters.md. + rv.numWorkGroups.x = problemNumGroupTiles.x; // nWG0 (M-tiles) + // rv.numWorkGroups.y already = nWG1 * gsu (N-tiles); z stays batch. + } + else if(sizeMapping.streamKDualMulticast != 0 && sizeMapping.clusterDim.y > 1) + { + // STANDARD two-tile StreamK 2-D DUAL-multicast (StreamKDualMulticast). + // Launch a PERSISTENT 2-D cluster grid [nWG0, gridY, batch]: gridX is + // PINNED to nWG0 so the kernel's dense DP fold StreamKIdx = + // WorkGroup1*nWG0 + WorkGroup0 (StreamK.preLoop) is dense in + // [0, skGrid) and its Cs X-peers are M-ADJACENT (share B) / Ck Y-peers + // are N-ADJACENT (share A). gridY = sk.grid / nWG0 is a multiple of Ck + // (getSKGridImpl), so gridDimY % Ck == 0 and gridDimX (== nWG0) % Cs == + // 0 (ClusterDimCheck) -- a legal cluster launch. When gridY < nWG1 the + // leftover tiles form the SK partial round (standard two-tile + // accounting). See docs/design/streamk-wg-clusters.md. + rv.numWorkGroups.x = problemNumGroupTiles.x; // nWG0 + rv.numWorkGroups.y = sk.grid / problemNumGroupTiles.x; // gridY (mult of Ck) + // rv.numWorkGroups.z stays batch. + } + else if(sizeMapping.clusterDim.y > 1) + { + // Genuine 2-D StreamK cluster with Ck = clusterDim.y > 1 that is NOT + // dual multicast: the pure K-split reduction [1, C]. Launch a 2-D grid + // so the cluster Y-extent is legal (gridDimY % Ck == 0) and every WG + // gets a unique index via StreamKIdx = WorkGroup0*Ck + WorkGroup1 + // (kernel preLoop). sk.grid is rounded to a multiple of C = Cs*Ck + // (getSKGridImpl), so gridDimX = skGrid/Ck is a whole number. uint32_t ck = sizeMapping.clusterDim.y; rv.numWorkGroups.x = sk.grid / ck; rv.numWorkGroups.y = ck; @@ -4386,19 +4418,81 @@ namespace TensileLite } } - // StreamKMulticast (gfx1250): DP cooperative B-multicast. The multicast - // cluster is SPATIAL: the C peers of a cluster process C DISTINCT, - // M-adjacent tiles and share the B (N-block) tile. v1 is single-round, so - // the launched grid is `tiles` rounded UP to a multiple of C = clusterDim.x. - // Rounding up keeps every launched HW cluster full while giving each WG one - // tile in the single DP round; a trailing partial cluster (tiles % C != 0) - // has idle-but-present tail WGs whose cluster is disabled by the kernel's - // clusterMulticastValid runtime predicate (so a masked target is never left - // without a matching load). See docs/design/cluster-load-component-and-streamk-multicast.md. - if(self.sizeMapping.streamK == 3 && self.sizeMapping.clusterDim.x > 1) + // StreamK cluster grid (gfx1250, ClusterDim-driven). The 1-D + // pure-multicast [C,1] cluster is SPATIAL: the Cs peers process Cs + // DISTINCT M-adjacent tiles and share B, so the launched grid is `tiles` + // rounded UP to a multiple of Cs = clusterDim.x (a trailing partial + // cluster is disabled by the kernel's clusterMulticastValid runtime + // guard). The genuine 2-D DUAL-multicast cluster ([Cs,Ck] both > 1) + // instead tiles the M x N space (ForceDPOnly-2D) or reshapes the DP grid + // into a 2-D cluster grid (standard StreamKDualMulticast); Ck is an + // N-tiling / A-multicast axis, not a K-split. See + // docs/design/streamk-wg-clusters.md. + if(self.sizeMapping.streamK == 3 + && (static_cast(self.sizeMapping.clusterDim.x) + * static_cast(self.sizeMapping.clusterDim.y)) + > 1) { - size_t c = self.sizeMapping.clusterDim.x; - skGrid = ((tiles + c - 1) / c) * c; + if(self.sizeMapping.streamKForceDPOnly) + { + // ForceDPOnly 2-D dual-operand multicast: one WG per output tile + // (Cs peers M-adjacent -> share B, Ck peers N-adjacent -> share + // A). Not a K-split, so the grid is exactly `tiles` (already a + // multiple of C=Cs*Ck since nWG0 % Cs == 0 and nWG1 % Ck == 0 via + // ClusterDimCheck). The 2-D launch splits this into [nWG0, nWG1, 1] + // (generateSingleCall). See docs/design/streamk-wg-clusters.md. + skGrid = tiles; + } + else if(self.sizeMapping.streamKDualMulticast) + { + // STANDARD two-tile StreamK (streamKForceDPOnly==0, + // StreamKDualMulticast) 2-D DUAL-multicast. UNLIKE ForceDPOnly + // (skGrid==tiles, no SK round), the standard schedule keeps a real + // DP + SK split, so we RESHAPE the base DP grid (skGrid computed + // above from the CU budget / origami) into a 2-D cluster grid + // [nWG0, gridY]: + // * gridX is PINNED to nWG0 so the kernel's dense DP fold + // StreamKIdx = WorkGroup1*nWG0 + WorkGroup0 (StreamK.preLoop) + // is dense in [0, skGrid) AND its Cs X-peers land on + // M-ADJACENT tiles (share B) while its Ck Y-peers land on + // N-ADJACENT tiles (share A) -- the 2-D dual multicast. + // * gridY = round(baseSkGrid / nWG0) up to a multiple of Ck, + // clamped to the full N-extent floor(nWG1/Ck)*Ck. + // skGrid = nWG0 * gridY is a multiple of C = Cs*Ck (nWG0 % Cs == 0 + // via ClusterDimCheck, gridY % Ck == 0 by construction), so every + // launched HW cluster is full. A genuine SK partial round runs + // whenever nWG1 % gridY != 0 (tiles % skGrid != 0), driving the + // standard SK3 two-tile accounting below; at the DP->SK boundary + // the kernel drops BOTH masks to self-only + // (streamKMulticastBoundaryClear). batch==1 only (the fold's batch + // stride is nWG0*nWG1). See docs/design/streamk-wg-clusters.md. + size_t ck = static_cast(self.sizeMapping.clusterDim.y); + // Derive nWG0/nWG1 with the SAME routine the launch uses + // (calculateGrid): it folds packBatchDims into nWG0/nWG1, applies + // transposeC01(), and CeilDivides by the macroTile, so the reshaped + // grid matches the launched grid. + dim3 wgSizeTmp, nWGTmp; + self.calculateGrid(wgSizeTmp, nWGTmp, problem); + size_t nwg0 = static_cast(nWGTmp.x); + size_t nwg1 = static_cast(nWGTmp.y); + size_t nwg1Ck = (nwg1 / ck) * ck; // largest multiple of Ck <= nWG1 + if(nwg1Ck < ck) + nwg1Ck = ck; // at least one full cluster row + size_t gridY = nwg0 > 0 ? (skGrid + nwg0 - 1) / nwg0 : ck; // base rows + gridY = ((gridY + ck - 1) / ck) * ck; // round up to a multiple of Ck + if(gridY < ck) + gridY = ck; + if(gridY > nwg1Ck) + gridY = nwg1Ck; // clamp to full N-extent + skGrid = nwg0 * gridY; + } + else + { + size_t c = static_cast(self.sizeMapping.clusterDim.x) + * static_cast(self.sizeMapping.clusterDim.y); + size_t ck = static_cast(self.sizeMapping.clusterDim.y); + skGrid = ((ck * tiles + c - 1) / c) * c; + } } // StreamKClusterReduction (gfx1250): fixed even split / one-tile-per-cluster