[AMDGPU] Prevent GFX11 VALU Hazard Wait merging into terminators (#214935)

Fix an issue where a pending wait would be moved into the block
terminators causing a validation error.
Flush all pending waits and exit optimization loop when reaching first
terminator within a block.

Author: perlfu

llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp

@@ -478,9 +478,6 @@ class AMDGPUWaitSGPRHazards {
       };
 
       for (MachineInstr &MI : MBB) {
-        if (MI.isMetaInstruction())
-          continue;
-
         if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
             (MI.getOperand(0).getImm() & ConstantMaskBits) ==
                 ConstantMaskBits) {
@@ -500,12 +497,20 @@ class AMDGPUWaitSGPRHazards {
           continue;
         }
 
-        // Do not optimize over branches
-        if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch())) {
+        // Do not optimize over branches or terminators
+        if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch() ||
+                         MI.isTerminator())) {
           PrevWait->moveBefore(&MI);
           PrevWait = nullptr;
           Changed = true;
         }
+        if (MI.isTerminator())
+          break;
+
+        // Note: test for meta instructions after terminators.
+        // Required to handle terminator meta instruction.
+        if (MI.isMetaInstruction())
+          continue;
 
         const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
         const bool IsSALU = SIInstrInfo::isSALU(MI);

llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir

@@ -60,6 +60,7 @@
   define amdgpu_gs void @mask_hazard_optimize1() { ret void }
   define amdgpu_gs void @mask_hazard_optimize2() { ret void }
   define amdgpu_gs void @mask_hazard_optimize3() { ret void }
+  define amdgpu_gs void @mask_hazard_optimize_terminators() { ret void }
 ...
 
 ---
@@ -1268,3 +1269,53 @@ body:            |
     $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
     $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
 ...
+
+---
+name:            mask_hazard_optimize_terminators
+body:            |
+  ; GFX11-LABEL: name: mask_hazard_optimize_terminators
+  ; GFX11: bb.0:
+  ; GFX11-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX11-NEXT: {{  $}}
+  ; GFX11-NEXT:   $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX11-NEXT:   V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+  ; GFX11-NEXT:   $sgpr4 = S_MOV_B32 $sgpr1
+  ; GFX11-NEXT:   $sgpr5 = S_MOV_B32 $sgpr2
+  ; GFX11-NEXT:   S_WAITCNT_DEPCTR .VaVcc_0
+  ; GFX11-NEXT:   $sgpr6 = S_MOV_B32_term $sgpr4
+  ; GFX11-NEXT:   $sgpr7 = S_MOV_B32_term $sgpr5
+  ; GFX11-NEXT:   S_BRANCH %bb.1
+  ; GFX11-NEXT: {{  $}}
+  ; GFX11-NEXT: bb.1:
+  ; GFX11-NEXT:   $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX11-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX12-LABEL: name: mask_hazard_optimize_terminators
+  ; GFX12: bb.0:
+  ; GFX12-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX12-NEXT: {{  $}}
+  ; GFX12-NEXT:   $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX12-NEXT:   V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+  ; GFX12-NEXT:   $sgpr4 = S_MOV_B32 $sgpr1
+  ; GFX12-NEXT:   $sgpr5 = S_MOV_B32 $sgpr2
+  ; GFX12-NEXT:   $sgpr6 = S_MOV_B32_term $sgpr4
+  ; GFX12-NEXT:   $sgpr7 = S_MOV_B32_term $sgpr5
+  ; GFX12-NEXT:   S_BRANCH %bb.1
+  ; GFX12-NEXT: {{  $}}
+  ; GFX12-NEXT: bb.1:
+  ; GFX12-NEXT:   S_WAITCNT_DEPCTR .VaVcc_0
+  ; GFX12-NEXT:   $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+  ; GFX12-NEXT:   S_ENDPGM 0
+  bb.0:
+    $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+    V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+    $sgpr4 = S_MOV_B32 $sgpr1
+    $sgpr5 = S_MOV_B32 $sgpr2
+    $sgpr6 = S_MOV_B32_term $sgpr4
+    $sgpr7 = S_MOV_B32_term $sgpr5
+    S_BRANCH %bb.1
+
+  bb.1:
+    $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+    S_ENDPGM 0
+...