[AMDGPU][AMDGPUDemoteSCCBranchToExecz] demote s_cbranch_scc0/1 branches into vcmp + s_cbranch_execz branches

llvm/lib/Target/AMDGPU/AMDGPU.h

@@ -373,6 +373,9 @@ extern char &AMDGPUCodeGenPrepareID;
 void initializeAMDGPURemoveIncompatibleFunctionsPass(PassRegistry &);
 extern char &AMDGPURemoveIncompatibleFunctionsID;
 
+void initializeAMDGPUDemoteSCCBranchToExeczLegacyPass(PassRegistry &);
+extern char &AMDGPUDemoteSCCBranchToExeczLegacyID;
+
 void initializeAMDGPULateCodeGenPrepareLegacyPass(PassRegistry &);
 extern char &AMDGPULateCodeGenPrepareLegacyID;
 

llvm/lib/Target/AMDGPU/AMDGPUDemoteSCCBranchToExecz.cpp

@@ -0,0 +1,251 @@
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+#include "AMDGPU.h"
+#include "AMDGPUDemoteSCCBranchToExecz.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+
+using namespace llvm;
+
+namespace {
+#define DEBUG_TYPE "amdgpu-demote-scc-to-execz"
+const char PassName[] = "AMDGPU s_cbranch_scc to s_cbranch_execz conversion";
+
+std::optional<unsigned> getVALUOpc(const MachineInstr &MI,
+                                   bool Reverse = false) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
+#define HandleOpcAndReverse(Opc, ReverseOpc, VOpc, ReverseVOpc)                \
+  case Opc:                                                                    \
+    return Reverse ? ReverseVOpc : VOpc;                                       \
+  case ReverseOpc:                                                             \
+    return Reverse ? VOpc : ReverseVOpc
+    HandleOpcAndReverse(AMDGPU::S_CMP_EQ_I32, AMDGPU::S_CMP_LG_I32,
+                        AMDGPU::V_CMP_EQ_I32_e64, AMDGPU::V_CMP_NE_I32_e64);
+    HandleOpcAndReverse(AMDGPU::S_CMP_EQ_U32, AMDGPU::S_CMP_LG_U32,
+                        AMDGPU::V_CMP_EQ_U32_e64, AMDGPU::V_CMP_NE_U32_e64);
+    HandleOpcAndReverse(AMDGPU::S_CMP_GT_I32, AMDGPU::S_CMP_LE_I32,
+                        AMDGPU::V_CMP_GT_I32_e64, AMDGPU::V_CMP_LE_I32_e64);
+    HandleOpcAndReverse(AMDGPU::S_CMP_GT_U32, AMDGPU::S_CMP_LE_U32,
+                        AMDGPU::V_CMP_GT_U32_e64, AMDGPU::V_CMP_LE_U32_e64);
+    HandleOpcAndReverse(AMDGPU::S_CMP_GE_I32, AMDGPU::S_CMP_LT_I32,
+                        AMDGPU::V_CMP_GE_I32_e64, AMDGPU::V_CMP_LT_I32_e64);
+    HandleOpcAndReverse(AMDGPU::S_CMP_GE_U32, AMDGPU::S_CMP_LT_U32,
+                        AMDGPU::V_CMP_GE_U32_e64, AMDGPU::V_CMP_LT_U32_e64);
+    HandleOpcAndReverse(AMDGPU::S_CMP_EQ_U64, AMDGPU::S_CMP_LG_U64,
+                        AMDGPU::V_CMP_EQ_U64_e64, AMDGPU::V_CMP_NE_U64_e64);
+#undef HandleOpcAndReverse
+  default:
+    break;
+  }
+  return std::nullopt;
+}
+
+bool isSCmpPromotableToVCmp(const MachineInstr &MI) {
+  return getVALUOpc(MI).has_value();
+}
+
+bool isTriangular(MachineBasicBlock &Head, MachineBasicBlock *&Then,
+                  MachineBasicBlock *&Tail) {
+  if (Head.succ_size() != 2)
+    return false;
+
+  Then = Head.succ_begin()[0];
+  Tail = Head.succ_begin()[1];
+
+  // Canonicalize so Succ0 has MBB as its single predecessor.
+  if (Then->pred_size() != 1)
+    std::swap(Then, Tail);
+
+  if (Then->pred_size() != 1 || Then->succ_size() != 1)
+    return false;
+
+  return *Then->succ_begin() == Tail;
+}
+
+bool hasPromotableCmpConditon(MachineInstr &Term, MachineInstr *&Cmp) {
+  auto CmpIt = std::next(Term.getReverseIterator());
+  if (CmpIt == Term.getParent()->instr_rend())
+    return false;
+
+  if (!isSCmpPromotableToVCmp(*CmpIt))
+    return false;
+
+  Cmp = &*CmpIt;
+  return true;
+}
+
+bool hasCbranchSCCTerm(MachineBasicBlock &Head, MachineInstr *&Term) {
+  auto TermIt = Head.getFirstInstrTerminator();
+  if (TermIt == Head.end())
+    return false;
+
+  switch (TermIt->getOpcode()) {
+  case AMDGPU::S_CBRANCH_SCC0:
+  case AMDGPU::S_CBRANCH_SCC1:
+    Term = &*TermIt;
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool isTriangularSCCBranch(MachineBasicBlock &Head, MachineInstr *&Term,
+                           MachineInstr *&Cmp, MachineBasicBlock *&Then,
+                           MachineBasicBlock *&Tail) {
+
+  if (!hasCbranchSCCTerm(Head, Term))
+    return false;
+
+  bool SCCIsUsedOutsideHead = any_of(
+      Head.liveouts(), [](const auto &P) { return P.PhysReg == AMDGPU::SCC; });
+  if (SCCIsUsedOutsideHead)
+    return false;
+
+  if (!isTriangular(Head, Then, Tail))
+    return false;
+
+  // phi-nodes in the tail can prevent splicing the instructions of the then
+  // and tail blocks in the head
+  if (!Tail->empty() && Tail->begin()->isPHI())
+    return false;
+
+  if (!hasPromotableCmpConditon(*Term, Cmp))
+    return false;
+
+  return true;
+}
+
+bool SCC1JumpsToThen(const MachineInstr &Term, const MachineBasicBlock &Then) {
+  MachineBasicBlock *TBB = Term.getOperand(0).getMBB();
+  return (TBB == &Then) == (Term.getOpcode() == AMDGPU::S_CBRANCH_SCC1);
+}
+
+class AMDGPUDemoteSCCBranchToExecz {
+  MachineFunction &MF;
+  const GCNSubtarget &ST;
+  const SIInstrInfo &TII;
+  const SIRegisterInfo &RegInfo;
+  const TargetSchedModel &SchedModel;
+
+public:
+  AMDGPUDemoteSCCBranchToExecz(MachineFunction &MF)
+      : MF(MF), ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
+        RegInfo(*ST.getRegisterInfo()), SchedModel(TII.getSchedModel()) {}
+
+  bool mustRetainSCCBranch(const MachineInstr &Term, const MachineInstr &Cmp,
+                           const MachineBasicBlock &Then,
+                           const MachineBasicBlock &Tail) {
+    bool IsWave32 = TII.isWave32();
+    unsigned AndSaveExecOpc =
+        IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
+    unsigned Mov = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
+    unsigned NewOps[] = {*getVALUOpc(Cmp, !SCC1JumpsToThen(Term, Then)),
+                         AndSaveExecOpc, Mov};
+    unsigned NewOpsCost = 0;
+    for (unsigned Opc : NewOps)
+      NewOpsCost += SchedModel.computeInstrLatency(Opc);
+    unsigned OldCmpCost = SchedModel.computeInstrLatency(&Cmp, false);
+
+    assert(NewOpsCost >= OldCmpCost);
+    return !TII.mustRetainExeczBranch(Term, Then, Tail,
+                                      NewOpsCost - OldCmpCost);
+  }
+
+  void demoteCmp(MachineInstr &Term, MachineInstr &Cmp, MachineBasicBlock &Head,
+                 MachineBasicBlock &Then, MachineBasicBlock &Tail) {
+    unsigned NewCmpOpc = *getVALUOpc(Cmp, !SCC1JumpsToThen(Term, Then));
+    Cmp.setDesc(TII.get(NewCmpOpc));
+
+    Cmp.removeOperand(2);
+
+    auto VCC = RegInfo.getVCC();
+    auto Exec = RegInfo.getExec();
+
+    auto &MRI = MF.getRegInfo();
+    MCRegister ExecBackup =
+        MRI.createVirtualRegister(RegInfo.getPhysRegBaseClass(Exec));
+
+    Cmp.insert(Cmp.operands_begin(), MachineOperand::CreateReg(VCC, true));
+    Cmp.addImplicitDefUseOperands(MF);
+
+    TII.legalizeOperands(Cmp);
+
+    bool IsWave32 = TII.isWave32();
+    unsigned AndSaveExecOpc =
+        IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
+    auto SaveAndMaskExec = BuildMI(*Term.getParent(), Term, Cmp.getDebugLoc(),
+                                   TII.get(AndSaveExecOpc), ExecBackup);
+    SaveAndMaskExec.addReg(VCC, RegState::Kill);
+    SaveAndMaskExec->getOperand(3).setIsDead(); // mark SCC as dead
+
+    DebugLoc DL = Term.getDebugLoc();
+    TII.removeBranch(Head);
+    MachineOperand Cond[] = {
+        MachineOperand::CreateImm(SIInstrInfo::BranchPredicate::EXECZ),
+        MachineOperand::CreateReg(RegInfo.getExec(), false)};
+    TII.insertBranch(Head, &Tail, &Then, Cond, DL);
+
+    TII.restoreExec(MF, Tail, Tail.instr_begin(), DebugLoc(), ExecBackup);
+  }
+
+  bool run() {
+    if (!SchedModel.hasInstrSchedModel())
+      return false;
+    bool Changed = false;
+
+    for (MachineBasicBlock &Head : MF) {
+      MachineInstr *Term;
+      MachineInstr *Cmp;
+      MachineBasicBlock *Then;
+      MachineBasicBlock *Tail;
+      if (!isTriangularSCCBranch(Head, Term, Cmp, Then, Tail))
+        continue;
+
+      if (!mustRetainSCCBranch(*Term, *Cmp, *Then, *Tail))
+        continue;
+
+      demoteCmp(*Term, *Cmp, Head, *Then, *Tail);
+      Changed = true;
+    }
+    return Changed;
+  }
+};
+
+class AMDGPUDemoteSCCBranchToExeczLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  AMDGPUDemoteSCCBranchToExeczLegacy() : MachineFunctionPass(ID) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    AMDGPUDemoteSCCBranchToExecz IfCvt{MF};
+    return IfCvt.run();
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override { return PassName; }
+};
+
+char AMDGPUDemoteSCCBranchToExeczLegacy::ID = 0;
+
+} // namespace
+
+PreservedAnalyses llvm::AMDGPUDemoteSCCBranchToExeczPass::run(
+    MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
+  AMDGPUDemoteSCCBranchToExecz IfCvt{MF};
+  if (!IfCvt.run())
+    return PreservedAnalyses::all();
+  return PreservedAnalyses::none();
+}
+
+char &llvm::AMDGPUDemoteSCCBranchToExeczLegacyID =
+    AMDGPUDemoteSCCBranchToExeczLegacy::ID;
+INITIALIZE_PASS_BEGIN(AMDGPUDemoteSCCBranchToExeczLegacy, DEBUG_TYPE, PassName,
+                      false, false)
+INITIALIZE_PASS_END(AMDGPUDemoteSCCBranchToExeczLegacy, DEBUG_TYPE, PassName,
+                    false, false)

llvm/lib/Target/AMDGPU/AMDGPUDemoteSCCBranchToExecz.h

@@ -0,0 +1,31 @@
+//===- AMDGPURDemoteSCCBranchToExecz.h --- demote s_cbranch_scc -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// \brief Pass used to demote s_cbranch_scc0/1 branches to s_cbranch_execz
+/// branches. These can be later removed by SIPreEmitPeephole.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUDEMOTESCCBRANCHTOEXECZ_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUDEMOTESCCBRANCHTOEXECZ_H
+
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+class AMDGPUDemoteSCCBranchToExeczPass
+    : public PassInfoMixin<AMDGPUDemoteSCCBranchToExeczPass> {
+public:
+  AMDGPUDemoteSCCBranchToExeczPass() = default;
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif

llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def

@@ -95,6 +95,7 @@ FUNCTION_PASS_WITH_PARAMS(
 #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
 #endif
 MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
+MACHINE_FUNCTION_PASS("amdgpu-demote-scc-to-execz", AMDGPUDemoteSCCBranchToExeczPass())
 MACHINE_FUNCTION_PASS("si-fix-sgpr-copies", SIFixSGPRCopiesPass())
 MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
 MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

@@ -18,6 +18,7 @@
 #include "AMDGPU.h"
 #include "AMDGPUAliasAnalysis.h"
 #include "AMDGPUCtorDtorLowering.h"
+#include "AMDGPUDemoteSCCBranchToExecz.h"
 #include "AMDGPUExportClustering.h"
 #include "AMDGPUIGroupLP.h"
 #include "AMDGPUISelDAGToDAG.h"
@@ -498,6 +499,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
   initializeAMDGPUUnifyMetadataPass(*PR);
   initializeSIAnnotateControlFlowLegacyPass(*PR);
+  initializeAMDGPUDemoteSCCBranchToExeczLegacyPass(*PR);
   initializeAMDGPUInsertDelayAluPass(*PR);
   initializeSIInsertHardClausesPass(*PR);
   initializeSIInsertWaitcntsPass(*PR);
@@ -1336,7 +1338,7 @@ void GCNPassConfig::addMachineSSAOptimization() {
 bool GCNPassConfig::addILPOpts() {
   if (EnableEarlyIfConversion)
     addPass(&EarlyIfConverterID);
-
+  addPass(&AMDGPUDemoteSCCBranchToExeczLegacyID);
   TargetPassConfig::addILPOpts();
   return false;
 }

llvm/lib/Target/AMDGPU/CMakeLists.txt

@@ -59,6 +59,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUGlobalISelDivergenceLowering.cpp
   AMDGPUGlobalISelUtils.cpp
   AMDGPUHSAMetadataStreamer.cpp
+  AMDGPUDemoteSCCBranchToExecz.cpp
   AMDGPUInsertDelayAlu.cpp
   AMDGPUInstCombineIntrinsic.cpp
   AMDGPUInstrInfo.cpp