[MCA] Refactor the logic that computes the critical memory dependency info. NFCI

CriticalRegDep has been renamed CriticalDependency, and it is now used by class
Instruction to store information about the critical register dependency and the
critical memory dependency. No functional change intendend.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@361737 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/MCA/Instruction.h b/include/llvm/MCA/Instruction.h
index 9ac1fffb4430ca5d88613f3bda84f99f843c60a1..74be06e3c1c089e58007453d7aa689f909dd144f 100644 (file)
--- a/include/llvm/MCA/Instruction.h
+++ b/include/llvm/MCA/Instruction.h
@@ -80,11 +80,10 @@ struct ReadDescriptor {
 
 class ReadState;
 
-/// Longest register dependency.
+/// A critical data dependency descriptor.
 ///
-/// Used internally by WriteState/ReadState/InstructionBase to help with the
-/// computation of the longest register dependency for an instruction.
-struct CriticalRegDep {
+/// Field RegID is set to the invalid register for memory dependencies.
+struct CriticalDependency {
   unsigned IID;
   unsigned RegID;
   unsigned Cycles;
@@ -136,7 +135,7 @@ class WriteState {
   unsigned DependentWriteCyclesLeft;
 
   // Critical register dependency for this write.
-  CriticalRegDep CRD;
+  CriticalDependency CRD;
 
   // A list of dependent reads. Users is a set of dependent
   // reads. A dependent read is added to the set only if CyclesLeft
@@ -166,7 +165,7 @@ public:
     return DependentWriteCyclesLeft;
   }
   const WriteState *getDependentWrite() const { return DependentWrite; }
-  const CriticalRegDep &getCriticalRegDep() const { return CRD; }
+  const CriticalDependency &getCriticalRegDep() const { return CRD; }
 
   // This method adds Use to the set of data dependent reads. IID is the
   // instruction identifier associated with this write. ReadAdvance is the
@@ -244,7 +243,7 @@ class ReadState {
   // propagated to field CyclesLeft.
   unsigned TotalCycles;
   // Longest register dependency.
-  CriticalRegDep CRD;
+  CriticalDependency CRD;
   // This field is set to true only if there are no dependent writes, and
   // there are no `CyclesLeft' to wait.
   bool IsReady;
@@ -263,7 +262,7 @@ public:
   unsigned getSchedClass() const { return RD->SchedClassID; }
   unsigned getRegisterID() const { return RegisterID; }
   unsigned getRegisterFileID() const { return PRFID; }
-  const CriticalRegDep &getCriticalRegDep() const { return CRD; }
+  const CriticalDependency &getCriticalRegDep() const { return CRD; }
 
   bool isPending() const { return !IndependentFromDef && CyclesLeft > 0; }
   bool isReady() const { return IsReady; }
@@ -405,12 +404,8 @@ class InstructionBase {
   // One entry per each implicit and explicit register use.
   SmallVector<ReadState, 4> Uses;
 
-  // Critical register dependency.
-  CriticalRegDep CRD;
-
 public:
-  InstructionBase(const InstrDesc &D)
-      : Desc(D), IsOptimizableMove(false), CRD() {}
+  InstructionBase(const InstrDesc &D) : Desc(D), IsOptimizableMove(false) {}
 
   SmallVectorImpl<WriteState> &getDefs() { return Defs; }
   const ArrayRef<WriteState> getDefs() const { return Defs; }
@@ -420,9 +415,6 @@ public:
 
   unsigned getLatency() const { return Desc.MaxLatency; }
 
-  const CriticalRegDep &getCriticalRegDep() const { return CRD; }
-  const CriticalRegDep &computeCriticalRegDep();
-
   bool hasDependentUsers() const {
     return any_of(Defs,
                   [](const WriteState &Def) { return Def.getNumUsers() > 0; });
@@ -466,14 +458,19 @@ class Instruction : public InstructionBase {
   // Retire Unit token ID for this instruction.
   unsigned RCUTokenID;
 
+  // Critical register dependency.
+  CriticalDependency CriticalRegDep;
+
+  // Critical memory dependency.
+  CriticalDependency CriticalMemDep;
+
   // A bitmask of busy processor resource units.
   // This field is set to zero only if execution is not delayed during this
   // cycle because of unavailable pipeline resources.
   uint64_t CriticalResourceMask;
 
-  // An instruction identifier. This field is only set if execution is delayed
-  // by a memory dependency.
-  unsigned CriticalMemDep;
+  // Used internally by the logic that computes the critical memory dependency.
+  const Instruction *CurrentMemDep;
 
   // True if this instruction has been optimized at register renaming stage.
   bool IsEliminated;
@@ -481,8 +478,8 @@ class Instruction : public InstructionBase {
 public:
   Instruction(const InstrDesc &D)
       : InstructionBase(D), Stage(IS_INVALID), CyclesLeft(UNKNOWN_CYCLES),
-        RCUTokenID(0), CriticalResourceMask(0), CriticalMemDep(0),
-        IsEliminated(false) {}
+        RCUTokenID(0), CriticalRegDep(), CriticalMemDep(),
+        CriticalResourceMask(0), CurrentMemDep(nullptr), IsEliminated(false) {}
 
   unsigned getRCUTokenID() const { return RCUTokenID; }
   int getCyclesLeft() const { return CyclesLeft; }
@@ -523,12 +520,21 @@ public:
     Stage = IS_RETIRED;
   }
 
+  const CriticalDependency &getCriticalRegDep() const { return CriticalRegDep; }
+  const CriticalDependency &getCriticalMemDep() const { return CriticalMemDep; }
+  const CriticalDependency &computeCriticalRegDep();
+
+  void setCriticalMemDep(unsigned IID, unsigned Cycles) {
+    CriticalMemDep.IID = IID;
+    CriticalMemDep.Cycles = Cycles;
+  }
+  const Instruction *getCurrentMemDep() const { return CurrentMemDep; }
+  void setCurrentMemDep(const Instruction *CMD) { CurrentMemDep = CMD; }
+
   uint64_t getCriticalResourceMask() const { return CriticalResourceMask; }
-  unsigned getCriticalMemDep() const { return CriticalMemDep; }
   void setCriticalResourceMask(uint64_t ResourceMask) {
     CriticalResourceMask = ResourceMask;
   }
-  void setCriticalMemDep(unsigned IID) { CriticalMemDep = IID; }
 
   void cycleEvent();
 };

diff --git a/lib/MCA/HardwareUnits/Scheduler.cpp b/lib/MCA/HardwareUnits/Scheduler.cpp
index bf48d9288994cf71cd80dd804a105a44392b54a4..b2928ed1b1223182a5cab539ed671bf99454280d 100644 (file)
--- a/lib/MCA/HardwareUnits/Scheduler.cpp
+++ b/lib/MCA/HardwareUnits/Scheduler.cpp
@@ -82,6 +82,8 @@ void Scheduler::issueInstructionImpl(
   // This updates the internal state of each write.
   IS->execute(IR.getSourceIndex());
 
+  IS->computeCriticalRegDep();
+
   if (IS->isExecuting())
     IssuedSet.emplace_back(IR);
   else if (IS->isExecuted())
@@ -107,6 +109,59 @@ void Scheduler::issueInstruction(
     promoteToReadySet(ReadyInstructions);
 }
 
+static bool initializeCriticalMemDepInfo(InstRef &IR, const LSUnit &LSU) {
+  Instruction &IS = *IR.getInstruction();
+  assert(IS.isMemOp() && "Not a memory operation!");
+
+  // Check if this instruction depends on another memory operation.
+  InstRef DependentMemOp = LSU.isReady(IR);
+  const Instruction *MemOp = DependentMemOp.getInstruction();
+  IS.setCurrentMemDep(MemOp);
+
+  // Initialize the CriticalMemDep structure.
+  unsigned Cycles = 0;
+  if (MemOp->isExecuting())
+    Cycles = static_cast<unsigned>(MemOp->getCyclesLeft());
+  IS.setCriticalMemDep(DependentMemOp.getSourceIndex(), Cycles);
+  return IR.getSourceIndex() == DependentMemOp.getSourceIndex();
+}
+
+static bool updateMemoryDependencyInfo(InstRef &IR, const LSUnit &LSU) {
+  Instruction &IS = *IR.getInstruction();
+  assert(IS.isMemOp() && "Not a memory operation!");
+
+  const Instruction *MemOp = IS.getCurrentMemDep();
+  if (!MemOp && initializeCriticalMemDepInfo(IR, LSU))
+    return true;
+
+  MemOp = IS.getCurrentMemDep();
+  if (MemOp == IR.getInstruction())
+    return true;
+
+  const CriticalDependency &CMD = IS.getCriticalMemDep();
+  if (MemOp->isExecuting() && !CMD.Cycles) {
+    // Update the critical memory dependency info.
+    IS.setCriticalMemDep(CMD.IID, MemOp->getCyclesLeft());
+    return false;
+  }
+
+  if (!MemOp->isExecuted() && !MemOp->isRetired())
+    return false;
+
+  // Check if there are still unsolved memory dependencies.
+  InstRef DependentMemOp = LSU.isReady(IR);
+  MemOp = DependentMemOp.getInstruction();
+  IS.setCurrentMemDep(MemOp);
+  if (DependentMemOp == IR)
+    return true;
+
+  unsigned Cycles = 0;
+  if (MemOp->isExecuting())
+    Cycles = static_cast<unsigned>(MemOp->getCyclesLeft());
+  IS.setCriticalMemDep(DependentMemOp.getSourceIndex(), Cycles);
+  return false;
+}
+
 bool Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
   // Scan the set of waiting instructions and promote them to the
   // ready set if operands are all ready.
@@ -116,19 +171,14 @@ bool Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
     if (!IR)
       break;
 
-    // Check if there are still unsolved memory dependencies.
+    // Check if there are unsolved memory dependencies.
     Instruction &IS = *IR.getInstruction();
-    if (IS.isMemOp()) {
-      const InstRef &CriticalMemDep = LSU.isReady(IR);
-      if (CriticalMemDep != IR) {
-        IS.setCriticalMemDep(CriticalMemDep.getSourceIndex());
-        ++I;
-        continue;
-      }
+    if (IS.isMemOp() && !updateMemoryDependencyInfo(IR, LSU)) {
+      ++I;
+      continue;
     }
 
-    // Check if this instruction is now ready. In case, force
-    // a transition in state using method 'update()'.
+    // Check if there are unsolved register dependencies.
     if (!IS.isReady() && !IS.updatePending()) {
       ++I;
       continue;
@@ -301,7 +351,7 @@ bool Scheduler::dispatch(const InstRef &IR) {
   }
 
   // Memory operations that are not in a ready state are initially assigned to
-  // the WaitSet. 
+  // the WaitSet.
   if (!IS.isReady() || (IS.isMemOp() && LSU.isReady(IR) != IR)) {
     LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n");
     WaitSet.push_back(IR);

diff --git a/lib/MCA/Instruction.cpp b/lib/MCA/Instruction.cpp
index 5e2fb771e4f5fb948aadb8dade2bf9dfd03f3df1..001842bca3185585da047a1692ff3d301e0a7074 100644 (file)
--- a/lib/MCA/Instruction.cpp
+++ b/lib/MCA/Instruction.cpp
@@ -18,7 +18,8 @@
 namespace llvm {
 namespace mca {
 
-void WriteState::writeStartEvent(unsigned IID, unsigned RegID, unsigned Cycles) {
+void WriteState::writeStartEvent(unsigned IID, unsigned RegID,
+                                 unsigned Cycles) {
   CRD.IID = IID;
   CRD.RegID = RegID;
   CRD.Cycles = Cycles;
@@ -134,23 +135,24 @@ void WriteRef::dump() const {
 }
 #endif
 
-const CriticalRegDep &InstructionBase::computeCriticalRegDep() {
-  if (CRD.Cycles || (Defs.empty() && Uses.empty()))
-    return CRD;
+const CriticalDependency &Instruction::computeCriticalRegDep() {
+  if (CriticalRegDep.Cycles)
+    return CriticalRegDep;
+
   unsigned MaxLatency = 0;
-  for (const WriteState &WS : Defs) {
-    const CriticalRegDep &WriteCRD = WS.getCriticalRegDep();
+  for (const WriteState &WS : getDefs()) {
+    const CriticalDependency &WriteCRD = WS.getCriticalRegDep();
     if (WriteCRD.Cycles > MaxLatency)
-      CRD = WriteCRD;
+      CriticalRegDep = WriteCRD;
   }
 
-  for (const ReadState &RS : Uses) {
-    const CriticalRegDep &ReadCRD = RS.getCriticalRegDep();
+  for (const ReadState &RS : getUses()) {
+    const CriticalDependency &ReadCRD = RS.getCriticalRegDep();
     if (ReadCRD.Cycles > MaxLatency)
-      CRD = ReadCRD;
+      CriticalRegDep = ReadCRD;
   }
 
-  return CRD;
+  return CriticalRegDep;
 }
 
 void Instruction::dispatch(unsigned RCUToken) {

diff --git a/lib/MCA/Stages/DispatchStage.cpp b/lib/MCA/Stages/DispatchStage.cpp
index 80d6da09b5e965125e60da0dcf8535d7b496276e..7334a268e9a6f7faadfc5b0dc4b2a1cea2dfac22 100644 (file)
--- a/lib/MCA/Stages/DispatchStage.cpp
+++ b/lib/MCA/Stages/DispatchStage.cpp
@@ -102,9 +102,6 @@ Error DispatchStage::dispatch(InstRef IR) {
       IS.setEliminated();
   }
 
-  if (IS.isMemOp())
-    IS.setCriticalMemDep(IR.getSourceIndex());
-
   // A dependency-breaking instruction doesn't have to wait on the register
   // input operands, and it is often optimized at register renaming stage.
   // Update RAW dependencies if this instruction is not a dependency-breaking