#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/DataTypes.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/Dominators.h"
#include <functional>
namespace llvm {
}
};
+/// Attributes of a target dependent hardware loop.
+struct HardwareLoopInfo {
+ HardwareLoopInfo() = delete;
+ HardwareLoopInfo(Loop *L) : L(L) {}
+ Loop *L = nullptr;
+ BasicBlock *ExitBlock = nullptr;
+ BranchInst *ExitBranch = nullptr;
+ const SCEV *ExitCount = nullptr;
+ IntegerType *CountType = nullptr;
+ Value *LoopDecrement = nullptr; // Decrement the loop counter by this
+ // value in every iteration.
+ bool IsNestingLegal = false; // Can a hardware loop be a parent to
+ // another hardware loop?
+ bool CounterInReg = false; // Should loop counter be updated in
+ // the loop via a phi?
+ bool isHardwareLoopCandidate(ScalarEvolution &SE, LoopInfo &LI,
+ DominatorTree &DT, bool ForceNestedLoop = false,
+ bool ForceHardwareLoopPHI = false);
+};
+
/// This pass provides access to the codegen interfaces that are needed
/// for IR-level transformations.
class TargetTransformInfo {
void getUnrollingPreferences(Loop *L, ScalarEvolution &,
UnrollingPreferences &UP) const;
- /// Attributes of a target dependent hardware loop.
- struct HardwareLoopInfo {
- HardwareLoopInfo() = delete;
- HardwareLoopInfo(Loop *L) : L(L) { }
- Loop *L = nullptr;
- BasicBlock *ExitBlock = nullptr;
- BranchInst *ExitBranch = nullptr;
- const SCEV *ExitCount = nullptr;
- IntegerType *CountType = nullptr;
- Value *LoopDecrement = nullptr; // Decrement the loop counter by this
- // value in every iteration.
- bool IsNestingLegal = false; // Can a hardware loop be a parent to
- // another hardware loop?
- bool CounterInReg = false; // Should loop counter be updated in
- // the loop via a phi?
- };
-
/// Query the target whether it would be profitable to convert the given loop
/// into a hardware loop.
bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC,
TargetLibraryInfo *LibInfo,
- TTI::HardwareLoopInfo &HWLoopInfo) {
+ HardwareLoopInfo &HWLoopInfo) {
return false;
}
bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC,
TargetLibraryInfo *LibInfo,
- TTI::HardwareLoopInfo &HWLoopInfo) {
+ HardwareLoopInfo &HWLoopInfo) {
return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
}
};
}
+bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
+ LoopInfo &LI, DominatorTree &DT,
+ bool ForceNestedLoop,
+ bool ForceHardwareLoopPHI) {
+ SmallVector<BasicBlock *, 4> ExitingBlocks;
+ L->getExitingBlocks(ExitingBlocks);
+
+ for (SmallVectorImpl<BasicBlock *>::iterator I = ExitingBlocks.begin(),
+ IE = ExitingBlocks.end();
+ I != IE; ++I) {
+ BasicBlock *BB = *I;
+
+ // If we pass the updated counter back through a phi, we need to know
+ // which latch the updated value will be coming from.
+ if (!L->isLoopLatch(BB)) {
+ if (ForceHardwareLoopPHI || CounterInReg)
+ continue;
+ }
+
+ const SCEV *EC = SE.getExitCount(L, BB);
+ if (isa<SCEVCouldNotCompute>(EC))
+ continue;
+ if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) {
+ if (ConstEC->getValue()->isZero())
+ continue;
+ } else if (!SE.isLoopInvariant(EC, L))
+ continue;
+
+ if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth())
+ continue;
+
+ // If this exiting block is contained in a nested loop, it is not eligible
+ // for insertion of the branch-and-decrement since the inner loop would
+ // end up messing up the value in the CTR.
+ if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
+ continue;
+
+ // We now have a loop-invariant count of loop iterations (which is not the
+ // constant zero) for which we know that this loop will not exit via this
+ // existing block.
+
+ // We need to make sure that this block will run on every loop iteration.
+ // For this to be true, we must dominate all blocks with backedges. Such
+ // blocks are in-loop predecessors to the header block.
+ bool NotAlways = false;
+ for (pred_iterator PI = pred_begin(L->getHeader()),
+ PIE = pred_end(L->getHeader());
+ PI != PIE; ++PI) {
+ if (!L->contains(*PI))
+ continue;
+
+ if (!DT.dominates(*I, *PI)) {
+ NotAlways = true;
+ break;
+ }
+ }
+
+ if (NotAlways)
+ continue;
+
+ // Make sure this blocks ends with a conditional branch.
+ Instruction *TI = BB->getTerminator();
+ if (!TI)
+ continue;
+
+ if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
+ if (!BI->isConditional())
+ continue;
+
+ ExitBranch = BI;
+ } else
+ continue;
+
+ // Note that this block may not be the loop latch block, even if the loop
+ // has a latch block.
+ ExitBlock = *I;
+ ExitCount = EC;
+ break;
+ }
+
+ if (!ExitBlock)
+ return false;
+ return true;
+}
+
TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
: TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
// Given that the target believes the loop to be profitable, try to
// convert it.
- bool TryConvertLoop(TTI::HardwareLoopInfo &HWLoopInfo);
+ bool TryConvertLoop(HardwareLoopInfo &HWLoopInfo);
private:
ScalarEvolution *SE = nullptr;
void UpdateBranch(Value *EltsRem);
public:
- HardwareLoop(TTI::HardwareLoopInfo &Info, ScalarEvolution &SE,
+ HardwareLoop(HardwareLoopInfo &Info, ScalarEvolution &SE,
const DataLayout &DL) :
SE(SE), DL(DL), L(Info.L), M(L->getHeader()->getModule()),
ExitCount(Info.ExitCount),
if (containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI))
return false;
- TTI::HardwareLoopInfo HWLoopInfo(L);
+ HardwareLoopInfo HWLoopInfo(L);
if (TTI->isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo) ||
ForceHardwareLoops) {
return false;
}
-bool HardwareLoops::TryConvertLoop(TTI::HardwareLoopInfo &HWLoopInfo) {
+bool HardwareLoops::TryConvertLoop(HardwareLoopInfo &HWLoopInfo) {
Loop *L = HWLoopInfo.L;
LLVM_DEBUG(dbgs() << "HWLoops: Try to convert profitable loop: " << *L);
- SmallVector<BasicBlock*, 4> ExitingBlocks;
- L->getExitingBlocks(ExitingBlocks);
-
- for (SmallVectorImpl<BasicBlock *>::iterator I = ExitingBlocks.begin(),
- IE = ExitingBlocks.end(); I != IE; ++I) {
- BasicBlock *BB = *I;
-
- // If we pass the updated counter back through a phi, we need to know
- // which latch the updated value will be coming from.
- if (!L->isLoopLatch(BB)) {
- if ((ForceHardwareLoopPHI.getNumOccurrences() && ForceHardwareLoopPHI) ||
- HWLoopInfo.CounterInReg)
- continue;
- }
-
- const SCEV *EC = SE->getExitCount(L, BB);
- if (isa<SCEVCouldNotCompute>(EC))
- continue;
- if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) {
- if (ConstEC->getValue()->isZero())
- continue;
- } else if (!SE->isLoopInvariant(EC, L))
- continue;
-
- if (SE->getTypeSizeInBits(EC->getType()) >
- HWLoopInfo.CountType->getBitWidth())
- continue;
-
- // If this exiting block is contained in a nested loop, it is not eligible
- // for insertion of the branch-and-decrement since the inner loop would
- // end up messing up the value in the CTR.
- if (!HWLoopInfo.IsNestingLegal && LI->getLoopFor(BB) != L &&
- !ForceNestedLoop)
- continue;
-
- // We now have a loop-invariant count of loop iterations (which is not the
- // constant zero) for which we know that this loop will not exit via this
- // existing block.
-
- // We need to make sure that this block will run on every loop iteration.
- // For this to be true, we must dominate all blocks with backedges. Such
- // blocks are in-loop predecessors to the header block.
- bool NotAlways = false;
- for (pred_iterator PI = pred_begin(L->getHeader()),
- PIE = pred_end(L->getHeader()); PI != PIE; ++PI) {
- if (!L->contains(*PI))
- continue;
-
- if (!DT->dominates(*I, *PI)) {
- NotAlways = true;
- break;
- }
- }
-
- if (NotAlways)
- continue;
-
- // Make sure this blocks ends with a conditional branch.
- Instruction *TI = BB->getTerminator();
- if (!TI)
- continue;
-
- if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
- if (!BI->isConditional())
- continue;
-
- HWLoopInfo.ExitBranch = BI;
- } else
- continue;
-
- // Note that this block may not be the loop latch block, even if the loop
- // has a latch block.
- HWLoopInfo.ExitBlock = *I;
- HWLoopInfo.ExitCount = EC;
- break;
- }
-
- if (!HWLoopInfo.ExitBlock)
+ if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT, ForceNestedLoop,
+ ForceHardwareLoopPHI))
return false;
+ assert(
+ (HWLoopInfo.ExitBlock && HWLoopInfo.ExitBranch && HWLoopInfo.ExitCount) &&
+ "Hardware Loop must have set exit info.");
+
BasicBlock *Preheader = L->getLoopPreheader();
// If we don't have a preheader, then insert one.
bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC,
TargetLibraryInfo *LibInfo,
- TTI::HardwareLoopInfo &HWLoopInfo) {
+ HardwareLoopInfo &HWLoopInfo) {
// Low-overhead branches are only supported in the 'low-overhead branch'
// extension of v8.1-m.
if (!ST->hasLOB() || DisableLowOverheadLoops)
bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC,
TargetLibraryInfo *LibInfo,
- TTI::HardwareLoopInfo &HWLoopInfo);
+ HardwareLoopInfo &HWLoopInfo);
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP);
bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC,
TargetLibraryInfo *LibInfo,
- TTI::HardwareLoopInfo &HWLoopInfo) {
+ HardwareLoopInfo &HWLoopInfo) {
const PPCTargetMachine &TM = ST->getTargetMachine();
TargetSchedModel SchedModel;
SchedModel.init(ST);
bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC,
TargetLibraryInfo *LibInfo,
- TTI::HardwareLoopInfo &HWLoopInfo);
+ HardwareLoopInfo &HWLoopInfo);
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP);
projects / llvm / commitdiff