From: Alina Sbirlea Date: Fri, 12 Apr 2019 19:16:07 +0000 (+0000) Subject: [SCEV] Add option to forget everything in SCEV. X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=3a08fe17897830311a6c3b033b5682a174b8220e;p=llvm [SCEV] Add option to forget everything in SCEV. Summary: Create a method to forget everything in SCEV. Add a cl::opt and PassManagerBuilder option to use this in LoopUnroll. Motivation: Certain Halide applications spend a very long time compiling in forgetLoop, and prefer to forget everything and rebuild SCEV from scratch. Sample difference in compile time reduction: 21.04 to 14.78 using current ToT release build. Testcase showcasing this cannot be opensourced and is fairly large. The option disabled by default, but it may be desirable to enable by default. Evidence in favor (two difference runs on different days/ToT state): File Before (s) After (s) clang-9.bc 7267.91 6639.14 llvm-as.bc 194.12 194.12 llvm-dis.bc 62.50 62.50 opt.bc 1855.85 1857.53 File Before (s) After (s) clang-9.bc 8588.70 7812.83 llvm-as.bc 196.20 194.78 llvm-dis.bc 61.55 61.97 opt.bc 1739.78 1886.26 Reviewers: sanjoy Subscribers: mehdi_amini, jlebar, zzheng, javed.absar, dmgreen, jdoerfert, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D60144 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@358304 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index ba8648b4f44..45dea7dae27 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -782,6 +782,13 @@ public: /// backedge-taken count. bool hasLoopInvariantBackedgeTakenCount(const Loop *L); + // This method should be called by the client when it made any change that + // would invalidate SCEV's answers, and the client wants to remove all loop + // information held internally by ScalarEvolution. This is intended to be used + // when the alternative to forget a loop is too expensive (i.e. large loop + // bodies). + void forgetAllLoops(); + /// This method should be called by the client when it has changed a loop in /// a way that may effect ScalarEvolution's ability to compute a trip count, /// or if the loop is deleted. This call is potentially expensive for large diff --git a/include/llvm/Transforms/IPO/PassManagerBuilder.h b/include/llvm/Transforms/IPO/PassManagerBuilder.h index c5d79165dfa..82b84495106 100644 --- a/include/llvm/Transforms/IPO/PassManagerBuilder.h +++ b/include/llvm/Transforms/IPO/PassManagerBuilder.h @@ -148,6 +148,7 @@ public: bool RerollLoops; bool NewGVN; bool DisableGVNLoadPRE; + bool ForgetAllSCEVInLoopUnroll; bool VerifyInput; bool VerifyOutput; bool MergeFunctions; diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h index 97a4c06cdc9..402ee6f951b 100644 --- a/include/llvm/Transforms/Scalar.h +++ b/include/llvm/Transforms/Scalar.h @@ -183,11 +183,13 @@ Pass *createLoopInstSimplifyPass(); // LoopUnroll - This pass is a simple loop unrolling pass. // Pass *createLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false, - int Threshold = -1, int Count = -1, - int AllowPartial = -1, int Runtime = -1, - int UpperBound = -1, int AllowPeeling = -1); + bool ForgetAllSCEV = false, int Threshold = -1, + int Count = -1, int AllowPartial = -1, + int Runtime = -1, int UpperBound = -1, + int AllowPeeling = -1); // Create an unrolling pass for full unrolling that uses exact trip count only. -Pass *createSimpleLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false); +Pass *createSimpleLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false, + bool ForgetAllSCEV = false); //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/UnrollLoop.h b/include/llvm/Transforms/Utils/UnrollLoop.h index 8f3824e150a..20a3980817a 100644 --- a/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/include/llvm/Transforms/Utils/UnrollLoop.h @@ -67,17 +67,17 @@ LoopUnrollResult UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, bool AllowExpensiveTripCount, bool PreserveCondBr, bool PreserveOnlyFirst, unsigned TripMultiple, unsigned PeelCount, bool UnrollRemainder, - LoopInfo *LI, ScalarEvolution *SE, - DominatorTree *DT, AssumptionCache *AC, - OptimizationRemarkEmitter *ORE, bool PreserveLCSSA, - Loop **RemainderLoop = nullptr); + bool ForgetAllSCEV, LoopInfo *LI, + ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, OptimizationRemarkEmitter *ORE, + bool PreserveLCSSA, Loop **RemainderLoop = nullptr); bool UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool AllowExpensiveTripCount, bool UseEpilogRemainder, bool UnrollRemainder, - LoopInfo *LI, ScalarEvolution *SE, - DominatorTree *DT, AssumptionCache *AC, - bool PreserveLCSSA, + bool ForgetAllSCEV, LoopInfo *LI, + ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, bool PreserveLCSSA, Loop **ResultLoop = nullptr); void computePeelCount(Loop *L, unsigned LoopSize, diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index dff80fbb24d..350eb3d8a7b 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -6789,6 +6789,28 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { return BackedgeTakenCounts.find(L)->second = std::move(Result); } +void ScalarEvolution::forgetAllLoops() { + // This method is intended to forget all info about loops. It should + // invalidate caches as if the following happened: + // - The trip counts of all loops have changed arbitrarily + // - Every llvm::Value has been updated in place to produce a different + // result. + BackedgeTakenCounts.clear(); + PredicatedBackedgeTakenCounts.clear(); + LoopPropertiesCache.clear(); + ConstantEvolutionLoopExitValue.clear(); + ValueExprMap.clear(); + ValuesAtScopes.clear(); + LoopDispositions.clear(); + BlockDispositions.clear(); + UnsignedRanges.clear(); + SignedRanges.clear(); + ExprValueMap.clear(); + HasRecMap.clear(); + MinTrailingZerosCache.clear(); + PredicatedSCEVRewrites.clear(); +} + void ScalarEvolution::forgetLoop(const Loop *L) { // Drop any stored trip count value. auto RemoveLoopFromBackedgeMap = diff --git a/lib/Transforms/IPO/PassManagerBuilder.cpp b/lib/Transforms/IPO/PassManagerBuilder.cpp index 9e241aa335a..7491fe06841 100644 --- a/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -158,6 +158,12 @@ cl::opt EnableOrderFileInstrumentation( "enable-order-file-instrumentation", cl::init(false), cl::Hidden, cl::desc("Enable order file instrumentation (default = off)")); +cl::opt ForgetSCEVInLoopUnroll( + "forget-scev-loop-unroll", cl::init(false), cl::Hidden, + cl::desc("Forget everything in SCEV when doing LoopUnroll, instead of just" + " the current top-most loop. This is somtimes preferred to reduce" + " compile time.")); + PassManagerBuilder::PassManagerBuilder() { OptLevel = 2; SizeLevel = 0; @@ -169,6 +175,7 @@ PassManagerBuilder::PassManagerBuilder() { RerollLoops = RunLoopRerolling; NewGVN = RunNewGVN; DisableGVNLoadPRE = false; + ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll; VerifyInput = false; VerifyOutput = false; MergeFunctions = false; @@ -386,8 +393,9 @@ void PassManagerBuilder::addFunctionSimplificationPasses( if (EnableLoopInterchange) MPM.add(createLoopInterchangePass()); // Interchange loops - MPM.add(createSimpleLoopUnrollPass(OptLevel, - DisableUnrollLoops)); // Unroll small loops + // Unroll small loops + MPM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops, + ForgetAllSCEVInLoopUnroll)); addExtensionsToPM(EP_LoopOptimizerEnd, MPM); // This ends the loop pass pipelines. @@ -724,8 +732,9 @@ void PassManagerBuilder::populateModulePassManager( MPM.add(createLoopUnrollAndJamPass(OptLevel)); } - MPM.add(createLoopUnrollPass(OptLevel, - DisableUnrollLoops)); // Unroll small loops + // Unroll small loops + MPM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops, + ForgetAllSCEVInLoopUnroll)); if (!DisableUnrollLoops) { // LoopUnroll may generate some redundency to cleanup. @@ -919,11 +928,13 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) { if (EnableLoopInterchange) PM.add(createLoopInterchangePass()); - PM.add(createSimpleLoopUnrollPass(OptLevel, - DisableUnrollLoops)); // Unroll small loops + // Unroll small loops + PM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops, + ForgetAllSCEVInLoopUnroll)); PM.add(createLoopVectorizePass(true, !LoopVectorize)); // The vectorizer may have significantly shortened a loop body; unroll again. - PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops)); + PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops, + ForgetAllSCEVInLoopUnroll)); PM.add(createWarnMissedTransformationsPass()); diff --git a/lib/Transforms/Scalar/LoopUnrollPass.cpp b/lib/Transforms/Scalar/LoopUnrollPass.cpp index ef2831d7217..53d4ae84456 100644 --- a/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -964,7 +964,7 @@ static LoopUnrollResult tryToUnrollLoop( Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, const TargetTransformInfo &TTI, AssumptionCache &AC, OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, int OptLevel, - bool OnlyWhenForced, Optional ProvidedCount, + bool OnlyWhenForced, bool ForgetAllSCEV, Optional ProvidedCount, Optional ProvidedThreshold, Optional ProvidedAllowPartial, Optional ProvidedRuntime, Optional ProvidedUpperBound, Optional ProvidedAllowPeeling) { @@ -1082,7 +1082,7 @@ static LoopUnrollResult tryToUnrollLoop( LoopUnrollResult UnrollResult = UnrollLoop( L, UP.Count, TripCount, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount, UseUpperBound, MaxOrZero, TripMultiple, UP.PeelCount, UP.UnrollRemainder, - LI, &SE, &DT, &AC, &ORE, PreserveLCSSA, &RemainderLoop); + ForgetAllSCEV, LI, &SE, &DT, &AC, &ORE, PreserveLCSSA, &RemainderLoop); if (UnrollResult == LoopUnrollResult::Unmodified) return LoopUnrollResult::Unmodified; @@ -1131,6 +1131,11 @@ public: /// metadata are considered. All other loops are skipped. bool OnlyWhenForced; + /// If false, when SCEV is invalidated, only forget everything in the + /// top-most loop (call forgetTopMostLoop), of the loop being processed. + /// Otherwise, forgetAllLoops and rebuild when needed next. + bool ForgetAllSCEV; + Optional ProvidedCount; Optional ProvidedThreshold; Optional ProvidedAllowPartial; @@ -1139,15 +1144,16 @@ public: Optional ProvidedAllowPeeling; LoopUnroll(int OptLevel = 2, bool OnlyWhenForced = false, - Optional Threshold = None, + bool ForgetAllSCEV = false, Optional Threshold = None, Optional Count = None, Optional AllowPartial = None, Optional Runtime = None, Optional UpperBound = None, Optional AllowPeeling = None) : LoopPass(ID), OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced), - ProvidedCount(std::move(Count)), ProvidedThreshold(Threshold), - ProvidedAllowPartial(AllowPartial), ProvidedRuntime(Runtime), - ProvidedUpperBound(UpperBound), ProvidedAllowPeeling(AllowPeeling) { + ForgetAllSCEV(ForgetAllSCEV), ProvidedCount(std::move(Count)), + ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial), + ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound), + ProvidedAllowPeeling(AllowPeeling) { initializeLoopUnrollPass(*PassRegistry::getPassRegistry()); } @@ -1171,8 +1177,8 @@ public: LoopUnrollResult Result = tryToUnrollLoop( L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel, OnlyWhenForced, - ProvidedCount, ProvidedThreshold, ProvidedAllowPartial, ProvidedRuntime, - ProvidedUpperBound, ProvidedAllowPeeling); + ForgetAllSCEV, ProvidedCount, ProvidedThreshold, ProvidedAllowPartial, + ProvidedRuntime, ProvidedUpperBound, ProvidedAllowPeeling); if (Result == LoopUnrollResult::FullyUnrolled) LPM.markLoopAsDeleted(*L); @@ -1202,14 +1208,14 @@ INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_END(LoopUnroll, "loop-unroll", "Unroll loops", false, false) Pass *llvm::createLoopUnrollPass(int OptLevel, bool OnlyWhenForced, - int Threshold, int Count, int AllowPartial, - int Runtime, int UpperBound, + bool ForgetAllSCEV, int Threshold, int Count, + int AllowPartial, int Runtime, int UpperBound, int AllowPeeling) { // TODO: It would make more sense for this function to take the optionals // directly, but that's dangerous since it would silently break out of tree // callers. return new LoopUnroll( - OptLevel, OnlyWhenForced, + OptLevel, OnlyWhenForced, ForgetAllSCEV, Threshold == -1 ? None : Optional(Threshold), Count == -1 ? None : Optional(Count), AllowPartial == -1 ? None : Optional(AllowPartial), @@ -1218,8 +1224,10 @@ Pass *llvm::createLoopUnrollPass(int OptLevel, bool OnlyWhenForced, AllowPeeling == -1 ? None : Optional(AllowPeeling)); } -Pass *llvm::createSimpleLoopUnrollPass(int OptLevel, bool OnlyWhenForced) { - return createLoopUnrollPass(OptLevel, OnlyWhenForced, -1, -1, 0, 0, 0, 0); +Pass *llvm::createSimpleLoopUnrollPass(int OptLevel, bool OnlyWhenForced, + bool ForgetAllSCEV) { + return createLoopUnrollPass(OptLevel, OnlyWhenForced, ForgetAllSCEV, -1, -1, + 0, 0, 0, 0); } PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM, @@ -1250,7 +1258,7 @@ PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM, bool Changed = tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, *ORE, /*PreserveLCSSA*/ true, OptLevel, OnlyWhenForced, - /*Count*/ None, + /*ForgetAllSCEV*/ false, /*Count*/ None, /*Threshold*/ None, /*AllowPartial*/ false, /*Runtime*/ false, /*UpperBound*/ false, /*AllowPeeling*/ false) != LoopUnrollResult::Unmodified; @@ -1388,7 +1396,7 @@ PreservedAnalyses LoopUnrollPass::run(Function &F, LoopUnrollResult Result = tryToUnrollLoop( &L, DT, &LI, SE, TTI, AC, ORE, /*PreserveLCSSA*/ true, UnrollOpts.OptLevel, UnrollOpts.OnlyWhenForced, - /*Count*/ None, + /*ForgetAllSCEV*/ false, /*Count*/ None, /*Threshold*/ None, UnrollOpts.AllowPartial, UnrollOpts.AllowRuntime, UnrollOpts.AllowUpperBound, LocalAllowPeeling); Changed |= Result != LoopUnrollResult::Unmodified; diff --git a/lib/Transforms/Utils/LoopUnroll.cpp b/lib/Transforms/Utils/LoopUnroll.cpp index 0d62a81c923..fa8a0ab342c 100644 --- a/lib/Transforms/Utils/LoopUnroll.cpp +++ b/lib/Transforms/Utils/LoopUnroll.cpp @@ -335,8 +335,9 @@ LoopUnrollResult llvm::UnrollLoop( Loop *L, unsigned Count, unsigned TripCount, bool Force, bool AllowRuntime, bool AllowExpensiveTripCount, bool PreserveCondBr, bool PreserveOnlyFirst, unsigned TripMultiple, unsigned PeelCount, bool UnrollRemainder, - LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, - OptimizationRemarkEmitter *ORE, bool PreserveLCSSA, Loop **RemainderLoop) { + bool ForgetAllSCEV, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, OptimizationRemarkEmitter *ORE, bool PreserveLCSSA, + Loop **RemainderLoop) { BasicBlock *Preheader = L->getLoopPreheader(); if (!Preheader) { @@ -469,8 +470,9 @@ LoopUnrollResult llvm::UnrollLoop( if (RuntimeTripCount && TripMultiple % Count != 0 && !UnrollRuntimeLoopRemainder(L, Count, AllowExpensiveTripCount, - EpilogProfitability, UnrollRemainder, LI, SE, - DT, AC, PreserveLCSSA, RemainderLoop)) { + EpilogProfitability, UnrollRemainder, + ForgetAllSCEV, LI, SE, DT, AC, PreserveLCSSA, + RemainderLoop)) { if (Force) RuntimeTripCount = false; else { @@ -554,8 +556,12 @@ LoopUnrollResult llvm::UnrollLoop( // and if something changes inside them then any of outer loops may also // change. When we forget outermost loop, we also forget all contained loops // and this is what we need here. - if (SE) - SE->forgetTopmostLoop(L); + if (SE) { + if (ForgetAllSCEV) + SE->forgetAllLoops(); + else + SE->forgetTopmostLoop(L); + } bool ContinueOnTrue = L->contains(BI->getSuccessor(0)); BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue); diff --git a/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/lib/Transforms/Utils/LoopUnrollAndJam.cpp index d5ccadde3f9..a53ee9d2687 100644 --- a/lib/Transforms/Utils/LoopUnrollAndJam.cpp +++ b/lib/Transforms/Utils/LoopUnrollAndJam.cpp @@ -197,8 +197,8 @@ LoopUnrollResult llvm::UnrollAndJamLoop( if (TripMultiple == 1 || TripMultiple % Count != 0) { if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false, /*UseEpilogRemainder*/ true, - UnrollRemainder, LI, SE, DT, AC, true, - EpilogueLoop)) { + UnrollRemainder, /*ForgetAllSCEV*/ false, + LI, SE, DT, AC, true, EpilogueLoop)) { LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be " "generated when assuming runtime trip count\n"); return LoopUnrollResult::Unmodified; diff --git a/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/lib/Transforms/Utils/LoopUnrollRuntime.cpp index 2f1a759870c..20d687fbcc8 100644 --- a/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -553,10 +553,10 @@ static bool canProfitablyUnrollMultiExitLoop( bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool AllowExpensiveTripCount, bool UseEpilogRemainder, - bool UnrollRemainder, LoopInfo *LI, - ScalarEvolution *SE, DominatorTree *DT, - AssumptionCache *AC, bool PreserveLCSSA, - Loop **ResultLoop) { + bool UnrollRemainder, bool ForgetAllSCEV, + LoopInfo *LI, ScalarEvolution *SE, + DominatorTree *DT, AssumptionCache *AC, + bool PreserveLCSSA, Loop **ResultLoop) { LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n"); LLVM_DEBUG(L->dump()); LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n" @@ -953,8 +953,8 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, /*Force*/ false, /*AllowRuntime*/ false, /*AllowExpensiveTripCount*/ false, /*PreserveCondBr*/ true, /*PreserveOnlyFirst*/ false, /*TripMultiple*/ 1, - /*PeelCount*/ 0, /*UnrollRemainder*/ false, LI, SE, DT, AC, - /*ORE*/ nullptr, PreserveLCSSA); + /*PeelCount*/ 0, /*UnrollRemainder*/ false, ForgetAllSCEV, + LI, SE, DT, AC, /*ORE*/ nullptr, PreserveLCSSA); } if (ResultLoop && UnrollResult != LoopUnrollResult::FullyUnrolled) diff --git a/unittests/Transforms/Utils/UnrollLoopTest.cpp b/unittests/Transforms/Utils/UnrollLoopTest.cpp index 319498479c3..fba21a76867 100644 --- a/unittests/Transforms/Utils/UnrollLoopTest.cpp +++ b/unittests/Transforms/Utils/UnrollLoopTest.cpp @@ -70,6 +70,7 @@ while.end: ; preds = %while.cond bool PreserveLCSSA = L->isRecursivelyLCSSAForm(DT,LI); - bool ret = UnrollRuntimeLoopRemainder(L, 4, true, false, false, &LI, &SE, &DT, &AC, PreserveLCSSA); + bool ret = UnrollRuntimeLoopRemainder(L, 4, true, false, false, false, &LI, + &SE, &DT, &AC, PreserveLCSSA); EXPECT_FALSE(ret); }