From aebcb9ce1d00dbc1686cec915bb4b44ba902e28d Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Thu, 3 Aug 2017 17:52:38 +0000 Subject: [PATCH] Disable loop peeling during full unrolling pass. Summary: Peeling should not occur during the full unrolling invocation early in the pipeline, but rather later with partial and runtime loop unrolling. The later loop unrolling invocation will also eventually utilize profile summary and branch frequency information, which we would like to use to control peeling. And for ThinLTO we want to delay peeling until the backend (post thin link) phase, just as we do for most types of unrolling. Ensure peeling doesn't occur during the full unrolling invocation by adding a parameter to the shared implementation function, similar to the way partial and runtime loop unrolling are disabled. Performance results for ThinLTO suggest this has a neutral to positive effect on some internal benchmarks. Reviewers: chandlerc, davidxl Subscribers: mzolotukhin, llvm-commits, mehdi_amini Differential Revision: https://reviews.llvm.org/D36258 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@309966 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/Transforms/Scalar.h | 2 +- lib/Transforms/Scalar/LoopUnrollPass.cpp | 47 ++++++++++++--------- test/Transforms/LoopUnroll/peel-loop-pgo.ll | 5 +++ 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h index 1913a9d5da0..7bc522a9653 100644 --- a/include/llvm/Transforms/Scalar.h +++ b/include/llvm/Transforms/Scalar.h @@ -184,7 +184,7 @@ Pass *createLoopInstSimplifyPass(); // Pass *createLoopUnrollPass(int OptLevel = 2, int Threshold = -1, int Count = -1, int AllowPartial = -1, int Runtime = -1, - int UpperBound = -1); + int UpperBound = -1, int AllowPeeling = -1); // Create an unrolling pass for full unrolling that uses exact trip count only. Pass *createSimpleLoopUnrollPass(int OptLevel = 2); diff --git a/lib/Transforms/Scalar/LoopUnrollPass.cpp b/lib/Transforms/Scalar/LoopUnrollPass.cpp index d86f6ab720c..cd0e4328697 100644 --- a/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -134,7 +134,7 @@ static TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences( Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, int OptLevel, Optional UserThreshold, Optional UserCount, Optional UserAllowPartial, Optional UserRuntime, - Optional UserUpperBound) { + Optional UserUpperBound, Optional UserAllowPeeling) { TargetTransformInfo::UnrollingPreferences UP; // Set up the defaults @@ -201,6 +201,8 @@ static TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences( UP.Runtime = *UserRuntime; if (UserUpperBound.hasValue()) UP.UpperBound = *UserUpperBound; + if (UserAllowPeeling.hasValue()) + UP.AllowPeeling = *UserAllowPeeling; return UP; } @@ -927,15 +929,13 @@ static bool computeUnrollCount( return ExplicitUnroll; } -static bool tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, - ScalarEvolution &SE, const TargetTransformInfo &TTI, - AssumptionCache &AC, OptimizationRemarkEmitter &ORE, - bool PreserveLCSSA, int OptLevel, - Optional ProvidedCount, - Optional ProvidedThreshold, - Optional ProvidedAllowPartial, - Optional ProvidedRuntime, - Optional ProvidedUpperBound) { +static bool tryToUnrollLoop( + Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, + const TargetTransformInfo &TTI, AssumptionCache &AC, + OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, int OptLevel, + Optional ProvidedCount, Optional ProvidedThreshold, + Optional ProvidedAllowPartial, Optional ProvidedRuntime, + Optional ProvidedUpperBound, Optional ProvidedAllowPeeling) { DEBUG(dbgs() << "Loop Unroll: F[" << L->getHeader()->getParent()->getName() << "] Loop %" << L->getHeader()->getName() << "\n"); if (HasUnrollDisablePragma(L)) @@ -951,7 +951,8 @@ static bool tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, bool Convergent; TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences( L, SE, TTI, OptLevel, ProvidedThreshold, ProvidedCount, - ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound); + ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound, + ProvidedAllowPeeling); // Exit early if unrolling is disabled. if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0)) return false; @@ -1053,10 +1054,12 @@ public: LoopUnroll(int OptLevel = 2, Optional Threshold = None, Optional Count = None, Optional AllowPartial = None, Optional Runtime = None, - Optional UpperBound = None) + Optional UpperBound = None, + Optional AllowPeeling = None) : LoopPass(ID), OptLevel(OptLevel), ProvidedCount(std::move(Count)), ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial), - ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound) { + ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound), + ProvidedAllowPeeling(AllowPeeling) { initializeLoopUnrollPass(*PassRegistry::getPassRegistry()); } @@ -1066,6 +1069,7 @@ public: Optional ProvidedAllowPartial; Optional ProvidedRuntime; Optional ProvidedUpperBound; + Optional ProvidedAllowPeeling; bool runOnLoop(Loop *L, LPPassManager &) override { if (skipLoop(L)) @@ -1088,7 +1092,7 @@ public: return tryToUnrollLoop(L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel, ProvidedCount, ProvidedThreshold, ProvidedAllowPartial, ProvidedRuntime, - ProvidedUpperBound); + ProvidedUpperBound, ProvidedAllowPeeling); } /// This transformation requires natural loop information & requires that @@ -1112,8 +1116,8 @@ INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_END(LoopUnroll, "loop-unroll", "Unroll loops", false, false) Pass *llvm::createLoopUnrollPass(int OptLevel, int Threshold, int Count, - int AllowPartial, int Runtime, - int UpperBound) { + int AllowPartial, int Runtime, int UpperBound, + int AllowPeeling) { // TODO: It would make more sense for this function to take the optionals // directly, but that's dangerous since it would silently break out of tree // callers. @@ -1122,11 +1126,12 @@ Pass *llvm::createLoopUnrollPass(int OptLevel, int Threshold, int Count, Count == -1 ? None : Optional(Count), AllowPartial == -1 ? None : Optional(AllowPartial), Runtime == -1 ? None : Optional(Runtime), - UpperBound == -1 ? None : Optional(UpperBound)); + UpperBound == -1 ? None : Optional(UpperBound), + AllowPeeling == -1 ? None : Optional(AllowPeeling)); } Pass *llvm::createSimpleLoopUnrollPass(int OptLevel) { - return llvm::createLoopUnrollPass(OptLevel, -1, -1, 0, 0, 0); + return llvm::createLoopUnrollPass(OptLevel, -1, -1, 0, 0, 0, 0); } PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM, @@ -1156,7 +1161,8 @@ PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM, tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, *ORE, /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None, /*Threshold*/ None, /*AllowPartial*/ false, - /*Runtime*/ false, /*UpperBound*/ false); + /*Runtime*/ false, /*UpperBound*/ false, + /*AllowPeeling*/ false); if (!Changed) return PreservedAnalyses::all(); @@ -1278,7 +1284,8 @@ PreservedAnalyses LoopUnrollPass::run(Function &F, bool CurChanged = tryToUnrollLoop( &L, DT, &LI, SE, TTI, AC, ORE, /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None, - /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam); + /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam, + /*AllowPeeling*/ None); Changed |= CurChanged; // The parent must not be damaged by unrolling! diff --git a/test/Transforms/LoopUnroll/peel-loop-pgo.ll b/test/Transforms/LoopUnroll/peel-loop-pgo.ll index 20c3878d03a..ee5846c6325 100644 --- a/test/Transforms/LoopUnroll/peel-loop-pgo.ll +++ b/test/Transforms/LoopUnroll/peel-loop-pgo.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll 2>&1 | FileCheck %s +; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,unroll' 2>&1 | FileCheck %s ; REQUIRES: asserts ; Make sure we use the profile information correctly to peel-off 3 iterations @@ -9,6 +10,10 @@ ; CHECK: Loop Unroll: F[optsize] ; CHECK-NOT: PEELING +; Confirm that no peeling occurs when we are performing full unrolling. +; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=FULLUNROLL +; FULLUNROLL-NOT: PEELING + ; CHECK-LABEL: @basic ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !1 ; CHECK: [[NEXT0]]: -- 2.50.1