From: Sanjoy Das Date: Sun, 10 Sep 2017 03:54:22 +0000 (+0000) Subject: [SCEV] Re-arrange public and private sections to be contiguous; NFC X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=2ff7d65548772f818dc85a9c3a545876f95cbb00;p=llvm [SCEV] Re-arrange public and private sections to be contiguous; NFC git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@312876 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index d63aaf6d055..5409949c6fb 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -482,1259 +482,1257 @@ public: return (SCEV::NoWrapFlags)(Flags & ~OffFlags); } -private: - /// A CallbackVH to arrange for ScalarEvolution to be notified whenever a - /// Value is deleted. - class SCEVCallbackVH final : public CallbackVH { - ScalarEvolution *SE; - - void deleted() override; - void allUsesReplacedWith(Value *New) override; - - public: - SCEVCallbackVH(Value *V, ScalarEvolution *SE = nullptr); - }; - - friend class SCEVCallbackVH; - friend class SCEVExpander; - friend class SCEVUnknown; + ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, + DominatorTree &DT, LoopInfo &LI); + ScalarEvolution(ScalarEvolution &&Arg); + ~ScalarEvolution(); - /// The function we are analyzing. - Function &F; + LLVMContext &getContext() const { return F.getContext(); } - /// Does the module have any calls to the llvm.experimental.guard intrinsic - /// at all? If this is false, we avoid doing work that will only help if - /// thare are guards present in the IR. - bool HasGuards; + /// Test if values of the given type are analyzable within the SCEV + /// framework. This primarily includes integer types, and it can optionally + /// include pointer types if the ScalarEvolution class has access to + /// target-specific information. + bool isSCEVable(Type *Ty) const; - /// The target library information for the target we are targeting. - TargetLibraryInfo &TLI; + /// Return the size in bits of the specified type, for which isSCEVable must + /// return true. + uint64_t getTypeSizeInBits(Type *Ty) const; - /// The tracker for @llvm.assume intrinsics in this function. - AssumptionCache &AC; + /// Return a type with the same bitwidth as the given type and which + /// represents how SCEV will treat the given type, for which isSCEVable must + /// return true. For pointer types, this is the pointer-sized integer type. + Type *getEffectiveSCEVType(Type *Ty) const; - /// The dominator tree. - DominatorTree &DT; + // Returns a wider type among {Ty1, Ty2}. + Type *getWiderType(Type *Ty1, Type *Ty2) const; - /// The loop information for the function we are currently analyzing. - LoopInfo &LI; + /// Return true if the SCEV is a scAddRecExpr or it contains + /// scAddRecExpr. The result will be cached in HasRecMap. + bool containsAddRecurrence(const SCEV *S); - /// This SCEV is used to represent unknown trip counts and things. - std::unique_ptr CouldNotCompute; + /// Erase Value from ValueExprMap and ExprValueMap. + void eraseValueFromMap(Value *V); - /// The type for HasRecMap. - using HasRecMapType = DenseMap; + /// Return a SCEV expression for the full generality of the specified + /// expression. + const SCEV *getSCEV(Value *V); - /// This is a cache to record whether a SCEV contains any scAddRecExpr. - HasRecMapType HasRecMap; + const SCEV *getConstant(ConstantInt *V); + const SCEV *getConstant(const APInt &Val); + const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); + const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty); + const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); + const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); + const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty); + const SCEV *getAddExpr(SmallVectorImpl &Ops, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0); + const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector Ops = {LHS, RHS}; + return getAddExpr(Ops, Flags, Depth); + } + const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector Ops = {Op0, Op1, Op2}; + return getAddExpr(Ops, Flags, Depth); + } + const SCEV *getMulExpr(SmallVectorImpl &Ops, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0); + const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector Ops = {LHS, RHS}; + return getMulExpr(Ops, Flags, Depth); + } + const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector Ops = {Op0, Op1, Op2}; + return getMulExpr(Ops, Flags, Depth); + } + const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUDivExactExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getURemExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, + SCEV::NoWrapFlags Flags); + const SCEV *getAddRecExpr(SmallVectorImpl &Operands, + const Loop *L, SCEV::NoWrapFlags Flags); + const SCEV *getAddRecExpr(const SmallVectorImpl &Operands, + const Loop *L, SCEV::NoWrapFlags Flags) { + SmallVector NewOp(Operands.begin(), Operands.end()); + return getAddRecExpr(NewOp, L, Flags); + } - /// The type for ExprValueMap. - using ValueOffsetPair = std::pair; - using ExprValueMapType = DenseMap>; + /// Checks if \p SymbolicPHI can be rewritten as an AddRecExpr under some + /// Predicates. If successful return these ; + /// The function is intended to be called from PSCEV (the caller will decide + /// whether to actually add the predicates and carry out the rewrites). + Optional>> + createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI); - /// ExprValueMap -- This map records the original values from which - /// the SCEV expr is generated from. - /// - /// We want to represent the mapping as SCEV -> ValueOffsetPair instead - /// of SCEV -> Value: - /// Suppose we know S1 expands to V1, and - /// S1 = S2 + C_a - /// S3 = S2 + C_b - /// where C_a and C_b are different SCEVConstants. Then we'd like to - /// expand S3 as V1 - C_a + C_b instead of expanding S2 literally. - /// It is helpful when S2 is a complex SCEV expr. - /// - /// In order to do that, we represent ExprValueMap as a mapping from - /// SCEV to ValueOffsetPair. We will save both S1->{V1, 0} and - /// S2->{V1, C_a} into the map when we create SCEV for V1. When S3 - /// is expanded, it will first expand S2 to V1 - C_a because of - /// S2->{V1, C_a} in the map, then expand S3 to V1 - C_a + C_b. + /// Returns an expression for a GEP /// - /// Note: S->{V, Offset} in the ExprValueMap means S can be expanded - /// to V - Offset. - ExprValueMapType ExprValueMap; - - /// The type for ValueExprMap. - using ValueExprMapType = - DenseMap>; + /// \p GEP The GEP. The indices contained in the GEP itself are ignored, + /// instead we use IndexExprs. + /// \p IndexExprs The expressions for the indices. + const SCEV *getGEPExpr(GEPOperator *GEP, + const SmallVectorImpl &IndexExprs); + const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getSMaxExpr(SmallVectorImpl &Operands); + const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUMaxExpr(SmallVectorImpl &Operands); + const SCEV *getSMinExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUnknown(Value *V); + const SCEV *getCouldNotCompute(); - /// This is a cache of the values we have analyzed so far. - ValueExprMapType ValueExprMap; + /// Return a SCEV for the constant 0 of a specific type. + const SCEV *getZero(Type *Ty) { return getConstant(Ty, 0); } - /// Mark predicate values currently being processed by isImpliedCond. - SmallPtrSet PendingLoopPredicates; + /// Return a SCEV for the constant 1 of a specific type. + const SCEV *getOne(Type *Ty) { return getConstant(Ty, 1); } - /// Set to true by isLoopBackedgeGuardedByCond when we're walking the set of - /// conditions dominating the backedge of a loop. - bool WalkingBEDominatingConds = false; + /// Return an expression for sizeof AllocTy that is type IntTy + const SCEV *getSizeOfExpr(Type *IntTy, Type *AllocTy); - /// Set to true by isKnownPredicateViaSplitting when we're trying to prove a - /// predicate by splitting it into a set of independent predicates. - bool ProvingSplitPredicate = false; + /// Return an expression for offsetof on the given field with type IntTy + const SCEV *getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo); - /// Memoized values for the GetMinTrailingZeros - DenseMap MinTrailingZerosCache; + /// Return the SCEV object corresponding to -V. + const SCEV *getNegativeSCEV(const SCEV *V, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); - /// Private helper method for the GetMinTrailingZeros method - uint32_t GetMinTrailingZerosImpl(const SCEV *S); + /// Return the SCEV object corresponding to ~V. + const SCEV *getNotSCEV(const SCEV *V); - /// Information about the number of loop iterations for which a loop exit's - /// branch condition evaluates to the not-taken path. This is a temporary - /// pair of exact and max expressions that are eventually summarized in - /// ExitNotTakenInfo and BackedgeTakenInfo. - struct ExitLimit { - const SCEV *ExactNotTaken; // The exit is not taken exactly this many times - const SCEV *MaxNotTaken; // The exit is not taken at most this many times + /// Return LHS-RHS. Minus is represented in SCEV as A+B*-1. + const SCEV *getMinusSCEV(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0); - // Not taken either exactly MaxNotTaken or zero times - bool MaxOrZero = false; + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is zero extended. + const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty); - /// A set of predicate guards for this ExitLimit. The result is only valid - /// if all of the predicates in \c Predicates evaluate to 'true' at - /// run-time. - SmallPtrSet Predicates; + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is sign extended. + const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty); - void addPredicate(const SCEVPredicate *P) { - assert(!isa(P) && "Only add leaf predicates here!"); - Predicates.insert(P); - } + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is zero extended. The + /// conversion must not be narrowing. + const SCEV *getNoopOrZeroExtend(const SCEV *V, Type *Ty); - /*implicit*/ ExitLimit(const SCEV *E); + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is sign extended. The + /// conversion must not be narrowing. + const SCEV *getNoopOrSignExtend(const SCEV *V, Type *Ty); - ExitLimit( - const SCEV *E, const SCEV *M, bool MaxOrZero, - ArrayRef *> PredSetList); + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is extended with + /// unspecified bits. The conversion must not be narrowing. + const SCEV *getNoopOrAnyExtend(const SCEV *V, Type *Ty); - ExitLimit(const SCEV *E, const SCEV *M, bool MaxOrZero, - const SmallPtrSetImpl &PredSet); + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. The conversion must not be widening. + const SCEV *getTruncateOrNoop(const SCEV *V, Type *Ty); - ExitLimit(const SCEV *E, const SCEV *M, bool MaxOrZero); + /// Promote the operands to the wider of the types using zero-extension, and + /// then perform a umax operation with them. + const SCEV *getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); - /// Test whether this ExitLimit contains any computed information, or - /// whether it's all SCEVCouldNotCompute values. - bool hasAnyInfo() const { - return !isa(ExactNotTaken) || - !isa(MaxNotTaken); - } + /// Promote the operands to the wider of the types using zero-extension, and + /// then perform a umin operation with them. + const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); - bool hasOperand(const SCEV *S) const; + /// Transitively follow the chain of pointer-type operands until reaching a + /// SCEV that does not have a single pointer operand. This returns a + /// SCEVUnknown pointer for well-formed pointer-type expressions, but corner + /// cases do exist. + const SCEV *getPointerBase(const SCEV *V); - /// Test whether this ExitLimit contains all information. - bool hasFullInfo() const { - return !isa(ExactNotTaken); - } - }; + /// Return a SCEV expression for the specified value at the specified scope + /// in the program. The L value specifies a loop nest to evaluate the + /// expression at, where null is the top-level or a specified loop is + /// immediately inside of the loop. + /// + /// This method can be used to compute the exit value for a variable defined + /// in a loop by querying what the value will hold in the parent loop. + /// + /// In the case that a relevant loop exit value cannot be computed, the + /// original value V is returned. + const SCEV *getSCEVAtScope(const SCEV *S, const Loop *L); - /// Information about the number of times a particular loop exit may be - /// reached before exiting the loop. - struct ExitNotTakenInfo { - PoisoningVH ExitingBlock; - const SCEV *ExactNotTaken; - std::unique_ptr Predicate; + /// This is a convenience function which does getSCEVAtScope(getSCEV(V), L). + const SCEV *getSCEVAtScope(Value *V, const Loop *L); - explicit ExitNotTakenInfo(PoisoningVH ExitingBlock, - const SCEV *ExactNotTaken, - std::unique_ptr Predicate) - : ExitingBlock(ExitingBlock), ExactNotTaken(ExactNotTaken), - Predicate(std::move(Predicate)) {} + /// Test whether entry to the loop is protected by a conditional between LHS + /// and RHS. This is used to help avoid max expressions in loop trip + /// counts, and to eliminate casts. + bool isLoopEntryGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); - bool hasAlwaysTruePredicate() const { - return !Predicate || Predicate->isAlwaysTrue(); - } - }; + /// Test whether the backedge of the loop is protected by a conditional + /// between LHS and RHS. This is used to to eliminate casts. + bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); - /// Information about the backedge-taken count of a loop. This currently - /// includes an exact count and a maximum count. + /// Returns the maximum trip count of the loop if it is a single-exit + /// loop and we can compute a small maximum for that loop. /// - class BackedgeTakenInfo { - /// A list of computable exits and their not-taken counts. Loops almost - /// never have more than one computable exit. - SmallVector ExitNotTaken; + /// Implemented in terms of the \c getSmallConstantTripCount overload with + /// the single exiting block passed to it. See that routine for details. + unsigned getSmallConstantTripCount(const Loop *L); - /// The pointer part of \c MaxAndComplete is an expression indicating the - /// least maximum backedge-taken count of the loop that is known, or a - /// SCEVCouldNotCompute. This expression is only valid if the predicates - /// associated with all loop exits are true. - /// - /// The integer part of \c MaxAndComplete is a boolean indicating if \c - /// ExitNotTaken has an element for every exiting block in the loop. - PointerIntPair MaxAndComplete; + /// Returns the maximum trip count of this loop as a normal unsigned + /// value. Returns 0 if the trip count is unknown or not constant. This + /// "trip count" assumes that control exits via ExitingBlock. More + /// precisely, it is the number of times that control may reach ExitingBlock + /// before taking the branch. For loops with multiple exits, it may not be + /// the number times that the loop header executes if the loop exits + /// prematurely via another branch. + unsigned getSmallConstantTripCount(const Loop *L, BasicBlock *ExitingBlock); - /// True iff the backedge is taken either exactly Max or zero times. - bool MaxOrZero = false; + /// Returns the upper bound of the loop trip count as a normal unsigned + /// value. + /// Returns 0 if the trip count is unknown or not constant. + unsigned getSmallConstantMaxTripCount(const Loop *L); - /// \name Helper projection functions on \c MaxAndComplete. - /// @{ - bool isComplete() const { return MaxAndComplete.getInt(); } - const SCEV *getMax() const { return MaxAndComplete.getPointer(); } - /// @} + /// Returns the largest constant divisor of the trip count of the + /// loop if it is a single-exit loop and we can compute a small maximum for + /// that loop. + /// + /// Implemented in terms of the \c getSmallConstantTripMultiple overload with + /// the single exiting block passed to it. See that routine for details. + unsigned getSmallConstantTripMultiple(const Loop *L); - public: - BackedgeTakenInfo() : MaxAndComplete(nullptr, 0) {} - BackedgeTakenInfo(BackedgeTakenInfo &&) = default; - BackedgeTakenInfo &operator=(BackedgeTakenInfo &&) = default; + /// Returns the largest constant divisor of the trip count of this loop as a + /// normal unsigned value, if possible. This means that the actual trip + /// count is always a multiple of the returned value (don't forget the trip + /// count could very well be zero as well!). As explained in the comments + /// for getSmallConstantTripCount, this assumes that control exits the loop + /// via ExitingBlock. + unsigned getSmallConstantTripMultiple(const Loop *L, + BasicBlock *ExitingBlock); - using EdgeExitInfo = std::pair; + /// Get the expression for the number of loop iterations for which this loop + /// is guaranteed not to exit via ExitingBlock. Otherwise return + /// SCEVCouldNotCompute. + const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock); - /// Initialize BackedgeTakenInfo from a list of exact exit counts. - BackedgeTakenInfo(SmallVectorImpl &&ExitCounts, bool Complete, - const SCEV *MaxCount, bool MaxOrZero); + /// If the specified loop has a predictable backedge-taken count, return it, + /// otherwise return a SCEVCouldNotCompute object. The backedge-taken count is + /// the number of times the loop header will be branched to from within the + /// loop, assuming there are no abnormal exists like exception throws. This is + /// one less than the trip count of the loop, since it doesn't count the first + /// iteration, when the header is branched to from outside the loop. + /// + /// Note that it is not valid to call this method on a loop without a + /// loop-invariant backedge-taken count (see + /// hasLoopInvariantBackedgeTakenCount). + const SCEV *getBackedgeTakenCount(const Loop *L); - /// Test whether this BackedgeTakenInfo contains any computed information, - /// or whether it's all SCEVCouldNotCompute values. - bool hasAnyInfo() const { - return !ExitNotTaken.empty() || !isa(getMax()); - } + /// Similar to getBackedgeTakenCount, except it will add a set of + /// SCEV predicates to Predicates that are required to be true in order for + /// the answer to be correct. Predicates can be checked with run-time + /// checks and can be used to perform loop versioning. + const SCEV *getPredicatedBackedgeTakenCount(const Loop *L, + SCEVUnionPredicate &Predicates); - /// Test whether this BackedgeTakenInfo contains complete information. - bool hasFullInfo() const { return isComplete(); } + /// When successful, this returns a SCEVConstant that is greater than or equal + /// to (i.e. a "conservative over-approximation") of the value returend by + /// getBackedgeTakenCount. If such a value cannot be computed, it returns the + /// SCEVCouldNotCompute object. + const SCEV *getMaxBackedgeTakenCount(const Loop *L); - /// Return an expression indicating the exact *backedge-taken* - /// count of the loop if it is known or SCEVCouldNotCompute - /// otherwise. If execution makes it to the backedge on every - /// iteration (i.e. there are no abnormal exists like exception - /// throws and thread exits) then this is the number of times the - /// loop header will execute minus one. - /// - /// If the SCEV predicate associated with the answer can be different - /// from AlwaysTrue, we must add a (non null) Predicates argument. - /// The SCEV predicate associated with the answer will be added to - /// Predicates. A run-time check needs to be emitted for the SCEV - /// predicate in order for the answer to be valid. - /// - /// Note that we should always know if we need to pass a predicate - /// argument or not from the way the ExitCounts vector was computed. - /// If we allowed SCEV predicates to be generated when populating this - /// vector, this information can contain them and therefore a - /// SCEVPredicate argument should be added to getExact. - const SCEV *getExact(ScalarEvolution *SE, - SCEVUnionPredicate *Predicates = nullptr) const; + /// Return true if the backedge taken count is either the value returned by + /// getMaxBackedgeTakenCount or zero. + bool isBackedgeTakenCountMaxOrZero(const Loop *L); - /// Return the number of times this loop exit may fall through to the back - /// edge, or SCEVCouldNotCompute. The loop is guaranteed not to exit via - /// this block before this number of iterations, but may exit via another - /// block. - const SCEV *getExact(BasicBlock *ExitingBlock, ScalarEvolution *SE) const; + /// Return true if the specified loop has an analyzable loop-invariant + /// backedge-taken count. + bool hasLoopInvariantBackedgeTakenCount(const Loop *L); - /// Get the max backedge taken count for the loop. - const SCEV *getMax(ScalarEvolution *SE) const; + /// This method should be called by the client when it has changed a loop in + /// a way that may effect ScalarEvolution's ability to compute a trip count, + /// or if the loop is deleted. This call is potentially expensive for large + /// loop bodies. + void forgetLoop(const Loop *L); - /// Return true if the number of times this backedge is taken is either the - /// value returned by getMax or zero. - bool isMaxOrZero(ScalarEvolution *SE) const; + /// This method should be called by the client when it has changed a value + /// in a way that may effect its value, or which may disconnect it from a + /// def-use chain linking it to a loop. + void forgetValue(Value *V); - /// Return true if any backedge taken count expressions refer to the given - /// subexpression. - bool hasOperand(const SCEV *S, ScalarEvolution *SE) const; + /// Called when the client has changed the disposition of values in + /// this loop. + /// + /// We don't have a way to invalidate per-loop dispositions. Clear and + /// recompute is simpler. + void forgetLoopDispositions(const Loop *L) { LoopDispositions.clear(); } - /// Invalidate this result and free associated memory. - void clear(); - }; + /// Determine the minimum number of zero bits that S is guaranteed to end in + /// (at every loop iteration). It is, at the same time, the minimum number + /// of times S is divisible by 2. For example, given {4,+,8} it returns 2. + /// If S is guaranteed to be 0, it returns the bitwidth of S. + uint32_t GetMinTrailingZeros(const SCEV *S); - /// Cache the backedge-taken count of the loops for this function as they - /// are computed. - DenseMap BackedgeTakenCounts; + /// Determine the unsigned range for a particular SCEV. + /// NOTE: This returns a copy of the reference returned by getRangeRef. + ConstantRange getUnsignedRange(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_UNSIGNED); + } - /// Cache the predicated backedge-taken count of the loops for this - /// function as they are computed. - DenseMap PredicatedBackedgeTakenCounts; + /// Determine the min of the unsigned range for a particular SCEV. + APInt getUnsignedRangeMin(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMin(); + } - // Cache the calculated exit limits for the loops. - DenseMap ExitLimits; + /// Determine the max of the unsigned range for a particular SCEV. + APInt getUnsignedRangeMax(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMax(); + } - /// This map contains entries for all of the PHI instructions that we - /// attempt to compute constant evolutions for. This allows us to avoid - /// potentially expensive recomputation of these properties. An instruction - /// maps to null if we are unable to compute its exit value. - DenseMap ConstantEvolutionLoopExitValue; + /// Determine the signed range for a particular SCEV. + /// NOTE: This returns a copy of the reference returned by getRangeRef. + ConstantRange getSignedRange(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_SIGNED); + } - /// This map contains entries for all the expressions that we attempt to - /// compute getSCEVAtScope information for, which can be expensive in - /// extreme cases. - DenseMap, 2>> - ValuesAtScopes; + /// Determine the min of the signed range for a particular SCEV. + APInt getSignedRangeMin(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMin(); + } - /// Memoized computeLoopDisposition results. - DenseMap, 2>> - LoopDispositions; + /// Determine the max of the signed range for a particular SCEV. + APInt getSignedRangeMax(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMax(); + } - struct LoopProperties { - /// Set to true if the loop contains no instruction that can have side - /// effects (i.e. via throwing an exception, volatile or atomic access). - bool HasNoAbnormalExits; + /// Test if the given expression is known to be negative. + bool isKnownNegative(const SCEV *S); - /// Set to true if the loop contains no instruction that can abnormally exit - /// the loop (i.e. via throwing an exception, by terminating the thread - /// cleanly or by infinite looping in a called function). Strictly - /// speaking, the last one is not leaving the loop, but is identical to - /// leaving the loop for reasoning about undefined behavior. - bool HasNoSideEffects; - }; + /// Test if the given expression is known to be positive. + bool isKnownPositive(const SCEV *S); - /// Cache for \c getLoopProperties. - DenseMap LoopPropertiesCache; + /// Test if the given expression is known to be non-negative. + bool isKnownNonNegative(const SCEV *S); - /// Return a \c LoopProperties instance for \p L, creating one if necessary. - LoopProperties getLoopProperties(const Loop *L); + /// Test if the given expression is known to be non-positive. + bool isKnownNonPositive(const SCEV *S); - bool loopHasNoSideEffects(const Loop *L) { - return getLoopProperties(L).HasNoSideEffects; - } + /// Test if the given expression is known to be non-zero. + bool isKnownNonZero(const SCEV *S); - bool loopHasNoAbnormalExits(const Loop *L) { - return getLoopProperties(L).HasNoAbnormalExits; - } + /// Test if the given expression is known to satisfy the condition described + /// by Pred, LHS, and RHS. + bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS); - /// Compute a LoopDisposition value. - LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L); + /// Return true if, for all loop invariant X, the predicate "LHS `Pred` X" + /// is monotonically increasing or decreasing. In the former case set + /// `Increasing` to true and in the latter case set `Increasing` to false. + /// + /// A predicate is said to be monotonically increasing if may go from being + /// false to being true as the loop iterates, but never the other way + /// around. A predicate is said to be monotonically decreasing if may go + /// from being true to being false as the loop iterates, but never the other + /// way around. + bool isMonotonicPredicate(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred, + bool &Increasing); - /// Memoized computeBlockDisposition results. - DenseMap< - const SCEV *, - SmallVector, 2>> - BlockDispositions; + /// Return true if the result of the predicate LHS `Pred` RHS is loop + /// invariant with respect to L. Set InvariantPred, InvariantLHS and + /// InvariantLHS so that InvariantLHS `InvariantPred` InvariantRHS is the + /// loop invariant form of LHS `Pred` RHS. + bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const Loop *L, + ICmpInst::Predicate &InvariantPred, + const SCEV *&InvariantLHS, + const SCEV *&InvariantRHS); - /// Compute a BlockDisposition value. - BlockDisposition computeBlockDisposition(const SCEV *S, const BasicBlock *BB); + /// Simplify LHS and RHS in a comparison with predicate Pred. Return true + /// iff any changes were made. If the operands are provably equal or + /// unequal, LHS and RHS are set to the same value and Pred is set to either + /// ICMP_EQ or ICMP_NE. + bool SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, + const SCEV *&RHS, unsigned Depth = 0); - /// Memoized results from getRange - DenseMap UnsignedRanges; + /// Return the "disposition" of the given SCEV with respect to the given + /// loop. + LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L); - /// Memoized results from getRange - DenseMap SignedRanges; + /// Return true if the value of the given SCEV is unchanging in the + /// specified loop. + bool isLoopInvariant(const SCEV *S, const Loop *L); - /// Used to parameterize getRange - enum RangeSignHint { HINT_RANGE_UNSIGNED, HINT_RANGE_SIGNED }; + /// Determine if the SCEV can be evaluated at loop's entry. It is true if it + /// doesn't depend on a SCEVUnknown of an instruction which is dominated by + /// the header of loop L. + bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L); - /// Set the memoized range for the given SCEV. - const ConstantRange &setRange(const SCEV *S, RangeSignHint Hint, - ConstantRange CR) { - DenseMap &Cache = - Hint == HINT_RANGE_UNSIGNED ? UnsignedRanges : SignedRanges; + /// Return true if the given SCEV changes value in a known way in the + /// specified loop. This property being true implies that the value is + /// variant in the loop AND that we can emit an expression to compute the + /// value of the expression at any particular loop iteration. + bool hasComputableLoopEvolution(const SCEV *S, const Loop *L); - auto Pair = Cache.try_emplace(S, std::move(CR)); - if (!Pair.second) - Pair.first->second = std::move(CR); - return Pair.first->second; - } + /// Return the "disposition" of the given SCEV with respect to the given + /// block. + BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB); - /// Determine the range for a particular SCEV. - /// NOTE: This returns a reference to an entry in a cache. It must be - /// copied if its needed for longer. - const ConstantRange &getRangeRef(const SCEV *S, RangeSignHint Hint); + /// Return true if elements that makes up the given SCEV dominate the + /// specified basic block. + bool dominates(const SCEV *S, const BasicBlock *BB); - /// Determines the range for the affine SCEVAddRecExpr {\p Start,+,\p Stop}. - /// Helper for \c getRange. - ConstantRange getRangeForAffineAR(const SCEV *Start, const SCEV *Stop, - const SCEV *MaxBECount, unsigned BitWidth); + /// Return true if elements that makes up the given SCEV properly dominate + /// the specified basic block. + bool properlyDominates(const SCEV *S, const BasicBlock *BB); - /// Try to compute a range for the affine SCEVAddRecExpr {\p Start,+,\p - /// Stop} by "factoring out" a ternary expression from the add recurrence. - /// Helper called by \c getRange. - ConstantRange getRangeViaFactoring(const SCEV *Start, const SCEV *Stop, - const SCEV *MaxBECount, unsigned BitWidth); + /// Test whether the given SCEV has Op as a direct or indirect operand. + bool hasOperand(const SCEV *S, const SCEV *Op) const; - /// We know that there is no SCEV for the specified value. Analyze the - /// expression. - const SCEV *createSCEV(Value *V); + /// Return the size of an element read or written by Inst. + const SCEV *getElementSize(Instruction *Inst); - /// Provide the special handling we need to analyze PHI SCEVs. - const SCEV *createNodeForPHI(PHINode *PN); + /// Compute the array dimensions Sizes from the set of Terms extracted from + /// the memory access function of this SCEVAddRecExpr (second step of + /// delinearization). + void findArrayDimensions(SmallVectorImpl &Terms, + SmallVectorImpl &Sizes, + const SCEV *ElementSize); - /// Helper function called from createNodeForPHI. - const SCEV *createAddRecFromPHI(PHINode *PN); + void print(raw_ostream &OS) const; + void verify() const; + bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); - /// A helper function for createAddRecFromPHI to handle simple cases. - const SCEV *createSimpleAffineAddRec(PHINode *PN, Value *BEValueV, - Value *StartValueV); + /// Collect parametric terms occurring in step expressions (first step of + /// delinearization). + void collectParametricTerms(const SCEV *Expr, + SmallVectorImpl &Terms); - /// Helper function called from createNodeForPHI. - const SCEV *createNodeFromSelectLikePHI(PHINode *PN); + /// Return in Subscripts the access functions for each dimension in Sizes + /// (third step of delinearization). + void computeAccessFunctions(const SCEV *Expr, + SmallVectorImpl &Subscripts, + SmallVectorImpl &Sizes); - /// Provide special handling for a select-like instruction (currently this - /// is either a select instruction or a phi node). \p I is the instruction - /// being processed, and it is assumed equivalent to "Cond ? TrueVal : - /// FalseVal". - const SCEV *createNodeForSelectOrPHI(Instruction *I, Value *Cond, - Value *TrueVal, Value *FalseVal); + /// Split this SCEVAddRecExpr into two vectors of SCEVs representing the + /// subscripts and sizes of an array access. + /// + /// The delinearization is a 3 step process: the first two steps compute the + /// sizes of each subscript and the third step computes the access functions + /// for the delinearized array: + /// + /// 1. Find the terms in the step functions + /// 2. Compute the array size + /// 3. Compute the access function: divide the SCEV by the array size + /// starting with the innermost dimensions found in step 2. The Quotient + /// is the SCEV to be divided in the next step of the recursion. The + /// Remainder is the subscript of the innermost dimension. Loop over all + /// array dimensions computed in step 2. + /// + /// To compute a uniform array size for several memory accesses to the same + /// object, one can collect in step 1 all the step terms for all the memory + /// accesses, and compute in step 2 a unique array shape. This guarantees + /// that the array shape will be the same across all memory accesses. + /// + /// FIXME: We could derive the result of steps 1 and 2 from a description of + /// the array shape given in metadata. + /// + /// Example: + /// + /// A[][n][m] + /// + /// for i + /// for j + /// for k + /// A[j+k][2i][5i] = + /// + /// The initial SCEV: + /// + /// A[{{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k] + /// + /// 1. Find the different terms in the step functions: + /// -> [2*m, 5, n*m, n*m] + /// + /// 2. Compute the array size: sort and unique them + /// -> [n*m, 2*m, 5] + /// find the GCD of all the terms = 1 + /// divide by the GCD and erase constant terms + /// -> [n*m, 2*m] + /// GCD = m + /// divide by GCD -> [n, 2] + /// remove constant terms + /// -> [n] + /// size of the array is A[unknown][n][m] + /// + /// 3. Compute the access function + /// a. Divide {{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k by the innermost size m + /// Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k + /// Remainder: {{{0,+,5}_i, +, 0}_j, +, 0}_k + /// The remainder is the subscript of the innermost array dimension: [5i]. + /// + /// b. Divide Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k by next outer size n + /// Quotient: {{{0,+,0}_i, +, 1}_j, +, 1}_k + /// Remainder: {{{0,+,2}_i, +, 0}_j, +, 0}_k + /// The Remainder is the subscript of the next array dimension: [2i]. + /// + /// The subscript of the outermost dimension is the Quotient: [j+k]. + /// + /// Overall, we have: A[][n][m], and the access function: A[j+k][2i][5i]. + void delinearize(const SCEV *Expr, SmallVectorImpl &Subscripts, + SmallVectorImpl &Sizes, + const SCEV *ElementSize); - /// Provide the special handling we need to analyze GEP SCEVs. - const SCEV *createNodeForGEP(GEPOperator *GEP); + /// Return the DataLayout associated with the module this SCEV instance is + /// operating on. + const DataLayout &getDataLayout() const { + return F.getParent()->getDataLayout(); + } - /// Implementation code for getSCEVAtScope; called at most once for each - /// SCEV+Loop pair. - const SCEV *computeSCEVAtScope(const SCEV *S, const Loop *L); + const SCEVPredicate *getEqualPredicate(const SCEV *LHS, const SCEV *RHS); - /// This looks up computed SCEV values for all instructions that depend on - /// the given instruction and removes them from the ValueExprMap map if they - /// reference SymName. This is used during PHI resolution. - void forgetSymbolicName(Instruction *I, const SCEV *SymName); + const SCEVPredicate * + getWrapPredicate(const SCEVAddRecExpr *AR, + SCEVWrapPredicate::IncrementWrapFlags AddedFlags); - /// Return the BackedgeTakenInfo for the given loop, lazily computing new - /// values if the loop hasn't been analyzed yet. The returned result is - /// guaranteed not to be predicated. - const BackedgeTakenInfo &getBackedgeTakenInfo(const Loop *L); + /// Re-writes the SCEV according to the Predicates in \p A. + const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L, + SCEVUnionPredicate &A); + /// Tries to convert the \p S expression to an AddRec expression, + /// adding additional predicates to \p Preds as required. + const SCEVAddRecExpr *convertSCEVToAddRecWithPredicates( + const SCEV *S, const Loop *L, + SmallPtrSetImpl &Preds); - /// Similar to getBackedgeTakenInfo, but will add predicates as required - /// with the purpose of returning complete information. - const BackedgeTakenInfo &getPredicatedBackedgeTakenInfo(const Loop *L); +private: + /// A CallbackVH to arrange for ScalarEvolution to be notified whenever a + /// Value is deleted. + class SCEVCallbackVH final : public CallbackVH { + ScalarEvolution *SE; - /// Compute the number of times the specified loop will iterate. - /// If AllowPredicates is set, we will create new SCEV predicates as - /// necessary in order to return an exact answer. - BackedgeTakenInfo computeBackedgeTakenCount(const Loop *L, - bool AllowPredicates = false); - - /// Compute the number of times the backedge of the specified loop will - /// execute if it exits via the specified block. If AllowPredicates is set, - /// this call will try to use a minimal set of SCEV predicates in order to - /// return an exact answer. - ExitLimit computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, - bool AllowPredicates = false); - - ExitLimit computeExitLimitImpl(const Loop *L, BasicBlock *ExitingBlock, - bool AllowPredicates = false); - - /// Compute the number of times the backedge of the specified loop will - /// execute if its exit condition were a conditional branch of ExitCond, - /// TBB, and FBB. - /// - /// \p ControlsExit is true if ExitCond directly controls the exit - /// branch. In this case, we can assume that the loop exits only if the - /// condition is true and can infer that failing to meet the condition prior - /// to integer wraparound results in undefined behavior. - /// - /// If \p AllowPredicates is set, this call will try to use a minimal set of - /// SCEV predicates in order to return an exact answer. - ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, - BasicBlock *TBB, BasicBlock *FBB, - bool ControlsExit, - bool AllowPredicates = false); - - // Helper functions for computeExitLimitFromCond to avoid exponential time - // complexity. - - class ExitLimitCache { - // It may look like we need key on the whole (L, TBB, FBB, ControlsExit, - // AllowPredicates) tuple, but recursive calls to - // computeExitLimitFromCondCached from computeExitLimitFromCondImpl only - // vary the in \c ExitCond and \c ControlsExit parameters. We remember the - // initial values of the other values to assert our assumption. - SmallDenseMap, ExitLimit> TripCountMap; - - const Loop *L; - BasicBlock *TBB; - BasicBlock *FBB; - bool AllowPredicates; + void deleted() override; + void allUsesReplacedWith(Value *New) override; public: - ExitLimitCache(const Loop *L, BasicBlock *TBB, BasicBlock *FBB, - bool AllowPredicates) - : L(L), TBB(TBB), FBB(FBB), AllowPredicates(AllowPredicates) {} + SCEVCallbackVH(Value *V, ScalarEvolution *SE = nullptr); + }; - Optional find(const Loop *L, Value *ExitCond, BasicBlock *TBB, - BasicBlock *FBB, bool ControlsExit, - bool AllowPredicates); + friend class SCEVCallbackVH; + friend class SCEVExpander; + friend class SCEVUnknown; - void insert(const Loop *L, Value *ExitCond, BasicBlock *TBB, - BasicBlock *FBB, bool ControlsExit, bool AllowPredicates, - const ExitLimit &EL); - }; + /// The function we are analyzing. + Function &F; - using ExitLimitCacheTy = ExitLimitCache; + /// Does the module have any calls to the llvm.experimental.guard intrinsic + /// at all? If this is false, we avoid doing work that will only help if + /// thare are guards present in the IR. + bool HasGuards; - ExitLimit computeExitLimitFromCondCached(ExitLimitCacheTy &Cache, - const Loop *L, Value *ExitCond, - BasicBlock *TBB, BasicBlock *FBB, - bool ControlsExit, - bool AllowPredicates); - ExitLimit computeExitLimitFromCondImpl(ExitLimitCacheTy &Cache, const Loop *L, - Value *ExitCond, BasicBlock *TBB, - BasicBlock *FBB, bool ControlsExit, - bool AllowPredicates); + /// The target library information for the target we are targeting. + TargetLibraryInfo &TLI; - /// Compute the number of times the backedge of the specified loop will - /// execute if its exit condition were a conditional branch of the ICmpInst - /// ExitCond, TBB, and FBB. If AllowPredicates is set, this call will try - /// to use a minimal set of SCEV predicates in order to return an exact - /// answer. - ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, - BasicBlock *TBB, BasicBlock *FBB, - bool IsSubExpr, - bool AllowPredicates = false); + /// The tracker for @llvm.assume intrinsics in this function. + AssumptionCache &AC; - /// Compute the number of times the backedge of the specified loop will - /// execute if its exit condition were a switch with a single exiting case - /// to ExitingBB. - ExitLimit computeExitLimitFromSingleExitSwitch(const Loop *L, - SwitchInst *Switch, - BasicBlock *ExitingBB, - bool IsSubExpr); + /// The dominator tree. + DominatorTree &DT; - /// Given an exit condition of 'icmp op load X, cst', try to see if we can - /// compute the backedge-taken count. - ExitLimit computeLoadConstantCompareExitLimit(LoadInst *LI, Constant *RHS, - const Loop *L, - ICmpInst::Predicate p); + /// The loop information for the function we are currently analyzing. + LoopInfo &LI; - /// Compute the exit limit of a loop that is controlled by a - /// "(IV >> 1) != 0" type comparison. We cannot compute the exact trip - /// count in these cases (since SCEV has no way of expressing them), but we - /// can still sometimes compute an upper bound. - /// - /// Return an ExitLimit for a loop whose backedge is guarded by `LHS Pred - /// RHS`. - ExitLimit computeShiftCompareExitLimit(Value *LHS, Value *RHS, const Loop *L, - ICmpInst::Predicate Pred); + /// This SCEV is used to represent unknown trip counts and things. + std::unique_ptr CouldNotCompute; - /// If the loop is known to execute a constant number of times (the - /// condition evolves only from constants), try to evaluate a few iterations - /// of the loop until we get the exit condition gets a value of ExitWhen - /// (true or false). If we cannot evaluate the exit count of the loop, - /// return CouldNotCompute. - const SCEV *computeExitCountExhaustively(const Loop *L, Value *Cond, - bool ExitWhen); + /// The type for HasRecMap. + using HasRecMapType = DenseMap; - /// Return the number of times an exit condition comparing the specified - /// value to zero will execute. If not computable, return CouldNotCompute. - /// If AllowPredicates is set, this call will try to use a minimal set of - /// SCEV predicates in order to return an exact answer. - ExitLimit howFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr, - bool AllowPredicates = false); + /// This is a cache to record whether a SCEV contains any scAddRecExpr. + HasRecMapType HasRecMap; - /// Return the number of times an exit condition checking the specified - /// value for nonzero will execute. If not computable, return - /// CouldNotCompute. - ExitLimit howFarToNonZero(const SCEV *V, const Loop *L); + /// The type for ExprValueMap. + using ValueOffsetPair = std::pair; + using ExprValueMapType = DenseMap>; - /// Return the number of times an exit condition containing the specified - /// less-than comparison will execute. If not computable, return - /// CouldNotCompute. + /// ExprValueMap -- This map records the original values from which + /// the SCEV expr is generated from. /// - /// \p isSigned specifies whether the less-than is signed. + /// We want to represent the mapping as SCEV -> ValueOffsetPair instead + /// of SCEV -> Value: + /// Suppose we know S1 expands to V1, and + /// S1 = S2 + C_a + /// S3 = S2 + C_b + /// where C_a and C_b are different SCEVConstants. Then we'd like to + /// expand S3 as V1 - C_a + C_b instead of expanding S2 literally. + /// It is helpful when S2 is a complex SCEV expr. /// - /// \p ControlsExit is true when the LHS < RHS condition directly controls - /// the branch (loops exits only if condition is true). In this case, we can - /// use NoWrapFlags to skip overflow checks. + /// In order to do that, we represent ExprValueMap as a mapping from + /// SCEV to ValueOffsetPair. We will save both S1->{V1, 0} and + /// S2->{V1, C_a} into the map when we create SCEV for V1. When S3 + /// is expanded, it will first expand S2 to V1 - C_a because of + /// S2->{V1, C_a} in the map, then expand S3 to V1 - C_a + C_b. /// - /// If \p AllowPredicates is set, this call will try to use a minimal set of - /// SCEV predicates in order to return an exact answer. - ExitLimit howManyLessThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, - bool isSigned, bool ControlsExit, - bool AllowPredicates = false); + /// Note: S->{V, Offset} in the ExprValueMap means S can be expanded + /// to V - Offset. + ExprValueMapType ExprValueMap; - ExitLimit howManyGreaterThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, - bool isSigned, bool IsSubExpr, - bool AllowPredicates = false); + /// The type for ValueExprMap. + using ValueExprMapType = + DenseMap>; - /// Return a predecessor of BB (which may not be an immediate predecessor) - /// which has exactly one successor from which BB is reachable, or null if - /// no such block is found. - std::pair - getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB); + /// This is a cache of the values we have analyzed so far. + ValueExprMapType ValueExprMap; - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the given FoundCondValue value evaluates to true. - bool isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, - Value *FoundCondValue, bool Inverse); + /// Mark predicate values currently being processed by isImpliedCond. + SmallPtrSet PendingLoopPredicates; - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the condition described by FoundPred, FoundLHS, FoundRHS is - /// true. - bool isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, - ICmpInst::Predicate FoundPred, const SCEV *FoundLHS, - const SCEV *FoundRHS); + /// Set to true by isLoopBackedgeGuardedByCond when we're walking the set of + /// conditions dominating the backedge of a loop. + bool WalkingBEDominatingConds = false; - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the condition described by Pred, FoundLHS, and FoundRHS is - /// true. - bool isImpliedCondOperands(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS, const SCEV *FoundLHS, - const SCEV *FoundRHS); + /// Set to true by isKnownPredicateViaSplitting when we're trying to prove a + /// predicate by splitting it into a set of independent predicates. + bool ProvingSplitPredicate = false; - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the condition described by Pred, FoundLHS, and FoundRHS is - /// true. Here LHS is an operation that includes FoundLHS as one of its - /// arguments. - bool isImpliedViaOperations(ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS, - const SCEV *FoundLHS, const SCEV *FoundRHS, - unsigned Depth = 0); + /// Memoized values for the GetMinTrailingZeros + DenseMap MinTrailingZerosCache; - /// Test whether the condition described by Pred, LHS, and RHS is true. - /// Use only simple non-recursive types of checks, such as range analysis etc. - bool isKnownViaSimpleReasoning(ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); + /// Return the Value set from which the SCEV expr is generated. + SetVector *getSCEVValues(const SCEV *S); - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the condition described by Pred, FoundLHS, and FoundRHS is - /// true. - bool isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS, const SCEV *FoundLHS, - const SCEV *FoundRHS); + /// Private helper method for the GetMinTrailingZeros method + uint32_t GetMinTrailingZerosImpl(const SCEV *S); - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the condition described by Pred, FoundLHS, and FoundRHS is - /// true. Utility function used by isImpliedCondOperands. Tries to get - /// cases like "X `sgt` 0 => X - 1 `sgt` -1". - bool isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS, const SCEV *FoundLHS, - const SCEV *FoundRHS); + /// Information about the number of loop iterations for which a loop exit's + /// branch condition evaluates to the not-taken path. This is a temporary + /// pair of exact and max expressions that are eventually summarized in + /// ExitNotTakenInfo and BackedgeTakenInfo. + struct ExitLimit { + const SCEV *ExactNotTaken; // The exit is not taken exactly this many times + const SCEV *MaxNotTaken; // The exit is not taken at most this many times - /// Return true if the condition denoted by \p LHS \p Pred \p RHS is implied - /// by a call to \c @llvm.experimental.guard in \p BB. - bool isImpliedViaGuard(BasicBlock *BB, ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); + // Not taken either exactly MaxNotTaken or zero times + bool MaxOrZero = false; - /// Test whether the condition described by Pred, LHS, and RHS is true - /// whenever the condition described by Pred, FoundLHS, and FoundRHS is - /// true. - /// - /// This routine tries to rule out certain kinds of integer overflow, and - /// then tries to reason about arithmetic properties of the predicates. - bool isImpliedCondOperandsViaNoOverflow(ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS, - const SCEV *FoundLHS, - const SCEV *FoundRHS); + /// A set of predicate guards for this ExitLimit. The result is only valid + /// if all of the predicates in \c Predicates evaluate to 'true' at + /// run-time. + SmallPtrSet Predicates; - /// If we know that the specified Phi is in the header of its containing - /// loop, we know the loop executes a constant number of times, and the PHI - /// node is just a recurrence involving constants, fold it. - Constant *getConstantEvolutionLoopExitValue(PHINode *PN, const APInt &BEs, - const Loop *L); + void addPredicate(const SCEVPredicate *P) { + assert(!isa(P) && "Only add leaf predicates here!"); + Predicates.insert(P); + } - /// Test if the given expression is known to satisfy the condition described - /// by Pred and the known constant ranges of LHS and RHS. - bool isKnownPredicateViaConstantRanges(ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); + /*implicit*/ ExitLimit(const SCEV *E); - /// Try to prove the condition described by "LHS Pred RHS" by ruling out - /// integer overflow. - /// - /// For instance, this will return true for "A s< (A + C)" if C is - /// positive. - bool isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS); + ExitLimit( + const SCEV *E, const SCEV *M, bool MaxOrZero, + ArrayRef *> PredSetList); - /// Try to split Pred LHS RHS into logical conjunctions (and's) and try to - /// prove them individually. - bool isKnownPredicateViaSplitting(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS); + ExitLimit(const SCEV *E, const SCEV *M, bool MaxOrZero, + const SmallPtrSetImpl &PredSet); - /// Try to match the Expr as "(L + R)". - bool splitBinaryAdd(const SCEV *Expr, const SCEV *&L, const SCEV *&R, - SCEV::NoWrapFlags &Flags); + ExitLimit(const SCEV *E, const SCEV *M, bool MaxOrZero); - /// Compute \p LHS - \p RHS and returns the result as an APInt if it is a - /// constant, and None if it isn't. + /// Test whether this ExitLimit contains any computed information, or + /// whether it's all SCEVCouldNotCompute values. + bool hasAnyInfo() const { + return !isa(ExactNotTaken) || + !isa(MaxNotTaken); + } + + bool hasOperand(const SCEV *S) const; + + /// Test whether this ExitLimit contains all information. + bool hasFullInfo() const { + return !isa(ExactNotTaken); + } + }; + + /// Information about the number of times a particular loop exit may be + /// reached before exiting the loop. + struct ExitNotTakenInfo { + PoisoningVH ExitingBlock; + const SCEV *ExactNotTaken; + std::unique_ptr Predicate; + + explicit ExitNotTakenInfo(PoisoningVH ExitingBlock, + const SCEV *ExactNotTaken, + std::unique_ptr Predicate) + : ExitingBlock(ExitingBlock), ExactNotTaken(ExactNotTaken), + Predicate(std::move(Predicate)) {} + + bool hasAlwaysTruePredicate() const { + return !Predicate || Predicate->isAlwaysTrue(); + } + }; + + /// Information about the backedge-taken count of a loop. This currently + /// includes an exact count and a maximum count. /// - /// This is intended to be a cheaper version of getMinusSCEV. We can be - /// frugal here since we just bail out of actually constructing and - /// canonicalizing an expression in the cases where the result isn't going - /// to be a constant. - Optional computeConstantDifference(const SCEV *LHS, const SCEV *RHS); + class BackedgeTakenInfo { + /// A list of computable exits and their not-taken counts. Loops almost + /// never have more than one computable exit. + SmallVector ExitNotTaken; - /// Drop memoized information computed for S. Only erase Exit Limits info if - /// we expect that the operation we have made is going to change it. - void forgetMemoizedResults(const SCEV *S, bool EraseExitLimit = true); + /// The pointer part of \c MaxAndComplete is an expression indicating the + /// least maximum backedge-taken count of the loop that is known, or a + /// SCEVCouldNotCompute. This expression is only valid if the predicates + /// associated with all loop exits are true. + /// + /// The integer part of \c MaxAndComplete is a boolean indicating if \c + /// ExitNotTaken has an element for every exiting block in the loop. + PointerIntPair MaxAndComplete; - /// Return an existing SCEV for V if there is one, otherwise return nullptr. - const SCEV *getExistingSCEV(Value *V); + /// True iff the backedge is taken either exactly Max or zero times. + bool MaxOrZero = false; - /// Return false iff given SCEV contains a SCEVUnknown with NULL value- - /// pointer. - bool checkValidity(const SCEV *S) const; + /// \name Helper projection functions on \c MaxAndComplete. + /// @{ + bool isComplete() const { return MaxAndComplete.getInt(); } + const SCEV *getMax() const { return MaxAndComplete.getPointer(); } + /// @} - /// Return true if `ExtendOpTy`({`Start`,+,`Step`}) can be proved to be - /// equal to {`ExtendOpTy`(`Start`),+,`ExtendOpTy`(`Step`)}. This is - /// equivalent to proving no signed (resp. unsigned) wrap in - /// {`Start`,+,`Step`} if `ExtendOpTy` is `SCEVSignExtendExpr` - /// (resp. `SCEVZeroExtendExpr`). - template - bool proveNoWrapByVaryingStart(const SCEV *Start, const SCEV *Step, - const Loop *L); + public: + BackedgeTakenInfo() : MaxAndComplete(nullptr, 0) {} + BackedgeTakenInfo(BackedgeTakenInfo &&) = default; + BackedgeTakenInfo &operator=(BackedgeTakenInfo &&) = default; - /// Try to prove NSW or NUW on \p AR relying on ConstantRange manipulation. - SCEV::NoWrapFlags proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR); + using EdgeExitInfo = std::pair; - bool isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS, - ICmpInst::Predicate Pred, bool &Increasing); + /// Initialize BackedgeTakenInfo from a list of exact exit counts. + BackedgeTakenInfo(SmallVectorImpl &&ExitCounts, bool Complete, + const SCEV *MaxCount, bool MaxOrZero); - /// Return SCEV no-wrap flags that can be proven based on reasoning about - /// how poison produced from no-wrap flags on this value (e.g. a nuw add) - /// would trigger undefined behavior on overflow. - SCEV::NoWrapFlags getNoWrapFlagsFromUB(const Value *V); + /// Test whether this BackedgeTakenInfo contains any computed information, + /// or whether it's all SCEVCouldNotCompute values. + bool hasAnyInfo() const { + return !ExitNotTaken.empty() || !isa(getMax()); + } - /// Return true if the SCEV corresponding to \p I is never poison. Proving - /// this is more complex than proving that just \p I is never poison, since - /// SCEV commons expressions across control flow, and you can have cases - /// like: - /// - /// idx0 = a + b; - /// ptr[idx0] = 100; - /// if () { - /// idx1 = a +nsw b; - /// ptr[idx1] = 200; - /// } - /// - /// where the SCEV expression (+ a b) is guaranteed to not be poison (and - /// hence not sign-overflow) only if "" is true. Since both - /// `idx0` and `idx1` will be mapped to the same SCEV expression, (+ a b), - /// it is not okay to annotate (+ a b) with in the above example. - bool isSCEVExprNeverPoison(const Instruction *I); + /// Test whether this BackedgeTakenInfo contains complete information. + bool hasFullInfo() const { return isComplete(); } - /// This is like \c isSCEVExprNeverPoison but it specifically works for - /// instructions that will get mapped to SCEV add recurrences. Return true - /// if \p I will never generate poison under the assumption that \p I is an - /// add recurrence on the loop \p L. - bool isAddRecNeverPoison(const Instruction *I, const Loop *L); + /// Return an expression indicating the exact *backedge-taken* + /// count of the loop if it is known or SCEVCouldNotCompute + /// otherwise. If execution makes it to the backedge on every + /// iteration (i.e. there are no abnormal exists like exception + /// throws and thread exits) then this is the number of times the + /// loop header will execute minus one. + /// + /// If the SCEV predicate associated with the answer can be different + /// from AlwaysTrue, we must add a (non null) Predicates argument. + /// The SCEV predicate associated with the answer will be added to + /// Predicates. A run-time check needs to be emitted for the SCEV + /// predicate in order for the answer to be valid. + /// + /// Note that we should always know if we need to pass a predicate + /// argument or not from the way the ExitCounts vector was computed. + /// If we allowed SCEV predicates to be generated when populating this + /// vector, this information can contain them and therefore a + /// SCEVPredicate argument should be added to getExact. + const SCEV *getExact(ScalarEvolution *SE, + SCEVUnionPredicate *Predicates = nullptr) const; -public: - ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, - DominatorTree &DT, LoopInfo &LI); - ScalarEvolution(ScalarEvolution &&Arg); - ~ScalarEvolution(); + /// Return the number of times this loop exit may fall through to the back + /// edge, or SCEVCouldNotCompute. The loop is guaranteed not to exit via + /// this block before this number of iterations, but may exit via another + /// block. + const SCEV *getExact(BasicBlock *ExitingBlock, ScalarEvolution *SE) const; - LLVMContext &getContext() const { return F.getContext(); } + /// Get the max backedge taken count for the loop. + const SCEV *getMax(ScalarEvolution *SE) const; - /// Test if values of the given type are analyzable within the SCEV - /// framework. This primarily includes integer types, and it can optionally - /// include pointer types if the ScalarEvolution class has access to - /// target-specific information. - bool isSCEVable(Type *Ty) const; + /// Return true if the number of times this backedge is taken is either the + /// value returned by getMax or zero. + bool isMaxOrZero(ScalarEvolution *SE) const; - /// Return the size in bits of the specified type, for which isSCEVable must - /// return true. - uint64_t getTypeSizeInBits(Type *Ty) const; + /// Return true if any backedge taken count expressions refer to the given + /// subexpression. + bool hasOperand(const SCEV *S, ScalarEvolution *SE) const; - /// Return a type with the same bitwidth as the given type and which - /// represents how SCEV will treat the given type, for which isSCEVable must - /// return true. For pointer types, this is the pointer-sized integer type. - Type *getEffectiveSCEVType(Type *Ty) const; + /// Invalidate this result and free associated memory. + void clear(); + }; - // Returns a wider type among {Ty1, Ty2}. - Type *getWiderType(Type *Ty1, Type *Ty2) const; + /// Cache the backedge-taken count of the loops for this function as they + /// are computed. + DenseMap BackedgeTakenCounts; - /// Return true if the SCEV is a scAddRecExpr or it contains - /// scAddRecExpr. The result will be cached in HasRecMap. - bool containsAddRecurrence(const SCEV *S); + /// Cache the predicated backedge-taken count of the loops for this + /// function as they are computed. + DenseMap PredicatedBackedgeTakenCounts; - /// Return the Value set from which the SCEV expr is generated. - SetVector *getSCEVValues(const SCEV *S); + // Cache the calculated exit limits for the loops. + DenseMap ExitLimits; - /// Erase Value from ValueExprMap and ExprValueMap. - void eraseValueFromMap(Value *V); + /// This map contains entries for all of the PHI instructions that we + /// attempt to compute constant evolutions for. This allows us to avoid + /// potentially expensive recomputation of these properties. An instruction + /// maps to null if we are unable to compute its exit value. + DenseMap ConstantEvolutionLoopExitValue; - /// Return a SCEV expression for the full generality of the specified - /// expression. - const SCEV *getSCEV(Value *V); + /// This map contains entries for all the expressions that we attempt to + /// compute getSCEVAtScope information for, which can be expensive in + /// extreme cases. + DenseMap, 2>> + ValuesAtScopes; - const SCEV *getConstant(ConstantInt *V); - const SCEV *getConstant(const APInt &Val); - const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); - const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty); - const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); - const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); - const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty); - const SCEV *getAddExpr(SmallVectorImpl &Ops, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0); - const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector Ops = {LHS, RHS}; - return getAddExpr(Ops, Flags, Depth); - } - const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector Ops = {Op0, Op1, Op2}; - return getAddExpr(Ops, Flags, Depth); - } - const SCEV *getMulExpr(SmallVectorImpl &Ops, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0); - const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector Ops = {LHS, RHS}; - return getMulExpr(Ops, Flags, Depth); - } - const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector Ops = {Op0, Op1, Op2}; - return getMulExpr(Ops, Flags, Depth); - } - const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUDivExactExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getURemExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, - SCEV::NoWrapFlags Flags); - const SCEV *getAddRecExpr(SmallVectorImpl &Operands, - const Loop *L, SCEV::NoWrapFlags Flags); - const SCEV *getAddRecExpr(const SmallVectorImpl &Operands, - const Loop *L, SCEV::NoWrapFlags Flags) { - SmallVector NewOp(Operands.begin(), Operands.end()); - return getAddRecExpr(NewOp, L, Flags); - } + /// Memoized computeLoopDisposition results. + DenseMap, 2>> + LoopDispositions; - /// Checks if \p SymbolicPHI can be rewritten as an AddRecExpr under some - /// Predicates. If successful return these ; - /// The function is intended to be called from PSCEV (the caller will decide - /// whether to actually add the predicates and carry out the rewrites). - Optional>> - createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI); + struct LoopProperties { + /// Set to true if the loop contains no instruction that can have side + /// effects (i.e. via throwing an exception, volatile or atomic access). + bool HasNoAbnormalExits; - /// Returns an expression for a GEP - /// - /// \p GEP The GEP. The indices contained in the GEP itself are ignored, - /// instead we use IndexExprs. - /// \p IndexExprs The expressions for the indices. - const SCEV *getGEPExpr(GEPOperator *GEP, - const SmallVectorImpl &IndexExprs); - const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getSMaxExpr(SmallVectorImpl &Operands); - const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUMaxExpr(SmallVectorImpl &Operands); - const SCEV *getSMinExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUnknown(Value *V); - const SCEV *getCouldNotCompute(); + /// Set to true if the loop contains no instruction that can abnormally exit + /// the loop (i.e. via throwing an exception, by terminating the thread + /// cleanly or by infinite looping in a called function). Strictly + /// speaking, the last one is not leaving the loop, but is identical to + /// leaving the loop for reasoning about undefined behavior. + bool HasNoSideEffects; + }; - /// Return a SCEV for the constant 0 of a specific type. - const SCEV *getZero(Type *Ty) { return getConstant(Ty, 0); } + /// Cache for \c getLoopProperties. + DenseMap LoopPropertiesCache; - /// Return a SCEV for the constant 1 of a specific type. - const SCEV *getOne(Type *Ty) { return getConstant(Ty, 1); } + /// Return a \c LoopProperties instance for \p L, creating one if necessary. + LoopProperties getLoopProperties(const Loop *L); - /// Return an expression for sizeof AllocTy that is type IntTy - const SCEV *getSizeOfExpr(Type *IntTy, Type *AllocTy); + bool loopHasNoSideEffects(const Loop *L) { + return getLoopProperties(L).HasNoSideEffects; + } - /// Return an expression for offsetof on the given field with type IntTy - const SCEV *getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo); + bool loopHasNoAbnormalExits(const Loop *L) { + return getLoopProperties(L).HasNoAbnormalExits; + } - /// Return the SCEV object corresponding to -V. - const SCEV *getNegativeSCEV(const SCEV *V, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); + /// Compute a LoopDisposition value. + LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L); - /// Return the SCEV object corresponding to ~V. - const SCEV *getNotSCEV(const SCEV *V); + /// Memoized computeBlockDisposition results. + DenseMap< + const SCEV *, + SmallVector, 2>> + BlockDispositions; - /// Return LHS-RHS. Minus is represented in SCEV as A+B*-1. - const SCEV *getMinusSCEV(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0); + /// Compute a BlockDisposition value. + BlockDisposition computeBlockDisposition(const SCEV *S, const BasicBlock *BB); - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is zero extended. - const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty); + /// Memoized results from getRange + DenseMap UnsignedRanges; - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is sign extended. - const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty); + /// Memoized results from getRange + DenseMap SignedRanges; - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is zero extended. The - /// conversion must not be narrowing. - const SCEV *getNoopOrZeroExtend(const SCEV *V, Type *Ty); + /// Used to parameterize getRange + enum RangeSignHint { HINT_RANGE_UNSIGNED, HINT_RANGE_SIGNED }; - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is sign extended. The - /// conversion must not be narrowing. - const SCEV *getNoopOrSignExtend(const SCEV *V, Type *Ty); + /// Set the memoized range for the given SCEV. + const ConstantRange &setRange(const SCEV *S, RangeSignHint Hint, + ConstantRange CR) { + DenseMap &Cache = + Hint == HINT_RANGE_UNSIGNED ? UnsignedRanges : SignedRanges; - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is extended with - /// unspecified bits. The conversion must not be narrowing. - const SCEV *getNoopOrAnyExtend(const SCEV *V, Type *Ty); + auto Pair = Cache.try_emplace(S, std::move(CR)); + if (!Pair.second) + Pair.first->second = std::move(CR); + return Pair.first->second; + } - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. The conversion must not be widening. - const SCEV *getTruncateOrNoop(const SCEV *V, Type *Ty); + /// Determine the range for a particular SCEV. + /// NOTE: This returns a reference to an entry in a cache. It must be + /// copied if its needed for longer. + const ConstantRange &getRangeRef(const SCEV *S, RangeSignHint Hint); - /// Promote the operands to the wider of the types using zero-extension, and - /// then perform a umax operation with them. - const SCEV *getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); + /// Determines the range for the affine SCEVAddRecExpr {\p Start,+,\p Stop}. + /// Helper for \c getRange. + ConstantRange getRangeForAffineAR(const SCEV *Start, const SCEV *Stop, + const SCEV *MaxBECount, unsigned BitWidth); - /// Promote the operands to the wider of the types using zero-extension, and - /// then perform a umin operation with them. - const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); + /// Try to compute a range for the affine SCEVAddRecExpr {\p Start,+,\p + /// Stop} by "factoring out" a ternary expression from the add recurrence. + /// Helper called by \c getRange. + ConstantRange getRangeViaFactoring(const SCEV *Start, const SCEV *Stop, + const SCEV *MaxBECount, unsigned BitWidth); - /// Transitively follow the chain of pointer-type operands until reaching a - /// SCEV that does not have a single pointer operand. This returns a - /// SCEVUnknown pointer for well-formed pointer-type expressions, but corner - /// cases do exist. - const SCEV *getPointerBase(const SCEV *V); + /// We know that there is no SCEV for the specified value. Analyze the + /// expression. + const SCEV *createSCEV(Value *V); - /// Return a SCEV expression for the specified value at the specified scope - /// in the program. The L value specifies a loop nest to evaluate the - /// expression at, where null is the top-level or a specified loop is - /// immediately inside of the loop. - /// - /// This method can be used to compute the exit value for a variable defined - /// in a loop by querying what the value will hold in the parent loop. - /// - /// In the case that a relevant loop exit value cannot be computed, the - /// original value V is returned. - const SCEV *getSCEVAtScope(const SCEV *S, const Loop *L); + /// Provide the special handling we need to analyze PHI SCEVs. + const SCEV *createNodeForPHI(PHINode *PN); - /// This is a convenience function which does getSCEVAtScope(getSCEV(V), L). - const SCEV *getSCEVAtScope(Value *V, const Loop *L); + /// Helper function called from createNodeForPHI. + const SCEV *createAddRecFromPHI(PHINode *PN); - /// Test whether entry to the loop is protected by a conditional between LHS - /// and RHS. This is used to help avoid max expressions in loop trip - /// counts, and to eliminate casts. - bool isLoopEntryGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); + /// A helper function for createAddRecFromPHI to handle simple cases. + const SCEV *createSimpleAffineAddRec(PHINode *PN, Value *BEValueV, + Value *StartValueV); - /// Test whether the backedge of the loop is protected by a conditional - /// between LHS and RHS. This is used to to eliminate casts. - bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); + /// Helper function called from createNodeForPHI. + const SCEV *createNodeFromSelectLikePHI(PHINode *PN); - /// Returns the maximum trip count of the loop if it is a single-exit - /// loop and we can compute a small maximum for that loop. - /// - /// Implemented in terms of the \c getSmallConstantTripCount overload with - /// the single exiting block passed to it. See that routine for details. - unsigned getSmallConstantTripCount(const Loop *L); + /// Provide special handling for a select-like instruction (currently this + /// is either a select instruction or a phi node). \p I is the instruction + /// being processed, and it is assumed equivalent to "Cond ? TrueVal : + /// FalseVal". + const SCEV *createNodeForSelectOrPHI(Instruction *I, Value *Cond, + Value *TrueVal, Value *FalseVal); - /// Returns the maximum trip count of this loop as a normal unsigned - /// value. Returns 0 if the trip count is unknown or not constant. This - /// "trip count" assumes that control exits via ExitingBlock. More - /// precisely, it is the number of times that control may reach ExitingBlock - /// before taking the branch. For loops with multiple exits, it may not be - /// the number times that the loop header executes if the loop exits - /// prematurely via another branch. - unsigned getSmallConstantTripCount(const Loop *L, BasicBlock *ExitingBlock); + /// Provide the special handling we need to analyze GEP SCEVs. + const SCEV *createNodeForGEP(GEPOperator *GEP); - /// Returns the upper bound of the loop trip count as a normal unsigned - /// value. - /// Returns 0 if the trip count is unknown or not constant. - unsigned getSmallConstantMaxTripCount(const Loop *L); + /// Implementation code for getSCEVAtScope; called at most once for each + /// SCEV+Loop pair. + const SCEV *computeSCEVAtScope(const SCEV *S, const Loop *L); - /// Returns the largest constant divisor of the trip count of the - /// loop if it is a single-exit loop and we can compute a small maximum for - /// that loop. - /// - /// Implemented in terms of the \c getSmallConstantTripMultiple overload with - /// the single exiting block passed to it. See that routine for details. - unsigned getSmallConstantTripMultiple(const Loop *L); + /// This looks up computed SCEV values for all instructions that depend on + /// the given instruction and removes them from the ValueExprMap map if they + /// reference SymName. This is used during PHI resolution. + void forgetSymbolicName(Instruction *I, const SCEV *SymName); - /// Returns the largest constant divisor of the trip count of this loop as a - /// normal unsigned value, if possible. This means that the actual trip - /// count is always a multiple of the returned value (don't forget the trip - /// count could very well be zero as well!). As explained in the comments - /// for getSmallConstantTripCount, this assumes that control exits the loop - /// via ExitingBlock. - unsigned getSmallConstantTripMultiple(const Loop *L, - BasicBlock *ExitingBlock); + /// Return the BackedgeTakenInfo for the given loop, lazily computing new + /// values if the loop hasn't been analyzed yet. The returned result is + /// guaranteed not to be predicated. + const BackedgeTakenInfo &getBackedgeTakenInfo(const Loop *L); - /// Get the expression for the number of loop iterations for which this loop - /// is guaranteed not to exit via ExitingBlock. Otherwise return - /// SCEVCouldNotCompute. - const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock); + /// Similar to getBackedgeTakenInfo, but will add predicates as required + /// with the purpose of returning complete information. + const BackedgeTakenInfo &getPredicatedBackedgeTakenInfo(const Loop *L); - /// If the specified loop has a predictable backedge-taken count, return it, - /// otherwise return a SCEVCouldNotCompute object. The backedge-taken count is - /// the number of times the loop header will be branched to from within the - /// loop, assuming there are no abnormal exists like exception throws. This is - /// one less than the trip count of the loop, since it doesn't count the first - /// iteration, when the header is branched to from outside the loop. + /// Compute the number of times the specified loop will iterate. + /// If AllowPredicates is set, we will create new SCEV predicates as + /// necessary in order to return an exact answer. + BackedgeTakenInfo computeBackedgeTakenCount(const Loop *L, + bool AllowPredicates = false); + + /// Compute the number of times the backedge of the specified loop will + /// execute if it exits via the specified block. If AllowPredicates is set, + /// this call will try to use a minimal set of SCEV predicates in order to + /// return an exact answer. + ExitLimit computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, + bool AllowPredicates = false); + + ExitLimit computeExitLimitImpl(const Loop *L, BasicBlock *ExitingBlock, + bool AllowPredicates = false); + + /// Compute the number of times the backedge of the specified loop will + /// execute if its exit condition were a conditional branch of ExitCond, + /// TBB, and FBB. /// - /// Note that it is not valid to call this method on a loop without a - /// loop-invariant backedge-taken count (see - /// hasLoopInvariantBackedgeTakenCount). - const SCEV *getBackedgeTakenCount(const Loop *L); + /// \p ControlsExit is true if ExitCond directly controls the exit + /// branch. In this case, we can assume that the loop exits only if the + /// condition is true and can infer that failing to meet the condition prior + /// to integer wraparound results in undefined behavior. + /// + /// If \p AllowPredicates is set, this call will try to use a minimal set of + /// SCEV predicates in order to return an exact answer. + ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, + BasicBlock *TBB, BasicBlock *FBB, + bool ControlsExit, + bool AllowPredicates = false); - /// Similar to getBackedgeTakenCount, except it will add a set of - /// SCEV predicates to Predicates that are required to be true in order for - /// the answer to be correct. Predicates can be checked with run-time - /// checks and can be used to perform loop versioning. - const SCEV *getPredicatedBackedgeTakenCount(const Loop *L, - SCEVUnionPredicate &Predicates); + // Helper functions for computeExitLimitFromCond to avoid exponential time + // complexity. - /// When successful, this returns a SCEVConstant that is greater than or equal - /// to (i.e. a "conservative over-approximation") of the value returend by - /// getBackedgeTakenCount. If such a value cannot be computed, it returns the - /// SCEVCouldNotCompute object. - const SCEV *getMaxBackedgeTakenCount(const Loop *L); + class ExitLimitCache { + // It may look like we need key on the whole (L, TBB, FBB, ControlsExit, + // AllowPredicates) tuple, but recursive calls to + // computeExitLimitFromCondCached from computeExitLimitFromCondImpl only + // vary the in \c ExitCond and \c ControlsExit parameters. We remember the + // initial values of the other values to assert our assumption. + SmallDenseMap, ExitLimit> TripCountMap; - /// Return true if the backedge taken count is either the value returned by - /// getMaxBackedgeTakenCount or zero. - bool isBackedgeTakenCountMaxOrZero(const Loop *L); + const Loop *L; + BasicBlock *TBB; + BasicBlock *FBB; + bool AllowPredicates; - /// Return true if the specified loop has an analyzable loop-invariant - /// backedge-taken count. - bool hasLoopInvariantBackedgeTakenCount(const Loop *L); + public: + ExitLimitCache(const Loop *L, BasicBlock *TBB, BasicBlock *FBB, + bool AllowPredicates) + : L(L), TBB(TBB), FBB(FBB), AllowPredicates(AllowPredicates) {} - /// This method should be called by the client when it has changed a loop in - /// a way that may effect ScalarEvolution's ability to compute a trip count, - /// or if the loop is deleted. This call is potentially expensive for large - /// loop bodies. - void forgetLoop(const Loop *L); + Optional find(const Loop *L, Value *ExitCond, BasicBlock *TBB, + BasicBlock *FBB, bool ControlsExit, + bool AllowPredicates); - /// This method should be called by the client when it has changed a value - /// in a way that may effect its value, or which may disconnect it from a - /// def-use chain linking it to a loop. - void forgetValue(Value *V); + void insert(const Loop *L, Value *ExitCond, BasicBlock *TBB, + BasicBlock *FBB, bool ControlsExit, bool AllowPredicates, + const ExitLimit &EL); + }; - /// Called when the client has changed the disposition of values in - /// this loop. - /// - /// We don't have a way to invalidate per-loop dispositions. Clear and - /// recompute is simpler. - void forgetLoopDispositions(const Loop *L) { LoopDispositions.clear(); } + using ExitLimitCacheTy = ExitLimitCache; - /// Determine the minimum number of zero bits that S is guaranteed to end in - /// (at every loop iteration). It is, at the same time, the minimum number - /// of times S is divisible by 2. For example, given {4,+,8} it returns 2. - /// If S is guaranteed to be 0, it returns the bitwidth of S. - uint32_t GetMinTrailingZeros(const SCEV *S); + ExitLimit computeExitLimitFromCondCached(ExitLimitCacheTy &Cache, + const Loop *L, Value *ExitCond, + BasicBlock *TBB, BasicBlock *FBB, + bool ControlsExit, + bool AllowPredicates); + ExitLimit computeExitLimitFromCondImpl(ExitLimitCacheTy &Cache, const Loop *L, + Value *ExitCond, BasicBlock *TBB, + BasicBlock *FBB, bool ControlsExit, + bool AllowPredicates); - /// Determine the unsigned range for a particular SCEV. - /// NOTE: This returns a copy of the reference returned by getRangeRef. - ConstantRange getUnsignedRange(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_UNSIGNED); - } + /// Compute the number of times the backedge of the specified loop will + /// execute if its exit condition were a conditional branch of the ICmpInst + /// ExitCond, TBB, and FBB. If AllowPredicates is set, this call will try + /// to use a minimal set of SCEV predicates in order to return an exact + /// answer. + ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, + BasicBlock *TBB, BasicBlock *FBB, + bool IsSubExpr, + bool AllowPredicates = false); - /// Determine the min of the unsigned range for a particular SCEV. - APInt getUnsignedRangeMin(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMin(); - } + /// Compute the number of times the backedge of the specified loop will + /// execute if its exit condition were a switch with a single exiting case + /// to ExitingBB. + ExitLimit computeExitLimitFromSingleExitSwitch(const Loop *L, + SwitchInst *Switch, + BasicBlock *ExitingBB, + bool IsSubExpr); - /// Determine the max of the unsigned range for a particular SCEV. - APInt getUnsignedRangeMax(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMax(); - } + /// Given an exit condition of 'icmp op load X, cst', try to see if we can + /// compute the backedge-taken count. + ExitLimit computeLoadConstantCompareExitLimit(LoadInst *LI, Constant *RHS, + const Loop *L, + ICmpInst::Predicate p); - /// Determine the signed range for a particular SCEV. - /// NOTE: This returns a copy of the reference returned by getRangeRef. - ConstantRange getSignedRange(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_SIGNED); - } + /// Compute the exit limit of a loop that is controlled by a + /// "(IV >> 1) != 0" type comparison. We cannot compute the exact trip + /// count in these cases (since SCEV has no way of expressing them), but we + /// can still sometimes compute an upper bound. + /// + /// Return an ExitLimit for a loop whose backedge is guarded by `LHS Pred + /// RHS`. + ExitLimit computeShiftCompareExitLimit(Value *LHS, Value *RHS, const Loop *L, + ICmpInst::Predicate Pred); - /// Determine the min of the signed range for a particular SCEV. - APInt getSignedRangeMin(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMin(); - } + /// If the loop is known to execute a constant number of times (the + /// condition evolves only from constants), try to evaluate a few iterations + /// of the loop until we get the exit condition gets a value of ExitWhen + /// (true or false). If we cannot evaluate the exit count of the loop, + /// return CouldNotCompute. + const SCEV *computeExitCountExhaustively(const Loop *L, Value *Cond, + bool ExitWhen); - /// Determine the max of the signed range for a particular SCEV. - APInt getSignedRangeMax(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMax(); - } + /// Return the number of times an exit condition comparing the specified + /// value to zero will execute. If not computable, return CouldNotCompute. + /// If AllowPredicates is set, this call will try to use a minimal set of + /// SCEV predicates in order to return an exact answer. + ExitLimit howFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr, + bool AllowPredicates = false); - /// Test if the given expression is known to be negative. - bool isKnownNegative(const SCEV *S); + /// Return the number of times an exit condition checking the specified + /// value for nonzero will execute. If not computable, return + /// CouldNotCompute. + ExitLimit howFarToNonZero(const SCEV *V, const Loop *L); - /// Test if the given expression is known to be positive. - bool isKnownPositive(const SCEV *S); + /// Return the number of times an exit condition containing the specified + /// less-than comparison will execute. If not computable, return + /// CouldNotCompute. + /// + /// \p isSigned specifies whether the less-than is signed. + /// + /// \p ControlsExit is true when the LHS < RHS condition directly controls + /// the branch (loops exits only if condition is true). In this case, we can + /// use NoWrapFlags to skip overflow checks. + /// + /// If \p AllowPredicates is set, this call will try to use a minimal set of + /// SCEV predicates in order to return an exact answer. + ExitLimit howManyLessThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, + bool isSigned, bool ControlsExit, + bool AllowPredicates = false); - /// Test if the given expression is known to be non-negative. - bool isKnownNonNegative(const SCEV *S); + ExitLimit howManyGreaterThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, + bool isSigned, bool IsSubExpr, + bool AllowPredicates = false); - /// Test if the given expression is known to be non-positive. - bool isKnownNonPositive(const SCEV *S); + /// Return a predecessor of BB (which may not be an immediate predecessor) + /// which has exactly one successor from which BB is reachable, or null if + /// no such block is found. + std::pair + getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB); - /// Test if the given expression is known to be non-zero. - bool isKnownNonZero(const SCEV *S); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the given FoundCondValue value evaluates to true. + bool isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, + Value *FoundCondValue, bool Inverse); - /// Test if the given expression is known to satisfy the condition described - /// by Pred, LHS, and RHS. - bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by FoundPred, FoundLHS, FoundRHS is + /// true. + bool isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, + ICmpInst::Predicate FoundPred, const SCEV *FoundLHS, + const SCEV *FoundRHS); - /// Return true if, for all loop invariant X, the predicate "LHS `Pred` X" - /// is monotonically increasing or decreasing. In the former case set - /// `Increasing` to true and in the latter case set `Increasing` to false. - /// - /// A predicate is said to be monotonically increasing if may go from being - /// false to being true as the loop iterates, but never the other way - /// around. A predicate is said to be monotonically decreasing if may go - /// from being true to being false as the loop iterates, but never the other - /// way around. - bool isMonotonicPredicate(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred, - bool &Increasing); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. + bool isImpliedCondOperands(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const SCEV *FoundLHS, + const SCEV *FoundRHS); + + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. Here LHS is an operation that includes FoundLHS as one of its + /// arguments. + bool isImpliedViaOperations(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS, + const SCEV *FoundLHS, const SCEV *FoundRHS, + unsigned Depth = 0); - /// Return true if the result of the predicate LHS `Pred` RHS is loop - /// invariant with respect to L. Set InvariantPred, InvariantLHS and - /// InvariantLHS so that InvariantLHS `InvariantPred` InvariantRHS is the - /// loop invariant form of LHS `Pred` RHS. - bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS, const Loop *L, - ICmpInst::Predicate &InvariantPred, - const SCEV *&InvariantLHS, - const SCEV *&InvariantRHS); + /// Test whether the condition described by Pred, LHS, and RHS is true. + /// Use only simple non-recursive types of checks, such as range analysis etc. + bool isKnownViaSimpleReasoning(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); - /// Simplify LHS and RHS in a comparison with predicate Pred. Return true - /// iff any changes were made. If the operands are provably equal or - /// unequal, LHS and RHS are set to the same value and Pred is set to either - /// ICMP_EQ or ICMP_NE. - bool SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, - const SCEV *&RHS, unsigned Depth = 0); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. + bool isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const SCEV *FoundLHS, + const SCEV *FoundRHS); - /// Return the "disposition" of the given SCEV with respect to the given - /// loop. - LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. Utility function used by isImpliedCondOperands. Tries to get + /// cases like "X `sgt` 0 => X - 1 `sgt` -1". + bool isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const SCEV *FoundLHS, + const SCEV *FoundRHS); - /// Return true if the value of the given SCEV is unchanging in the - /// specified loop. - bool isLoopInvariant(const SCEV *S, const Loop *L); + /// Return true if the condition denoted by \p LHS \p Pred \p RHS is implied + /// by a call to \c @llvm.experimental.guard in \p BB. + bool isImpliedViaGuard(BasicBlock *BB, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); - /// Determine if the SCEV can be evaluated at loop's entry. It is true if it - /// doesn't depend on a SCEVUnknown of an instruction which is dominated by - /// the header of loop L. - bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. + /// + /// This routine tries to rule out certain kinds of integer overflow, and + /// then tries to reason about arithmetic properties of the predicates. + bool isImpliedCondOperandsViaNoOverflow(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS, + const SCEV *FoundLHS, + const SCEV *FoundRHS); - /// Return true if the given SCEV changes value in a known way in the - /// specified loop. This property being true implies that the value is - /// variant in the loop AND that we can emit an expression to compute the - /// value of the expression at any particular loop iteration. - bool hasComputableLoopEvolution(const SCEV *S, const Loop *L); + /// If we know that the specified Phi is in the header of its containing + /// loop, we know the loop executes a constant number of times, and the PHI + /// node is just a recurrence involving constants, fold it. + Constant *getConstantEvolutionLoopExitValue(PHINode *PN, const APInt &BEs, + const Loop *L); - /// Return the "disposition" of the given SCEV with respect to the given - /// block. - BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB); + /// Test if the given expression is known to satisfy the condition described + /// by Pred and the known constant ranges of LHS and RHS. + bool isKnownPredicateViaConstantRanges(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); - /// Return true if elements that makes up the given SCEV dominate the - /// specified basic block. - bool dominates(const SCEV *S, const BasicBlock *BB); + /// Try to prove the condition described by "LHS Pred RHS" by ruling out + /// integer overflow. + /// + /// For instance, this will return true for "A s< (A + C)" if C is + /// positive. + bool isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS); - /// Return true if elements that makes up the given SCEV properly dominate - /// the specified basic block. - bool properlyDominates(const SCEV *S, const BasicBlock *BB); + /// Try to split Pred LHS RHS into logical conjunctions (and's) and try to + /// prove them individually. + bool isKnownPredicateViaSplitting(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS); - /// Test whether the given SCEV has Op as a direct or indirect operand. - bool hasOperand(const SCEV *S, const SCEV *Op) const; + /// Try to match the Expr as "(L + R)". + bool splitBinaryAdd(const SCEV *Expr, const SCEV *&L, const SCEV *&R, + SCEV::NoWrapFlags &Flags); - /// Return the size of an element read or written by Inst. - const SCEV *getElementSize(Instruction *Inst); + /// Compute \p LHS - \p RHS and returns the result as an APInt if it is a + /// constant, and None if it isn't. + /// + /// This is intended to be a cheaper version of getMinusSCEV. We can be + /// frugal here since we just bail out of actually constructing and + /// canonicalizing an expression in the cases where the result isn't going + /// to be a constant. + Optional computeConstantDifference(const SCEV *LHS, const SCEV *RHS); - /// Compute the array dimensions Sizes from the set of Terms extracted from - /// the memory access function of this SCEVAddRecExpr (second step of - /// delinearization). - void findArrayDimensions(SmallVectorImpl &Terms, - SmallVectorImpl &Sizes, - const SCEV *ElementSize); + /// Drop memoized information computed for S. Only erase Exit Limits info if + /// we expect that the operation we have made is going to change it. + void forgetMemoizedResults(const SCEV *S, bool EraseExitLimit = true); - void print(raw_ostream &OS) const; - void verify() const; - bool invalidate(Function &F, const PreservedAnalyses &PA, - FunctionAnalysisManager::Invalidator &Inv); + /// Return an existing SCEV for V if there is one, otherwise return nullptr. + const SCEV *getExistingSCEV(Value *V); - /// Collect parametric terms occurring in step expressions (first step of - /// delinearization). - void collectParametricTerms(const SCEV *Expr, - SmallVectorImpl &Terms); + /// Return false iff given SCEV contains a SCEVUnknown with NULL value- + /// pointer. + bool checkValidity(const SCEV *S) const; - /// Return in Subscripts the access functions for each dimension in Sizes - /// (third step of delinearization). - void computeAccessFunctions(const SCEV *Expr, - SmallVectorImpl &Subscripts, - SmallVectorImpl &Sizes); + /// Return true if `ExtendOpTy`({`Start`,+,`Step`}) can be proved to be + /// equal to {`ExtendOpTy`(`Start`),+,`ExtendOpTy`(`Step`)}. This is + /// equivalent to proving no signed (resp. unsigned) wrap in + /// {`Start`,+,`Step`} if `ExtendOpTy` is `SCEVSignExtendExpr` + /// (resp. `SCEVZeroExtendExpr`). + template + bool proveNoWrapByVaryingStart(const SCEV *Start, const SCEV *Step, + const Loop *L); - /// Split this SCEVAddRecExpr into two vectors of SCEVs representing the - /// subscripts and sizes of an array access. - /// - /// The delinearization is a 3 step process: the first two steps compute the - /// sizes of each subscript and the third step computes the access functions - /// for the delinearized array: - /// - /// 1. Find the terms in the step functions - /// 2. Compute the array size - /// 3. Compute the access function: divide the SCEV by the array size - /// starting with the innermost dimensions found in step 2. The Quotient - /// is the SCEV to be divided in the next step of the recursion. The - /// Remainder is the subscript of the innermost dimension. Loop over all - /// array dimensions computed in step 2. - /// - /// To compute a uniform array size for several memory accesses to the same - /// object, one can collect in step 1 all the step terms for all the memory - /// accesses, and compute in step 2 a unique array shape. This guarantees - /// that the array shape will be the same across all memory accesses. - /// - /// FIXME: We could derive the result of steps 1 and 2 from a description of - /// the array shape given in metadata. - /// - /// Example: - /// - /// A[][n][m] - /// - /// for i - /// for j - /// for k - /// A[j+k][2i][5i] = - /// - /// The initial SCEV: - /// - /// A[{{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k] - /// - /// 1. Find the different terms in the step functions: - /// -> [2*m, 5, n*m, n*m] - /// - /// 2. Compute the array size: sort and unique them - /// -> [n*m, 2*m, 5] - /// find the GCD of all the terms = 1 - /// divide by the GCD and erase constant terms - /// -> [n*m, 2*m] - /// GCD = m - /// divide by GCD -> [n, 2] - /// remove constant terms - /// -> [n] - /// size of the array is A[unknown][n][m] - /// - /// 3. Compute the access function - /// a. Divide {{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k by the innermost size m - /// Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k - /// Remainder: {{{0,+,5}_i, +, 0}_j, +, 0}_k - /// The remainder is the subscript of the innermost array dimension: [5i]. - /// - /// b. Divide Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k by next outer size n - /// Quotient: {{{0,+,0}_i, +, 1}_j, +, 1}_k - /// Remainder: {{{0,+,2}_i, +, 0}_j, +, 0}_k - /// The Remainder is the subscript of the next array dimension: [2i]. - /// - /// The subscript of the outermost dimension is the Quotient: [j+k]. - /// - /// Overall, we have: A[][n][m], and the access function: A[j+k][2i][5i]. - void delinearize(const SCEV *Expr, SmallVectorImpl &Subscripts, - SmallVectorImpl &Sizes, - const SCEV *ElementSize); + /// Try to prove NSW or NUW on \p AR relying on ConstantRange manipulation. + SCEV::NoWrapFlags proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR); - /// Return the DataLayout associated with the module this SCEV instance is - /// operating on. - const DataLayout &getDataLayout() const { - return F.getParent()->getDataLayout(); - } + bool isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS, + ICmpInst::Predicate Pred, bool &Increasing); - const SCEVPredicate *getEqualPredicate(const SCEV *LHS, const SCEV *RHS); + /// Return SCEV no-wrap flags that can be proven based on reasoning about + /// how poison produced from no-wrap flags on this value (e.g. a nuw add) + /// would trigger undefined behavior on overflow. + SCEV::NoWrapFlags getNoWrapFlagsFromUB(const Value *V); - const SCEVPredicate * - getWrapPredicate(const SCEVAddRecExpr *AR, - SCEVWrapPredicate::IncrementWrapFlags AddedFlags); + /// Return true if the SCEV corresponding to \p I is never poison. Proving + /// this is more complex than proving that just \p I is never poison, since + /// SCEV commons expressions across control flow, and you can have cases + /// like: + /// + /// idx0 = a + b; + /// ptr[idx0] = 100; + /// if () { + /// idx1 = a +nsw b; + /// ptr[idx1] = 200; + /// } + /// + /// where the SCEV expression (+ a b) is guaranteed to not be poison (and + /// hence not sign-overflow) only if "" is true. Since both + /// `idx0` and `idx1` will be mapped to the same SCEV expression, (+ a b), + /// it is not okay to annotate (+ a b) with in the above example. + bool isSCEVExprNeverPoison(const Instruction *I); - /// Re-writes the SCEV according to the Predicates in \p A. - const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L, - SCEVUnionPredicate &A); - /// Tries to convert the \p S expression to an AddRec expression, - /// adding additional predicates to \p Preds as required. - const SCEVAddRecExpr *convertSCEVToAddRecWithPredicates( - const SCEV *S, const Loop *L, - SmallPtrSetImpl &Preds); + /// This is like \c isSCEVExprNeverPoison but it specifically works for + /// instructions that will get mapped to SCEV add recurrences. Return true + /// if \p I will never generate poison under the assumption that \p I is an + /// add recurrence on the loop \p L. + bool isAddRecNeverPoison(const Instruction *I, const Loop *L); -private: - /// Similar to createAddRecFromPHI, but with the additional flexibility of + /// Similar to createAddRecFromPHI, but with the additional flexibility of /// suggesting runtime overflow checks in case casts are encountered. /// If successful, the analysis records that for this loop, \p SymbolicPHI, /// which is the UnknownSCEV currently representing the PHI, can be rewritten /// into an AddRec, assuming some predicates; The function then returns the /// AddRec and the predicates as a pair, and caches this pair in /// PredicatedSCEVRewrites. - /// If the analysis is not successful, a mapping from the \p SymbolicPHI to + /// If the analysis is not successful, a mapping from the \p SymbolicPHI to /// itself (with no predicates) is recorded, and a nullptr with an empty - /// predicates vector is returned as a pair. + /// predicates vector is returned as a pair. Optional>> createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI); @@ -1763,7 +1761,6 @@ private: const SCEV *getOrCreateMulExpr(SmallVectorImpl &Ops, SCEV::NoWrapFlags Flags); -private: FoldingSet UniqueSCEVs; FoldingSet UniquePreds; BumpPtrAllocator SCEVAllocator;