/// not already be in the cache.
void registerAssumption(CallInst *CI);
+ /// Remove an \@llvm.assume intrinsic from this function's cache if it has
+ /// been added to the cache earlier.
+ void unregisterAssumption(CallInst *CI);
+
/// Update the cache of values being affected by this assumption (i.e.
/// the values about which this assumption provides information).
void updateAffectedValues(CallInst *CI);
/// existing cache will be returned.
AssumptionCache &getAssumptionCache(Function &F);
+ /// Return the cached assumptions for a function if it has already been
+ /// scanned. Otherwise return nullptr.
+ AssumptionCache *lookupAssumptionCache(Function &F);
+
AssumptionCacheTracker();
~AssumptionCacheTracker() override;
class BlockFrequency;
class BlockFrequencyInfo;
class BranchProbabilityInfo;
+class AssumptionCache;
class CallInst;
class DominatorTree;
class Function;
const bool AggregateArgs;
BlockFrequencyInfo *BFI;
BranchProbabilityInfo *BPI;
+ AssumptionCache *AC;
// If true, varargs functions can be extracted.
bool AllowVarArgs;
CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr,
bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr,
BranchProbabilityInfo *BPI = nullptr,
+ AssumptionCache *AC = nullptr,
bool AllowVarArgs = false, bool AllowAlloca = false,
std::string Suffix = "");
CodeExtractor(DominatorTree &DT, Loop &L, bool AggregateArgs = false,
BlockFrequencyInfo *BFI = nullptr,
BranchProbabilityInfo *BPI = nullptr,
+ AssumptionCache *AC = nullptr,
std::string Suffix = "");
/// Perform the extraction, returning the new function.
return AVIP.first->second;
}
-void AssumptionCache::updateAffectedValues(CallInst *CI) {
+static void findAffectedValues(CallInst *CI,
+ SmallVectorImpl<Value *> &Affected) {
// Note: This code must be kept in-sync with the code in
// computeKnownBitsFromAssume in ValueTracking.
- SmallVector<Value *, 16> Affected;
auto AddAffected = [&Affected](Value *V) {
if (isa<Argument>(V)) {
Affected.push_back(V);
AddAffectedFromEq(B);
}
}
+}
+
+void AssumptionCache::updateAffectedValues(CallInst *CI) {
+ SmallVector<Value *, 16> Affected;
+ findAffectedValues(CI, Affected);
for (auto &AV : Affected) {
auto &AVV = getOrInsertAffectedValues(AV);
}
}
+void AssumptionCache::unregisterAssumption(CallInst *CI) {
+ SmallVector<Value *, 16> Affected;
+ findAffectedValues(CI, Affected);
+
+ for (auto &AV : Affected) {
+ auto AVI = AffectedValues.find_as(AV);
+ if (AVI != AffectedValues.end())
+ AffectedValues.erase(AVI);
+ }
+ remove_if(AssumeHandles, [CI](WeakTrackingVH &VH) { return CI == VH; });
+}
+
void AssumptionCache::AffectedValueCallbackVH::deleted() {
auto AVI = AC->AffectedValues.find(getValPtr());
if (AVI != AC->AffectedValues.end())
return *IP.first->second;
}
+AssumptionCache *AssumptionCacheTracker::lookupAssumptionCache(Function &F) {
+ auto I = AssumptionCaches.find_as(&F);
+ if (I != AssumptionCaches.end())
+ return I->second.get();
+ return nullptr;
+}
+
void AssumptionCacheTracker::verifyAnalysis() const {
// FIXME: In the long term the verifier should not be controllable with a
// flag. We should either fix all passes to correctly update the assumption
HotColdSplitting(ProfileSummaryInfo *ProfSI,
function_ref<BlockFrequencyInfo *(Function &)> GBFI,
function_ref<TargetTransformInfo &(Function &)> GTTI,
- std::function<OptimizationRemarkEmitter &(Function &)> *GORE)
- : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE) {}
+ std::function<OptimizationRemarkEmitter &(Function &)> *GORE,
+ function_ref<AssumptionCache *(Function &)> LAC)
+ : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE), LookupAC(LAC) {}
bool run(Module &M);
private:
bool outlineColdRegions(Function &F, bool HasProfileSummary);
Function *extractColdRegion(const BlockSequence &Region, DominatorTree &DT,
BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
- OptimizationRemarkEmitter &ORE, unsigned Count);
+ OptimizationRemarkEmitter &ORE,
+ AssumptionCache *AC, unsigned Count);
ProfileSummaryInfo *PSI;
function_ref<BlockFrequencyInfo *(Function &)> GetBFI;
function_ref<TargetTransformInfo &(Function &)> GetTTI;
std::function<OptimizationRemarkEmitter &(Function &)> *GetORE;
+ function_ref<AssumptionCache *(Function &)> LookupAC;
};
class HotColdSplittingLegacyPass : public ModulePass {
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<AssumptionCacheTracker>();
AU.addRequired<BlockFrequencyInfoWrapperPass>();
AU.addRequired<ProfileSummaryInfoWrapperPass>();
AU.addRequired<TargetTransformInfoWrapperPass>();
+ AU.addUsedIfAvailable<AssumptionCacheTracker>();
}
bool runOnModule(Module &M) override;
BlockFrequencyInfo *BFI,
TargetTransformInfo &TTI,
OptimizationRemarkEmitter &ORE,
+ AssumptionCache *AC,
unsigned Count) {
assert(!Region.empty());
// TODO: Pass BFI and BPI to update profile information.
CodeExtractor CE(Region, &DT, /* AggregateArgs */ false, /* BFI */ nullptr,
- /* BPI */ nullptr, /* AllowVarArgs */ false,
+ /* BPI */ nullptr, AC, /* AllowVarArgs */ false,
/* AllowAlloca */ false,
/* Suffix */ "cold." + std::to_string(Count));
TargetTransformInfo &TTI = GetTTI(F);
OptimizationRemarkEmitter &ORE = (*GetORE)(F);
+ AssumptionCache *AC = LookupAC(F);
// Find all cold regions.
for (BasicBlock *BB : RPOT) {
BB->dump();
});
- Function *Outlined =
- extractColdRegion(SubRegion, *DT, BFI, TTI, ORE, OutlinedFunctionID);
+ Function *Outlined = extractColdRegion(SubRegion, *DT, BFI, TTI, ORE, AC,
+ OutlinedFunctionID);
if (Outlined) {
++OutlinedFunctionID;
Changed = true;
ORE.reset(new OptimizationRemarkEmitter(&F));
return *ORE.get();
};
+ auto LookupAC = [this](Function &F) -> AssumptionCache * {
+ if (auto *ACT = getAnalysisIfAvailable<AssumptionCacheTracker>())
+ return ACT->lookupAssumptionCache(F);
+ return nullptr;
+ };
- return HotColdSplitting(PSI, GBFI, GTTI, &GetORE).run(M);
+ return HotColdSplitting(PSI, GBFI, GTTI, &GetORE, LookupAC).run(M);
}
PreservedAnalyses
HotColdSplittingPass::run(Module &M, ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- std::function<AssumptionCache &(Function &)> GetAssumptionCache =
- [&FAM](Function &F) -> AssumptionCache & {
- return FAM.getResult<AssumptionAnalysis>(F);
+ auto LookupAC = [&FAM](Function &F) -> AssumptionCache * {
+ return FAM.getCachedResult<AssumptionAnalysis>(F);
};
auto GBFI = [&FAM](Function &F) {
ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
- if (HotColdSplitting(PSI, GBFI, GTTI, &GetORE).run(M))
+ if (HotColdSplitting(PSI, GBFI, GTTI, &GetORE, LookupAC).run(M))
return PreservedAnalyses::none();
return PreservedAnalyses::all();
}
//===----------------------------------------------------------------------===//
#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Instructions.h"
AU.addRequiredID(LoopSimplifyID);
AU.addRequired<DominatorTreeWrapperPass>();
AU.addRequired<LoopInfoWrapperPass>();
+ AU.addUsedIfAvailable<AssumptionCacheTracker>();
}
};
}
if (ShouldExtractLoop) {
if (NumLoops == 0) return Changed;
--NumLoops;
- CodeExtractor Extractor(DT, *L);
+ AssumptionCache *AC = nullptr;
+ if (auto *ACT = getAnalysisIfAvailable<AssumptionCacheTracker>())
+ AC = ACT->lookupAssumptionCache(*L->getHeader()->getParent());
+ CodeExtractor Extractor(DT, *L, false, nullptr, nullptr, AC);
if (Extractor.extractCodeRegion() != nullptr) {
Changed = true;
// After extraction, the loop is replaced by a function call, so
PartialInlinerImpl(
std::function<AssumptionCache &(Function &)> *GetAC,
+ function_ref<AssumptionCache *(Function &)> LookupAC,
std::function<TargetTransformInfo &(Function &)> *GTTI,
Optional<function_ref<BlockFrequencyInfo &(Function &)>> GBFI,
ProfileSummaryInfo *ProfSI)
- : GetAssumptionCache(GetAC), GetTTI(GTTI), GetBFI(GBFI), PSI(ProfSI) {}
+ : GetAssumptionCache(GetAC), LookupAssumptionCache(LookupAC),
+ GetTTI(GTTI), GetBFI(GBFI), PSI(ProfSI) {}
bool run(Module &M);
// Main part of the transformation that calls helper functions to find
// Two constructors, one for single region outlining, the other for
// multi-region outlining.
FunctionCloner(Function *F, FunctionOutliningInfo *OI,
- OptimizationRemarkEmitter &ORE);
+ OptimizationRemarkEmitter &ORE,
+ function_ref<AssumptionCache *(Function &)> LookupAC);
FunctionCloner(Function *F, FunctionOutliningMultiRegionInfo *OMRI,
- OptimizationRemarkEmitter &ORE);
+ OptimizationRemarkEmitter &ORE,
+ function_ref<AssumptionCache *(Function &)> LookupAC);
~FunctionCloner();
// Prepare for function outlining: making sure there is only
std::unique_ptr<FunctionOutliningMultiRegionInfo> ClonedOMRI = nullptr;
std::unique_ptr<BlockFrequencyInfo> ClonedFuncBFI = nullptr;
OptimizationRemarkEmitter &ORE;
+ function_ref<AssumptionCache *(Function &)> LookupAC;
};
private:
int NumPartialInlining = 0;
std::function<AssumptionCache &(Function &)> *GetAssumptionCache;
+ function_ref<AssumptionCache *(Function &)> LookupAssumptionCache;
std::function<TargetTransformInfo &(Function &)> *GetTTI;
Optional<function_ref<BlockFrequencyInfo &(Function &)>> GetBFI;
ProfileSummaryInfo *PSI;
return ACT->getAssumptionCache(F);
};
+ auto LookupAssumptionCache = [ACT](Function &F) -> AssumptionCache * {
+ return ACT->lookupAssumptionCache(F);
+ };
+
std::function<TargetTransformInfo &(Function &)> GetTTI =
[&TTIWP](Function &F) -> TargetTransformInfo & {
return TTIWP->getTTI(F);
};
- return PartialInlinerImpl(&GetAssumptionCache, &GetTTI, NoneType::None, PSI)
+ return PartialInlinerImpl(&GetAssumptionCache, LookupAssumptionCache,
+ &GetTTI, NoneType::None, PSI)
.run(M);
}
};
}
PartialInlinerImpl::FunctionCloner::FunctionCloner(
- Function *F, FunctionOutliningInfo *OI, OptimizationRemarkEmitter &ORE)
- : OrigFunc(F), ORE(ORE) {
+ Function *F, FunctionOutliningInfo *OI, OptimizationRemarkEmitter &ORE,
+ function_ref<AssumptionCache *(Function &)> LookupAC)
+ : OrigFunc(F), ORE(ORE), LookupAC(LookupAC) {
ClonedOI = llvm::make_unique<FunctionOutliningInfo>();
// Clone the function, so that we can hack away on it.
PartialInlinerImpl::FunctionCloner::FunctionCloner(
Function *F, FunctionOutliningMultiRegionInfo *OI,
- OptimizationRemarkEmitter &ORE)
- : OrigFunc(F), ORE(ORE) {
+ OptimizationRemarkEmitter &ORE,
+ function_ref<AssumptionCache *(Function &)> LookupAC)
+ : OrigFunc(F), ORE(ORE), LookupAC(LookupAC) {
ClonedOMRI = llvm::make_unique<FunctionOutliningMultiRegionInfo>();
// Clone the function, so that we can hack away on it.
int CurrentOutlinedRegionCost = ComputeRegionCost(RegionInfo.Region);
CodeExtractor CE(RegionInfo.Region, &DT, /*AggregateArgs*/ false,
- ClonedFuncBFI.get(), &BPI, /* AllowVarargs */ false);
+ ClonedFuncBFI.get(), &BPI,
+ LookupAC(*RegionInfo.EntryBlock->getParent()),
+ /* AllowVarargs */ false);
CE.findInputsOutputs(Inputs, Outputs, Sinks);
// Extract the body of the if.
Function *OutlinedFunc =
CodeExtractor(ToExtract, &DT, /*AggregateArgs*/ false,
- ClonedFuncBFI.get(), &BPI,
+ ClonedFuncBFI.get(), &BPI, LookupAC(*ClonedFunc),
/* AllowVarargs */ true)
.extractCodeRegion();
std::unique_ptr<FunctionOutliningMultiRegionInfo> OMRI =
computeOutliningColdRegionsInfo(F, ORE);
if (OMRI) {
- FunctionCloner Cloner(F, OMRI.get(), ORE);
+ FunctionCloner Cloner(F, OMRI.get(), ORE, LookupAssumptionCache);
#ifndef NDEBUG
if (TracePartialInlining) {
if (!OI)
return {false, nullptr};
- FunctionCloner Cloner(F, OI.get(), ORE);
+ FunctionCloner Cloner(F, OI.get(), ORE, LookupAssumptionCache);
Cloner.NormalizeReturnBlock();
Function *OutlinedFunction = Cloner.doSingleRegionFunctionOutlining();
return FAM.getResult<AssumptionAnalysis>(F);
};
+ auto LookupAssumptionCache = [&FAM](Function &F) -> AssumptionCache * {
+ return FAM.getCachedResult<AssumptionAnalysis>(F);
+ };
+
std::function<BlockFrequencyInfo &(Function &)> GetBFI =
[&FAM](Function &F) -> BlockFrequencyInfo & {
return FAM.getResult<BlockFrequencyAnalysis>(F);
ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
- if (PartialInlinerImpl(&GetAssumptionCache, &GetTTI, {GetBFI}, PSI)
+ if (PartialInlinerImpl(&GetAssumptionCache, LookupAssumptionCache, &GetTTI,
+ {GetBFI}, PSI)
.run(M))
return PreservedAnalyses::none();
return PreservedAnalyses::all();
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include <vector>
using namespace llvm;
+using namespace llvm::PatternMatch;
using ProfileCount = Function::ProfileCount;
#define DEBUG_TYPE "code-extractor"
CodeExtractor::CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT,
bool AggregateArgs, BlockFrequencyInfo *BFI,
- BranchProbabilityInfo *BPI, bool AllowVarArgs,
- bool AllowAlloca, std::string Suffix)
+ BranchProbabilityInfo *BPI, AssumptionCache *AC,
+ bool AllowVarArgs, bool AllowAlloca,
+ std::string Suffix)
: DT(DT), AggregateArgs(AggregateArgs || AggregateArgsOpt), BFI(BFI),
- BPI(BPI), AllowVarArgs(AllowVarArgs),
+ BPI(BPI), AC(AC), AllowVarArgs(AllowVarArgs),
Blocks(buildExtractionBlockSet(BBs, DT, AllowVarArgs, AllowAlloca)),
Suffix(Suffix) {}
CodeExtractor::CodeExtractor(DominatorTree &DT, Loop &L, bool AggregateArgs,
BlockFrequencyInfo *BFI,
- BranchProbabilityInfo *BPI, std::string Suffix)
+ BranchProbabilityInfo *BPI, AssumptionCache *AC,
+ std::string Suffix)
: DT(&DT), AggregateArgs(AggregateArgs || AggregateArgsOpt), BFI(BFI),
- BPI(BPI), AllowVarArgs(false),
+ BPI(BPI), AC(AC), AllowVarArgs(false),
Blocks(buildExtractionBlockSet(L.getBlocks(), &DT,
/* AllowVarArgs */ false,
/* AllowAlloca */ false)),
// Insert this basic block into the new function
newBlocks.push_back(Block);
+
+ // Remove @llvm.assume calls that were moved to the new function from the
+ // old function's assumption cache.
+ if (AC)
+ for (auto &I : *Block)
+ if (match(&I, m_Intrinsic<Intrinsic::assume>()))
+ AC->unregisterAssumption(cast<CallInst>(&I));
}
}
--- /dev/null
+; RUN: opt -passes="function(slp-vectorizer),module(hotcoldsplit),function(slp-vectorizer,print<assumptions>)" -disable-output %s 2>&1 | FileCheck %s
+;
+; Make sure this compiles. Check that function assumption cache is refreshed
+; after extracting blocks with assume calls from the function.
+
+; CHECK: Cached assumptions for function: fun
+; CHECK-NEXT: Cached assumptions for function: fun.cold
+; CHECK-NEXT: %cmp = icmp uge i32 %x, 64
+
+declare void @fun2(i32) #0
+
+define void @fun(i32 %x) {
+entry:
+ br i1 undef, label %if.then, label %if.else
+
+if.then:
+ ret void
+
+if.else:
+ %cmp = icmp uge i32 %x, 64
+ call void @llvm.assume(i1 %cmp)
+ call void @fun2(i32 %x)
+ unreachable
+}
+
+declare void @llvm.assume(i1) #1
+
+attributes #0 = { alwaysinline }
+attributes #1 = { nounwind }
projects / llvm / commitdiff