From d97a419cbe0511ab45b92bf67bb09aed868ba1b5 Mon Sep 17 00:00:00 2001 From: Vlad Tsyrklevich Date: Wed, 27 Feb 2019 03:45:28 +0000 Subject: [PATCH] Revert "[PGO] Context sensitive PGO (part 1)" This reverts commit r354930, it was causing UBSan failures. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@354953 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/InitializePasses.h | 1 - include/llvm/LTO/Config.h | 6 - include/llvm/LinkAllPasses.h | 1 - include/llvm/ProfileData/InstrProf.h | 16 -- include/llvm/ProfileData/InstrProfData.inc | 2 - include/llvm/Transforms/Instrumentation.h | 18 +- .../Instrumentation/InstrProfiling.h | 6 +- .../Instrumentation/PGOInstrumentation.h | 31 +-- lib/Passes/PassBuilder.cpp | 3 +- lib/ProfileData/InstrProf.cpp | 19 -- .../Instrumentation/InstrProfiling.cpp | 53 +---- .../Instrumentation/PGOInstrumentation.cpp | 203 +++++------------- tools/gold/gold-plugin.cpp | 14 +- tools/llvm-lto2/llvm-lto2.cpp | 11 - 14 files changed, 75 insertions(+), 309 deletions(-) diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index 9b549b07044..acc7065de98 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -299,7 +299,6 @@ void initializePEIPass(PassRegistry&); void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&); void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&); void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&); -void initializePGOInstrumentationGenCreateVarLegacyPassPass(PassRegistry&); void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&); void initializePHIEliminationPass(PassRegistry&); void initializePartialInlinerLegacyPassPass(PassRegistry&); diff --git a/include/llvm/LTO/Config.h b/include/llvm/LTO/Config.h index bf8cc98ac4d..f0e1b1d1649 100644 --- a/include/llvm/LTO/Config.h +++ b/include/llvm/LTO/Config.h @@ -55,9 +55,6 @@ struct Config { /// Disable entirely the optimizer, including importing for ThinLTO bool CodeGenOnly = false; - /// Run PGO context sensitive IR instrumentation. - bool RunCSIRInstr = false; - /// If this field is set, the set of passes run in the middle-end optimizer /// will be the one specified by the string. Only works with the new pass /// manager as the old one doesn't have this ability. @@ -76,9 +73,6 @@ struct Config { /// with this triple. std::string DefaultTriple; - /// Context Sensitive PGO profile path. - std::string CSIRProfile; - /// Sample PGO profile path. std::string SampleProfile; diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h index fb3b19e2dab..ec81b1618cc 100644 --- a/include/llvm/LinkAllPasses.h +++ b/include/llvm/LinkAllPasses.h @@ -102,7 +102,6 @@ namespace { (void) llvm::createGCOVProfilerPass(); (void) llvm::createPGOInstrumentationGenLegacyPass(); (void) llvm::createPGOInstrumentationUseLegacyPass(); - (void) llvm::createPGOInstrumentationGenCreateVarLegacyPass(); (void) llvm::createPGOIndirectCallPromotionLegacyPass(); (void) llvm::createPGOMemOPSizeOptLegacyPass(); (void) llvm::createInstrProfilingLegacyPass(); diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index 1d55a8ea2c6..9ea1b9bd2fe 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -767,20 +767,10 @@ struct NamedInstrProfRecord : InstrProfRecord { StringRef Name; uint64_t Hash; - // We reserve this bit as the flag for context sensitive profile record. - static const int CS_FLAG_IN_FUNC_HASH = 60; - NamedInstrProfRecord() = default; NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} - - static bool hasCSFlagInHash(uint64_t FuncHash) { - return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); - } - static void setCSFlagInHash(uint64_t &FuncHash) { - FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH); - } }; uint32_t InstrProfRecord::getNumValueKinds() const { @@ -1014,8 +1004,6 @@ namespace RawInstrProf { // from control data struct is changed from raw pointer to Name's MD5 value. // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the // raw header. -// Version 5: Bit 60 of FuncHash is reserved for the flag for the context -// sensitive records. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); @@ -1052,10 +1040,6 @@ struct Header { void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast); -// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime -// aware this is an ir_level profile so it can set the version flag. -void createIRLevelProfileFlagVar(Module &M, bool IsCS); - // Create the variable for the profile file name. void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc index 19d46512925..e1e2df55699 100644 --- a/include/llvm/ProfileData/InstrProfData.inc +++ b/include/llvm/ProfileData/InstrProfData.inc @@ -635,12 +635,10 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. - * 1 in bit 57 indicates there are context-sensitive records in the profile. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) -#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h index 9eea552b238..ff0ebf75245 100644 --- a/include/llvm/Transforms/Instrumentation.h +++ b/include/llvm/Transforms/Instrumentation.h @@ -87,14 +87,10 @@ struct GCOVOptions { ModulePass *createGCOVProfilerPass(const GCOVOptions &Options = GCOVOptions::getDefault()); -// PGO Instrumention. Parameter IsCS indicates if this is the context senstive -// instrumentation. -ModulePass *createPGOInstrumentationGenLegacyPass(bool IsCS = false); +// PGO Instrumention +ModulePass *createPGOInstrumentationGenLegacyPass(); ModulePass * -createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""), - bool IsCS = false); -ModulePass *createPGOInstrumentationGenCreateVarLegacyPass( - StringRef CSInstrName = StringRef("")); +createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef("")); ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false, bool SamplePGO = false); FunctionPass *createPGOMemOPSizeOptLegacyPass(); @@ -136,19 +132,15 @@ struct InstrProfOptions { // Use atomic profile counter increments. bool Atomic = false; - // Use BFI to guide register promotion - bool UseBFIInPromotion = false; - // Name of the profile file to use as output std::string InstrProfileOutput; InstrProfOptions() = default; }; -/// Insert frontend instrumentation based profiling. Parameter IsCS indicates if -// this is the context senstive instrumentation. +/// Insert frontend instrumentation based profiling. ModulePass *createInstrProfilingLegacyPass( - const InstrProfOptions &Options = InstrProfOptions(), bool IsCS = false); + const InstrProfOptions &Options = InstrProfOptions()); FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, bool Recover = false); diff --git a/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/include/llvm/Transforms/Instrumentation/InstrProfiling.h index 4428be7e4d4..4fad6506810 100644 --- a/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -35,8 +35,7 @@ using LoadStorePair = std::pair; class InstrProfiling : public PassInfoMixin { public: InstrProfiling() = default; - InstrProfiling(const InstrProfOptions &Options, bool IsCS) - : Options(Options), IsCS(IsCS) {} + InstrProfiling(const InstrProfOptions &Options) : Options(Options) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); bool run(Module &M, const TargetLibraryInfo &TLI); @@ -61,9 +60,6 @@ private: GlobalVariable *NamesVar; size_t NamesSize; - // Is this lowering for the context-sensitive instrumentation. - bool IsCS; - // vector of counter load/store pairs to be register promoted. std::vector PromotionCandidates; diff --git a/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h index e59d1ba64cf..4f22e28232e 100644 --- a/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ b/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -17,7 +17,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/PassManager.h" -#include "llvm/ProfileData/InstrProf.h" #include #include @@ -27,51 +26,23 @@ class Function; class Instruction; class Module; -/// The instrumentation (profile-instr-gen) pass for IR based PGO. -// We use this pass to create COMDAT profile variables for context -// sensitive PGO (CSPGO). The reason to have a pass for this is CSPGO -// can be run after LTO/ThinLTO linking. Lld linker needs to see -// all the COMDAT variables before linking. So we have this pass -// always run before linking for CSPGO. -class PGOInstrumentationGenCreateVar - : public PassInfoMixin { -public: - PGOInstrumentationGenCreateVar(std::string CSInstrName = "") - : CSInstrName(CSInstrName) {} - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { - createProfileFileNameVar(M, CSInstrName); - createIRLevelProfileFlagVar(M, /* IsCS */ true); - return PreservedAnalyses::all(); - } - -private: - std::string CSInstrName; -}; - /// The instrumentation (profile-instr-gen) pass for IR based PGO. class PGOInstrumentationGen : public PassInfoMixin { public: - PGOInstrumentationGen(bool IsCS = false) : IsCS(IsCS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); - -private: - // If this is a context sensitive instrumentation. - bool IsCS; }; /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin { public: PGOInstrumentationUse(std::string Filename = "", - std::string RemappingFilename = "", bool IsCS = false); + std::string RemappingFilename = ""); PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; std::string ProfileRemappingFileName; - // If this is a context sensitive instrumentation. - bool IsCS; }; /// The indirect function call promotion pass. diff --git a/lib/Passes/PassBuilder.cpp b/lib/Passes/PassBuilder.cpp index cfc436183eb..9d5f5aa4b78 100644 --- a/lib/Passes/PassBuilder.cpp +++ b/lib/Passes/PassBuilder.cpp @@ -569,8 +569,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, if (!ProfileGenFile.empty()) Options.InstrProfileOutput = ProfileGenFile; Options.DoCounterPromotion = true; - Options.UseBFIInPromotion = false; - MPM.addPass(InstrProfiling(Options, false)); + MPM.addPass(InstrProfiling(Options)); } if (!ProfileUseFile.empty()) diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp index ccb651a22b3..8a2ff7769f1 100644 --- a/lib/ProfileData/InstrProf.cpp +++ b/lib/ProfileData/InstrProf.cpp @@ -1011,25 +1011,6 @@ void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart, assert(RangeLast >= RangeStart); } -// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime -// aware this is an ir_level profile so it can set the version flag. -void createIRLevelProfileFlagVar(Module &M, bool IsCS) { - const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); - Type *IntTy64 = Type::getInt64Ty(M.getContext()); - uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); - if (IsCS) - ProfileVersion |= VARIANT_MASK_CSIR_PROF; - auto IRLevelVersionVariable = new GlobalVariable( - M, IntTy64, true, GlobalValue::WeakAnyLinkage, - Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName); - IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); - Triple TT(M.getTargetTriple()); - if (TT.supportsCOMDAT()) { - IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage); - IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName)); - } -} - // Create the variable for the profile file name. void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) { if (InstrProfileOutput.empty()) diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp index 9c1dc5c8e86..e77427aa91d 100644 --- a/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -18,8 +18,6 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Attributes.h" @@ -149,8 +147,8 @@ public: static char ID; InstrProfilingLegacyPass() : ModulePass(ID) {} - InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS) - : ModulePass(ID), InstrProf(Options, IsCS) {} + InstrProfilingLegacyPass(const InstrProfOptions &Options) + : ModulePass(ID), InstrProf(Options) {} StringRef getPassName() const override { return "Frontend instrumentation-based coverage lowering"; @@ -234,9 +232,9 @@ class PGOCounterPromoter { public: PGOCounterPromoter( DenseMap> &LoopToCands, - Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI) + Loop &CurLoop, LoopInfo &LI) : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop), - LI(LI), BFI(BFI) { + LI(LI) { SmallVector LoopExitBlocks; SmallPtrSet BlockSet; @@ -265,20 +263,6 @@ public: SSAUpdater SSA(&NewPHIs); Value *InitVal = ConstantInt::get(Cand.first->getType(), 0); - // If BFI is set, we will use it to guide the promotions. - if (BFI) { - auto *BB = Cand.first->getParent(); - auto InstrCount = BFI->getBlockProfileCount(BB); - if (!InstrCount) - continue; - auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader()); - // If the average loop trip count is not greater than 1.5, we skip - // promotion. - if (PreheaderCount && - (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2)) - continue; - } - PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, L.getLoopPreheader(), ExitBlocks, InsertPts, LoopToCandidates, LI); @@ -328,11 +312,6 @@ private: SmallVector ExitingBlocks; LP->getExitingBlocks(ExitingBlocks); - - // If BFI is set, we do more aggressive promotions based on BFI. - if (BFI) - return (unsigned)-1; - // Not considierered speculative. if (ExitingBlocks.size() == 1) return MaxNumOfPromotionsPerLoop; @@ -364,7 +343,6 @@ private: SmallVector InsertPts; Loop &L; LoopInfo &LI; - BlockFrequencyInfo *BFI; }; } // end anonymous namespace @@ -387,9 +365,8 @@ INITIALIZE_PASS_END( "Frontend instrumentation-based coverage lowering.", false, false) ModulePass * -llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options, - bool IsCS) { - return new InstrProfilingLegacyPass(Options, IsCS); +llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) { + return new InstrProfilingLegacyPass(Options); } static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) { @@ -438,13 +415,6 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) { LoopInfo LI(DT); DenseMap> LoopPromotionCandidates; - std::unique_ptr BFI; - if (Options.UseBFIInPromotion) { - std::unique_ptr BPI; - BPI.reset(new BranchProbabilityInfo(*F, LI, TLI)); - BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI)); - } - for (const auto &LoadStore : PromotionCandidates) { auto *CounterLoad = LoadStore.first; auto *CounterStore = LoadStore.second; @@ -460,7 +430,7 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) { // Do a post-order traversal of the loops so that counter updates can be // iteratively hoisted outside the loop nest. for (auto *Loop : llvm::reverse(Loops)) { - PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get()); + PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI); Promoter.run(&TotalCountersPromoted); } } @@ -711,6 +681,7 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { // Don't do this for Darwin. compiler-rt uses linker magic. if (TT.isOSDarwin()) return false; + // Use linker script magic to get data/cnts/name start/end. if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() || TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows()) @@ -1014,12 +985,8 @@ void InstrProfiling::emitUses() { } void InstrProfiling::emitInitialization() { - // Create ProfileFileName variable. Don't don't this for the - // context-sensitive instrumentation lowering: This lowering is after - // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should - // have already create the variable before LTO/ThinLTO linking. - if (!IsCS) - createProfileFileNameVar(*M, Options.InstrProfileOutput); + // Create variable for profile name. + createProfileFileNameVar(*M, Options.InstrProfileOutput); Function *RegisterF = M->getFunction(getInstrProfRegFuncsName()); if (!RegisterF) return; diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index c956f4e32e9..bb2e3359881 100644 --- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -65,7 +65,6 @@ #include "llvm/Analysis/IndirectCallVisitor.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" -#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" @@ -133,19 +132,6 @@ STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts."); STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile."); STATISTIC(NumOfPGOMissing, "Number of functions without profile."); STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations."); -STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO."); -STATISTIC(NumOfCSPGOSelectInsts, - "Number of select instruction instrumented in CSPGO."); -STATISTIC(NumOfCSPGOMemIntrinsics, - "Number of mem intrinsics instrumented in CSPGO."); -STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO."); -STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO."); -STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO."); -STATISTIC(NumOfCSPGOFunc, - "Number of functions having valid profile counts in CSPGO."); -STATISTIC(NumOfCSPGOMismatch, - "Number of functions having mismatch profile in CSPGO."); -STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO."); // Command line option to specify the file to read profile from. This is // mainly used for testing. @@ -397,8 +383,7 @@ class PGOInstrumentationGenLegacyPass : public ModulePass { public: static char ID; - PGOInstrumentationGenLegacyPass(bool IsCS = false) - : ModulePass(ID), IsCS(IsCS) { + PGOInstrumentationGenLegacyPass() : ModulePass(ID) { initializePGOInstrumentationGenLegacyPassPass( *PassRegistry::getPassRegistry()); } @@ -406,8 +391,6 @@ public: StringRef getPassName() const override { return "PGOInstrumentationGenPass"; } private: - // Is this is context-sensitive instrumentation. - bool IsCS; bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -420,8 +403,8 @@ public: static char ID; // Provide the profile filename as the parameter. - PGOInstrumentationUseLegacyPass(std::string Filename = "", bool IsCS = false) - : ModulePass(ID), ProfileFileName(std::move(Filename)), IsCS(IsCS) { + PGOInstrumentationUseLegacyPass(std::string Filename = "") + : ModulePass(ID), ProfileFileName(std::move(Filename)) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; initializePGOInstrumentationUseLegacyPassPass( @@ -432,38 +415,14 @@ public: private: std::string ProfileFileName; - // Is this is context-sensitive instrumentation use. - bool IsCS; bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); AU.addRequired(); } }; -class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass { -public: - static char ID; - StringRef getPassName() const override { - return "PGOInstrumentationGenCreateVarPass"; - } - PGOInstrumentationGenCreateVarLegacyPass(std::string CSInstrName = "") - : ModulePass(ID), InstrProfileOutput(CSInstrName) { - initializePGOInstrumentationGenCreateVarLegacyPassPass( - *PassRegistry::getPassRegistry()); - } - -private: - bool runOnModule(Module &M) override { - createProfileFileNameVar(M, InstrProfileOutput); - createIRLevelProfileFlagVar(M, true); - return false; - } - std::string InstrProfileOutput; -}; - } // end anonymous namespace char PGOInstrumentationGenLegacyPass::ID = 0; @@ -475,8 +434,8 @@ INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) -ModulePass *llvm::createPGOInstrumentationGenLegacyPass(bool IsCS) { - return new PGOInstrumentationGenLegacyPass(IsCS); +ModulePass *llvm::createPGOInstrumentationGenLegacyPass() { + return new PGOInstrumentationGenLegacyPass(); } char PGOInstrumentationUseLegacyPass::ID = 0; @@ -485,25 +444,11 @@ INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) -INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) -ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename, - bool IsCS) { - return new PGOInstrumentationUseLegacyPass(Filename.str(), IsCS); -} - -char PGOInstrumentationGenCreateVarLegacyPass::ID = 0; - -INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass, - "pgo-instr-gen-create-var", - "Create PGO instrumentation version variable for CSPGO.", false, - false) - -ModulePass * -llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) { - return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName); +ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) { + return new PGOInstrumentationUseLegacyPass(Filename.str()); } namespace { @@ -551,9 +496,6 @@ template class FuncPGOInstrumentation { private: Function &F; - // Is this is context-sensitive instrumentation. - bool IsCS; - // A map that stores the Comdat group in function F. std::unordered_multimap &ComdatMembers; @@ -593,23 +535,15 @@ public: Function &Func, std::unordered_multimap &ComdatMembers, bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr, - BlockFrequencyInfo *BFI = nullptr, bool IsCS = false) - : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), - ValueSites(IPVK_Last + 1), SIVisitor(Func), MIVisitor(Func), - MST(F, BPI, BFI) { + BlockFrequencyInfo *BFI = nullptr) + : F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1), + SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) { // This should be done before CFG hash computation. SIVisitor.countSelects(Func); MIVisitor.countMemIntrinsics(Func); - if (!IsCS) { - NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); - NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); - NumOfPGOBB += MST.BBInfos.size(); - ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func); - } else { - NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); - NumOfCSPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); - NumOfCSPGOBB += MST.BBInfos.size(); - } + NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); + NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); + ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func); ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func); FuncName = getPGOFuncName(F); @@ -618,12 +552,13 @@ public: renameComdatFunction(); LLVM_DEBUG(dumpInfo("after CFGMST")); + NumOfPGOBB += MST.BBInfos.size(); for (auto &E : MST.AllEdges) { if (E->Removed) continue; - IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++; + NumOfPGOEdge++; if (!E->InMST) - IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++; + NumOfPGOInstrument++; } if (CreateGlobalVar) @@ -662,17 +597,9 @@ void FuncPGOInstrumentation::computeCFGHash() { } } JC.update(Indexes); - - // Hash format for context sensitive profile. Reserve 4 bits for other - // information. FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 | (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 | - //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 | (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC(); - // Reserve bit 60-63 for other information purpose. - FunctionHash &= 0x0FFFFFFFFFFFFFFF; - if (IsCS) - NamedInstrProfRecord::setCSFlagInHash(FunctionHash); LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n" << " CRC = " << JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts() @@ -778,7 +705,7 @@ BasicBlock *FuncPGOInstrumentation::getInstrBB(Edge *E) { // For a critical edge, we have to split. Instrument the newly // created BB. - IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++; + NumOfPGOSplit++; LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index << " --> " << getBBInfo(DestBB).Index << "\n"); unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); @@ -793,14 +720,12 @@ BasicBlock *FuncPGOInstrumentation::getInstrBB(Edge *E) { // Critical edges will be split. static void instrumentOneFunc( Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, - std::unordered_multimap &ComdatMembers, - bool IsCS) { + std::unordered_multimap &ComdatMembers) { // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); - FuncPGOInstrumentation FuncInfo(F, ComdatMembers, true, BPI, - BFI, IsCS); + BFI); unsigned NumCounters = FuncInfo.getNumCounters(); uint32_t I = 0; @@ -927,10 +852,10 @@ public: PGOUseFunc(Function &Func, Module *Modu, std::unordered_multimap &ComdatMembers, BranchProbabilityInfo *BPI = nullptr, - BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false) + BlockFrequencyInfo *BFIin = nullptr) : F(Func), M(Modu), BFI(BFIin), - FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS), - FreqAttr(FFA_Normal), IsCS(IsCS) {} + FuncInfo(Func, ComdatMembers, false, BPI, BFIin), + FreqAttr(FFA_Normal) {} // Read counts for the instrumented BB from profile. bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros); @@ -1003,9 +928,6 @@ private: // Function hotness info derived from profile. FuncFreqAttr FreqAttr; - // Is to use the context sensitive profile. - bool IsCS; - // Find the Instrumented BB and set the value. void setInstrumentedCounts(const std::vector &CountFromProfile); @@ -1099,31 +1021,23 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { auto Err = IPE.get(); bool SkipWarning = false; - LLVM_DEBUG(dbgs() << "Error in reading profile for Func " - << FuncInfo.FuncName << ": "); if (Err == instrprof_error::unknown_function) { - IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++; + NumOfPGOMissing++; SkipWarning = !PGOWarnMissing; - LLVM_DEBUG(dbgs() << "unknown function"); } else if (Err == instrprof_error::hash_mismatch || Err == instrprof_error::malformed) { - IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++; + NumOfPGOMismatch++; SkipWarning = NoPGOWarnMismatch || (NoPGOWarnMismatchComdat && (F.hasComdat() || F.getLinkage() == GlobalValue::AvailableExternallyLinkage)); - LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")"); } - LLVM_DEBUG(dbgs() << " IsCS=" << IsCS << "\n"); if (SkipWarning) return; - std::string Msg = IPE.message() + std::string(" ") + F.getName().str() + - std::string(" Hash = ") + - std::to_string(FuncInfo.FunctionHash); - + std::string Msg = IPE.message() + std::string(" ") + F.getName().str(); Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); }); @@ -1132,7 +1046,7 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) ProfileRecord = std::move(Result.get()); std::vector &CountFromProfile = ProfileRecord.Counts; - IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++; + NumOfPGOFunc++; LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n"); uint64_t ValueSum = 0; for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) { @@ -1147,11 +1061,7 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) getBBInfo(nullptr).UnknownCountInEdge = 2; setInstrumentedCounts(CountFromProfile); -#if 0 - ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS); -#else ProgramMaxCount = PGOReader->getMaximumFunctionCount(); -#endif return true; } @@ -1256,8 +1166,7 @@ void PGOUseFunc::populateCounters() { // Assign the scaled count values to the BB with multiple out edges. void PGOUseFunc::setBranchWeights() { // Generate MD_prof metadata for every branch instruction. - LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName() - << " IsCS=" << IsCS << "\n"); + LLVM_DEBUG(dbgs() << "\nSetting branch weights.\n"); for (auto &BB : F) { Instruction *TI = BB.getTerminator(); if (TI->getNumSuccessors() < 2) @@ -1265,7 +1174,6 @@ void PGOUseFunc::setBranchWeights() { if (!(isa(TI) || isa(TI) || isa(TI))) continue; - if (getBBInfo(&BB).CountValue == 0) continue; @@ -1443,6 +1351,24 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) { } } +// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime +// aware this is an ir_level profile so it can set the version flag. +static void createIRLevelProfileFlagVariable(Module &M) { + Type *IntTy64 = Type::getInt64Ty(M.getContext()); + uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); + auto IRLevelVersionVariable = new GlobalVariable( + M, IntTy64, true, GlobalVariable::ExternalLinkage, + Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), + INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); + IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); + Triple TT(M.getTargetTriple()); + if (!TT.supportsCOMDAT()) + IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage); + else + IRLevelVersionVariable->setComdat(M.getOrInsertComdat( + StringRef(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)))); +} + // Collect the set of members for each Comdat in module M and store // in ComdatMembers. static void collectComdatMembers( @@ -1463,11 +1389,8 @@ static void collectComdatMembers( static bool InstrumentAllFunctions( Module &M, function_ref LookupBPI, - function_ref LookupBFI, bool IsCS) { - // For the context-sensitve instrumentation, we should have a separated pass - // (before LTO/ThinLTO linking) to create these variables. - if (!IsCS) - createIRLevelProfileFlagVar(M, /* IsCS */ false); + function_ref LookupBFI) { + createIRLevelProfileFlagVariable(M); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers); @@ -1476,7 +1399,7 @@ static bool InstrumentAllFunctions( continue; auto *BPI = LookupBPI(F); auto *BFI = LookupBFI(F); - instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS); + instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers); } return true; } @@ -1491,7 +1414,7 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { auto LookupBFI = [this](Function &F) { return &this->getAnalysis(F).getBFI(); }; - return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS); + return InstrumentAllFunctions(M, LookupBPI, LookupBFI); } PreservedAnalyses PGOInstrumentationGen::run(Module &M, @@ -1505,7 +1428,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M, return &FAM.getResult(F); }; - if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS)) + if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1514,7 +1437,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M, static bool annotateAllFunctions( Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref LookupBPI, - function_ref LookupBFI, bool IsCS) { + function_ref LookupBFI) { LLVM_DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. @@ -1535,11 +1458,6 @@ static bool annotateAllFunctions( StringRef("Cannot get PGOReader"))); return false; } -#if 0 - if (!PGOReader->hasCSIRLevelProfile() && IsCS) - return false; -#endif - // TODO: might need to change the warning once the clang option is finalized. if (!PGOReader->isIRLevelProfile()) { Ctx.diagnose(DiagnosticInfoPGOProfile( @@ -1559,7 +1477,7 @@ static bool annotateAllFunctions( // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); - PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS); + PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI); bool AllZeros = false; if (!Func.readCounters(PGOReader.get(), AllZeros)) continue; @@ -1607,14 +1525,7 @@ static bool annotateAllFunctions( } } } -#if 0 - M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), - IsCS ? ProfileSummary::PSK_CSInstr - : ProfileSummary::PSK_Instr); -#else M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext())); -#endif - // Set function hotness attribute from the profile. // We have to apply these attributes at the end because their presence // can affect the BranchProbabilityInfo of any callers, resulting in an @@ -1633,10 +1544,9 @@ static bool annotateAllFunctions( } PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename, - std::string RemappingFilename, - bool IsCS) + std::string RemappingFilename) : ProfileFileName(std::move(Filename)), - ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS) { + ProfileRemappingFileName(std::move(RemappingFilename)) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; if (!PGOTestProfileRemappingFile.empty()) @@ -1656,7 +1566,7 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M, }; if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, - LookupBPI, LookupBFI, IsCS)) + LookupBPI, LookupBFI)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1673,8 +1583,7 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { return &this->getAnalysis(F).getBFI(); }; - return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, - IsCS); + return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI); } static std::string getSimpleNodeName(const BasicBlock *Node) { diff --git a/tools/gold/gold-plugin.cpp b/tools/gold/gold-plugin.cpp index fbf78b02551..626fb5f6363 100644 --- a/tools/gold/gold-plugin.cpp +++ b/tools/gold/gold-plugin.cpp @@ -209,10 +209,6 @@ namespace options { static std::string OptRemarksFilename; static bool OptRemarksWithHotness = false; - // Context sensitive PGO options. - static std::string cs_profile_path; - static bool cs_pgo_gen = false; - static void process_plugin_option(const char *opt_) { if (opt_ == nullptr) @@ -272,11 +268,7 @@ namespace options { } else if (opt == "disable-verify") { DisableVerify = true; } else if (opt.startswith("sample-profile=")) { - sample_profile = opt.substr(strlen("sample-profile=")); - } else if (opt == "cs-profile-generate") { - cs_pgo_gen = true; - } else if (opt.startswith("cs-profile-path=")) { - cs_profile_path = opt.substr(strlen("cs-profile-path=")); + sample_profile= opt.substr(strlen("sample-profile=")); } else if (opt == "new-pass-manager") { new_pass_manager = true; } else if (opt == "debug-pass-manager") { @@ -900,10 +892,6 @@ static std::unique_ptr createLTO(IndexWriteCallback OnIndexWrite, if (!options::sample_profile.empty()) Conf.SampleProfile = options::sample_profile; - if (!options::cs_profile_path.empty()) - Conf.CSIRProfile = options::cs_profile_path; - Conf.RunCSIRInstr = options::cs_pgo_gen; - Conf.DwoDir = options::dwo_dir; // Set up optimization remarks handling. diff --git a/tools/llvm-lto2/llvm-lto2.cpp b/tools/llvm-lto2/llvm-lto2.cpp index 6cceb8e4cf8..d9950815a61 100644 --- a/tools/llvm-lto2/llvm-lto2.cpp +++ b/tools/llvm-lto2/llvm-lto2.cpp @@ -104,15 +104,6 @@ static cl::opt SamplePGOFile("lto-sample-profile-file", cl::desc("Specify a SamplePGO profile file")); -static cl::opt - CSPGOFile("lto-cspgo-profile-file", - cl::desc("Specify a context sensitive PGO profile file")); - -static cl::opt - RunCSIRInstr("lto-cspgo-gen", - cl::desc("Run PGO context sensitive IR instrumentation"), - cl::init(false), cl::Hidden); - static cl::opt UseNewPM("use-new-pm", cl::desc("Run LTO passes using the new pass manager"), @@ -223,8 +214,6 @@ static int run(int argc, char **argv) { Conf.RemarksWithHotness = OptRemarksWithHotness; Conf.SampleProfile = SamplePGOFile; - Conf.CSIRProfile = CSPGOFile; - Conf.RunCSIRInstr = RunCSIRInstr; // Run a custom pipeline, if asked for. Conf.OptPipeline = OptPipeline; -- 2.40.0