auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); };
CompileThreads.async(std::move(Work));
});
- JITEvaluatedSymbol SpeculatorSymbol(JITTargetAddress(&S),
- JITSymbolFlags::Exported);
- ExitOnErr(this->ES->getMainJITDylib().define(
- absoluteSymbols({{Mangle("__orc_speculator"), SpeculatorSymbol}})));
+ ExitOnErr(S.addSpeculationRuntime(this->ES->getMainJITDylib(), Mangle));
LocalCXXRuntimeOverrides CXXRuntimeoverrides;
ExitOnErr(CXXRuntimeoverrides.enable(this->ES->getMainJITDylib(), Mangle));
}
//===----------------------------------------------------------------------===//
// \file
/// Contains the Analyses and Result Interpretation to select likely functions
-/// to Speculatively compile before they are called. [Experimentation]
+/// to Speculatively compile before they are called. [Purely Experimentation]
//===----------------------------------------------------------------------===//
#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H
#define LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
#include "llvm/ExecutionEngine/Orc/Speculation.h"
namespace orc {
-// Direct calls in high frequency basic blocks are extracted.
-class BlockFreqQuery {
-private:
+// Provides common code.
+class SpeculateQuery {
+protected:
void findCalles(const BasicBlock *, DenseSet<StringRef> &);
- size_t numBBToGet(size_t);
+ bool isStraightLine(const Function &F);
public:
using ResultTy = Optional<DenseMap<StringRef, DenseSet<StringRef>>>;
+};
+// Direct calls in high frequency basic blocks are extracted.
+class BlockFreqQuery : public SpeculateQuery {
+ size_t numBBToGet(size_t);
+
+public:
// Find likely next executables based on IR Block Frequency
- ResultTy operator()(Function &F, FunctionAnalysisManager &FAM);
+ ResultTy operator()(Function &F);
+};
+
+// This Query generates a sequence of basic blocks which follows the order of
+// execution.
+// A handful of BB with higher block frequencies are taken, then path to entry
+// and end BB are discovered by traversing up & down the CFG.
+class SequenceBBQuery : public SpeculateQuery {
+ struct WalkDirection {
+ bool Upward = true, Downward = true;
+ // the block associated contain a call
+ bool CallerBlock = false;
+ };
+
+public:
+ using VisitedBlocksInfoTy = DenseMap<const BasicBlock *, WalkDirection>;
+ using BlockListTy = SmallVector<const BasicBlock *, 8>;
+ using BackEdgesInfoTy =
+ SmallVector<std::pair<const BasicBlock *, const BasicBlock *>, 8>;
+ using BlockFreqInfoTy =
+ SmallVector<std::pair<const BasicBlock *, uint64_t>, 8>;
+
+private:
+ std::size_t getHottestBlocks(std::size_t TotalBlocks);
+ BlockListTy rearrangeBB(const Function &, const BlockListTy &);
+ BlockListTy queryCFG(Function &, const BlockListTy &);
+ void traverseToEntryBlock(const BasicBlock *, const BlockListTy &,
+ const BackEdgesInfoTy &,
+ const BranchProbabilityInfo *,
+ VisitedBlocksInfoTy &);
+ void traverseToExitBlock(const BasicBlock *, const BlockListTy &,
+ const BackEdgesInfoTy &,
+ const BranchProbabilityInfo *,
+ VisitedBlocksInfoTy &);
+
+public:
+ ResultTy operator()(Function &F);
};
} // namespace orc
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/Debug.h"
#include <mutex>
#include <type_traits>
{
std::lock_guard<std::mutex> Lockit(ConcurrentAccess);
auto It = GlobalSpecMap.find(FAddr);
- // Kill this when jump on first call instrumentation is in place;
- auto Iv = AlreadyExecuted.insert(FAddr);
- if (It == GlobalSpecMap.end() || Iv.second == false)
+ if (It == GlobalSpecMap.end())
return;
- else
- CandidateSet = It->getSecond();
+ CandidateSet = It->getSecond();
}
- // Try to distinguish pre-compiled symbols!
+ SymbolDependenceMap SpeculativeLookUpImpls;
+
for (auto &Callee : CandidateSet) {
auto ImplSymbol = AliaseeImplTable.getImplFor(Callee);
+ // try to distinguish already compiled & library symbols
if (!ImplSymbol.hasValue())
continue;
const auto &ImplSymbolName = ImplSymbol.getPointer()->first;
- auto *ImplJD = ImplSymbol.getPointer()->second;
- ES.lookup(JITDylibSearchList({{ImplJD, true}}),
- SymbolNameSet({ImplSymbolName}), SymbolState::Ready,
+ JITDylib *ImplJD = ImplSymbol.getPointer()->second;
+ auto &SymbolsInJD = SpeculativeLookUpImpls[ImplJD];
+ SymbolsInJD.insert(ImplSymbolName);
+ }
+
+ DEBUG_WITH_TYPE("orc", for (auto &I
+ : SpeculativeLookUpImpls) {
+ llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib ";
+ for (auto &N : I.second)
+ llvm::dbgs() << "\n Likely Symbol : " << N;
+ });
+
+ // for a given symbol, there may be no symbol qualified for speculatively
+ // compile try to fix this before jumping to this code if possible.
+ for (auto &LookupPair : SpeculativeLookUpImpls)
+ ES.lookup(JITDylibSearchList({{LookupPair.first, true}}),
+ LookupPair.second, SymbolState::Ready,
[this](Expected<SymbolMap> Result) {
if (auto Err = Result.takeError())
ES.reportError(std::move(Err));
},
NoDependenciesToRegister);
- }
}
public:
Speculator(Speculator &&) = delete;
Speculator &operator=(const Speculator &) = delete;
Speculator &operator=(Speculator &&) = delete;
- ~Speculator() {}
+
+ /// Define symbols for this Speculator object (__orc_speculator) and the
+ /// speculation runtime entry point symbol (__orc_speculate_for) in the
+ /// given JITDylib.
+ Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle);
// Speculatively compile likely functions for the given Stub Address.
// destination of __orc_speculate_for jump
ExecutionSession &getES() { return ES; }
private:
+ static void speculateForEntryPoint(Speculator *Ptr, uint64_t StubId);
std::mutex ConcurrentAccess;
ImplSymbolMap &AliaseeImplTable;
ExecutionSession &ES;
- DenseSet<TargetFAddr> AlreadyExecuted;
StubAddrLikelies GlobalSpecMap;
};
-// replace DenseMap with Pair
+
class IRSpeculationLayer : public IRLayer {
public:
using IRlikiesStrRef = Optional<DenseMap<StringRef, DenseSet<StringRef>>>;
- using ResultEval =
- std::function<IRlikiesStrRef(Function &, FunctionAnalysisManager &)>;
+ using ResultEval = std::function<IRlikiesStrRef(Function &)>;
using TargetAndLikelies = DenseMap<SymbolStringPtr, SymbolNameSet>;
IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer,
Speculator &Spec, MangleAndInterner &Mangle,
ResultEval Interpreter)
: IRLayer(ES), NextLayer(BaseLayer), S(Spec), Mangle(Mangle),
- QueryAnalysis(Interpreter) {
- PB.registerFunctionAnalyses(FAM);
- }
+ QueryAnalysis(Interpreter) {}
void emit(MaterializationResponsibility R, ThreadSafeModule TSM);
IRCompileLayer &NextLayer;
Speculator &S;
MangleAndInterner &Mangle;
- PassBuilder PB;
- FunctionAnalysisManager FAM;
ResultEval QueryAnalysis;
};
-// Runtime Function Interface
-extern "C" {
-void __orc_speculate_for(Speculator *, uint64_t stub_id);
-}
-
} // namespace orc
} // namespace llvm
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/SpeculateAnalyses.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/CFG.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <algorithm>
namespace {
using namespace llvm;
-std::vector<const BasicBlock *> findBBwithCalls(const Function &F,
- bool IndirectCall = false) {
- std::vector<const BasicBlock *> BBs;
+SmallVector<const BasicBlock *, 8> findBBwithCalls(const Function &F,
+ bool IndirectCall = false) {
+ SmallVector<const BasicBlock *, 8> BBs;
auto findCallInst = [&IndirectCall](const Instruction &I) {
- if (auto Call = dyn_cast<CallBase>(&I)) {
- if (Call->isIndirectCall())
- return IndirectCall;
- else
- return true;
- } else
+ if (auto Call = dyn_cast<CallBase>(&I))
+ return Call->isIndirectCall() ? IndirectCall : true;
+ else
return false;
};
for (auto &BB : F)
// Implementations of Queries shouldn't need to lock the resources
// such as LLVMContext, each argument (function) has a non-shared LLVMContext
+// Plus, if Queries contain states necessary locking scheme should be provided.
namespace llvm {
namespace orc {
// Collect direct calls only
-void BlockFreqQuery::findCalles(const BasicBlock *BB,
+void SpeculateQuery::findCalles(const BasicBlock *BB,
DenseSet<StringRef> &CallesNames) {
assert(BB != nullptr && "Traversing Null BB to find calls?");
getCalledFunction(II);
}
-// blind calculation
+bool SpeculateQuery::isStraightLine(const Function &F) {
+ return llvm::all_of(F.getBasicBlockList(), [](const BasicBlock &BB) {
+ return BB.getSingleSuccessor() != nullptr;
+ });
+}
+
+// BlockFreqQuery Implementations
+
size_t BlockFreqQuery::numBBToGet(size_t numBB) {
// small CFG
if (numBB < 4)
return (numBB / 2) + (numBB / 4);
}
-BlockFreqQuery::ResultTy BlockFreqQuery::
-operator()(Function &F, FunctionAnalysisManager &FAM) {
+BlockFreqQuery::ResultTy BlockFreqQuery::operator()(Function &F) {
DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles;
DenseSet<StringRef> Calles;
SmallVector<std::pair<const BasicBlock *, uint64_t>, 8> BBFreqs;
+ PassBuilder PB;
+ FunctionAnalysisManager FAM;
+ PB.registerFunctionAnalyses(FAM);
+
auto IBBs = findBBwithCalls(F);
if (IBBs.empty())
return CallerAndCalles;
}
+
+// SequenceBBQuery Implementation
+std::size_t SequenceBBQuery::getHottestBlocks(std::size_t TotalBlocks) {
+ if (TotalBlocks == 1)
+ return TotalBlocks;
+ return TotalBlocks / 2;
+}
+
+// FIXME : find good implementation.
+SequenceBBQuery::BlockListTy
+SequenceBBQuery::rearrangeBB(const Function &F, const BlockListTy &BBList) {
+ BlockListTy RearrangedBBSet;
+
+ for (auto &Block : F.getBasicBlockList())
+ if (llvm::is_contained(BBList, &Block))
+ RearrangedBBSet.push_back(&Block);
+
+ assert(RearrangedBBSet.size() == BBList.size() &&
+ "BasicBlock missing while rearranging?");
+ return RearrangedBBSet;
+}
+
+void SequenceBBQuery::traverseToEntryBlock(const BasicBlock *AtBB,
+ const BlockListTy &CallerBlocks,
+ const BackEdgesInfoTy &BackEdgesInfo,
+ const BranchProbabilityInfo *BPI,
+ VisitedBlocksInfoTy &VisitedBlocks) {
+ auto Itr = VisitedBlocks.find(AtBB);
+ if (Itr != VisitedBlocks.end()) { // already visited.
+ if (!Itr->second.Upward)
+ return;
+ Itr->second.Upward = false;
+ } else {
+ // Create hint for newly discoverd blocks.
+ WalkDirection BlockHint;
+ BlockHint.Upward = false;
+ // FIXME: Expensive Check
+ if (llvm::is_contained(CallerBlocks, AtBB))
+ BlockHint.CallerBlock = true;
+ VisitedBlocks.insert(std::make_pair(AtBB, BlockHint));
+ }
+
+ const_pred_iterator PIt = pred_begin(AtBB), EIt = pred_end(AtBB);
+ // Move this check to top, when we have code setup to launch speculative
+ // compiles for function in entry BB, this triggers the speculative compiles
+ // before running the program.
+ if (PIt == EIt) // No Preds.
+ return;
+
+ DenseSet<const BasicBlock *> PredSkipNodes;
+
+ // Since we are checking for predecessor's backedges, this Block
+ // occurs in second position.
+ for (auto &I : BackEdgesInfo)
+ if (I.second == AtBB)
+ PredSkipNodes.insert(I.first);
+
+ // Skip predecessors which source of back-edges.
+ for (; PIt != EIt; ++PIt)
+ // checking EdgeHotness is cheaper
+ if (BPI->isEdgeHot(*PIt, AtBB) && !PredSkipNodes.count(*PIt))
+ traverseToEntryBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+}
+
+void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB,
+ const BlockListTy &CallerBlocks,
+ const BackEdgesInfoTy &BackEdgesInfo,
+ const BranchProbabilityInfo *BPI,
+ VisitedBlocksInfoTy &VisitedBlocks) {
+ auto Itr = VisitedBlocks.find(AtBB);
+ if (Itr != VisitedBlocks.end()) { // already visited.
+ if (!Itr->second.Downward)
+ return;
+ Itr->second.Downward = false;
+ } else {
+ // Create hint for newly discoverd blocks.
+ WalkDirection BlockHint;
+ BlockHint.Downward = false;
+ // FIXME: Expensive Check
+ if (llvm::is_contained(CallerBlocks, AtBB))
+ BlockHint.CallerBlock = true;
+ VisitedBlocks.insert(std::make_pair(AtBB, BlockHint));
+ }
+
+ succ_const_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB);
+ if (PIt == EIt) // No succs.
+ return;
+
+ // If there are hot edges, then compute SuccSkipNodes.
+ DenseSet<const BasicBlock *> SuccSkipNodes;
+
+ // Since we are checking for successor's backedges, this Block
+ // occurs in first position.
+ for (auto &I : BackEdgesInfo)
+ if (I.first == AtBB)
+ SuccSkipNodes.insert(I.second);
+
+ for (; PIt != EIt; ++PIt)
+ if (BPI->isEdgeHot(AtBB, *PIt) && !SuccSkipNodes.count(*PIt))
+ traverseToExitBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+}
+
+// Get Block frequencies for blocks and take most frquently executed block,
+// walk towards the entry block from those blocks and discover the basic blocks
+// with call.
+SequenceBBQuery::BlockListTy
+SequenceBBQuery::queryCFG(Function &F, const BlockListTy &CallerBlocks) {
+
+ BlockFreqInfoTy BBFreqs;
+ VisitedBlocksInfoTy VisitedBlocks;
+ BackEdgesInfoTy BackEdgesInfo;
+
+ PassBuilder PB;
+ FunctionAnalysisManager FAM;
+ PB.registerFunctionAnalyses(FAM);
+
+ auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
+
+ llvm::FindFunctionBackedges(F, BackEdgesInfo);
+
+ for (const auto I : CallerBlocks)
+ BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()});
+
+ llvm::sort(BBFreqs, [](decltype(BBFreqs)::const_reference Bbf,
+ decltype(BBFreqs)::const_reference Bbs) {
+ return Bbf.second > Bbs.second;
+ });
+
+ ArrayRef<std::pair<const BasicBlock *, uint64_t>> HotBlocksRef(BBFreqs);
+ HotBlocksRef =
+ HotBlocksRef.drop_back(BBFreqs.size() - getHottestBlocks(BBFreqs.size()));
+
+ BranchProbabilityInfo *BPI =
+ FAM.getCachedResult<BranchProbabilityAnalysis>(F);
+
+ // visit NHotBlocks,
+ // traverse upwards to entry
+ // traverse downwards to end.
+
+ for (auto I : HotBlocksRef) {
+ traverseToEntryBlock(I.first, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+ traverseToExitBlock(I.first, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+ }
+
+ BlockListTy MinCallerBlocks;
+ for (auto &I : VisitedBlocks)
+ if (I.second.CallerBlock)
+ MinCallerBlocks.push_back(std::move(I.first));
+
+ return rearrangeBB(F, MinCallerBlocks);
+}
+
+SpeculateQuery::ResultTy SequenceBBQuery::operator()(Function &F) {
+ // reduce the number of lists!
+ DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles;
+ DenseSet<StringRef> Calles;
+ BlockListTy SequencedBlocks;
+ BlockListTy CallerBlocks;
+
+ CallerBlocks = findBBwithCalls(F);
+ if (CallerBlocks.empty())
+ return None;
+
+ if (isStraightLine(F))
+ SequencedBlocks = rearrangeBB(F, CallerBlocks);
+ else
+ SequencedBlocks = queryCFG(F, CallerBlocks);
+
+ for (auto BB : SequencedBlocks)
+ findCalles(BB, Calles);
+
+ CallerAndCalles.insert({F.getName(), std::move(Calles)});
+ return CallerAndCalles;
+}
+
} // namespace orc
} // namespace llvm
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/Speculation.h"
-
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Verifier.h"
+#include "llvm/Support/Debug.h"
#include <vector>
}
}
+// Trigger Speculative Compiles.
+void Speculator::speculateForEntryPoint(Speculator *Ptr, uint64_t StubId) {
+ assert(Ptr && " Null Address Received in orc_speculate_for ");
+ Ptr->speculateFor(StubId);
+}
+
+Error Speculator::addSpeculationRuntime(JITDylib &JD,
+ MangleAndInterner &Mangle) {
+ JITEvaluatedSymbol ThisPtr(pointerToJITTargetAddress(this),
+ JITSymbolFlags::Exported);
+ JITEvaluatedSymbol SpeculateForEntryPtr(
+ pointerToJITTargetAddress(&speculateForEntryPoint),
+ JITSymbolFlags::Exported);
+ return JD.define(absoluteSymbols({
+ {Mangle("__orc_speculator"), ThisPtr}, // Data Symbol
+ {Mangle("__orc_speculate_for"), SpeculateForEntryPtr} // Callable Symbol
+ }));
+}
+
// If two modules, share the same LLVMContext, different threads must
-// not access those modules concurrently, doing so leave the
-// LLVMContext in in-consistent state.
-// But here since each TSM has a unique Context associated with it,
-// on locking is necessary!
+// not access them concurrently without locking the associated LLVMContext
+// this implementation follows this contract.
void IRSpeculationLayer::emit(MaterializationResponsibility R,
ThreadSafeModule TSM) {
assert(TSM.getContext().getContext() != nullptr &&
"Module with null LLVMContext?");
- // Instrumentation of runtime calls
- auto &InContext = *TSM.getContext().getContext();
- auto SpeculatorVTy = StructType::create(InContext, "Class.Speculator");
- auto RuntimeCallTy = FunctionType::get(
- Type::getVoidTy(InContext),
- {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(InContext)}, false);
- auto RuntimeCall =
- Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage,
- "__orc_speculate_for", TSM.getModuleUnlocked());
- auto SpeclAddr = new GlobalVariable(
- *TSM.getModuleUnlocked(), SpeculatorVTy, false,
- GlobalValue::LinkageTypes::ExternalLinkage, nullptr, "__orc_speculator");
-
- IRBuilder<> Mutator(InContext);
-
- // QueryAnalysis allowed to transform the IR source, one such example is
- // Simplify CFG helps the static branch prediction heuristics!
- for (auto &Fn : TSM.getModuleUnlocked()->getFunctionList()) {
- if (!Fn.isDeclaration()) {
- auto IRNames = QueryAnalysis(Fn, FAM);
- // Instrument and register if Query has result
- if (IRNames.hasValue()) {
- Mutator.SetInsertPoint(&(Fn.getEntryBlock().front()));
- auto ImplAddrToUint =
- Mutator.CreatePtrToInt(&Fn, Type::getInt64Ty(InContext));
- Mutator.CreateCall(RuntimeCallTy, RuntimeCall,
- {SpeclAddr, ImplAddrToUint});
- S.registerSymbols(internToJITSymbols(IRNames.getValue()),
- &R.getTargetJITDylib());
+ // Instrumentation of runtime calls, lock the Module
+ TSM.withModuleDo([this, &R](Module &M) {
+ auto &MContext = M.getContext();
+ auto SpeculatorVTy = StructType::create(MContext, "Class.Speculator");
+ auto RuntimeCallTy = FunctionType::get(
+ Type::getVoidTy(MContext),
+ {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(MContext)}, false);
+ auto RuntimeCall =
+ Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage,
+ "__orc_speculate_for", &M);
+ auto SpeclAddr = new GlobalVariable(
+ M, SpeculatorVTy, false, GlobalValue::LinkageTypes::ExternalLinkage,
+ nullptr, "__orc_speculator");
+
+ IRBuilder<> Mutator(MContext);
+
+ // QueryAnalysis allowed to transform the IR source, one such example is
+ // Simplify CFG helps the static branch prediction heuristics!
+ for (auto &Fn : M.getFunctionList()) {
+ if (!Fn.isDeclaration()) {
+
+ auto IRNames = QueryAnalysis(Fn);
+ // Instrument and register if Query has result
+ if (IRNames.hasValue()) {
+
+ // Emit globals for each function.
+ auto LoadValueTy = Type::getInt8Ty(MContext);
+ auto SpeculatorGuard = new GlobalVariable(
+ M, LoadValueTy, false, GlobalValue::LinkageTypes::InternalLinkage,
+ ConstantInt::get(LoadValueTy, 0),
+ "__orc_speculate.guard.for." + Fn.getName());
+ SpeculatorGuard->setAlignment(1);
+ SpeculatorGuard->setUnnamedAddr(GlobalValue::UnnamedAddr::Local);
+
+ BasicBlock &ProgramEntry = Fn.getEntryBlock();
+ // Create BasicBlocks before the program's entry basicblock
+ BasicBlock *SpeculateBlock = BasicBlock::Create(
+ MContext, "__orc_speculate.block", &Fn, &ProgramEntry);
+ BasicBlock *SpeculateDecisionBlock = BasicBlock::Create(
+ MContext, "__orc_speculate.decision.block", &Fn, SpeculateBlock);
+
+ assert(SpeculateDecisionBlock == &Fn.getEntryBlock() &&
+ "SpeculateDecisionBlock not updated?");
+ Mutator.SetInsertPoint(SpeculateDecisionBlock);
+
+ auto LoadGuard =
+ Mutator.CreateLoad(LoadValueTy, SpeculatorGuard, "guard.value");
+ // if just loaded value equal to 0,return true.
+ auto CanSpeculate =
+ Mutator.CreateICmpEQ(LoadGuard, ConstantInt::get(LoadValueTy, 0),
+ "compare.to.speculate");
+ Mutator.CreateCondBr(CanSpeculate, SpeculateBlock, &ProgramEntry);
+
+ Mutator.SetInsertPoint(SpeculateBlock);
+ auto ImplAddrToUint =
+ Mutator.CreatePtrToInt(&Fn, Type::getInt64Ty(MContext));
+ Mutator.CreateCall(RuntimeCallTy, RuntimeCall,
+ {SpeclAddr, ImplAddrToUint});
+ Mutator.CreateStore(ConstantInt::get(LoadValueTy, 1),
+ SpeculatorGuard);
+ Mutator.CreateBr(&ProgramEntry);
+
+ assert(Mutator.GetInsertBlock()->getParent() == &Fn &&
+ "IR builder association mismatch?");
+ S.registerSymbols(internToJITSymbols(IRNames.getValue()),
+ &R.getTargetJITDylib());
+ }
}
}
- }
- // No locking needed read only operation.
- assert(!(verifyModule(*TSM.getModuleUnlocked())) &&
+ });
+
+ assert(!TSM.withModuleDo([](const Module &M) { return verifyModule(M); }) &&
"Speculation Instrumentation breaks IR?");
NextLayer.emit(std::move(R), std::move(TSM));
}
-// Runtime Function Implementation
-extern "C" void __orc_speculate_for(Speculator *Ptr, uint64_t StubId) {
- assert(Ptr && " Null Address Received in orc_speculate_for ");
- Ptr->speculateFor(StubId);
-}
-
} // namespace orc
} // namespace llvm
projects / llvm / commitdiff