EmitLambdaBlockInvokeBody();
else {
PGO.assignRegionCounters(blockDecl, fn);
- RegionCounter Cnt = getPGORegionCounter(blockDecl->getBody());
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(blockDecl->getBody());
EmitStmt(blockDecl->getBody());
}
if (IsTryBody)
EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
- RegionCounter Cnt = getPGORegionCounter(Body);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(Body);
RunCleanupsScope RunCleanups(*this);
case Dtor_Base:
assert(Body);
- RegionCounter Cnt = getPGORegionCounter(Body);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(Body);
// Enter the cleanup scopes for fields and non-virtual bases.
EnterDtorCleanups(Dtor, Dtor_Base);
CGM.getCXXABI().emitBeginCatch(*this, C);
// Emit the PGO counter increment.
- RegionCounter CatchCnt = getPGORegionCounter(C);
- CatchCnt.beginRegion(Builder);
+ incrementProfileCounter(C);
// Perform the body of the catch.
EmitStmt(C->getHandlerBlock());
Builder.CreateBr(ContBB);
}
- RegionCounter ContCnt = getPGORegionCounter(&S);
EmitBlock(ContBB);
- ContCnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
}
namespace {
}
OpaqueValueMapping binding(*this, expr);
- RegionCounter Cnt = getPGORegionCounter(expr);
const Expr *condExpr = expr->getCond();
bool CondExprBool;
if (!ContainsLabel(dead)) {
// If the true case is live, we need to track its region.
if (CondExprBool)
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(expr);
return EmitLValue(live);
}
}
llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
ConditionalEvaluation eval(*this);
- EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, Cnt.getCount());
+ EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, getProfileCount(expr));
// Any temporaries created here are conditional.
EmitBlock(lhsBlock);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(expr);
eval.begin(*this);
Optional<LValue> lhs =
EmitLValueOrThrowExpression(*this, expr->getTrueExpr());
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
CodeGenFunction::ConditionalEvaluation eval(CGF);
- CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
+ CGF.getProfileCount(E));
// Save whether the destination's lifetime is externally managed.
bool isExternallyDestructed = Dest.isExternallyDestructed();
eval.begin(CGF);
CGF.EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Visit(E->getTrueExpr());
eval.end(CGF);
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
+
CodeGenFunction::ConditionalEvaluation eval(CGF);
- CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
+ CGF.getProfileCount(E));
eval.begin(CGF);
CGF.EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
ComplexPairTy LHS = Visit(E->getTrueExpr());
LHSBlock = Builder.GetInsertBlock();
CGF.EmitBranch(ContBlock);
}
Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
-
// Perform vector logical and on comparisons with zero vectors.
if (E->getType()->isVectorType()) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *LHS = Visit(E->getLHS());
Value *RHS = Visit(E->getRHS());
bool LHSCondVal;
if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
if (LHSCondVal) { // If we have 1 && X, just emit X.
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
// ZExt result to int or bool.
CodeGenFunction::ConditionalEvaluation eval(CGF);
// Branch on the LHS first. If it is false, go to the failure (cont) block.
- CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock,
+ CGF.getProfileCount(E->getRHS()));
// Any edges into the ContBlock are now from an (indeterminate number of)
// edges from this first condition. All of these values will be false. Start
eval.begin(CGF);
CGF.EmitBlock(RHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
eval.end(CGF);
}
Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
-
// Perform vector logical or on comparisons with zero vectors.
if (E->getType()->isVectorType()) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *LHS = Visit(E->getLHS());
Value *RHS = Visit(E->getRHS());
bool LHSCondVal;
if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
if (!LHSCondVal) { // If we have 0 || X, just emit X.
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
// ZExt result to int or bool.
// Branch on the LHS first. If it is true, go to the success (cont) block.
CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock,
- Cnt.getParentCount() - Cnt.getCount());
+ CGF.getCurrentProfileCount() -
+ CGF.getProfileCount(E->getRHS()));
// Any edges into the ContBlock are now from an (indeterminate number of)
// edges from this first condition. All of these values will be true. Start
// Emit the RHS condition as a bool value.
CGF.EmitBlock(RHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
eval.end(CGF);
// Bind the common expression if necessary.
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
- RegionCounter Cnt = CGF.getPGORegionCounter(E);
Expr *condExpr = E->getCond();
Expr *lhsExpr = E->getTrueExpr();
// If the dead side doesn't have labels we need, just emit the Live part.
if (!CGF.ContainsLabel(dead)) {
if (CondExprBool)
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
Value *Result = Visit(live);
// If the live part is a throw expression, it acts like it has a void
// the select function.
if (CGF.getLangOpts().OpenCL
&& condExpr->getType()->isVectorType()) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
llvm::Value *LHS = Visit(lhsExpr);
// safe to evaluate the LHS and RHS unconditionally.
if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) &&
isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) {
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr);
llvm::Value *LHS = Visit(lhsExpr);
llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
CodeGenFunction::ConditionalEvaluation eval(CGF);
- CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, Cnt.getCount());
+ CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock,
+ CGF.getProfileCount(lhsExpr));
CGF.EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ CGF.incrementProfileCounter(E);
eval.begin(CGF);
Value *LHS = Visit(lhsExpr);
eval.end(CGF);
StartObjCMethod(OMD, OMD->getClassInterface());
PGO.assignRegionCounters(OMD, CurFn);
assert(isa<CompoundStmt>(OMD->getBody()));
- RegionCounter Cnt = getPGORegionCounter(OMD->getBody());
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(OMD->getBody());
EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody()));
FinishFunction(OMD->getBodyRBrace());
}
// If the limit pointer was zero to begin with, the collection is
// empty; skip all this. Set the branch weight assuming this has the same
// probability of exiting the loop as any other loop exit.
- uint64_t EntryCount = PGO.getCurrentRegionCount();
- RegionCounter Cnt = getPGORegionCounter(&S);
- Builder.CreateCondBr(Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"),
- EmptyBB, LoopInitBB,
- PGO.createBranchWeights(EntryCount, Cnt.getCount()));
+ uint64_t EntryCount = getCurrentProfileCount();
+ Builder.CreateCondBr(
+ Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"), EmptyBB,
+ LoopInitBB,
+ PGO.createBranchWeights(EntryCount, getProfileCount(S.getBody())));
// Otherwise, initialize the loop.
EmitBlock(LoopInitBB);
llvm::PHINode *count = Builder.CreatePHI(UnsignedLongLTy, 3, "forcoll.count");
count->addIncoming(initialBufferLimit, LoopInitBB);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
// Check whether the mutations value has changed from where it was
// at start. StateMutationsPtr should actually be invariant between
// Set the branch weights based on the simplifying assumption that this is
// like a while-loop, i.e., ignoring that the false branch fetches more
// elements and then returns to the loop.
- Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count),
- LoopBodyBB, FetchMoreBB,
- PGO.createBranchWeights(Cnt.getCount(), EntryCount));
+ Builder.CreateCondBr(
+ Builder.CreateICmpULT(indexPlusOne, count), LoopBodyBB, FetchMoreBB,
+ PGO.createBranchWeights(getProfileCount(S.getBody()), EntryCount));
index->addIncoming(indexPlusOne, AfterBody.getBlock());
count->addIncoming(count, AfterBody.getBlock());
ResolveBranchFixups(Dest.getBlock());
}
- RegionCounter Cnt = getPGORegionCounter(D->getStmt());
EmitBlock(Dest.getBlock());
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(D->getStmt());
}
/// Change the cleanup scope of the labels in this lexical scope to
// C99 6.8.4.1: The first substatement is executed if the expression compares
// unequal to 0. The condition must be a scalar type.
LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
- RegionCounter Cnt = getPGORegionCounter(&S);
if (S.getConditionVariable())
EmitAutoVarDecl(*S.getConditionVariable());
// This avoids emitting dead code and simplifies the CFG substantially.
if (!ContainsLabel(Skipped)) {
if (CondConstant)
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
if (Executed) {
RunCleanupsScope ExecutedScope(*this);
EmitStmt(Executed);
if (S.getElse())
ElseBlock = createBasicBlock("if.else");
- EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Cnt.getCount());
+ EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock,
+ getProfileCount(S.getThen()));
// Emit the 'then' code.
EmitBlock(ThenBlock);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
{
RunCleanupsScope ThenScope(*this);
EmitStmt(S.getThen());
void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
ArrayRef<const Attr *> WhileAttrs) {
- RegionCounter Cnt = getPGORegionCounter(&S);
-
// Emit the header for the loop, which will also become
// the continue target.
JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
if (ConditionScope.requiresCleanups())
ExitBlock = createBasicBlock("while.exit");
- llvm::BranchInst *CondBr =
- Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock,
- PGO.createLoopWeights(S.getCond(), Cnt));
+ llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ BoolCondVal, LoopBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), getProfileCount(S.getBody())));
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
{
RunCleanupsScope BodyScope(*this);
EmitBlock(LoopBody);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
EmitStmt(S.getBody());
}
JumpDest LoopExit = getJumpDestInCurrentScope("do.end");
JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");
- RegionCounter Cnt = getPGORegionCounter(&S);
+ uint64_t ParentCount = getCurrentProfileCount();
// Store the blocks to use for break and continue.
BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
LoopStack.push(LoopBody);
- EmitBlockWithFallThrough(LoopBody, Cnt);
+ EmitBlockWithFallThrough(LoopBody, &S);
{
RunCleanupsScope BodyScope(*this);
EmitStmt(S.getBody());
// As long as the condition is true, iterate the loop.
if (EmitBoolCondBranch) {
+ uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;
llvm::BranchInst *CondBr =
Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(),
- PGO.createLoopWeights(S.getCond(), Cnt));
+ PGO.createLoopWeights(S.getCond(), BackedgeCount));
// Attach metadata to loop body conditional branch.
EmitCondBrHints(LoopBody->getContext(), CondBr, DoAttrs);
if (S.getInit())
EmitStmt(S.getInit());
- RegionCounter Cnt = getPGORegionCounter(&S);
-
// Start the loop with a block that tests the condition.
// If there's an increment, the continue scope will be overwritten
// later.
// C99 6.8.5p2/p4: The first substatement is executed if the expression
// compares unequal to 0. The condition must be a scalar type.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- llvm::BranchInst *CondBr =
- Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock,
- PGO.createLoopWeights(S.getCond(), Cnt));
+ llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ BoolCondVal, ForBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), getProfileCount(S.getBody())));
// Attach metadata to loop body conditional branch.
EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs);
// Treat it as a non-zero constant. Don't even create a new block for the
// body, just fall into it.
}
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
{
// Create a separate cleanup scope for the body, in case it is not
EmitStmt(S.getRangeStmt());
EmitStmt(S.getBeginEndStmt());
- RegionCounter Cnt = getPGORegionCounter(&S);
-
// Start the loop with a block that tests the condition.
// If there's an increment, the continue scope will be overwritten
// later.
// to bool, is true.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
llvm::BranchInst *CondBr = Builder.CreateCondBr(
- BoolCondVal, ForBody, ExitBlock, PGO.createLoopWeights(S.getCond(), Cnt));
+ BoolCondVal, ForBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), getProfileCount(S.getBody())));
// Attach metadata to loop body conditional branch.
EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs);
}
EmitBlock(ForBody);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
// Create a block for the increment. In case of a 'continue', we jump there.
JumpDest Continue = getJumpDestInCurrentScope("for.inc");
llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
- RegionCounter CaseCnt = getPGORegionCounter(&S);
-
// Emit the code for this case. We do this first to make sure it is
// properly chained from our predecessor before generating the
// switch machinery to enter this block.
llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
- EmitBlockWithFallThrough(CaseDest, CaseCnt);
+ EmitBlockWithFallThrough(CaseDest, &S);
EmitStmt(S.getSubStmt());
// If range is empty, do nothing.
// FIXME: parameters such as this should not be hardcoded.
if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
// Range is small enough to add multiple switch instruction cases.
- uint64_t Total = CaseCnt.getCount();
+ uint64_t Total = getProfileCount(&S);
unsigned NCases = Range.getZExtValue() + 1;
// We only have one region counter for the entire set of cases here, so we
// need to divide the weights evenly between the generated cases, ensuring
llvm::MDNode *Weights = nullptr;
if (SwitchWeights) {
- uint64_t ThisCount = CaseCnt.getCount();
+ uint64_t ThisCount = getProfileCount(&S);
uint64_t DefaultCount = (*SwitchWeights)[0];
Weights = PGO.createBranchWeights(ThisCount, DefaultCount);
return;
}
- RegionCounter CaseCnt = getPGORegionCounter(&S);
llvm::ConstantInt *CaseVal =
Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
// Only do this optimization if there are no cleanups that need emitting.
if (isObviouslyBranchWithoutCleanups(Block)) {
if (SwitchWeights)
- SwitchWeights->push_back(CaseCnt.getCount());
+ SwitchWeights->push_back(getProfileCount(&S));
SwitchInsn->addCase(CaseVal, Block.getBlock());
// If there was a fallthrough into this case, make sure to redirect it to
}
llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
- EmitBlockWithFallThrough(CaseDest, CaseCnt);
+ EmitBlockWithFallThrough(CaseDest, &S);
if (SwitchWeights)
- SwitchWeights->push_back(CaseCnt.getCount());
+ SwitchWeights->push_back(getProfileCount(&S));
SwitchInsn->addCase(CaseVal, CaseDest);
// Recursively emitting the statement is acceptable, but is not wonderful for
llvm::ConstantInt *CaseVal =
Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
- CaseCnt = getPGORegionCounter(NextCase);
if (SwitchWeights)
- SwitchWeights->push_back(CaseCnt.getCount());
+ SwitchWeights->push_back(getProfileCount(NextCase));
if (CGM.getCodeGenOpts().ProfileInstrGenerate) {
CaseDest = createBasicBlock("sw.bb");
- EmitBlockWithFallThrough(CaseDest, CaseCnt);
+ EmitBlockWithFallThrough(CaseDest, &S);
}
SwitchInsn->addCase(CaseVal, CaseDest);
assert(DefaultBlock->empty() &&
"EmitDefaultStmt: Default block already defined?");
- RegionCounter Cnt = getPGORegionCounter(&S);
- EmitBlockWithFallThrough(DefaultBlock, Cnt);
+ EmitBlockWithFallThrough(DefaultBlock, &S);
EmitStmt(S.getSubStmt());
}
const SwitchCase *Case = nullptr;
if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
getContext(), Case)) {
- if (Case) {
- RegionCounter CaseCnt = getPGORegionCounter(Case);
- CaseCnt.beginRegion(Builder);
- }
+ if (Case)
+ incrementProfileCounter(Case);
RunCleanupsScope ExecutedScope(*this);
// Emit the condition variable if needed inside the entire cleanup scope
// specified series of statements and we're good.
for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
EmitStmt(CaseStmts[i]);
- RegionCounter ExitCnt = getPGORegionCounter(&S);
- ExitCnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
// Now we want to restore the saved switch instance so that nested
// switches continue to function properly
Case;
Case = Case->getNextSwitchCase()) {
if (isa<DefaultStmt>(Case))
- DefaultCount = getPGORegionCounter(Case).getCount();
+ DefaultCount = getProfileCount(Case);
NumCases += 1;
}
SwitchWeights = new SmallVector<uint64_t, 16>();
// Emit continuation.
EmitBlock(SwitchExit.getBlock(), true);
- RegionCounter ExitCnt = getPGORegionCounter(&S);
- ExitCnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
if (SwitchWeights) {
assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
- auto Cnt = getPGORegionCounter(&S);
// Start the loop with a block that tests the condition.
auto CondBlock = createBasicBlock("omp.inner.for.cond");
auto LoopBody = createBasicBlock("omp.inner.for.body");
// Emit condition.
- EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, Cnt.getCount());
+ EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
EmitBranchThroughCleanup(LoopExit);
}
EmitBlock(LoopBody);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
// Create a block for the increment.
auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
if (!CondConstant)
return;
} else {
- RegionCounter Cnt = CGF.getPGORegionCounter(&S);
auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
ContBlock = CGF.createBasicBlock("simd.if.end");
- emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount());
+ emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
+ CGF.getProfileCount(&S));
CGF.EmitBlock(ThenBlock);
- Cnt.beginRegion(CGF.Builder);
+ CGF.incrementProfileCounter(&S);
}
// Walk clauses and process safelen/lastprivate.
bool SeparateIter = false;
if (!CondConstant)
return false;
} else {
- RegionCounter Cnt = getPGORegionCounter(&S);
auto *ThenBlock = createBasicBlock("omp.precond.then");
ContBlock = createBasicBlock("omp.precond.end");
emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
- Cnt.getCount());
+ getProfileCount(&S));
EmitBlock(ThenBlock);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(&S);
}
// Emit 'then' code.
{
void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) {
llvm_unreachable("CodeGen for 'omp teams' is not supported yet.");
}
-
void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args,
const Stmt *Body) {
- RegionCounter Cnt = getPGORegionCounter(Body);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(Body);
if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
EmitCompoundStmtWithoutScope(*S);
else
/// emit a branch around the instrumentation code. When not instrumenting,
/// this just calls EmitBlock().
void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
- RegionCounter &Cnt) {
+ const Stmt *S) {
llvm::BasicBlock *SkipCountBB = nullptr;
if (HaveInsertPoint() && CGM.getCodeGenOpts().ProfileInstrGenerate) {
// When instrumenting for profiling, the fallthrough to certain
EmitBranch(SkipCountBB);
}
EmitBlock(BB);
- Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true);
+ uint64_t CurrentCount = getCurrentProfileCount();
+ incrementProfileCounter(S);
+ setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
if (SkipCountBB)
EmitBlock(SkipCountBB);
}
ResTy = CGM.getContext().VoidPtrTy;
CGM.getCXXABI().buildThisParam(*this, Args);
}
-
+
Args.append(FD->param_begin(), FD->param_end());
if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
// Handle X && Y in a condition.
if (CondBOp->getOpcode() == BO_LAnd) {
- RegionCounter Cnt = getPGORegionCounter(CondBOp);
-
// If we have "1 && X", simplify the code. "0 && X" would have constant
// folded if the case was simple enough.
bool ConstantBool = false;
if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
ConstantBool) {
// br(1 && X) -> br(X).
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(CondBOp);
return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
TrueCount);
}
llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
// The counter tells us how often we evaluate RHS, and all of TrueCount
// can be propagated to that branch.
- uint64_t RHSCount = Cnt.getCount();
+ uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
ConditionalEvaluation eval(*this);
{
EmitBlock(LHSTrue);
}
+ incrementProfileCounter(CondBOp);
+ setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
+
// Any temporaries created here are conditional.
- Cnt.beginRegion(Builder);
eval.begin(*this);
EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount);
eval.end(*this);
}
if (CondBOp->getOpcode() == BO_LOr) {
- RegionCounter Cnt = getPGORegionCounter(CondBOp);
-
// If we have "0 || X", simplify the code. "1 || X" would have constant
// folded if the case was simple enough.
bool ConstantBool = false;
if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
!ConstantBool) {
// br(0 || X) -> br(X).
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(CondBOp);
return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
TrueCount);
}
// We have the count for entry to the RHS and for the whole expression
// being true, so we can divy up True count between the short circuit and
// the RHS.
- uint64_t LHSCount = Cnt.getParentCount() - Cnt.getCount();
+ uint64_t LHSCount =
+ getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
uint64_t RHSCount = TrueCount - LHSCount;
ConditionalEvaluation eval(*this);
EmitBlock(LHSFalse);
}
+ incrementProfileCounter(CondBOp);
+ setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
+
// Any temporaries created here are conditional.
- Cnt.beginRegion(Builder);
eval.begin(*this);
EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount);
// br(!x, t, f) -> br(x, f, t)
if (CondUOp->getOpcode() == UO_LNot) {
// Negate the count.
- uint64_t FalseCount = PGO.getCurrentRegionCount() - TrueCount;
+ uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
// Negate the condition and swap the destination blocks.
return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
FalseCount);
llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
- RegionCounter Cnt = getPGORegionCounter(CondOp);
ConditionalEvaluation cond(*this);
- EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
+ EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
+ getProfileCount(CondOp));
// When computing PGO branch weights, we only know the overall count for
// the true block. This code is essentially doing tail duplication of the
// the conditional operator.
uint64_t LHSScaledTrueCount = 0;
if (TrueCount) {
- double LHSRatio = Cnt.getCount() / (double) Cnt.getParentCount();
+ double LHSRatio =
+ getProfileCount(CondOp) / (double)getCurrentProfileCount();
LHSScaledTrueCount = TrueCount * LHSRatio;
}
cond.begin(*this);
EmitBlock(LHSBlock);
- Cnt.beginRegion(Builder);
+ incrementProfileCounter(CondOp);
{
ApplyDebugLocation DL(*this, Cond);
EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
// Create branch weights based on the number of times we get here and the
// number of times the condition should be true.
- uint64_t CurrentCount = std::max(PGO.getCurrentRegionCount(), TrueCount);
+ uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
llvm::MDNode *Weights = PGO.createBranchWeights(TrueCount,
CurrentCount - TrueCount);
/// \brief Emit the captured statement body.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
- RegionCounter Cnt = CGF.getPGORegionCounter(S);
- Cnt.beginRegion(CGF.Builder);
+ CGF.incrementProfileCounter(S);
CGF.EmitStmt(S);
}
CodeGenPGO PGO;
public:
- /// Get a counter for instrumentation of the region associated with the given
- /// statement.
- RegionCounter getPGORegionCounter(const Stmt *S) {
- return RegionCounter(PGO, S);
+ /// Increment the profiler's counter for the given statement.
+ void incrementProfileCounter(const Stmt *S) {
+ if (CGM.getCodeGenOpts().ProfileInstrGenerate)
+ PGO.emitCounterIncrement(Builder, S);
+ PGO.setCurrentStmt(S);
}
+
+ /// Get the profiler's count for the given statement.
+ uint64_t getProfileCount(const Stmt *S) {
+ Optional<uint64_t> Count = PGO.getStmtCount(S);
+ if (!Count.hasValue())
+ return 0;
+ return *Count;
+ }
+
+ /// Set the profiler's current count.
+ void setCurrentProfileCount(uint64_t Count) {
+ PGO.setCurrentRegionCount(Count);
+ }
+
+ /// Get the profiler's current count. This is generally the count for the most
+ /// recently incremented counter.
+ uint64_t getCurrentProfileCount() {
+ return PGO.getCurrentRegionCount();
+ }
+
private:
/// SwitchInsn - This is nearest current switch instruction. It is null if
void EmitDestructorBody(FunctionArgList &Args);
void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
void EmitFunctionBody(FunctionArgList &Args, const Stmt *Body);
- void EmitBlockWithFallThrough(llvm::BasicBlock *BB, RegionCounter &Cnt);
+ void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
CallArgList &CallArgs);
Fn->addFnAttr(llvm::Attribute::Cold);
}
-void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter) {
+void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) {
if (!CGM.getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap)
return;
if (!Builder.GetInsertPoint())
return;
+
+ unsigned Counter = (*RegionCounterMap)[S];
auto *I8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
Builder.CreateCall4(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment),
llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
}
llvm::MDNode *CodeGenPGO::createLoopWeights(const Stmt *Cond,
- RegionCounter &Cnt) {
+ uint64_t LoopCount) {
if (!haveRegionCounts())
return nullptr;
- uint64_t LoopCount = Cnt.getCount();
Optional<uint64_t> CondCount = getStmtCount(Cond);
assert(CondCount.hasValue() && "missing expected loop condition count");
if (*CondCount == 0)
namespace clang {
namespace CodeGen {
-class RegionCounter;
-/// Per-function PGO state. This class should generally not be used directly,
-/// but instead through the CodeGenFunction and RegionCounter types.
+/// Per-function PGO state.
class CodeGenPGO {
private:
CodeGenModule &CGM;
/// Calculate branch weights appropriate for PGO data
llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount);
llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights);
- llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt);
+ llvm::MDNode *createLoopWeights(const Stmt *Cond, uint64_t LoopCount);
/// Check if we need to emit coverage mapping for a given declaration
void checkGlobalDecl(GlobalDecl GD);
void emitCounterVariables();
void emitCounterRegionMapping(const Decl *D);
- /// Emit code to increment the counter at the given index
- void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter);
+public:
+ void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S);
- /// Return the region counter for the given statement. This should only be
- /// called on statements that have a dedicated counter.
- unsigned getRegionCounter(const Stmt *S) {
+ /// Return the region count for the counter at the given index.
+ uint64_t getRegionCount(const Stmt *S) {
if (!RegionCounterMap)
return 0;
- return (*RegionCounterMap)[S];
- }
-
- /// Return the region count for the counter at the given index.
- uint64_t getRegionCount(unsigned Counter) {
if (!haveRegionCounts())
return 0;
- return RegionCounts[Counter];
+ return RegionCounts[(*RegionCounterMap)[S]];
}
-
- friend class RegionCounter;
};
/// A counter for a particular region. This is the primary interface through
/// which clients manage PGO counters and their values.
class RegionCounter {
CodeGenPGO *PGO;
- unsigned Counter;
uint64_t Count;
uint64_t ParentCount;
uint64_t RegionCount;
int64_t Adjust;
- RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex)
- : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)),
- ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
-
public:
RegionCounter(CodeGenPGO &PGO, const Stmt *S)
- : PGO(&PGO), Counter(PGO.getRegionCounter(S)),
- Count(PGO.getRegionCount(Counter)),
+ : PGO(&PGO), Count(PGO.getRegionCount(S)),
ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
/// Get the value of the counter. In most cases this is the number of times
/// or the normal exits of a loop.
uint64_t getParentCount() const { return ParentCount; }
- /// Activate the counter by emitting an increment and starting to track
- /// adjustments. If AddIncomingFallThrough is true, the current region count
- /// will be added to the counter for the purposes of tracking the region.
- void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) {
- beginRegion(AddIncomingFallThrough);
- PGO->emitCounterIncrement(Builder, Counter);
- }
void beginRegion(bool AddIncomingFallThrough=false) {
RegionCount = Count;
if (AddIncomingFallThrough)
projects / clang / commitdiff