EmitBlock(LoopExit.getBlock());
// Tell the runtime we are done.
- if (!DynamicOrOrdered)
- RT.emitForStaticFinish(*this, S.getLocEnd());
-
+ auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
+ if (!DynamicOrOrdered)
+ CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd());
+ };
+ OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
}
void CodeGenFunction::EmitOMPForOuterLoop(
*this, OMPD_distribute_parallel_for,
[&S](CodeGenFunction &CGF, PrePostActionTy &) {
OMPLoopScope PreInitScope(CGF, S);
+ OMPCancelStackRAII CancelRegion(CGF, OMPD_distribute_parallel_for,
+ /*HasCancel=*/false);
CGF.EmitStmt(
cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
});
[](CodeGenFunction &) {});
EmitBlock(LoopExit.getBlock());
// Tell the runtime we are done.
- RT.emitForStaticFinish(*this, S.getLocStart());
+ auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+ CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd());
+ };
+ OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
} else {
const bool IsMonotonic =
Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
bool HasLastprivates = false;
auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
PrePostActionTy &) {
+ OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
};
{
CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
[](CodeGenFunction &) {});
// Tell the runtime we are done.
- CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart());
+ auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+ CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd());
+ };
+ CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
CGF.EmitOMPReductionClauseFinal(S);
// Emit post-update of the reduction variables if IsLastIter != 0.
emitPostUpdateForReductionClause(
HasCancel = OSD->hasCancel();
else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
HasCancel = OPSD->hasCancel();
+ OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
HasCancel);
// Emit barrier for lastprivates only if 'sections' directive has 'nowait'
// Emit directive as a combined directive that consists of two implicit
// directives: 'parallel' with 'for' directive.
auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
+ OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
CGF.EmitOMPWorksharingLoop(S);
};
emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen);
CodeGenFunction::JumpDest
CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
- if (Kind == OMPD_parallel || Kind == OMPD_task)
+ if (Kind == OMPD_parallel || Kind == OMPD_task ||
+ Kind == OMPD_target_parallel)
return ReturnBlock;
assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
- Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for);
- return BreakContinueStack.back().BreakBlock;
+ Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
+ Kind == OMPD_distribute_parallel_for ||
+ Kind == OMPD_target_parallel_for);
+ return OMPCancelStack.getExitBlock();
}
void CodeGenFunction::EmitOMPUseDevicePtrClause(
};
SmallVector<BreakContinue, 8> BreakContinueStack;
+ /// Handles cancellation exit points in OpenMP-related constructs.
+ class OpenMPCancelExitStack {
+ /// Tracks cancellation exit point and join point for cancel-related exit
+ /// and normal exit.
+ struct CancelExit {
+ CancelExit() = default;
+ CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
+ JumpDest ContBlock)
+ : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
+ OpenMPDirectiveKind Kind = OMPD_unknown;
+ /// true if the exit block has been emitted already by the special
+ /// emitExit() call, false if the default codegen is used.
+ bool HasBeenEmitted = false;
+ JumpDest ExitBlock;
+ JumpDest ContBlock;
+ };
+
+ SmallVector<CancelExit, 8> Stack;
+
+ public:
+ OpenMPCancelExitStack() : Stack(1) {}
+ ~OpenMPCancelExitStack() = default;
+ /// Fetches the exit block for the current OpenMP construct.
+ JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
+ /// Emits exit block with special codegen procedure specific for the related
+ /// OpenMP construct + emits code for normal construct cleanup.
+ void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
+ const llvm::function_ref<void(CodeGenFunction &)> &CodeGen) {
+ if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
+ assert(CGF.getOMPCancelDestination(Kind).isValid());
+ assert(CGF.HaveInsertPoint());
+ assert(!Stack.back().HasBeenEmitted);
+ auto IP = CGF.Builder.saveAndClearIP();
+ CGF.EmitBlock(Stack.back().ExitBlock.getBlock());
+ CodeGen(CGF);
+ CGF.EmitBranchThroughCleanup(Stack.back().ContBlock);
+ CGF.Builder.restoreIP(IP);
+ Stack.back().HasBeenEmitted = true;
+ }
+ CodeGen(CGF);
+ }
+ /// Enter the cancel supporting \a Kind construct.
+ /// \param Kind OpenMP directive that supports cancel constructs.
+ /// \param HasCancel true, if the construct has inner cancel directive,
+ /// false otherwise.
+ void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
+ Stack.push_back({Kind,
+ HasCancel ? CGF.getJumpDestInCurrentScope("cancel.exit")
+ : JumpDest(),
+ HasCancel ? CGF.getJumpDestInCurrentScope("cancel.cont")
+ : JumpDest()});
+ }
+ /// Emits default exit point for the cancel construct (if the special one
+ /// has not be used) + join point for cancel/normal exits.
+ void exit(CodeGenFunction &CGF) {
+ if (getExitBlock().isValid()) {
+ assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
+ bool HaveIP = CGF.HaveInsertPoint();
+ if (!Stack.back().HasBeenEmitted) {
+ if (HaveIP)
+ CGF.EmitBranchThroughCleanup(Stack.back().ContBlock);
+ CGF.EmitBlock(Stack.back().ExitBlock.getBlock());
+ CGF.EmitBranchThroughCleanup(Stack.back().ContBlock);
+ }
+ CGF.EmitBlock(Stack.back().ContBlock.getBlock());
+ if (!HaveIP) {
+ CGF.Builder.CreateUnreachable();
+ CGF.Builder.ClearInsertionPoint();
+ }
+ }
+ Stack.pop_back();
+ }
+ };
+ OpenMPCancelExitStack OMPCancelStack;
+
+ /// Controls insertion of cancellation exit blocks in worksharing constructs.
+ class OMPCancelStackRAII {
+ CodeGenFunction &CGF;
+
+ public:
+ OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
+ bool HasCancel)
+ : CGF(CGF) {
+ CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
+ }
+ ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
+ };
+
CodeGenPGO PGO;
/// Calculate branch weights appropriate for PGO data
projects / clang / commitdiff