The ``#pragma clang loop`` directive is used to specify hints for optimizing the
subsequent for, while, do-while, or c++11 range-based for loop. The directive
-provides options for vectorization, interleaving, and unrolling. Loop hints can
-be specified before any loop and will be ignored if the optimization is not safe
-to apply.
+provides options for vectorization, interleaving, unrolling and
+distribution. Loop hints can be specified before any loop and will be ignored if
+the optimization is not safe to apply.
Vectorization and Interleaving
------------------------------
Unrolling of a loop can be prevented by specifying ``unroll(disable)``.
+Loop Distribution
+-----------------
+
+Loop Distribution allows splitting a loop into multiple loops. This is
+beneficial for example when the entire loop cannot be vectorized but some of the
+resulting loops can.
+
+If ``distribute(enable))'' is specified and the loop has memory dependencies
+that inhibit vectorization, the compiler will attempt to isolate the offending
+operations into a new loop. This optimization is not enabled by default, only
+loops marked with the pragma are considered.
+
+.. code-block:: c++
+
+ #pragma clang loop distribute(enable)
+ for (i = 0; i < N; ++i) {
+ S1: A[i + 1] = A[i] + B[i];
+ S2: C[i] = D[i] * E[i];
+ }
+
+This loop will be split into two loops between statements S1 and S2. The
+second loop containing S2 will be vectorized.
+
+Loop Distribution is currently not enabled by default in the optimizer because
+it can hurt performance in some cases. For example, instruction-level
+parallelism could be reduced by sequentializing the execution of the
+statements S1 and S2 above.
+
+If Loop Distribution is turned on globally with
+``-mllvm -enable-loop-distribution``, specifying ``distribute(disable)`` can
+be used the disable it on a per-loop basis.
+
Additional Information
----------------------
/// interleave_count: interleaves 'Value' loop interations.
/// unroll: fully unroll loop if State == Enable.
/// unroll_count: unrolls loop 'Value' times.
+ /// distribute: attempt to distribute loop if State == Enable
/// #pragma unroll <argument> directive
/// <no arg>: fully unrolls loop.
/// State of the loop optimization specified by the spelling.
let Args = [EnumArgument<"Option", "OptionType",
["vectorize", "vectorize_width", "interleave", "interleave_count",
- "unroll", "unroll_count"],
+ "unroll", "unroll_count", "distribute"],
["Vectorize", "VectorizeWidth", "Interleave", "InterleaveCount",
- "Unroll", "UnrollCount"]>,
+ "Unroll", "UnrollCount", "Distribute"]>,
EnumArgument<"State", "LoopHintState",
["enable", "disable", "numeric", "assume_safety", "full"],
["Enable", "Disable", "Numeric", "AssumeSafety", "Full"]>,
case InterleaveCount: return "interleave_count";
case Unroll: return "unroll";
case UnrollCount: return "unroll_count";
+ case Distribute: return "distribute";
}
llvm_unreachable("Unhandled LoopHint option.");
}
// Pragma loop support.
def err_pragma_loop_missing_argument : Error<
"missing argument; expected %select{an integer value|"
- "'enable', %select{'assume_safety'|'full'}1 or 'disable'}0">;
+ "'enable'%select{|, 'full'}1%select{|, 'assume_safety'}2 or 'disable'}0">;
def err_pragma_loop_invalid_option : Error<
"%select{invalid|missing}0 option%select{ %1|}0; expected vectorize, "
- "vectorize_width, interleave, interleave_count, unroll, or unroll_count">;
+ "vectorize_width, interleave, interleave_count, unroll, unroll_count, or distribute">;
def err_pragma_invalid_keyword : Error<
- "invalid argument; expected 'enable', %select{'assume_safety'|'full'}0 or 'disable'">;
+ "invalid argument; expected 'enable'%select{|, 'full'}0%select{|, 'assume_safety'}1 or 'disable'">;
// Pragma unroll support.
def warn_pragma_unroll_cuda_value_in_parens : Warning<
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
+ Attrs.DistributeEnable == LoopAttributes::Unspecified &&
!Location)
return nullptr;
Args.push_back(MDNode::get(Ctx, Vals));
}
+ if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
+ LoopAttributes::Enable)))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
// Set the first operand to itself.
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
LoopAttributes::LoopAttributes(bool IsParallel)
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
- InterleaveCount(0), UnrollCount(0) {}
+ InterleaveCount(0), UnrollCount(0),
+ DistributeEnable(LoopAttributes::Unspecified) {}
void LoopAttributes::clear() {
IsParallel = false;
case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Disable);
break;
+ case LoopHintAttr::Distribute:
+ setDistributeState(false);
+ break;
case LoopHintAttr::UnrollCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
case LoopHintAttr::Unroll:
setUnrollState(LoopAttributes::Enable);
break;
+ case LoopHintAttr::Distribute:
+ setDistributeState(true);
+ break;
case LoopHintAttr::UnrollCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
case LoopHintAttr::UnrollCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
+ case LoopHintAttr::Distribute:
llvm_unreachable("Options cannot be used to assume mem safety.");
break;
}
case LoopHintAttr::UnrollCount:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::InterleaveCount:
+ case LoopHintAttr::Distribute:
llvm_unreachable("Options cannot be used with 'full' hint.");
break;
}
case LoopHintAttr::Unroll:
case LoopHintAttr::Vectorize:
case LoopHintAttr::Interleave:
+ case LoopHintAttr::Distribute:
llvm_unreachable("Options cannot be assigned a value.");
break;
}
/// \brief llvm.unroll.
unsigned UnrollCount;
+
+ /// \brief Value for llvm.loop.distribute.enable metadata.
+ LVEnableState DistributeEnable;
};
/// \brief Information used when generating a structured loop.
Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
}
+ /// \brief Set the next pushed loop as a distribution candidate.
+ void setDistributeState(bool Enable = true) {
+ StagedAttrs.DistributeEnable =
+ Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
+ }
+
/// \brief Set the next pushed loop unroll state.
void setUnrollState(const LoopAttributes::LVEnableState &State) {
StagedAttrs.UnrollEnable = State;
// If no option is specified the argument is assumed to be a constant expr.
bool OptionUnroll = false;
+ bool OptionDistribute = false;
bool StateOption = false;
if (OptionInfo) { // Pragma Unroll does not specify an option.
OptionUnroll = OptionInfo->isStr("unroll");
+ OptionDistribute = OptionInfo->isStr("distribute");
StateOption = llvm::StringSwitch<bool>(OptionInfo->getName())
.Case("vectorize", true)
.Case("interleave", true)
- .Default(false) || OptionUnroll;
+ .Default(false) ||
+ OptionUnroll || OptionDistribute;
}
+ bool AssumeSafetyArg = !OptionUnroll && !OptionDistribute;
// Verify loop hint has an argument.
if (Toks[0].is(tok::eof)) {
ConsumeToken(); // The annotation token.
Diag(Toks[0].getLocation(), diag::err_pragma_loop_missing_argument)
- << /*StateArgument=*/StateOption << /*FullKeyword=*/OptionUnroll;
+ << /*StateArgument=*/StateOption << /*FullKeyword=*/OptionUnroll
+ << /*AssumeSafetyKeyword=*/AssumeSafetyArg;
return false;
}
llvm::StringSwitch<bool>(StateInfo->getName())
.Cases("enable", "disable", true)
.Case("full", OptionUnroll)
- .Case("assume_safety", !OptionUnroll)
+ .Case("assume_safety", AssumeSafetyArg)
.Default(false);
if (!Valid) {
Diag(Toks[0].getLocation(), diag::err_pragma_invalid_keyword)
- << /*FullKeyword=*/OptionUnroll;
+ << /*FullKeyword=*/OptionUnroll
+ << /*AssumeSafetyKeyword=*/AssumeSafetyArg;
return false;
}
if (Toks.size() > 2)
.Case("vectorize", true)
.Case("interleave", true)
.Case("unroll", true)
+ .Case("distribute", true)
.Case("vectorize_width", true)
.Case("interleave_count", true)
.Case("unroll_count", true)
.Case("interleave_count", LoopHintAttr::InterleaveCount)
.Case("unroll", LoopHintAttr::Unroll)
.Case("unroll_count", LoopHintAttr::UnrollCount)
+ .Case("distribute", LoopHintAttr::Distribute)
.Default(LoopHintAttr::Vectorize);
if (Option == LoopHintAttr::VectorizeWidth ||
Option == LoopHintAttr::InterleaveCount ||
State = LoopHintAttr::Numeric;
} else if (Option == LoopHintAttr::Vectorize ||
Option == LoopHintAttr::Interleave ||
- Option == LoopHintAttr::Unroll) {
+ Option == LoopHintAttr::Unroll ||
+ Option == LoopHintAttr::Distribute) {
assert(StateLoc && StateLoc->Ident && "Loop hint must have an argument");
if (StateLoc->Ident->isStr("disable"))
State = LoopHintAttr::Disable;
static void
CheckForIncompatibleAttributes(Sema &S,
const SmallVectorImpl<const Attr *> &Attrs) {
- // There are 3 categories of loop hints attributes: vectorize, interleave,
- // and unroll. Each comes in two variants: a state form and a numeric form.
- // The state form selectively defaults/enables/disables the transformation
- // for the loop (for unroll, default indicates full unrolling rather than
- // enabling the transformation). The numeric form form provides an integer
- // hint (for example, unroll count) to the transformer. The following array
- // accumulates the hints encountered while iterating through the attributes
- // to check for compatibility.
+ // There are 4 categories of loop hints attributes: vectorize, interleave,
+ // unroll and distribute. Except for distribute they come in two variants: a
+ // state form and a numeric form. The state form selectively
+ // defaults/enables/disables the transformation for the loop (for unroll,
+ // default indicates full unrolling rather than enabling the transformation).
+ // The numeric form form provides an integer hint (for example, unroll count)
+ // to the transformer. The following array accumulates the hints encountered
+ // while iterating through the attributes to check for compatibility.
struct {
const LoopHintAttr *StateAttr;
const LoopHintAttr *NumericAttr;
- } HintAttrs[] = {{nullptr, nullptr}, {nullptr, nullptr}, {nullptr, nullptr}};
+ } HintAttrs[] = {{nullptr, nullptr},
+ {nullptr, nullptr},
+ {nullptr, nullptr},
+ {nullptr, nullptr}};
for (const auto *I : Attrs) {
const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(I);
continue;
LoopHintAttr::OptionType Option = LH->getOption();
- enum { Vectorize, Interleave, Unroll } Category;
+ enum { Vectorize, Interleave, Unroll, Distribute } Category;
switch (Option) {
case LoopHintAttr::Vectorize:
case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::UnrollCount:
Category = Unroll;
break;
+ case LoopHintAttr::Distribute:
+ // Perform the check for duplicated 'distribute' hints.
+ Category = Distribute;
+ break;
};
auto &CategoryState = HintAttrs[Category];
const LoopHintAttr *PrevAttr;
if (Option == LoopHintAttr::Vectorize ||
- Option == LoopHintAttr::Interleave || Option == LoopHintAttr::Unroll) {
+ Option == LoopHintAttr::Interleave || Option == LoopHintAttr::Unroll ||
+ Option == LoopHintAttr::Distribute) {
// Enable|Disable|AssumeSafety hint. For example, vectorize(enable).
PrevAttr = CategoryState.StateAttr;
CategoryState.StateAttr = LH;
#pragma clang loop interleave_count(4)
#pragma clang loop vectorize_width(4)
#pragma clang loop unroll(full)
+#pragma clang loop distribute(enable)
while (i < Length) {
// CHECK: br label {{.*}}, !llvm.loop ![[LOOP_1:.*]]
List[i] = i * 2;
void do_test(int *List, int Length) {
int i = 0;
-#pragma clang loop vectorize_width(8) interleave_count(4) unroll(disable)
+#pragma clang loop vectorize_width(8) interleave_count(4) unroll(disable) distribute(disable)
do {
// CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_2:.*]]
List[i] = i * 2;
// Verify disable pragma clang loop directive generates correct metadata
void disable_test(int *List, int Length) {
-#pragma clang loop vectorize(disable) unroll(disable)
+#pragma clang loop vectorize(disable) unroll(disable) distribute(disable)
for (int i = 0; i < Length; i++) {
// CHECK: br label {{.*}}, !llvm.loop ![[LOOP_5:.*]]
List[i] = i * 2;
for_template_constant_expression_test<double, 2, 4, 8>(List, Length);
}
-// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[WIDTH_4:.*]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[UNROLL_FULL:.*]]}
+// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[WIDTH_4:.*]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[UNROLL_FULL:.*]], ![[DISTRIBUTE_ENABLE:.*]]}
// CHECK: ![[WIDTH_4]] = !{!"llvm.loop.vectorize.width", i32 4}
// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
// CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
-// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_DISABLE:.*]]}
+// CHECK: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
+// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
+// CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false}
// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]], ![[INTENABLE_1:.*]]}
// CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8}
// CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]]}
// CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2}
// CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2}
-// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[WIDTH_1:.*]], ![[UNROLL_DISABLE:.*]]}
+// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[WIDTH_1:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[WIDTH_5:.*]]}
// CHECK: #pragma clang loop interleave(disable)
// CHECK-NEXT: #pragma clang loop vectorize(enable)
+// CHECK-NEXT: #pragma clang loop distribute(disable)
+#pragma clang loop distribute(disable)
#pragma clang loop vectorize(enable)
#pragma clang loop interleave(disable)
// CHECK-NEXT: while (i - 1 < Length)
// CHECK: #pragma clang loop interleave(enable)
// CHECK-NEXT: #pragma clang loop vectorize(disable)
+// CHECK-NEXT: #pragma clang loop distribute(enable)
+#pragma clang loop distribute(enable)
#pragma clang loop vectorize(disable)
#pragma clang loop interleave(enable)
// CHECK-NEXT: while (i - 2 < Length)
// CHECK: #pragma clang loop unroll_count(16)
// CHECK: #pragma clang loop interleave_count(8)
// CHECK: #pragma clang loop vectorize_width(4)
+// CHECK: #pragma clang loop distribute(enable)
// CHECK: #pragma clang loop unroll(disable)
// CHECK: #pragma clang loop interleave(disable)
// CHECK: #pragma clang loop vectorize(enable)
+// CHECK: #pragma clang loop distribute(disable)
// CHECK: #pragma clang loop unroll(full)
// CHECK: #pragma clang loop interleave(enable)
// CHECK: #pragma clang loop vectorize(disable)
#pragma clang loop vectorize(enable)
#pragma clang loop interleave(disable)
#pragma clang loop unroll(disable)
+#pragma clang loop distribute(enable)
while (i - 1 < Length) {
List[i] = i;
i++;
#pragma clang loop vectorize(disable)
#pragma clang loop interleave(enable)
#pragma clang loop unroll(full)
+#pragma clang loop distribute(disable)
while (i - 3 < Length) {
List[i] = i;
i++;
/* expected-error {{expected ')'}} */ #pragma clang loop interleave(assume_safety
/* expected-error {{invalid argument; expected 'enable', 'full' or 'disable'}} */ #pragma clang loop unroll(assume_safety)
+/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */ #pragma clang loop distribute(assume_safety)
/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop vectorize(badidentifier)
/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop interleave(badidentifier)
VList[j] = List[j];
}
+#pragma clang loop distribute(enable)
+ for (int j : VList) {
+ VList[j] = List[j];
+ }
+
+#pragma clang loop distribute(disable)
+ for (int j : VList) {
+ VList[j] = List[j];
+ }
+
test_nontype_template_param<4, 8>(List, Length);
/* expected-error {{expected '('}} */ #pragma clang loop vectorize
/* expected-error {{expected '('}} */ #pragma clang loop interleave
/* expected-error {{expected '('}} */ #pragma clang loop unroll
+/* expected-error {{expected '('}} */ #pragma clang loop distribute
/* expected-error {{expected ')'}} */ #pragma clang loop vectorize(enable
/* expected-error {{expected ')'}} */ #pragma clang loop interleave(enable
/* expected-error {{expected ')'}} */ #pragma clang loop unroll(full
+/* expected-error {{expected ')'}} */ #pragma clang loop distribute(enable
/* expected-error {{expected ')'}} */ #pragma clang loop vectorize_width(4
/* expected-error {{expected ')'}} */ #pragma clang loop interleave_count(4
/* expected-error {{missing argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop vectorize()
/* expected-error {{missing argument; expected an integer value}} */ #pragma clang loop interleave_count()
/* expected-error {{missing argument; expected 'enable', 'full' or 'disable'}} */ #pragma clang loop unroll()
+/* expected-error {{missing argument; expected 'enable' or 'disable'}} */ #pragma clang loop distribute()
-/* expected-error {{missing option; expected vectorize, vectorize_width, interleave, interleave_count, unroll, or unroll_count}} */ #pragma clang loop
+/* expected-error {{missing option; expected vectorize, vectorize_width, interleave, interleave_count, unroll, unroll_count, or distribute}} */ #pragma clang loop
/* expected-error {{invalid option 'badkeyword'}} */ #pragma clang loop badkeyword
/* expected-error {{invalid option 'badkeyword'}} */ #pragma clang loop badkeyword(enable)
/* expected-error {{invalid option 'badkeyword'}} */ #pragma clang loop vectorize(enable) badkeyword(4)
/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop vectorize(badidentifier)
/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop interleave(badidentifier)
/* expected-error {{invalid argument; expected 'enable', 'full' or 'disable'}} */ #pragma clang loop unroll(badidentifier)
+/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */ #pragma clang loop distribute(badidentifier)
while (i-7 < Length) {
List[i] = i;
}
/* expected-error {{expected ')'}} */ #pragma clang loop vectorize(()
/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop interleave(*)
/* expected-error {{invalid argument; expected 'enable', 'full' or 'disable'}} */ #pragma clang loop unroll(=)
+/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */ #pragma clang loop distribute(+)
/* expected-error {{type name requires a specifier or qualifier}} expected-error {{expected expression}} */ #pragma clang loop vectorize_width(^)
/* expected-error {{expected expression}} expected-error {{expected expression}} */ #pragma clang loop interleave_count(/)
/* expected-error {{expected expression}} expected-error {{expected expression}} */ #pragma clang loop unroll_count(==)
#pragma clang loop interleave(disable)
/* expected-error {{duplicate directives 'unroll(disable)' and 'unroll(full)'}} */ #pragma clang loop unroll(full)
#pragma clang loop unroll(disable)
+/* expected-error {{duplicate directives 'distribute(disable)' and 'distribute(enable)'}} */ #pragma clang loop distribute(enable)
+#pragma clang loop distribute(disable)
while (i-9 < Length) {
List[i] = i;
}