From: Tyler Nowicki Date: Thu, 11 Jun 2015 23:23:17 +0000 (+0000) Subject: Add assume_safety option for pragma loop vectorize and interleave. X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=addc7c62d7c181179884998f6f3a8fad9525e081;p=clang Add assume_safety option for pragma loop vectorize and interleave. Specifying #pragma clang loop vectorize(assume_safety) on a loop adds the mem.parallel_loop_access metadata to each load/store operation in the loop. This metadata tells loop access analysis (LAA) to skip memory dependency checking. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@239572 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/Basic/Attr.td b/include/clang/Basic/Attr.td index c310d25a55..bcc230a46f 100644 --- a/include/clang/Basic/Attr.td +++ b/include/clang/Basic/Attr.td @@ -1935,8 +1935,8 @@ def LoopHint : Attr { ["Vectorize", "VectorizeWidth", "Interleave", "InterleaveCount", "Unroll", "UnrollCount"]>, EnumArgument<"State", "LoopHintState", - ["default", "enable", "disable"], - ["Default", "Enable", "Disable"]>, + ["default", "enable", "disable", "assume_safety"], + ["Default", "Enable", "Disable", "AssumeSafety"]>, ExprArgument<"Value">]; let AdditionalMembers = [{ @@ -1982,6 +1982,8 @@ def LoopHint : Attr { return ""; else if (state == Enable) OS << (option == Unroll ? "full" : "enable"); + else if (state == AssumeSafety) + OS << "assume_safety"; else OS << "disable"; OS << ")"; diff --git a/include/clang/Basic/DiagnosticParseKinds.td b/include/clang/Basic/DiagnosticParseKinds.td index f00a3b39d2..c119c4a7d1 100644 --- a/include/clang/Basic/DiagnosticParseKinds.td +++ b/include/clang/Basic/DiagnosticParseKinds.td @@ -989,12 +989,12 @@ def err_omp_expected_identifier_for_critical : Error< // Pragma loop support. def err_pragma_loop_missing_argument : Error< "missing argument; expected %select{an integer value|" - "'%select{enable|full}1' or 'disable'}0">; + "%select{'enable', 'assume_safety'|'full'}1 or 'disable'}0">; def err_pragma_loop_invalid_option : Error< "%select{invalid|missing}0 option%select{ %1|}0; expected vectorize, " "vectorize_width, interleave, interleave_count, unroll, or unroll_count">; def err_pragma_invalid_keyword : Error< - "invalid argument; expected '%select{enable|full}0' or 'disable'">; + "invalid argument; expected %select{'enable', 'assume_safety'|'full'}0 or 'disable'">; // Pragma unroll support. def warn_pragma_unroll_cuda_value_in_parens : Warning< diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp index 0675544bed..3254fb81dc 100644 --- a/lib/CodeGen/CGLoopInfo.cpp +++ b/lib/CodeGen/CGLoopInfo.cpp @@ -8,6 +8,8 @@ //===----------------------------------------------------------------------===// #include "CGLoopInfo.h" +#include "clang/AST/Attr.h" +#include "clang/Sema/LoopHint.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/InstrTypes.h" @@ -76,7 +78,33 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs) LoopID = createMetadata(Header->getContext(), Attrs); } -void LoopInfoStack::push(BasicBlock *Header) { +void LoopInfoStack::push(BasicBlock *Header, ArrayRef Attrs) { + for (const auto *Attr : Attrs) { + const LoopHintAttr *LH = dyn_cast(Attr); + + // Skip non loop hint attributes + if (!LH) + continue; + + LoopHintAttr::OptionType Option = LH->getOption(); + LoopHintAttr::LoopHintState State = LH->getState(); + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + if (State == LoopHintAttr::AssumeSafety) { + // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. + setParallel(true); + } + break; + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::Unroll: + case LoopHintAttr::UnrollCount: + // Nothing to do here for these loop hints. + break; + } + } + Active.push_back(LoopInfo(Header, StagedAttrs)); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); diff --git a/lib/CodeGen/CGLoopInfo.h b/lib/CodeGen/CGLoopInfo.h index aee162118a..2249937cd0 100644 --- a/lib/CodeGen/CGLoopInfo.h +++ b/lib/CodeGen/CGLoopInfo.h @@ -15,6 +15,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H #define LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/Value.h" @@ -27,6 +28,7 @@ class MDNode; } // end namespace llvm namespace clang { +class Attr; namespace CodeGen { /// \brief Attributes that may be specified on loops. @@ -86,7 +88,8 @@ public: /// \brief Begin a new structured loop. The set of staged attributes will be /// applied to the loop and then cleared. - void push(llvm::BasicBlock *Header); + void push(llvm::BasicBlock *Header, + llvm::ArrayRef Attrs = llvm::None); /// \brief End the current loop. void pop(); diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp index 744c61384c..57f55aacfa 100644 --- a/lib/CodeGen/CGStmt.cpp +++ b/lib/CodeGen/CGStmt.cpp @@ -682,7 +682,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); EmitBlock(LoopHeader.getBlock()); - LoopStack.push(LoopHeader.getBlock()); + LoopStack.push(LoopHeader.getBlock(), WhileAttrs); // Create an exit block for when the condition fails, which will // also become the break target. @@ -776,7 +776,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); - LoopStack.push(LoopBody); + LoopStack.push(LoopBody, DoAttrs); EmitBlockWithFallThrough(LoopBody, &S); { @@ -842,7 +842,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, llvm::BasicBlock *CondBlock = Continue.getBlock(); EmitBlock(CondBlock); - LoopStack.push(CondBlock); + LoopStack.push(CondBlock, ForAttrs); // If the for loop doesn't have an increment we can just use the // condition as the continue block. Otherwise we'll need to create @@ -940,7 +940,7 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, llvm::BasicBlock *CondBlock = createBasicBlock("for.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock); + LoopStack.push(CondBlock, ForAttrs); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. diff --git a/lib/Parse/ParsePragma.cpp b/lib/Parse/ParsePragma.cpp index 84256dfd8e..892d3c6a52 100644 --- a/lib/Parse/ParsePragma.cpp +++ b/lib/Parse/ParsePragma.cpp @@ -823,9 +823,11 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) { ConsumeToken(); // The annotation token. SourceLocation StateLoc = Toks[0].getLocation(); IdentifierInfo *StateInfo = Toks[0].getIdentifierInfo(); - if (!StateInfo || ((OptionUnroll ? !StateInfo->isStr("full") - : !StateInfo->isStr("enable")) && - !StateInfo->isStr("disable"))) { + if (!StateInfo || + ((OptionUnroll ? !StateInfo->isStr("full") + : !StateInfo->isStr("enable") && + !StateInfo->isStr("assume_safety")) && + !StateInfo->isStr("disable"))) { Diag(Toks[0].getLocation(), diag::err_pragma_invalid_keyword) << /*FullKeyword=*/OptionUnroll; return false; @@ -1954,6 +1956,7 @@ static bool ParseLoopHintValue(Preprocessor &PP, Token &Tok, Token PragmaName, /// loop-hint-keyword: /// 'enable' /// 'disable' +/// 'assume_safety' /// /// unroll-hint-keyword: /// 'full' diff --git a/lib/Sema/SemaStmtAttr.cpp b/lib/Sema/SemaStmtAttr.cpp index 19e2c8ed65..5b71c11b52 100644 --- a/lib/Sema/SemaStmtAttr.cpp +++ b/lib/Sema/SemaStmtAttr.cpp @@ -105,6 +105,8 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const AttributeList &A, if (StateLoc && StateLoc->Ident) { if (StateLoc->Ident->isStr("disable")) State = LoopHintAttr::Disable; + else if (StateLoc->Ident->isStr("assume_safety")) + State = LoopHintAttr::AssumeSafety; else State = LoopHintAttr::Enable; } @@ -159,7 +161,7 @@ CheckForIncompatibleAttributes(Sema &S, const LoopHintAttr *PrevAttr; if (Option == LoopHintAttr::Vectorize || Option == LoopHintAttr::Interleave || Option == LoopHintAttr::Unroll) { - // Enable|disable hint. For example, vectorize(enable). + // Enable|Disable|AssumeSafety hint. For example, vectorize(enable). PrevAttr = CategoryState.StateAttr; CategoryState.StateAttr = LH; } else { diff --git a/test/CodeGenCXX/pragma-loop-safety.cpp b/test/CodeGenCXX/pragma-loop-safety.cpp new file mode 100644 index 0000000000..d12e412749 --- /dev/null +++ b/test/CodeGenCXX/pragma-loop-safety.cpp @@ -0,0 +1,49 @@ +// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s + +// Verify assume_safety vectorization is recognized. +void vectorize_test(int *List, int Length) { +// CHECK: define {{.*}} @_Z14vectorize_testPii +// CHECK: [[LOAD1_IV:.+]] = load i32, i32* [[IV1:[^,]+]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP1_ID:[0-9]+]] +// CHECK-NEXT: [[LOAD1_LEN:.+]] = load i32, i32* [[LEN1:.+]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP1_ID]] +// CHECK-NEXT: [[CMP1:.+]] = icmp slt i32[[LOAD1_IV]],[[LOAD1_LEN]] +// CHECK-NEXT: br i1[[CMP1]], label %[[LOOP1_BODY:[^,]+]], label %[[LOOP1_END:[^,]+]], !llvm.loop ![[LOOP1_HINTS:[0-9]+]] +#pragma clang loop vectorize(assume_safety) + for (int i = 0; i < Length; i++) { + // CHECK: [[LOOP1_BODY]] + // CHECK-NEXT: [[RHIV1:.+]] = load i32, i32* [[IV1]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP1_ID]] + // CHECK-NEXT: [[CALC1:.+]] = mul nsw i32[[RHIV1]], 2 + // CHECK-NEXT: [[SIV1:.+]] = load i32, i32* [[IV1]]{{.*}}!llvm.mem.parallel_loop_access ![[LOOP1_ID]] + // CHECK-NEXT: [[INDEX1:.+]] = sext i32[[SIV1]] to i64 + // CHECK-NEXT: [[ARRAY1:.+]] = load i32*, i32** [[LIST1:.*]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP1_ID]] + // CHECK-NEXT: [[PTR1:.+]] = getelementptr inbounds i32, i32*[[ARRAY1]], i64[[INDEX1]] + // CHECK-NEXT: store i32[[CALC1]], i32*[[PTR1]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP1_ID]] + List[i] = i * 2; + } + // CHECK: [[LOOP1_END]] +} + +// Verify assume_safety interleaving is recognized. +void interleave_test(int *List, int Length) { +// CHECK: define {{.*}} @_Z15interleave_testPii +// CHECK: [[LOAD2_IV:.+]] = load i32, i32* [[IV2:[^,]+]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP2_ID:[0-9]+]] +// CHECK-NEXT: [[LOAD2_LEN:.+]] = load i32, i32* [[LEN2:.+]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP2_ID]] +// CHECK-NEXT: [[CMP2:.+]] = icmp slt i32[[LOAD2_IV]],[[LOAD2_LEN]] +// CHECK-NEXT: br i1[[CMP2]], label %[[LOOP2_BODY:[^,]+]], label %[[LOOP2_END:[^,]+]], !llvm.loop ![[LOOP2_HINTS:[0-9]+]] +#pragma clang loop interleave(assume_safety) + for (int i = 0; i < Length; i++) { + // CHECK: [[LOOP2_BODY]] + // CHECK-NEXT: [[RHIV2:.+]] = load i32, i32* [[IV2]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP2_ID]] + // CHECK-NEXT: [[CALC2:.+]] = mul nsw i32[[RHIV2]], 2 + // CHECK-NEXT: [[SIV2:.+]] = load i32, i32* [[IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[LOOP2_ID]] + // CHECK-NEXT: [[INDEX2:.+]] = sext i32[[SIV2]] to i64 + // CHECK-NEXT: [[ARRAY2:.+]] = load i32*, i32** [[LIST2:.*]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP2_ID]] + // CHECK-NEXT: [[PTR2:.+]] = getelementptr inbounds i32, i32*[[ARRAY2]], i64[[INDEX2]] + // CHECK-NEXT: store i32[[CALC2]], i32*[[PTR2]], {{.*}}!llvm.mem.parallel_loop_access ![[LOOP2_ID]] + List[i] = i * 2; + } + // CHECK: [[LOOP2_END]] +} + +// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[INTENABLE_1:.*]]} +// CHCCK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[INTENABLE_1:.*]]} diff --git a/test/Parser/pragma-loop-safety.cpp b/test/Parser/pragma-loop-safety.cpp new file mode 100644 index 0000000000..cc98c775e5 --- /dev/null +++ b/test/Parser/pragma-loop-safety.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -std=c++11 -verify %s + +// Note that this puts the expected lines before the directives to work around +// limitations in the -verify mode. + +void test(int *List, int Length) { + int i = 0; + +#pragma clang loop vectorize(assume_safety) +#pragma clang loop interleave(assume_safety) + while (i + 1 < Length) { + List[i] = i; + } + +/* expected-error {{expected ')'}} */ #pragma clang loop vectorize(assume_safety +/* expected-error {{expected ')'}} */ #pragma clang loop interleave(assume_safety + +/* expected-error {{invalid argument; expected 'full' or 'disable'}} */ #pragma clang loop unroll(assume_safety) + +/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop vectorize(badidentifier) +/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop interleave(badidentifier) +/* expected-error {{invalid argument; expected 'full' or 'disable'}} */ #pragma clang loop unroll(badidentifier) + while (i-7 < Length) { + List[i] = i; + } + +/* expected-error {{duplicate directives 'vectorize(assume_safety)' and 'vectorize(enable)'}} */ #pragma clang loop vectorize(enable) +#pragma clang loop vectorize(assume_safety) +/* expected-error {{duplicate directives 'interleave(assume_safety)' and 'interleave(enable)'}} */ #pragma clang loop interleave(enable) +#pragma clang loop interleave(assume_safety) + while (i-9 < Length) { + List[i] = i; + } +} diff --git a/test/Parser/pragma-loop.cpp b/test/Parser/pragma-loop.cpp index a0213ac50d..60820584a9 100644 --- a/test/Parser/pragma-loop.cpp +++ b/test/Parser/pragma-loop.cpp @@ -130,7 +130,7 @@ void test(int *List, int Length) { /* expected-error {{expected ')'}} */ #pragma clang loop interleave_count(4 /* expected-error {{expected ')'}} */ #pragma clang loop unroll_count(4 -/* expected-error {{missing argument; expected 'enable' or 'disable'}} */ #pragma clang loop vectorize() +/* expected-error {{missing argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop vectorize() /* expected-error {{missing argument; expected an integer value}} */ #pragma clang loop interleave_count() /* expected-error {{missing argument; expected 'full' or 'disable'}} */ #pragma clang loop unroll() @@ -184,8 +184,8 @@ const int VV = 4; List[i] = i; } -/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */ #pragma clang loop vectorize(badidentifier) -/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */ #pragma clang loop interleave(badidentifier) +/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop vectorize(badidentifier) +/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop interleave(badidentifier) /* expected-error {{invalid argument; expected 'full' or 'disable'}} */ #pragma clang loop unroll(badidentifier) while (i-7 < Length) { List[i] = i; @@ -194,7 +194,7 @@ const int VV = 4; // PR20069 - Loop pragma arguments that are not identifiers or numeric // constants crash FE. /* expected-error {{expected ')'}} */ #pragma clang loop vectorize(() -/* expected-error {{invalid argument; expected 'enable' or 'disable'}} */ #pragma clang loop interleave(*) +/* expected-error {{invalid argument; expected 'enable', 'assume_safety' or 'disable'}} */ #pragma clang loop interleave(*) /* expected-error {{invalid argument; expected 'full' or 'disable'}} */ #pragma clang loop unroll(=) /* expected-error {{type name requires a specifier or qualifier}} expected-error {{expected expression}} */ #pragma clang loop vectorize_width(^) /* expected-error {{expected expression}} expected-error {{expected expression}} */ #pragma clang loop interleave_count(/)