Emit @llvm.assume for non-parameter lvalue align_value-attribute loads

We already add the align parameter attribute for function parameters that have
the align_value attribute (or those with a typedef type having that attribute),
which is an important special case, but does not handle pointers with value
alignment assumptions that come into scope in any other way. To handle the
general case, emit an @llvm.assume-based alignment assumption whenever we load
the pointer-typed lvalue of an align_value-attributed variable (except for
function parameters, which we already deal with at entry).

I'll also note that this is more general than Intel's described support in:
  https://software.intel.com/en-us/articles/data-alignment-to-assist-vectorization
which states that the compiler inserts __assume_aligned directives in response
to align_value-attributed variables only for function parameters and for the
initializers of local variables. I think that we can make the optimizer deal
with this more-general scheme (which could lead to a lot of calls to
@llvm.assume inside of loop bodies, for example), but if not, I'll rework this
to be less aggressive.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@219052 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 207f98f85f17a2a3a8773c9d1c539259216f7bd5..abde51f29e4f3f04c1ba2886ba68dd1b54824168 100644 (file)
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -91,12 +91,47 @@ public:
     return CGF.EmitLoadOfLValue(LV, Loc).getScalarVal();
   }
 
+  void EmitLValueAlignmentAssumption(const Expr *E, Value *V) {
+    const AlignValueAttr *AVAttr = nullptr;
+    if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
+      const ValueDecl *VD = DRE->getDecl();
+
+      if (VD->getType()->isReferenceType()) {
+        if (const auto *TTy =
+            dyn_cast<TypedefType>(VD->getType().getNonReferenceType()))
+          AVAttr = TTy->getDecl()->getAttr<AlignValueAttr>();
+      } else {
+        // Assumptions for function parameters are emitted at the start of the
+        // function, so there is no need to repeat that here.
+        if (isa<ParmVarDecl>(VD))
+          return;
+
+        AVAttr = VD->getAttr<AlignValueAttr>();
+      }
+    }
+
+    if (!AVAttr)
+      if (const auto *TTy =
+          dyn_cast<TypedefType>(E->getType()))
+        AVAttr = TTy->getDecl()->getAttr<AlignValueAttr>();
+
+    if (!AVAttr)
+      return;
+
+    Value *AlignmentValue = CGF.EmitScalarExpr(AVAttr->getAlignment());
+    llvm::ConstantInt *AlignmentCI = cast<llvm::ConstantInt>(AlignmentValue);
+    CGF.EmitAlignmentAssumption(V, AlignmentCI->getZExtValue());
+  }
+
   /// EmitLoadOfLValue - Given an expression with complex type that represents a
   /// value l-value, this method emits the address of the l-value, then loads
   /// and returns the result.
   Value *EmitLoadOfLValue(const Expr *E) {
-    return EmitLoadOfLValue(EmitCheckedLValue(E, CodeGenFunction::TCK_Load),
-                            E->getExprLoc());
+    Value *V = EmitLoadOfLValue(EmitCheckedLValue(E, CodeGenFunction::TCK_Load),
+                                E->getExprLoc());
+
+    EmitLValueAlignmentAssumption(E, V);
+    return V;
   }
 
   /// EmitConversionToBool - Convert the specified expression value to a
@@ -286,7 +321,10 @@ public:
     if (E->getCallReturnType()->isReferenceType())
       return EmitLoadOfLValue(E);
 
-    return CGF.EmitCallExpr(E).getScalarVal();
+    Value *V = CGF.EmitCallExpr(E).getScalarVal();
+
+    EmitLValueAlignmentAssumption(E, V);
+    return V;
   }
 
   Value *VisitStmtExpr(const StmtExpr *E);

diff --git a/test/CodeGen/align_value.cpp b/test/CodeGen/align_value.cpp
index 90bb37f322d5759e38a5b4913665b0d210b8440e..6d0e48128cb2f0e7036fc389093184fc163d7e01 100644 (file)
--- a/test/CodeGen/align_value.cpp
+++ b/test/CodeGen/align_value.cpp
@@ -6,3 +6,98 @@ void foo(aligned_double x, double * y __attribute__((align_value(32))),
          double & z __attribute__((align_value(128)))) { };
 // CHECK: define void @_Z3fooPdS_Rd(double* align 64 %x, double* align 32 %y, double* dereferenceable(8) align 128 %z)
 
+struct ad_struct {
+  aligned_double a;
+};
+
+double *foo(ad_struct& x) {
+// CHECK-LABEL: @_Z3fooR9ad_struct
+
+// CHECK: [[PTRINT1:%.+]] = ptrtoint
+// CHECK: [[MASKEDPTR1:%.+]] = and i64 [[PTRINT1]], 63
+// CHECK: [[MASKCOND1:%.+]] = icmp eq i64 [[MASKEDPTR1]], 0
+// CHECK: call void @llvm.assume(i1 [[MASKCOND1]])
+  return x.a;
+}
+
+double *goo(ad_struct *x) {
+// CHECK-LABEL: @_Z3gooP9ad_struct
+
+// CHECK: [[PTRINT2:%.+]] = ptrtoint
+// CHECK: [[MASKEDPTR2:%.+]] = and i64 [[PTRINT2]], 63
+// CHECK: [[MASKCOND2:%.+]] = icmp eq i64 [[MASKEDPTR2]], 0
+// CHECK: call void @llvm.assume(i1 [[MASKCOND2]])
+  return x->a;
+}
+
+double *bar(aligned_double *x) {
+// CHECK-LABEL: @_Z3barPPd
+
+// CHECK: [[PTRINT3:%.+]] = ptrtoint
+// CHECK: [[MASKEDPTR3:%.+]] = and i64 [[PTRINT3]], 63
+// CHECK: [[MASKCOND3:%.+]] = icmp eq i64 [[MASKEDPTR3]], 0
+// CHECK: call void @llvm.assume(i1 [[MASKCOND3]])
+  return *x;
+}
+
+double *car(aligned_double &x) {
+// CHECK-LABEL: @_Z3carRPd
+
+// CHECK: [[PTRINT4:%.+]] = ptrtoint
+// CHECK: [[MASKEDPTR4:%.+]] = and i64 [[PTRINT4]], 63
+// CHECK: [[MASKCOND4:%.+]] = icmp eq i64 [[MASKEDPTR4]], 0
+// CHECK: call void @llvm.assume(i1 [[MASKCOND4]])
+  return x;
+}
+
+double *dar(aligned_double *x) {
+// CHECK-LABEL: @_Z3darPPd
+
+// CHECK: [[PTRINT5:%.+]] = ptrtoint
+// CHECK: [[MASKEDPTR5:%.+]] = and i64 [[PTRINT5]], 63
+// CHECK: [[MASKCOND5:%.+]] = icmp eq i64 [[MASKEDPTR5]], 0
+// CHECK: call void @llvm.assume(i1 [[MASKCOND5]])
+  return x[5];
+}
+
+aligned_double eep();
+double *ret() {
+// CHECK-LABEL: @_Z3retv
+
+// CHECK: [[PTRINT6:%.+]] = ptrtoint
+// CHECK: [[MASKEDPTR6:%.+]] = and i64 [[PTRINT6]], 63
+// CHECK: [[MASKCOND6:%.+]] = icmp eq i64 [[MASKEDPTR6]], 0
+// CHECK: call void @llvm.assume(i1 [[MASKCOND6]])
+  return eep();
+}
+
+double **no1(aligned_double *x) {
+// CHECK-LABEL: @_Z3no1PPd
+  return x;
+// CHECK-NOT: call void @llvm.assume
+}
+
+double *&no2(aligned_double &x) {
+// CHECK-LABEL: @_Z3no2RPd
+  return x;
+// CHECK-NOT: call void @llvm.assume
+}
+
+double **no3(aligned_double &x) {
+// CHECK-LABEL: @_Z3no3RPd
+  return &x;
+// CHECK-NOT: call void @llvm.assume
+}
+
+double no3(aligned_double x) {
+// CHECK-LABEL: @_Z3no3Pd
+  return *x;
+// CHECK-NOT: call void @llvm.assume
+}
+
+double *no4(aligned_double x) {
+// CHECK-LABEL: @_Z3no4Pd
+  return x;
+// CHECK-NOT: call void @llvm.assume
+}
+