[hwasan] Inline instrumentation & fixed shadow.

Summary: This brings CPU overhead on bzip2 down from 5.5x to 2x.

Reviewers: kcc, alekseyshl

Subscribers: kubamracek, hiraditya, llvm-commits

Differential Revision: https://reviews.llvm.org/D41137

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@320538 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 7a1852c0da1de9b6d7771d05bc1f67143a72f007..2a25423e04bd663770aeb893be4f42cad8643c48 100644 (file)
--- a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -22,7 +22,10 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/IR/Function.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstVisitor.h"
@@ -51,11 +54,19 @@ static const char *const kHwasanInitName = "__hwasan_init";
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
 static const size_t kNumberOfAccessSizes = 5;
 
+static const size_t kShadowScale = 4;
+static const unsigned kPointerTagShift = 56;
+
 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
     "hwasan-memory-access-callback-prefix",
     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
     cl::init("__hwasan_"));
 
+static cl::opt<bool>
+    ClInstrumentWithCalls("hwasan-instrument-with-calls",
+                cl::desc("instrument reads and writes with callbacks"),
+                cl::Hidden, cl::init(false));
+
 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
                                        cl::desc("instrument read instructions"),
                                        cl::Hidden, cl::init(true));
@@ -86,6 +97,9 @@ public:
   bool doInitialization(Module &M) override;
 
   void initializeCallbacks(Module &M);
+  void instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
+                                 unsigned AccessSizeIndex,
+                                 Instruction *InsertBefore);
   bool instrumentMemAccess(Instruction *I);
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
                                    uint64_t *TypeSize, unsigned *Alignment,
@@ -219,6 +233,31 @@ static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
   return Res;
 }
 
+void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
+                                                   unsigned AccessSizeIndex,
+                                                   Instruction *InsertBefore) {
+  IRBuilder<> IRB(InsertBefore);
+  Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty());
+  Value *AddrLong =
+      IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
+                                              ~(0xFFULL << kPointerTagShift)));
+  Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
+  Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
+  Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
+
+  TerminatorInst *CheckTerm =
+      SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
+                                MDBuilder(*C).createBranchWeights(1, 100000));
+
+  IRB.SetInsertPoint(CheckTerm);
+  // The signal handler will find the data address in x0.
+  InlineAsm *Asm = InlineAsm::get(
+      FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
+      "hlt #" + itostr(0x100 + IsWrite * 0x10 + AccessSizeIndex), "{x0}",
+      /*hasSideEffects=*/true);
+  IRB.CreateCall(Asm, PtrLong);
+}
+
 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
   DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
   bool IsWrite = false;
@@ -237,10 +276,16 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
   IRBuilder<> IRB(I);
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   if (isPowerOf2_64(TypeSize) &&
-      (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1)))) {
+      (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
+      (Alignment >= (1UL << kShadowScale) || Alignment == 0 ||
+       Alignment >= TypeSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
-    IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
-                   AddrLong);
+    if (ClInstrumentWithCalls) {
+      IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
+                     AddrLong);
+    } else {
+      instrumentMemAccessInline(AddrLong, IsWrite, AccessSizeIndex, I);
+    }
   } else {
     IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
                    {AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8)});

diff --git a/test/Instrumentation/HWAddressSanitizer/atomic.ll b/test/Instrumentation/HWAddressSanitizer/atomic.ll
index 67ddac73de548c9c0fc0244c77342cae90888422..5492fda61bbba1f8a196a71c8ff9ec4c6abc4083 100644 (file)
--- a/test/Instrumentation/HWAddressSanitizer/atomic.ll
+++ b/test/Instrumentation/HWAddressSanitizer/atomic.ll
@@ -7,8 +7,10 @@ target triple = "aarch64--linux-android"
 
 define void @atomicrmw(i64* %ptr) sanitize_hwaddress {
 ; CHECK-LABEL: @atomicrmw(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %ptr to i64
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
+; CHECK: lshr i64 %[[A:[^ ]*]], 56
+; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: atomicrmw add i64* %ptr, i64 1 seq_cst
+; CHECK: ret void
 
 entry:
   %0 = atomicrmw add i64* %ptr, i64 1 seq_cst
@@ -17,8 +19,10 @@ entry:
 
 define void @cmpxchg(i64* %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
 ; CHECK-LABEL: @cmpxchg(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %ptr to i64
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
+; CHECK: lshr i64 %[[A:[^ ]*]], 56
+; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
+; CHECK: ret void
 
 entry:
   %0 = cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst

diff --git a/test/Instrumentation/HWAddressSanitizer/basic.ll b/test/Instrumentation/HWAddressSanitizer/basic.ll
index 93129a51d741de90760e874bac4aa8c8aef03c53..db1f52fc56e92d8353c85c5b077b0e5168075e57 100644 (file)
--- a/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -8,9 +8,20 @@ target triple = "aarch64--linux-android"
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: call void @__hwasan_load1(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i8, i8* %a
-; CHECK: ret i8 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #256", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
+; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
@@ -20,9 +31,20 @@ entry:
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: call void @__hwasan_load2(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i16, i16* %a
-; CHECK: ret i16 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #257", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
+; CHECK: ret i16 %[[G]]
 
 entry:
   %b = load i16, i16* %a, align 4
@@ -32,9 +54,20 @@ entry:
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: call void @__hwasan_load4(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i32, i32* %a
-; CHECK: ret i32 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #258", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
+; CHECK: ret i32 %[[G]]
 
 entry:
   %b = load i32, i32* %a, align 4
@@ -44,9 +77,20 @@ entry:
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_load8(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i64, i64* %a
-; CHECK: ret i64 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #259", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
+; CHECK: ret i64 %[[G]]
 
 entry:
   %b = load i64, i64* %a, align 8
@@ -56,9 +100,20 @@ entry:
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: call void @__hwasan_load16(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i128, i128* %a
-; CHECK: ret i128 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #260", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
+; CHECK: ret i128 %[[G]]
 
 entry:
   %b = load i128, i128* %a, align 16
@@ -80,8 +135,19 @@ entry:
 define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: call void @__hwasan_store1(i64 %[[A]])
-; CHECK: store i8 %b, i8* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #272", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
@@ -92,8 +158,19 @@ entry:
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: call void @__hwasan_store2(i64 %[[A]])
-; CHECK: store i16 %b, i16* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #273", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
 entry:
@@ -104,8 +181,19 @@ entry:
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: call void @__hwasan_store4(i64 %[[A]])
-; CHECK: store i32 %b, i32* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #274", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
 entry:
@@ -116,24 +204,46 @@ entry:
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
-; CHECK: store i64 %b, i64* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
 entry:
-  store i64 %b, i64* %a, align 4
+  store i64 %b, i64* %a, align 8
   ret void
 }
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: call void @__hwasan_store16(i64 %[[A]])
-; CHECK: store i128 %b, i128* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #276", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
 
 entry:
-  store i128 %b, i128* %a, align 4
+  store i128 %b, i128* %a, align 16
   ret void
 }
 
@@ -149,6 +259,18 @@ entry:
   ret void
 }
 
+define void @test_store_unaligned(i64* %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store_unaligned(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
+; CHECK: call void @__hwasan_store(i64 %[[A]], i64 8)
+; CHECK: store i64 %b, i64* %a, align 4
+; CHECK: ret void
+
+entry:
+  store i64 %b, i64* %a, align 4
+  ret void
+}
+
 define i8 @test_load_noattr(i8* %a) {
 ; CHECK-LABEL: @test_load_noattr(
 ; CHECK-NEXT: entry:

diff --git a/test/Instrumentation/HWAddressSanitizer/with-calls.ll b/test/Instrumentation/HWAddressSanitizer/with-calls.ll
new file mode 100644 (file)
index 0000000..be3b33d
--- /dev/null
+++ b/test/Instrumentation/HWAddressSanitizer/with-calls.ll
@@ -0,0 +1,190 @@
+; Test basic address sanitizer instrumentation.
+;
+; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+define i8 @test_load8(i8* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load8(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; CHECK: call void @__hwasan_load1(i64 %[[A]])
+; CHECK: %[[B:[^ ]*]] = load i8, i8* %a
+; CHECK: ret i8 %[[B]]
+
+entry:
+  %b = load i8, i8* %a, align 4
+  ret i8 %b
+}
+
+define i16 @test_load16(i16* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load16(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
+; CHECK: call void @__hwasan_load2(i64 %[[A]])
+; CHECK: %[[B:[^ ]*]] = load i16, i16* %a
+; CHECK: ret i16 %[[B]]
+
+entry:
+  %b = load i16, i16* %a, align 4
+  ret i16 %b
+}
+
+define i32 @test_load32(i32* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load32(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
+; CHECK: call void @__hwasan_load4(i64 %[[A]])
+; CHECK: %[[B:[^ ]*]] = load i32, i32* %a
+; CHECK: ret i32 %[[B]]
+
+entry:
+  %b = load i32, i32* %a, align 4
+  ret i32 %b
+}
+
+define i64 @test_load64(i64* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load64(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
+; CHECK: call void @__hwasan_load8(i64 %[[A]])
+; CHECK: %[[B:[^ ]*]] = load i64, i64* %a
+; CHECK: ret i64 %[[B]]
+
+entry:
+  %b = load i64, i64* %a, align 8
+  ret i64 %b
+}
+
+define i128 @test_load128(i128* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load128(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
+; CHECK: call void @__hwasan_load16(i64 %[[A]])
+; CHECK: %[[B:[^ ]*]] = load i128, i128* %a
+; CHECK: ret i128 %[[B]]
+
+entry:
+  %b = load i128, i128* %a, align 16
+  ret i128 %b
+}
+
+define i40 @test_load40(i40* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load40(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
+; CHECK: call void @__hwasan_load(i64 %[[A]], i64 5)
+; CHECK: %[[B:[^ ]*]] = load i40, i40* %a
+; CHECK: ret i40 %[[B]]
+
+entry:
+  %b = load i40, i40* %a, align 4
+  ret i40 %b
+}
+
+define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store8(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; CHECK: call void @__hwasan_store1(i64 %[[A]])
+; CHECK: store i8 %b, i8* %a
+; CHECK: ret void
+
+entry:
+  store i8 %b, i8* %a, align 4
+  ret void
+}
+
+define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store16(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
+; CHECK: call void @__hwasan_store2(i64 %[[A]])
+; CHECK: store i16 %b, i16* %a
+; CHECK: ret void
+
+entry:
+  store i16 %b, i16* %a, align 4
+  ret void
+}
+
+define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store32(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
+; CHECK: call void @__hwasan_store4(i64 %[[A]])
+; CHECK: store i32 %b, i32* %a
+; CHECK: ret void
+
+entry:
+  store i32 %b, i32* %a, align 4
+  ret void
+}
+
+define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store64(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
+; CHECK: call void @__hwasan_store8(i64 %[[A]])
+; CHECK: store i64 %b, i64* %a
+; CHECK: ret void
+
+entry:
+  store i64 %b, i64* %a, align 8
+  ret void
+}
+
+define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store128(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
+; CHECK: call void @__hwasan_store16(i64 %[[A]])
+; CHECK: store i128 %b, i128* %a
+; CHECK: ret void
+
+entry:
+  store i128 %b, i128* %a, align 16
+  ret void
+}
+
+define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store40(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
+; CHECK: call void @__hwasan_store(i64 %[[A]], i64 5)
+; CHECK: store i40 %b, i40* %a
+; CHECK: ret void
+
+entry:
+  store i40 %b, i40* %a, align 4
+  ret void
+}
+
+define i8 @test_load_noattr(i8* %a) {
+; CHECK-LABEL: @test_load_noattr(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %[[B:[^ ]*]] = load i8, i8* %a
+; CHECK-NEXT: ret i8 %[[B]]
+
+entry:
+  %b = load i8, i8* %a, align 4
+  ret i8 %b
+}
+
+define i8 @test_load_notmyattr(i8* %a) sanitize_address {
+; CHECK-LABEL: @test_load_notmyattr(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %[[B:[^ ]*]] = load i8, i8* %a
+; CHECK-NEXT: ret i8 %[[B]]
+
+entry:
+  %b = load i8, i8* %a, align 4
+  ret i8 %b
+}
+
+define i8 @test_load_addrspace(i8 addrspace(256)* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load_addrspace(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %[[B:[^ ]*]] = load i8, i8 addrspace(256)* %a
+; CHECK-NEXT: ret i8 %[[B]]
+
+entry:
+  %b = load i8, i8 addrspace(256)* %a, align 4
+  ret i8 %b
+}
+
+; CHECK: declare void @__hwasan_init()
+
+; CHECK:      define internal void @hwasan.module_ctor() {
+; CHECK-NEXT:   call void @__hwasan_init()
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }