[CodeGen][X86] Implement _InterlockedCompareExchange128 intrinsic

Summary:
InterlockedCompareExchange128 is a bit more complicated than the other
InterlockedCompareExchange functions, so it requires a bit more work. It
doesn't directly refer to 128bit ints, instead it takes pointers to
64bit ints for Destination and ComparandResult, and exchange is taken as
two 64bit ints (high & low). The previous value is written to
ComparandResult, and success is returned. This implementation does the
following in order to produce a cmpxchg instruction:

  1. Cast everything to 128bit ints or int pointers, and glues together
     the Exchange values
  2. Reads from CompareandResult to get the comparand
  3. Calls cmpxchg volatile (on X86 this will produce a lock cmpxchg16b
     instruction)
    1. Result 0 (previous value) is written back to ComparandResult
    2. Result 1 (success bool) is zext'ed to a uchar and returned

Resolves bug https://llvm.org/PR35251

Patch by Colden Cullen!

Reviewers: rnk, agutowski

Reviewed By: rnk

Subscribers: majnemer, cfe-commits

Differential Revision: https://reviews.llvm.org/D41032

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@320730 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/clang/Basic/BuiltinsX86_64.def b/include/clang/Basic/BuiltinsX86_64.def
index 3d879386f73aa36fae12747215ac3d54c09d240b..fe2c887c7e0f5520d8c289fea3d42ca4bcba7b29 100644 (file)
--- a/include/clang/Basic/BuiltinsX86_64.def
+++ b/include/clang/Basic/BuiltinsX86_64.def
@@ -40,6 +40,7 @@ TARGET_HEADER_BUILTIN(_InterlockedExchangeSub64, "LLiLLiD*LLi", "nh", "intrin.h"
 TARGET_HEADER_BUILTIN(_InterlockedIncrement64,   "LLiLLiD*",    "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedOr64,          "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedXor64,         "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", "intrin.h", ALL_MS_LANGUAGES, "cx16")
 
 TARGET_BUILTIN(__builtin_ia32_readeflags_u64, "ULLi", "n", "")
 TARGET_BUILTIN(__builtin_ia32_writeeflags_u64, "vULLi", "n", "")

diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 527cbf2d9d45dbbd156d546379fcea882e6b9d95..cb052ebcaccbefd9707101edb40e063d0139bc20 100644 (file)
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -8432,6 +8432,45 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
   case X86::BI_InterlockedIncrement64:
     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
+  case X86::BI_InterlockedCompareExchange128: {
+    // InterlockedCompareExchange128 doesn't directly refer to 128bit ints,
+    // instead it takes pointers to 64bit ints for Destination and
+    // ComparandResult, and exchange is taken as two 64bit ints (high & low).
+    // The previous value is written to ComparandResult, and success is
+    // returned.
+
+    llvm::Type *Int128Ty = Builder.getInt128Ty();
+    llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
+
+    Value *Destination =
+        Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PtrTy);
+    Value *ExchangeHigh128 =
+        Builder.CreateZExt(EmitScalarExpr(E->getArg(1)), Int128Ty);
+    Value *ExchangeLow128 =
+        Builder.CreateZExt(EmitScalarExpr(E->getArg(2)), Int128Ty);
+    Address ComparandResult(
+        Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int128PtrTy),
+        getContext().toCharUnitsFromBits(128));
+
+    Value *Exchange = Builder.CreateOr(
+        Builder.CreateShl(ExchangeHigh128, 64, "", false, false),
+        ExchangeLow128);
+
+    Value *Comparand = Builder.CreateLoad(ComparandResult);
+
+    AtomicCmpXchgInst *CXI =
+        Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
+                                    AtomicOrdering::SequentiallyConsistent,
+                                    AtomicOrdering::SequentiallyConsistent);
+    CXI->setVolatile(true);
+
+    // Write the result back to the inout pointer.
+    Builder.CreateStore(Builder.CreateExtractValue(CXI, 0), ComparandResult);
+
+    // Get the success boolean and zero extend it to i8.
+    Value *Success = Builder.CreateExtractValue(CXI, 1);
+    return Builder.CreateZExt(Success, ConvertType(E->getType()));
+  }
 
   case X86::BI_AddressOfReturnAddress: {
     Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress);

diff --git a/test/CodeGen/ms-intrinsics.c b/test/CodeGen/ms-intrinsics.c
index 818be7fd7ffd8bed4e2e90441043d5a7ef8160ff..38cda978502940242e2c1d93c13450f7b3681baa 100644 (file)
--- a/test/CodeGen/ms-intrinsics.c
+++ b/test/CodeGen/ms-intrinsics.c
@@ -5,7 +5,7 @@
 // RUN:         -triple thumbv7--windows -Oz -emit-llvm %s -o - \
 // RUN:         | FileCheck %s --check-prefixes CHECK,CHECK-ARM,CHECK-ARM-X64
 // RUN: %clang_cc1 -ffreestanding -fms-extensions -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:         -triple x86_64--windows -Oz -emit-llvm %s -o - \
+// RUN:         -triple x86_64--windows -Oz -emit-llvm -target-feature +cx16 %s -o - \
 // RUN:         | FileCheck %s --check-prefixes CHECK,CHECK-X64,CHECK-ARM-X64,CHECK-INTEL
 
 // intrin.h needs size_t, but -ffreestanding prevents us from getting it from
@@ -329,6 +329,27 @@ __int64 test_InterlockedCompareExchange64(__int64 volatile *Destination, __int64
 // CHECK: ret i64 [[RESULT]]
 // CHECK: }
 
+#if defined(__x86_64__)
+unsigned char test_InterlockedCompareExchange128(__int64 volatile *Destination, __int64 ExchangeHigh, __int64 ExchangeLow, __int64* ComparandResult) {
+  return _InterlockedCompareExchange128(Destination, ExchangeHigh, ExchangeLow, ComparandResult);
+}
+// CHECK-X64: define{{.*}}i8 @test_InterlockedCompareExchange128(i64*{{[a-z_ ]*}}%Destination, i64{{[a-z_ ]*}}%ExchangeHigh, i64{{[a-z_ ]*}}%ExchangeLow, i64*{{[a-z_ ]*}}%ComparandResult){{.*}}{
+// CHECK-X64: [[DST:%[0-9]+]] = bitcast i64* %Destination to i128*
+// CHECK-X64: [[EH:%[0-9]+]] = zext i64 %ExchangeHigh to i128
+// CHECK-X64: [[EL:%[0-9]+]] = zext i64 %ExchangeLow to i128
+// CHECK-X64: [[CNR:%[0-9]+]] = bitcast i64* %ComparandResult to i128*
+// CHECK-X64: [[EHS:%[0-9]+]] = shl nuw i128 [[EH]], 64
+// CHECK-X64: [[EXP:%[0-9]+]] = or i128 [[EHS]], [[EL]]
+// CHECK-X64: [[ORG:%[0-9]+]] = load i128, i128* [[CNR]], align 16
+// CHECK-X64: [[RES:%[0-9]+]] = cmpxchg volatile i128* [[DST]], i128 [[ORG]], i128 [[EXP]] seq_cst seq_cst
+// CHECK-X64: [[OLD:%[0-9]+]] = extractvalue { i128, i1 } [[RES]], 0
+// CHECK-X64: store i128 [[OLD]], i128* [[CNR]], align 16
+// CHECK-X64: [[SUC1:%[0-9]+]] = extractvalue { i128, i1 } [[RES]], 1
+// CHECK-X64: [[SUC8:%[0-9]+]] = zext i1 [[SUC1]] to i8
+// CHECK-X64: ret i8 [[SUC8]]
+// CHECK-X64: }
+#endif
+
 short test_InterlockedIncrement16(short volatile *Addend) {
   return _InterlockedIncrement16(Addend);
 }