From 3a91316f3bfef8e9ea77fe34c54ac47015f6950e Mon Sep 17 00:00:00 2001 From: Piotr Padlewski Date: Tue, 15 Sep 2015 21:46:55 +0000 Subject: [PATCH] Decorating vptr load & stores with !invariant.group Adding !invariant.group to vptr load/stores for devirtualization purposes. For more goto: http://lists.llvm.org/pipermail/cfe-dev/2015-July/044227.html http://reviews.llvm.org/D12026 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@247725 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/CodeGen/CGAtomic.cpp | 4 +- lib/CodeGen/CGClass.cpp | 24 ++++-- lib/CodeGen/CGExpr.cpp | 8 +- lib/CodeGen/CGExprCXX.cpp | 2 +- lib/CodeGen/CodeGenFunction.h | 3 +- lib/CodeGen/CodeGenModule.cpp | 16 +++- lib/CodeGen/CodeGenModule.h | 10 ++- lib/CodeGen/ItaniumCXXABI.cpp | 23 ++++-- lib/CodeGen/MicrosoftCXXABI.cpp | 7 +- test/CodeGenCXX/invariant.group-for-vptrs.cpp | 75 +++++++++++++++++++ 10 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 test/CodeGenCXX/invariant.group-for-vptrs.cpp diff --git a/lib/CodeGen/CGAtomic.cpp b/lib/CodeGen/CGAtomic.cpp index 4e457fd43f..7c9f3daaa7 100644 --- a/lib/CodeGen/CGAtomic.cpp +++ b/lib/CodeGen/CGAtomic.cpp @@ -1246,7 +1246,7 @@ llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, if (IsVolatile) Load->setVolatile(true); if (LVal.getTBAAInfo()) - CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo()); + CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); return Load; } @@ -1769,7 +1769,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, if (IsVolatile) store->setVolatile(true); if (dest.getTBAAInfo()) - CGM.DecorateInstruction(store, dest.getTBAAInfo()); + CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); return; } diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp index 4c46d56590..7ec9c15aa5 100644 --- a/lib/CodeGen/CGClass.cpp +++ b/lib/CodeGen/CGClass.cpp @@ -25,6 +25,7 @@ #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Metadata.h" using namespace clang; using namespace CodeGen; @@ -2087,7 +2088,8 @@ void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) { ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr, Vptr.VTableClass, Vptr.NearestVBase); - llvm::Value *VPtrValue = GetVTablePtr(This, VTableGlobal->getType()); + llvm::Value *VPtrValue = + GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass); llvm::Value *Cmp = Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables"); Builder.CreateAssumption(Cmp); @@ -2306,7 +2308,10 @@ void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) { VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy); llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField); - CGM.DecorateInstruction(Store, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithTBAA(Store, CGM.getTBAAInfoForVTablePtr()); + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGM.getCodeGenOpts().StrictVTablePointers) + CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass); } CodeGenFunction::VPtrsVector @@ -2393,10 +2398,16 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) { } llvm::Value *CodeGenFunction::GetVTablePtr(Address This, - llvm::Type *Ty) { - Address VTablePtrSrc = Builder.CreateElementBitCast(This, Ty); + llvm::Type *VTableTy, + const CXXRecordDecl *RD) { + Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy); llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable"); - CGM.DecorateInstruction(VTable, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithTBAA(VTable, CGM.getTBAAInfoForVTablePtr()); + + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGM.getCodeGenOpts().StrictVTablePointers) + CGM.DecorateInstructionWithInvariantGroup(VTable, RD); + return VTable; } @@ -2481,7 +2492,8 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, } llvm::Value *VTable = - GetVTablePtr(Address(Derived, getPointerAlign()), Int8PtrTy); + GetVTablePtr(Address(Derived, getPointerAlign()), Int8PtrTy, ClassDecl); + EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); if (MayBeNull) { diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp index cda7766884..c38dbdd7d3 100644 --- a/lib/CodeGen/CGExpr.cpp +++ b/lib/CodeGen/CGExpr.cpp @@ -1277,7 +1277,8 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo, TBAAOffset); if (TBAAPath) - CGM.DecorateInstruction(Load, TBAAPath, false/*ConvertTypeToTag*/); + CGM.DecorateInstructionWithTBAA(Load, TBAAPath, + false /*ConvertTypeToTag*/); } bool NeedsBoolCheck = @@ -1391,7 +1392,8 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo, TBAAOffset); if (TBAAPath) - CGM.DecorateInstruction(Store, TBAAPath, false/*ConvertTypeToTag*/); + CGM.DecorateInstructionWithTBAA(Store, TBAAPath, + false /*ConvertTypeToTag*/); } } @@ -3115,7 +3117,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, else tbaa = CGM.getTBAAInfo(type); if (tbaa) - CGM.DecorateInstruction(load, tbaa); + CGM.DecorateInstructionWithTBAA(load, tbaa); } mayAlias = false; diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp index 1952282a07..7e0a6abc23 100644 --- a/lib/CodeGen/CGExprCXX.cpp +++ b/lib/CodeGen/CGExprCXX.cpp @@ -258,7 +258,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( } else { if (SanOpts.has(SanitizerKind::CFINVCall) && MD->getParent()->isDynamicClass()) { - llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy); + llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy, MD->getParent()); EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart()); } diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h index 76296dee13..cf04596772 100644 --- a/lib/CodeGen/CodeGenFunction.h +++ b/lib/CodeGen/CodeGenFunction.h @@ -1380,7 +1380,8 @@ public: /// GetVTablePtr - Return the Value of the vtable pointer member pointed /// to by This. - llvm::Value *GetVTablePtr(Address This, llvm::Type *Ty); + llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy, + const CXXRecordDecl *VTableClass); enum CFITypeCheckKind { CFITCK_VCall, diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp index d9bb24efff..bb3bbf2818 100644 --- a/lib/CodeGen/CodeGenModule.cpp +++ b/lib/CodeGen/CodeGenModule.cpp @@ -507,9 +507,9 @@ llvm::MDNode *CodeGenModule::getTBAAStructTagInfo(QualType BaseTy, /// and struct-path aware TBAA, the tag has the same format: /// base type, access type and offset. /// When ConvertTypeToTag is true, we create a tag based on the scalar type. -void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst, - llvm::MDNode *TBAAInfo, - bool ConvertTypeToTag) { +void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst, + llvm::MDNode *TBAAInfo, + bool ConvertTypeToTag) { if (ConvertTypeToTag && TBAA) Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAA->getTBAAScalarTagInfo(TBAAInfo)); @@ -517,6 +517,16 @@ void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst, Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo); } +void CodeGenModule::DecorateInstructionWithInvariantGroup( + llvm::Instruction *I, const CXXRecordDecl *RD) { + llvm::Metadata *MD = CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + auto *MetaDataNode = dyn_cast(MD); + // Check if we have to wrap MDString in MDNode. + if (!MetaDataNode) + MetaDataNode = llvm::MDNode::get(getLLVMContext(), MD); + I->setMetadata("invariant.group", MetaDataNode); +} + void CodeGenModule::Error(SourceLocation loc, StringRef message) { unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0"); getDiags().Report(Context.getFullLoc(loc), diagID) << message; diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h index 754e3d1d29..e7ae20fce0 100644 --- a/lib/CodeGen/CodeGenModule.h +++ b/lib/CodeGen/CodeGenModule.h @@ -658,9 +658,13 @@ public: /// is the same as the type. For struct-path aware TBAA, the tag /// is different from the type: base type, access type and offset. /// When ConvertTypeToTag is true, we create a tag based on the scalar type. - void DecorateInstruction(llvm::Instruction *Inst, - llvm::MDNode *TBAAInfo, - bool ConvertTypeToTag = true); + void DecorateInstructionWithTBAA(llvm::Instruction *Inst, + llvm::MDNode *TBAAInfo, + bool ConvertTypeToTag = true); + + /// Adds !invariant.barrier !tag to instruction + void DecorateInstructionWithInvariantGroup(llvm::Instruction *I, + const CXXRecordDecl *RD); /// Emit the given number of characters as a value of type size_t. llvm::ConstantInt *getSize(CharUnits numChars); diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp index 7228a6d8c1..da67b16c24 100644 --- a/lib/CodeGen/ItaniumCXXABI.cpp +++ b/lib/CodeGen/ItaniumCXXABI.cpp @@ -584,7 +584,7 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( CGF.CGM.getDynamicOffsetAlignment(ThisAddr.getAlignment(), RD, CGF.getPointerAlign()); llvm::Value *VTable = - CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy); + CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy, RD); // Apply the offset. llvm::Value *VTableOffset = FnAsInt; @@ -1012,7 +1012,10 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, // to pass to the deallocation function. // Grab the vtable pointer as an intptr_t*. - llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo()); + auto *ClassDecl = + cast(ElementType->getAs()->getDecl()); + llvm::Value *VTable = + CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl); // Track back to entry -2 and pull out the offset there. llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64( @@ -1211,8 +1214,10 @@ llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, Address ThisPtr, llvm::Type *StdTypeInfoPtrTy) { + auto *ClassDecl = + cast(SrcRecordTy->getAs()->getDecl()); llvm::Value *Value = - CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo()); + CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl); // Load the type info. Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL); @@ -1275,8 +1280,11 @@ llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, CGF.ConvertType(CGF.getContext().getPointerDiffType()); llvm::Type *DestLTy = CGF.ConvertType(DestTy); + auto *ClassDecl = + cast(SrcRecordTy->getAs()->getDecl()); // Get the vtable pointer. - llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo()); + llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo(), + ClassDecl); // Get the offset-to-top from the vtable. llvm::Value *OffsetToTop = @@ -1305,7 +1313,7 @@ ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) { - llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy); + llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl); CharUnits VBaseOffsetOffset = CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl); @@ -1591,10 +1599,11 @@ llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, SourceLocation Loc) { GD = GD.getCanonicalDecl(); Ty = Ty->getPointerTo()->getPointerTo(); - llvm::Value *VTable = CGF.GetVTablePtr(This, Ty); + auto *MethodDecl = cast(GD.getDecl()); + llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) - CGF.EmitVTablePtrCheckForCall(cast(GD.getDecl()), VTable, + CGF.EmitVTablePtrCheckForCall(MethodDecl, VTable, CodeGenFunction::CFITCK_VCall, Loc); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); diff --git a/lib/CodeGen/MicrosoftCXXABI.cpp b/lib/CodeGen/MicrosoftCXXABI.cpp index 87080b0cef..a2f7165a38 100644 --- a/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/lib/CodeGen/MicrosoftCXXABI.cpp @@ -1831,7 +1831,9 @@ llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, Ty = Ty->getPointerTo()->getPointerTo(); Address VPtr = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty); + + auto *MethodDecl = cast(GD.getDecl()); + llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent()); MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); @@ -1958,7 +1960,8 @@ llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk( // Load the vfptr and then callee from the vftable. The callee should have // adjusted 'this' so that the vfptr is at offset zero. llvm::Value *VTable = CGF.GetVTablePtr( - getThisAddress(CGF), ThunkTy->getPointerTo()->getPointerTo()); + getThisAddress(CGF), ThunkTy->getPointerTo()->getPointerTo(), MD->getParent()); + llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); llvm::Value *Callee = diff --git a/test/CodeGenCXX/invariant.group-for-vptrs.cpp b/test/CodeGenCXX/invariant.group-for-vptrs.cpp new file mode 100644 index 0000000000..8458aeb5b0 --- /dev/null +++ b/test/CodeGenCXX/invariant.group-for-vptrs.cpp @@ -0,0 +1,75 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm %s -fstrict-vtable-pointers -O1 -o - -disable-llvm-optzns | FileCheck %s + +struct A { + virtual void foo(); +}; + +struct D : A { + void foo(); +}; + +// CHECK-LABEL: define void @_Z21testExternallyVisiblev() +void testExternallyVisible() { + A *a = new A; + + // CHECK: load {{.*}} !invariant.group ![[A_MD:[0-9]+]] + a->foo(); + + D *d = new D; + // CHECK: call void @_ZN1DC1Ev( + // CHECK: load {{.*}} !invariant.group ![[D_MD:[0-9]+]] + d->foo(); + A *a2 = d; + // CHECK: load {{.*}} !invariant.group ![[A_MD]] + a2->foo(); +} +// CHECK-LABEL: } + +namespace { + +struct B { + virtual void bar(); +}; + +struct C : B { + void bar(); +}; + +} + +// CHECK-LABEL: define void @_Z21testInternallyVisibleb( +void testInternallyVisible(bool p) { + B *b = new B; + // CHECK: = load {{.*}}, !invariant.group ![[B_MD:[0-9]+]] + b->bar(); + + // CHECK: call void @_ZN12_GLOBAL__N_11CC1Ev( + C *c = new C; + // CHECK: = load {{.*}}, !invariant.group ![[C_MD:[0-9]+]] + c->bar(); +} + +// Checking A::A() +// CHECK-LABEL: define linkonce_odr void @_ZN1AC2Ev( +// CHECK: store {{.*}}, !invariant.group ![[A_MD]] +// CHECK-LABEL: } + +// Checking D::D() +// CHECK-LABEL: define linkonce_odr void @_ZN1DC2Ev( + +// CHECK: call void @_ZN1AC2Ev(%struct.A* +// CHECK: = call i8* @llvm.invariant.group.barrier(i8* +// CHECK: store {{.*}} !invariant.group ![[D_MD]] + +// Checking B::B() +// CHECK-LABEL: define internal void @_ZN12_GLOBAL__N_11BC2Ev( +// CHECK: store {{.*}}, !invariant.group ![[B_MD]] + +// Checking C::C() +// CHECK-LABEL: define internal void @_ZN12_GLOBAL__N_11CC2Ev( +// CHECK: store {{.*}}, !invariant.group ![[C_MD]] + +// CHECK: ![[A_MD]] = !{!"_ZTS1A"} +// CHECK: ![[D_MD]] = !{!"_ZTS1D"} +// CHECK: ![[B_MD]] = distinct !{} +// CHECK: ![[C_MD]] = distinct !{} -- 2.40.0