From 495cfa46300979642acde8d93a1f21c9291dac98 Mon Sep 17 00:00:00 2001 From: Nadav Rotem Date: Sat, 23 Mar 2013 06:43:35 +0000 Subject: [PATCH] Make clang to mark static stack allocations with lifetime markers to enable a more aggressive stack coloring. Patch by John McCall with help by Shuxin Yang. rdar://13115369 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@177819 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/CodeGen/CGDecl.cpp | 69 ++++++++++++++++++++++++++++++++- lib/CodeGen/CGStmt.cpp | 34 ++++++++++++++-- lib/CodeGen/CodeGenFunction.cpp | 2 +- lib/CodeGen/CodeGenFunction.h | 44 +++++++++++++++++++-- lib/CodeGen/CodeGenModule.cpp | 1 + lib/CodeGen/CodeGenModule.h | 9 +++++ test/CodeGen/lifetime2.c | 17 ++++++++ test/CodeGenObjC/arc-blocks.m | 20 +++++----- test/CodeGenObjC/arc.m | 5 ++- 9 files changed, 180 insertions(+), 21 deletions(-) create mode 100644 test/CodeGen/lifetime2.c diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp index a99fe29bda..c095ff96fc 100644 --- a/lib/CodeGen/CGDecl.cpp +++ b/lib/CodeGen/CGDecl.cpp @@ -452,6 +452,22 @@ namespace { CGF.EmitCall(FnInfo, CleanupFn, ReturnValueSlot(), Args); } }; + + /// A cleanup to call @llvm.lifetime.end. + class CallLifetimeEnd : public EHScopeStack::Cleanup { + llvm::Value *Addr; + llvm::Value *Size; + public: + CallLifetimeEnd(llvm::Value *addr, llvm::Value *size) + : Addr(addr), Size(size) {} + + void Emit(CodeGenFunction &CGF, Flags flags) { + llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy); + CGF.Builder.CreateCall2(CGF.CGM.getLLVMLifetimeEndFn(), + Size, castAddr) + ->setDoesNotThrow(); + } + }; } /// EmitAutoVarWithLifetime - Does the setup required for an automatic @@ -756,7 +772,6 @@ static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init, // If a global is all zeros, always use a memset. if (isa(Init)) return true; - // If a non-zero global is <= 32 bytes, always use a memcpy. If it is large, // do it if it will require 6 or fewer scalar stores. // TODO: Should budget depends on the size? Avoiding a large global warrants @@ -768,6 +783,20 @@ static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init, canEmitInitWithFewStoresAfterMemset(Init, StoreBudget); } +/// Should we use the LLVM lifetime intrinsics for the given local variable? +static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, const VarDecl &D, + unsigned Size) { + // For now, only in optimized builds. + if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) + return false; + + // Limit the size of marked objects to 32 bytes. We don't want to increase + // compile time by marking tiny objects. + unsigned SizeThreshold = 32; + + return Size > SizeThreshold; +} + /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a /// variable declaration with auto, register, or no storage class specifier. @@ -870,6 +899,20 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { getContext().toCharUnitsFromBits(Target.getPointerAlign(0))); Alloc->setAlignment(allocaAlignment.getQuantity()); DeclPtr = Alloc; + + // Emit a lifetime intrinsic if meaningful. There's no point + // in doing this if we don't have a valid insertion point (?). + uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy); + if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) { + llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size); + + emission.SizeForLifetimeMarkers = sizeV; + llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy); + Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr) + ->setDoesNotThrow(); + } else { + assert(!emission.useLifetimeMarkers()); + } } } else { // Targets that don't support recursion emit locals as globals. @@ -1215,6 +1258,14 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) { const VarDecl &D = *emission.Variable; + // Make sure we call @llvm.lifetime.end. This needs to happen + // *last*, so the cleanup needs to be pushed *first*. + if (emission.useLifetimeMarkers()) { + EHStack.pushCleanup(NormalCleanup, + emission.getAllocatedAddress(), + emission.getSizeForLifetimeMarkers()); + } + // Check the type for a cleanup. if (QualType::DestructionKind dtorKind = D.getType().isDestructedType()) emitAutoVarTypeCleanup(emission, dtorKind); @@ -1485,6 +1536,22 @@ void CodeGenFunction::pushRegularPartialArrayCleanup(llvm::Value *arrayBegin, elementType, destroyer); } +/// Lazily declare the @llvm.lifetime.start intrinsic. +llvm::Constant *CodeGenModule::getLLVMLifetimeStartFn() { + if (LifetimeStartFn) return LifetimeStartFn; + LifetimeStartFn = llvm::Intrinsic::getDeclaration(&getModule(), + llvm::Intrinsic::lifetime_start); + return LifetimeStartFn; +} + +/// Lazily declare the @llvm.lifetime.end intrinsic. +llvm::Constant *CodeGenModule::getLLVMLifetimeEndFn() { + if (LifetimeEndFn) return LifetimeEndFn; + LifetimeEndFn = llvm::Intrinsic::getDeclaration(&getModule(), + llvm::Intrinsic::lifetime_end); + return LifetimeEndFn; +} + namespace { /// A cleanup to perform a release of an object at the end of a /// function. This is used to balance out the incoming +1 of a diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp index 9e7ddfbdc1..06f74600b3 100644 --- a/lib/CodeGen/CGStmt.cpp +++ b/lib/CodeGen/CGStmt.cpp @@ -319,6 +319,12 @@ CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) { } void CodeGenFunction::EmitLabel(const LabelDecl *D) { + // Add this label to the current lexical scope if we're within any + // normal cleanups. Jumps "in" to this label --- when permitted by + // the language --- may need to be routed around such cleanups. + if (EHStack.hasNormalCleanups() && CurLexicalScope) + CurLexicalScope->addLabel(D); + JumpDest &Dest = LabelMap[D]; // If we didn't need a forward reference to this label, just go @@ -330,16 +336,36 @@ void CodeGenFunction::EmitLabel(const LabelDecl *D) { // it from the branch-fixups list. } else { assert(!Dest.getScopeDepth().isValid() && "already emitted label!"); - Dest = JumpDest(Dest.getBlock(), - EHStack.stable_begin(), - Dest.getDestIndex()); - + Dest.setScopeDepth(EHStack.stable_begin()); ResolveBranchFixups(Dest.getBlock()); } EmitBlock(Dest.getBlock()); } +/// Change the cleanup scope of the labels in this lexical scope to +/// match the scope of the enclosing context. +void CodeGenFunction::LexicalScope::rescopeLabels() { + assert(!Labels.empty()); + EHScopeStack::stable_iterator innermostScope + = CGF.EHStack.getInnermostNormalCleanup(); + + // Change the scope depth of all the labels. + for (SmallVectorImpl::const_iterator + i = Labels.begin(), e = Labels.end(); i != e; ++i) { + assert(CGF.LabelMap.count(*i)); + JumpDest &dest = CGF.LabelMap.find(*i)->second; + assert(dest.getScopeDepth().isValid()); + assert(innermostScope.encloses(dest.getScopeDepth())); + dest.setScopeDepth(innermostScope); + } + + // Reparent the labels if the new scope also has cleanups. + if (innermostScope != EHScopeStack::stable_end() && ParentScope) { + ParentScope->Labels.append(Labels.begin(), Labels.end()); + } +} + void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) { EmitLabel(S.getDecl()); diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp index bd244355f0..04de55d517 100644 --- a/lib/CodeGen/CodeGenFunction.cpp +++ b/lib/CodeGen/CodeGenFunction.cpp @@ -45,7 +45,7 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) IndirectBranch(0), SwitchInsn(0), CaseRangeBlock(0), UnreachableBlock(0), CXXABIThisDecl(0), CXXABIThisValue(0), CXXThisValue(0), CXXStructorImplicitParamDecl(0), CXXStructorImplicitParamValue(0), - OutermostConditional(0), TerminateLandingPad(0), + OutermostConditional(0), CurLexicalScope(0), TerminateLandingPad(0), TerminateHandler(0), TrapBB(0) { if (!suppressNewContext) CGM.getCXXABI().getMangleContext().startNewFunction(); diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h index 46848ae2d7..e40a731825 100644 --- a/lib/CodeGen/CodeGenFunction.h +++ b/lib/CodeGen/CodeGenFunction.h @@ -562,6 +562,11 @@ public: EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; } unsigned getDestIndex() const { return Index; } + // This should be used cautiously. + void setScopeDepth(EHScopeStack::stable_iterator depth) { + ScopeDepth = depth; + } + private: llvm::BasicBlock *Block; EHScopeStack::stable_iterator ScopeDepth; @@ -853,6 +858,8 @@ public: class LexicalScope: protected RunCleanupsScope { SourceRange Range; + SmallVector Labels; + LexicalScope *ParentScope; LexicalScope(const LexicalScope &) LLVM_DELETED_FUNCTION; void operator=(const LexicalScope &) LLVM_DELETED_FUNCTION; @@ -860,15 +867,23 @@ public: public: /// \brief Enter a new cleanup scope. explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range) - : RunCleanupsScope(CGF), Range(Range) { + : RunCleanupsScope(CGF), Range(Range), ParentScope(CGF.CurLexicalScope) { + CGF.CurLexicalScope = this; if (CGDebugInfo *DI = CGF.getDebugInfo()) DI->EmitLexicalBlockStart(CGF.Builder, Range.getBegin()); } + void addLabel(const LabelDecl *label) { + assert(PerformCleanup && "adding label to dead scope?"); + Labels.push_back(label); + } + /// \brief Exit this cleanup scope, emitting any accumulated /// cleanups. ~LexicalScope() { - if (PerformCleanup) endLexicalScope(); + // If we should perform a cleanup, force them now. Note that + // this ends the cleanup scope before rescoping any labels. + if (PerformCleanup) ForceCleanup(); } /// \brief Force the emission of cleanups now, instead of waiting @@ -880,9 +895,14 @@ public: private: void endLexicalScope() { + CGF.CurLexicalScope = ParentScope; if (CGDebugInfo *DI = CGF.getDebugInfo()) DI->EmitLexicalBlockEnd(CGF.Builder, Range.getEnd()); + if (!Labels.empty()) + rescopeLabels(); } + + void rescopeLabels(); }; @@ -1205,6 +1225,8 @@ private: /// temporary should be destroyed conditionally. ConditionalEvaluation *OutermostConditional; + /// The current lexical scope. + LexicalScope *CurLexicalScope; /// ByrefValueInfoMap - For each __block variable, contains a pair of the LLVM /// type as well as the field number that contains the actual data. @@ -2001,18 +2023,34 @@ public: /// initializer. bool IsConstantAggregate; + /// Non-null if we should use lifetime annotations. + llvm::Value *SizeForLifetimeMarkers; + struct Invalid {}; AutoVarEmission(Invalid) : Variable(0) {} AutoVarEmission(const VarDecl &variable) : Variable(&variable), Address(0), NRVOFlag(0), - IsByRef(false), IsConstantAggregate(false) {} + IsByRef(false), IsConstantAggregate(false), + SizeForLifetimeMarkers(0) {} bool wasEmittedAsGlobal() const { return Address == 0; } public: static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); } + bool useLifetimeMarkers() const { return SizeForLifetimeMarkers != 0; } + llvm::Value *getSizeForLifetimeMarkers() const { + assert(useLifetimeMarkers()); + return SizeForLifetimeMarkers; + } + + /// Returns the raw, allocated address, which is not necessarily + /// the address of the object itself. + llvm::Value *getAllocatedAddress() const { + return Address; + } + /// Returns the address of the object within this declaration. /// Note that this does not chase the forwarding pointer for /// __block decls. diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp index 173774452e..9f604e860e 100644 --- a/lib/CodeGen/CodeGenModule.cpp +++ b/lib/CodeGen/CodeGenModule.cpp @@ -86,6 +86,7 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO, NSConcreteGlobalBlock(0), NSConcreteStackBlock(0), BlockObjectAssign(0), BlockObjectDispose(0), BlockDescriptorType(0), GenericBlockLiteralType(0), + LifetimeStartFn(0), LifetimeEndFn(0), SanitizerBlacklist(CGO.SanitizerBlacklistFile), SanOpts(SanitizerBlacklist.isIn(M) ? SanitizerOptions::Disabled : LangOpts.Sanitize) { diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h index a5f0689f96..fcd48302ae 100644 --- a/lib/CodeGen/CodeGenModule.h +++ b/lib/CodeGen/CodeGenModule.h @@ -381,6 +381,12 @@ class CodeGenModule : public CodeGenTypeCache { int GlobalUniqueCount; } Block; + /// void @llvm.lifetime.start(i64 %size, i8* nocapture ) + llvm::Constant *LifetimeStartFn; + + /// void @llvm.lifetime.end(i64 %size, i8* nocapture ) + llvm::Constant *LifetimeEndFn; + GlobalDecl initializedGlobalDecl; llvm::BlackList SanitizerBlacklist; @@ -757,6 +763,9 @@ public: ///@} + llvm::Constant *getLLVMLifetimeStartFn(); + llvm::Constant *getLLVMLifetimeEndFn(); + // UpdateCompleteType - Make sure that this type is translated. void UpdateCompletedType(const TagDecl *TD); diff --git a/test/CodeGen/lifetime2.c b/test/CodeGen/lifetime2.c new file mode 100644 index 0000000000..ffff5cca12 --- /dev/null +++ b/test/CodeGen/lifetime2.c @@ -0,0 +1,17 @@ +// RUN: %clang -S -emit-llvm -o - -O2 %s | FileCheck %s -check-prefix=O2 +// RUN: %clang -S -emit-llvm -o - -O0 %s | FileCheck %s -check-prefix=O0 + +extern int bar(char *A, int n); + +// O0-NOT: @llvm.lifetime.start +int foo (int n) { + if (n) { +// O2: @llvm.lifetime.start + char A[100]; + return bar(A, 1); + } else { +// O2: @llvm.lifetime.start + char A[100]; + return bar(A, 2); + } +} diff --git a/test/CodeGenObjC/arc-blocks.m b/test/CodeGenObjC/arc-blocks.m index c7bc502ca0..3281b2aab8 100644 --- a/test/CodeGenObjC/arc-blocks.m +++ b/test/CodeGenObjC/arc-blocks.m @@ -128,7 +128,7 @@ void test4(void) { // CHECK-NEXT: call void @_Block_object_dispose(i8* [[T0]], i32 8) // CHECK-NEXT: [[T0:%.*]] = load i8** [[SLOT]] // CHECK-NEXT: call void @objc_release(i8* [[T0]]) - // CHECK-NEXT: ret void + // CHECK: ret void // CHECK: define internal void @__Block_byref_object_copy_ // CHECK: [[T0:%.*]] = getelementptr inbounds [[BYREF_T]]* {{%.*}}, i32 0, i32 6 @@ -207,7 +207,7 @@ void test6(void) { // CHECK: [[T0:%.*]] = bitcast [[BYREF_T]]* [[VAR]] to i8* // CHECK-NEXT: call void @_Block_object_dispose(i8* [[T0]], i32 8) // CHECK-NEXT: call void @objc_destroyWeak(i8** [[SLOT]]) - // CHECK-NEXT: ret void + // CHECK: ret void // CHECK: define internal void @__Block_byref_object_copy_ // CHECK: [[T0:%.*]] = getelementptr inbounds [[BYREF_T]]* {{%.*}}, i32 0, i32 6 @@ -256,14 +256,14 @@ void test7(void) { // CHECK: call void @test7_helper( // CHECK-NEXT: call void @objc_destroyWeak(i8** {{%.*}}) // CHECK-NEXT: call void @objc_destroyWeak(i8** [[VAR]]) - // CHECK-NEXT: ret void + // CHECK: ret void // CHECK: define internal void @__test7_block_invoke // CHECK: [[SLOT:%.*]] = getelementptr inbounds [[BLOCK_T]]* {{%.*}}, i32 0, i32 5 // CHECK-NEXT: [[T0:%.*]] = call i8* @objc_loadWeakRetained(i8** [[SLOT]]) // CHECK-NEXT: call void @test7_consume(i8* [[T0]]) // CHECK-NEXT: call void @objc_release(i8* [[T0]]) - // CHECK-NEXT: ret void + // CHECK: ret void // CHECK: define internal void @__copy_helper_block_ // CHECK: getelementptr @@ -296,7 +296,7 @@ void test7(void) { // CHECK-NEXT: [[T1:%.*]] = load [[TEST8]]** [[D0]] // CHECK-NEXT: [[T2:%.*]] = bitcast [[TEST8]]* [[T1]] to i8* // CHECK-NEXT: call void @objc_release(i8* [[T2]]) -// CHECK-NEXT: ret void +// CHECK: ret void extern void test8_helper(void (^)(void)); test8_helper(^{ (void) self; }); @@ -354,7 +354,7 @@ void test10a(void) { // CHECK-NEXT: [[T1:%.*]] = load void ()** [[SLOT]] // CHECK-NEXT: [[T2:%.*]] = bitcast void ()* [[T1]] to i8* // CHECK-NEXT: call void @objc_release(i8* [[T2]]) - // CHECK-NEXT: ret void + // CHECK: ret void } // : do this copy and dispose with @@ -374,7 +374,7 @@ void test10a(void) { // CHECK-NEXT: [[T2:%.*]] = call i8* @objc_retainBlock(i8* [[T1]]) // CHECK-NEXT: [[T3:%.*]] = bitcast i8* [[T2]] to void ()* // CHECK-NEXT: store void ()* [[T3]], void ()** [[D2]], align 8 -// CHECK-NEXT: ret void +// CHECK: ret void // CHECK: define internal void @__Block_byref_object_dispose // CHECK: [[T0:%.*]] = load i8** {{%.*}} @@ -418,7 +418,7 @@ void test10b(void) { // CHECK-NEXT: [[T1:%.*]] = load void ()** [[SLOT]] // CHECK-NEXT: [[T2:%.*]] = bitcast void ()* [[T1]] to i8* // CHECK-NEXT: call void @objc_release(i8* [[T2]]) - // CHECK-NEXT: ret void + // CHECK: ret void } // rdar://problem/10088932 @@ -438,7 +438,7 @@ void test11a(void) { // CHECK-NEXT: call void @test11_helper(i8* [[T4]]) // CHECK-NEXT: [[T5:%.*]] = bitcast void ()* [[T3]] to i8* // CHECK-NEXT: call void @objc_release(i8* [[T5]]) - // CHECK-NEXT: ret void + // CHECK: ret void } void test11b(void) { int x; @@ -456,7 +456,7 @@ void test11b(void) { // CHECK-NEXT: store i8* [[T4]], i8** [[B]], align 8 // CHECK-NEXT: [[T5:%.*]] = load i8** [[B]] // CHECK-NEXT: call void @objc_release(i8* [[T5]]) - // CHECK-NEXT: ret void + // CHECK: ret void } // rdar://problem/9979150 diff --git a/test/CodeGenObjC/arc.m b/test/CodeGenObjC/arc.m index 45cb5d761e..7262dc8d7b 100644 --- a/test/CodeGenObjC/arc.m +++ b/test/CodeGenObjC/arc.m @@ -353,7 +353,7 @@ void test12(void) { // CHECK-NEXT: [[T4:%.*]] = load i8** [[Y]] // CHECK-NEXT: call void @objc_release(i8* [[T4]]) [[NUW]], !clang.imprecise_release // CHECK-NEXT: call void @objc_destroyWeak(i8** [[X]]) - // CHECK-NEXT: ret void + // CHECK: ret void } // Indirect consuming calls. @@ -460,8 +460,9 @@ void test13(void) { void test19() { // CHECK: define void @test19() // CHECK: [[X:%.*]] = alloca [5 x i8*], align 16 + // CHECK: call void @llvm.lifetime.start // CHECK-NEXT: [[T0:%.*]] = bitcast [5 x i8*]* [[X]] to i8* - // CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* [[T0]], i8 0, i64 40, i32 16, i1 false) + // CHECK: call void @llvm.memset.p0i8.i64(i8* [[T0]], i8 0, i64 40, i32 16, i1 false) id x[5]; extern id test19_helper(void); -- 2.40.0