From: Endre Fulop Date: Mon, 8 Jul 2019 12:37:10 +0000 (+0000) Subject: [analyzer] Add analyzer option to limit the number of imported TUs X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=fbc4d7bcd574afb4b7cd6ad9c49058fce6f53955;p=clang [analyzer] Add analyzer option to limit the number of imported TUs Summary: During CTU analysis of complex projects, the loaded AST-contents of imported TUs can grow bigger than available system memory. This option introduces a threshold on the number of TUs to be imported for a single TU in order to prevent such cases. Differential Revision: https://reviews.llvm.org/D59798 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@365314 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/CrossTU/CrossTranslationUnit.h b/include/clang/CrossTU/CrossTranslationUnit.h index d1d38e09ef..d64329cdff 100644 --- a/include/clang/CrossTU/CrossTranslationUnit.h +++ b/include/clang/CrossTU/CrossTranslationUnit.h @@ -45,7 +45,8 @@ enum class index_error_code { failed_to_generate_usr, triple_mismatch, lang_mismatch, - lang_dialect_mismatch + lang_dialect_mismatch, + load_threshold_reached }; class IndexError : public llvm::ErrorInfo { @@ -134,7 +135,8 @@ public: /// A definition with the same declaration will be looked up in the /// index file which should be in the \p CrossTUDir directory, called /// \p IndexName. In case the declaration is found in the index the - /// corresponding AST file will be loaded. + /// corresponding AST file will be loaded. If the number of TUs imported + /// reaches \p CTULoadTreshold, no loading is performed. /// /// \return Returns a pointer to the ASTUnit that contains the definition of /// the looked up name or an Error. @@ -182,6 +184,10 @@ private: CompilerInstance &CI; ASTContext &Context; std::shared_ptr ImporterSharedSt; + /// \p CTULoadTreshold should serve as an upper limit to the number of TUs + /// imported in order to reduce the memory footprint of CTU analysis. + const unsigned CTULoadThreshold; + unsigned NumASTLoaded{0u}; }; } // namespace cross_tu diff --git a/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def b/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def index 8589610048..70bd476b6c 100644 --- a/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def +++ b/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def @@ -300,6 +300,14 @@ ANALYZER_OPTION(bool, ShouldTrackConditionsDebug, "track-conditions-debug", "Whether to place an event at each tracked condition.", false) +ANALYZER_OPTION(unsigned, CTUImportThreshold, "ctu-import-threshold", + "The maximal amount of translation units that is considered " + "for import when inlining functions during CTU analysis. " + "Lowering this threshold can alleviate the memory burder of " + "analysis with many interdependent definitions located in " + "various translation units.", + 100u) + //===----------------------------------------------------------------------===// // Unsinged analyzer options. //===----------------------------------------------------------------------===// diff --git a/lib/CrossTU/CrossTranslationUnit.cpp b/lib/CrossTU/CrossTranslationUnit.cpp index 166bdc0fb3..977fd4b8dd 100644 --- a/lib/CrossTU/CrossTranslationUnit.cpp +++ b/lib/CrossTU/CrossTranslationUnit.cpp @@ -47,6 +47,8 @@ STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter " STATISTIC(NumTripleMismatch, "The # of triple mismatches"); STATISTIC(NumLangMismatch, "The # of language mismatches"); STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches"); +STATISTIC(NumASTLoadThresholdReached, + "The # of ASTs not loaded because of threshold"); // Same as Triple's equality operator, but we check a field only if that is // known in both instances. @@ -106,6 +108,8 @@ public: return "Language mismatch"; case index_error_code::lang_dialect_mismatch: return "Language dialect mismatch"; + case index_error_code::load_threshold_reached: + return "Load threshold reached"; } llvm_unreachable("Unrecognized index_error_code."); } @@ -184,7 +188,8 @@ template static bool hasBodyOrInit(const T *D) { } CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI) - : CI(CI), Context(CI.getASTContext()) {} + : CI(CI), Context(CI.getASTContext()), + CTULoadThreshold(CI.getAnalyzerOpts()->CTUImportThreshold) {} CrossTranslationUnitContext::~CrossTranslationUnitContext() {} @@ -232,8 +237,8 @@ llvm::Expected CrossTranslationUnitContext::getCrossTUDefinitionImpl( if (LookupName.empty()) return llvm::make_error( index_error_code::failed_to_generate_usr); - llvm::Expected ASTUnitOrError = - loadExternalAST(LookupName, CrossTUDir, IndexName, DisplayCTUProgress); + llvm::Expected ASTUnitOrError = loadExternalAST( + LookupName, CrossTUDir, IndexName, DisplayCTUProgress); if (!ASTUnitOrError) return ASTUnitOrError.takeError(); ASTUnit *Unit = *ASTUnitOrError; @@ -342,6 +347,13 @@ llvm::Expected CrossTranslationUnitContext::loadExternalAST( // a lookup name from a single translation unit. If multiple // translation units contains decls with the same lookup name an // error will be returned. + + if (NumASTLoaded >= CTULoadThreshold) { + ++NumASTLoadThresholdReached; + return llvm::make_error( + index_error_code::load_threshold_reached); + } + ASTUnit *Unit = nullptr; auto NameUnitCacheEntry = NameASTUnitMap.find(LookupName); if (NameUnitCacheEntry == NameASTUnitMap.end()) { @@ -379,6 +391,7 @@ llvm::Expected CrossTranslationUnitContext::loadExternalAST( ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts())); Unit = LoadedUnit.get(); FileASTUnitMap[ASTFileName] = std::move(LoadedUnit); + ++NumASTLoaded; if (DisplayCTUProgress) { llvm::errs() << "CTU loaded AST file: " << ASTFileName << "\n"; diff --git a/test/Analysis/analyzer-config.c b/test/Analysis/analyzer-config.c index 9ceb8e9472..26fa5d2206 100644 --- a/test/Analysis/analyzer-config.c +++ b/test/Analysis/analyzer-config.c @@ -27,6 +27,7 @@ // CHECK-NEXT: cplusplus.Move:WarnOn = KnownsAndLocals // CHECK-NEXT: crosscheck-with-z3 = false // CHECK-NEXT: ctu-dir = "" +// CHECK-NEXT: ctu-import-threshold = 100 // CHECK-NEXT: ctu-index-name = externalDefMap.txt // CHECK-NEXT: debug.AnalysisOrder:* = false // CHECK-NEXT: debug.AnalysisOrder:Bind = false @@ -90,4 +91,4 @@ // CHECK-NEXT: unroll-loops = false // CHECK-NEXT: widen-loops = false // CHECK-NEXT: [stats] -// CHECK-NEXT: num-entries = 87 +// CHECK-NEXT: num-entries = 88 diff --git a/test/Analysis/ctu-import-threshold.c b/test/Analysis/ctu-import-threshold.c new file mode 100644 index 0000000000..82711b873d --- /dev/null +++ b/test/Analysis/ctu-import-threshold.c @@ -0,0 +1,5 @@ +// Ensure analyzer option 'ctu-import-threshold' is a recognized option. +// +// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-threshold=30 -verify %s +// +// expected-no-diagnostics diff --git a/unittests/CrossTU/CrossTranslationUnitTest.cpp b/unittests/CrossTU/CrossTranslationUnitTest.cpp index 43e0e75c31..b3e7243ce1 100644 --- a/unittests/CrossTU/CrossTranslationUnitTest.cpp +++ b/unittests/CrossTU/CrossTranslationUnitTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/CrossTU/CrossTranslationUnit.h" +#include "clang/Frontend/CompilerInstance.h" #include "clang/AST/ASTConsumer.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Tooling/Tooling.h" @@ -70,12 +71,14 @@ public: EXPECT_TRUE(llvm::sys::fs::exists(ASTFileName)); // Load the definition from the AST file. - llvm::Expected NewFDorError = - CTU.getCrossTUDefinition(FD, "", IndexFileName); - EXPECT_TRUE((bool)NewFDorError); - const FunctionDecl *NewFD = *NewFDorError; + llvm::Expected NewFDorError = handleExpected( + CTU.getCrossTUDefinition(FD, "", IndexFileName, false), + []() { return nullptr; }, [](IndexError &) {}); - *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody; + if (NewFDorError) { + const FunctionDecl *NewFD = *NewFDorError; + *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody; + } } private: @@ -85,26 +88,37 @@ private: class CTUAction : public clang::ASTFrontendAction { public: - CTUAction(bool *Success) : Success(Success) {} + CTUAction(bool *Success, unsigned OverrideLimit) + : Success(Success), OverrideLimit(OverrideLimit) {} protected: std::unique_ptr CreateASTConsumer(clang::CompilerInstance &CI, StringRef) override { + CI.getAnalyzerOpts()->CTUImportThreshold = OverrideLimit; return llvm::make_unique(CI, Success); } private: bool *Success; + const unsigned OverrideLimit; }; } // end namespace TEST(CrossTranslationUnit, CanLoadFunctionDefinition) { bool Success = false; - EXPECT_TRUE(tooling::runToolOnCode(new CTUAction(&Success), "int f(int);")); + EXPECT_TRUE( + tooling::runToolOnCode(new CTUAction(&Success, 1u), "int f(int);")); EXPECT_TRUE(Success); } +TEST(CrossTranslationUnit, RespectsLoadThreshold) { + bool Success = false; + EXPECT_TRUE( + tooling::runToolOnCode(new CTUAction(&Success, 0u), "int f(int);")); + EXPECT_FALSE(Success); +} + TEST(CrossTranslationUnit, IndexFormatCanBeParsed) { llvm::StringMap Index; Index["a"] = "/b/f1";