From abc563f554951259bbe0315055cad92ee14d87e4 Mon Sep 17 00:00:00 2001 From: Douglas Gregor Date: Mon, 19 Jul 2010 21:46:24 +0000 Subject: [PATCH] Introduce a new libclang API, clang_reparseTranslationUnit(), which reparses an already-parsed translation unit. At the moment it's just a convenience function, but we hope to use it for performance optimizations. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@108756 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang-c/Index.h | 38 +++++ include/clang/Basic/Diagnostic.h | 4 + include/clang/Frontend/ASTUnit.h | 14 ++ include/clang/Frontend/PreprocessorOptions.h | 4 + lib/Basic/Diagnostic.cpp | 57 +++---- lib/Frontend/ASTUnit.cpp | 152 +++++++++++++------ test/Index/cindex-from-source.m | 3 + tools/c-index-test/c-index-test.c | 64 +++++++- tools/libclang/CIndex.cpp | 19 +++ tools/libclang/libclang.darwin.exports | 1 + tools/libclang/libclang.exports | 1 + 11 files changed, 280 insertions(+), 77 deletions(-) diff --git a/include/clang-c/Index.h b/include/clang-c/Index.h index b377b6d5fe..08178e61e6 100644 --- a/include/clang-c/Index.h +++ b/include/clang-c/Index.h @@ -639,6 +639,44 @@ CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnit(CXIndex, */ CINDEX_LINKAGE void clang_disposeTranslationUnit(CXTranslationUnit); +/** + * \brief Reparse the source files that produced this translation unit. + * + * This routine can be used to re-parse the source files that originally + * created the given translation unit, for example because those source files + * have changed (either on disk or as passed via \p unsaved_files). The + * source code will be reparsed with the same command-line options as it + * was originally parsed. + * + * Reparsing a translation unit invalidates all cursors and source locations + * that refer into that translation unit. This makes reparsing a translation + * unit semantically equivalent to destroying the translation unit and then + * creating a new translation unit with the same command-line arguments. + * However, it may be more efficient to reparse a translation + * unit using this routine. + * + * \param TU The translation unit whose contents will be re-parsed. The + * translation unit must originally have been built with + * \c clang_createTranslationUnitFromSourceFile(). + * + * \param num_unsaved_files The number of unsaved file entries in \p + * unsaved_files. + * + * \param unsaved_files The files that have not yet been saved to disk + * but may be required for parsing, including the contents of + * those files. The contents and name of these files (as specified by + * CXUnsavedFile) are copied when necessary, so the client only needs to + * guarantee their validity until the call to this function returns. + * + * \returns 0 if the sources could be reparsed. A non-zero value will be + * returned if reparsing was impossible, such that the translation unit is + * invalid. In such cases, the only valid call for \p TU is + * \c clang_disposeTranslationUnit(TU). + */ +CINDEX_LINKAGE int clang_reparseTranslationUnit(CXTranslationUnit TU, + unsigned num_unsaved_files, + struct CXUnsavedFile *unsaved_files); + /** * @} */ diff --git a/include/clang/Basic/Diagnostic.h b/include/clang/Basic/Diagnostic.h index 1fe0d8154c..b09fd814b9 100644 --- a/include/clang/Basic/Diagnostic.h +++ b/include/clang/Basic/Diagnostic.h @@ -404,6 +404,10 @@ public: ArgToStringCookie = Cookie; } + /// \brief Reset the state of the diagnostic object to its initial + /// configuration. + void Reset(); + //===--------------------------------------------------------------------===// // Diagnostic classification and reporting interfaces. // diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h index 9252358f42..b7a48881ef 100644 --- a/include/clang/Frontend/ASTUnit.h +++ b/include/clang/Frontend/ASTUnit.h @@ -70,6 +70,9 @@ private: // FIXME: This is temporary; eventually, CIndex will always do this. bool OnlyLocalDecls; + /// \brief Whether to capture any diagnostics produced. + bool CaptureDiagnostics; + /// Track whether the main file was loaded from an AST or not. bool MainFileIsAST; @@ -120,6 +123,9 @@ private: explicit ASTUnit(bool MainFileIsAST); + void CleanTemporaryFiles(); + bool Parse(); + public: class ConcurrencyCheck { volatile ASTUnit &Self; @@ -259,6 +265,14 @@ public: RemappedFile *RemappedFiles = 0, unsigned NumRemappedFiles = 0, bool CaptureDiagnostics = false); + + /// \brief Reparse the source files using the same command-line options that + /// were originally used to produce this translation unit. + /// + /// \returns True if a failure occurred that causes the ASTUnit not to + /// contain any translation-unit information, false otherwise. + bool Reparse(RemappedFile *RemappedFiles = 0, + unsigned NumRemappedFiles = 0); }; } // namespace clang diff --git a/include/clang/Frontend/PreprocessorOptions.h b/include/clang/Frontend/PreprocessorOptions.h index 891359b747..e2c1ca25a2 100644 --- a/include/clang/Frontend/PreprocessorOptions.h +++ b/include/clang/Frontend/PreprocessorOptions.h @@ -95,6 +95,10 @@ public: void addRemappedFile(llvm::StringRef From, const llvm::MemoryBuffer * To) { RemappedFileBuffers.push_back(std::make_pair(From, To)); } + void clearRemappedFiles() { + RemappedFiles.clear(); + RemappedFileBuffers.clear(); + } }; } // end namespace clang diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp index 641d87bb9a..68548da0cb 100644 --- a/lib/Basic/Diagnostic.cpp +++ b/lib/Basic/Diagnostic.cpp @@ -244,35 +244,10 @@ static void DummyArgToStringFn(Diagnostic::ArgumentKind AK, intptr_t QT, Diagnostic::Diagnostic(DiagnosticClient *client) : Client(client) { - AllExtensionsSilenced = 0; - IgnoreAllWarnings = false; - WarningsAsErrors = false; - ErrorsAsFatal = false; - SuppressSystemWarnings = false; - SuppressAllDiagnostics = false; - ShowOverloads = Ovl_All; - ExtBehavior = Ext_Ignore; - - ErrorOccurred = false; - FatalErrorOccurred = false; - ErrorLimit = 0; - TemplateBacktraceLimit = 0; - - NumWarnings = 0; - NumErrors = 0; - NumErrorsSuppressed = 0; - CustomDiagInfo = 0; - CurDiagID = ~0U; - LastDiagLevel = Ignored; - ArgToStringFn = DummyArgToStringFn; ArgToStringCookie = 0; - DelayedDiagID = 0; - - // Set all mappings to 'unset'. - DiagMappings BlankDiags(diag::DIAG_UPPER_LIMIT/2, 0); - DiagMappingsStack.push_back(BlankDiags); + Reset(); } Diagnostic::~Diagnostic() { @@ -335,6 +310,36 @@ bool Diagnostic::isBuiltinExtensionDiag(unsigned DiagID, return true; } +void Diagnostic::Reset() { + AllExtensionsSilenced = 0; + IgnoreAllWarnings = false; + WarningsAsErrors = false; + ErrorsAsFatal = false; + SuppressSystemWarnings = false; + SuppressAllDiagnostics = false; + ShowOverloads = Ovl_All; + ExtBehavior = Ext_Ignore; + + ErrorOccurred = false; + FatalErrorOccurred = false; + ErrorLimit = 0; + TemplateBacktraceLimit = 0; + + NumWarnings = 0; + NumErrors = 0; + NumErrorsSuppressed = 0; + CustomDiagInfo = 0; + CurDiagID = ~0U; + LastDiagLevel = Ignored; + DelayedDiagID = 0; + + // Set all mappings to 'unset'. + while (!DiagMappingsStack.empty()) + DiagMappingsStack.pop_back(); + + DiagMappings BlankDiags(diag::DIAG_UPPER_LIMIT/2, 0); + DiagMappingsStack.push_back(BlankDiags); +} /// getDescription - Given a diagnostic ID, return a description of the /// issue. diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp index d8e1c51529..4bfefd60d5 100644 --- a/lib/Frontend/ASTUnit.cpp +++ b/lib/Frontend/ASTUnit.cpp @@ -36,12 +36,18 @@ using namespace clang; ASTUnit::ASTUnit(bool _MainFileIsAST) - : MainFileIsAST(_MainFileIsAST), ConcurrencyCheckValue(CheckUnlocked) { } + : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), + ConcurrencyCheckValue(CheckUnlocked) { } ASTUnit::~ASTUnit() { ConcurrencyCheckValue = CheckLocked; + CleanTemporaryFiles(); +} + +void ASTUnit::CleanTemporaryFiles() { for (unsigned I = 0, N = TemporaryFiles.size(); I != N; ++I) TemporaryFiles[I].eraseFromDisk(); + TemporaryFiles.clear(); } namespace { @@ -156,7 +162,8 @@ ASTUnit *ASTUnit::LoadFromPCHFile(const std::string &Filename, DiagnosticOptions DiagOpts; Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0); } - + + AST->CaptureDiagnostics = CaptureDiagnostics; AST->OnlyLocalDecls = OnlyLocalDecls; AST->Diagnostics = Diags; AST->FileMgr.reset(new FileManager); @@ -298,41 +305,38 @@ public: } -ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI, - llvm::IntrusiveRefCntPtr Diags, - bool OnlyLocalDecls, - bool CaptureDiagnostics) { +/// Parse the source file into a translation unit using the given compiler +/// invocation, replacing the current translation unit. +/// +/// \returns True if a failure occurred that causes the ASTUnit not to +/// contain any translation-unit information, false otherwise. +bool ASTUnit::Parse() { + if (!Invocation.get()) + return true; + // Create the compiler instance to use for building the AST. CompilerInstance Clang; - llvm::OwningPtr AST; - llvm::OwningPtr Act; - - if (!Diags.getPtr()) { - // No diagnostics engine was provided, so create our own diagnostics object - // with the default options. - DiagnosticOptions DiagOpts; - Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0); - } + Clang.setInvocation(Invocation.take()); + OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second; + + // Set up diagnostics. + Clang.setDiagnostics(&getDiagnostics()); + Clang.setDiagnosticClient(getDiagnostics().getClient()); - Clang.setInvocation(CI); - - Clang.setDiagnostics(Diags.getPtr()); - Clang.setDiagnosticClient(Diags->getClient()); - // Create the target instance. Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(), Clang.getTargetOpts())); if (!Clang.hasTarget()) { Clang.takeDiagnosticClient(); - return 0; + return true; } - + // Inform the target of the language options. // // FIXME: We shouldn't need to do this, the target should be immutable once // created. This complexity should be lifted elsewhere. Clang.getTarget().setForcedLangOptions(Clang.getLangOpts()); - + assert(Clang.getFrontendOpts().Inputs.size() == 1 && "Invocation must have exactly one source file!"); assert(Clang.getFrontendOpts().Inputs[0].first != IK_AST && @@ -340,52 +344,84 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI, assert(Clang.getFrontendOpts().Inputs[0].first != IK_LLVM_IR && "IR inputs not support here!"); - // Create the AST unit. - AST.reset(new ASTUnit(false)); - AST->Diagnostics = Diags; - AST->FileMgr.reset(new FileManager); - AST->SourceMgr.reset(new SourceManager(AST->getDiagnostics())); - AST->OnlyLocalDecls = OnlyLocalDecls; - AST->OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second; - + // Configure the various subsystems. + // FIXME: Should we retain the previous file manager? + FileMgr.reset(new FileManager); + SourceMgr.reset(new SourceManager(getDiagnostics())); + Ctx.reset(); + PP.reset(); + + // Clear out old caches and data. + TopLevelDecls.clear(); + StoredDiagnostics.clear(); + CleanTemporaryFiles(); + PreprocessedEntitiesByFile.clear(); + // Capture any diagnostics that would otherwise be dropped. CaptureDroppedDiagnostics Capture(CaptureDiagnostics, Clang.getDiagnostics(), - AST->StoredDiagnostics); - + StoredDiagnostics); + // Create a file manager object to provide access to and cache the filesystem. - Clang.setFileManager(&AST->getFileManager()); - + Clang.setFileManager(&getFileManager()); + // Create the source manager. - Clang.setSourceManager(&AST->getSourceManager()); - - Act.reset(new TopLevelDeclTrackerAction(*AST)); + Clang.setSourceManager(&getSourceManager()); + + llvm::OwningPtr Act; + Act.reset(new TopLevelDeclTrackerAction(*this)); if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second, Clang.getFrontendOpts().Inputs[0].first)) goto error; - + Act->Execute(); - + // Steal the created target, context, and preprocessor, and take back the // source and file managers. - AST->Ctx.reset(Clang.takeASTContext()); - AST->PP.reset(Clang.takePreprocessor()); + Ctx.reset(Clang.takeASTContext()); + PP.reset(Clang.takePreprocessor()); Clang.takeSourceManager(); Clang.takeFileManager(); - AST->Target.reset(Clang.takeTarget()); - + Target.reset(Clang.takeTarget()); + Act->EndSourceFile(); - + Clang.takeDiagnosticClient(); - Clang.takeInvocation(); - - AST->Invocation.reset(Clang.takeInvocation()); - return AST.take(); - + + Invocation.reset(Clang.takeInvocation()); + return false; + error: Clang.takeSourceManager(); Clang.takeFileManager(); Clang.takeDiagnosticClient(); + Invocation.reset(Clang.takeInvocation()); + return true; +} + + +ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI, + llvm::IntrusiveRefCntPtr Diags, + bool OnlyLocalDecls, + bool CaptureDiagnostics) { + if (!Diags.getPtr()) { + // No diagnostics engine was provided, so create our own diagnostics object + // with the default options. + DiagnosticOptions DiagOpts; + Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0); + } + + // Create the AST unit. + llvm::OwningPtr AST; + AST.reset(new ASTUnit(false)); + AST->Diagnostics = Diags; + AST->CaptureDiagnostics = CaptureDiagnostics; + AST->OnlyLocalDecls = OnlyLocalDecls; + AST->Invocation.reset(CI); + + if (!AST->Parse()) + return AST.take(); + return 0; } @@ -459,3 +495,19 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin, return LoadFromCompilerInvocation(CI.take(), Diags, OnlyLocalDecls, CaptureDiagnostics); } + +bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) { + if (!Invocation.get()) + return true; + + // Clear out the diagnostics state. + getDiagnostics().Reset(); + + // Remap files. + Invocation->getPreprocessorOpts().clearRemappedFiles(); + for (unsigned I = 0; I != NumRemappedFiles; ++I) + Invocation->getPreprocessorOpts().addRemappedFile(RemappedFiles[I].first, + RemappedFiles[I].second); + + return Parse(); +} diff --git a/test/Index/cindex-from-source.m b/test/Index/cindex-from-source.m index 86e794db89..f226e45335 100644 --- a/test/Index/cindex-from-source.m +++ b/test/Index/cindex-from-source.m @@ -7,3 +7,6 @@ // CHECK: cindex-from-source.m:9:1: TypeRef=t0:1:13 Extent=[9:1 - 9:3] struct s0 {}; t0 g0; + +// RUN: c-index-test -test-load-source-reparse 5 local %s -include %t.pfx.h > %t +// RUN: FileCheck %s < %t diff --git a/tools/c-index-test/c-index-test.c b/tools/c-index-test/c-index-test.c index 4ed24b15c9..569ef206e3 100644 --- a/tools/c-index-test/c-index-test.c +++ b/tools/c-index-test/c-index-test.c @@ -558,7 +558,7 @@ int perform_test_load_source(int argc, const char **argv, struct CXUnsavedFile *unsaved_files = 0; int num_unsaved_files = 0; int result; - + Idx = clang_createIndex(/* excludeDeclsFromPCH */ !strcmp(filter, "local") ? 1 : 0, /* displayDiagnosics=*/1); @@ -578,6 +578,7 @@ int perform_test_load_source(int argc, const char **argv, unsaved_files); if (!TU) { fprintf(stderr, "Unable to load translation unit!\n"); + free_remapped_files(unsaved_files, num_unsaved_files); clang_disposeIndex(Idx); return 1; } @@ -588,6 +589,57 @@ int perform_test_load_source(int argc, const char **argv, return result; } +int perform_test_reparse_source(int argc, const char **argv, int trials, + const char *filter, CXCursorVisitor Visitor, + PostVisitTU PV) { + const char *UseExternalASTs = + getenv("CINDEXTEST_USE_EXTERNAL_AST_GENERATION"); + CXIndex Idx; + CXTranslationUnit TU; + struct CXUnsavedFile *unsaved_files = 0; + int num_unsaved_files = 0; + int result; + int trial; + + Idx = clang_createIndex(/* excludeDeclsFromPCH */ + !strcmp(filter, "local") ? 1 : 0, + /* displayDiagnosics=*/1); + + if (UseExternalASTs && strlen(UseExternalASTs)) + clang_setUseExternalASTGeneration(Idx, 1); + + if (parse_remapped_files(argc, argv, 0, &unsaved_files, &num_unsaved_files)) { + clang_disposeIndex(Idx); + return -1; + } + + TU = clang_createTranslationUnitFromSourceFile(Idx, 0, + argc - num_unsaved_files, + argv + num_unsaved_files, + num_unsaved_files, + unsaved_files); + if (!TU) { + fprintf(stderr, "Unable to load translation unit!\n"); + free_remapped_files(unsaved_files, num_unsaved_files); + clang_disposeIndex(Idx); + return 1; + } + + for (trial = 0; trial < trials; ++trial) { + if (clang_reparseTranslationUnit(TU, num_unsaved_files, unsaved_files)) { + clang_disposeTranslationUnit(TU); + free_remapped_files(unsaved_files, num_unsaved_files); + clang_disposeIndex(Idx); + return -1; + } + } + + result = perform_test_load(Idx, TU, filter, NULL, Visitor, PV); + free_remapped_files(unsaved_files, num_unsaved_files); + clang_disposeIndex(Idx); + return result; +} + /******************************************************************************/ /* Logic for testing clang_getCursor(). */ /******************************************************************************/ @@ -1219,6 +1271,8 @@ static void print_usage(void) { "[FileCheck prefix]\n" " c-index-test -test-load-source {}*\n"); fprintf(stderr, + " c-index-test -test-load-source-reparse " + " {}*\n" " c-index-test -test-load-source-usrs {}*\n" " c-index-test -test-annotate-tokens= {}*\n" " c-index-test -test-inclusion-stack-source {}*\n" @@ -1252,6 +1306,14 @@ int main(int argc, const char **argv) { return perform_test_load_tu(argv[2], argv[3], argc >= 5 ? argv[4] : 0, I, NULL); } + else if (argc >= 5 && strncmp(argv[1], "-test-load-source-reparse", 25) == 0){ + CXCursorVisitor I = GetVisitor(argv[1] + 25); + if (I) { + int trials = atoi(argv[2]); + return perform_test_reparse_source(argc - 4, argv + 4, trials, argv[3], I, + NULL); + } + } else if (argc >= 4 && strncmp(argv[1], "-test-load-source", 17) == 0) { CXCursorVisitor I = GetVisitor(argv[1] + 17); if (I) diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp index 7f32a1c148..efc61a0645 100644 --- a/tools/libclang/CIndex.cpp +++ b/tools/libclang/CIndex.cpp @@ -1390,6 +1390,25 @@ void clang_disposeTranslationUnit(CXTranslationUnit CTUnit) { delete static_cast(CTUnit); } +int clang_reparseTranslationUnit(CXTranslationUnit TU, + unsigned num_unsaved_files, + struct CXUnsavedFile *unsaved_files) { + if (!TU) + return 1; + + llvm::SmallVector RemappedFiles; + for (unsigned I = 0; I != num_unsaved_files; ++I) { + llvm::StringRef Data(unsaved_files[I].Contents, unsaved_files[I].Length); + const llvm::MemoryBuffer *Buffer + = llvm::MemoryBuffer::getMemBufferCopy(Data, unsaved_files[I].Filename); + RemappedFiles.push_back(std::make_pair(unsaved_files[I].Filename, + Buffer)); + } + + return static_cast(TU)->Reparse(RemappedFiles.data(), + RemappedFiles.size())? 1 : 0; +} + CXString clang_getTranslationUnitSpelling(CXTranslationUnit CTUnit) { if (!CTUnit) return createCXString(""); diff --git a/tools/libclang/libclang.darwin.exports b/tools/libclang/libclang.darwin.exports index f21fec63f4..e09a6a00af 100644 --- a/tools/libclang/libclang.darwin.exports +++ b/tools/libclang/libclang.darwin.exports @@ -86,6 +86,7 @@ _clang_isReference _clang_isStatement _clang_isTranslationUnit _clang_isUnexposed +_clang_reparseTranslationUnit _clang_setUseExternalASTGeneration _clang_tokenize _clang_visitChildren diff --git a/tools/libclang/libclang.exports b/tools/libclang/libclang.exports index dcb40d413c..4b6ddd1f5c 100644 --- a/tools/libclang/libclang.exports +++ b/tools/libclang/libclang.exports @@ -86,6 +86,7 @@ clang_isReference clang_isStatement clang_isTranslationUnit clang_isUnexposed +clang_reparseTranslationUnit clang_setUseExternalASTGeneration clang_tokenize clang_visitChildren -- 2.40.0