From 44c181aec37789f25f6c15543c164416f72e562a Mon Sep 17 00:00:00 2001 From: Douglas Gregor Date: Fri, 23 Jul 2010 00:33:23 +0000 Subject: [PATCH] Basic plumbing for generating a precompiled preamble for an ASTUnit/CXTranslationUnit. We can't actually use this preamble yet, however. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@109202 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang-c/Index.h | 44 +++- include/clang/Frontend/ASTUnit.h | 15 +- include/clang/Frontend/PreprocessorOptions.h | 39 +++- lib/Frontend/ASTUnit.cpp | 201 ++++++++++++++++++- lib/Frontend/InitPreprocessor.cpp | 8 +- tools/c-index-test/c-index-test.c | 21 +- tools/libclang/CIndex.cpp | 19 +- 7 files changed, 314 insertions(+), 33 deletions(-) diff --git a/include/clang-c/Index.h b/include/clang-c/Index.h index 467a9826eb..30feb41f33 100644 --- a/include/clang-c/Index.h +++ b/include/clang-c/Index.h @@ -634,13 +634,13 @@ CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnitFromSourceFile( CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnit(CXIndex, const char *ast_filename); - /** - * \brief Flags that control the creation of translation units. - * - * The enumerators in this enumeration type are meant to be bitwise - * ORed together to specify which options should be used when - * constructing the translation unit. - */ +/** + * \brief Flags that control the creation of translation units. + * + * The enumerators in this enumeration type are meant to be bitwise + * ORed together to specify which options should be used when + * constructing the translation unit. + */ enum CXTranslationUnit_Flags { /** * \brief Used to indicate that no special translation-unit options are @@ -658,7 +658,35 @@ enum CXTranslationUnit_Flags { * applications that require more detailed information about the * behavior of the preprocessor. */ - CXTranslationUnit_DetailedPreprocessingRecord = 0x01 + CXTranslationUnit_DetailedPreprocessingRecord = 0x01, + + /** + * \brief A flag that indicates that the intent of parsing the + * given translation unit is for live editing of the file. + * + * This flag is essentially a meta-flag that callers can use to indicate + * that the translation unit is being edited and, therefore, is likely to + * be reparsed many times. It enables an unspecified set of optimizations + * (e.g., the precompiled preamble) geared toward improving the performance + * of \c clang_reparseTranslationUnit(). + */ + CXTranslationUnit_Editing = 0x02, + + /** + * \brief Used to indicate that the translation unit should be built with an + * implicit precompiled header for the preamble. + * + * An implicit precompiled header is used as an optimization when a + * particular translation unit is likely to be reparsed many times + * when the sources aren't changing that often. In this case, an + * implicit precompiled header will be built containing all of the + * initial includes at the top of the main file (what we refer to as + * the "preamble" of the file). In subsequent parses, if the + * preamble or the files in it have not changed, \c + * clang_reparseTranslationUnit() will re-use the implicit + * precompiled header to improve parsing performance. + */ + CXTranslationUnit_PrecompiledPreamble = 0x04 }; /** diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h index b7a48881ef..950da64c22 100644 --- a/include/clang/Frontend/ASTUnit.h +++ b/include/clang/Frontend/ASTUnit.h @@ -46,6 +46,14 @@ class TargetInfo; using namespace idx; +class PrecompiledPreamble { + llvm::sys::Path PreambleFile; + +public: + ~PrecompiledPreamble(); + +}; + /// \brief Utility class for loading a ASTContext from a PCH file. /// class ASTUnit { @@ -125,6 +133,7 @@ private: void CleanTemporaryFiles(); bool Parse(); + void BuildPrecompiledPreamble(); public: class ConcurrencyCheck { @@ -241,7 +250,8 @@ public: static ASTUnit *LoadFromCompilerInvocation(CompilerInvocation *CI, llvm::IntrusiveRefCntPtr Diags, bool OnlyLocalDecls = false, - bool CaptureDiagnostics = false); + bool CaptureDiagnostics = false, + bool PrecompilePreamble = false); /// LoadFromCommandLine - Create an ASTUnit from a vector of command line /// arguments, which must specify exactly one source file. @@ -264,7 +274,8 @@ public: bool OnlyLocalDecls = false, RemappedFile *RemappedFiles = 0, unsigned NumRemappedFiles = 0, - bool CaptureDiagnostics = false); + bool CaptureDiagnostics = false, + bool PrecompilePreamble = false); /// \brief Reparse the source files using the same command-line options that /// were originally used to produce this translation unit. diff --git a/include/clang/Frontend/PreprocessorOptions.h b/include/clang/Frontend/PreprocessorOptions.h index e2c1ca25a2..05159335ad 100644 --- a/include/clang/Frontend/PreprocessorOptions.h +++ b/include/clang/Frontend/PreprocessorOptions.h @@ -62,21 +62,37 @@ public: std::vector > RemappedFileBuffers; - typedef std::vector >::const_iterator + typedef std::vector >::iterator remapped_file_iterator; - remapped_file_iterator remapped_file_begin() const { + typedef std::vector >::const_iterator + const_remapped_file_iterator; + remapped_file_iterator remapped_file_begin() { return RemappedFiles.begin(); } - remapped_file_iterator remapped_file_end() const { + const_remapped_file_iterator remapped_file_begin() const { + return RemappedFiles.begin(); + } + remapped_file_iterator remapped_file_end() { + return RemappedFiles.end(); + } + const_remapped_file_iterator remapped_file_end() const { return RemappedFiles.end(); } typedef std::vector >:: - const_iterator remapped_file_buffer_iterator; - remapped_file_buffer_iterator remapped_file_buffer_begin() const { + iterator remapped_file_buffer_iterator; + typedef std::vector >:: + const_iterator const_remapped_file_buffer_iterator; + remapped_file_buffer_iterator remapped_file_buffer_begin() { + return RemappedFileBuffers.begin(); + } + const_remapped_file_buffer_iterator remapped_file_buffer_begin() const { return RemappedFileBuffers.begin(); } - remapped_file_buffer_iterator remapped_file_buffer_end() const { + remapped_file_buffer_iterator remapped_file_buffer_end() { + return RemappedFileBuffers.end(); + } + const_remapped_file_buffer_iterator remapped_file_buffer_end() const { return RemappedFileBuffers.end(); } @@ -92,9 +108,20 @@ public: void addRemappedFile(llvm::StringRef From, llvm::StringRef To) { RemappedFiles.push_back(std::make_pair(From, To)); } + + remapped_file_iterator eraseRemappedFile(remapped_file_iterator Remapped) { + return RemappedFiles.erase(Remapped); + } + void addRemappedFile(llvm::StringRef From, const llvm::MemoryBuffer * To) { RemappedFileBuffers.push_back(std::make_pair(From, To)); } + + remapped_file_buffer_iterator + eraseRemappedFile(remapped_file_buffer_iterator Remapped) { + return RemappedFileBuffers.erase(Remapped); + } + void clearRemappedFiles() { RemappedFiles.clear(); RemappedFileBuffers.clear(); diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp index 4bfefd60d5..5b2af15c0d 100644 --- a/lib/Frontend/ASTUnit.cpp +++ b/lib/Frontend/ASTUnit.cpp @@ -33,8 +33,13 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/System/Host.h" #include "llvm/System/Path.h" +#include using namespace clang; +PrecompiledPreamble::~PrecompiledPreamble() { + PreambleFile.eraseFromDisk(); +} + ASTUnit::ASTUnit(bool _MainFileIsAST) : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), ConcurrencyCheckValue(CheckUnlocked) { } @@ -399,11 +404,195 @@ error: return true; } +/// \brief Simple function to retrieve a path for a preamble precompiled header. +static std::string GetPreamblePCHPath() { + // FIXME: This is lame; sys::Path should provide this function (in particular, + // it should know how to find the temporary files dir). + // FIXME: This is really lame. I copied this code from the Driver! + std::string Error; + const char *TmpDir = ::getenv("TMPDIR"); + if (!TmpDir) + TmpDir = ::getenv("TEMP"); + if (!TmpDir) + TmpDir = ::getenv("TMP"); + if (!TmpDir) + TmpDir = "/tmp"; + llvm::sys::Path P(TmpDir); + P.appendComponent("preamble"); + if (P.createTemporaryFileOnDisk()) + return std::string(); + + P.appendSuffix("pch"); + return P.str(); +} + +void ASTUnit::BuildPrecompiledPreamble() { + CompilerInvocation PreambleInvocation(*Invocation); + FrontendOptions &FrontendOpts = PreambleInvocation.getFrontendOpts(); + PreprocessorOptions &PreprocessorOpts + = PreambleInvocation.getPreprocessorOpts(); + + // Try to determine if the main file has been remapped, either from the + // command line (to another file) or directly through the compiler invocation + // (to a memory buffer). + llvm::MemoryBuffer *Buffer = 0; + llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].second); + if (const llvm::sys::FileStatus *MainFileStatus = MainFilePath.getFileStatus()) { + // Check whether there is a file-file remapping of the main file + for (PreprocessorOptions::remapped_file_iterator + M = PreprocessorOpts.remapped_file_begin(), + E = PreprocessorOpts.remapped_file_end(); + M != E; + ++M) { + llvm::sys::PathWithStatus MPath(M->first); + if (const llvm::sys::FileStatus *MStatus = MPath.getFileStatus()) { + if (MainFileStatus->uniqueID == MStatus->uniqueID) { + // We found a remapping. Try to load the resulting, remapped source. + if (Buffer) + delete Buffer; + Buffer = llvm::MemoryBuffer::getFile(M->second); + if (!Buffer) + return; + + // Remove the file-file remapping. + M = PreprocessorOpts.eraseRemappedFile(M); + E = PreprocessorOpts.remapped_file_end(); + } + } + } + + // Check whether there is a file-buffer remapping. It supercedes the + // file-file remapping. + for (PreprocessorOptions::remapped_file_buffer_iterator + M = PreprocessorOpts.remapped_file_buffer_begin(), + E = PreprocessorOpts.remapped_file_buffer_end(); + M != E; + ++M) { + llvm::sys::PathWithStatus MPath(M->first); + if (const llvm::sys::FileStatus *MStatus = MPath.getFileStatus()) { + if (MainFileStatus->uniqueID == MStatus->uniqueID) { + // We found a remapping. + if (Buffer) + delete Buffer; + Buffer = const_cast(M->second); + + // Remove the file-buffer remapping. + M = PreprocessorOpts.eraseRemappedFile(M); + E = PreprocessorOpts.remapped_file_buffer_end(); + } + } + } + } + + // If the main source file was not remapped, load it now. + if (!Buffer) { + Buffer = llvm::MemoryBuffer::getFile(FrontendOpts.Inputs[0].second); + if (!Buffer) + return; + } + + // Try to compute the preamble. + unsigned PreambleLength = Lexer::ComputePreamble(Buffer); + if (PreambleLength == 0) + return; + + // Create a new buffer that stores the preamble. The buffer also contains + // extra space for the original contents of the file (which will be present + // when we actually parse the file) along with more room in case the file + // grows. + unsigned PreambleBufferSize = Buffer->getBufferSize(); + if (PreambleBufferSize < 4096) + PreambleBufferSize = 8192; + else + PreambleBufferSize *= 2; + + llvm::MemoryBuffer *PreambleBuffer + = llvm::MemoryBuffer::getNewUninitMemBuffer(PreambleBufferSize, + FrontendOpts.Inputs[0].second); + memcpy(const_cast(PreambleBuffer->getBufferStart()), + Buffer->getBufferStart(), PreambleLength); + memset(const_cast(PreambleBuffer->getBufferStart()) + PreambleLength, + ' ', PreambleBufferSize - PreambleLength - 1); + const_cast(PreambleBuffer->getBufferEnd())[-1] = 0; + delete Buffer; + + // Remap the main source file to the preamble buffer. + PreprocessorOpts.addRemappedFile(MainFilePath.str(), PreambleBuffer); + + // Tell the compiler invocation to generate a temporary precompiled header. + FrontendOpts.ProgramAction = frontend::GeneratePCH; + // FIXME: Set ChainedPCH, once it is ready. + // FIXME: Generate the precompiled header into memory? + FrontendOpts.OutputFile = GetPreamblePCHPath(); + + // Create the compiler instance to use for building the precompiled preamble. + CompilerInstance Clang; + Clang.setInvocation(&PreambleInvocation); + OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second; + + // Set up diagnostics. + Clang.setDiagnostics(&getDiagnostics()); + Clang.setDiagnosticClient(getDiagnostics().getClient()); + + // Create the target instance. + Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(), + Clang.getTargetOpts())); + if (!Clang.hasTarget()) { + Clang.takeDiagnosticClient(); + return; + } + + // Inform the target of the language options. + // + // FIXME: We shouldn't need to do this, the target should be immutable once + // created. This complexity should be lifted elsewhere. + Clang.getTarget().setForcedLangOptions(Clang.getLangOpts()); + + assert(Clang.getFrontendOpts().Inputs.size() == 1 && + "Invocation must have exactly one source file!"); + assert(Clang.getFrontendOpts().Inputs[0].first != IK_AST && + "FIXME: AST inputs not yet supported here!"); + assert(Clang.getFrontendOpts().Inputs[0].first != IK_LLVM_IR && + "IR inputs not support here!"); + + // Clear out old caches and data. + StoredDiagnostics.clear(); + + // Capture any diagnostics that would otherwise be dropped. + CaptureDroppedDiagnostics Capture(CaptureDiagnostics, + Clang.getDiagnostics(), + StoredDiagnostics); + + // Create a file manager object to provide access to and cache the filesystem. + Clang.setFileManager(new FileManager); + + // Create the source manager. + Clang.setSourceManager(new SourceManager(getDiagnostics())); + + // FIXME: Eventually, we'll have to track top-level declarations here, too. + llvm::OwningPtr Act; + Act.reset(new GeneratePCHAction); + if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second, + Clang.getFrontendOpts().Inputs[0].first)) { + Clang.takeDiagnosticClient(); + Clang.takeInvocation(); + return; + } + + Act->Execute(); + Act->EndSourceFile(); + Clang.takeDiagnosticClient(); + Clang.takeInvocation(); + + // FIXME: Keep track of the actual preamble header we created! + fprintf(stderr, "Preamble PCH: %s\n", FrontendOpts.OutputFile.c_str()); +} ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI, llvm::IntrusiveRefCntPtr Diags, bool OnlyLocalDecls, - bool CaptureDiagnostics) { + bool CaptureDiagnostics, + bool PrecompilePreamble) { if (!Diags.getPtr()) { // No diagnostics engine was provided, so create our own diagnostics object // with the default options. @@ -419,6 +608,9 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI, AST->OnlyLocalDecls = OnlyLocalDecls; AST->Invocation.reset(CI); + if (PrecompilePreamble) + AST->BuildPrecompiledPreamble(); + if (!AST->Parse()) return AST.take(); @@ -432,7 +624,8 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin, bool OnlyLocalDecls, RemappedFile *RemappedFiles, unsigned NumRemappedFiles, - bool CaptureDiagnostics) { + bool CaptureDiagnostics, + bool PrecompilePreamble) { if (!Diags.getPtr()) { // No diagnostics engine was provided, so create our own diagnostics object // with the default options. @@ -480,7 +673,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin, CompilerInvocation::CreateFromArgs(*CI, const_cast(CCArgs.data()), const_cast(CCArgs.data()) + - CCArgs.size(), + CCArgs.size(), *Diags); // Override any files that need remapping @@ -493,7 +686,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin, CI->getFrontendOpts().DisableFree = true; return LoadFromCompilerInvocation(CI.take(), Diags, OnlyLocalDecls, - CaptureDiagnostics); + CaptureDiagnostics, PrecompilePreamble); } bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) { diff --git a/lib/Frontend/InitPreprocessor.cpp b/lib/Frontend/InitPreprocessor.cpp index 889b6e52a4..15d804f7b4 100644 --- a/lib/Frontend/InitPreprocessor.cpp +++ b/lib/Frontend/InitPreprocessor.cpp @@ -477,7 +477,7 @@ static void InitializeFileRemapping(Diagnostic &Diags, FileManager &FileMgr, const PreprocessorOptions &InitOpts) { // Remap files in the source manager (with buffers). - for (PreprocessorOptions::remapped_file_buffer_iterator + for (PreprocessorOptions::const_remapped_file_buffer_iterator Remap = InitOpts.remapped_file_buffer_begin(), RemapEnd = InitOpts.remapped_file_buffer_end(); Remap != RemapEnd; @@ -499,9 +499,9 @@ static void InitializeFileRemapping(Diagnostic &Diags, } // Remap files in the source manager (with other files). - for (PreprocessorOptions::remapped_file_iterator - Remap = InitOpts.remapped_file_begin(), - RemapEnd = InitOpts.remapped_file_end(); + for (PreprocessorOptions::const_remapped_file_iterator + Remap = InitOpts.remapped_file_begin(), + RemapEnd = InitOpts.remapped_file_end(); Remap != RemapEnd; ++Remap) { // Find the file that we're mapping to. diff --git a/tools/c-index-test/c-index-test.c b/tools/c-index-test/c-index-test.c index 9d56eec8eb..db8ce18c73 100644 --- a/tools/c-index-test/c-index-test.c +++ b/tools/c-index-test/c-index-test.c @@ -28,6 +28,16 @@ char *basename(const char* path) extern char *basename(const char *); #endif +/// \brief Return the default parsing options. +static unsigned getDefaultParsingOptions() { + unsigned options = CXTranslationUnit_DetailedPreprocessingRecord; + + if (getenv("CINDEXTEST_EDITING")) + options |= CXTranslationUnit_Editing; + + return options; +} + static void PrintExtent(FILE *out, unsigned begin_line, unsigned begin_column, unsigned end_line, unsigned end_column) { fprintf(out, "[%d:%d - %d:%d]", begin_line, begin_column, @@ -613,11 +623,12 @@ int perform_test_reparse_source(int argc, const char **argv, int trials, return -1; } - TU = clang_createTranslationUnitFromSourceFile(Idx, 0, - argc - num_unsaved_files, - argv + num_unsaved_files, - num_unsaved_files, - unsaved_files); + TU = clang_parseTranslationUnit(Idx, 0, + argv + num_unsaved_files, + argc - num_unsaved_files, + unsaved_files, + num_unsaved_files, + getDefaultParsingOptions()); if (!TU) { fprintf(stderr, "Unable to load translation unit!\n"); free_remapped_files(unsaved_files, num_unsaved_files); diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp index 51965bb8a1..c3095e7c91 100644 --- a/tools/libclang/CIndex.cpp +++ b/tools/libclang/CIndex.cpp @@ -1176,6 +1176,11 @@ CXTranslationUnit clang_parseTranslationUnit(CXIndex CIdx, CIndexer *CXXIdx = static_cast(CIdx); + // The "editing" option implies other options. + if (options & CXTranslationUnit_Editing) + options |= CXTranslationUnit_PrecompiledPreamble; + bool PrecompilePreamble = options & CXTranslationUnit_PrecompiledPreamble; + // Configure the diagnostics. DiagnosticOptions DiagOpts; llvm::IntrusiveRefCntPtr Diags; @@ -1210,10 +1215,12 @@ CXTranslationUnit clang_parseTranslationUnit(CXIndex CIdx, Args.insert(Args.end(), command_line_args, command_line_args + num_command_line_args); + // Do we need the detailed preprocessing record? if (options & CXTranslationUnit_DetailedPreprocessingRecord) { Args.push_back("-Xclang"); Args.push_back("-detailed-preprocessing-record"); } + unsigned NumErrors = Diags->getNumErrors(); #ifdef USE_CRASHTRACER @@ -1227,7 +1234,8 @@ CXTranslationUnit clang_parseTranslationUnit(CXIndex CIdx, CXXIdx->getOnlyLocalDecls(), RemappedFiles.data(), RemappedFiles.size(), - /*CaptureDiagnostics=*/true)); + /*CaptureDiagnostics=*/true, + PrecompilePreamble)); if (NumErrors != Diags->getNumErrors()) { // Make sure to check that 'Unit' is non-NULL. @@ -1317,9 +1325,12 @@ CXTranslationUnit clang_parseTranslationUnit(CXIndex CIdx, TemporaryFiles.push_back(DiagnosticsFile); argv.push_back("-fdiagnostics-binary"); - argv.push_back("-Xclang"); - argv.push_back("-detailed-preprocessing-record"); - + // Do we need the detailed preprocessing record? + if (options & CXTranslationUnit_DetailedPreprocessingRecord) { + argv.push_back("-Xclang"); + argv.push_back("-detailed-preprocessing-record"); + } + // Add the null terminator. argv.push_back(NULL); -- 2.40.0