From 9414f34c578b4fa0a2755b727c65aa4d03c951a0 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Mon, 5 Oct 2015 13:55:14 +0000 Subject: [PATCH] [VFS] Add an in-memory file system implementation. This is a simple file system tree of memory buffers that can be filled by a client. In conjunction with an OverlayFS it can be used to make virtual files accessible right next to physical files. This can be used as a replacement for the virtual file handling in FileManager and which I intend to remove eventually. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@249315 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Basic/VirtualFileSystem.h | 21 ++ lib/Basic/VirtualFileSystem.cpp | 246 ++++++++++++++++++++++ unittests/Basic/VirtualFileSystemTest.cpp | 67 ++++++ unittests/Tooling/RewriterTestContext.h | 29 ++- 4 files changed, 352 insertions(+), 11 deletions(-) diff --git a/include/clang/Basic/VirtualFileSystem.h b/include/clang/Basic/VirtualFileSystem.h index c458c070b4..d263e15c91 100644 --- a/include/clang/Basic/VirtualFileSystem.h +++ b/include/clang/Basic/VirtualFileSystem.h @@ -241,6 +241,27 @@ public: iterator overlays_end() { return FSList.rend(); } }; +namespace detail { +class InMemoryDirectory; +} // end namespace detail + +/// An in-memory file system. +class InMemoryFileSystem : public FileSystem { + std::unique_ptr Root; + +public: + InMemoryFileSystem(); + ~InMemoryFileSystem() override; + void addFile(const Twine &Path, time_t ModificationTime, + std::unique_ptr Buffer); + StringRef toString() const; + + llvm::ErrorOr status(const Twine &Path) override; + llvm::ErrorOr> + openFileForRead(const Twine &Path) override; + directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override; +}; + /// \brief Get a globally unique ID for a virtual file or directory. llvm::sys::fs::UniqueID getNextVirtualUniqueID(); diff --git a/lib/Basic/VirtualFileSystem.cpp b/lib/Basic/VirtualFileSystem.cpp index 32e485f6c2..b258bc4252 100644 --- a/lib/Basic/VirtualFileSystem.cpp +++ b/lib/Basic/VirtualFileSystem.cpp @@ -320,6 +320,252 @@ directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir, std::make_shared(Dir, *this, EC)); } +namespace clang { +namespace vfs { +namespace detail { + +enum InMemoryNodeKind { IME_File, IME_Directory }; + +/// The in memory file system is a tree of Nodes. Every node can either be a +/// file or a directory. +class InMemoryNode { + Status Stat; + InMemoryNodeKind Kind; + +public: + InMemoryNode(Status Stat, InMemoryNodeKind Kind) + : Stat(std::move(Stat)), Kind(Kind) {} + virtual ~InMemoryNode() {} + const Status &getStatus() const { return Stat; } + InMemoryNodeKind getKind() const { return Kind; } + virtual std::string toString(unsigned Indent) const = 0; +}; + +namespace { +class InMemoryFile : public InMemoryNode { + std::unique_ptr Buffer; + +public: + InMemoryFile(Status Stat, std::unique_ptr Buffer) + : InMemoryNode(std::move(Stat), IME_File), Buffer(std::move(Buffer)) {} + + llvm::MemoryBuffer *getBuffer() { return Buffer.get(); } + std::string toString(unsigned Indent) const override { + return (std::string(Indent, ' ') + getStatus().getName() + "\n").str(); + } + static bool classof(const InMemoryNode *N) { + return N->getKind() == IME_File; + } +}; + +/// Adapt a InMemoryFile for VFS' File interface. +class InMemoryFileAdaptor : public File { + InMemoryFile &Node; + +public: + explicit InMemoryFileAdaptor(InMemoryFile &Node) : Node(Node) {} + + llvm::ErrorOr status() override { return Node.getStatus(); } + llvm::ErrorOr> + getBuffer(const Twine &Name, int64_t FileSize = -1, + bool RequiresNullTerminator = true, + bool IsVolatile = false) override { + llvm::MemoryBuffer *Buf = Node.getBuffer(); + return llvm::MemoryBuffer::getMemBuffer( + Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator); + } + std::error_code close() override { return std::error_code(); } +}; +} // end anonymous namespace + +class InMemoryDirectory : public InMemoryNode { + std::map> Entries; + +public: + InMemoryDirectory(Status Stat) + : InMemoryNode(std::move(Stat), IME_Directory) {} + InMemoryNode *getChild(StringRef Name) { + auto I = Entries.find(Name); + if (I != Entries.end()) + return I->second.get(); + return nullptr; + } + InMemoryNode *addChild(StringRef Name, std::unique_ptr Child) { + return Entries.insert(make_pair(Name, std::move(Child))) + .first->second.get(); + } + + typedef decltype(Entries)::const_iterator const_iterator; + const_iterator begin() const { return Entries.begin(); } + const_iterator end() const { return Entries.end(); } + + std::string toString(unsigned Indent) const override { + std::string Result = + (std::string(Indent, ' ') + getStatus().getName() + "\n").str(); + for (const auto &Entry : Entries) { + Result += Entry.second->toString(Indent + 2); + } + return Result; + } + static bool classof(const InMemoryNode *N) { + return N->getKind() == IME_Directory; + } +}; +} + +InMemoryFileSystem::InMemoryFileSystem() + : Root(new detail::InMemoryDirectory( + Status("", getNextVirtualUniqueID(), llvm::sys::TimeValue::MinTime(), + 0, 0, 0, llvm::sys::fs::file_type::directory_file, + llvm::sys::fs::perms::all_all))) {} + +InMemoryFileSystem::~InMemoryFileSystem() {} + +StringRef InMemoryFileSystem::toString() const { + return Root->toString(/*Indent=*/0); +} + +void InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime, + std::unique_ptr Buffer) { + SmallString<128> Path; + P.toVector(Path); + + // Fix up relative paths. This just prepends the current working directory. + std::error_code EC = sys::fs::make_absolute(Path); + assert(!EC); + (void)EC; + + detail::InMemoryDirectory *Dir = Root.get(); + auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path); + while (true) { + StringRef Name = *I; + detail::InMemoryNode *Node = Dir->getChild(Name); + ++I; + if (!Node) { + if (I == E) { + // End of the path, create a new file. + // FIXME: expose the status details in the interface. + Status Stat(Path, getNextVirtualUniqueID(), + llvm::sys::TimeValue(ModificationTime), 0, 0, + Buffer->getBufferSize(), + llvm::sys::fs::file_type::regular_file, + llvm::sys::fs::all_all); + Dir->addChild(Name, llvm::make_unique( + std::move(Stat), std::move(Buffer))); + return; + } + + // Create a new directory. Use the path up to here. + // FIXME: expose the status details in the interface. + Status Stat( + StringRef(Path.str().begin(), Name.end() - Path.str().begin()), + getNextVirtualUniqueID(), llvm::sys::TimeValue(ModificationTime), 0, + 0, Buffer->getBufferSize(), llvm::sys::fs::file_type::directory_file, + llvm::sys::fs::all_all); + Dir = cast(Dir->addChild( + Name, llvm::make_unique(std::move(Stat)))); + continue; + } + + if (auto *NewDir = dyn_cast(Node)) + Dir = NewDir; + } +} + +static ErrorOr +lookupInMemoryNode(detail::InMemoryDirectory *Dir, const Twine &P) { + SmallString<128> Path; + P.toVector(Path); + + // Fix up relative paths. This just prepends the current working directory. + std::error_code EC = sys::fs::make_absolute(Path); + assert(!EC); + (void)EC; + + auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path); + while (true) { + detail::InMemoryNode *Node = Dir->getChild(*I); + ++I; + if (!Node) + return errc::no_such_file_or_directory; + + // Return the file if it's at the end of the path. + if (auto File = dyn_cast(Node)) { + if (I == E) + return File; + return errc::no_such_file_or_directory; + } + + // Traverse directories. + Dir = cast(Node); + if (I == E) + return Dir; + } +} + +llvm::ErrorOr InMemoryFileSystem::status(const Twine &Path) { + auto Node = lookupInMemoryNode(Root.get(), Path); + if (Node) + return (*Node)->getStatus(); + return Node.getError(); +} + +llvm::ErrorOr> +InMemoryFileSystem::openFileForRead(const Twine &Path) { + auto Node = lookupInMemoryNode(Root.get(), Path); + if (!Node) + return Node.getError(); + + // When we have a file provide a heap-allocated wrapper for the memory buffer + // to match the ownership semantics for File. + if (auto *F = dyn_cast(*Node)) + return std::unique_ptr(new detail::InMemoryFileAdaptor(*F)); + + // FIXME: errc::not_a_file? + return make_error_code(llvm::errc::invalid_argument); +} + +namespace { +/// Adaptor from InMemoryDir::iterator to directory_iterator. +class InMemoryDirIterator : public clang::vfs::detail::DirIterImpl { + detail::InMemoryDirectory::const_iterator I; + detail::InMemoryDirectory::const_iterator E; + +public: + InMemoryDirIterator() {} + explicit InMemoryDirIterator(detail::InMemoryDirectory &Dir) + : I(Dir.begin()), E(Dir.end()) { + if (I != E) + CurrentEntry = I->second->getStatus(); + } + + std::error_code increment() override { + ++I; + // When we're at the end, make CurrentEntry invalid and DirIterImpl will do + // the rest. + CurrentEntry = I != E ? I->second->getStatus() : Status(); + return std::error_code(); + } +}; +} // end anonymous namespace + +directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir, + std::error_code &EC) { + auto Node = lookupInMemoryNode(Root.get(), Dir); + if (!Node) { + EC = Node.getError(); + return directory_iterator(std::make_shared()); + } + + if (auto *DirNode = dyn_cast(*Node)) + return directory_iterator(std::make_shared(*DirNode)); + + EC = make_error_code(llvm::errc::not_a_directory); + return directory_iterator(std::make_shared()); +} +} +} + //===-----------------------------------------------------------------------===/ // VFSFromYAML implementation //===-----------------------------------------------------------------------===/ diff --git a/unittests/Basic/VirtualFileSystemTest.cpp b/unittests/Basic/VirtualFileSystemTest.cpp index 13ae501235..4e5ec874ba 100644 --- a/unittests/Basic/VirtualFileSystemTest.cpp +++ b/unittests/Basic/VirtualFileSystemTest.cpp @@ -515,6 +515,73 @@ TEST(VirtualFileSystemTest, HiddenInIteration) { } } +class InMemoryFileSystemTest : public ::testing::Test { +protected: + clang::vfs::InMemoryFileSystem FS; +}; + +TEST_F(InMemoryFileSystemTest, IsEmpty) { + auto Stat = FS.status("/a"); + ASSERT_EQ(errc::no_such_file_or_directory, Stat.getError()) << FS.toString(); + Stat = FS.status("/"); + ASSERT_EQ(errc::no_such_file_or_directory, Stat.getError()) << FS.toString(); +} + +TEST_F(InMemoryFileSystemTest, WindowsPath) { + FS.addFile("c:/windows/system128/foo.cpp", 0, MemoryBuffer::getMemBuffer("")); + auto Stat = FS.status("c:"); + ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString(); + Stat = FS.status("c:/windows/system128/foo.cpp"); + ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString(); + FS.addFile("d:/windows/foo.cpp", 0, MemoryBuffer::getMemBuffer("")); + Stat = FS.status("d:/windows/foo.cpp"); + ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString(); +} + +TEST_F(InMemoryFileSystemTest, OverlayFile) { + FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a")); + auto Stat = FS.status("/"); + ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString(); + Stat = FS.status("/a"); + ASSERT_FALSE(Stat.getError()) << Stat.getError() << "\n" << FS.toString(); + ASSERT_EQ("/a", Stat->getName()); +} + +TEST_F(InMemoryFileSystemTest, OpenFileForRead) { + FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a")); + auto File = FS.openFileForRead("/a"); + ASSERT_EQ("a", (*(*File)->getBuffer("ignored"))->getBuffer()); + File = FS.openFileForRead("/a"); // Open again. + ASSERT_EQ("a", (*(*File)->getBuffer("ignored"))->getBuffer()); + File = FS.openFileForRead("/"); + ASSERT_EQ(errc::invalid_argument, File.getError()) << FS.toString(); + File = FS.openFileForRead("/b"); + ASSERT_EQ(errc::no_such_file_or_directory, File.getError()) << FS.toString(); +} + +TEST_F(InMemoryFileSystemTest, DirectoryIteration) { + FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("")); + FS.addFile("/b/c", 0, MemoryBuffer::getMemBuffer("")); + + std::error_code EC; + vfs::directory_iterator I = FS.dir_begin("/", EC); + ASSERT_FALSE(EC); + ASSERT_EQ("/a", I->getName()); + I.increment(EC); + ASSERT_FALSE(EC); + ASSERT_EQ("/b", I->getName()); + I.increment(EC); + ASSERT_FALSE(EC); + ASSERT_EQ(vfs::directory_iterator(), I); + + I = FS.dir_begin("/b", EC); + ASSERT_FALSE(EC); + ASSERT_EQ("/b/c", I->getName()); + I.increment(EC); + ASSERT_FALSE(EC); + ASSERT_EQ(vfs::directory_iterator(), I); +} + // NOTE: in the tests below, we use '//root/' as our root directory, since it is // a legal *absolute* path on Windows as well as *nix. class VFSFromYAMLTest : public ::testing::Test { diff --git a/unittests/Tooling/RewriterTestContext.h b/unittests/Tooling/RewriterTestContext.h index 112efac52e..eee7ea1873 100644 --- a/unittests/Tooling/RewriterTestContext.h +++ b/unittests/Tooling/RewriterTestContext.h @@ -34,15 +34,20 @@ namespace clang { /// methods, which help with writing tests that change files. class RewriterTestContext { public: - RewriterTestContext() - : DiagOpts(new DiagnosticOptions()), - Diagnostics(IntrusiveRefCntPtr(new DiagnosticIDs), - &*DiagOpts), - DiagnosticPrinter(llvm::outs(), &*DiagOpts), - Files((FileSystemOptions())), - Sources(Diagnostics, Files), - Rewrite(Sources, Options) { + RewriterTestContext() + : DiagOpts(new DiagnosticOptions()), + Diagnostics(IntrusiveRefCntPtr(new DiagnosticIDs), + &*DiagOpts), + DiagnosticPrinter(llvm::outs(), &*DiagOpts), + InMemoryFileSystem(new vfs::InMemoryFileSystem), + OverlayFileSystem( + new vfs::OverlayFileSystem(vfs::getRealFileSystem())), + Files(FileSystemOptions(), OverlayFileSystem), + Sources(Diagnostics, Files), Rewrite(Sources, Options) { Diagnostics.setClient(&DiagnosticPrinter, false); + // FIXME: To make these tests truly in-memory, we need to overlay the + // builtin headers. + OverlayFileSystem->pushOverlay(InMemoryFileSystem); } ~RewriterTestContext() {} @@ -50,9 +55,9 @@ class RewriterTestContext { FileID createInMemoryFile(StringRef Name, StringRef Content) { std::unique_ptr Source = llvm::MemoryBuffer::getMemBuffer(Content); - const FileEntry *Entry = - Files.getVirtualFile(Name, Source->getBufferSize(), 0); - Sources.overrideFileContents(Entry, std::move(Source)); + InMemoryFileSystem->addFile(Name, 0, std::move(Source)); + + const FileEntry *Entry = Files.getFile(Name); assert(Entry != nullptr); return Sources.createFileID(Entry, SourceLocation(), SrcMgr::C_User); } @@ -109,6 +114,8 @@ class RewriterTestContext { IntrusiveRefCntPtr DiagOpts; DiagnosticsEngine Diagnostics; TextDiagnosticPrinter DiagnosticPrinter; + IntrusiveRefCntPtr InMemoryFileSystem; + IntrusiveRefCntPtr OverlayFileSystem; FileManager Files; SourceManager Sources; LangOptions Options; -- 2.40.0