From de63fff9046d6771c5093d36f2e83ec4e9721153 Mon Sep 17 00:00:00 2001 From: Jan Korous Date: Fri, 12 Jul 2019 19:54:36 +0000 Subject: [PATCH] Revert "Reland [clang] DirectoryWatcher" This reverts commit fdcb7f47e783933e0af8a5fae91132269a208268. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@365948 91177308-0d34-0410-b5e6-96231b3b80d8 --- .../clang/DirectoryWatcher/DirectoryWatcher.h | 123 ----- lib/CMakeLists.txt | 1 - lib/DirectoryWatcher/CMakeLists.txt | 29 -- lib/DirectoryWatcher/DirectoryScanner.cpp | 54 --- lib/DirectoryWatcher/DirectoryScanner.h | 29 -- .../DirectoryWatcher-not-implemented.cpp | 19 - .../linux/DirectoryWatcher-linux.cpp | 345 -------------- .../mac/DirectoryWatcher-mac.cpp | 233 ---------- unittests/CMakeLists.txt | 1 - unittests/DirectoryWatcher/CMakeLists.txt | 17 - .../DirectoryWatcher/DirectoryWatcherTest.cpp | 426 ------------------ 11 files changed, 1277 deletions(-) delete mode 100644 include/clang/DirectoryWatcher/DirectoryWatcher.h delete mode 100644 lib/DirectoryWatcher/CMakeLists.txt delete mode 100644 lib/DirectoryWatcher/DirectoryScanner.cpp delete mode 100644 lib/DirectoryWatcher/DirectoryScanner.h delete mode 100644 lib/DirectoryWatcher/default/DirectoryWatcher-not-implemented.cpp delete mode 100644 lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp delete mode 100644 lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp delete mode 100644 unittests/DirectoryWatcher/CMakeLists.txt delete mode 100644 unittests/DirectoryWatcher/DirectoryWatcherTest.cpp diff --git a/include/clang/DirectoryWatcher/DirectoryWatcher.h b/include/clang/DirectoryWatcher/DirectoryWatcher.h deleted file mode 100644 index 0bf966bb83..0000000000 --- a/include/clang/DirectoryWatcher/DirectoryWatcher.h +++ /dev/null @@ -1,123 +0,0 @@ -//===- DirectoryWatcher.h - Listens for directory file changes --*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H -#define LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringRef.h" -#include -#include -#include - -namespace clang { -/// Provides notifications for file changes in a directory. -/// -/// Invokes client-provided function on every filesystem event in the watched -/// directory. Initially the the watched directory is scanned and for every file -/// found, an event is synthesized as if the file was added. -/// -/// This is not a general purpose directory monitoring tool - list of -/// limitations follows. -/// -/// Only flat directories with no subdirectories are supported. In case -/// subdirectories are present the behavior is unspecified - events *might* be -/// passed to Receiver on macOS (due to FSEvents being used) while they -/// *probably* won't be passed on Linux (due to inotify being used). -/// -/// Known potential inconsistencies -/// - For files that are deleted befor the initial scan processed them, clients -/// might receive Removed notification without any prior Added notification. -/// - Multiple notifications might be produced when a file is added to the -/// watched directory during the initial scan. We are choosing the lesser evil -/// here as the only known alternative strategy would be to invalidate the -/// watcher instance and force user to create a new one whenever filesystem -/// event occurs during the initial scan but that would introduce continuous -/// restarting failure mode (watched directory is not always "owned" by the same -/// process that is consuming it). Since existing clients can handle duplicate -/// events well, we decided for simplicity. -/// -/// Notifications are provided only for changes done through local user-space -/// filesystem interface. Specifically, it's unspecified if notification would -/// be provided in case of a: -/// - a file mmap-ed and changed -/// - a file changed via remote (NFS) or virtual (/proc) FS access to monitored -/// directory -/// - another filesystem mounted to the watched directory -/// -/// No support for LLVM VFS. -/// -/// It is unspecified whether notifications for files being deleted are sent in -/// case the whole watched directory is sent. -/// -/// Directories containing "too many" files and/or receiving events "too -/// frequently" are not supported - if the initial scan can't be finished before -/// the watcher instance gets invalidated (see WatcherGotInvalidated) there's no -/// good error handling strategy - the only option for client is to destroy the -/// watcher, restart watching with new instance and hope it won't repeat. -class DirectoryWatcher { -public: - struct Event { - enum class EventKind { - Removed, - /// Content of a file was modified. - Modified, - /// The watched directory got deleted. - WatchedDirRemoved, - /// The DirectoryWatcher that originated this event is no longer valid and - /// its behavior is unspecified. - /// - /// The prime case is kernel signalling to OS-specific implementation of - /// DirectoryWatcher some resource limit being hit. - /// *Usually* kernel starts dropping or squashing events together after - /// that and so would DirectoryWatcher. This means that *some* events - /// might still be passed to Receiver but this behavior is unspecified. - /// - /// Another case is after the watched directory itself is deleted. - /// WatcherGotInvalidated will be received at least once during - /// DirectoryWatcher instance lifetime - when handling errors this is done - /// on best effort basis, when an instance is being destroyed then this is - /// guaranteed. - /// - /// The only proper response to this kind of event is to destruct the - /// originating DirectoryWatcher instance and create a new one. - WatcherGotInvalidated - }; - - EventKind Kind; - /// Filename that this event is related to or an empty string in - /// case this event is related to the watched directory itself. - std::string Filename; - - Event(EventKind Kind, llvm::StringRef Filename) - : Kind(Kind), Filename(Filename) {} - }; - - /// Returns nullptr if \param Path doesn't exist. - /// Returns nullptr if \param Path isn't a directory. - /// Returns nullptr if OS kernel API told us we can't start watching. In such - /// case it's unclear whether just retrying has any chance to succeeed. - static std::unique_ptr - create(llvm::StringRef Path, - std::function Events, - bool IsInitial)> - Receiver, - bool WaitForInitialSync); - - virtual ~DirectoryWatcher() = default; - DirectoryWatcher(const DirectoryWatcher &) = delete; - DirectoryWatcher &operator=(const DirectoryWatcher &) = delete; - DirectoryWatcher(DirectoryWatcher &&) = default; - -protected: - DirectoryWatcher() = default; -}; - -} // namespace clang - -#endif // LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 0c03f5972b..b3fa93555b 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -18,7 +18,6 @@ add_subdirectory(Serialization) add_subdirectory(Frontend) add_subdirectory(FrontendTool) add_subdirectory(Tooling) -add_subdirectory(DirectoryWatcher) add_subdirectory(Index) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(StaticAnalyzer) diff --git a/lib/DirectoryWatcher/CMakeLists.txt b/lib/DirectoryWatcher/CMakeLists.txt deleted file mode 100644 index 45a6443a17..0000000000 --- a/lib/DirectoryWatcher/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -include(CheckIncludeFiles) - -set(LLVM_LINK_COMPONENTS support) - -set(DIRECTORY_WATCHER_SOURCES DirectoryScanner.cpp) -set(DIRECTORY_WATCHER_LINK_LIBS "") - -if(APPLE) - check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES) - if(HAVE_CORESERVICES) - list(APPEND DIRECTORY_WATCHER_SOURCES mac/DirectoryWatcher-mac.cpp) - set(DIRECTORY_WATCHER_LINK_LIBS "-framework CoreServices") - endif() -elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") - check_include_files("sys/inotify.h" HAVE_INOTIFY) - if(HAVE_INOTIFY) - list(APPEND DIRECTORY_WATCHER_SOURCES linux/DirectoryWatcher-linux.cpp) - find_package(Threads REQUIRED) - set(DIRECTORY_WATCHER_LINK_LIBS ${CMAKE_THREAD_LIBS_INIT}) - endif() -else() - list(APPEND DIRECTORY_WATCHER_SOURCES default/DirectoryWatcher-not-implemented.cpp) -endif() - -add_clang_library(clangDirectoryWatcher - ${DIRECTORY_WATCHER_SOURCES} - ) - -target_link_libraries(clangDirectoryWatcher PRIVATE ${DIRECTORY_WATCHER_LINK_LIBS}) diff --git a/lib/DirectoryWatcher/DirectoryScanner.cpp b/lib/DirectoryWatcher/DirectoryScanner.cpp deleted file mode 100644 index ecfec52f45..0000000000 --- a/lib/DirectoryWatcher/DirectoryScanner.cpp +++ /dev/null @@ -1,54 +0,0 @@ -//===- DirectoryScanner.cpp - Utility functions for DirectoryWatcher ------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "DirectoryScanner.h" - -#include "llvm/Support/Path.h" - -namespace clang { - -using namespace llvm; - -Optional getFileStatus(StringRef Path) { - sys::fs::file_status Status; - std::error_code EC = status(Path, Status); - if (EC) - return None; - return Status; -} - -std::vector scanDirectory(StringRef Path) { - using namespace llvm::sys; - std::vector Result; - - std::error_code EC; - for (auto It = fs::directory_iterator(Path, EC), - End = fs::directory_iterator(); - !EC && It != End; It.increment(EC)) { - auto status = getFileStatus(It->path()); - if (!status.hasValue()) - continue; - Result.emplace_back(sys::path::filename(It->path())); - } - - return Result; -} - -std::vector -getAsFileEvents(const std::vector &Scan) { - std::vector Events; - Events.reserve(Scan.size()); - - for (const auto &File : Scan) { - Events.emplace_back(DirectoryWatcher::Event{ - DirectoryWatcher::Event::EventKind::Modified, File}); - } - return Events; -} - -} // namespace clang \ No newline at end of file diff --git a/lib/DirectoryWatcher/DirectoryScanner.h b/lib/DirectoryWatcher/DirectoryScanner.h deleted file mode 100644 index 55731225e2..0000000000 --- a/lib/DirectoryWatcher/DirectoryScanner.h +++ /dev/null @@ -1,29 +0,0 @@ -//===- DirectoryScanner.h - Utility functions for DirectoryWatcher --------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/DirectoryWatcher/DirectoryWatcher.h" -#include "llvm/Support/FileSystem.h" -#include -#include - -namespace clang { - -/// Gets names (filenames) of items in directory at \p Path. -/// \returns empty vector if \p Path is not a directory, doesn't exist or can't -/// be read from. -std::vector scanDirectory(llvm::StringRef Path); - -/// Create event with EventKind::Added for every element in \p Scan. -std::vector -getAsFileEvents(const std::vector &Scan); - -/// Gets status of file (or directory) at \p Path. -/// \returns llvm::None if \p Path doesn't exist or can't get the status. -llvm::Optional getFileStatus(llvm::StringRef Path); - -} // namespace clang \ No newline at end of file diff --git a/lib/DirectoryWatcher/default/DirectoryWatcher-not-implemented.cpp b/lib/DirectoryWatcher/default/DirectoryWatcher-not-implemented.cpp deleted file mode 100644 index e330ff06f5..0000000000 --- a/lib/DirectoryWatcher/default/DirectoryWatcher-not-implemented.cpp +++ /dev/null @@ -1,19 +0,0 @@ -//===- DirectoryWatcher-not-implemented.cpp -------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/DirectoryWatcher/DirectoryWatcher.h" - -using namespace llvm; -using namespace clang; - -std::unique_ptr clang::DirectoryWatcher::create( - StringRef Path, - std::function, bool)> Receiver, - bool WaitForInitialSync) { - return nullptr; -} \ No newline at end of file diff --git a/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp deleted file mode 100644 index 986ebc5d95..0000000000 --- a/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp +++ /dev/null @@ -1,345 +0,0 @@ -//===- DirectoryWatcher-linux.cpp - Linux-platform directory watching -----===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "DirectoryScanner.h" -#include "clang/DirectoryWatcher/DirectoryWatcher.h" - -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/Support/AlignOf.h" -#include "llvm/Support/Errno.h" -#include "llvm/Support/Mutex.h" -#include "llvm/Support/Path.h" -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace { - -using namespace llvm; -using namespace clang; - -/// Pipe for inter-thread synchronization - for epoll-ing on multiple -/// conditions. It is meant for uni-directional 1:1 signalling - specifically: -/// no multiple consumers, no data passing. Thread waiting for signal should -/// poll the FDRead. Signalling thread should call signal() which writes single -/// character to FDRead. -struct SemaphorePipe { - // Expects two file-descriptors opened as a pipe in the canonical POSIX - // order: pipefd[0] refers to the read end of the pipe. pipefd[1] refers to - // the write end of the pipe. - SemaphorePipe(int pipefd[2]) - : FDRead(pipefd[0]), FDWrite(pipefd[1]), OwnsFDs(true) {} - SemaphorePipe(const SemaphorePipe &) = delete; - void operator=(const SemaphorePipe &) = delete; - SemaphorePipe(SemaphorePipe &&other) - : FDRead(other.FDRead), FDWrite(other.FDWrite), - OwnsFDs(other.OwnsFDs) // Someone could have moved from the other - // instance before. - { - other.OwnsFDs = false; - }; - - void signal() { - ssize_t Result = llvm::sys::RetryAfterSignal(-1, write, FDWrite, "A", 1); - assert(Result != -1); - } - ~SemaphorePipe() { - if (OwnsFDs) { - close(FDWrite); - close(FDRead); - } - } - const int FDRead; - const int FDWrite; - bool OwnsFDs; - - static llvm::Optional create() { - int InotifyPollingStopperFDs[2]; - if (pipe2(InotifyPollingStopperFDs, O_CLOEXEC) == -1) - return llvm::None; - return SemaphorePipe(InotifyPollingStopperFDs); - } -}; - -/// Mutex-protected queue of Events. -class EventQueue { - std::mutex Mtx; - std::condition_variable NonEmpty; - std::queue Events; - -public: - void push_back(const DirectoryWatcher::Event::EventKind K, - StringRef Filename) { - { - std::unique_lock L(Mtx); - Events.emplace(K, Filename); - } - NonEmpty.notify_one(); - } - - // Blocks on caller thread and uses codition_variable to wait until there's an - // event to return. - DirectoryWatcher::Event pop_front_blocking() { - std::unique_lock L(Mtx); - while (true) { - // Since we might have missed all the prior notifications on NonEmpty we - // have to check the queue first (under lock). - if (!Events.empty()) { - DirectoryWatcher::Event Front = Events.front(); - Events.pop(); - return Front; - } - NonEmpty.wait(L, [this]() { return !Events.empty(); }); - } - } -}; - -class DirectoryWatcherLinux : public clang::DirectoryWatcher { -public: - DirectoryWatcherLinux( - llvm::StringRef WatchedDirPath, - std::function, bool)> Receiver, - bool WaitForInitialSync, int InotifyFD, int InotifyWD, - SemaphorePipe &&InotifyPollingStopSignal); - - ~DirectoryWatcherLinux() override { - StopWork(); - InotifyPollingThread.join(); - EventsReceivingThread.join(); - inotify_rm_watch(InotifyFD, InotifyWD); - llvm::sys::RetryAfterSignal(-1, close, InotifyFD); - } - -private: - const std::string WatchedDirPath; - // inotify file descriptor - int InotifyFD = -1; - // inotify watch descriptor - int InotifyWD = -1; - - EventQueue Queue; - - // Make sure lifetime of Receiver fully contains lifetime of - // EventsReceivingThread. - std::function, bool)> Receiver; - - // Consumes inotify events and pushes directory watcher events to the Queue. - void InotifyPollingLoop(); - std::thread InotifyPollingThread; - // Using pipe so we can epoll two file descriptors at once - inotify and - // stopping condition. - SemaphorePipe InotifyPollingStopSignal; - - // Does the initial scan of the directory - directly calling Receiver, - // bypassing the Queue. Both InitialScan and EventReceivingLoop use Receiver - // which isn't necessarily thread-safe. - void InitialScan(); - - // Processing events from the Queue. - // In case client doesn't want to do the initial scan synchronously - // (WaitForInitialSync=false in ctor) we do the initial scan at the beginning - // of this thread. - std::thread EventsReceivingThread; - // Push event of WatcherGotInvalidated kind to the Queue to stop the loop. - // Both InitialScan and EventReceivingLoop use Receiver which isn't - // necessarily thread-safe. - void EventReceivingLoop(); - - // Stops all the async work. Reentrant. - void StopWork() { - Queue.push_back(DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, - ""); - InotifyPollingStopSignal.signal(); - } -}; - -void DirectoryWatcherLinux::InotifyPollingLoop() { - // We want to be able to read ~30 events at once even in the worst case - // (obscenely long filenames). - constexpr size_t EventBufferLength = - 30 * (sizeof(struct inotify_event) + NAME_MAX + 1); - // http://man7.org/linux/man-pages/man7/inotify.7.html - // Some systems cannot read integer variables if they are not - // properly aligned. On other systems, incorrect alignment may - // decrease performance. Hence, the buffer used for reading from - // the inotify file descriptor should have the same alignment as - // struct inotify_event. - - auto ManagedBuffer = - llvm::make_unique>(); - char *const Buf = ManagedBuffer->buffer; - - const int EpollFD = epoll_create1(EPOLL_CLOEXEC); - if (EpollFD == -1) { - StopWork(); - return; - } - auto EpollFDGuard = llvm::make_scope_exit([EpollFD]() { close(EpollFD); }); - - struct epoll_event EventSpec; - EventSpec.events = EPOLLIN; - EventSpec.data.fd = InotifyFD; - if (epoll_ctl(EpollFD, EPOLL_CTL_ADD, InotifyFD, &EventSpec) == -1) { - StopWork(); - return; - } - - EventSpec.data.fd = InotifyPollingStopSignal.FDRead; - if (epoll_ctl(EpollFD, EPOLL_CTL_ADD, InotifyPollingStopSignal.FDRead, - &EventSpec) == -1) { - StopWork(); - return; - } - - std::array EpollEventBuffer; - - while (true) { - const int EpollWaitResult = llvm::sys::RetryAfterSignal( - -1, epoll_wait, EpollFD, EpollEventBuffer.data(), - EpollEventBuffer.size(), /*timeout=*/-1 /*== infinity*/); - if (EpollWaitResult == -1) { - StopWork(); - return; - } - - // Multiple epoll_events can be received for a single file descriptor per - // epoll_wait call. - for (const auto &EpollEvent : EpollEventBuffer) { - if (EpollEvent.data.fd == InotifyPollingStopSignal.FDRead) { - StopWork(); - return; - } - } - - // epoll_wait() always return either error or >0 events. Since there was no - // event for stopping, it must be an inotify event ready for reading. - ssize_t NumRead = llvm::sys::RetryAfterSignal(-1, read, InotifyFD, Buf, - EventBufferLength); - for (char *P = Buf; P < Buf + NumRead;) { - if (P + sizeof(struct inotify_event) > Buf + NumRead) { - StopWork(); - llvm_unreachable("an incomplete inotify_event was read"); - return; - } - - struct inotify_event *Event = reinterpret_cast(P); - P += sizeof(struct inotify_event) + Event->len; - - if (Event->mask & (IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_DELETE) && - Event->len <= 0) { - StopWork(); - llvm_unreachable("expected a filename from inotify"); - return; - } - - if (Event->mask & (IN_CREATE | IN_MOVED_TO | IN_MODIFY)) { - Queue.push_back(DirectoryWatcher::Event::EventKind::Modified, - Event->name); - } else if (Event->mask & (IN_DELETE | IN_MOVED_FROM)) { - Queue.push_back(DirectoryWatcher::Event::EventKind::Removed, - Event->name); - } else if (Event->mask & (IN_DELETE_SELF | IN_MOVE_SELF)) { - Queue.push_back(DirectoryWatcher::Event::EventKind::WatchedDirRemoved, - ""); - StopWork(); - return; - } else if (Event->mask & IN_IGNORED) { - StopWork(); - return; - } else { - StopWork(); - llvm_unreachable("Unknown event type."); - return; - } - } - } -} - -void DirectoryWatcherLinux::InitialScan() { - this->Receiver(getAsFileEvents(scanDirectory(WatchedDirPath)), - /*IsInitial=*/true); -} - -void DirectoryWatcherLinux::EventReceivingLoop() { - while (true) { - DirectoryWatcher::Event Event = this->Queue.pop_front_blocking(); - this->Receiver(Event, false); - if (Event.Kind == - DirectoryWatcher::Event::EventKind::WatcherGotInvalidated) { - StopWork(); - return; - } - } -} - -DirectoryWatcherLinux::DirectoryWatcherLinux( - StringRef WatchedDirPath, - std::function, bool)> Receiver, - bool WaitForInitialSync, int InotifyFD, int InotifyWD, - SemaphorePipe &&InotifyPollingStopSignal) - : WatchedDirPath(WatchedDirPath), InotifyFD(InotifyFD), - InotifyWD(InotifyWD), Receiver(Receiver), - InotifyPollingStopSignal(std::move(InotifyPollingStopSignal)) { - - InotifyPollingThread = std::thread([this]() { InotifyPollingLoop(); }); - // We have no guarantees about thread safety of the Receiver which is being - // used in both InitialScan and EventReceivingLoop. We shouldn't run these - // only synchronously. - if (WaitForInitialSync) { - InitialScan(); - EventsReceivingThread = std::thread([this]() { EventReceivingLoop(); }); - } else { - EventsReceivingThread = std::thread([this]() { - // FIXME: We might want to terminate an async initial scan early in case - // of a failure in EventsReceivingThread. - InitialScan(); - EventReceivingLoop(); - }); - } -} - -} // namespace - -std::unique_ptr clang::DirectoryWatcher::create( - StringRef Path, - std::function, bool)> Receiver, - bool WaitForInitialSync) { - if (Path.empty()) - return nullptr; - - const int InotifyFD = inotify_init1(IN_CLOEXEC); - if (InotifyFD == -1) - return nullptr; - - const int InotifyWD = inotify_add_watch( - InotifyFD, Path.str().c_str(), - IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_EXCL_UNLINK | IN_MODIFY | - IN_MOVED_FROM | IN_MOVE_SELF | IN_MOVED_TO | IN_ONLYDIR | IN_IGNORED); - if (InotifyWD == -1) - return nullptr; - - auto InotifyPollingStopper = SemaphorePipe::create(); - - if (!InotifyPollingStopper) - return nullptr; - - return llvm::make_unique( - Path, Receiver, WaitForInitialSync, InotifyFD, InotifyWD, - std::move(*InotifyPollingStopper)); -} \ No newline at end of file diff --git a/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp b/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp deleted file mode 100644 index 3df79ac48a..0000000000 --- a/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp +++ /dev/null @@ -1,233 +0,0 @@ -//===- DirectoryWatcher-mac.cpp - Mac-platform directory watching ---------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "DirectoryScanner.h" -#include "clang/DirectoryWatcher/DirectoryWatcher.h" - -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/Path.h" -#include - -using namespace llvm; -using namespace clang; - -static FSEventStreamRef createFSEventStream( - StringRef Path, - std::function, bool)>, - dispatch_queue_t); -static void stopFSEventStream(FSEventStreamRef); - -namespace { - -class DirectoryWatcherMac : public clang::DirectoryWatcher { -public: - DirectoryWatcherMac( - FSEventStreamRef EventStream, - std::function, bool)> - Receiver, - llvm::StringRef WatchedDirPath) - : EventStream(EventStream), Receiver(Receiver), - WatchedDirPath(WatchedDirPath) {} - - ~DirectoryWatcherMac() override { - stopFSEventStream(EventStream); - EventStream = nullptr; - // Now it's safe to use Receiver as the only other concurrent use would have - // been in EventStream processing. - Receiver(DirectoryWatcher::Event( - DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""), - false); - } - -private: - FSEventStreamRef EventStream; - std::function, bool)> Receiver; - const std::string WatchedDirPath; -}; - -struct EventStreamContextData { - std::string WatchedPath; - std::function, bool)> Receiver; - - EventStreamContextData( - std::string &&WatchedPath, - std::function, bool)> - Receiver) - : WatchedPath(std::move(WatchedPath)), Receiver(Receiver) {} - - // Needed for FSEvents - static void dispose(const void *ctx) { - delete static_cast(ctx); - } -}; -} // namespace - -constexpr const FSEventStreamEventFlags StreamInvalidatingFlags = - kFSEventStreamEventFlagUserDropped | kFSEventStreamEventFlagKernelDropped | - kFSEventStreamEventFlagMustScanSubDirs; - -constexpr const FSEventStreamEventFlags ModifyingFileEvents = - kFSEventStreamEventFlagItemCreated | kFSEventStreamEventFlagItemRenamed | - kFSEventStreamEventFlagItemModified; - -static void eventStreamCallback(ConstFSEventStreamRef Stream, - void *ClientCallBackInfo, size_t NumEvents, - void *EventPaths, - const FSEventStreamEventFlags EventFlags[], - const FSEventStreamEventId EventIds[]) { - auto *ctx = static_cast(ClientCallBackInfo); - - std::vector Events; - for (size_t i = 0; i < NumEvents; ++i) { - StringRef Path = ((const char **)EventPaths)[i]; - const FSEventStreamEventFlags Flags = EventFlags[i]; - - if (Flags & StreamInvalidatingFlags) { - Events.emplace_back(DirectoryWatcher::Event{ - DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); - break; - } else if (!(Flags & kFSEventStreamEventFlagItemIsFile)) { - // Subdirectories aren't supported - if some directory got removed it - // must've been the watched directory itself. - if ((Flags & kFSEventStreamEventFlagItemRemoved) && - Path == ctx->WatchedPath) { - Events.emplace_back(DirectoryWatcher::Event{ - DirectoryWatcher::Event::EventKind::WatchedDirRemoved, ""}); - Events.emplace_back(DirectoryWatcher::Event{ - DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); - break; - } - // No support for subdirectories - just ignore everything. - continue; - } else if (Flags & kFSEventStreamEventFlagItemRemoved) { - Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, - llvm::sys::path::filename(Path)); - continue; - } else if (Flags & ModifyingFileEvents) { - if (!getFileStatus(Path).hasValue()) { - Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, - llvm::sys::path::filename(Path)); - } else { - Events.emplace_back(DirectoryWatcher::Event::EventKind::Modified, - llvm::sys::path::filename(Path)); - } - continue; - } - - // default - Events.emplace_back(DirectoryWatcher::Event{ - DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); - llvm_unreachable("Unknown FSEvent type."); - } - - if (!Events.empty()) { - ctx->Receiver(Events, /*IsInitial=*/false); - } -} - -FSEventStreamRef createFSEventStream( - StringRef Path, - std::function, bool)> Receiver, - dispatch_queue_t Queue) { - if (Path.empty()) - return nullptr; - - CFMutableArrayRef PathsToWatch = [&]() { - CFMutableArrayRef PathsToWatch = - CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); - CFStringRef CfPathStr = - CFStringCreateWithBytes(nullptr, (const UInt8 *)Path.data(), - Path.size(), kCFStringEncodingUTF8, false); - CFArrayAppendValue(PathsToWatch, CfPathStr); - CFRelease(CfPathStr); - return PathsToWatch; - }(); - - FSEventStreamContext Context = [&]() { - std::string RealPath; - { - SmallString<128> Storage; - StringRef P = llvm::Twine(Path).toNullTerminatedStringRef(Storage); - char Buffer[PATH_MAX]; - if (::realpath(P.begin(), Buffer) != nullptr) - RealPath = Buffer; - else - RealPath = Path; - } - - FSEventStreamContext Context; - Context.version = 0; - Context.info = new EventStreamContextData(std::move(RealPath), Receiver); - Context.retain = nullptr; - Context.release = EventStreamContextData::dispose; - Context.copyDescription = nullptr; - return Context; - }(); - - FSEventStreamRef Result = FSEventStreamCreate( - nullptr, eventStreamCallback, &Context, PathsToWatch, - kFSEventStreamEventIdSinceNow, /* latency in seconds */ 0.0, - kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagNoDefer); - CFRelease(PathsToWatch); - - return Result; -} - -void stopFSEventStream(FSEventStreamRef EventStream) { - if (!EventStream) - return; - FSEventStreamStop(EventStream); - FSEventStreamInvalidate(EventStream); - FSEventStreamRelease(EventStream); -} - -std::unique_ptr clang::DirectoryWatcher::create( - StringRef Path, - std::function, bool)> Receiver, - bool WaitForInitialSync) { - dispatch_queue_t Queue = - dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); - - if (Path.empty()) - return nullptr; - - auto EventStream = createFSEventStream(Path, Receiver, Queue); - if (!EventStream) { - return nullptr; - } - - std::unique_ptr Result = - llvm::make_unique(EventStream, Receiver, Path); - - // We need to copy the data so the lifetime is ok after a const copy is made - // for the block. - const std::string CopiedPath = Path; - - auto InitWork = ^{ - // We need to start watching the directory before we start scanning in order - // to not miss any event. By dispatching this on the same serial Queue as - // the FSEvents will be handled we manage to start watching BEFORE the - // inital scan and handling events ONLY AFTER the scan finishes. - FSEventStreamSetDispatchQueue(EventStream, Queue); - FSEventStreamStart(EventStream); - // We need to decrement the ref count for Queue as initialize() will return - // and FSEvents has incremented it. Since we have to wait for FSEvents to - // take ownership it's the easiest to do it here rather than main thread. - dispatch_release(Queue); - Receiver(getAsFileEvents(scanDirectory(CopiedPath)), /*IsInitial=*/true); - }; - - if (WaitForInitialSync) { - dispatch_sync(Queue, InitWork); - } else { - dispatch_async(Queue, InitWork); - } - - return Result; -} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 9a41000cf4..4c8a3a840d 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -30,7 +30,6 @@ add_subdirectory(CodeGen) if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) add_subdirectory(libclang) endif() -add_subdirectory(DirectoryWatcher) add_subdirectory(Rename) add_subdirectory(Index) add_subdirectory(Serialization) diff --git a/unittests/DirectoryWatcher/CMakeLists.txt b/unittests/DirectoryWatcher/CMakeLists.txt deleted file mode 100644 index fc93323b37..0000000000 --- a/unittests/DirectoryWatcher/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Linux") - - set(LLVM_LINK_COMPONENTS - Support - ) - - add_clang_unittest(DirectoryWatcherTests - DirectoryWatcherTest.cpp - ) - - target_link_libraries(DirectoryWatcherTests - PRIVATE - clangDirectoryWatcher - clangBasic - ) - -endif() \ No newline at end of file diff --git a/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp b/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp deleted file mode 100644 index a2c50fc7d0..0000000000 --- a/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp +++ /dev/null @@ -1,426 +0,0 @@ -//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/DirectoryWatcher/DirectoryWatcher.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Mutex.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/raw_ostream.h" -#include "gtest/gtest.h" -#include -#include -#include -#include - -using namespace llvm; -using namespace llvm::sys; -using namespace llvm::sys::fs; -using namespace clang; - -namespace clang { -static bool operator==(const DirectoryWatcher::Event &lhs, - const DirectoryWatcher::Event &rhs) { - return lhs.Filename == rhs.Filename && - static_cast(lhs.Kind) == static_cast(rhs.Kind); -} -} // namespace clang - -namespace { - -struct DirectoryWatcherTestFixture { - std::string TestRootDir; - std::string TestWatchedDir; - - DirectoryWatcherTestFixture() { - SmallString<128> pathBuf; - std::error_code UniqDirRes = createUniqueDirectory("dirwatcher", pathBuf); - assert(!UniqDirRes); - TestRootDir = pathBuf.str(); - path::append(pathBuf, "watch"); - TestWatchedDir = pathBuf.str(); - std::error_code CreateDirRes = create_directory(TestWatchedDir, false); - assert(!CreateDirRes); - } - - ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); } - - SmallString<128> getPathInWatched(const std::string &testFile) { - SmallString<128> pathBuf; - pathBuf = TestWatchedDir; - path::append(pathBuf, testFile); - return pathBuf; - } - - void addFile(const std::string &testFile) { - Expected ft = openNativeFileForWrite(getPathInWatched(testFile), - CD_CreateNew, OF_None); - if (ft) { - closeFile(*ft); - } else { - llvm::errs() << llvm::toString(ft.takeError()) << "\n"; - llvm::errs() << getPathInWatched(testFile) << "\n"; - llvm_unreachable("Couldn't create test file."); - } - } - - void deleteFile(const std::string &testFile) { - std::error_code EC = - remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false); - ASSERT_FALSE(EC); - } -}; - -std::string eventKindToString(const DirectoryWatcher::Event::EventKind K) { - switch (K) { - case DirectoryWatcher::Event::EventKind::Removed: - return "Removed"; - case DirectoryWatcher::Event::EventKind::Modified: - return "Modified"; - case DirectoryWatcher::Event::EventKind::WatchedDirRemoved: - return "WatchedDirRemoved"; - case DirectoryWatcher::Event::EventKind::WatcherGotInvalidated: - return "WatcherGotInvalidated"; - } - llvm_unreachable("unknown event kind"); -} - -struct VerifyingConsumer { - std::vector ExpectedInitial; - std::vector ExpectedNonInitial; - std::vector OptionalNonInitial; - std::vector UnexpectedInitial; - std::vector UnexpectedNonInitial; - std::mutex Mtx; - std::condition_variable ResultIsReady; - - VerifyingConsumer( - const std::vector &ExpectedInitial, - const std::vector &ExpectedNonInitial, - const std::vector &OptionalNonInitial = {}) - : ExpectedInitial(ExpectedInitial), - ExpectedNonInitial(ExpectedNonInitial), - OptionalNonInitial(OptionalNonInitial) {} - - // This method is used by DirectoryWatcher. - void consume(DirectoryWatcher::Event E, bool IsInitial) { - if (IsInitial) - consumeInitial(E); - else - consumeNonInitial(E); - } - - void consumeInitial(DirectoryWatcher::Event E) { - std::unique_lock L(Mtx); - auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E); - if (It == ExpectedInitial.end()) { - UnexpectedInitial.push_back(E); - } else { - ExpectedInitial.erase(It); - } - if (result()) - ResultIsReady.notify_one(); - } - - void consumeNonInitial(DirectoryWatcher::Event E) { - std::unique_lock L(Mtx); - auto It = - std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E); - if (It == ExpectedNonInitial.end()) { - auto OptIt = - std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E); - if (OptIt != OptionalNonInitial.end()) { - OptionalNonInitial.erase(OptIt); - } else { - UnexpectedNonInitial.push_back(E); - } - } else { - ExpectedNonInitial.erase(It); - } - if (result()) - ResultIsReady.notify_one(); - } - - // This method is used by DirectoryWatcher. - void consume(llvm::ArrayRef Es, bool IsInitial) { - for (const auto &E : Es) - consume(E, IsInitial); - } - - // Not locking - caller has to lock Mtx. - llvm::Optional result() const { - if (ExpectedInitial.empty() && ExpectedNonInitial.empty() && - UnexpectedInitial.empty() && UnexpectedNonInitial.empty()) - return true; - if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty()) - return false; - return llvm::None; - } - - // This method is used by tests. - // \returns true on success - bool blockUntilResult() { - std::unique_lock L(Mtx); - while (true) { - if (result()) - return *result(); - - ResultIsReady.wait(L, [this]() { return result().hasValue(); }); - } - return false; // Just to make compiler happy. - } - - void printUnmetExpectations(llvm::raw_ostream &OS) { - if (!ExpectedInitial.empty()) { - OS << "Expected but not seen initial events: \n"; - for (const auto &E : ExpectedInitial) { - OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; - } - } - if (!ExpectedNonInitial.empty()) { - OS << "Expected but not seen non-initial events: \n"; - for (const auto &E : ExpectedNonInitial) { - OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; - } - } - if (!UnexpectedInitial.empty()) { - OS << "Unexpected initial events seen: \n"; - for (const auto &E : UnexpectedInitial) { - OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; - } - } - if (!UnexpectedNonInitial.empty()) { - OS << "Unexpected non-initial events seen: \n"; - for (const auto &E : UnexpectedNonInitial) { - OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; - } - } - } -}; - -void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) { - std::packaged_task task( - [&TestConsumer]() { return TestConsumer.blockUntilResult(); }); - std::future WaitForExpectedStateResult = task.get_future(); - std::thread worker(std::move(task)); - worker.detach(); - - EXPECT_TRUE(WaitForExpectedStateResult.wait_for(std::chrono::seconds(3)) == - std::future_status::ready) - << "The expected result state wasn't reached before the time-out."; - EXPECT_TRUE(TestConsumer.result().hasValue()); - if (TestConsumer.result().hasValue()) { - EXPECT_TRUE(*TestConsumer.result()); - } - if ((TestConsumer.result().hasValue() && !TestConsumer.result().getValue()) || - !TestConsumer.result().hasValue()) - TestConsumer.printUnmetExpectations(llvm::outs()); -} - -} // namespace - -TEST(DirectoryWatcherTest, InitialScanSync) { - DirectoryWatcherTestFixture fixture; - - fixture.addFile("a"); - fixture.addFile("b"); - fixture.addFile("c"); - - VerifyingConsumer TestConsumer{ - {{DirectoryWatcher::Event::EventKind::Modified, "a"}, - {DirectoryWatcher::Event::EventKind::Modified, "b"}, - {DirectoryWatcher::Event::EventKind::Modified, "c"}}, - {}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, InitialScanAsync) { - DirectoryWatcherTestFixture fixture; - - fixture.addFile("a"); - fixture.addFile("b"); - fixture.addFile("c"); - - VerifyingConsumer TestConsumer{ - {{DirectoryWatcher::Event::EventKind::Modified, "a"}, - {DirectoryWatcher::Event::EventKind::Modified, "b"}, - {DirectoryWatcher::Event::EventKind::Modified, "c"}}, - {}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/false); - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, AddFiles) { - DirectoryWatcherTestFixture fixture; - - VerifyingConsumer TestConsumer{ - {}, - {{DirectoryWatcher::Event::EventKind::Modified, "a"}, - {DirectoryWatcher::Event::EventKind::Modified, "b"}, - {DirectoryWatcher::Event::EventKind::Modified, "c"}}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - - fixture.addFile("a"); - fixture.addFile("b"); - fixture.addFile("c"); - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, ModifyFile) { - DirectoryWatcherTestFixture fixture; - - fixture.addFile("a"); - - VerifyingConsumer TestConsumer{ - {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, - {{DirectoryWatcher::Event::EventKind::Modified, "a"}}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - - // modify the file - { - std::error_code error; - llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error, - CD_OpenExisting); - assert(!error); - bStream << "foo"; - } - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, DeleteFile) { - DirectoryWatcherTestFixture fixture; - - fixture.addFile("a"); - - VerifyingConsumer TestConsumer{ - {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, - {{DirectoryWatcher::Event::EventKind::Removed, "a"}}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - - fixture.deleteFile("a"); - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, DeleteWatchedDir) { - DirectoryWatcherTestFixture fixture; - - VerifyingConsumer TestConsumer{ - {}, - {{DirectoryWatcher::Event::EventKind::WatchedDirRemoved, ""}, - {DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - - remove_directories(fixture.TestWatchedDir); - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, InvalidatedWatcher) { - DirectoryWatcherTestFixture fixture; - - VerifyingConsumer TestConsumer{ - {}, {{DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}}}; - - { - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - } // DW is destructed here. - - checkEventualResultWithTimeout(TestConsumer); -} - -TEST(DirectoryWatcherTest, ChangeMetadata) { - DirectoryWatcherTestFixture fixture; - fixture.addFile("a"); - - VerifyingConsumer TestConsumer{ - {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, - // We don't expect any notification for file having access file changed. - {}, - // Given the timing we are ok with receiving the duplicate event. - {{DirectoryWatcher::Event::EventKind::Modified, "a"}}}; - - auto DW = DirectoryWatcher::create( - fixture.TestWatchedDir, - [&TestConsumer](llvm::ArrayRef Events, - bool IsInitial) { - TestConsumer.consume(Events, IsInitial); - }, - /*waitForInitialSync=*/true); - - { // Change access and modification time of file a. - Expected HopefullyTheFD = llvm::sys::fs::openNativeFileForWrite( - fixture.getPathInWatched("a"), CD_OpenExisting, OF_None); - if (!HopefullyTheFD) { - llvm::outs() << HopefullyTheFD.takeError(); - } - - const int FD = HopefullyTheFD.get(); - const TimePoint<> NewTimePt = - std::chrono::system_clock::now() - std::chrono::minutes(1); - - std::error_code setTimeRes = - llvm::sys::fs::setLastAccessAndModificationTime(FD, NewTimePt, - NewTimePt); - assert(!setTimeRes); - } - - checkEventualResultWithTimeout(TestConsumer); -} -- 2.40.0