From 92e1fd5269ae700b8a59b253701951f09cb4ae0e Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Tue, 12 Sep 2017 11:07:29 +0200 Subject: [PATCH 01/37] refactor the queue implementation --- CMakeLists.txt | 9 +++++ includes/NativeInterface.h | 18 +++++++-- includes/Queue.h | 30 +++++++-------- src/CMakeLists.txt | 12 ++++++ src/NativeInterface.cpp | 38 +++---------------- src/Queue.cpp | 77 +++++++++++++++----------------------- 6 files changed, 85 insertions(+), 99 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 src/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..b2af2baf --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required (VERSION 3.1.0 FATAL_ERROR) +set (PROJECT_NAME "NSFW") +project (${PROJECT_NAME}) + +message (STATUS "Running CMake version ${CMAKE_VERSION}") + +set (NSFW_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") + +add_subdirectory (src) diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index d4bf6a17..6137f261 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -4,19 +4,29 @@ #include "Queue.h" #include +#if defined(_WIN32) +#include "../includes/win32/ReadLoop.h" +using NativeImplementation = ReadLoop; +#elif defined(__APPLE_CC__) || defined(BSD) +#include "../includes/osx/FSEventsService.h" +using NativeImplementation = FSEventsService; +#elif defined(__linux__) +#include "../includes/linux/InotifyService.h" +using NativeImplementation = InotifyService; +#endif + class NativeInterface { public: - NativeInterface(std::string path); + NativeInterface(const std::string path); std::string getError(); - std::vector *getEvents(); + std::vector* getEvents(); bool hasErrored(); bool isWatching(); - ~NativeInterface(); private: EventQueue mQueue; - void *mNativeInterface; + std::unique_ptr mNativeInterface; }; #endif diff --git a/includes/Queue.h b/includes/Queue.h index 70c5893d..ebb4e82a 100644 --- a/includes/Queue.h +++ b/includes/Queue.h @@ -2,10 +2,10 @@ #define NSFW_QUEUE_H #include -extern "C" { -# include -# include -} +#include +#include +#include +#include enum EventType { CREATED = 0, @@ -15,32 +15,30 @@ enum EventType { }; struct Event { + Event(const EventType type, const std::string& directory, const std::string& fileA, const std::string& fileB) : + type(type), directory(directory), fileA(fileA), fileB(fileB) {} EventType type; std::string directory, fileA, fileB; }; class EventQueue { public: - EventQueue(); - ~EventQueue(); void clear(); int count(); - Event *dequeue(); // Free this pointer when you are done with it + std::unique_ptr dequeue(); + std::unique_ptr> dequeueAll(); void enqueue( EventType type, - std::string directory, - std::string fileA, - std::string fileB = "" + const std::string& directory, + const std::string& fileA, + const std::string& fileB = "" ); private: - struct EventNode { - OPA_Queue_element_hdr_t header; - Event *event; - }; - OPA_Queue_info_t mQueue; - OPA_int_t mNumEvents; + std::deque> queue; + std::mutex mutex; + }; #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..4135d67e --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,12 @@ + +set (NSFW_LIBRARY_SOURCES + Queue.cpp + NativeInterface.cpp + win32/ReadLoop.cpp + win32/ReadLoopRunner.cpp +) + +set (CMAKE_CXX_STANDARD 11) + +add_library(nsfw STATIC ${NSFW_LIBRARY_SOURCES}) +target_include_directories(nsfw PUBLIC ${NSFW_INCLUDE_DIR}) diff --git a/src/NativeInterface.cpp b/src/NativeInterface.cpp index 098a411d..64672d3b 100644 --- a/src/NativeInterface.cpp +++ b/src/NativeInterface.cpp @@ -1,47 +1,21 @@ #include "../includes/NativeInterface.h" -#if defined(_WIN32) -#define SERVICE ReadLoop -#include "../includes/win32/ReadLoop.h" -#elif defined(__APPLE_CC__) || defined(BSD) -#define SERVICE FSEventsService -#include "../includes/osx/FSEventsService.h" -#elif defined(__linux__) -#define SERVICE InotifyService -#include "../includes/linux/InotifyService.h" -#endif - NativeInterface::NativeInterface(std::string path) { - mNativeInterface = new SERVICE(mQueue, path); -} - -NativeInterface::~NativeInterface() { - delete (SERVICE *)mNativeInterface; + mNativeInterface.reset(new NativeImplementation(mQueue, path)); } std::string NativeInterface::getError() { - return ((SERVICE *)mNativeInterface)->getError(); + return mNativeInterface->getError(); } -std::vector *NativeInterface::getEvents() { - if (mQueue.count() == 0) { - return NULL; - } - - int count = mQueue.count(); - std::vector *events = new std::vector; - events->reserve(count); - for (int i = 0; i < count; ++i) { - events->push_back(mQueue.dequeue()); - } - - return events; +std::vector* NativeInterface::getEvents() { + return mQueue.dequeueAll().release(); } bool NativeInterface::hasErrored() { - return ((SERVICE *)mNativeInterface)->hasErrored(); + return mNativeInterface->hasErrored(); } bool NativeInterface::isWatching() { - return ((SERVICE *)mNativeInterface)->isWatching(); + return mNativeInterface->isWatching(); } diff --git a/src/Queue.cpp b/src/Queue.cpp index 808df6bc..98ea61f2 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -1,65 +1,48 @@ #include "../includes/Queue.h" #pragma unmanaged -EventQueue::EventQueue() { - OPA_Queue_init(&mQueue); - OPA_store_int(&mNumEvents, 0); -} - -EventQueue::~EventQueue() { - while(!OPA_Queue_is_empty(&mQueue)) { - EventNode *node; - - OPA_Queue_dequeue(&mQueue, node, EventNode, header); - - delete node->event; - delete node; - } -} void EventQueue::clear() { - while(!OPA_Queue_is_empty(&mQueue)) { - EventNode *node; - - OPA_decr_int(&mNumEvents); - OPA_Queue_dequeue(&mQueue, node, EventNode, header); - - delete node->event; - delete node; - } + std::lock_guard lock(mutex); + queue.clear(); } int EventQueue::count() { - return OPA_load_int(&mNumEvents); - return 0; + std::lock_guard lock(mutex); + return queue.size(); } -Event *EventQueue::dequeue() { - if (!OPA_Queue_is_empty(&mQueue)) { - EventNode *node; - - OPA_decr_int(&mNumEvents); - OPA_Queue_dequeue(&mQueue, node, EventNode, header); +std::unique_ptr EventQueue::dequeue() { + std::lock_guard lock(mutex); + if (queue.empty()) { + return nullptr; + } - Event *event = node->event; - delete node; + auto& front = queue.front(); + auto retVal = std::move(front); + queue.pop_front(); - return event; - } - return NULL; + return retVal; } -void EventQueue::enqueue(EventType type, std::string directory, std::string fileA, std::string fileB) { - EventNode *node = new EventNode; +std::unique_ptr> EventQueue::dequeueAll() { + std::lock_guard lock(mutex); + if (queue.empty()) { + return nullptr; + } - OPA_Queue_header_init(&node->header); + const auto queueSize = queue.size(); + std::unique_ptr> events(new std::vector(queueSize, nullptr)); + for (auto i = 0; i < queueSize; ++i) { + auto& front = queue.front(); + (*events)[i] = front.release(); + queue.pop_front(); + } - node->event = new Event; - node->event->type = type; - node->event->directory = directory; - node->event->fileA = fileA; - node->event->fileB = fileB; + return events; +} - OPA_Queue_enqueue(&mQueue, node, EventNode, header); - OPA_incr_int(&mNumEvents); +void EventQueue::enqueue(const EventType type, const std::string& directory, const std::string& fileA, const std::string& fileB) { + std::lock_guard lock(mutex); + queue.emplace_back(std::unique_ptr(new Event(type, directory, fileA, fileB))); } From 5ec8b6e63e210334bf6d7f4d158c96a52c0b67bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Tue, 12 Sep 2017 11:30:49 +0200 Subject: [PATCH 02/37] FIX: remove dependency to uv_sem_t in osx service --- includes/osx/RunLoop.h | 5 +++-- src/osx/RunLoop.cpp | 13 ++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/includes/osx/RunLoop.h b/includes/osx/RunLoop.h index 48f22fc4..5395d357 100644 --- a/includes/osx/RunLoop.h +++ b/includes/osx/RunLoop.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include void *scheduleRunLoopWork(void *runLoop); @@ -38,7 +38,8 @@ class RunLoop { std::string mPath; CFRunLoopRef mRunLoop; pthread_t mRunLoopThread; - uv_sem_t mReadyForCleanup; + std::condition_variable mReadyForCleanup; + std::mutex mReadyForCleanupMutex; bool mStarted; }; diff --git a/src/osx/RunLoop.cpp b/src/osx/RunLoop.cpp index a04d1a8e..023acecd 100644 --- a/src/osx/RunLoop.cpp +++ b/src/osx/RunLoop.cpp @@ -11,11 +11,6 @@ RunLoop::RunLoop(FSEventsService *eventsService, std::string path): mPath(path), mRunLoop(NULL), mStarted(false) { - if (uv_sem_init(&mReadyForCleanup, 0) != 0) { - mStarted = false; - return; - } - mStarted = !pthread_create( &mRunLoopThread, NULL, @@ -33,12 +28,12 @@ RunLoop::~RunLoop() { return; } - uv_sem_wait(&mReadyForCleanup); + std::unique_lock lk(mReadyForCleanupMutex); + mReadyForCleanup.wait(lk); CFRunLoopStop(mRunLoop); pthread_join(mRunLoopThread, NULL); - uv_sem_destroy(&mReadyForCleanup); } void RunLoop::work() { @@ -58,9 +53,9 @@ void RunLoop::work() { mRunLoop = CFRunLoopGetCurrent(); - __block uv_sem_t *runLoopHasStarted = &mReadyForCleanup; + __block auto *runLoopHasStarted = &mReadyForCleanup; CFRunLoopPerformBlock(mRunLoop, kCFRunLoopDefaultMode, ^ { - uv_sem_post(runLoopHasStarted); + runLoopHasStarted->notify_all(); }); CFRunLoopWakeUp(mRunLoop); From 4425157a790422db36f27a570317296ab8656442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Tue, 12 Sep 2017 11:31:14 +0200 Subject: [PATCH 03/37] extend CMakeLists.txt for platform dependent code --- src/CMakeLists.txt | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4135d67e..9c64f56b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,11 +1,35 @@ +# platform independent code set (NSFW_LIBRARY_SOURCES Queue.cpp NativeInterface.cpp - win32/ReadLoop.cpp - win32/ReadLoopRunner.cpp ) +if (WIN32) + message (STATUS "compiling windows specific file system service") + set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} + win32/ReadLoop.cpp + win32/ReadLoopRunner.cpp + ) +endif(WIN32) + +if (UNIX) + if (APPLE) + message (STATUS "compiling macOS specific file system service") + set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} + osx/RunLoop.cpp + osx/FSEventsService.cpp + ) + else (APPLE) + message (STATUS "compiling linux specific file system service") + set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} + linux/InotifyEventLoop.cpp + linux/InotifyService.cpp + linux/InotifyTree.cpp + ) + endif(APPLE) +endif (UNIX) + set (CMAKE_CXX_STANDARD 11) add_library(nsfw STATIC ${NSFW_LIBRARY_SOURCES}) From 62939cd20633e0520ad2c3f2fa0880a260808379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Tue, 12 Sep 2017 11:47:43 +0200 Subject: [PATCH 04/37] FIX: use const& of std::string --- includes/NativeInterface.h | 2 +- src/NativeInterface.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index 6137f261..cc6291d4 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -17,7 +17,7 @@ using NativeImplementation = InotifyService; class NativeInterface { public: - NativeInterface(const std::string path); + NativeInterface(const std::string &path); std::string getError(); std::vector* getEvents(); diff --git a/src/NativeInterface.cpp b/src/NativeInterface.cpp index 64672d3b..85e907ee 100644 --- a/src/NativeInterface.cpp +++ b/src/NativeInterface.cpp @@ -1,6 +1,6 @@ #include "../includes/NativeInterface.h" -NativeInterface::NativeInterface(std::string path) { +NativeInterface::NativeInterface(const std::string &path) { mNativeInterface.reset(new NativeImplementation(mQueue, path)); } From 3015c9dd96c018ec5e47f90c3b7816f3504277d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Tue, 12 Sep 2017 13:40:48 +0200 Subject: [PATCH 05/37] FIX: usage of condition variable --- includes/osx/RunLoop.h | 3 ++- src/osx/RunLoop.cpp | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/includes/osx/RunLoop.h b/includes/osx/RunLoop.h index 5395d357..0ea962a9 100644 --- a/includes/osx/RunLoop.h +++ b/includes/osx/RunLoop.h @@ -38,8 +38,9 @@ class RunLoop { std::string mPath; CFRunLoopRef mRunLoop; pthread_t mRunLoopThread; - std::condition_variable mReadyForCleanup; + std::condition_variable mReadyForCleanupCond; std::mutex mReadyForCleanupMutex; + bool mReadyForCleanup; bool mStarted; }; diff --git a/src/osx/RunLoop.cpp b/src/osx/RunLoop.cpp index 023acecd..d22140fc 100644 --- a/src/osx/RunLoop.cpp +++ b/src/osx/RunLoop.cpp @@ -10,6 +10,7 @@ RunLoop::RunLoop(FSEventsService *eventsService, std::string path): mExited(false), mPath(path), mRunLoop(NULL), + mReadyForCleanup(false), mStarted(false) { mStarted = !pthread_create( &mRunLoopThread, @@ -28,8 +29,12 @@ RunLoop::~RunLoop() { return; } - std::unique_lock lk(mReadyForCleanupMutex); - mReadyForCleanup.wait(lk); + { + std::unique_lock lk(mReadyForCleanupMutex); + while (!mReadyForCleanup) { + mReadyForCleanupCond.wait(lk); + } + } CFRunLoopStop(mRunLoop); @@ -53,9 +58,13 @@ void RunLoop::work() { mRunLoop = CFRunLoopGetCurrent(); - __block auto *runLoopHasStarted = &mReadyForCleanup; + __block auto *runLoopHasStartedMutex = &mReadyForCleanupMutex; + __block auto *runLoopHasStartedCond = &mReadyForCleanupCond; + __block auto *runLoopHasStarted = &mReadyForCleanup; CFRunLoopPerformBlock(mRunLoop, kCFRunLoopDefaultMode, ^ { - runLoopHasStarted->notify_all(); + std::unique_lock lk(*runLoopHasStartedMutex); + *runLoopHasStarted = true; + runLoopHasStartedCond->notify_all(); }); CFRunLoopWakeUp(mRunLoop); From 5fc4dbd4f96c02abb0512650a89c657b4dfdeb39 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Tue, 12 Sep 2017 13:50:08 +0200 Subject: [PATCH 06/37] use the javascript tests with windows --- includes/NativeInterface.h | 6 +++--- includes/Queue.h | 2 +- includes/win32/ReadLoop.h | 4 +++- src/CMakeLists.txt | 1 + src/Queue.cpp | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index cc6291d4..6c7f69d3 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -1,9 +1,6 @@ #ifndef NSFW_NATIVE_INTERFACE_H #define NSFW_NATIVE_INTERFACE_H -#include "Queue.h" -#include - #if defined(_WIN32) #include "../includes/win32/ReadLoop.h" using NativeImplementation = ReadLoop; @@ -15,6 +12,9 @@ using NativeImplementation = FSEventsService; using NativeImplementation = InotifyService; #endif +#include "Queue.h" +#include + class NativeInterface { public: NativeInterface(const std::string &path); diff --git a/includes/Queue.h b/includes/Queue.h index ebb4e82a..b2744edc 100644 --- a/includes/Queue.h +++ b/includes/Queue.h @@ -25,7 +25,7 @@ class EventQueue { public: void clear(); - int count(); + std::size_t count(); std::unique_ptr dequeue(); std::unique_ptr> dequeueAll(); void enqueue( diff --git a/includes/win32/ReadLoop.h b/includes/win32/ReadLoop.h index b2fae67d..02000b0d 100644 --- a/includes/win32/ReadLoop.h +++ b/includes/win32/ReadLoop.h @@ -1,13 +1,15 @@ #ifndef READ_LOOP_H #define READ_LOOP_H +#include #include +#include "ReadLoopRunner.h" + #include #include #include #include "../Queue.h" -#include "ReadLoopRunner.h" class ReadLoop { public: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9c64f56b..d390be1a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,6 +23,7 @@ if (UNIX) else (APPLE) message (STATUS "compiling linux specific file system service") set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} + Lock.cpp linux/InotifyEventLoop.cpp linux/InotifyService.cpp linux/InotifyTree.cpp diff --git a/src/Queue.cpp b/src/Queue.cpp index 98ea61f2..5858145c 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -7,7 +7,7 @@ void EventQueue::clear() { queue.clear(); } -int EventQueue::count() { +std::size_t EventQueue::count() { std::lock_guard lock(mutex); return queue.size(); } From 87b2e5950f4f5c93a8fda07b795c22a1f1b9f284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Tue, 12 Sep 2017 14:01:02 +0200 Subject: [PATCH 07/37] remove dependency to OpenPA --- binding.gyp | 19 - openpa/CHANGELOG | 191 -------- openpa/COPYRIGHT | 38 -- openpa/README | 103 ---- openpa/VERSION | 21 - openpa/openpa.gyp | 77 --- openpa/src/opa_config.h | 162 ------- openpa/src/opa_primitives.c | 38 -- openpa/src/opa_primitives.h | 160 ------- openpa/src/opa_queue.c | 40 -- openpa/src/opa_queue.h | 286 ----------- openpa/src/opa_util.h | 27 -- openpa/src/primitives/opa_by_lock.h | 205 -------- openpa/src/primitives/opa_emulated.h | 192 -------- openpa/src/primitives/opa_gcc_ia64.h | 216 --------- openpa/src/primitives/opa_gcc_intel_32_64.h | 15 - .../primitives/opa_gcc_intel_32_64_barrier.h | 25 - .../src/primitives/opa_gcc_intel_32_64_ops.h | 173 ------- .../src/primitives/opa_gcc_intel_32_64_p3.h | 15 - .../opa_gcc_intel_32_64_p3barrier.h | 34 -- openpa/src/primitives/opa_gcc_intrinsics.h | 129 ----- openpa/src/primitives/opa_gcc_ppc.h | 180 ------- openpa/src/primitives/opa_gcc_sicortex.h | 443 ------------------ openpa/src/primitives/opa_nt_intrinsics.h | 153 ------ openpa/src/primitives/opa_sun_atomic_ops.h | 135 ------ openpa/src/primitives/opa_unsafe.h | 152 ------ package.json | 1 - 27 files changed, 3230 deletions(-) delete mode 100644 openpa/CHANGELOG delete mode 100644 openpa/COPYRIGHT delete mode 100644 openpa/README delete mode 100644 openpa/VERSION delete mode 100644 openpa/openpa.gyp delete mode 100644 openpa/src/opa_config.h delete mode 100644 openpa/src/opa_primitives.c delete mode 100644 openpa/src/opa_primitives.h delete mode 100644 openpa/src/opa_queue.c delete mode 100644 openpa/src/opa_queue.h delete mode 100644 openpa/src/opa_util.h delete mode 100644 openpa/src/primitives/opa_by_lock.h delete mode 100644 openpa/src/primitives/opa_emulated.h delete mode 100644 openpa/src/primitives/opa_gcc_ia64.h delete mode 100644 openpa/src/primitives/opa_gcc_intel_32_64.h delete mode 100644 openpa/src/primitives/opa_gcc_intel_32_64_barrier.h delete mode 100644 openpa/src/primitives/opa_gcc_intel_32_64_ops.h delete mode 100644 openpa/src/primitives/opa_gcc_intel_32_64_p3.h delete mode 100644 openpa/src/primitives/opa_gcc_intel_32_64_p3barrier.h delete mode 100644 openpa/src/primitives/opa_gcc_intrinsics.h delete mode 100644 openpa/src/primitives/opa_gcc_ppc.h delete mode 100644 openpa/src/primitives/opa_gcc_sicortex.h delete mode 100644 openpa/src/primitives/opa_nt_intrinsics.h delete mode 100644 openpa/src/primitives/opa_sun_atomic_ops.h delete mode 100644 openpa/src/primitives/opa_unsafe.h diff --git a/binding.gyp b/binding.gyp index be587152..88022216 100644 --- a/binding.gyp +++ b/binding.gyp @@ -2,10 +2,6 @@ "targets": [{ "target_name": "nsfw", - "dependencies": [ - "openpa/openpa.gyp:openpa" - ], - "sources": [ "src/NSFW.cpp", "src/Queue.cpp", @@ -73,10 +69,6 @@ ] }], ["OS=='win'", { - "defines": [ - "OPA_HAVE_NT_INTRINSICS=1", - "_opa_inline=__inline" - ], "conditions": [ ["target_arch=='x64'", { "VCLibrarianTool": { @@ -95,22 +87,11 @@ }], ["OS=='mac' or OS=='linux'", { "defines": [ - "OPA_HAVE_GCC_INTRINSIC_ATOMICS=1", "HAVE_STDDEF_H=1", "HAVE_STDLIB_H=1", "HAVE_UNISTD_H=1" ] }], - ["target_arch=='x64' or target_arch=='arm64'", { - "defines": [ - "OPA_SIZEOF_VOID_P=8" - ] - }], - ["target_arch=='ia32' or target_arch=='armv7'", { - "defines": [ - "OPA_SIZEOF_VOID_P=4" - ] - }] ], }] } diff --git a/openpa/CHANGELOG b/openpa/CHANGELOG deleted file mode 100644 index e67db0a1..00000000 --- a/openpa/CHANGELOG +++ /dev/null @@ -1,191 +0,0 @@ -============= -OpenPA v1.0.4 -============= -Major Changes: - * native ARM (v7 and higher) support - * numerous memory barrier placement improvements in queue and test code - * x86 memory barrier improvements, including for Intel MIC - * numerous build system improvements - -Individual Change Summary By Developer: - -Dave Goodell (17): - * [svn-r125] fix quoting in configure macros to avoid ac-2.68 "no - AC_LANG_SOURCE" warnings - * [svn-r126] OPA build system fixes - * [svn-r129] fix and optimize GCC+x86/x86_64 memory barriers - * [svn-r130] improve AC_ARG_WITH m4 quoting - * [svn-r131] do not install the README - * [svn-r132] store-release/load-acquire functionality - * [svn-r133] add a "coverage" target to keep MPICH2's recursive "make - coverage" happy - * [svn-r134] support automatic dependency tracking for pgcc - * [svn-r137] improve opa_queue.h comments - * [svn-r138] add read barrier to opa_queue.h - * [svn-r140] build-sys: fixes for automake-1.12 compatibility - * [svn-r142] pkg-config: unconditionally add -lopa to Libs - * [svn-r147] Revert r145: "Since we now depend on automake-1.12.3 - [...]" - * [svn-r149] tweak intel header includes - * [svn-r152] fix incorrect PPC LL/SC comment - * add .gitignore file - * bump version number to 1.0.4 for a new release - -Kazutomo Yoshii (1): - * [svn-r151] added preliminary ARM support. only tested it on panda - board(ARMv7) and gcc 4.6.3 - -Neil Fortner (3): - * [svn-r136] Fix problem in queue that manifested in an occasional - infinite loop on POWER7 machines. Added a write barrier to - OPA_Queue_enqueue() after the invocation of OPA_SHM_SET_REL_NULL to - prevent the "next" pointer from being set to NULL after the element - was enqueued (and possibly modified by another thread). - * [svn-r139] Add memory barriers to LL/SC ABA tests. Should fix - occasional failure on POWER7 systems. - * Add more memory barriers to LL/SC ABA tests. This should fix the - recent errors on POWER7. - -Pavan Balaji (9): - * [svn-r127] Some x86 processors don't seem to like sfence either. - Making the write barrier even more stringent by forcing it to do a - full barrier. - * [svn-r128] Forgot to update the comment when we fixed the sfence - problem in r127. - * [svn-r135] configure.in --> configure.ac to match the naming - convention of newer autotools. - * [svn-r143] Propagate autoreconf errors upstream. - * [svn-r144] Remove internally maintained pgcc patch since - automake-1.12.3 fixes this. - * [svn-r145] Since we now depend on automake-1.12.3, we no longer need - to check if AM_PROG_AR is available. - * [svn-r146] svn:ignore. - * [svn-r148] svn:ignores. - * [svn-r150] IBM-contributed patch: Added support for non-versioned - libraries - -William Gropp (1): - * [svn-r141] Added missing svn:ignore entries - - -============= -OpenPA v1.0.3 -============= -Major Changes: - * Libtool shared library support for OPA, avoiding shared/static mixed linking problems on some platforms. - * The build system should now work more portably with broken/exotic shells. - * pkg-config support - * Emulated atomics can now be detected by the presence of an OPA_EXPLICIT_EMULATION #define in opa_config.h. - * PPC types are now correctly aligned to 8 bytes instead of 16 bytes. - * many more tests for "make check", fixed missing memory barriers in one test - -Individual Change Summary By Developer: - -balaji (2): - * Shared-library support for OPA. - * Initial draft of shared library versioning support for OPA. - -buntinas (1): - * added svn:ignores - -fortnern (3): - * Added tests for OPA_swap_int and OPA_swap_ptr. Other minor cleanup in the test suite. - * Added tests for queue, and fixed bugs in queue implementation. Other misc cleanup. - * Add tests for OPA_LL_int, OPA_SC_int, OPA_LL_ptr and OPA_SC_ptr (skipped test 4 from plan). - -goodell (16): - * Fix const usage in OPA_load_xxx - * Change PPC type sizes back to 8-bytes. - * Add pkg-config support. - * Add configure check for stddef.h, fixes ticket #15. - * AC_DEFINE OPA_EXPLICIT_EMULATION upon --with-atomic-primitives=no - * remove nonsense "bit" word in configure message - * fix AX_PREFIX_CONFIG_H to work with dash - * redo r113 with printf to be more portable - * fix AX_PREFIX_CONFIG sed issue with AS_ECHO this time - * include VERSION in EXTRA_DIST to avoid "make distcheck" errors - * add "color-tests" option for automake - * use "silent rules" by default, like other MPICH2 projects - * ensure that config.status has a dependency on the VERSION file - * make the age=0 field explicit in the VERSION file - * update CHANGELOG, etc. for the upcoming 1.0.3 release - * add missing memory barriers to test_primitive.c's stack tests - -jayesh (1): - * Fixing the type casts of atomic func params for 64-bit builds on windows - - -============= -OpenPA v1.0.2 -============= -Major Changes: - * Add support for 64-bit PPC. - * Static initializer macros for OPA types. - -Individual Change Summary By Developer: - -balaji (1): - * Fix pthread_mutex usage for inter-process shared memory regions. - -buntinas (1): - * added OPA typedef for pthread_mutex_t - -fortnern (4): - * Add more tests for compare-and-swap. - * Add integer compare-and-swap fairness test. - * Add pointer version of compare-and-swap fairness test. - * Added configure test for pthread_yield. - -goodell (6): - * Fix bad include guard in the opa_by_lock.h header. - * Add new "unsafe" primitives. Also minor updates to the docs. - * Add support for 64-bit PPC. - * Update README to reflect 64-bit PPC support. - * Add static initializer macros for OPA_int_t/OPA_ptr_t. - * Actually include the COPYRIGHT and CHANGELOG files in the distribution. - -jayesh (1): - * Fixed compiler warnings in NT intrinsics. Now type casting the arguments to NT intrinsics correctly - - -============= -OpenPA v1.0.1 -============= - -Major Changes: - * Fix for x86/x86_64 machines that don't support SSE2 and therefore lfence/mfence. - * Fix major bug in SC on PPC. Fixes ticket #8. - -Individual Change Summary By Developer: - -buntinas (2): - * Work around PGI compiler bug by rearranging input parameters - * check for pre-Pentium 4 machines which don't support mfence and lfence - -fortnern (2): - * Add/improve tests for fetch_and_{incr,decr}_int and fetch_and_add_int. - * Add some tests for OPA_cas_int. Also fix a bug in the fetch and * tests. - -goodell (6): - * Remove erroneous "C" mode from some emacs modelines. - * Fix Darius' email address in the COPYRIGHT file. - * Update the README version number to match configure.in. - * Add an "all-executable" target to support parallel make in MPICH2. - * Fix major bug in SC on PPC. Fixes ticket #8. - * Add new header files to the appropriate automake variables. - - -============= -OpenPA v1.0.0 -============= - -everyone: - * This is the initial release of OpenPA. - * support for GCC + x86/x86_64 - * support for GCC + IA64 - * support for GCC intrinsic atomic operations - * support for GCC + PPC450 (IBM Blue Gene/P compute nodes) - * support for GCC + MIPS (specifically, SiCortex compute nodes) - * support for SUN Solaris' atomic operations library - * support for Windows NT intrinsic atomic operations - * Includes a partially completed test suite covering a substantial portion of the API. diff --git a/openpa/COPYRIGHT b/openpa/COPYRIGHT deleted file mode 100644 index 6f6c81ec..00000000 --- a/openpa/COPYRIGHT +++ /dev/null @@ -1,38 +0,0 @@ - - COPYRIGHT - -The following is a notice of limited availability of the code, and disclaimer -which must be included in the prologue of the code and in all source listings -of the code. - -Copyright Notice - + 2008 University of Chicago - -Permission is hereby granted to use, reproduce, prepare derivative works, and -to redistribute to others. This software was authored by: - -Argonne National Laboratory Group -D. Goodell: (630) 252-6082; FAX: (630) 252-5986; e-mail: goodell@mcs.anl.gov -D. Buntinas: (630) 252-7928; FAX: (630) 252-5986; e-mail: buntinas@mcs.anl.gov -Mathematics and Computer Science Division -Argonne National Laboratory, Argonne IL 60439 - - - GOVERNMENT LICENSE - -Portions of this material resulted from work developed under a U.S. -Government Contract and are subject to the following license: the Government -is granted for itself and others acting on its behalf a paid-up, nonexclusive, -irrevocable worldwide license in this computer software to reproduce, prepare -derivative works, and perform publicly and display publicly. - - DISCLAIMER - -This computer code material was prepared, in part, as an account of work -sponsored by an agency of the United States Government. Neither the United -States, nor the University of Chicago, nor any of their employees, makes any -warranty express or implied, or assumes any legal liability or responsibility -for the accuracy, completeness, or usefulness of any information, apparatus, -product, or process disclosed, or represents that its use would not infringe -privately owned rights. - diff --git a/openpa/README b/openpa/README deleted file mode 100644 index bf6ea8be..00000000 --- a/openpa/README +++ /dev/null @@ -1,103 +0,0 @@ -OpenPA v1.0.4 -------------- - -The goal of this project is to provide an open source, highly-portable -library that provides atomic primitives (and related constructs) for -high performance, concurrent software. This project is a collaboration -between the Mathematics and Computer Science (MCS) division at Argonne -National Laboratory (ANL) and the HDF Group. The code was originally -derived from work on the MPICH2 project. - -Project documentation and bug tracking can be found at: - - https://trac.mcs.anl.gov/projects/openpa/ - -If you would like to email questions or discuss topics related to OpenPA -you can send mail to opa-discuss@lists.mcs.anl.gov. - - -Building --------- - -If you checked out the project from source control then you will need to -generate configure files and makefiles with autogen.sh: - -% ./autogen.sh - -Otherwise, the build procedure is basically the same as any other -autoconfiscated software: - -% ./configure [configure_args] -% make -% make install - -OpenPA does support Microsoft Windows but the build system -infrastructure is unfortunately not yet in place for general use. - - -Supported Platforms -------------------- - -The following header files in the src/primitives directory support the -listed platforms: -opa_gcc_ia64.h - GCC on Intel's IA64 (Itanium) architecture -opa_gcc_intel_32_64.h - GCC (and some GCC-like compilers) on x86 and - x86_64 architectures -opa_gcc_intrinsics.h - GCC on many other platforms. These use compiler - intrinsics which are not always implemented on - every platform -opa_gcc_ppc.h - GCC and IBM's XLC on PowerPC 4xx and 970 systems. - Specifically, this supports the modified-PPC440 - processor in IBM's Blue Gene/P supercomputers and most - 64-bit PPC machines such as BG/P login nodes and G5 - Macs. -opa_gcc_sicortex.h - GCC on SiCortex machines. This is a MIPS 5K based - architecture, so it may work on similar platforms. -opa_gcc_arm.h - GCC on ARMv7 (and later) -opa_nt_intrinsics.h - Windows support. These use compiler intrinsics - available in Microsoft's Visual Studio compiler. -opa_sun_atomic_ops.h - Solaris support. This uses Solaris' built-in - atomic operations library. Tested on a Niagara - (T5240) machine with Solaris (s10s_u4wos_12b). - -We also support two pseudo-platforms: - -opa_by_lock.h - Used when you specify "--with-atomic-primitives=no" or when - auto-detecting the primitive implementation and lock-based fall - back is selected. This uses pthread mutexes to emulate the - atomic behavior. This option typically has dramatically slower - performance on most platforms where native primitives are - available. You should usually only use it for testing or on - platforms where pthreads are available but no native primitives - are currently implemented. The library initialization function - *must* be called when using this primitives implementation. -opa_unsafe.h - Used when you specify "--with-atomic-primitives=unsafe". This - can be used to improve performance in code that uses OPA already - and is conditionally compiled to be single-threaded without - having to modify said code. It is also potentially useful for - meta-testing to ensure that any threading tests you might have - will catch bugs when you have a broken atomics implementation. - The OPA test suite itself fails spectacularly when compiled this - way. This header can also be used by defining the preprocessor - macro OPA_USE_UNSAFE_PRIMITIVES prior to including - opa_primitives.h. - -Known Issues ------------- - -* One known issue is that the gcc atomic intrinsics aren't supported by - compilers prior to GCC 4.1. In particular the default Mac OS X compiler is - gcc 4.0.1 so these result in a linker error when using this set of - primitives. The good news is that on OSX/Intel we use native inline - assembly anyway, so this isn't a big problem. -* The PGI compilers currently are not supported. There is at least one known - bug in the PGI compiler's handling of of inline assembly for which we are - awaiting a fix from PGI. Once a fixed version of the compiler is available - this issue should be rectified in an upcoming release. -* As mentioned earlier, Windows is supported but the build system is not - yet present. -* We've had reports of trouble with older IA64 machines running GCC 3.2.2. - Unfortunately we don't have access to a machine with this configuration so we - have been unable to debug and fix the problem. Patches and detailed bug - reports on this issue are very welcome. - diff --git a/openpa/VERSION b/openpa/VERSION deleted file mode 100644 index 356af1c3..00000000 --- a/openpa/VERSION +++ /dev/null @@ -1,21 +0,0 @@ -# -*- Mode: c-basic-offset:4 ; indent-tabs-mode:nil ; -*- -# -# (C) 2008 by Argonne National Laboratory. -# See COPYRIGHT in top-level directory. -# - -# For libtool ABI versioning rules see: -# http://www.gnu.org/software/libtool/manual/libtool.html#Updating-version-info - -# 1. If the library source code has changed at all since the last -# update, then increment revision (`c:r:a' becomes `c:r+1:a'). -# -# 2. If any interfaces have been added, removed, or changed since -# the last update, increment current, and set revision to 0. -# -# 3. If any interfaces have been added since the last public -# release, then increment age. -# -# 4. If any interfaces have been removed since the last public -# release, then set age to 0. -libopa_so_version=1:0:0 diff --git a/openpa/openpa.gyp b/openpa/openpa.gyp deleted file mode 100644 index 0ceeb4bc..00000000 --- a/openpa/openpa.gyp +++ /dev/null @@ -1,77 +0,0 @@ -{ - "targets": [ - { - "target_name": "openpa", - "target_arch%": "x86", - "type": "static_library", - - "sources": [ - "src/primitives/opa_emulated.h", - "src/primitives/opa_nt_intrinics.h", - "src/primitives/opa_gcc_intrinsics.h", - "src/opa_config.h", - "src/opa_primitives.h", - "src/opa_primitives.c", - "src/opa_util.h", - "src/opa_queue.h", - "src/opa_queue.c" - ], - "include_dirs": [ - "src/primitves", - "src" - ], - "conditions": [ - ["OS=='win'", { - "sources": [ - "src/primitives/opa_nt_intrinsics.h" - ], - "defines": [ - "OPA_HAVE_NT_INTRINSICS=1", - "_opa_inline=__inline" - ], - "conditions": [ - ["target_arch=='x64'", { - "VCLibrarianTool": { - "AdditionalOptions": [ - "/MACHINE:X64", - ], - }, - }, { - "VCLibrarianTool": { - "AdditionalOptions": [ - "/MACHINE:x86", - ], - }, - }], - ] - }], - ["OS=='mac' or OS=='linux'", { - "sources": [ - "src/primitives/opa_gcc_intrinsics.h" - ], - "defines": [ - "OPA_HAVE_GCC_INTRINSIC_ATOMICS=1", - "HAVE_STDDEF_H=1", - "HAVE_STDLIB_H=1", - "HAVE_UNISTD_H=1" - ] - }], - ["target_arch=='x64' or target_arch=='arm64'", { - "defines": [ - "OPA_SIZEOF_VOID_P=8" - ] - }], - ["target_arch=='ia32' or target_arch=='armv7'", { - "defines": [ - "OPA_SIZEOF_VOID_P=4" - ] - }] - ], - "direct_dependent_settings": { - "include_dirs": [ - "src" - ] - } - } - ] -} diff --git a/openpa/src/opa_config.h b/openpa/src/opa_config.h deleted file mode 100644 index d1685918..00000000 --- a/openpa/src/opa_config.h +++ /dev/null @@ -1,162 +0,0 @@ -/* src/config.h.in. Generated from configure.ac by autoheader. */ - -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - - -// /* define if lock-based emulation was explicitly requested at configure time -// via --with-atomic-primitives=no */ -// #undef EXPLICIT_EMULATION -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_ATOMIC_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_DLFCN_H -// -// /* define to 1 if we have support for gcc ARM atomics */ -// #undef HAVE_GCC_AND_ARM_ASM -// -// /* define to 1 if we have support for gcc ia64 primitives */ -// #undef HAVE_GCC_AND_IA64_ASM -// -// /* define to 1 if we have support for gcc PowerPC atomics */ -// #undef HAVE_GCC_AND_POWERPC_ASM -// -// /* define to 1 if we have support for gcc SiCortex atomics */ -// #undef HAVE_GCC_AND_SICORTEX_ASM -// -// /* Define if GNU __attribute__ is supported */ -// #undef HAVE_GCC_ATTRIBUTE -// -// /* define to 1 if we have support for gcc atomic intrinsics */ -// #undef HAVE_GCC_INTRINSIC_ATOMICS -// -// /* define to 1 if we have support for gcc x86/x86_64 primitives */ -// #undef HAVE_GCC_X86_32_64 -// -// /* define to 1 if we have support for gcc x86 primitives for pre-Pentium 4 */ -// #undef HAVE_GCC_X86_32_64_P3 -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_INTRIN_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_INTTYPES_H -// -// /* Define to 1 if you have the `pthread' library (-lpthread). */ -// #undef HAVE_LIBPTHREAD -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_MEMORY_H -// -// /* define to 1 if we have support for Windows NT atomic intrinsics */ -// #undef HAVE_NT_INTRINSICS -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_PTHREAD_H -// -// /* Define to 1 if you have the `pthread_yield' function. */ -// #undef HAVE_PTHREAD_YIELD -// -// /* Define to 1 if you have the `sched_yield' function. */ -// #undef HAVE_SCHED_YIELD -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_STDDEF_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_STDINT_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_STDLIB_H -// -// /* Define if strict checking of atomic operation fairness is desired */ -// #undef HAVE_STRICT_FAIRNESS_CHECKS -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_STRINGS_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_STRING_H -// -// /* define to 1 if we have support for Sun atomic operations library */ -// #undef HAVE_SUN_ATOMIC_OPS -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_SYS_STAT_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_SYS_TYPES_H -// -// /* Define to 1 if you have the header file. */ -// #undef HAVE_UNISTD_H -// -// /* Define to the sub-directory in which libtool stores uninstalled libraries. -// */ -// #undef LT_OBJDIR -// -// /* define to the maximum number of simultaneous threads */ -// #undef MAX_NTHREADS -// -// /* Define to 1 if assertions should be disabled. */ -// #undef NDEBUG -// -// /* Name of package */ -// #undef PACKAGE -// -// /* Define to the address where bug reports for this package should be sent. */ -// #undef PACKAGE_BUGREPORT -// -// /* Define to the full name of this package. */ -// #undef PACKAGE_NAME -// -// /* Define to the full name and version of this package. */ -// #undef PACKAGE_STRING -// -// /* Define to the one symbol short name of this package. */ -// #undef PACKAGE_TARNAME -// -// /* Define to the home page for this package. */ -// #undef PACKAGE_URL -// -// /* Define to the version of this package. */ -// #undef PACKAGE_VERSION -// -// /* The size of `int', as computed by sizeof. */ -// #undef SIZEOF_INT -// -// /* The size of `void *', as computed by sizeof. */ -// #undef SIZEOF_VOID_P -// -// /* Define to 1 if you have the ANSI C header files. */ -// #undef STDC_HEADERS -// -// /* define to 1 to force using lock-based atomic primitives */ -// #undef USE_LOCK_BASED_PRIMITIVES -// -// /* define to 1 if unsafe (non-atomic) primitives should be used */ -// #undef USE_UNSAFE_PRIMITIVES -// -// /* Version number of package */ -// #undef VERSION -// -// /* Define to empty if `const' does not conform to ANSI C. */ -// #undef const -// -// /* Define to the equivalent of the C99 'restrict' keyword, or to -// nothing if this is not supported. Do not define if restrict is -// supported directly. */ -// #undef restrict -// /* Work around a bug in Sun C++: it does not support _Restrict or -// __restrict__, even though the corresponding Sun C compiler ends up with -// "#define restrict _Restrict" or "#define restrict __restrict__" in the -// previous line. Perhaps some future version of Sun C++ will work with -// restrict; if so, hopefully it defines __RESTRICT like Sun C does. */ -// #if defined __SUNPRO_CC && !defined __RESTRICT -// # define _Restrict -// # define __restrict__ -// #endif diff --git a/openpa/src/opa_primitives.c b/openpa/src/opa_primitives.c deleted file mode 100644 index 45a239ef..00000000 --- a/openpa/src/opa_primitives.c +++ /dev/null @@ -1,38 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#include "opa_config.h" - -/* FIXME For now we rely on pthreads for our IPC locks. This is fairly - portable, although it is obviously not 100% portable. We need to - figure out how to support other threading packages and lock - implementations, such as the BG/P lockbox. */ -#if defined(OPA_HAVE_PTHREAD_H) -#include -#include - -pthread_mutex_t *OPA_emulation_lock = NULL; - -int OPA_Interprocess_lock_init(OPA_emulation_ipl_t *shm_lock, int isLeader) -{ - int mpi_errno = 0; /*MPI_SUCCESS*/ - pthread_mutexattr_t attr; - OPA_emulation_lock = shm_lock; - - if (isLeader) { - /* Set the mutex attributes to work correctly on inter-process - * shared memory as well. This is required for some compilers - * (such as SUN Studio) that don't enable it by default. */ - if (pthread_mutexattr_init(&attr) || - pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) || - pthread_mutex_init(OPA_emulation_lock, &attr)) - mpi_errno = 16; /* MPI_ERR_INTERN */ - } - - return mpi_errno; -} -#endif /* defined(OPA_HAVE_PTHREAD_H) */ - diff --git a/openpa/src/opa_primitives.h b/openpa/src/opa_primitives.h deleted file mode 100644 index 507c96e4..00000000 --- a/openpa/src/opa_primitives.h +++ /dev/null @@ -1,160 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_PRIMITIVES_H_INCLUDED -#define OPA_PRIMITIVES_H_INCLUDED - -#include "opa_config.h" -#include "opa_util.h" - -/* Clean up some of the opa_config.h definitions. This is a consequence - of using the AX_PREFIX_CONFIG_H macro. Autoconf won't define inline - or _opa_inline when a real "inline" works. Since we are - unconditionally using _opa_inline we must define it ourselves in this - case. */ -#ifndef _opa_inline -#define _opa_inline inline -#endif -#ifndef _opa_restrict -#define _opa_restrict restrict -#endif -#ifndef _opa_const -#define _opa_const const -#endif - -/* - Primitive atomic functions - -------------------------- - - The included file is responsible for defining the types of OPA_int_t and - OPA_ptr_t as well as a set of functions for operating on these - types. If you have the following declaration: - - OPA_int_t atomic_var; - - Then in order for the emulation functions to compile, the underlying value of - atomic_var should be accessible via: - - atomic_var.v; - - The same goes for OPA_ptr_t. - - The atomic functions that must be ported for each architecture: - - static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr); - static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val); - static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr); - static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val); - - static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val); - static _opa_inline void OPA_incr_int(OPA_int_t *ptr); - static _opa_inline void OPA_decr_int(OPA_int_t *ptr); - - static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr); - static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val); - static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr); - static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr); - - static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv); - static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv); - - static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val); - static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val); - - // (the memory barriers may be macros instead of inline functions) - static _opa_inline void OPA_write_barrier(); - static _opa_inline void OPA_read_barrier(); - static _opa_inline void OPA_read_write_barrier(); - - // Loads and stores with memory ordering guarantees (also may be macros): - static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr); - static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val); - static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr); - static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val); - - // Compiler barrier, only preventing compiler reordering, *not* CPU - // reordering (may be a macro): - static _opa_inline void OPA_compiler_barrier(); - - // The following need to be ported only for architectures supporting LL/SC: - static _opa_inline int OPA_LL_int(OPA_int_t *ptr); - static _opa_inline int OPA_SC_int(OPA_int_t *ptr, int val); - static _opa_inline void *OPA_LL_ptr(OPA_ptr_t *ptr); - static _opa_inline int OPA_SC_ptr(OPA_ptr_t *ptr, void *val); - - // Additionally, the following initializer macros must be defined: - #define OPA_INT_T_INITIALIZER(val_) ... - #define OPA_PTR_T_INITIALIZER(val_) ... - - // They should be useable as C89 static initializers like so: - - struct { int x; OPA_int_t y; OPA_ptr_t z; } foo = { 35, OPA_INT_T_INITIALIZER(1), OPA_PTR_T_INITIALIZER(NULL) }; -*/ - -/* Include the appropriate header for the architecture */ -#if defined(OPA_USE_UNSAFE_PRIMITIVES) -/* comes first to permit user overrides in the style of NDEBUG */ -#include "primitives/opa_unsafe.h" -#elif defined(OPA_HAVE_GCC_AND_POWERPC_ASM) -#include "primitives/opa_gcc_ppc.h" -#elif defined(OPA_HAVE_GCC_AND_ARM_ASM) -#include "primitives/opa_gcc_arm.h" -#elif defined(OPA_HAVE_GCC_X86_32_64) -#include "primitives/opa_gcc_intel_32_64.h" -#elif defined(OPA_HAVE_GCC_X86_32_64_P3) -#include "primitives/opa_gcc_intel_32_64_p3.h" -#elif defined(OPA_HAVE_GCC_AND_IA64_ASM) -#include "primitives/opa_gcc_ia64.h" -#elif defined(OPA_HAVE_GCC_AND_SICORTEX_ASM) -#include "primitives/opa_gcc_sicortex.h" -#elif defined(OPA_HAVE_GCC_INTRINSIC_ATOMICS) -#include "primitives/opa_gcc_intrinsics.h" -#elif defined(OPA_HAVE_SUN_ATOMIC_OPS) -#include "primitives/opa_sun_atomic_ops.h" -#elif defined(OPA_HAVE_NT_INTRINSICS) -#include "primitives/opa_nt_intrinsics.h" -#elif defined(OPA_USE_LOCK_BASED_PRIMITIVES) -#include "primitives/opa_by_lock.h" -#else -#error no primitives implementation specified -#endif - -/* - This routine is needed because the MPIU_THREAD_XXX_CS_{ENTER,EXIT} macros do - not provide synchronization across multiple processes, only across multiple - threads within a process. In order to safely emulate atomic operations on a - shared memory region, we need a shared memory backed lock mechanism. - - This routine must be called by any subsystem that intends to use the atomic - abstractions if the cpp directive OPA_USE_LOCK_BASED_PRIMITIVES is defined. It must - be called exactly once by _all_ processes, not just a single leader. This - function will initialize the contents of the lock variable if the caller - specifies (isLeader==true). Note that multiple initialization is forbidden - by several lock implementations, especially pthreads. - - Inputs: - shm_lock - A pointer to an allocated piece of shared memory that can hold - a mutex (e.g., pthread_mutex_t). This is not portable to - non-pthreads systems at this time. - isLeader - This boolean value should be set to true for exactly one - thread/process of the group that calls this function. -*/ -#if defined(OPA_HAVE_PTHREAD_H) -# include -typedef pthread_mutex_t OPA_emulation_ipl_t; -int OPA_Interprocess_lock_init(OPA_emulation_ipl_t *shm_lock, int isLeader); -#endif - - -/* FIXME This should probably be pushed down into the platform-specific headers. */ -#if defined(OPA_HAVE_SCHED_YIELD) -# include -# define OPA_busy_wait() sched_yield() -#else -# define OPA_busy_wait() do { } while (0) -#endif - -#endif /* defined(OPA_PRIMITIVES_H_INCLUDED) */ diff --git a/openpa/src/opa_queue.c b/openpa/src/opa_queue.c deleted file mode 100644 index 347f4289..00000000 --- a/openpa/src/opa_queue.c +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -/* FIXME remove MPI error code references in a meaningful way */ - -#include "opa_queue.h" - -char *OPA_Shm_asymm_base_addr = (char *)(OPA_SHM_ASYMM_NULL_VAL); - -int OPA_Shm_asymm_init(char *base) -{ - int mpi_errno = 0/*MPI_SUCCESS*/; - - if (OPA_Shm_asymm_base_addr != (char *)OPA_SHM_ASYMM_NULL_VAL) { - /* the _base_addr has already been initialized */ - mpi_errno = 16/*MPI_ERR_INTERN*/; - goto fn_exit; - } - - OPA_Shm_asymm_base_addr = base; - -fn_exit: - return mpi_errno; -} - -void OPA_Queue_init(OPA_Queue_info_t *qhead) -{ - OPA_SHM_SET_REL_NULL((qhead)->head); - OPA_SHM_SET_REL_NULL((qhead)->tail); - OPA_SHM_SET_REL_NULL((qhead)->shadow_head); -} - -void OPA_Queue_header_init(OPA_Queue_element_hdr_t *hdr) -{ - OPA_SHM_SET_REL_NULL(hdr->next); -} - diff --git a/openpa/src/opa_queue.h b/openpa/src/opa_queue.h deleted file mode 100644 index e86d4250..00000000 --- a/openpa/src/opa_queue.h +++ /dev/null @@ -1,286 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -/* Implements a fast, lockfree, multi-producer, single-consumer queue. It's - * important to note the *single-consumer* piece of this, since multithreaded - * consumption will surely lead to data corruption and/or other problems. */ - -#ifndef OPA_QUEUE_H_INCLUDED -#define OPA_QUEUE_H_INCLUDED - -#include "opa_primitives.h" -#ifdef HAVE_STDDEF_H -#include -#endif /* OPA_HAVE_STDDEF_H */ - -/* This value is used to indicate NULL in the OPA_Shm_asymm_base_addr - variable. It is non-zero because one of the likely base addresses is zero - (indicating memory is actually symmetrically mapped). The value 64 was used - because it is unlikely that mmap will choose an address this low for a - mapping. */ -/* XXX DJG TODO put some conditionally compiled error checking in that uses this */ -#define OPA_SHM_ASYMM_NULL_VAL 64 - -extern char *OPA_Shm_asymm_base_addr; - -/* Used to initialize the base address for relative pointers. This interface - assumes that there is only one shared memory segment. If this turns out to - not be the case in the future, we should probably add support for multiple - shm segments. - - This function will return an error if it has already been called. */ -int OPA_Shm_asymm_init(char *base); - -/* Relative addressing macros. These are for manipulating addresses relative - to the start of a shared memory region. */ -#define OPA_SHM_REL_NULL (0x0) -#define OPA_SHM_IS_REL_NULL(rel_ptr) (OPA_load_ptr(&(rel_ptr).offset) == OPA_SHM_REL_NULL) -#define OPA_SHM_SET_REL_NULL(rel_ptr) (OPA_store_ptr(&(rel_ptr).offset, OPA_SHM_REL_NULL)) -#define OPA_SHM_REL_ARE_EQUAL(rel_ptr1, rel_ptr2) \ - (OPA_load_ptr(&(rel_ptr1).offset) == OPA_load_ptr(&(rel_ptr2).offset)) - -/* This structure exists such that it is possible to expand the expressiveness - of a relative address at some point in the future. It also provides a - modicum of type safety to help prevent certain flavors of errors. - - For example, instead of referencing an offset from a global base address, it - might make sense for there to be multiple base addresses. These base - addresses could correspond to the start of a segment or region of shared - memory. This structure could store the segment number that is used to lookup - a base address in a non-shared table. Note that you would have to be very - careful about all of this because if you add the segment number as a separate - field you can no longer (compare and) swap a relative address atomically. So - you'll either have to shave bits from the pointer or make some sort of - requirement that relative addresses can only be swapped within the same - segment. */ -typedef struct OPA_Shm_rel_addr_t { - OPA_ptr_t offset; -} OPA_Shm_rel_addr_t; - -/* converts a relative pointer to an absolute pointer */ -static _opa_inline -void *OPA_Shm_rel_to_abs(OPA_Shm_rel_addr_t r) -{ - void *offset = OPA_load_ptr(&r.offset); - OPA_assert((size_t)OPA_Shm_asymm_base_addr != OPA_SHM_ASYMM_NULL_VAL); - return (void*)(OPA_Shm_asymm_base_addr + (size_t)offset); -} - -/* converts an absolute pointer to a relative pointer */ -static _opa_inline -OPA_Shm_rel_addr_t OPA_Shm_abs_to_rel(void *a) -{ - OPA_Shm_rel_addr_t ret; - - OPA_assert((size_t)OPA_Shm_asymm_base_addr != OPA_SHM_ASYMM_NULL_VAL); - OPA_store_ptr(&ret.offset, (void*)((size_t)a - (size_t)OPA_Shm_asymm_base_addr)); - return ret; -} - -static _opa_inline -OPA_Shm_rel_addr_t OPA_Shm_swap_rel(OPA_Shm_rel_addr_t *addr, OPA_Shm_rel_addr_t newv) { - OPA_Shm_rel_addr_t oldv; - OPA_store_ptr(&oldv.offset, OPA_swap_ptr(&addr->offset, OPA_load_ptr(&newv.offset))); - return oldv; -} - -/* Compare the relative pointer to (relative) null and swap if equal. Prevents - the guts of the _rel_addr_t abstraction from bleeding up into the - enqueue/dequeue operations. */ -static _opa_inline -OPA_Shm_rel_addr_t OPA_Shm_cas_rel_null(OPA_Shm_rel_addr_t *addr, OPA_Shm_rel_addr_t oldv) { - OPA_Shm_rel_addr_t prev; - OPA_store_ptr(&prev.offset, OPA_cas_ptr(&(addr->offset), OPA_load_ptr(&oldv.offset), (void*)OPA_SHM_REL_NULL)); - return prev; -} - -/* The shadow head pointer makes this queue implementation perform much better - than a standard queue. Unfortunately, it also makes it a bit non-intuitive - when read the code. The following is an excerpt from "Design and Evaluation - of Nemesis, a Scalable, Low-Latency, Message-Passing Communication - Subsystem" by D. Buntinas, G. Mercier, and W. Gropp that gives an - explanation: - - A process must access both the head and tail of the queue when it is - enqueuing an element on an empty queue or when it is dequeuing an element - that is the last element in the queue. In these cases, if the head and - tail were in the same cache line, only one L2 cache miss would be - encountered. If the queue has more elements in it, however, then the - enqueuer only needs to access the tail, and the dequeuer only needs to - access the head. If the head and tail were in the same cache line, then - there would be L2 misses encountered as a result of false sharing each - time a process enqueues an element after another has been dequeued from - the same queue, and vice versa. In this case it would be better if the - head and tail were in separate cache lines. - - Our solution is to put the head and tail in the same cache line and have a - shadow head pointer in a separate cache line. The shadow head is - initialized to NULL. The dequeuer uses the shadow head in place of the - real head except when the shadow head is NULL, meaning that the queue has - become empty. If the shadow head is NULL when the dequeuer tries to - dequeue, it checks the value of the real head. If the real head is not - NULL, meaning that an element has been enqueued on the queue since the - last time the queue became empty, the dequeuer initializes its shadow head - to the value of the real head and sets the real head to NULL. In this way, - only one L2 cache miss is encountered when enqueuing onto an empty queue - or dequeuing from a queue with one element. And because the tail and - shadow head are in separate cache lines, there are no L2 cache misses from - false sharing. - - We found that using a shadow head pointer reduced one-way latency by about - 200 ns on a dual 2 GHz Xeon node. -*/ - -/* Pick an arbitrary cacheline size for now, we can setup a mechanism to detect - it at build time later on. This should work well on most intel systems at - the very least. */ -#define OPA_QUEUE_CACHELINE_PADDING 128 - -/* All absolute and relative pointers point to the start of the enclosing element. */ -typedef struct OPA_Queue_info_t { - OPA_Shm_rel_addr_t head; /* holds the offset pointer, not the abs ptr */ - OPA_Shm_rel_addr_t tail; /* holds the offset pointer, not the abs ptr */ - char padding1[OPA_QUEUE_CACHELINE_PADDING-2*sizeof(OPA_Shm_rel_addr_t)]; - OPA_Shm_rel_addr_t shadow_head; /* holds the offset pointer, not the abs ptr */ - char padding2[OPA_QUEUE_CACHELINE_PADDING-sizeof(OPA_Shm_rel_addr_t)]; -} OPA_Queue_info_t; - -/* Using this header struct even though it's just one element gives us the - opportunity to vary the implementation more easily in the future without - updating all callers. */ -typedef struct OPA_Queue_element_hdr_t { - OPA_Shm_rel_addr_t next; /* holds the offset pointer, not the abs ptr */ -} OPA_Queue_element_hdr_t; - - -/* Used to initialize a queue structure. */ -void OPA_Queue_init(OPA_Queue_info_t *qhead); - -/* Used to initialize a queue element header. */ -void OPA_Queue_header_init(OPA_Queue_element_hdr_t *hdr); - -static _opa_inline -int OPA_Queue_is_empty(OPA_Queue_info_t *qhead) -{ - int __ret = 0; - if (OPA_SHM_IS_REL_NULL (qhead->shadow_head)) { - if (OPA_SHM_IS_REL_NULL(qhead->head)) { - __ret = 1; - } - else { - qhead->shadow_head = qhead->head; - OPA_SHM_SET_REL_NULL(qhead->head); /* reset it for next time */ - } - } - return __ret; -} - -/* Returns a pointer to the element at the head of the queue. The current - implementation of these queue algorithms imposes several notable - restrictions on the use of this function: - - The caller must currently hold the critical section, just as if you were - calling OPA_Queue_dequeue. Failure to do could easily produce incorrect - results. - - OPA_Queue_is_empty must be called on this queue prior to calling this - function. Furthermore, there must be no intervening calls to any - OPA_Queue_* functions (for this queue) between _is_empty and _peek_head. - Failure to do so will produce incorrect results. - - This operation is effectively the same as the dequeue operation (insofar as - it shares the same calling restrictions) but it does not disturb the actual - contents of the queue. */ -static _opa_inline -void *OPA_Queue_peek_head(OPA_Queue_info_t *qhead_ptr) -{ - OPA_assert(qhead_ptr != NULL); - - if (OPA_SHM_IS_REL_NULL(qhead_ptr->shadow_head)) { - return NULL; - } - return OPA_Shm_rel_to_abs(qhead_ptr->shadow_head); -} - -/* This macro atomically enqueues an element (elt for short) into the queue - indicated by qhead_ptr. You need to pass several arguments: - qhead_ptr - a pointer to a OPA_Queue_info_t structure to which the - element should be enqueued - elt_ptr - a pointer to an element structure type that is to be enqueued - elt_type - The base type of elt_ptr. That is, if elt_ptr is a - '(struct example_t *)' then elt_type is 'struct example_t'. - elt_hdr_field - This is the member name of elt_type that is a - OPA_Queue_element_hdr_t. - - This queue implementation is loosely modeled after the linked lists used in - the linux kernel. You put the list header structure inside of the client - structure, rather than the other way around. - */ -#define OPA_Queue_enqueue(qhead_ptr, elt_ptr, elt_type, elt_hdr_field) \ -do { \ - OPA_Shm_rel_addr_t r_prev; \ - OPA_Shm_rel_addr_t r_elt = OPA_Shm_abs_to_rel(elt_ptr); \ - \ - OPA_SHM_SET_REL_NULL((elt_ptr)->elt_hdr_field.next); \ - \ - OPA_write_barrier(); \ - \ - r_prev = OPA_Shm_swap_rel(&((qhead_ptr)->tail), r_elt); \ - \ - if (OPA_SHM_IS_REL_NULL(r_prev)) { \ - (qhead_ptr)->head = r_elt; \ - } \ - else { \ - elt_type *abs_prev = (elt_type *)OPA_Shm_rel_to_abs(r_prev); \ - abs_prev->elt_hdr_field.next = r_elt; \ - } \ -} while (0) - -/* This macro atomically dequeues an element (elt for short) from the queue - indicated by qhead_ptr. You need to pass several arguments: - qhead_ptr - a pointer to a OPA_Queue_info_t structure to which the - element should be dequeued - elt_ptr - A pointer to an element structure type that should be populated - with the dequeued value. Must be an lvalue. - elt_type - The base type of elt_ptr. That is, if elt_ptr is a - '(struct example_t *)' then elt_type is 'struct example_t'. - elt_hdr_field - This is the member name of elt_type that is a - OPA_Queue_element_hdr_t. - - This queue implementation is loosely modeled after the linked lists used in - the linux kernel. You put the list header structure inside of the client - structure, rather than the other way around. - - NOTE: you must *always* call _is_empty() prior to this function */ -#define OPA_Queue_dequeue(qhead_ptr, elt_ptr, elt_type, elt_hdr_field) \ -do { \ - elt_type *_e; \ - OPA_Shm_rel_addr_t _r_e; \ - \ - _r_e = (qhead_ptr)->shadow_head; \ - _e = (elt_type *)OPA_Shm_rel_to_abs(_r_e); \ - \ - if (!OPA_SHM_IS_REL_NULL(_e->elt_hdr_field.next)) { \ - (qhead_ptr)->shadow_head = _e->elt_hdr_field.next; \ - } \ - else { \ - OPA_Shm_rel_addr_t old_tail; \ - \ - OPA_SHM_SET_REL_NULL((qhead_ptr)->shadow_head); \ - \ - old_tail = OPA_Shm_cas_rel_null(&((qhead_ptr)->tail), _r_e); \ - \ - if (!OPA_SHM_REL_ARE_EQUAL(old_tail, _r_e)) { \ - while (OPA_SHM_IS_REL_NULL(_e->elt_hdr_field.next)) { \ - OPA_busy_wait(); \ - } \ - (qhead_ptr)->shadow_head = _e->elt_hdr_field.next; \ - } \ - } \ - OPA_SHM_SET_REL_NULL(_e->elt_hdr_field.next); \ - OPA_read_barrier(); \ - elt_ptr = _e; \ -} while (0) - -#endif /* OPA_QUEUE_H_INCLUDED */ diff --git a/openpa/src/opa_util.h b/openpa/src/opa_util.h deleted file mode 100644 index f6819ba1..00000000 --- a/openpa/src/opa_util.h +++ /dev/null @@ -1,27 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_UTIL_H_INCLUDED -#define OPA_UTIL_H_INCLUDED - -#define OPA_QUOTE(x_) OPA_QUOTE2(x_) -#define OPA_QUOTE2(x_) #x_ - -#if defined(OPA_HAVE_GCC_ATTRIBUTE) -# define OPA_ATTRIBUTE(x_) __attribute__ (x_) -#else -# define OPA_ATTRIBUTE(x_) -#endif - -/* FIXME this just needs a total rework in general with an OPA_NDEBUG or similar. */ -#define OPA_assert(expr_) do {} while (0) -#define OPA_assertp(expr_) do { if (!(expr_)) ++((int *)NULL) } while (0) /* SEGV intentionally */ - -/* A compile-time assertion macro. It should cause a compilation error if (expr_) is false. */ -#define OPA_COMPILE_TIME_ASSERT(expr_) \ - do { switch(0) { case 0: case (expr_): default: break; } } while (0) - -#endif /* OPA_UTIL_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_by_lock.h b/openpa/src/primitives/opa_by_lock.h deleted file mode 100644 index 2d7cd284..00000000 --- a/openpa/src/primitives/opa_by_lock.h +++ /dev/null @@ -1,205 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_BY_LOCK_H_INCLUDED -#define OPA_BY_LOCK_H_INCLUDED - -/* FIXME For now we rely on pthreads for our IPC locks. This is fairly - portable, although it is obviously not 100% portable. Some day when we - refactor the OPA_Process_locks code we should be able to use that again. */ -#if defined(OPA_HAVE_PTHREAD_H) -#include - -/* defined in opa_primitives.c */ -extern pthread_mutex_t *OPA_emulation_lock; - -/* FIXME these make less sense now that OPA is not inside of MPICH2. Is there a - simpler name/scheme that could be used here instead? [goodell@ 2009-02-19] */ -#define OPA_IPC_SINGLE_CS_ENTER(msg) \ - do { \ - OPA_assert(OPA_emulation_lock); \ - pthread_mutex_lock(OPA_emulation_lock); \ - } while (0) - -#define OPA_IPC_SINGLE_CS_EXIT(msg) \ - do { \ - OPA_assert(OPA_emulation_lock); \ - pthread_mutex_unlock(OPA_emulation_lock); \ - } while (0) - -typedef struct { volatile int v; } OPA_int_t; -typedef struct { int * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -/* - Emulated atomic primitives - -------------------------- - - These are versions of the atomic primitives that emulate the proper behavior - via the use of an inter-process lock. For more information on their - individual behavior, please see the comment on the corresponding top level - function. - - In general, these emulated primitives should _not_ be used. Most algorithms - can be more efficiently implemented by putting most or all of the algorithm - inside of a single critical section. These emulated primitives exist to - ensure that there is always a fallback if no machine-dependent version of a - particular operation has been defined. They also serve as a very readable - reference for the exact semantics of our OPA_* ops. -*/ - -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - int retval; - OPA_IPC_SINGLE_CS_ENTER("atomic_add"); - retval = ptr->v; - OPA_IPC_SINGLE_CS_EXIT("atomic_add"); - return retval; -} - -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - OPA_IPC_SINGLE_CS_ENTER("atomic_add"); - ptr->v = val; - OPA_IPC_SINGLE_CS_EXIT("atomic_add"); -} - -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - int *retval; - OPA_IPC_SINGLE_CS_ENTER("atomic_add"); - retval = ptr->v; - OPA_IPC_SINGLE_CS_EXIT("atomic_add"); - return retval; -} - -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - OPA_IPC_SINGLE_CS_ENTER("atomic_add"); - ptr->v = val; - OPA_IPC_SINGLE_CS_EXIT("atomic_add"); -} - -/* normal loads/stores are fully ordered, so just use them */ -#define OPA_load_acquire_int(ptr_) OPA_load_int((ptr_)) -#define OPA_store_release_int(ptr_,val_) OPA_store_int((ptr_),(val_)) -#define OPA_load_acquire_ptr(ptr_) OPA_load_ptr((ptr_)) -#define OPA_store_release_ptr(ptr_,val_) OPA_store_ptr((ptr_),(val_)) - -static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val) -{ - OPA_IPC_SINGLE_CS_ENTER("atomic_add"); - ptr->v += val; - OPA_IPC_SINGLE_CS_EXIT("atomic_add"); -} - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, int *oldv, int *newv) -{ - int *prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_cas"); - prev = ptr->v; - if (prev == oldv) { - ptr->v = newv; - } - OPA_IPC_SINGLE_CS_EXIT("atomic_cas"); - return prev; -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - int prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_cas"); - prev = ptr->v; - if (prev == oldv) { - ptr->v = newv; - } - OPA_IPC_SINGLE_CS_EXIT("atomic_cas"); - return prev; -} - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - int new_val; - OPA_IPC_SINGLE_CS_ENTER("atomic_decr_and_test"); - new_val = --(ptr->v); - OPA_IPC_SINGLE_CS_EXIT("atomic_decr_and_test"); - return (0 == new_val); -} - -static _opa_inline void OPA_decr_int(OPA_int_t *ptr) -{ - OPA_IPC_SINGLE_CS_ENTER("atomic_decr"); - --(ptr->v); - OPA_IPC_SINGLE_CS_EXIT("atomic_decr"); -} - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - int prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_fetch_and_add"); - prev = ptr->v; - ptr->v += val; - OPA_IPC_SINGLE_CS_EXIT("atomic_fetch_and_add"); - return prev; -} - -static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr) -{ - int prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_fetch_and_decr"); - prev = ptr->v; - --(ptr->v); - OPA_IPC_SINGLE_CS_EXIT("atomic_fetch_and_decr"); - return prev; -} - -static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr) -{ - int prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_fetch_and_incr"); - prev = ptr->v; - ++(ptr->v); - OPA_IPC_SINGLE_CS_EXIT("atomic_fetch_and_incr"); - return prev; -} - -static _opa_inline void OPA_incr_int(OPA_int_t *ptr) -{ - OPA_IPC_SINGLE_CS_ENTER("atomic_incr"); - ++(ptr->v); - OPA_IPC_SINGLE_CS_EXIT("atomic_incr"); -} - -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ - int *prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_swap_ptr"); - prev = ptr->v; - ptr->v = val; - OPA_IPC_SINGLE_CS_EXIT("atomic_swap_ptr"); - return prev; -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - int prev; - OPA_IPC_SINGLE_CS_ENTER("atomic_swap_int"); - prev = ptr->v; - ptr->v = val; - OPA_IPC_SINGLE_CS_EXIT("atomic_swap_int"); - return (int)prev; -} - -/* lock/unlock provides barrier */ -#define OPA_write_barrier() do {} while (0) -#define OPA_read_barrier() do {} while (0) -#define OPA_read_write_barrier() do {} while (0) - - -#endif /* defined(OPA_HAVE_PTHREAD_H) */ -#endif /* !defined(OPA_BY_LOCK_H_INCLUDED) */ diff --git a/openpa/src/primitives/opa_emulated.h b/openpa/src/primitives/opa_emulated.h deleted file mode 100644 index c526fe43..00000000 --- a/openpa/src/primitives/opa_emulated.h +++ /dev/null @@ -1,192 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_EMULATED_H_INCLUDED -#define OPA_EMULATED_H_INCLUDED - -/* Functions emulated using other atomics - - This header should be included at the bottom of any atomic - primitives header that needs to implement an atomic op in terms of - another atomic. -*/ - -/* Emulating using LL/SC */ -#if defined(OPA_LL_SC_SUPPORTED) -static _opa_inline int OPA_fetch_and_add_int_by_llsc(OPA_int_t *ptr, int val) -{ - int prev; - do { - prev = OPA_LL_int(ptr); - } while (!OPA_SC_int(ptr, prev + val)); - return prev; -} - - -static _opa_inline void OPA_add_int_by_llsc(OPA_int_t *ptr, int val) -{ - OPA_fetch_and_add_int_by_llsc(ptr, val); -} - -static _opa_inline void OPA_incr_int_by_llsc(OPA_int_t *ptr) -{ - OPA_add_int_by_llsc(ptr, 1); -} - -static _opa_inline void OPA_decr_int_by_llsc(OPA_int_t *ptr) -{ - OPA_add_int_by_llsc(ptr, -1); -} - - -static _opa_inline int OPA_fetch_and_decr_int_by_llsc(OPA_int_t *ptr) -{ - return OPA_fetch_and_add_int_by_llsc(ptr, -1); -} - -static _opa_inline int OPA_fetch_and_incr_int_by_llsc(OPA_int_t *ptr) -{ - return OPA_fetch_and_add_int_by_llsc(ptr, 1); -} - -static _opa_inline int OPA_decr_and_test_int_by_llsc(OPA_int_t *ptr) -{ - int prev = OPA_fetch_and_decr_int_by_llsc(ptr); - return prev == 1; -} - -static _opa_inline void *OPA_cas_ptr_by_llsc(OPA_ptr_t *ptr, void *oldv, void *newv) -{ - void *prev; - do { - prev = OPA_LL_ptr(ptr); - } while (prev == oldv && !OPA_SC_ptr(ptr, newv)); - return prev; -} - -static _opa_inline int OPA_cas_int_by_llsc(OPA_int_t *ptr, int oldv, int newv) -{ - int prev; - do { - prev = OPA_LL_int(ptr); - } while (prev == oldv && !OPA_SC_int(ptr, newv)); - return prev; -} - - -static _opa_inline void *OPA_swap_ptr_by_llsc(OPA_ptr_t *ptr, void *val) -{ - void *prev; - do { - prev = OPA_LL_ptr(ptr); - } while (!OPA_SC_ptr(ptr, val)); - return prev; -} - -static _opa_inline int OPA_swap_int_by_llsc(OPA_int_t *ptr, int val) -{ - int prev; - do { - prev = OPA_LL_int(ptr); - } while (!OPA_SC_int(ptr, val)); - return prev; -} - -#endif /* OPA_LL_SC_SUPPORTED */ - -static _opa_inline int OPA_fetch_and_add_int_by_cas(OPA_int_t *ptr, int val) -{ - int cmp; - int prev = OPA_load_int(ptr); - - do { - cmp = prev; - prev = OPA_cas_int(ptr, cmp, prev + val); - } while (prev != cmp); - - return prev; -} - -static _opa_inline int OPA_fetch_and_incr_int_by_faa(OPA_int_t *ptr) -{ - return OPA_fetch_and_add_int(ptr, 1); -} - -static _opa_inline int OPA_fetch_and_decr_int_by_faa(OPA_int_t *ptr) -{ - return OPA_fetch_and_add_int(ptr, -1); -} - -static _opa_inline int OPA_decr_and_test_int_by_fad(OPA_int_t *ptr) -{ - return OPA_fetch_and_decr_int(ptr) == 1; -} - -static _opa_inline void OPA_add_int_by_faa(OPA_int_t *ptr, int val) -{ - OPA_fetch_and_add_int(ptr, val); -} - -static _opa_inline int OPA_incr_int_by_faa(OPA_int_t *ptr) -{ - return OPA_fetch_and_add_int(ptr, 1); -} - -static _opa_inline void OPA_incr_int_by_add(OPA_int_t *ptr) -{ - OPA_add_int(ptr, 1); -} - -static _opa_inline void OPA_incr_int_by_fai(OPA_int_t *ptr) -{ - OPA_fetch_and_incr_int(ptr); -} - -static _opa_inline int OPA_decr_int_by_faa(OPA_int_t *ptr) -{ - return OPA_fetch_and_add_int(ptr, -1); -} - -static _opa_inline void OPA_decr_int_by_add(OPA_int_t *ptr) -{ - OPA_add_int(ptr, -1); -} - -static _opa_inline void OPA_decr_int_by_fad(OPA_int_t *ptr) -{ - OPA_fetch_and_decr_int(ptr); -} - - -/* Swap using CAS */ - -static _opa_inline void *OPA_swap_ptr_by_cas(OPA_ptr_t *ptr, void *val) -{ - void *cmp; - void *prev = OPA_load_ptr(ptr); - - do { - cmp = prev; - prev = OPA_cas_ptr(ptr, cmp, val); - } while (prev != cmp); - - return prev; -} - -static _opa_inline int OPA_swap_int_by_cas(OPA_int_t *ptr, int val) -{ - int cmp; - int prev = OPA_load_int(ptr); - - do { - cmp = prev; - prev = OPA_cas_int(ptr, cmp, val); - } while (prev != cmp); - - return prev; -} - -#endif /* OPA_EMULATED_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_ia64.h b/openpa/src/primitives/opa_gcc_ia64.h deleted file mode 100644 index 8f13523c..00000000 --- a/openpa/src/primitives/opa_gcc_ia64.h +++ /dev/null @@ -1,216 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -/* NOTE: We (@ANL) do not have easy access to any ia64 hosts, so it is difficult - * to develop and test updates to this file. Any help in this regard is greatly - * appreciated. */ - -#ifndef OPA_GCC_IA64_H_INCLUDED -#define OPA_GCC_IA64_H_INCLUDED - -/* FIXME do we need to align these? */ -typedef struct { volatile int v; } OPA_int_t; -typedef struct { void * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -/* Aligned loads and stores are atomic on ia64. */ -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic on ia64. */ -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = val; -} - -/* Aligned loads and stores are atomic on ia64. */ -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic on ia64. */ -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - - -/* NOTE-IA64-1 ia64 suports half-fence suffixes to ld/st instructions. It seems - * that compilers also treat any C-language load/store from/to a volatile - * variable as acquire/release and append the corresponding suffix accordingly. - * But we don't have extensive testing to back this up, so we'll explicitly - * force these instructions for now. When using the C-level approach, it's not - * clear what instructions the compiler is allowed to reorder. */ - -#if 0 -/* acquire means operations after the acquire will see any memory opeations - * performed before the corresponding paired release operation */ -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - int tmp; - __asm__ __volatile__ ("ld.acq %0=[%1]" - : "=r" (tmp) - : "r" (ptr->v) - : "memory"); - return tmp; -} -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - __asm__ __volatile__ ("st.rel [%0]=%1" - : "=m" (ptr->v) - : "r" (val) - : "memory"); -} -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - int tmp; - __asm__ __volatile__ ("ld.acq %0=[%1]" - : "=r" (tmp) - : "r" (ptr->v) - : "memory"); - return tmp; -} -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - __asm__ __volatile__ ("st.rel [%0]=%1" - : "=m" (ptr->v) - : "r" (val) - : "memory"); -} -#else -/* FIXME because we can't test these implementations, they are currently - * disabled. The above impls are rough starting points, but probably won't - * compile/assemble correctly as-is. Patches are welcome :) */ -#define OPA_load_acquire_int(ptr_) ::"choke me" -#define OPA_store_release_int(ptr_,val_) ::"choke me" -#define OPA_load_acquire_ptr(ptr_) ::"choke me" -#define OPA_store_release_ptr(ptr_,val_) ::"choke me" -#endif - - -#define OPA_add_int_by_faa OPA_add_int -#define OPA_incr_int_by_faa OPA_incr_int -#define OPA_decr_int_by_faa OPA_decr_int -#define OPA_fetch_and_decr_int_by_faa OPA_fetch_and_decr_int -#define OPA_fetch_and_incr_int_by_faa OPA_fetch_and_incr_int - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - int val; - __asm__ __volatile__ ("fetchadd4.rel %0=[%2],%3" - : "=r"(val), "=m"(ptr->v) - : "r"(&ptr->v), "i"(-1)); - return val == 1; -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - int prev; - -#if OPA_SIZEOF_INT == 8 - __asm__ __volatile__ ("mov ar.ccv=%1;;" - "cmpxchg8.rel %0=[%3],%4,ar.ccv" - : "=r"(prev), "=m"(ptr->v) - : "rO"(oldv), "r"(&ptr->v), "r"(newv) - : "memory"); - break; -#elif OPA_SIZEOF_INT == 4 - __asm__ __volatile__ ("zxt4 %1=%1;;" /* don't want oldv sign-extended to 64 bits */ - "mov ar.ccv=%1;;" - "cmpxchg4.rel %0=[%3],%4,ar.ccv" - : "=r"(prev), "=m"(ptr->v) - : "r0"(oldv), "r"(&ptr->v), "r"(newv) - : "memory"); -#else -#error OPA_SIZEOF_INT is not 4 or 8 -#endif - - return prev; -} - -/* IA64 has a fetch-and-add instruction that only accepts immediate - values of -16, -8, -4, -1, 1, 4, 8, and 16. So we check for these - values before falling back to the CAS implementation. */ -#define OPA_IA64_FAA_CASE_MACRO(ptr, val) case val: { \ - int prev; \ - __asm__ __volatile__ ("fetchadd4.rel %0=[%2],%3" \ - : "=r"(prev), "=m"(ptr->v) \ - : "r"(&ptr->v), "i"(val)); \ - return prev; \ - } \ - break - - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - switch (val) - { - OPA_IA64_FAA_CASE_MACRO(ptr, -16); - OPA_IA64_FAA_CASE_MACRO(ptr, -8); - OPA_IA64_FAA_CASE_MACRO(ptr, -4); - OPA_IA64_FAA_CASE_MACRO(ptr, -1); - OPA_IA64_FAA_CASE_MACRO(ptr, 1); - OPA_IA64_FAA_CASE_MACRO(ptr, 4); - OPA_IA64_FAA_CASE_MACRO(ptr, 8); - OPA_IA64_FAA_CASE_MACRO(ptr, 16); - default: - { - int cmp; - int prev = OPA_load_int(ptr); - - do { - cmp = prev; - prev = OPA_cas_int(ptr, cmp, val); - } while (prev != cmp); - - return prev; - } - } -} -#undef OPA_IA64_FAA_CASE_MACRO - - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv) -{ - void *prev; - __asm__ __volatile__ ("mov ar.ccv=%1;;" - "cmpxchg8.rel %0=[%3],%4,ar.ccv" - : "=r"(prev), "=m"(ptr->v) - : "rO"(oldv), "r"(&ptr->v), "r"(newv)); - return prev; -} - -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ - __asm__ __volatile__ ("xchg8 %0=[%2],%3" - : "=r" (val), "=m" (ptr->v) - : "r" (&ptr->v), "0" (val)); - return val; -} - - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - __asm__ __volatile__ ("xchg8 %0=[%2],%3" - : "=r" (val), "=m" (ptr->v) - : "r" (&ptr->v), "0" (val)); - return val; -} - - -#define OPA_write_barrier() __asm__ __volatile__ ("mf" ::: "memory" ) -#define OPA_read_barrier() __asm__ __volatile__ ("mf" ::: "memory" ) -#define OPA_read_write_barrier() __asm__ __volatile__ ("mf" ::: "memory" ) -#define OPA_compiler_barrier() __asm__ __volatile__ ("" ::: "memory" ) - -#include "opa_emulated.h" - -#endif /* OPA_GCC_IA64_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_intel_32_64.h b/openpa/src/primitives/opa_gcc_intel_32_64.h deleted file mode 100644 index d435311f..00000000 --- a/openpa/src/primitives/opa_gcc_intel_32_64.h +++ /dev/null @@ -1,15 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_GCC_INTEL_32_64_H_INCLUDED -#define OPA_GCC_INTEL_32_64_H_INCLUDED - -#include "opa_gcc_intel_32_64_barrier.h" -#include "opa_gcc_intel_32_64_ops.h" - -#include "opa_emulated.h" - -#endif /* OPA_GCC_INTEL_32_64_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_intel_32_64_barrier.h b/openpa/src/primitives/opa_gcc_intel_32_64_barrier.h deleted file mode 100644 index 36aaf474..00000000 --- a/openpa/src/primitives/opa_gcc_intel_32_64_barrier.h +++ /dev/null @@ -1,25 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_GCC_INTEL_32_64_BARRIER_H_INCLUDED -#define OPA_GCC_INTEL_32_64_BARRIER_H_INCLUDED - -#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" ) - -/* For all regular memory (write-back cacheable, not driver/graphics - * memory), there is only one general ordering relaxation permitted by - * x86/x86_64 processors: earlier stores may be retired after later - * stores. The "clflush" and "movnt*" instructions also don't follow - * general ordering constraints, although any code using these - * instructions should be responsible for ensuring proper ordering - * itself. So our read and write barriers may be implemented as simple - * compiler barriers. */ -#define OPA_write_barrier() OPA_compiler_barrier() -#define OPA_read_barrier() OPA_compiler_barrier() - -#define OPA_read_write_barrier() __asm__ __volatile__ ( "mfence" ::: "memory" ) - -#endif /* OPA_GCC_INTEL_32_64_BARRIER_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_intel_32_64_ops.h b/openpa/src/primitives/opa_gcc_intel_32_64_ops.h deleted file mode 100644 index 6482c199..00000000 --- a/openpa/src/primitives/opa_gcc_intel_32_64_ops.h +++ /dev/null @@ -1,173 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_GCC_INTEL_32_64_OPS_H_INCLUDED -#define OPA_GCC_INTEL_32_64_OPS_H_INCLUDED - -#ifndef OPA_SIZEOF_INT -#error OPA_SIZEOF_INT is not defined -#endif - -/* Set OPA_SS (Size Suffix) which is to be appended to asm ops for - specifying 4 or 8 byte operands */ -#if OPA_SIZEOF_INT == 4 -#define OPA_SS "l" -#elif OPA_SIZEOF_INT == 8 -#define OPA_SS "q" -#else -#error OPA_SIZEOF_INT is not 4 or 8 -#endif - -/* XXX DJG FIXME do we need to align these? */ -typedef struct { volatile int v; } OPA_int_t; -typedef struct { void * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -/* Aligned loads and stores are atomic on x86(-64). */ -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic on x86(-64). */ -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = val; -} - -/* Aligned loads and stores are atomic on x86(-64). */ -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic on x86(-64). */ -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -/* NOTE-X86-1 x86 and x86_64 processors execute instructions such that their - * effects are visible in issue order, with one exception: earlier stores may - * pass later loads. Also, non-temporal stores (with the "movnt*" instructions) - * do not follow these ordering constraints. So all normal load/store - * instructions on x86(_64) already have acquire/release semantics. We just - * have to make sure that the compiler does not reorder the instructions. */ - -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - int tmp; - tmp = ptr->v; - OPA_compiler_barrier(); /* NOTE-X86-1 */ - return tmp; -} - -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - OPA_compiler_barrier(); /* NOTE-X86-1 */ - ptr->v = val; -} - -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - void *tmp; - tmp = ptr->v; - OPA_compiler_barrier(); /* NOTE-X86-1 */ - return tmp; -} - -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - OPA_compiler_barrier(); /* NOTE-X86-1 */ - ptr->v = val; -} - - -static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val) -{ - __asm__ __volatile__ ("lock ; add"OPA_SS" %1,%0" - :"=m" (ptr->v) - :"ir" (val), "m" (ptr->v)); - return; -} - -static _opa_inline void OPA_incr_int(OPA_int_t *ptr) -{ - __asm__ __volatile__ ("lock ; inc"OPA_SS" %0" - :"=m" (ptr->v) - :"m" (ptr->v)); - return; -} - -static _opa_inline void OPA_decr_int(OPA_int_t *ptr) -{ - __asm__ __volatile__ ("lock ; dec"OPA_SS" %0" - :"=m" (ptr->v) - :"m" (ptr->v)); - return; -} - - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - char result; - __asm__ __volatile__ ("lock ; dec"OPA_SS" %0; setz %1" - :"=m" (ptr->v), "=q" (result) - :"m" (ptr->v)); - return result; -} - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - __asm__ __volatile__ ("lock ; xadd %0,%1" - : "=r" (val), "=m" (ptr->v) - : "0" (val), "m" (ptr->v)); - return val; -} - -#define OPA_fetch_and_incr_int_by_faa OPA_fetch_and_incr_int -#define OPA_fetch_and_decr_int_by_faa OPA_fetch_and_decr_int - - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv) -{ - void *prev; - __asm__ __volatile__ ("lock ; cmpxchg %3,%4" - : "=a" (prev), "=m" (ptr->v) - : "0" (oldv), "q" (newv), "m" (ptr->v)); - return prev; -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - int prev; - __asm__ __volatile__ ("lock ; cmpxchg %3,%4" - : "=a" (prev), "=m" (ptr->v) - : "0" (oldv), "q" (newv), "m" (ptr->v)); - return prev; -} - -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ - __asm__ __volatile__ ("xchg %0,%1" - :"=r" (val), "=m" (ptr->v) - : "0" (val), "m" (ptr->v)); - return val; -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - __asm__ __volatile__ ("xchg %0,%1" - :"=r" (val), "=m" (ptr->v) - : "0" (val), "m" (ptr->v)); - return val; -} - -#undef OPA_SS - -#endif /* OPA_GCC_INTEL_32_64_OPS_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_intel_32_64_p3.h b/openpa/src/primitives/opa_gcc_intel_32_64_p3.h deleted file mode 100644 index 7fad12a3..00000000 --- a/openpa/src/primitives/opa_gcc_intel_32_64_p3.h +++ /dev/null @@ -1,15 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_GCC_INTEL_32_64_P3_H_INCLUDED -#define OPA_GCC_INTEL_32_64_P3_H_INCLUDED - -#include "opa_gcc_intel_32_64_p3barrier.h" -#include "opa_gcc_intel_32_64_ops.h" - -#include "opa_emulated.h" - -#endif /* OPA_GCC_INTEL_32_64_P3_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_intel_32_64_p3barrier.h b/openpa/src/primitives/opa_gcc_intel_32_64_p3barrier.h deleted file mode 100644 index ecd68527..00000000 --- a/openpa/src/primitives/opa_gcc_intel_32_64_p3barrier.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_GCC_INTEL_32_64_P3BARRIER_H_INCLUDED -#define OPA_GCC_INTEL_32_64_P3BARRIER_H_INCLUDED - -#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" ) - -/* For all regular memory (write-back cacheable, not driver/graphics - * memory), there is only one general ordering relaxation permitted by - * x86/x86_64 processors: earlier stores may be retired after later - * stores. The "clflush" and "movnt*" instructions also don't follow - * general ordering constraints, although any code using these - * instructions should be responsible for ensuring proper ordering - * itself. So our read and write barriers may be implemented as simple - * compiler barriers. */ -#define OPA_write_barrier() OPA_compiler_barrier() -#define OPA_read_barrier() OPA_compiler_barrier() - -/* Some Pentium III and earlier processors have store barriers - (sfence), but no full barrier or load barriers. Some other very - recent x86-like processors don't seem to have sfence either. A - lock-prefixed atomic instruction (operating on any memory) behaves as - a full memory barrier, so we use that here. */ -static inline void OPA_read_write_barrier(void) -{ - int a = 0; - __asm__ __volatile__ ("lock orl $0, %0" : "+m" (a) : /*no outputs*/ : "memory"); -} - -#endif /* OPA_GCC_INTEL_32_64_P3BARRIER_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_intrinsics.h b/openpa/src/primitives/opa_gcc_intrinsics.h deleted file mode 100644 index 9857770c..00000000 --- a/openpa/src/primitives/opa_gcc_intrinsics.h +++ /dev/null @@ -1,129 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -/* FIXME needs to be converted to new style functions with OPA_int_t/OPA_ptr_t-types */ -#ifndef OPA_GCC_INTRINSICS_H_INCLUDED -#define OPA_GCC_INTRINSICS_H_INCLUDED - -/* FIXME do we need to align these? */ -typedef struct { volatile int v; } OPA_int_t; -typedef struct { void * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -/* Assume that loads/stores are atomic on the current platform, even though this - may not be true at all. */ -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return ptr->v; -} - -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = val; -} - -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ptr->v; -} - -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - volatile int i = 0; - int tmp; - tmp = ptr->v; - __sync_lock_test_and_set(&i, 1); /* guarantees acquire semantics */ - return tmp; -} - -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - volatile int i = 1; - __sync_lock_release(&i); /* guarantees release semantics */ - ptr->v = val; -} - -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - volatile int i = 0; - void *tmp; - tmp = ptr->v; - __sync_lock_test_and_set(&i, 1); /* guarantees acquire semantics */ - return tmp; -} - -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - volatile int i = 1; - __sync_lock_release(&i); /* guarantees release semantics */ - ptr->v = val; -} - - -/* gcc atomic intrinsics accept an optional list of variables to be - protected by a memory barrier. These variables are labeled - below by "protected variables :". */ - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - return __sync_fetch_and_add(&ptr->v, val, /* protected variables: */ &ptr->v); -} - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - return __sync_sub_and_fetch(&ptr->v, 1, /* protected variables: */ &ptr->v) == 0; -} - -#define OPA_fetch_and_incr_int_by_faa OPA_fetch_and_incr_int -#define OPA_fetch_and_decr_int_by_faa OPA_fetch_and_decr_int -#define OPA_add_int_by_faa OPA_add_int -#define OPA_incr_int_by_fai OPA_incr_int -#define OPA_decr_int_by_fad OPA_decr_int - - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv) -{ - return __sync_val_compare_and_swap(&ptr->v, oldv, newv, /* protected variables: */ &ptr->v); -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - return __sync_val_compare_and_swap(&ptr->v, oldv, newv, /* protected variables: */ &ptr->v); -} - -#ifdef SYNC_LOCK_TEST_AND_SET_IS_SWAP -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ - return __sync_lock_test_and_set(&ptr->v, val, /* protected variables: */ &ptr->v); -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - return __sync_lock_test_and_set(&ptr->v, val, /* protected variables: */ &ptr->v); -} - -#else -#define OPA_swap_ptr_by_cas OPA_swap_ptr -#define OPA_swap_int_by_cas OPA_swap_int -#endif - -#define OPA_write_barrier() __sync_synchronize() -#define OPA_read_barrier() __sync_synchronize() -#define OPA_read_write_barrier() __sync_synchronize() -#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" ) - - - -#include"opa_emulated.h" - -#endif /* OPA_GCC_INTRINSICS_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_ppc.h b/openpa/src/primitives/opa_gcc_ppc.h deleted file mode 100644 index 410d5616..00000000 --- a/openpa/src/primitives/opa_gcc_ppc.h +++ /dev/null @@ -1,180 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_GCC_PPC_H_INCLUDED -#define OPA_GCC_PPC_H_INCLUDED - -/* these need to be aligned on an 8-byte boundary to work on a BG/P */ -typedef struct { volatile int v OPA_ATTRIBUTE((aligned (8))); } OPA_int_t; -typedef struct { void * volatile v OPA_ATTRIBUTE((aligned (8))); } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -/* Aligned loads and stores are atomic. */ -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic. */ -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = val; -} - -/* Aligned loads and stores are atomic. */ -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic. */ -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -/* a useful link for PPC memory ordering issues: - * http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2009.02.22a.html - * - * lwsync: orders L-L, S-S, L-S, but *not* S-L (i.e. gives x86-ish ordering) - * eieio: orders S-S (but only for cacheable memory, not for MMIO) - * sync: totally orders memops - * isync: force all preceeding insns to appear complete before starting - * subsequent insns, but w/o cumulativity (very confusing) - */ -/* helper macros */ -#define OPA_ppc_lwsync_() __asm__ __volatile__ ( "lwsync" ::: "memory" ) -#define OPA_ppc_hwsync_() __asm__ __volatile__ ( "sync" ::: "memory" ) -#define OPA_ppc_eieio_() __asm__ __volatile__ ( "eieio" ::: "memory" ) - -#define OPA_write_barrier() OPA_ppc_eieio_() -#define OPA_read_barrier() OPA_ppc_lwsync_() -#define OPA_read_write_barrier() OPA_ppc_hwsync_() -#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" ) - -/* NOTE-PPC-1 we use lwsync, although I think we might be able to use - * conditional-branch+isync in some cases (load_acquire?) once we understand it - * better */ - -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - int tmp; - tmp = ptr->v; - OPA_ppc_lwsync_(); /* NOTE-PPC-1 */ - return tmp; -} -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - OPA_ppc_lwsync_(); - ptr->v = val; -} -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - void *tmp; - tmp = ptr->v; - OPA_ppc_lwsync_(); /* NOTE-PPC-1 */ - return tmp; -} -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - OPA_ppc_lwsync_(); - ptr->v = val; -} - - -/* - load-link/store-conditional (LL/SC) primitives. We LL/SC implement - these here, which are arch-specific, then use the generic - implementations from opa_emulated.h */ - -static _opa_inline int OPA_LL_int(OPA_int_t *ptr) -{ - int val; - __asm__ __volatile__ ("lwarx %[val],0,%[ptr]" - : [val] "=r" (val) - : [ptr] "r" (&ptr->v) - : "cc"); - - return val; -} - -/* Returns non-zero if the store was successful, zero otherwise. */ -static _opa_inline int OPA_SC_int(OPA_int_t *ptr, int val) -{ - int ret = 1; /* init to non-zero, will be reset to 0 if SC was unsuccessful */ - __asm__ __volatile__ ("stwcx. %[val],0,%[ptr];\n" - "beq 1f;\n" - "li %[ret], 0;\n" - "1: ;\n" - : [ret] "=r" (ret) - : [ptr] "r" (&ptr->v), [val] "r" (val), "0" (ret) - : "cc", "memory"); - return ret; -} - - -/* Pointer versions of LL/SC. */ - -/* Set OPA_SS (Size Suffix) which is used to choose between lwarx/stwcx and - * ldarx/stdcx when using 4 or 8 byte pointer operands */ -#if OPA_SIZEOF_VOID_P == 4 -#define OPA_SS "w" -#elif OPA_SIZEOF_VOID_P == 8 -#define OPA_SS "d" -#else -#error OPA_SIZEOF_VOID_P is not 4 or 8 -#endif - - -static _opa_inline void *OPA_LL_ptr(OPA_ptr_t *ptr) -{ - void *val; - __asm__ __volatile__ ("l"OPA_SS"arx %[val],0,%[ptr]" - : [val] "=r" (val) - : [ptr] "r" (&ptr->v) - : "cc"); - - return val; -} - -/* Returns non-zero if the store was successful, zero otherwise. */ -static _opa_inline int OPA_SC_ptr(OPA_ptr_t *ptr, void *val) -{ - int ret = 1; /* init to non-zero, will be reset to 0 if SC was unsuccessful */ - __asm__ __volatile__ ("st"OPA_SS"cx. %[val],0,%[ptr];\n" - "beq 1f;\n" - "li %[ret], 0;\n" - "1: ;\n" - : [ret] "=r" (ret) - : [ptr] "r" (&ptr->v), [val] "r" (val), "0" (ret) - : "cc", "memory"); - return ret; -} - -#undef OPA_SS - -/* necessary to enable LL/SC emulation support */ -#define OPA_LL_SC_SUPPORTED 1 - -/* Implement all function using LL/SC */ -#define OPA_add_int_by_llsc OPA_add_int -#define OPA_incr_int_by_llsc OPA_incr_int -#define OPA_decr_int_by_llsc OPA_decr_int -#define OPA_decr_and_test_int_by_llsc OPA_decr_and_test_int -#define OPA_fetch_and_add_int_by_llsc OPA_fetch_and_add_int -#define OPA_fetch_and_decr_int_by_llsc OPA_fetch_and_decr_int -#define OPA_fetch_and_incr_int_by_llsc OPA_fetch_and_incr_int -#define OPA_cas_ptr_by_llsc OPA_cas_ptr -#define OPA_cas_int_by_llsc OPA_cas_int -#define OPA_swap_ptr_by_llsc OPA_swap_ptr -#define OPA_swap_int_by_llsc OPA_swap_int - - -#include "opa_emulated.h" - -#endif /* OPA_GCC_PPC_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_gcc_sicortex.h b/openpa/src/primitives/opa_gcc_sicortex.h deleted file mode 100644 index abe3dfdc..00000000 --- a/openpa/src/primitives/opa_gcc_sicortex.h +++ /dev/null @@ -1,443 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -/* Atomic primitives for SiCortex machines. - * Originally contributed by Lawrence Stewart at SiCortex. - */ - -#ifndef OPA_GCC_SICORTEX_H -#define OPA_GCC_SICORTEX_H - -/* FIXME do these need alignment? */ -typedef struct { volatile int v; } OPA_int_t; -typedef struct { int * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -#define OPA_write_barrier() __asm__ __volatile__ ("sync" ::: "memory" ) -#define OPA_read_barrier() __asm__ __volatile__ ("sync" ::: "memory" ) -#define OPA_read_write_barrier() __asm__ __volatile__ ("sync" ::: "memory" ) -#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" ) - -/* Aligned loads and stores are atomic. */ -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic. */ -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = val; -} - -/* Aligned loads and stores are atomic. */ -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ptr->v; -} - -/* Aligned loads and stores are atomic. */ -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -/* NOTE: these acquire/release operations have not been optimized, I just threw - * down a full memory barrier. Spending much time working on the SiCortex platform - * doesn't really make a lot of sense since there are so few machines in - * existence and no more will ever be built. */ -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - int tmp; - tmp = ptr->v; - OPA_read_write_barrier(); - return tmp; -} - -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - OPA_read_write_barrier(); - ptr->v = val; -} - -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - void *tmp; - tmp = ptr->v; - OPA_read_write_barrier(); - return tmp; -} - -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - OPA_read_write_barrier(); - ptr->v = val; -} - -#include - -/* ICE9 rev A1 chips have a low-frequency bug that causes LL to - fail. The workaround is to do the LL twice to make sure the data - is in L1 - - very few systems are affected - - FIXME We should either remove the workaround entirely or make it - configurable/autodetected somehow. [goodell@ 2008/01/06] - */ -#define ICE9A_LLSC_WAR 0 - -/* For a description of the _opa_inline assembly constraints, see the MIPS section of: - http://gcc.gnu.org/onlinedocs/gcc-4.3.2/gcc/Machine-Constraints.html#Machine-Constraints - - relevant excerpt: - I - A signed 16-bit constant (for arithmetic instructions). - J - Integer zero. - - Other _opa_inline asm knowledge worth remembering: - r - a general register operand is allowed - m - a memory address operand - & - earlyclobber; operand is modified before instruction is finished using the input operands - = - this operand is write-only - - general format: - asm volatile ("instructions" : [outputs] : [inputs] : [clobbered_regs]); - "memory" should be included as a clobbered reg for most of these operations. - */ - -/* Atomic increment of a 32 bit value, returning the old value */ -static _opa_inline int OPA_shmemi_fetch_add_4(volatile int * v, int inc) -{ - unsigned long result; - if (ICE9A_LLSC_WAR) { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: ll %0, %2 # fetch_add_4 \n" - " ll %0, %2 # fetch_add_4 \n" - " addu %1, %0, %3 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "Ir" (inc) - : "memory"); - } else { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: ll %0, %2 # fetch_add_4 \n" - " addu %1, %0, %3 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "Ir" (inc) - : "memory"); - } - - return result; -} - -/* Atomic increment of a 64 bit value, returning the old value */ -static _opa_inline long int OPA_shmemi_fetch_add_8(volatile long int * v, long int inc) -{ - unsigned long result; - if (ICE9A_LLSC_WAR) { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: lld %0, %2 # fetch_add_8 \n" - " lld %0, %2 # fetch_add_8 \n" - " daddu %1, %0, %3 \n" - " scd %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "Ir" (inc) - : "memory"); - } else { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: lld %0, %2 # fetch_add_8 \n" - " daddu %1, %0, %3 \n" - " scd %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "Ir" (inc) - : "memory"); - } - - return result; -} - -/* Atomic swap of a 32 bit value, returning the old contents */ -static _opa_inline int OPA_shmemi_swap_4(volatile int * v, int val) -{ - unsigned long result; - if (ICE9A_LLSC_WAR) { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: ll %0, %2 # swap_4 \n" - " ll %0, %2 # swap_4 \n" - " move %1, %3 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val) - : "memory"); - } else { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: ll %0, %2 # swap_4 \n" - " move %1, %3 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val) - : "memory"); - } - - return result; -} - -/* Atomic swap of a 64 bit value, returning the old contents */ -static _opa_inline long int OPA_shmemi_swap_8(volatile long int * v, long int val) -{ - unsigned long result; - if (ICE9A_LLSC_WAR) { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: lld %0, %2 # swap_8 \n" - " lld %0, %2 # swap_8 \n" - " move %1, %3 \n" - " scd %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val) - : "memory"); - } else { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: lld %0, %2 # swap_8 \n" - " move %1, %3 \n" - " scd %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val) - : "memory"); - } - - return result; -} - -/* Atomic compare and swap of a 32 bit value, returns the old value - * but only does the store of the val value if the old value == expect */ -static _opa_inline int OPA_shmemi_cswap_4(volatile int * v, int expect, int val) -{ - unsigned long result; - if (ICE9A_LLSC_WAR) { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: ll %0, %2 # cswap_4 \n" - " ll %0, %2 # cswap_4 \n" - " bne %0, %4, 1f \n" - " move %1, %3 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - "1: \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val), "Jr" (expect) - : "memory"); - } else { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: ll %0, %2 # cswap_4 \n" - " bne %0, %4, 1f \n" - " move %1, %3 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - "1: \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val), "Jr" (expect) - : "memory"); - } - - return result; -} - -/* Atomic compare and swap of a 64 bit value, returns the old value - * but only does the store of the val value if the old value == expect */ -static _opa_inline long int OPA_shmemi_cswap_8(volatile long int * v, long int expect, long int val) -{ - unsigned long result; - if (ICE9A_LLSC_WAR) { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: lld %0, %2 # cswap_8 \n" - " lld %0, %2 # cswap_8 \n" - " bne %0, %4, 1f \n" - " move %1, %3 \n" - " scd %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - "1: \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val), "Jr" (expect) - : "memory"); - } else { - unsigned long temp; - - __asm__ __volatile__( - " .set mips3 \n" - " .set noreorder \n" - "1: lld %0, %2 # cswap_8 \n" - " bne %0, %4, 1f \n" - " move %1, %3 \n" - " scd %1, %2 \n" - " beqz %1, 1b \n" - " nop \n" - " .set reorder \n" - "1: \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), "=m" (*v) - : "r" (val), "Jr" (expect) - : "memory"); - } - - return result; -} - -static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val) -{ - OPA_shmemi_fetch_add_4(&ptr->v, val); -} - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv) -{ -#if (OPA_SIZEOF_VOID_P == 8) - return((int *) OPA_shmemi_cswap_8((volatile long int *) &ptr->v, (uintptr_t) oldv, (uintptr_t) newv)); -#elif (OPA_SIZEOF_VOID_P == 4) - return((int *) OPA_shmemi_cswap_4((volatile int *) &ptr->v, (uintptr_t) oldv, (uintptr_t) newv)); -#else -#error "OPA_SIZEOF_VOID_P has an unexpected value :" OPA_QUOTE(OPA_SIZEOF_VOID_P); -#endif -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - return(OPA_shmemi_cswap_4(&ptr->v, oldv, newv)); -} - -static _opa_inline void OPA_decr_int(OPA_int_t *ptr) -{ - OPA_shmemi_fetch_add_4(&ptr->v, -1); -} - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - int old = OPA_shmemi_fetch_add_4(&ptr->v, -1); - return (old == 1); -} - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - return(OPA_shmemi_fetch_add_4(&ptr->v, val)); -} - -static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr) -{ - return(OPA_shmemi_fetch_add_4(&ptr->v, -1)); -} - -static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr) -{ - return(OPA_shmemi_fetch_add_4(&ptr->v, 1)); -} - -static _opa_inline void OPA_incr_int(OPA_int_t *ptr) -{ - OPA_shmemi_fetch_add_4(&ptr->v, 1); -} - -static _opa_inline int *OPA_swap_ptr(OPA_ptr_t *ptr, int *val) -{ -#if (OPA_SIZEOF_VOID_P == 8) - return((int *) OPA_shmemi_swap_8((volatile long int *) &ptr->v, (uintptr_t) val)); -#elif (OPA_SIZEOF_VOID_P == 4) - return((int *) OPA_shmemi_swap_4((volatile int *) &ptr->v, (uintptr_t) val)); -#else -#error "OPA_SIZEOF_VOID_P has an unexpected value :" OPA_QUOTE(OPA_SIZEOF_VOID_P); -#endif -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - return(OPA_shmemi_swap_4(&ptr->v, val)); -} - -#endif /* OPA_GCC_SICORTEX_H */ diff --git a/openpa/src/primitives/opa_nt_intrinsics.h b/openpa/src/primitives/opa_nt_intrinsics.h deleted file mode 100644 index b5550b75..00000000 --- a/openpa/src/primitives/opa_nt_intrinsics.h +++ /dev/null @@ -1,153 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_NT_INTRINSICS_H_INCLUDED -#define OPA_NT_INTRINSICS_H_INCLUDED - -#define WIN32_LEAN_AND_MEAN -#include -#include - -/* OPA_int_t uses a long because the compiler intrinsics operate on - * longs instead of ints. */ -typedef struct { volatile long v; } OPA_int_t; -typedef struct { void * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -#define OPA_write_barrier() _WriteBarrier() -#define OPA_read_barrier() _ReadBarrier() -#define OPA_read_write_barrier() _ReadWriteBarrier() -/* FIXME there mut be a more efficient way to implement this. Is "asm {};" - * sufficient? */ -#define OPA_compiler_barrier() _ReadWriteBarrier() - -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return ((int)ptr->v); -} - -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = (long)val; -} - -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ((void *)ptr->v); -} - -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -/* NOTE: these acquire/release operations have not been optimized, I just threw - * down a full memory barrier. Someone with more Windows expertise should feel - * free to improve these (for Windows on x86/x86_64 these almost certainly only - * need to be compiler barriers). */ -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - int tmp; - tmp = ptr->v; - OPA_read_write_barrier(); - return tmp; -} - -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - OPA_read_write_barrier(); - ptr->v = val; -} - -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - void *tmp; - tmp = ptr->v; - OPA_read_write_barrier(); - return tmp; -} - -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - OPA_read_write_barrier(); - ptr->v = val; -} - - -static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val) -{ - _InterlockedExchangeAdd(&(ptr->v), val); -} - -static _opa_inline void OPA_incr_int(OPA_int_t *ptr) -{ - _InterlockedIncrement(&(ptr->v)); -} - -static _opa_inline void OPA_decr_int(OPA_int_t *ptr) -{ - _InterlockedDecrement(&(ptr->v)); -} - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - return (_InterlockedDecrement(&(ptr->v)) == 0); -} - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - return ((int)_InterlockedExchangeAdd(&(ptr->v), (long)val)); -} - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv) -{ -#if (OPA_SIZEOF_VOID_P == 4) - return ((void *)(LONG_PTR) _InterlockedCompareExchange((LONG volatile *)&(ptr->v), - (LONG)(LONG_PTR)newv, - (LONG)(LONG_PTR)oldv) - ); -#elif (OPA_SIZEOF_VOID_P == 8) - return ((void *)(LONG_PTR)_InterlockedCompareExchange64((__int64 *)&(ptr->v), - (INT64)(LONG_PTR)newv, - (INT64)(LONG_PTR)oldv) - ); -#else -#error "OPA_SIZEOF_VOID_P not valid" -#endif -} - -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ -#if (OPA_SIZEOF_VOID_P == 4) - return (void *)(LONG_PTR )_InterlockedExchange((LONG volatile *)&(ptr->v), - (LONG)(LONG_PTR)val); -#elif (OPA_SIZEOF_VOID_P == 8) - return (void *)(LONG_PTR)_InterlockedExchange64((LONG64 volatile *)&(ptr->v), - (INT64)(LONG_PTR)val); -#else -#error "OPA_SIZEOF_VOID_P not valid" -#endif -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - return _InterlockedCompareExchange((long *)&(ptr->v), newv, oldv); -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - return _InterlockedExchange(&(ptr->v), val); -} - -/* Implement fetch_and_incr/decr using fetch_and_add (*_faa) */ -#define OPA_fetch_and_incr_int_by_faa OPA_fetch_and_incr_int -#define OPA_fetch_and_decr_int_by_faa OPA_fetch_and_decr_int - -#include "opa_emulated.h" - -#endif /* defined(OPA_NT_INTRINSICS_H_INCLUDED) */ diff --git a/openpa/src/primitives/opa_sun_atomic_ops.h b/openpa/src/primitives/opa_sun_atomic_ops.h deleted file mode 100644 index e42d4c97..00000000 --- a/openpa/src/primitives/opa_sun_atomic_ops.h +++ /dev/null @@ -1,135 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - - -#ifndef OPA_SUN_ATOMIC_OPS_H_INCLUDED -#define OPA_SUN_ATOMIC_OPS_H_INCLUDED - -#include - -typedef struct { volatile uint_t v; } OPA_int_t; -typedef struct { void * volatile v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr) -{ - return (int)ptr->v; -} - -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = (uint)val; -} - -static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr) -{ - return ptr->v; -} - -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr) -{ - int tmp; - tmp = ptr->v; - membar_enter(); - return tmp; -} - -static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val) -{ - membar_exit(); - ptr->v = val; -} - -static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr) -{ - void *tmp; - tmp = ptr->v; - membar_enter(); - return tmp; -} - -static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val) -{ - membar_exit(); - ptr->v = val; -} - - -static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val) -{ - atomic_add_int(&ptr->v, val); -} - -static _opa_inline void OPA_incr_int(OPA_int_t *ptr) -{ - atomic_inc_uint(&ptr->v); -} - -static _opa_inline void OPA_decr_int(OPA_int_t *ptr) -{ - atomic_dec_uint(&ptr->v); -} - - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - return atomic_dec_uint_nv(&ptr->v) == 0; -} - - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - return (int)atomic_add_int_nv(&ptr->v, val) - val; -} - -static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr) -{ - return (int)atomic_dec_uint_nv(&ptr->v) + 1; -} - -static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr) -{ - return (int)atomic_inc_uint_nv(&ptr->v) - 1; -} - - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv) -{ - return atomic_cas_ptr(ptr, oldv, newv); -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - return (int)atomic_cas_uint(&ptr->v, (uint_t)oldv, (uint_t)newv); -} - - -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ - return atomic_swap_ptr(ptr, val); -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - return (int)atomic_swap_uint(&ptr->v, (uint_t) val); -} - - -#define OPA_write_barrier() membar_producer() -#define OPA_read_barrier() membar_consumer() -#define OPA_read_write_barrier() do { membar_consumer(); membar_producer(); } while (0) - -/* is this portable enough? */ -#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" ) - -#endif /* OPA_SUN_ATOMIC_OPS_H_INCLUDED */ diff --git a/openpa/src/primitives/opa_unsafe.h b/openpa/src/primitives/opa_unsafe.h deleted file mode 100644 index 99f3cd63..00000000 --- a/openpa/src/primitives/opa_unsafe.h +++ /dev/null @@ -1,152 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ -/* - * (C) 2008 by Argonne National Laboratory. - * See COPYRIGHT in top-level directory. - */ - -#ifndef OPA_UNSAFE_H_INCLUDED -#define OPA_UNSAFE_H_INCLUDED - -/* NOTE: These types intentionally do not use volatile in order to provide even - * better performance. If a good use case for having unsafe operations with - * volatile types comes up, we can make two definitions that are conditional on - * something sort of like NDEBUG. [goodell@ 2009-08-18] */ -typedef struct { int v; } OPA_int_t; -typedef struct { int *v; } OPA_ptr_t; - -#define OPA_INT_T_INITIALIZER(val_) { (val_) } -#define OPA_PTR_T_INITIALIZER(val_) { (val_) } - -/* - Unsafe Primitives - ----------------- - - These are versions of the atomic primitives that emulate the single-threaded - behavior of the primitives but do not attempt to provide any safety against - concurrent use. The primary use case for this implementation is to avoid - overhead without code changes in client libraries and applications that have - been compiled for single-threaded use only. They might also be useful for - testing (and testing of tests) in some cases. - - Thanks to Josh Haberman for inspiring this primitives implementation. -*/ - -static _opa_inline int OPA_load_int(const OPA_int_t *ptr) -{ - int retval; - retval = ptr->v; - return retval; -} - -static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val) -{ - ptr->v = val; -} - -static _opa_inline void *OPA_load_ptr(const OPA_ptr_t *ptr) -{ - int *retval; - retval = ptr->v; - return retval; -} - -static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val) -{ - ptr->v = val; -} - -/* no barrier needed, not even a compiler barrier */ -#define OPA_load_acquire_int(ptr_) OPA_load_int((ptr_)) -#define OPA_store_release_int(ptr_,val_) OPA_store_int((ptr_),(val_)) -#define OPA_load_acquire_ptr(ptr_) OPA_load_ptr((ptr_)) -#define OPA_store_release_ptr(ptr_,val_) OPA_store_ptr((ptr_),(val_)) - -static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val) -{ - ptr->v += val; -} - -static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, int *oldv, int *newv) -{ - int *prev; - prev = ptr->v; - if (prev == oldv) { - ptr->v = newv; - } - return prev; -} - -static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv) -{ - int prev; - prev = ptr->v; - if (prev == oldv) { - ptr->v = newv; - } - return prev; -} - -static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr) -{ - int new_val; - new_val = --(ptr->v); - return (0 == new_val); -} - -static _opa_inline void OPA_decr_int(OPA_int_t *ptr) -{ - --(ptr->v); -} - -static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val) -{ - int prev; - prev = ptr->v; - ptr->v += val; - return prev; -} - -static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr) -{ - int prev; - prev = ptr->v; - --(ptr->v); - return prev; -} - -static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr) -{ - int prev; - prev = ptr->v; - ++(ptr->v); - return prev; -} - -static _opa_inline void OPA_incr_int(OPA_int_t *ptr) -{ - ++(ptr->v); -} - -static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val) -{ - int *prev; - prev = ptr->v; - ptr->v = val; - return prev; -} - -static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val) -{ - int prev; - prev = ptr->v; - ptr->v = val; - return (int)prev; -} - -/* null barriers */ -#define OPA_write_barrier() do {} while (0) -#define OPA_read_barrier() do {} while (0) -#define OPA_read_write_barrier() do {} while (0) -#define OPA_compiler_barrier() do {} while (0) - -#endif /* !defined(OPA_UNSAFE_H_INCLUDED) */ diff --git a/package.json b/package.json index e9808983..2cf6c7b3 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "lib", "src", "includes", - "openpa", "binding.gyp" ], "homepage": "https://github.com/axosoft/node-simple-file-watcher", From 28701c9f4730d02597e5bc6bea30962c1f144eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Tue, 12 Sep 2017 14:01:16 +0200 Subject: [PATCH 08/37] FIX: signed/unsigned compiler warning --- src/Queue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Queue.cpp b/src/Queue.cpp index 5858145c..6bdb3138 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -33,7 +33,7 @@ std::unique_ptr> EventQueue::dequeueAll() { const auto queueSize = queue.size(); std::unique_ptr> events(new std::vector(queueSize, nullptr)); - for (auto i = 0; i < queueSize; ++i) { + for (size_t i = 0; i < queueSize; ++i) { auto& front = queue.front(); (*events)[i] = front.release(); queue.pop_front(); From 5d11a18db9c897c703a6bb66ab8431a94016b58f Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Tue, 12 Sep 2017 15:09:31 +0200 Subject: [PATCH 09/37] change xcode configuration for nodejs 4 and 6 --- binding.gyp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/binding.gyp b/binding.gyp index 88022216..8b76be71 100644 --- a/binding.gyp +++ b/binding.gyp @@ -41,6 +41,12 @@ "includes/osx/RunLoop.h", "includes/osx/FSEventsService.h" ], + "xcode_settings": { + "OTHER_CFLAGS": [ + "-std=c++0x", + "-stdlib=libc++" + ], + }, "link_settings": { "xcode_settings": { "OTHER_LDFLAGS": [ From 5c7fe7d6fb31ccb404c3b58f466530cd91adb348 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Tue, 12 Sep 2017 16:14:49 +0200 Subject: [PATCH 10/37] try to find a reace cond --- includes/NativeInterface.h | 1 + src/NativeInterface.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index 6c7f69d3..dc5d1505 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -18,6 +18,7 @@ using NativeImplementation = InotifyService; class NativeInterface { public: NativeInterface(const std::string &path); + ~NativeInterface(); std::string getError(); std::vector* getEvents(); diff --git a/src/NativeInterface.cpp b/src/NativeInterface.cpp index 85e907ee..48870866 100644 --- a/src/NativeInterface.cpp +++ b/src/NativeInterface.cpp @@ -4,6 +4,10 @@ NativeInterface::NativeInterface(const std::string &path) { mNativeInterface.reset(new NativeImplementation(mQueue, path)); } +NativeInterface::~NativeInterface() { + mNativeInterface.reset(); +} + std::string NativeInterface::getError() { return mNativeInterface->getError(); } From feb3b0ef78ec29153d617d4788f8c76ac86de887 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Tue, 12 Sep 2017 17:05:08 +0200 Subject: [PATCH 11/37] try msvs 2015 --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index f587668d..de7e0753 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,7 +15,7 @@ install: # Get the latest stable version of Node.js or io.js - ps: Install-Product node $env:nodejs_version # install modules - - npm install --msvs_version=2013 + - npm install --msvs_version=2015 # Post-install test scripts. test_script: From e7a94c318dd62620888e507a86b328aa4280d9d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Wed, 13 Sep 2017 09:35:43 +0200 Subject: [PATCH 12/37] FIX: build on macOS with node.js v4 --- binding.gyp | 1 + 1 file changed, 1 insertion(+) diff --git a/binding.gyp b/binding.gyp index 8b76be71..8e2f0f98 100644 --- a/binding.gyp +++ b/binding.gyp @@ -42,6 +42,7 @@ "includes/osx/FSEventsService.h" ], "xcode_settings": { + 'MACOSX_DEPLOYMENT_TARGET': '10.7', "OTHER_CFLAGS": [ "-std=c++0x", "-stdlib=libc++" From bfe9eee75b6613bca046500a1fafd5a324e73b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Wed, 13 Sep 2017 09:36:24 +0200 Subject: [PATCH 13/37] switch from c++0x to proper c++11 --- binding.gyp | 4 ++-- includes/Queue.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/binding.gyp b/binding.gyp index 8e2f0f98..04e1eb08 100644 --- a/binding.gyp +++ b/binding.gyp @@ -44,7 +44,7 @@ "xcode_settings": { 'MACOSX_DEPLOYMENT_TARGET': '10.7', "OTHER_CFLAGS": [ - "-std=c++0x", + "-std=c++11", "-stdlib=libc++" ], }, @@ -72,7 +72,7 @@ ], "cflags": [ "-Wno-unknown-pragmas", - "-std=c++0x" + "-std=c++11" ] }], ["OS=='win'", { diff --git a/includes/Queue.h b/includes/Queue.h index b2744edc..f7db63a3 100644 --- a/includes/Queue.h +++ b/includes/Queue.h @@ -38,7 +38,6 @@ class EventQueue { private: std::deque> queue; std::mutex mutex; - }; #endif From 4978e9a8283cdd3eede911fead9a683ae37b2f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Wed, 13 Sep 2017 09:42:58 +0200 Subject: [PATCH 14/37] Refactor: add a single shot semaphore for threadsafe state announcement --- includes/SingleshotSemaphore.h | 33 +++++++++++++++++++++++++++++++++ includes/osx/RunLoop.h | 7 ++----- src/CMakeLists.txt | 1 + src/SingleshotSemaphore.cpp | 24 ++++++++++++++++++++++++ src/osx/RunLoop.cpp | 17 +++-------------- 5 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 includes/SingleshotSemaphore.h create mode 100644 src/SingleshotSemaphore.cpp diff --git a/includes/SingleshotSemaphore.h b/includes/SingleshotSemaphore.h new file mode 100644 index 00000000..4acfc4f5 --- /dev/null +++ b/includes/SingleshotSemaphore.h @@ -0,0 +1,33 @@ +#ifndef NSFW_SEMAPHORE_H +#define NSFW_SEMAPHORE_H + +#include +#include + +class SingleshotSemaphore { + public: + SingleshotSemaphore() : mState(false) {} + + void wait() + { + std::unique_lock lk(mMutex); + + while (!mState) { + mCond.wait(lk); + } + } + + void signal() + { + std::unique_lock lk(mMutex); + mState = true; + mCond.notify_all(); + } + + private: + std::mutex mMutex; + std::condition_variable mCond; + bool mState; +}; + +#endif diff --git a/includes/osx/RunLoop.h b/includes/osx/RunLoop.h index 0ea962a9..11093a93 100644 --- a/includes/osx/RunLoop.h +++ b/includes/osx/RunLoop.h @@ -1,12 +1,11 @@ #ifndef NSFW_RUNLOOP_H #define NSFW_RUNLOOP_H -#include "../Lock.h" +#include "../SingleshotSemaphore.h" #include "FSEventsService.h" #include #include -#include #include void *scheduleRunLoopWork(void *runLoop); @@ -38,9 +37,7 @@ class RunLoop { std::string mPath; CFRunLoopRef mRunLoop; pthread_t mRunLoopThread; - std::condition_variable mReadyForCleanupCond; - std::mutex mReadyForCleanupMutex; - bool mReadyForCleanup; + SingleshotSemaphore mReadyForCleanup; bool mStarted; }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d390be1a..a4ab94d3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,6 +17,7 @@ if (UNIX) if (APPLE) message (STATUS "compiling macOS specific file system service") set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} + SingleshotSemaphore.cpp osx/RunLoop.cpp osx/FSEventsService.cpp ) diff --git a/src/SingleshotSemaphore.cpp b/src/SingleshotSemaphore.cpp new file mode 100644 index 00000000..cc41bb52 --- /dev/null +++ b/src/SingleshotSemaphore.cpp @@ -0,0 +1,24 @@ +#include "../includes/Semaphore.h" + +SingleshotSemaphore::SingleshotSemaphore() : mState(false) {} + +void Semaphore::wait() +{ + nsfw_sync::Guard lk(mMutex); + + while (!mState) { + wait(mMutex); + } +} + +void Semaphore::signal() +{ + std::unique_lock lk(mMutex); + mState = true; + mCond.notify_all(); +} + +void Semaphore::wait(nsfw_sync::Mutex &mutex) +{ + mCond.wait(mutex); +} diff --git a/src/osx/RunLoop.cpp b/src/osx/RunLoop.cpp index d22140fc..8ddf4a64 100644 --- a/src/osx/RunLoop.cpp +++ b/src/osx/RunLoop.cpp @@ -10,7 +10,6 @@ RunLoop::RunLoop(FSEventsService *eventsService, std::string path): mExited(false), mPath(path), mRunLoop(NULL), - mReadyForCleanup(false), mStarted(false) { mStarted = !pthread_create( &mRunLoopThread, @@ -29,13 +28,7 @@ RunLoop::~RunLoop() { return; } - { - std::unique_lock lk(mReadyForCleanupMutex); - while (!mReadyForCleanup) { - mReadyForCleanupCond.wait(lk); - } - } - + mReadyForCleanup.wait(); CFRunLoopStop(mRunLoop); pthread_join(mRunLoopThread, NULL); @@ -58,13 +51,9 @@ void RunLoop::work() { mRunLoop = CFRunLoopGetCurrent(); - __block auto *runLoopHasStartedMutex = &mReadyForCleanupMutex; - __block auto *runLoopHasStartedCond = &mReadyForCleanupCond; - __block auto *runLoopHasStarted = &mReadyForCleanup; + __block auto *runLoopHasStarted = &mReadyForCleanup; CFRunLoopPerformBlock(mRunLoop, kCFRunLoopDefaultMode, ^ { - std::unique_lock lk(*runLoopHasStartedMutex); - *runLoopHasStarted = true; - runLoopHasStartedCond->notify_all(); + runLoopHasStarted->signal(); }); CFRunLoopWakeUp(mRunLoop); From 747d16e8444602af072aab2eded8a2afeb0bd617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Wed, 13 Sep 2017 09:44:41 +0200 Subject: [PATCH 15/37] FIX: remove unnecessary implementation --- src/CMakeLists.txt | 1 - src/SingleshotSemaphore.cpp | 24 ------------------------ 2 files changed, 25 deletions(-) delete mode 100644 src/SingleshotSemaphore.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a4ab94d3..d390be1a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,7 +17,6 @@ if (UNIX) if (APPLE) message (STATUS "compiling macOS specific file system service") set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} - SingleshotSemaphore.cpp osx/RunLoop.cpp osx/FSEventsService.cpp ) diff --git a/src/SingleshotSemaphore.cpp b/src/SingleshotSemaphore.cpp deleted file mode 100644 index cc41bb52..00000000 --- a/src/SingleshotSemaphore.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "../includes/Semaphore.h" - -SingleshotSemaphore::SingleshotSemaphore() : mState(false) {} - -void Semaphore::wait() -{ - nsfw_sync::Guard lk(mMutex); - - while (!mState) { - wait(mMutex); - } -} - -void Semaphore::signal() -{ - std::unique_lock lk(mMutex); - mState = true; - mCond.notify_all(); -} - -void Semaphore::wait(nsfw_sync::Mutex &mutex) -{ - mCond.wait(mutex); -} From ed56d42b5aaf364c461c47115829b22f4de19ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Wed, 13 Sep 2017 13:42:11 +0200 Subject: [PATCH 16/37] FIX: code cosmetics --- includes/win32/ReadLoop.h | 46 ++-- includes/win32/ReadLoopRunner.h | 46 ++-- src/win32/ReadLoop.cpp | 221 ++++++++++---------- src/win32/ReadLoopRunner.cpp | 359 ++++++++++++++++---------------- 4 files changed, 336 insertions(+), 336 deletions(-) diff --git a/includes/win32/ReadLoop.h b/includes/win32/ReadLoop.h index 02000b0d..6010fda5 100644 --- a/includes/win32/ReadLoop.h +++ b/includes/win32/ReadLoop.h @@ -12,29 +12,29 @@ #include "../Queue.h" class ReadLoop { -public: - ReadLoop(EventQueue &queue, std::string path); - - static unsigned int WINAPI startReadLoop(LPVOID arg); - static void CALLBACK startRunner(__in ULONG_PTR arg); - static void CALLBACK killReadLoop(__in ULONG_PTR arg); - - std::string getError(); - bool hasErrored(); - bool isWatching(); - - ~ReadLoop(); -protected: - void run(); - void shutdown(); - std::shared_ptr *mRunner; - std::wstring mDirectory; - HANDLE mDirectoryHandle; - EventQueue &mQueue; -private: - unsigned int mThreadID; - HANDLE mThread; - bool mRunning; + public: + ReadLoop(EventQueue &queue, std::string path); + + static unsigned int WINAPI startReadLoop(LPVOID arg); + static void CALLBACK startRunner(__in ULONG_PTR arg); + static void CALLBACK killReadLoop(__in ULONG_PTR arg); + + std::string getError(); + bool hasErrored(); + bool isWatching(); + + ~ReadLoop(); + protected: + void run(); + void shutdown(); + std::shared_ptr *mRunner; + std::wstring mDirectory; + HANDLE mDirectoryHandle; + EventQueue &mQueue; + private: + unsigned int mThreadID; + HANDLE mThread; + bool mRunning; }; #endif diff --git a/includes/win32/ReadLoopRunner.h b/includes/win32/ReadLoopRunner.h index 284e33bf..0b85fa79 100644 --- a/includes/win32/ReadLoopRunner.h +++ b/includes/win32/ReadLoopRunner.h @@ -10,31 +10,31 @@ #define BUFFER_KB 1024 class ReadLoopRunner { -public: - ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle); - ~ReadLoopRunner(); + public: + ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle); + ~ReadLoopRunner(); - static VOID CALLBACK eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped); - std::string getError(); - std::string getUTF8Directory(std::wstring path); - void handleEvents(); - bool hasErrored(); - BOOL read(); - void resizeBuffers(unsigned int bufferSize); - void setError(std::string error); - void setSharedPointer(std::shared_ptr *pointer); - void swap(DWORD numBytes); - void prepareForShutdown(); + static VOID CALLBACK eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped); + std::string getError(); + std::string getUTF8Directory(std::wstring path); + void handleEvents(); + bool hasErrored(); + BOOL read(); + void resizeBuffers(unsigned int bufferSize); + void setError(std::string error); + void setSharedPointer(std::shared_ptr *pointer); + void swap(DWORD numBytes); + void prepareForShutdown(); -private: - unsigned int mBufferSize; - BYTE *mBuffer; - std::wstring mDirectory; - HANDLE mDirectoryHandle; - std::string mErrorMessage; - OVERLAPPED mOverlapped; - BYTE *mSwap; - EventQueue &mQueue; + private: + unsigned int mBufferSize; + BYTE *mBuffer; + std::wstring mDirectory; + HANDLE mDirectoryHandle; + std::string mErrorMessage; + OVERLAPPED mOverlapped; + BYTE *mSwap; + EventQueue &mQueue; }; #endif diff --git a/src/win32/ReadLoop.cpp b/src/win32/ReadLoop.cpp index 1263b45c..09bd1b7e 100644 --- a/src/win32/ReadLoop.cpp +++ b/src/win32/ReadLoop.cpp @@ -1,140 +1,141 @@ #include "../../includes/win32/ReadLoop.h" -ReadLoop::ReadLoop(EventQueue &queue, std::string path): - mDirectoryHandle(NULL), - mQueue(queue), - mRunner(NULL), - mThread(NULL), - mThreadID(0) { - - int wlen = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, 0, 0); - - if (wlen == 0) { - return; - } - - LPWSTR outputBuffer = new WCHAR[wlen](); - - int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, outputBuffer, wlen); - - if (failureToResolveUTF8 == 0) { - delete[] outputBuffer; - return; - } - - mDirectoryHandle = CreateFileW( - outputBuffer, - FILE_LIST_DIRECTORY, - FILE_SHARE_READ - | FILE_SHARE_WRITE - | FILE_SHARE_DELETE, - NULL, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS - | FILE_FLAG_OVERLAPPED, - NULL - ); - - mDirectory = outputBuffer; - - delete[] outputBuffer; - outputBuffer = NULL; - - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - mDirectoryHandle = NULL; - return; - } - - // TODO: handle errors - mThread = (HANDLE)_beginthreadex( - NULL, - 0, - ReadLoop::startReadLoop, - this, - 0, - &mThreadID - ); - - if (!mThread) { - CloseHandle(mDirectoryHandle); - mDirectoryHandle = NULL; - mThread = NULL; - return; - } - - QueueUserAPC(ReadLoop::startRunner, mThread, (ULONG_PTR)this); - int maxWait = 10 * 1000; // after 10 seconds, abandon hope - while (mRunner == NULL && maxWait > 0) { - Sleep(50); - maxWait--; - } +ReadLoop::ReadLoop(EventQueue &queue, std::string path) + : mDirectoryHandle(NULL) + , mQueue(queue), + , mRunner(NULL), + , mThread(NULL), + , mThreadID(0) +{ + + int wlen = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, 0, 0); + + if (wlen == 0) { + return; + } + + LPWSTR outputBuffer = new WCHAR[wlen](); + + int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, outputBuffer, wlen); + + if (failureToResolveUTF8 == 0) { + delete[] outputBuffer; + return; + } + + mDirectoryHandle = CreateFileW( + outputBuffer, + FILE_LIST_DIRECTORY, + FILE_SHARE_READ + | FILE_SHARE_WRITE + | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS + | FILE_FLAG_OVERLAPPED, + NULL + ); + + mDirectory = outputBuffer; + + delete[] outputBuffer; + outputBuffer = NULL; + + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + mDirectoryHandle = NULL; + return; + } + + // TODO: handle errors + mThread = (HANDLE)_beginthreadex( + NULL, + 0, + ReadLoop::startReadLoop, + this, + 0, + &mThreadID + ); + + if (!mThread) { + CloseHandle(mDirectoryHandle); + mDirectoryHandle = NULL; + mThread = NULL; + return; + } + + QueueUserAPC(ReadLoop::startRunner, mThread, (ULONG_PTR)this); + int maxWait = 10 * 1000; // after 10 seconds, abandon hope + while (mRunner == NULL && maxWait > 0) { + Sleep(50); + maxWait--; + } } std::string ReadLoop::getError() { - if (mDirectoryHandle == NULL || mThread == NULL) { - return "Failed to start watcher"; - } else if (mRunner == NULL) { - return "Watcher is not started"; - } else { - return mRunner->get()->getError(); - } + if (mDirectoryHandle == NULL || mThread == NULL) { + return "Failed to start watcher"; + } else if (mRunner == NULL) { + return "Watcher is not started"; + } else { + return mRunner->get()->getError(); + } } bool ReadLoop::hasErrored() { - return mDirectoryHandle == NULL || mThread == NULL || (mRunner != NULL && mRunner->get()->hasErrored()); + return mDirectoryHandle == NULL || mThread == NULL || (mRunner != NULL && mRunner->get()->hasErrored()); } bool ReadLoop::isWatching() { - return mDirectoryHandle != NULL && mThread != NULL && mRunner != NULL && !mRunner->get()->hasErrored(); + return mDirectoryHandle != NULL && mThread != NULL && mRunner != NULL && !mRunner->get()->hasErrored(); } unsigned int WINAPI ReadLoop::startReadLoop(LPVOID arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->run(); - return 0; + ReadLoop *readLoop = (ReadLoop *)arg; + readLoop->run(); + return 0; } void ReadLoop::run() { - while(mDirectoryHandle != NULL) { - SleepEx(INFINITE, true); - } + while(mDirectoryHandle != NULL) { + SleepEx(INFINITE, true); + } } void CALLBACK ReadLoop::startRunner(__in ULONG_PTR arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->mRunner = new std::shared_ptr(new ReadLoopRunner( - readLoop->mDirectory, - readLoop->mQueue, - readLoop->mDirectoryHandle - )); - readLoop->mRunner->get()->setSharedPointer(readLoop->mRunner); - readLoop->mRunner->get()->read(); + ReadLoop *readLoop = (ReadLoop *)arg; + readLoop->mRunner = new std::shared_ptr(new ReadLoopRunner( + readLoop->mDirectory, + readLoop->mQueue, + readLoop->mDirectoryHandle + )); + readLoop->mRunner->get()->setSharedPointer(readLoop->mRunner); + readLoop->mRunner->get()->read(); } void CALLBACK ReadLoop::killReadLoop(__in ULONG_PTR arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->shutdown(); + ReadLoop *readLoop = (ReadLoop *)arg; + readLoop->shutdown(); } void ReadLoop::shutdown() { - if (mRunner != NULL) { - mRunner->get()->prepareForShutdown(); - delete mRunner; - mRunner = NULL; - } - - CancelIo(mDirectoryHandle); - CloseHandle(mDirectoryHandle); - mDirectoryHandle = NULL; + if (mRunner != NULL) { + mRunner->get()->prepareForShutdown(); + delete mRunner; + mRunner = NULL; + } + + CancelIo(mDirectoryHandle); + CloseHandle(mDirectoryHandle); + mDirectoryHandle = NULL; } ReadLoop::~ReadLoop() { - if (mThread) { - QueueUserAPC(ReadLoop::killReadLoop, mThread, (ULONG_PTR)this); - WaitForSingleObjectEx(mThread, 10000, true); - CloseHandle(mThread); - - mThread = NULL; - mThreadID = 0; - } + if (mThread) { + QueueUserAPC(ReadLoop::killReadLoop, mThread, (ULONG_PTR)this); + WaitForSingleObjectEx(mThread, 10000, true); + CloseHandle(mThread); + + mThread = NULL; + mThreadID = 0; + } } diff --git a/src/win32/ReadLoopRunner.cpp b/src/win32/ReadLoopRunner.cpp index f00999ec..59396054 100644 --- a/src/win32/ReadLoopRunner.cpp +++ b/src/win32/ReadLoopRunner.cpp @@ -17,229 +17,228 @@ ReadLoopRunner::~ReadLoopRunner() { } VOID CALLBACK ReadLoopRunner::eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { - std::shared_ptr *runner = (std::shared_ptr *)overlapped->hEvent; - - if (errorCode != ERROR_SUCCESS) { - if (errorCode == ERROR_NOTIFY_ENUM_DIR) { - runner->get()->setError("Buffer filled up and service needs a restart"); - } else if (errorCode == ERROR_INVALID_PARAMETER) { - // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission - runner->get()->resizeBuffers(64); - if (!runner->get()->read()) { + std::shared_ptr *runner = (std::shared_ptr *)overlapped->hEvent; + + if (errorCode != ERROR_SUCCESS) { + if (errorCode == ERROR_NOTIFY_ENUM_DIR) { + runner->get()->setError("Buffer filled up and service needs a restart"); + } else if (errorCode == ERROR_INVALID_PARAMETER) { + // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission + runner->get()->resizeBuffers(64); + if (!runner->get()->read()) { + delete (std::shared_ptr *)runner; + } + return; + } else { + runner->get()->setError("Service shutdown unexpectedly"); + } + delete (std::shared_ptr *)runner; - } - return; - } else { - runner->get()->setError("Service shutdown unexpectedly"); + return; } - delete (std::shared_ptr *)runner; - return; - } - runner->get()->swap(numBytes); - BOOL readRequested = runner->get()->read(); - runner->get()->handleEvents(); + runner->get()->swap(numBytes); + BOOL readRequested = runner->get()->read(); + runner->get()->handleEvents(); - if (!readRequested) { - delete (std::shared_ptr *)runner; - } + if (!readRequested) { + delete (std::shared_ptr *)runner; + } } std::string ReadLoopRunner::getError() { - return mErrorMessage; + return mErrorMessage; } std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { - LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); - memcpy(nullTerminatedFileName, cFileName, length); - std::wstring fileName = nullTerminatedFileName; - delete[] nullTerminatedFileName; - return fileName; + LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); + memcpy(nullTerminatedFileName, cFileName, length); + std::wstring fileName = nullTerminatedFileName; + delete[] nullTerminatedFileName; + return fileName; } std::string ReadLoopRunner::getUTF8Directory(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - std::wstringstream utf16DirectoryStream; - - utf16DirectoryStream << mDirectory; - - if (found != std::wstring::npos) { - utf16DirectoryStream - << "\\" - << path.substr(0, found); - } - - std::wstring uft16DirectoryString = utf16DirectoryStream.str(); - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - 0, - 0, - NULL, - NULL - ); - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} + std::wstring::size_type found = path.rfind('\\'); + std::wstringstream utf16DirectoryStream; -std::string getUTF8FileName(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - if (found != std::wstring::npos) { - path = path.substr(found + 1); - } - - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - 0, - 0, - NULL, - NULL - ); + utf16DirectoryStream << mDirectory; - // TODO: failure cases for widechar conversion - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL + if (found != std::wstring::npos) { + utf16DirectoryStream + << "\\" + << path.substr(0, found); + } + + std::wstring uft16DirectoryString = utf16DirectoryStream.str(); + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + 0, + 0, + NULL, + NULL + ); + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL ); - std::string utf8Directory = utf8CString; - delete[] utf8CString; + std::string utf8Directory = utf8CString; + delete[] utf8CString; - return utf8Directory; + return utf8Directory; } -void ReadLoopRunner::handleEvents() { - BYTE *base = mSwap; - for (;;) { - PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; - std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); +std::string getUTF8FileName(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + if (found != std::wstring::npos) { + path = path.substr(found + 1); + } - if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { - if (info->NextEntryOffset != 0) { - base += info->NextEntryOffset; - info = (PFILE_NOTIFY_INFORMATION)base; - if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { - std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); - - mQueue.enqueue( - RENAMED, - getUTF8Directory(fileName), - getUTF8FileName(fileName), - getUTF8FileName(fileNameNew) - ); - } - else { - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - continue; + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + 0, + 0, + NULL, + NULL + ); + + // TODO: failure cases for widechar conversion + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; +} + +void ReadLoopRunner::handleEvents() { + BYTE *base = mSwap; + for (;;) { + PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; + std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); + + if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { + if (info->NextEntryOffset != 0) { + base += info->NextEntryOffset; + info = (PFILE_NOTIFY_INFORMATION)base; + if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { + std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); + + mQueue.enqueue( + RENAMED, + getUTF8Directory(fileName), + getUTF8FileName(fileName), + getUTF8FileName(fileNameNew) + ); + } else { + mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + continue; + } + } else { + mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + } } - } - else { - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - } - } - switch (info->Action) { - case FILE_ACTION_ADDED: - case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer - mQueue.enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_REMOVED: - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_MODIFIED: - default: - mQueue.enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - }; - - if (info->NextEntryOffset == 0) { - break; + switch (info->Action) { + case FILE_ACTION_ADDED: + case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer + mQueue.enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_REMOVED: + mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_MODIFIED: + default: + mQueue.enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + }; + + if (info->NextEntryOffset == 0) { + break; + } + base += info->NextEntryOffset; } - base += info->NextEntryOffset; - } } bool ReadLoopRunner::hasErrored() { - return mErrorMessage != ""; + return mErrorMessage != ""; } BOOL ReadLoopRunner::read() { - DWORD bytes; - - if (mDirectoryHandle == NULL) { - return FALSE; - } - - if (!ReadDirectoryChangesW( - mDirectoryHandle, - mBuffer, - mBufferSize * BUFFER_KB, - TRUE, - FILE_NOTIFY_CHANGE_FILE_NAME - | FILE_NOTIFY_CHANGE_DIR_NAME - | FILE_NOTIFY_CHANGE_ATTRIBUTES - | FILE_NOTIFY_CHANGE_SIZE - | FILE_NOTIFY_CHANGE_LAST_WRITE - | FILE_NOTIFY_CHANGE_LAST_ACCESS - | FILE_NOTIFY_CHANGE_CREATION - | FILE_NOTIFY_CHANGE_SECURITY, - &bytes, - &mOverlapped, - &ReadLoopRunner::eventCallback - )) { - setError("Service shutdown unexpectedly"); - return FALSE; - } - - return TRUE; + DWORD bytes; + + if (mDirectoryHandle == NULL) { + return FALSE; + } + + if (!ReadDirectoryChangesW( + mDirectoryHandle, + mBuffer, + mBufferSize * BUFFER_KB, + TRUE, + FILE_NOTIFY_CHANGE_FILE_NAME + | FILE_NOTIFY_CHANGE_DIR_NAME + | FILE_NOTIFY_CHANGE_ATTRIBUTES + | FILE_NOTIFY_CHANGE_SIZE + | FILE_NOTIFY_CHANGE_LAST_WRITE + | FILE_NOTIFY_CHANGE_LAST_ACCESS + | FILE_NOTIFY_CHANGE_CREATION + | FILE_NOTIFY_CHANGE_SECURITY, + &bytes, + &mOverlapped, + &ReadLoopRunner::eventCallback)) + { + setError("Service shutdown unexpectedly"); + return FALSE; + } + + return TRUE; } void ReadLoopRunner::resizeBuffers(unsigned int bufferSize) { - mBufferSize = bufferSize; - delete[] mBuffer; - delete[] mSwap; - mBuffer = new BYTE[mBufferSize * BUFFER_KB]; - mSwap = new BYTE[mBufferSize * BUFFER_KB]; + mBufferSize = bufferSize; + delete[] mBuffer; + delete[] mSwap; + mBuffer = new BYTE[mBufferSize * BUFFER_KB]; + mSwap = new BYTE[mBufferSize * BUFFER_KB]; } void ReadLoopRunner::setError(std::string error) { - mErrorMessage = error; + mErrorMessage = error; } void ReadLoopRunner::setSharedPointer(std::shared_ptr *ptr) { - mOverlapped.hEvent = new std::shared_ptr(*ptr); + mOverlapped.hEvent = new std::shared_ptr(*ptr); } void ReadLoopRunner::swap(DWORD numBytes) { - memcpy(mSwap, mBuffer, numBytes); + memcpy(mSwap, mBuffer, numBytes); } void ReadLoopRunner::prepareForShutdown() { - mDirectoryHandle = NULL; + mDirectoryHandle = NULL; } From 1e667d08385a795e989cbb8e1d5a6c0184d57efc Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 13 Sep 2017 15:10:16 +0200 Subject: [PATCH 17/37] refactor the windows native implementation --- binding.gyp | 8 +- includes/NativeInterface.h | 4 +- includes/Queue.h | 1 - includes/win32/ReadLoopRunner.h | 2 + includes/win32_2/Controller.h | 37 +++++ includes/win32_2/ReadLoop.h | 40 +++++ includes/win32_2/ReadLoopRunner.h | 42 +++++ includes/win32_2/Watcher.h | 40 +++++ src/CMakeLists.txt | 10 +- src/win32/ReadLoop.cpp | 6 +- src/win32_2/Controller.cpp | 44 ++++++ src/win32_2/ReadLoop.cpp | 141 +++++++++++++++++ src/win32_2/ReadLoopRunner.cpp | 244 ++++++++++++++++++++++++++++++ src/win32_2/Watcher.cpp | 42 +++++ 14 files changed, 649 insertions(+), 12 deletions(-) create mode 100644 includes/win32_2/Controller.h create mode 100644 includes/win32_2/ReadLoop.h create mode 100644 includes/win32_2/ReadLoopRunner.h create mode 100644 includes/win32_2/Watcher.h create mode 100644 src/win32_2/Controller.cpp create mode 100644 src/win32_2/ReadLoop.cpp create mode 100644 src/win32_2/ReadLoopRunner.cpp create mode 100644 src/win32_2/Watcher.cpp diff --git a/binding.gyp b/binding.gyp index 397856ae..d21b02b4 100644 --- a/binding.gyp +++ b/binding.gyp @@ -18,10 +18,10 @@ "conditions": [ ["OS=='win'", { "sources": [ - "src/win32/ReadLoop.cpp", - "src/win32/ReadLoopRunner.cpp", - "includes/win32/ReadLoop.h", - "includes/win32/ReadLoopRunner.h" + "src/win32_2/Controller.cpp", + "src/win32_2/Watcher.cpp", + "includes/win32_2/Controller.h", + "includes/win32_2/Watcher.h" ], "msvs_settings": { "VCCLCompilerTool": { diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index 160cfcf4..51bfd53b 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -2,8 +2,8 @@ #define NSFW_NATIVE_INTERFACE_H #if defined(_WIN32) -#include "../includes/win32/ReadLoop.h" -using NativeImplementation = ReadLoop; +#include "../includes/win32_2/Controller.h" +using NativeImplementation = Controller; #elif defined(__APPLE_CC__) #include "../includes/osx/FSEventsService.h" using NativeImplementation = FSEventsService; diff --git a/includes/Queue.h b/includes/Queue.h index f7db63a3..1d25967d 100644 --- a/includes/Queue.h +++ b/includes/Queue.h @@ -23,7 +23,6 @@ struct Event { class EventQueue { public: - void clear(); std::size_t count(); std::unique_ptr dequeue(); diff --git a/includes/win32/ReadLoopRunner.h b/includes/win32/ReadLoopRunner.h index 0b85fa79..04efc11e 100644 --- a/includes/win32/ReadLoopRunner.h +++ b/includes/win32/ReadLoopRunner.h @@ -6,6 +6,8 @@ #include #include "../Queue.h" +#include + // limited by over-the-network file watching #define BUFFER_KB 1024 diff --git a/includes/win32_2/Controller.h b/includes/win32_2/Controller.h new file mode 100644 index 00000000..bfc92f48 --- /dev/null +++ b/includes/win32_2/Controller.h @@ -0,0 +1,37 @@ +#ifndef NSFW_WIN32_CONTROLLER +#define NSFW_WIN32_CONTROLLER + +//#include "../Queue.h" +#include +#include +#include "Watcher.h" + +class EventQueue; + +class Controller { + public: + Controller(EventQueue &queue, const std::string &path); + + std::string getError(); + bool hasErrored(); + bool isWatching(); + + ~Controller(); + private: + std::unique_ptr mWatcher; +// protected: +// void run(); +// void shutdown(); +// std::shared_ptr *mRunner; +// std::wstring mDirectory; +// HANDLE mDirectoryHandle; +// EventQueue &mQueue; +// private: +// unsigned int mThreadID; +// HANDLE mThread; +// bool mRunning; + + +}; + +#endif diff --git a/includes/win32_2/ReadLoop.h b/includes/win32_2/ReadLoop.h new file mode 100644 index 00000000..6010fda5 --- /dev/null +++ b/includes/win32_2/ReadLoop.h @@ -0,0 +1,40 @@ +#ifndef READ_LOOP_H +#define READ_LOOP_H + +#include +#include +#include "ReadLoopRunner.h" + +#include +#include +#include + +#include "../Queue.h" + +class ReadLoop { + public: + ReadLoop(EventQueue &queue, std::string path); + + static unsigned int WINAPI startReadLoop(LPVOID arg); + static void CALLBACK startRunner(__in ULONG_PTR arg); + static void CALLBACK killReadLoop(__in ULONG_PTR arg); + + std::string getError(); + bool hasErrored(); + bool isWatching(); + + ~ReadLoop(); + protected: + void run(); + void shutdown(); + std::shared_ptr *mRunner; + std::wstring mDirectory; + HANDLE mDirectoryHandle; + EventQueue &mQueue; + private: + unsigned int mThreadID; + HANDLE mThread; + bool mRunning; +}; + +#endif diff --git a/includes/win32_2/ReadLoopRunner.h b/includes/win32_2/ReadLoopRunner.h new file mode 100644 index 00000000..04efc11e --- /dev/null +++ b/includes/win32_2/ReadLoopRunner.h @@ -0,0 +1,42 @@ +#ifndef READ_LOOP_RUNNER_H +#define READ_LOOP_RUNNER_H + +#include +#include +#include +#include "../Queue.h" + +#include + +// limited by over-the-network file watching +#define BUFFER_KB 1024 + +class ReadLoopRunner { + public: + ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle); + ~ReadLoopRunner(); + + static VOID CALLBACK eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped); + std::string getError(); + std::string getUTF8Directory(std::wstring path); + void handleEvents(); + bool hasErrored(); + BOOL read(); + void resizeBuffers(unsigned int bufferSize); + void setError(std::string error); + void setSharedPointer(std::shared_ptr *pointer); + void swap(DWORD numBytes); + void prepareForShutdown(); + + private: + unsigned int mBufferSize; + BYTE *mBuffer; + std::wstring mDirectory; + HANDLE mDirectoryHandle; + std::string mErrorMessage; + OVERLAPPED mOverlapped; + BYTE *mSwap; + EventQueue &mQueue; +}; + +#endif diff --git a/includes/win32_2/Watcher.h b/includes/win32_2/Watcher.h new file mode 100644 index 00000000..5737a727 --- /dev/null +++ b/includes/win32_2/Watcher.h @@ -0,0 +1,40 @@ +#ifndef NSFW_WIN32_WATCHER +#define NSFW_WIN32_WATCHER + +#include +#include +#include +#include +#include +#include + +#include "../SingleshotSemaphore.h" +#include "../Queue.h" + +class Watcher +{ + public: + Watcher(EventQueue &queue, HANDLE dirHandle, const std::wstring &path); + ~Watcher(); + + const std::wstring mPath; + EventQueue mQueue; + HANDLE mDirectoryHandle; + + std::thread mRunner; + + bool isRunning() const { return mRunning; } + + private: + void run(); + + void start(); + void stop(); + + std::atomic mRunning; + SingleshotSemaphore mHasStartedSemaphore; + +}; + + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d390be1a..24be8793 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,13 +3,19 @@ set (NSFW_LIBRARY_SOURCES Queue.cpp NativeInterface.cpp +#FileSystemWatcher.cpp ) if (WIN32) message (STATUS "compiling windows specific file system service") set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} - win32/ReadLoop.cpp - win32/ReadLoopRunner.cpp + #win32/ReadLoop.cpp + #win32/ReadLoopRunner.cpp + win32_2/ReadLoop.cpp + win32_2/ReadLoopRunner.cpp + win32_2/Controller.cpp + win32_2/Watcher.cpp + ) endif(WIN32) diff --git a/src/win32/ReadLoop.cpp b/src/win32/ReadLoop.cpp index 09bd1b7e..cfecdbb6 100644 --- a/src/win32/ReadLoop.cpp +++ b/src/win32/ReadLoop.cpp @@ -2,9 +2,9 @@ ReadLoop::ReadLoop(EventQueue &queue, std::string path) : mDirectoryHandle(NULL) - , mQueue(queue), - , mRunner(NULL), - , mThread(NULL), + , mQueue(queue) + , mRunner(NULL) + , mThread(NULL) , mThreadID(0) { diff --git a/src/win32_2/Controller.cpp b/src/win32_2/Controller.cpp new file mode 100644 index 00000000..23793eb0 --- /dev/null +++ b/src/win32_2/Controller.cpp @@ -0,0 +1,44 @@ +#include "../includes/win32_2/Controller.h" + + +static std::wstring convertMultiByteToWideChar(const std::string &multiByte) +{ + const int wlen = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, 0, 0); + + if (wlen == 0) { + return std::wstring(); + } + + std::wstring wideString; + wideString.reserve(wlen); + int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, const_cast(wideString.data()), wlen); + if (failureToResolveUTF8 == 0) { + return std::wstring(); + } + return wideString; +} + +Controller::Controller(EventQueue &queue, const std::string &path) +{ + auto wideString = convertMultiByteToWideChar(path); + mWatcher.reset(new Watcher(queue, nullptr, wideString)); +} + +Controller::~Controller() +{ +} + +std::string Controller::getError() +{ + return ""; +} + +bool Controller::hasErrored() +{ + return false; +} + +bool Controller::isWatching() +{ + return mWatcher->isRunning(); +} diff --git a/src/win32_2/ReadLoop.cpp b/src/win32_2/ReadLoop.cpp new file mode 100644 index 00000000..cfecdbb6 --- /dev/null +++ b/src/win32_2/ReadLoop.cpp @@ -0,0 +1,141 @@ +#include "../../includes/win32/ReadLoop.h" + +ReadLoop::ReadLoop(EventQueue &queue, std::string path) + : mDirectoryHandle(NULL) + , mQueue(queue) + , mRunner(NULL) + , mThread(NULL) + , mThreadID(0) +{ + + int wlen = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, 0, 0); + + if (wlen == 0) { + return; + } + + LPWSTR outputBuffer = new WCHAR[wlen](); + + int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, outputBuffer, wlen); + + if (failureToResolveUTF8 == 0) { + delete[] outputBuffer; + return; + } + + mDirectoryHandle = CreateFileW( + outputBuffer, + FILE_LIST_DIRECTORY, + FILE_SHARE_READ + | FILE_SHARE_WRITE + | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS + | FILE_FLAG_OVERLAPPED, + NULL + ); + + mDirectory = outputBuffer; + + delete[] outputBuffer; + outputBuffer = NULL; + + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + mDirectoryHandle = NULL; + return; + } + + // TODO: handle errors + mThread = (HANDLE)_beginthreadex( + NULL, + 0, + ReadLoop::startReadLoop, + this, + 0, + &mThreadID + ); + + if (!mThread) { + CloseHandle(mDirectoryHandle); + mDirectoryHandle = NULL; + mThread = NULL; + return; + } + + QueueUserAPC(ReadLoop::startRunner, mThread, (ULONG_PTR)this); + int maxWait = 10 * 1000; // after 10 seconds, abandon hope + while (mRunner == NULL && maxWait > 0) { + Sleep(50); + maxWait--; + } +} + +std::string ReadLoop::getError() { + if (mDirectoryHandle == NULL || mThread == NULL) { + return "Failed to start watcher"; + } else if (mRunner == NULL) { + return "Watcher is not started"; + } else { + return mRunner->get()->getError(); + } +} + +bool ReadLoop::hasErrored() { + return mDirectoryHandle == NULL || mThread == NULL || (mRunner != NULL && mRunner->get()->hasErrored()); +} + +bool ReadLoop::isWatching() { + return mDirectoryHandle != NULL && mThread != NULL && mRunner != NULL && !mRunner->get()->hasErrored(); +} + +unsigned int WINAPI ReadLoop::startReadLoop(LPVOID arg) { + ReadLoop *readLoop = (ReadLoop *)arg; + readLoop->run(); + return 0; +} + +void ReadLoop::run() { + while(mDirectoryHandle != NULL) { + SleepEx(INFINITE, true); + } +} + +void CALLBACK ReadLoop::startRunner(__in ULONG_PTR arg) { + ReadLoop *readLoop = (ReadLoop *)arg; + readLoop->mRunner = new std::shared_ptr(new ReadLoopRunner( + readLoop->mDirectory, + readLoop->mQueue, + readLoop->mDirectoryHandle + )); + readLoop->mRunner->get()->setSharedPointer(readLoop->mRunner); + readLoop->mRunner->get()->read(); +} + +void CALLBACK ReadLoop::killReadLoop(__in ULONG_PTR arg) { + ReadLoop *readLoop = (ReadLoop *)arg; + readLoop->shutdown(); +} + +void ReadLoop::shutdown() { + if (mRunner != NULL) { + mRunner->get()->prepareForShutdown(); + delete mRunner; + mRunner = NULL; + } + + CancelIo(mDirectoryHandle); + CloseHandle(mDirectoryHandle); + mDirectoryHandle = NULL; +} + +ReadLoop::~ReadLoop() { + if (mThread) { + QueueUserAPC(ReadLoop::killReadLoop, mThread, (ULONG_PTR)this); + WaitForSingleObjectEx(mThread, 10000, true); + CloseHandle(mThread); + + mThread = NULL; + mThreadID = 0; + } +} diff --git a/src/win32_2/ReadLoopRunner.cpp b/src/win32_2/ReadLoopRunner.cpp new file mode 100644 index 00000000..59396054 --- /dev/null +++ b/src/win32_2/ReadLoopRunner.cpp @@ -0,0 +1,244 @@ +#include "../../includes/win32/ReadLoopRunner.h" + +ReadLoopRunner::ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle): + mBufferSize(1024), + mDirectory(directory), + mDirectoryHandle(directoryHandle), + mErrorMessage(""), + mQueue(queue) { + ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); + mBuffer = new BYTE[mBufferSize * BUFFER_KB]; + mSwap = new BYTE[mBufferSize * BUFFER_KB]; +} + +ReadLoopRunner::~ReadLoopRunner() { + delete[] mBuffer; + delete[] mSwap; +} + +VOID CALLBACK ReadLoopRunner::eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { + std::shared_ptr *runner = (std::shared_ptr *)overlapped->hEvent; + + if (errorCode != ERROR_SUCCESS) { + if (errorCode == ERROR_NOTIFY_ENUM_DIR) { + runner->get()->setError("Buffer filled up and service needs a restart"); + } else if (errorCode == ERROR_INVALID_PARAMETER) { + // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission + runner->get()->resizeBuffers(64); + if (!runner->get()->read()) { + delete (std::shared_ptr *)runner; + } + return; + } else { + runner->get()->setError("Service shutdown unexpectedly"); + } + + delete (std::shared_ptr *)runner; + return; + } + + runner->get()->swap(numBytes); + BOOL readRequested = runner->get()->read(); + runner->get()->handleEvents(); + + if (!readRequested) { + delete (std::shared_ptr *)runner; + } +} + +std::string ReadLoopRunner::getError() { + return mErrorMessage; +} + +std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { + LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); + memcpy(nullTerminatedFileName, cFileName, length); + std::wstring fileName = nullTerminatedFileName; + delete[] nullTerminatedFileName; + return fileName; +} + +std::string ReadLoopRunner::getUTF8Directory(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + std::wstringstream utf16DirectoryStream; + + utf16DirectoryStream << mDirectory; + + if (found != std::wstring::npos) { + utf16DirectoryStream + << "\\" + << path.substr(0, found); + } + + std::wstring uft16DirectoryString = utf16DirectoryStream.str(); + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + 0, + 0, + NULL, + NULL + ); + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; +} + +std::string getUTF8FileName(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + if (found != std::wstring::npos) { + path = path.substr(found + 1); + } + + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + 0, + 0, + NULL, + NULL + ); + + // TODO: failure cases for widechar conversion + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; +} + +void ReadLoopRunner::handleEvents() { + BYTE *base = mSwap; + for (;;) { + PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; + std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); + + if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { + if (info->NextEntryOffset != 0) { + base += info->NextEntryOffset; + info = (PFILE_NOTIFY_INFORMATION)base; + if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { + std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); + + mQueue.enqueue( + RENAMED, + getUTF8Directory(fileName), + getUTF8FileName(fileName), + getUTF8FileName(fileNameNew) + ); + } else { + mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + continue; + } + } else { + mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + } + } + + switch (info->Action) { + case FILE_ACTION_ADDED: + case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer + mQueue.enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_REMOVED: + mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_MODIFIED: + default: + mQueue.enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + }; + + if (info->NextEntryOffset == 0) { + break; + } + base += info->NextEntryOffset; + } +} + +bool ReadLoopRunner::hasErrored() { + return mErrorMessage != ""; +} + +BOOL ReadLoopRunner::read() { + DWORD bytes; + + if (mDirectoryHandle == NULL) { + return FALSE; + } + + if (!ReadDirectoryChangesW( + mDirectoryHandle, + mBuffer, + mBufferSize * BUFFER_KB, + TRUE, + FILE_NOTIFY_CHANGE_FILE_NAME + | FILE_NOTIFY_CHANGE_DIR_NAME + | FILE_NOTIFY_CHANGE_ATTRIBUTES + | FILE_NOTIFY_CHANGE_SIZE + | FILE_NOTIFY_CHANGE_LAST_WRITE + | FILE_NOTIFY_CHANGE_LAST_ACCESS + | FILE_NOTIFY_CHANGE_CREATION + | FILE_NOTIFY_CHANGE_SECURITY, + &bytes, + &mOverlapped, + &ReadLoopRunner::eventCallback)) + { + setError("Service shutdown unexpectedly"); + return FALSE; + } + + return TRUE; +} + +void ReadLoopRunner::resizeBuffers(unsigned int bufferSize) { + mBufferSize = bufferSize; + delete[] mBuffer; + delete[] mSwap; + mBuffer = new BYTE[mBufferSize * BUFFER_KB]; + mSwap = new BYTE[mBufferSize * BUFFER_KB]; +} + +void ReadLoopRunner::setError(std::string error) { + mErrorMessage = error; +} + +void ReadLoopRunner::setSharedPointer(std::shared_ptr *ptr) { + mOverlapped.hEvent = new std::shared_ptr(*ptr); +} + +void ReadLoopRunner::swap(DWORD numBytes) { + memcpy(mSwap, mBuffer, numBytes); +} + +void ReadLoopRunner::prepareForShutdown() { + mDirectoryHandle = NULL; +} diff --git a/src/win32_2/Watcher.cpp b/src/win32_2/Watcher.cpp new file mode 100644 index 00000000..c1f839ce --- /dev/null +++ b/src/win32_2/Watcher.cpp @@ -0,0 +1,42 @@ +#include "../includes/win32_2/Watcher.h" + +Watcher::Watcher(EventQueue &queue, HANDLE dirHandle, const std::wstring &path) + : mRunning(false) +{ + start(); +} + +Watcher::~Watcher() +{ + stop(); +} + +void Watcher::run() +{ + while(mRunning) { + SleepEx(INFINITE, true); + } +} + +void Watcher::start() +{ + mRunner = std::thread([this]{ + // mRunning is set to false in the d'tor + mRunning = true; + run(); + }); + + QueueUserAPC([](__in ULONG_PTR self) { + auto watcher = reinterpret_cast(self); + watcher->mHasStartedSemaphore.signal(); + + } , mRunner.native_handle(), (ULONG_PTR)this); + mHasStartedSemaphore.wait(); +} + +void Watcher::stop() +{ + mRunning = false; + QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); + mRunner.join(); +} From 422d46bb377d3351e7a051eb5a75edbe48c4a0f6 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 13 Sep 2017 15:52:10 +0200 Subject: [PATCH 18/37] implement error handling in watcher and controller --- includes/SingleshotSemaphore.h | 12 ++++++++++++ includes/win32_2/Watcher.h | 16 ++++++++++------ src/win32_2/Controller.cpp | 4 ++-- src/win32_2/Watcher.cpp | 28 ++++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/includes/SingleshotSemaphore.h b/includes/SingleshotSemaphore.h index 4acfc4f5..13a391e6 100644 --- a/includes/SingleshotSemaphore.h +++ b/includes/SingleshotSemaphore.h @@ -17,6 +17,18 @@ class SingleshotSemaphore { } } + bool waitFor(std::chrono::milliseconds ms) + { + std::unique_lock lk(mMutex); + + if (mState) { + return true; + } + + mCond.wait_for(lk, ms); + return mState; + } + void signal() { std::unique_lock lk(mMutex); diff --git a/includes/win32_2/Watcher.h b/includes/win32_2/Watcher.h index 5737a727..702f6053 100644 --- a/includes/win32_2/Watcher.h +++ b/includes/win32_2/Watcher.h @@ -17,13 +17,8 @@ class Watcher Watcher(EventQueue &queue, HANDLE dirHandle, const std::wstring &path); ~Watcher(); - const std::wstring mPath; - EventQueue mQueue; - HANDLE mDirectoryHandle; - - std::thread mRunner; - bool isRunning() const { return mRunning; } + std::string getError() const; private: void run(); @@ -31,9 +26,18 @@ class Watcher void start(); void stop(); + void setError(const std::string &error); + std::atomic mRunning; SingleshotSemaphore mHasStartedSemaphore; + mutable std::mutex mErrorMutex; + std::string mError; + + const std::wstring mPath; + EventQueue mQueue; + HANDLE mDirectoryHandle; + std::thread mRunner; }; diff --git a/src/win32_2/Controller.cpp b/src/win32_2/Controller.cpp index 23793eb0..d0e0fcdf 100644 --- a/src/win32_2/Controller.cpp +++ b/src/win32_2/Controller.cpp @@ -30,12 +30,12 @@ Controller::~Controller() std::string Controller::getError() { - return ""; + return mWatcher->getError(); } bool Controller::hasErrored() { - return false; + return !mWatcher->getError().empty(); } bool Controller::isWatching() diff --git a/src/win32_2/Watcher.cpp b/src/win32_2/Watcher.cpp index c1f839ce..cf4c27d7 100644 --- a/src/win32_2/Watcher.cpp +++ b/src/win32_2/Watcher.cpp @@ -26,12 +26,19 @@ void Watcher::start() run(); }); + if (!mRunner.joinable()) { + mRunning = false; + return; + } + QueueUserAPC([](__in ULONG_PTR self) { auto watcher = reinterpret_cast(self); watcher->mHasStartedSemaphore.signal(); - } , mRunner.native_handle(), (ULONG_PTR)this); - mHasStartedSemaphore.wait(); + + if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { + setError("Watcher is not started"); + } } void Watcher::stop() @@ -40,3 +47,20 @@ void Watcher::stop() QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); mRunner.join(); } + +void Watcher::setError(const std::string &error) +{ + std::lock_guard lock(mErrorMutex); + mError = error; +} + + +std::string Watcher::getError() const +{ + if (!isRunning()) { + return "Failed to start watcher"; + } + + std::lock_guard lock(mErrorMutex); + return mError; +} From def8e5a0a0fecd81b50fc81b3adb37dcb08c4b21 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 13 Sep 2017 17:16:51 +0200 Subject: [PATCH 19/37] use c++11 threads instead of native implementation --- includes/NativeInterface.h | 2 +- includes/win32_2/Controller.h | 6 +- includes/win32_2/Watcher.h | 15 ++- src/NativeInterface.cpp | 3 +- src/Queue.cpp | 1 + src/win32_2/Controller.cpp | 38 +++++- src/win32_2/Watcher.cpp | 217 +++++++++++++++++++++++++++++++++- 7 files changed, 271 insertions(+), 11 deletions(-) diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index 51bfd53b..93cf0725 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -26,7 +26,7 @@ class NativeInterface { bool isWatching(); private: - EventQueue mQueue; + std::shared_ptr mQueue; std::unique_ptr mNativeInterface; }; diff --git a/includes/win32_2/Controller.h b/includes/win32_2/Controller.h index bfc92f48..de6d06c2 100644 --- a/includes/win32_2/Controller.h +++ b/includes/win32_2/Controller.h @@ -10,7 +10,7 @@ class EventQueue; class Controller { public: - Controller(EventQueue &queue, const std::string &path); + Controller(std::shared_ptr queue, const std::string &path); std::string getError(); bool hasErrored(); @@ -19,6 +19,10 @@ class Controller { ~Controller(); private: std::unique_ptr mWatcher; + + HANDLE openDirectory(const std::wstring &path); + HANDLE mDirectoryHandle; + // protected: // void run(); // void shutdown(); diff --git a/includes/win32_2/Watcher.h b/includes/win32_2/Watcher.h index 702f6053..0f2e35b9 100644 --- a/includes/win32_2/Watcher.h +++ b/includes/win32_2/Watcher.h @@ -14,7 +14,7 @@ class Watcher { public: - Watcher(EventQueue &queue, HANDLE dirHandle, const std::wstring &path); + Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path); ~Watcher(); bool isRunning() const { return mRunning; } @@ -22,11 +22,17 @@ class Watcher private: void run(); - + bool loop(); void start(); void stop(); void setError(const std::string &error); + void eventCallback(DWORD errorCode, DWORD numBytes); + void handleEvents(); + + void resizeBuffers(std::size_t size); + + std::string getUTF8Directory(std::wstring path) ; std::atomic mRunning; SingleshotSemaphore mHasStartedSemaphore; @@ -34,9 +40,12 @@ class Watcher std::string mError; const std::wstring mPath; - EventQueue mQueue; + std::shared_ptr mQueue; HANDLE mDirectoryHandle; + std::vector mReadBuffer, mWriteBuffer; + OVERLAPPED mOverlapped; + std::thread mRunner; }; diff --git a/src/NativeInterface.cpp b/src/NativeInterface.cpp index 48870866..9bbb547d 100644 --- a/src/NativeInterface.cpp +++ b/src/NativeInterface.cpp @@ -1,6 +1,7 @@ #include "../includes/NativeInterface.h" NativeInterface::NativeInterface(const std::string &path) { + mQueue = std::make_shared(); mNativeInterface.reset(new NativeImplementation(mQueue, path)); } @@ -13,7 +14,7 @@ std::string NativeInterface::getError() { } std::vector* NativeInterface::getEvents() { - return mQueue.dequeueAll().release(); + return mQueue->dequeueAll().release(); } bool NativeInterface::hasErrored() { diff --git a/src/Queue.cpp b/src/Queue.cpp index 6bdb3138..340672ae 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -1,4 +1,5 @@ #include "../includes/Queue.h" +#include #pragma unmanaged diff --git a/src/win32_2/Controller.cpp b/src/win32_2/Controller.cpp index d0e0fcdf..889dbec3 100644 --- a/src/win32_2/Controller.cpp +++ b/src/win32_2/Controller.cpp @@ -18,24 +18,54 @@ static std::wstring convertMultiByteToWideChar(const std::string &multiByte) return wideString; } -Controller::Controller(EventQueue &queue, const std::string &path) +HANDLE Controller::openDirectory(const std::wstring &path) { - auto wideString = convertMultiByteToWideChar(path); - mWatcher.reset(new Watcher(queue, nullptr, wideString)); + return CreateFileW( + path.data(), + FILE_LIST_DIRECTORY, + FILE_SHARE_READ + | FILE_SHARE_WRITE + | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS + | FILE_FLAG_OVERLAPPED, + NULL + ); +} + +Controller::Controller(std::shared_ptr queue, const std::string &path) + : mDirectoryHandle(INVALID_HANDLE_VALUE) +{ + auto widePath = convertMultiByteToWideChar(path); + mDirectoryHandle = openDirectory(widePath); + + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + return; + } + + mWatcher.reset(new Watcher(queue, mDirectoryHandle, widePath)); } Controller::~Controller() { + mWatcher.reset(); + CancelIo(mDirectoryHandle); + CloseHandle(mDirectoryHandle); + mDirectoryHandle = INVALID_HANDLE_VALUE; } std::string Controller::getError() { + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + return "Failed to open directory"; + } return mWatcher->getError(); } bool Controller::hasErrored() { - return !mWatcher->getError().empty(); + return mDirectoryHandle == INVALID_HANDLE_VALUE || !mWatcher->getError().empty(); } bool Controller::isWatching() diff --git a/src/win32_2/Watcher.cpp b/src/win32_2/Watcher.cpp index cf4c27d7..28f18c07 100644 --- a/src/win32_2/Watcher.cpp +++ b/src/win32_2/Watcher.cpp @@ -1,8 +1,104 @@ #include "../includes/win32_2/Watcher.h" -Watcher::Watcher(EventQueue &queue, HANDLE dirHandle, const std::wstring &path) +#include + +static +std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { + LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); + memcpy(nullTerminatedFileName, cFileName, length); + std::wstring fileName = nullTerminatedFileName; + delete[] nullTerminatedFileName; + return fileName; +} + + +std::string Watcher::getUTF8Directory(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + std::wstringstream utf16DirectoryStream; + + utf16DirectoryStream << mPath; + + if (found != std::wstring::npos) { + utf16DirectoryStream + << "\\" + << path.substr(0, found); + } + + std::wstring uft16DirectoryString = utf16DirectoryStream.str(); + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + 0, + 0, + NULL, + NULL + ); + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; +} + +static +std::string getUTF8FileName(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + if (found != std::wstring::npos) { + path = path.substr(found + 1); + } + + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + 0, + 0, + NULL, + NULL + ); + + // TODO: failure cases for widechar conversion + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; +} + +Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path) : mRunning(false) + , mDirectoryHandle(dirHandle) + , mQueue(queue) + , mPath(path) { + ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); + mOverlapped.hEvent = this; + resizeBuffers(1024 * 1024); start(); } @@ -11,6 +107,12 @@ Watcher::~Watcher() stop(); } +void Watcher::resizeBuffers(std::size_t size) +{ + mReadBuffer.resize(size); + mWriteBuffer.resize(size); +} + void Watcher::run() { while(mRunning) { @@ -18,6 +120,118 @@ void Watcher::run() } } +bool Watcher::loop() +{ + DWORD bytes = 0; + + if (!isRunning()) { + return false; + } + + if (!ReadDirectoryChangesW( + mDirectoryHandle, + mWriteBuffer.data(), + mWriteBuffer.size(), + TRUE, //recursive watching + FILE_NOTIFY_CHANGE_FILE_NAME + | FILE_NOTIFY_CHANGE_DIR_NAME + | FILE_NOTIFY_CHANGE_ATTRIBUTES + | FILE_NOTIFY_CHANGE_SIZE + | FILE_NOTIFY_CHANGE_LAST_WRITE + | FILE_NOTIFY_CHANGE_LAST_ACCESS + | FILE_NOTIFY_CHANGE_CREATION + | FILE_NOTIFY_CHANGE_SECURITY, + &bytes, //num bytes written + &mOverlapped, + [](DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { + auto watcher = reinterpret_cast(overlapped->hEvent); + watcher->eventCallback(errorCode, numBytes); + })) + { + setError("Service shutdown unexpectedly"); + return false; + } + + return true; +} + +void Watcher::eventCallback(DWORD errorCode, DWORD numBytes) +{ + if (errorCode != ERROR_SUCCESS) { + if (errorCode == ERROR_NOTIFY_ENUM_DIR) { + setError("Buffer filled up and service needs a restart"); + } else if (errorCode == ERROR_INVALID_PARAMETER) { + // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission + resizeBuffers(64 * 1024); + + if (!loop()) { + setError("failed resizing buffers for network traffic"); + } + } else { + setError("Service shutdown unexpectedly"); + } + return; + } + + std::swap(mWriteBuffer, mReadBuffer); + BOOL readRequested = loop(); + handleEvents(); + + if (!readRequested) { + //delete (std::shared_ptr *)runner; + } +} + +void Watcher::handleEvents() +{ + BYTE *base = mReadBuffer.data(); + while (true) { + PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; + std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); + + if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { + if (info->NextEntryOffset != 0) { + base += info->NextEntryOffset; + info = (PFILE_NOTIFY_INFORMATION)base; + if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { + std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); + + mQueue->enqueue( + RENAMED, + getUTF8Directory(fileName), + getUTF8FileName(fileName), + getUTF8FileName(fileNameNew) + ); + } else { + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + continue; + } + } else { + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + } + } + + switch (info->Action) { + case FILE_ACTION_ADDED: + case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer + mQueue->enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_REMOVED: + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_MODIFIED: + default: + mQueue->enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + }; + + if (info->NextEntryOffset == 0) { + break; + } + base += info->NextEntryOffset; + } +} + void Watcher::start() { mRunner = std::thread([this]{ @@ -34,6 +248,7 @@ void Watcher::start() QueueUserAPC([](__in ULONG_PTR self) { auto watcher = reinterpret_cast(self); watcher->mHasStartedSemaphore.signal(); + watcher->loop(); } , mRunner.native_handle(), (ULONG_PTR)this); if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { From 5352ae882a9ab8eb3a8023add4d161cab828d1ea Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 13 Sep 2017 18:10:33 +0200 Subject: [PATCH 20/37] quickfix for windows js tests --- src/win32_2/Controller.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/win32_2/Controller.cpp b/src/win32_2/Controller.cpp index 889dbec3..93e72c9d 100644 --- a/src/win32_2/Controller.cpp +++ b/src/win32_2/Controller.cpp @@ -10,8 +10,8 @@ static std::wstring convertMultiByteToWideChar(const std::string &multiByte) } std::wstring wideString; - wideString.reserve(wlen); - int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, const_cast(wideString.data()), wlen); + wideString.resize(wlen-1); + int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, &(wideString[0]), wlen); if (failureToResolveUTF8 == 0) { return std::wstring(); } From 94156e7a6c9f10baeacd8dafd1178dd81d278d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Wed, 13 Sep 2017 18:51:50 +0200 Subject: [PATCH 21/37] FIX: linux and macOS with std::shared_ptr --- includes/linux/InotifyService.h | 4 ++-- includes/osx/FSEventsService.h | 4 ++-- src/linux/InotifyService.cpp | 6 +++--- src/osx/FSEventsService.cpp | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/includes/linux/InotifyService.h b/includes/linux/InotifyService.h index bda0c7ab..feca01f2 100644 --- a/includes/linux/InotifyService.h +++ b/includes/linux/InotifyService.h @@ -12,7 +12,7 @@ class InotifyTree; class InotifyService { public: - InotifyService(EventQueue &queue, std::string path); + InotifyService(std::shared_ptr queue, std::string path); std::string getError(); bool hasErrored(); @@ -32,7 +32,7 @@ class InotifyService { void renameDirectory(int wd, std::string oldName, std::string newName); InotifyEventLoop *mEventLoop; - EventQueue &mQueue; + std::shared_ptr mQueue; InotifyTree *mTree; int mInotifyInstance; diff --git a/includes/osx/FSEventsService.h b/includes/osx/FSEventsService.h index 6efb9c29..f2008b87 100644 --- a/includes/osx/FSEventsService.h +++ b/includes/osx/FSEventsService.h @@ -24,7 +24,7 @@ class RunLoop; class FSEventsService { public: - FSEventsService(EventQueue &queue, std::string path); + FSEventsService(std::shared_ptr queue, std::string path); friend void FSEventsServiceCallback( ConstFSEventStreamRef streamRef, @@ -50,7 +50,7 @@ class FSEventsService { std::string mPath; RunLoop *mRunLoop; - EventQueue &mQueue; + std::shared_ptr mQueue; }; #endif diff --git a/src/linux/InotifyService.cpp b/src/linux/InotifyService.cpp index b39f9b84..f5797bca 100644 --- a/src/linux/InotifyService.cpp +++ b/src/linux/InotifyService.cpp @@ -1,6 +1,6 @@ #include "../../includes/linux/InotifyService.h" -InotifyService::InotifyService(EventQueue &queue, std::string path): +InotifyService::InotifyService(std::shared_ptr queue, std::string path): mEventLoop(NULL), mQueue(queue), mTree(NULL) { @@ -45,7 +45,7 @@ void InotifyService::dispatch(EventType action, int wd, std::string name) { return; } - mQueue.enqueue(action, path, name); + mQueue->enqueue(action, path, name); } void InotifyService::dispatchRename(int wd, std::string oldName, std::string newName) { @@ -54,7 +54,7 @@ void InotifyService::dispatchRename(int wd, std::string oldName, std::string new return; } - mQueue.enqueue(RENAMED, path, oldName, newName); + mQueue->enqueue(RENAMED, path, oldName, newName); } std::string InotifyService::getError() { diff --git a/src/osx/FSEventsService.cpp b/src/osx/FSEventsService.cpp index d9646cf9..d77fd873 100644 --- a/src/osx/FSEventsService.cpp +++ b/src/osx/FSEventsService.cpp @@ -1,7 +1,7 @@ #include "../../includes/osx/FSEventsService.h" #include -FSEventsService::FSEventsService(EventQueue &queue, std::string path): +FSEventsService::FSEventsService(std::shared_ptr queue, std::string path): mPath(path), mQueue(queue) { mRunLoop = new RunLoop(this, path); @@ -78,7 +78,7 @@ void FSEventsService::dispatch(EventType action, std::string path) { splitFilePath(directory, name, path); - mQueue.enqueue(action, directory, name); + mQueue->enqueue(action, directory, name); } std::string FSEventsService::getError() { @@ -125,9 +125,9 @@ void FSEventsService::rename(std::vector *paths) { sideBExists = stat(fullSideB.c_str(), &renameSideB) == 0; if (sideAExists && !sideBExists) { - mQueue.enqueue(RENAMED, binIterator->first, sideB, sideA); + mQueue->enqueue(RENAMED, binIterator->first, sideB, sideA); } else if (!sideAExists && sideBExists) { - mQueue.enqueue(RENAMED, binIterator->first, sideA, sideB); + mQueue->enqueue(RENAMED, binIterator->first, sideA, sideB); } else { demangle(fullSideA); demangle(fullSideB); From ce29e834da98a8955abc46913fbbe663d5085a4c Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Thu, 14 Sep 2017 11:17:37 +0200 Subject: [PATCH 22/37] fix deadlock in nodejs cpp interface --- includes/NSFW.h | 3 ++- src/NSFW.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/includes/NSFW.h b/includes/NSFW.h index b036526b..e821618f 100644 --- a/includes/NSFW.h +++ b/includes/NSFW.h @@ -5,6 +5,7 @@ #include #include #include +#include using namespace Nan; @@ -28,7 +29,7 @@ class NSFW : public ObjectWrap { bool mInterfaceLockValid; std::string mPath; uv_thread_t mPollThread; - bool mRunning; + std::atomic mRunning; private: NSFW(uint32_t debounceMS, std::string path, Callback *eventCallback, Callback *errorCallback); ~NSFW(); diff --git a/src/NSFW.cpp b/src/NSFW.cpp index 184a2709..2f4afef1 100644 --- a/src/NSFW.cpp +++ b/src/NSFW.cpp @@ -291,16 +291,18 @@ NSFW::StopWorker::StopWorker(NSFW *nsfw, Callback *callback): void NSFW::StopWorker::Execute() { uv_mutex_lock(&mNSFW->mInterfaceLock); - if (mNSFW->mInterface == NULL) { - uv_mutex_unlock(&mNSFW->mInterfaceLock); + uv_mutex_unlock(&mNSFW->mInterfaceLock); return; } + uv_mutex_unlock(&mNSFW->mInterfaceLock); + // unlock the mInterfaceLock mutex while operate on the running identifier mNSFW->mRunning = false; uv_thread_join(&mNSFW->mPollThread); + uv_mutex_lock(&mNSFW->mInterfaceLock); delete mNSFW->mInterface; mNSFW->mInterface = NULL; From 72459d67171068c82f90df3fe49dcf25ce63ba9f Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Thu, 14 Sep 2017 14:39:04 +0200 Subject: [PATCH 23/37] delete unused files --- binding.gyp | 8 +- includes/win32/ReadLoop.h | 40 ----- includes/win32/ReadLoopRunner.h | 42 ----- includes/win32_2/Controller.h | 41 ----- includes/win32_2/ReadLoop.h | 40 ----- includes/win32_2/ReadLoopRunner.h | 42 ----- includes/win32_2/Watcher.h | 53 ------ src/CMakeLists.txt | 10 +- src/win32/ReadLoop.cpp | 141 --------------- src/win32/ReadLoopRunner.cpp | 244 -------------------------- src/win32_2/Controller.cpp | 74 -------- src/win32_2/ReadLoop.cpp | 141 --------------- src/win32_2/ReadLoopRunner.cpp | 244 -------------------------- src/win32_2/Watcher.cpp | 281 ------------------------------ 14 files changed, 6 insertions(+), 1395 deletions(-) delete mode 100644 includes/win32/ReadLoop.h delete mode 100644 includes/win32/ReadLoopRunner.h delete mode 100644 includes/win32_2/Controller.h delete mode 100644 includes/win32_2/ReadLoop.h delete mode 100644 includes/win32_2/ReadLoopRunner.h delete mode 100644 includes/win32_2/Watcher.h delete mode 100644 src/win32/ReadLoop.cpp delete mode 100644 src/win32/ReadLoopRunner.cpp delete mode 100644 src/win32_2/Controller.cpp delete mode 100644 src/win32_2/ReadLoop.cpp delete mode 100644 src/win32_2/ReadLoopRunner.cpp delete mode 100644 src/win32_2/Watcher.cpp diff --git a/binding.gyp b/binding.gyp index d21b02b4..9176483b 100644 --- a/binding.gyp +++ b/binding.gyp @@ -18,10 +18,10 @@ "conditions": [ ["OS=='win'", { "sources": [ - "src/win32_2/Controller.cpp", - "src/win32_2/Watcher.cpp", - "includes/win32_2/Controller.h", - "includes/win32_2/Watcher.h" + "src/win32/Controller.cpp", + "src/win32/Watcher.cpp", + "includes/win32/Controller.h", + "includes/win32/Watcher.h" ], "msvs_settings": { "VCCLCompilerTool": { diff --git a/includes/win32/ReadLoop.h b/includes/win32/ReadLoop.h deleted file mode 100644 index 6010fda5..00000000 --- a/includes/win32/ReadLoop.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef READ_LOOP_H -#define READ_LOOP_H - -#include -#include -#include "ReadLoopRunner.h" - -#include -#include -#include - -#include "../Queue.h" - -class ReadLoop { - public: - ReadLoop(EventQueue &queue, std::string path); - - static unsigned int WINAPI startReadLoop(LPVOID arg); - static void CALLBACK startRunner(__in ULONG_PTR arg); - static void CALLBACK killReadLoop(__in ULONG_PTR arg); - - std::string getError(); - bool hasErrored(); - bool isWatching(); - - ~ReadLoop(); - protected: - void run(); - void shutdown(); - std::shared_ptr *mRunner; - std::wstring mDirectory; - HANDLE mDirectoryHandle; - EventQueue &mQueue; - private: - unsigned int mThreadID; - HANDLE mThread; - bool mRunning; -}; - -#endif diff --git a/includes/win32/ReadLoopRunner.h b/includes/win32/ReadLoopRunner.h deleted file mode 100644 index 04efc11e..00000000 --- a/includes/win32/ReadLoopRunner.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef READ_LOOP_RUNNER_H -#define READ_LOOP_RUNNER_H - -#include -#include -#include -#include "../Queue.h" - -#include - -// limited by over-the-network file watching -#define BUFFER_KB 1024 - -class ReadLoopRunner { - public: - ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle); - ~ReadLoopRunner(); - - static VOID CALLBACK eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped); - std::string getError(); - std::string getUTF8Directory(std::wstring path); - void handleEvents(); - bool hasErrored(); - BOOL read(); - void resizeBuffers(unsigned int bufferSize); - void setError(std::string error); - void setSharedPointer(std::shared_ptr *pointer); - void swap(DWORD numBytes); - void prepareForShutdown(); - - private: - unsigned int mBufferSize; - BYTE *mBuffer; - std::wstring mDirectory; - HANDLE mDirectoryHandle; - std::string mErrorMessage; - OVERLAPPED mOverlapped; - BYTE *mSwap; - EventQueue &mQueue; -}; - -#endif diff --git a/includes/win32_2/Controller.h b/includes/win32_2/Controller.h deleted file mode 100644 index de6d06c2..00000000 --- a/includes/win32_2/Controller.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef NSFW_WIN32_CONTROLLER -#define NSFW_WIN32_CONTROLLER - -//#include "../Queue.h" -#include -#include -#include "Watcher.h" - -class EventQueue; - -class Controller { - public: - Controller(std::shared_ptr queue, const std::string &path); - - std::string getError(); - bool hasErrored(); - bool isWatching(); - - ~Controller(); - private: - std::unique_ptr mWatcher; - - HANDLE openDirectory(const std::wstring &path); - HANDLE mDirectoryHandle; - -// protected: -// void run(); -// void shutdown(); -// std::shared_ptr *mRunner; -// std::wstring mDirectory; -// HANDLE mDirectoryHandle; -// EventQueue &mQueue; -// private: -// unsigned int mThreadID; -// HANDLE mThread; -// bool mRunning; - - -}; - -#endif diff --git a/includes/win32_2/ReadLoop.h b/includes/win32_2/ReadLoop.h deleted file mode 100644 index 6010fda5..00000000 --- a/includes/win32_2/ReadLoop.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef READ_LOOP_H -#define READ_LOOP_H - -#include -#include -#include "ReadLoopRunner.h" - -#include -#include -#include - -#include "../Queue.h" - -class ReadLoop { - public: - ReadLoop(EventQueue &queue, std::string path); - - static unsigned int WINAPI startReadLoop(LPVOID arg); - static void CALLBACK startRunner(__in ULONG_PTR arg); - static void CALLBACK killReadLoop(__in ULONG_PTR arg); - - std::string getError(); - bool hasErrored(); - bool isWatching(); - - ~ReadLoop(); - protected: - void run(); - void shutdown(); - std::shared_ptr *mRunner; - std::wstring mDirectory; - HANDLE mDirectoryHandle; - EventQueue &mQueue; - private: - unsigned int mThreadID; - HANDLE mThread; - bool mRunning; -}; - -#endif diff --git a/includes/win32_2/ReadLoopRunner.h b/includes/win32_2/ReadLoopRunner.h deleted file mode 100644 index 04efc11e..00000000 --- a/includes/win32_2/ReadLoopRunner.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef READ_LOOP_RUNNER_H -#define READ_LOOP_RUNNER_H - -#include -#include -#include -#include "../Queue.h" - -#include - -// limited by over-the-network file watching -#define BUFFER_KB 1024 - -class ReadLoopRunner { - public: - ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle); - ~ReadLoopRunner(); - - static VOID CALLBACK eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped); - std::string getError(); - std::string getUTF8Directory(std::wstring path); - void handleEvents(); - bool hasErrored(); - BOOL read(); - void resizeBuffers(unsigned int bufferSize); - void setError(std::string error); - void setSharedPointer(std::shared_ptr *pointer); - void swap(DWORD numBytes); - void prepareForShutdown(); - - private: - unsigned int mBufferSize; - BYTE *mBuffer; - std::wstring mDirectory; - HANDLE mDirectoryHandle; - std::string mErrorMessage; - OVERLAPPED mOverlapped; - BYTE *mSwap; - EventQueue &mQueue; -}; - -#endif diff --git a/includes/win32_2/Watcher.h b/includes/win32_2/Watcher.h deleted file mode 100644 index 0f2e35b9..00000000 --- a/includes/win32_2/Watcher.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef NSFW_WIN32_WATCHER -#define NSFW_WIN32_WATCHER - -#include -#include -#include -#include -#include -#include - -#include "../SingleshotSemaphore.h" -#include "../Queue.h" - -class Watcher -{ - public: - Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path); - ~Watcher(); - - bool isRunning() const { return mRunning; } - std::string getError() const; - - private: - void run(); - bool loop(); - void start(); - void stop(); - - void setError(const std::string &error); - void eventCallback(DWORD errorCode, DWORD numBytes); - void handleEvents(); - - void resizeBuffers(std::size_t size); - - std::string getUTF8Directory(std::wstring path) ; - - std::atomic mRunning; - SingleshotSemaphore mHasStartedSemaphore; - mutable std::mutex mErrorMutex; - std::string mError; - - const std::wstring mPath; - std::shared_ptr mQueue; - HANDLE mDirectoryHandle; - - std::vector mReadBuffer, mWriteBuffer; - OVERLAPPED mOverlapped; - - std::thread mRunner; -}; - - -#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 24be8793..154a56d5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,19 +3,13 @@ set (NSFW_LIBRARY_SOURCES Queue.cpp NativeInterface.cpp -#FileSystemWatcher.cpp ) if (WIN32) message (STATUS "compiling windows specific file system service") set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} - #win32/ReadLoop.cpp - #win32/ReadLoopRunner.cpp - win32_2/ReadLoop.cpp - win32_2/ReadLoopRunner.cpp - win32_2/Controller.cpp - win32_2/Watcher.cpp - + win32/Controller.cpp + win32/Watcher.cpp ) endif(WIN32) diff --git a/src/win32/ReadLoop.cpp b/src/win32/ReadLoop.cpp deleted file mode 100644 index cfecdbb6..00000000 --- a/src/win32/ReadLoop.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include "../../includes/win32/ReadLoop.h" - -ReadLoop::ReadLoop(EventQueue &queue, std::string path) - : mDirectoryHandle(NULL) - , mQueue(queue) - , mRunner(NULL) - , mThread(NULL) - , mThreadID(0) -{ - - int wlen = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, 0, 0); - - if (wlen == 0) { - return; - } - - LPWSTR outputBuffer = new WCHAR[wlen](); - - int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, outputBuffer, wlen); - - if (failureToResolveUTF8 == 0) { - delete[] outputBuffer; - return; - } - - mDirectoryHandle = CreateFileW( - outputBuffer, - FILE_LIST_DIRECTORY, - FILE_SHARE_READ - | FILE_SHARE_WRITE - | FILE_SHARE_DELETE, - NULL, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS - | FILE_FLAG_OVERLAPPED, - NULL - ); - - mDirectory = outputBuffer; - - delete[] outputBuffer; - outputBuffer = NULL; - - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - mDirectoryHandle = NULL; - return; - } - - // TODO: handle errors - mThread = (HANDLE)_beginthreadex( - NULL, - 0, - ReadLoop::startReadLoop, - this, - 0, - &mThreadID - ); - - if (!mThread) { - CloseHandle(mDirectoryHandle); - mDirectoryHandle = NULL; - mThread = NULL; - return; - } - - QueueUserAPC(ReadLoop::startRunner, mThread, (ULONG_PTR)this); - int maxWait = 10 * 1000; // after 10 seconds, abandon hope - while (mRunner == NULL && maxWait > 0) { - Sleep(50); - maxWait--; - } -} - -std::string ReadLoop::getError() { - if (mDirectoryHandle == NULL || mThread == NULL) { - return "Failed to start watcher"; - } else if (mRunner == NULL) { - return "Watcher is not started"; - } else { - return mRunner->get()->getError(); - } -} - -bool ReadLoop::hasErrored() { - return mDirectoryHandle == NULL || mThread == NULL || (mRunner != NULL && mRunner->get()->hasErrored()); -} - -bool ReadLoop::isWatching() { - return mDirectoryHandle != NULL && mThread != NULL && mRunner != NULL && !mRunner->get()->hasErrored(); -} - -unsigned int WINAPI ReadLoop::startReadLoop(LPVOID arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->run(); - return 0; -} - -void ReadLoop::run() { - while(mDirectoryHandle != NULL) { - SleepEx(INFINITE, true); - } -} - -void CALLBACK ReadLoop::startRunner(__in ULONG_PTR arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->mRunner = new std::shared_ptr(new ReadLoopRunner( - readLoop->mDirectory, - readLoop->mQueue, - readLoop->mDirectoryHandle - )); - readLoop->mRunner->get()->setSharedPointer(readLoop->mRunner); - readLoop->mRunner->get()->read(); -} - -void CALLBACK ReadLoop::killReadLoop(__in ULONG_PTR arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->shutdown(); -} - -void ReadLoop::shutdown() { - if (mRunner != NULL) { - mRunner->get()->prepareForShutdown(); - delete mRunner; - mRunner = NULL; - } - - CancelIo(mDirectoryHandle); - CloseHandle(mDirectoryHandle); - mDirectoryHandle = NULL; -} - -ReadLoop::~ReadLoop() { - if (mThread) { - QueueUserAPC(ReadLoop::killReadLoop, mThread, (ULONG_PTR)this); - WaitForSingleObjectEx(mThread, 10000, true); - CloseHandle(mThread); - - mThread = NULL; - mThreadID = 0; - } -} diff --git a/src/win32/ReadLoopRunner.cpp b/src/win32/ReadLoopRunner.cpp deleted file mode 100644 index 59396054..00000000 --- a/src/win32/ReadLoopRunner.cpp +++ /dev/null @@ -1,244 +0,0 @@ -#include "../../includes/win32/ReadLoopRunner.h" - -ReadLoopRunner::ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle): - mBufferSize(1024), - mDirectory(directory), - mDirectoryHandle(directoryHandle), - mErrorMessage(""), - mQueue(queue) { - ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); - mBuffer = new BYTE[mBufferSize * BUFFER_KB]; - mSwap = new BYTE[mBufferSize * BUFFER_KB]; -} - -ReadLoopRunner::~ReadLoopRunner() { - delete[] mBuffer; - delete[] mSwap; -} - -VOID CALLBACK ReadLoopRunner::eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { - std::shared_ptr *runner = (std::shared_ptr *)overlapped->hEvent; - - if (errorCode != ERROR_SUCCESS) { - if (errorCode == ERROR_NOTIFY_ENUM_DIR) { - runner->get()->setError("Buffer filled up and service needs a restart"); - } else if (errorCode == ERROR_INVALID_PARAMETER) { - // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission - runner->get()->resizeBuffers(64); - if (!runner->get()->read()) { - delete (std::shared_ptr *)runner; - } - return; - } else { - runner->get()->setError("Service shutdown unexpectedly"); - } - - delete (std::shared_ptr *)runner; - return; - } - - runner->get()->swap(numBytes); - BOOL readRequested = runner->get()->read(); - runner->get()->handleEvents(); - - if (!readRequested) { - delete (std::shared_ptr *)runner; - } -} - -std::string ReadLoopRunner::getError() { - return mErrorMessage; -} - -std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { - LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); - memcpy(nullTerminatedFileName, cFileName, length); - std::wstring fileName = nullTerminatedFileName; - delete[] nullTerminatedFileName; - return fileName; -} - -std::string ReadLoopRunner::getUTF8Directory(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - std::wstringstream utf16DirectoryStream; - - utf16DirectoryStream << mDirectory; - - if (found != std::wstring::npos) { - utf16DirectoryStream - << "\\" - << path.substr(0, found); - } - - std::wstring uft16DirectoryString = utf16DirectoryStream.str(); - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - 0, - 0, - NULL, - NULL - ); - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} - -std::string getUTF8FileName(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - if (found != std::wstring::npos) { - path = path.substr(found + 1); - } - - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - 0, - 0, - NULL, - NULL - ); - - // TODO: failure cases for widechar conversion - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} - -void ReadLoopRunner::handleEvents() { - BYTE *base = mSwap; - for (;;) { - PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; - std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); - - if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { - if (info->NextEntryOffset != 0) { - base += info->NextEntryOffset; - info = (PFILE_NOTIFY_INFORMATION)base; - if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { - std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); - - mQueue.enqueue( - RENAMED, - getUTF8Directory(fileName), - getUTF8FileName(fileName), - getUTF8FileName(fileNameNew) - ); - } else { - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - continue; - } - } else { - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - } - } - - switch (info->Action) { - case FILE_ACTION_ADDED: - case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer - mQueue.enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_REMOVED: - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_MODIFIED: - default: - mQueue.enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - }; - - if (info->NextEntryOffset == 0) { - break; - } - base += info->NextEntryOffset; - } -} - -bool ReadLoopRunner::hasErrored() { - return mErrorMessage != ""; -} - -BOOL ReadLoopRunner::read() { - DWORD bytes; - - if (mDirectoryHandle == NULL) { - return FALSE; - } - - if (!ReadDirectoryChangesW( - mDirectoryHandle, - mBuffer, - mBufferSize * BUFFER_KB, - TRUE, - FILE_NOTIFY_CHANGE_FILE_NAME - | FILE_NOTIFY_CHANGE_DIR_NAME - | FILE_NOTIFY_CHANGE_ATTRIBUTES - | FILE_NOTIFY_CHANGE_SIZE - | FILE_NOTIFY_CHANGE_LAST_WRITE - | FILE_NOTIFY_CHANGE_LAST_ACCESS - | FILE_NOTIFY_CHANGE_CREATION - | FILE_NOTIFY_CHANGE_SECURITY, - &bytes, - &mOverlapped, - &ReadLoopRunner::eventCallback)) - { - setError("Service shutdown unexpectedly"); - return FALSE; - } - - return TRUE; -} - -void ReadLoopRunner::resizeBuffers(unsigned int bufferSize) { - mBufferSize = bufferSize; - delete[] mBuffer; - delete[] mSwap; - mBuffer = new BYTE[mBufferSize * BUFFER_KB]; - mSwap = new BYTE[mBufferSize * BUFFER_KB]; -} - -void ReadLoopRunner::setError(std::string error) { - mErrorMessage = error; -} - -void ReadLoopRunner::setSharedPointer(std::shared_ptr *ptr) { - mOverlapped.hEvent = new std::shared_ptr(*ptr); -} - -void ReadLoopRunner::swap(DWORD numBytes) { - memcpy(mSwap, mBuffer, numBytes); -} - -void ReadLoopRunner::prepareForShutdown() { - mDirectoryHandle = NULL; -} diff --git a/src/win32_2/Controller.cpp b/src/win32_2/Controller.cpp deleted file mode 100644 index 93e72c9d..00000000 --- a/src/win32_2/Controller.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include "../includes/win32_2/Controller.h" - - -static std::wstring convertMultiByteToWideChar(const std::string &multiByte) -{ - const int wlen = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, 0, 0); - - if (wlen == 0) { - return std::wstring(); - } - - std::wstring wideString; - wideString.resize(wlen-1); - int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, &(wideString[0]), wlen); - if (failureToResolveUTF8 == 0) { - return std::wstring(); - } - return wideString; -} - -HANDLE Controller::openDirectory(const std::wstring &path) -{ - return CreateFileW( - path.data(), - FILE_LIST_DIRECTORY, - FILE_SHARE_READ - | FILE_SHARE_WRITE - | FILE_SHARE_DELETE, - NULL, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS - | FILE_FLAG_OVERLAPPED, - NULL - ); -} - -Controller::Controller(std::shared_ptr queue, const std::string &path) - : mDirectoryHandle(INVALID_HANDLE_VALUE) -{ - auto widePath = convertMultiByteToWideChar(path); - mDirectoryHandle = openDirectory(widePath); - - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - return; - } - - mWatcher.reset(new Watcher(queue, mDirectoryHandle, widePath)); -} - -Controller::~Controller() -{ - mWatcher.reset(); - CancelIo(mDirectoryHandle); - CloseHandle(mDirectoryHandle); - mDirectoryHandle = INVALID_HANDLE_VALUE; -} - -std::string Controller::getError() -{ - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - return "Failed to open directory"; - } - return mWatcher->getError(); -} - -bool Controller::hasErrored() -{ - return mDirectoryHandle == INVALID_HANDLE_VALUE || !mWatcher->getError().empty(); -} - -bool Controller::isWatching() -{ - return mWatcher->isRunning(); -} diff --git a/src/win32_2/ReadLoop.cpp b/src/win32_2/ReadLoop.cpp deleted file mode 100644 index cfecdbb6..00000000 --- a/src/win32_2/ReadLoop.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include "../../includes/win32/ReadLoop.h" - -ReadLoop::ReadLoop(EventQueue &queue, std::string path) - : mDirectoryHandle(NULL) - , mQueue(queue) - , mRunner(NULL) - , mThread(NULL) - , mThreadID(0) -{ - - int wlen = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, 0, 0); - - if (wlen == 0) { - return; - } - - LPWSTR outputBuffer = new WCHAR[wlen](); - - int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, path.data(), -1, outputBuffer, wlen); - - if (failureToResolveUTF8 == 0) { - delete[] outputBuffer; - return; - } - - mDirectoryHandle = CreateFileW( - outputBuffer, - FILE_LIST_DIRECTORY, - FILE_SHARE_READ - | FILE_SHARE_WRITE - | FILE_SHARE_DELETE, - NULL, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS - | FILE_FLAG_OVERLAPPED, - NULL - ); - - mDirectory = outputBuffer; - - delete[] outputBuffer; - outputBuffer = NULL; - - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - mDirectoryHandle = NULL; - return; - } - - // TODO: handle errors - mThread = (HANDLE)_beginthreadex( - NULL, - 0, - ReadLoop::startReadLoop, - this, - 0, - &mThreadID - ); - - if (!mThread) { - CloseHandle(mDirectoryHandle); - mDirectoryHandle = NULL; - mThread = NULL; - return; - } - - QueueUserAPC(ReadLoop::startRunner, mThread, (ULONG_PTR)this); - int maxWait = 10 * 1000; // after 10 seconds, abandon hope - while (mRunner == NULL && maxWait > 0) { - Sleep(50); - maxWait--; - } -} - -std::string ReadLoop::getError() { - if (mDirectoryHandle == NULL || mThread == NULL) { - return "Failed to start watcher"; - } else if (mRunner == NULL) { - return "Watcher is not started"; - } else { - return mRunner->get()->getError(); - } -} - -bool ReadLoop::hasErrored() { - return mDirectoryHandle == NULL || mThread == NULL || (mRunner != NULL && mRunner->get()->hasErrored()); -} - -bool ReadLoop::isWatching() { - return mDirectoryHandle != NULL && mThread != NULL && mRunner != NULL && !mRunner->get()->hasErrored(); -} - -unsigned int WINAPI ReadLoop::startReadLoop(LPVOID arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->run(); - return 0; -} - -void ReadLoop::run() { - while(mDirectoryHandle != NULL) { - SleepEx(INFINITE, true); - } -} - -void CALLBACK ReadLoop::startRunner(__in ULONG_PTR arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->mRunner = new std::shared_ptr(new ReadLoopRunner( - readLoop->mDirectory, - readLoop->mQueue, - readLoop->mDirectoryHandle - )); - readLoop->mRunner->get()->setSharedPointer(readLoop->mRunner); - readLoop->mRunner->get()->read(); -} - -void CALLBACK ReadLoop::killReadLoop(__in ULONG_PTR arg) { - ReadLoop *readLoop = (ReadLoop *)arg; - readLoop->shutdown(); -} - -void ReadLoop::shutdown() { - if (mRunner != NULL) { - mRunner->get()->prepareForShutdown(); - delete mRunner; - mRunner = NULL; - } - - CancelIo(mDirectoryHandle); - CloseHandle(mDirectoryHandle); - mDirectoryHandle = NULL; -} - -ReadLoop::~ReadLoop() { - if (mThread) { - QueueUserAPC(ReadLoop::killReadLoop, mThread, (ULONG_PTR)this); - WaitForSingleObjectEx(mThread, 10000, true); - CloseHandle(mThread); - - mThread = NULL; - mThreadID = 0; - } -} diff --git a/src/win32_2/ReadLoopRunner.cpp b/src/win32_2/ReadLoopRunner.cpp deleted file mode 100644 index 59396054..00000000 --- a/src/win32_2/ReadLoopRunner.cpp +++ /dev/null @@ -1,244 +0,0 @@ -#include "../../includes/win32/ReadLoopRunner.h" - -ReadLoopRunner::ReadLoopRunner(std::wstring directory, EventQueue &queue, HANDLE directoryHandle): - mBufferSize(1024), - mDirectory(directory), - mDirectoryHandle(directoryHandle), - mErrorMessage(""), - mQueue(queue) { - ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); - mBuffer = new BYTE[mBufferSize * BUFFER_KB]; - mSwap = new BYTE[mBufferSize * BUFFER_KB]; -} - -ReadLoopRunner::~ReadLoopRunner() { - delete[] mBuffer; - delete[] mSwap; -} - -VOID CALLBACK ReadLoopRunner::eventCallback(DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { - std::shared_ptr *runner = (std::shared_ptr *)overlapped->hEvent; - - if (errorCode != ERROR_SUCCESS) { - if (errorCode == ERROR_NOTIFY_ENUM_DIR) { - runner->get()->setError("Buffer filled up and service needs a restart"); - } else if (errorCode == ERROR_INVALID_PARAMETER) { - // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission - runner->get()->resizeBuffers(64); - if (!runner->get()->read()) { - delete (std::shared_ptr *)runner; - } - return; - } else { - runner->get()->setError("Service shutdown unexpectedly"); - } - - delete (std::shared_ptr *)runner; - return; - } - - runner->get()->swap(numBytes); - BOOL readRequested = runner->get()->read(); - runner->get()->handleEvents(); - - if (!readRequested) { - delete (std::shared_ptr *)runner; - } -} - -std::string ReadLoopRunner::getError() { - return mErrorMessage; -} - -std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { - LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); - memcpy(nullTerminatedFileName, cFileName, length); - std::wstring fileName = nullTerminatedFileName; - delete[] nullTerminatedFileName; - return fileName; -} - -std::string ReadLoopRunner::getUTF8Directory(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - std::wstringstream utf16DirectoryStream; - - utf16DirectoryStream << mDirectory; - - if (found != std::wstring::npos) { - utf16DirectoryStream - << "\\" - << path.substr(0, found); - } - - std::wstring uft16DirectoryString = utf16DirectoryStream.str(); - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - 0, - 0, - NULL, - NULL - ); - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} - -std::string getUTF8FileName(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - if (found != std::wstring::npos) { - path = path.substr(found + 1); - } - - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - 0, - 0, - NULL, - NULL - ); - - // TODO: failure cases for widechar conversion - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} - -void ReadLoopRunner::handleEvents() { - BYTE *base = mSwap; - for (;;) { - PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; - std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); - - if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { - if (info->NextEntryOffset != 0) { - base += info->NextEntryOffset; - info = (PFILE_NOTIFY_INFORMATION)base; - if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { - std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); - - mQueue.enqueue( - RENAMED, - getUTF8Directory(fileName), - getUTF8FileName(fileName), - getUTF8FileName(fileNameNew) - ); - } else { - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - continue; - } - } else { - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - } - } - - switch (info->Action) { - case FILE_ACTION_ADDED: - case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer - mQueue.enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_REMOVED: - mQueue.enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_MODIFIED: - default: - mQueue.enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - }; - - if (info->NextEntryOffset == 0) { - break; - } - base += info->NextEntryOffset; - } -} - -bool ReadLoopRunner::hasErrored() { - return mErrorMessage != ""; -} - -BOOL ReadLoopRunner::read() { - DWORD bytes; - - if (mDirectoryHandle == NULL) { - return FALSE; - } - - if (!ReadDirectoryChangesW( - mDirectoryHandle, - mBuffer, - mBufferSize * BUFFER_KB, - TRUE, - FILE_NOTIFY_CHANGE_FILE_NAME - | FILE_NOTIFY_CHANGE_DIR_NAME - | FILE_NOTIFY_CHANGE_ATTRIBUTES - | FILE_NOTIFY_CHANGE_SIZE - | FILE_NOTIFY_CHANGE_LAST_WRITE - | FILE_NOTIFY_CHANGE_LAST_ACCESS - | FILE_NOTIFY_CHANGE_CREATION - | FILE_NOTIFY_CHANGE_SECURITY, - &bytes, - &mOverlapped, - &ReadLoopRunner::eventCallback)) - { - setError("Service shutdown unexpectedly"); - return FALSE; - } - - return TRUE; -} - -void ReadLoopRunner::resizeBuffers(unsigned int bufferSize) { - mBufferSize = bufferSize; - delete[] mBuffer; - delete[] mSwap; - mBuffer = new BYTE[mBufferSize * BUFFER_KB]; - mSwap = new BYTE[mBufferSize * BUFFER_KB]; -} - -void ReadLoopRunner::setError(std::string error) { - mErrorMessage = error; -} - -void ReadLoopRunner::setSharedPointer(std::shared_ptr *ptr) { - mOverlapped.hEvent = new std::shared_ptr(*ptr); -} - -void ReadLoopRunner::swap(DWORD numBytes) { - memcpy(mSwap, mBuffer, numBytes); -} - -void ReadLoopRunner::prepareForShutdown() { - mDirectoryHandle = NULL; -} diff --git a/src/win32_2/Watcher.cpp b/src/win32_2/Watcher.cpp deleted file mode 100644 index 28f18c07..00000000 --- a/src/win32_2/Watcher.cpp +++ /dev/null @@ -1,281 +0,0 @@ -#include "../includes/win32_2/Watcher.h" - -#include - -static -std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { - LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); - memcpy(nullTerminatedFileName, cFileName, length); - std::wstring fileName = nullTerminatedFileName; - delete[] nullTerminatedFileName; - return fileName; -} - - -std::string Watcher::getUTF8Directory(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - std::wstringstream utf16DirectoryStream; - - utf16DirectoryStream << mPath; - - if (found != std::wstring::npos) { - utf16DirectoryStream - << "\\" - << path.substr(0, found); - } - - std::wstring uft16DirectoryString = utf16DirectoryStream.str(); - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - 0, - 0, - NULL, - NULL - ); - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} - -static -std::string getUTF8FileName(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - if (found != std::wstring::npos) { - path = path.substr(found + 1); - } - - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - 0, - 0, - NULL, - NULL - ); - - // TODO: failure cases for widechar conversion - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; -} - -Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path) - : mRunning(false) - , mDirectoryHandle(dirHandle) - , mQueue(queue) - , mPath(path) -{ - ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); - mOverlapped.hEvent = this; - resizeBuffers(1024 * 1024); - start(); -} - -Watcher::~Watcher() -{ - stop(); -} - -void Watcher::resizeBuffers(std::size_t size) -{ - mReadBuffer.resize(size); - mWriteBuffer.resize(size); -} - -void Watcher::run() -{ - while(mRunning) { - SleepEx(INFINITE, true); - } -} - -bool Watcher::loop() -{ - DWORD bytes = 0; - - if (!isRunning()) { - return false; - } - - if (!ReadDirectoryChangesW( - mDirectoryHandle, - mWriteBuffer.data(), - mWriteBuffer.size(), - TRUE, //recursive watching - FILE_NOTIFY_CHANGE_FILE_NAME - | FILE_NOTIFY_CHANGE_DIR_NAME - | FILE_NOTIFY_CHANGE_ATTRIBUTES - | FILE_NOTIFY_CHANGE_SIZE - | FILE_NOTIFY_CHANGE_LAST_WRITE - | FILE_NOTIFY_CHANGE_LAST_ACCESS - | FILE_NOTIFY_CHANGE_CREATION - | FILE_NOTIFY_CHANGE_SECURITY, - &bytes, //num bytes written - &mOverlapped, - [](DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { - auto watcher = reinterpret_cast(overlapped->hEvent); - watcher->eventCallback(errorCode, numBytes); - })) - { - setError("Service shutdown unexpectedly"); - return false; - } - - return true; -} - -void Watcher::eventCallback(DWORD errorCode, DWORD numBytes) -{ - if (errorCode != ERROR_SUCCESS) { - if (errorCode == ERROR_NOTIFY_ENUM_DIR) { - setError("Buffer filled up and service needs a restart"); - } else if (errorCode == ERROR_INVALID_PARAMETER) { - // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission - resizeBuffers(64 * 1024); - - if (!loop()) { - setError("failed resizing buffers for network traffic"); - } - } else { - setError("Service shutdown unexpectedly"); - } - return; - } - - std::swap(mWriteBuffer, mReadBuffer); - BOOL readRequested = loop(); - handleEvents(); - - if (!readRequested) { - //delete (std::shared_ptr *)runner; - } -} - -void Watcher::handleEvents() -{ - BYTE *base = mReadBuffer.data(); - while (true) { - PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; - std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); - - if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { - if (info->NextEntryOffset != 0) { - base += info->NextEntryOffset; - info = (PFILE_NOTIFY_INFORMATION)base; - if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { - std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); - - mQueue->enqueue( - RENAMED, - getUTF8Directory(fileName), - getUTF8FileName(fileName), - getUTF8FileName(fileNameNew) - ); - } else { - mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - continue; - } - } else { - mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - } - } - - switch (info->Action) { - case FILE_ACTION_ADDED: - case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer - mQueue->enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_REMOVED: - mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_MODIFIED: - default: - mQueue->enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - }; - - if (info->NextEntryOffset == 0) { - break; - } - base += info->NextEntryOffset; - } -} - -void Watcher::start() -{ - mRunner = std::thread([this]{ - // mRunning is set to false in the d'tor - mRunning = true; - run(); - }); - - if (!mRunner.joinable()) { - mRunning = false; - return; - } - - QueueUserAPC([](__in ULONG_PTR self) { - auto watcher = reinterpret_cast(self); - watcher->mHasStartedSemaphore.signal(); - watcher->loop(); - } , mRunner.native_handle(), (ULONG_PTR)this); - - if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { - setError("Watcher is not started"); - } -} - -void Watcher::stop() -{ - mRunning = false; - QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); - mRunner.join(); -} - -void Watcher::setError(const std::string &error) -{ - std::lock_guard lock(mErrorMutex); - mError = error; -} - - -std::string Watcher::getError() const -{ - if (!isRunning()) { - return "Failed to start watcher"; - } - - std::lock_guard lock(mErrorMutex); - return mError; -} From 31e3c18bf590e47cc3fc555fa70cfb6115958762 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Thu, 14 Sep 2017 14:43:22 +0200 Subject: [PATCH 24/37] move files and cleanup files --- includes/NativeInterface.h | 2 +- includes/win32/Controller.h | 26 ++++ includes/win32/Watcher.h | 53 ++++++++ src/win32/Controller.cpp | 73 +++++++++++ src/win32/Watcher.cpp | 251 ++++++++++++++++++++++++++++++++++++ 5 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 includes/win32/Controller.h create mode 100644 includes/win32/Watcher.h create mode 100644 src/win32/Controller.cpp create mode 100644 src/win32/Watcher.cpp diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index 93cf0725..bc297884 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -2,7 +2,7 @@ #define NSFW_NATIVE_INTERFACE_H #if defined(_WIN32) -#include "../includes/win32_2/Controller.h" +#include "../includes/win32/Controller.h" using NativeImplementation = Controller; #elif defined(__APPLE_CC__) #include "../includes/osx/FSEventsService.h" diff --git a/includes/win32/Controller.h b/includes/win32/Controller.h new file mode 100644 index 00000000..61578fef --- /dev/null +++ b/includes/win32/Controller.h @@ -0,0 +1,26 @@ +#ifndef NSFW_WIN32_CONTROLLER +#define NSFW_WIN32_CONTROLLER + +#include +#include +#include "Watcher.h" + +class EventQueue; + +class Controller { + public: + Controller(std::shared_ptr queue, const std::string &path); + + std::string getError(); + bool hasErrored(); + bool isWatching(); + + ~Controller(); + private: + std::unique_ptr mWatcher; + + HANDLE openDirectory(const std::wstring &path); + HANDLE mDirectoryHandle; +}; + +#endif diff --git a/includes/win32/Watcher.h b/includes/win32/Watcher.h new file mode 100644 index 00000000..591b8002 --- /dev/null +++ b/includes/win32/Watcher.h @@ -0,0 +1,53 @@ +#ifndef NSFW_WIN32_WATCHER +#define NSFW_WIN32_WATCHER + +#include +#include +#include +#include +#include +#include + +#include "../SingleshotSemaphore.h" +#include "../Queue.h" + +class Watcher +{ + public: + Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path); + ~Watcher(); + + bool isRunning() const { return mRunning; } + std::string getError() const; + + private: + void run(); + bool loop(); + void start(); + void stop(); + + void setError(const std::string &error); + void eventCallback(DWORD errorCode); + void handleEvents(); + + void resizeBuffers(std::size_t size); + + std::string getUTF8Directory(std::wstring path) ; + + std::atomic mRunning; + SingleshotSemaphore mHasStartedSemaphore; + mutable std::mutex mErrorMutex; + std::string mError; + + const std::wstring mPath; + std::shared_ptr mQueue; + HANDLE mDirectoryHandle; + + std::vector mReadBuffer, mWriteBuffer; + OVERLAPPED mOverlapped; + + std::thread mRunner; +}; + + +#endif diff --git a/src/win32/Controller.cpp b/src/win32/Controller.cpp new file mode 100644 index 00000000..9f6cf951 --- /dev/null +++ b/src/win32/Controller.cpp @@ -0,0 +1,73 @@ +#include "../includes/win32/Controller.h" + +static std::wstring convertMultiByteToWideChar(const std::string &multiByte) +{ + const int wlen = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, 0, 0); + + if (wlen == 0) { + return std::wstring(); + } + + std::wstring wideString; + wideString.resize(wlen-1); + int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, &(wideString[0]), wlen); + if (failureToResolveUTF8 == 0) { + return std::wstring(); + } + return wideString; +} + +HANDLE Controller::openDirectory(const std::wstring &path) +{ + return CreateFileW( + path.data(), + FILE_LIST_DIRECTORY, + FILE_SHARE_READ + | FILE_SHARE_WRITE + | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS + | FILE_FLAG_OVERLAPPED, + NULL + ); +} + +Controller::Controller(std::shared_ptr queue, const std::string &path) + : mDirectoryHandle(INVALID_HANDLE_VALUE) +{ + auto widePath = convertMultiByteToWideChar(path); + mDirectoryHandle = openDirectory(widePath); + + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + return; + } + + mWatcher.reset(new Watcher(queue, mDirectoryHandle, widePath)); +} + +Controller::~Controller() +{ + mWatcher.reset(); + CancelIo(mDirectoryHandle); + CloseHandle(mDirectoryHandle); + mDirectoryHandle = INVALID_HANDLE_VALUE; +} + +std::string Controller::getError() +{ + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + return "Failed to open directory"; + } + return mWatcher->getError(); +} + +bool Controller::hasErrored() +{ + return mDirectoryHandle == INVALID_HANDLE_VALUE || !mWatcher->getError().empty(); +} + +bool Controller::isWatching() +{ + return mWatcher->isRunning(); +} diff --git a/src/win32/Watcher.cpp b/src/win32/Watcher.cpp new file mode 100644 index 00000000..6ce893d6 --- /dev/null +++ b/src/win32/Watcher.cpp @@ -0,0 +1,251 @@ +#include "../includes/win32/Watcher.h" + +#include + +static std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { + return std::wstring(cFileName, length); +} + + +static std::string convertWideCharToMultiByte(const std::wstring &wideChar) +{ + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + wideChar.data(), + -1, + 0, + 0, + NULL, + NULL + ); + std::string utf8String; + utf8String.resize(utf8length-1); + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + wideChar.data(), + -1, + const_cast(utf8String.data()), + static_cast(utf8String.size()), + NULL, + NULL + ); + + return utf8String; +} + +std::string Watcher::getUTF8Directory(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + std::wstringstream utf16DirectoryStream; + + utf16DirectoryStream << mPath; + + if (found != std::wstring::npos) { + utf16DirectoryStream + << "\\" + << path.substr(0, found); + } + + return convertWideCharToMultiByte(utf16DirectoryStream.str()); +} + +static +std::string getUTF8FileName(std::wstring path) { + std::wstring::size_type found = path.rfind('\\'); + if (found != std::wstring::npos) { + path = path.substr(found + 1); + } + + return convertWideCharToMultiByte(path); +} + +Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path) + : mRunning(false) + , mDirectoryHandle(dirHandle) + , mQueue(queue) + , mPath(path) +{ + ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); + mOverlapped.hEvent = this; + resizeBuffers(1024 * 1024); + start(); +} + +Watcher::~Watcher() +{ + stop(); +} + +void Watcher::resizeBuffers(std::size_t size) +{ + mReadBuffer.resize(size); + mWriteBuffer.resize(size); +} + +void Watcher::run() +{ + while(mRunning) { + SleepEx(INFINITE, true); + } +} + +bool Watcher::loop() +{ + DWORD bytes = 0; + + if (!isRunning()) { + return false; + } + + if (!ReadDirectoryChangesW( + mDirectoryHandle, + mWriteBuffer.data(), + static_cast(mWriteBuffer.size()), + TRUE, //recursive watching + FILE_NOTIFY_CHANGE_FILE_NAME + | FILE_NOTIFY_CHANGE_DIR_NAME + | FILE_NOTIFY_CHANGE_ATTRIBUTES + | FILE_NOTIFY_CHANGE_SIZE + | FILE_NOTIFY_CHANGE_LAST_WRITE + | FILE_NOTIFY_CHANGE_LAST_ACCESS + | FILE_NOTIFY_CHANGE_CREATION + | FILE_NOTIFY_CHANGE_SECURITY, + &bytes, //num bytes written + &mOverlapped, + [](DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { + auto watcher = reinterpret_cast(overlapped->hEvent); + watcher->eventCallback(errorCode); + })) + { + setError("Service shutdown unexpectedly"); + return false; + } + + return true; +} + +void Watcher::eventCallback(DWORD errorCode) +{ + if (errorCode != ERROR_SUCCESS) { + if (errorCode == ERROR_NOTIFY_ENUM_DIR) { + setError("Buffer filled up and service needs a restart"); + } else if (errorCode == ERROR_INVALID_PARAMETER) { + // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission + resizeBuffers(64 * 1024); + + if (!loop()) { + setError("failed resizing buffers for network traffic"); + } + } else { + setError("Service shutdown unexpectedly"); + } + return; + } + + std::swap(mWriteBuffer, mReadBuffer); + BOOL readRequested = loop(); + handleEvents(); + + if (!readRequested) { + //delete (std::shared_ptr *)runner; + } +} + +void Watcher::handleEvents() +{ + BYTE *base = mReadBuffer.data(); + while (true) { + PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; + std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); + + if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { + if (info->NextEntryOffset != 0) { + base += info->NextEntryOffset; + info = (PFILE_NOTIFY_INFORMATION)base; + if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { + std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); + + mQueue->enqueue( + RENAMED, + getUTF8Directory(fileName), + getUTF8FileName(fileName), + getUTF8FileName(fileNameNew) + ); + } else { + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + continue; + } + } else { + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + } + } + + switch (info->Action) { + case FILE_ACTION_ADDED: + case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer + mQueue->enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_REMOVED: + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_MODIFIED: + default: + mQueue->enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + }; + + if (info->NextEntryOffset == 0) { + break; + } + base += info->NextEntryOffset; + } +} + +void Watcher::start() +{ + mRunner = std::thread([this]{ + // mRunning is set to false in the d'tor + mRunning = true; + run(); + }); + + if (!mRunner.joinable()) { + mRunning = false; + return; + } + + QueueUserAPC([](__in ULONG_PTR self) { + auto watcher = reinterpret_cast(self); + watcher->mHasStartedSemaphore.signal(); + watcher->loop(); + } , mRunner.native_handle(), (ULONG_PTR)this); + + if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { + setError("Watcher is not started"); + } +} + +void Watcher::stop() +{ + mRunning = false; + QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); + mRunner.join(); +} + +void Watcher::setError(const std::string &error) +{ + std::lock_guard lock(mErrorMutex); + mError = error; +} + + +std::string Watcher::getError() const +{ + if (!isRunning()) { + return "Failed to start watcher"; + } + + std::lock_guard lock(mErrorMutex); + return mError; +} From cd101160b96c01fd666887dfb149800f6345b0bc Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Thu, 14 Sep 2017 16:53:56 +0200 Subject: [PATCH 25/37] revert some changes --- src/win32/Watcher.cpp | 95 ++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 33 deletions(-) diff --git a/src/win32/Watcher.cpp b/src/win32/Watcher.cpp index 6ce893d6..52aebb3a 100644 --- a/src/win32/Watcher.cpp +++ b/src/win32/Watcher.cpp @@ -2,37 +2,13 @@ #include -static std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { - return std::wstring(cFileName, length); -} - - -static std::string convertWideCharToMultiByte(const std::wstring &wideChar) -{ - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - wideChar.data(), - -1, - 0, - 0, - NULL, - NULL - ); - std::string utf8String; - utf8String.resize(utf8length-1); - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - wideChar.data(), - -1, - const_cast(utf8String.data()), - static_cast(utf8String.size()), - NULL, - NULL - ); - - return utf8String; +static +std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { + LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); + memcpy(nullTerminatedFileName, cFileName, length); + std::wstring fileName = nullTerminatedFileName; + delete[] nullTerminatedFileName; + return fileName; } std::string Watcher::getUTF8Directory(std::wstring path) { @@ -47,7 +23,33 @@ std::string Watcher::getUTF8Directory(std::wstring path) { << path.substr(0, found); } - return convertWideCharToMultiByte(utf16DirectoryStream.str()); + std::wstring uft16DirectoryString = utf16DirectoryStream.str(); + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + 0, + 0, + NULL, + NULL + ); + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; } static @@ -57,7 +59,34 @@ std::string getUTF8FileName(std::wstring path) { path = path.substr(found + 1); } - return convertWideCharToMultiByte(path); + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + 0, + 0, + NULL, + NULL + ); + + // TODO: failure cases for widechar conversion + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; } Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path) From 85b10b301b87c7effa5bb4c72ba3da7e546fba09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Thu, 14 Sep 2017 17:59:31 +0200 Subject: [PATCH 26/37] Refactor: remove the self-tailored lock guard --- binding.gyp | 5 ----- includes/Lock.h | 14 -------------- includes/linux/InotifyEventLoop.h | 5 +++-- src/CMakeLists.txt | 1 - src/Lock.cpp | 10 ---------- src/linux/InotifyEventLoop.cpp | 10 ++-------- 6 files changed, 5 insertions(+), 40 deletions(-) delete mode 100644 includes/Lock.h delete mode 100644 src/Lock.cpp diff --git a/binding.gyp b/binding.gyp index 9176483b..c0b3e54e 100644 --- a/binding.gyp +++ b/binding.gyp @@ -34,10 +34,8 @@ }], ["OS=='mac'", { "sources": [ - "src/Lock.cpp", "src/osx/RunLoop.cpp", "src/osx/FSEventsService.cpp", - "includes/Lock.h", "includes/osx/RunLoop.h", "includes/osx/FSEventsService.h" ], @@ -61,11 +59,9 @@ }], ["OS=='linux' or OS=='freebsd'", { "sources": [ - "src/Lock.cpp", "src/linux/InotifyEventLoop.cpp", "src/linux/InotifyTree.cpp", "src/linux/InotifyService.cpp", - "includes/Lock.h", "includes/linux/InotifyEventLoop.h", "includes/linux/InotifyTree.h", "includes/linux/InotifyService.h" @@ -105,6 +101,5 @@ ] }], ] - }] } diff --git a/includes/Lock.h b/includes/Lock.h deleted file mode 100644 index 72e0866f..00000000 --- a/includes/Lock.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef LOCK_H -#define LOCK_H - -#include - -class Lock { -public: - Lock(pthread_mutex_t &mutex); - ~Lock(); -private: - pthread_mutex_t &mMutex; -}; - -#endif diff --git a/includes/linux/InotifyEventLoop.h b/includes/linux/InotifyEventLoop.h index 02e0f260..f59157ff 100644 --- a/includes/linux/InotifyEventLoop.h +++ b/includes/linux/InotifyEventLoop.h @@ -2,7 +2,7 @@ #define INOTIFY_EVENT_LOOP_H #include "InotifyService.h" -#include "../Lock.h" + #include #include #include @@ -10,6 +10,7 @@ #include #include #include +#include class InotifyService; class Lock; @@ -40,7 +41,7 @@ class InotifyEventLoop { InotifyService *mInotifyService; pthread_t mEventLoop; - pthread_mutex_t mMutex; + std::mutex mMutex; bool mStarted; }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 154a56d5..2bfb5f50 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,7 +23,6 @@ if (UNIX) else (APPLE) message (STATUS "compiling linux specific file system service") set (NSFW_LIBRARY_SOURCES ${NSFW_LIBRARY_SOURCES} - Lock.cpp linux/InotifyEventLoop.cpp linux/InotifyService.cpp linux/InotifyTree.cpp diff --git a/src/Lock.cpp b/src/Lock.cpp deleted file mode 100644 index 5d5173e4..00000000 --- a/src/Lock.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "../includes/Lock.h" - -Lock::Lock(pthread_mutex_t &mutex) - : mMutex(mutex) { - pthread_mutex_lock(&mMutex); -} - -Lock::~Lock() { - pthread_mutex_unlock(&mMutex); -} diff --git a/src/linux/InotifyEventLoop.cpp b/src/linux/InotifyEventLoop.cpp index 63f5d922..c9022d6a 100644 --- a/src/linux/InotifyEventLoop.cpp +++ b/src/linux/InotifyEventLoop.cpp @@ -7,11 +7,6 @@ InotifyEventLoop::InotifyEventLoop( mInotifyInstance(inotifyInstance), mInotifyService(inotifyService) { - if (pthread_mutex_init(&mMutex, NULL) != 0) { - mStarted = false; - return; - } - mStarted = !pthread_create( &mEventLoop, NULL, @@ -100,7 +95,7 @@ void InotifyEventLoop::work() { }; while((bytesRead = read(mInotifyInstance, &buffer, BUFFER_SIZE)) > 0) { - Lock syncWithDestructor(this->mMutex); + std::lock_guard syncWithDestructor(mMutex); do { event = (struct inotify_event *)(buffer + position); @@ -151,10 +146,9 @@ InotifyEventLoop::~InotifyEventLoop() { } { - Lock syncWithWork(this->mMutex); + std::lock_guard syncWithWork(mMutex); pthread_cancel(mEventLoop); } pthread_join(mEventLoop, NULL); - pthread_mutex_destroy(&mMutex); } From 7d6779fc96486c9a1711d7b957bf86a1508630bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Thu, 14 Sep 2017 18:12:05 +0200 Subject: [PATCH 27/37] Doc: SingleshotSemaphore --- includes/SingleshotSemaphore.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/includes/SingleshotSemaphore.h b/includes/SingleshotSemaphore.h index 13a391e6..45908757 100644 --- a/includes/SingleshotSemaphore.h +++ b/includes/SingleshotSemaphore.h @@ -4,10 +4,23 @@ #include #include +/** + * This is a convenience abstraction around std::condition_variable that allows + * for a one-shot synchronization point. Therefore the Semaphore has no way to + * reset its state. + * + * It doesn't matter if the waiting thread calls `wait()` before or after the + * signaling thread calls `signal()`. Only in the latter case the `wait()` won't + * block. + */ class SingleshotSemaphore { public: SingleshotSemaphore() : mState(false) {} + /** + * Blocks the calling thread until the semaphore is signaled asynchronously. + * If `signal()` has been called on the semaphore already, this won't block. + */ void wait() { std::unique_lock lk(mMutex); @@ -17,6 +30,13 @@ class SingleshotSemaphore { } } + /** + * Blocks the calling thread for a given time period and continues either + * when `signal()` was called asynchronously or when the time is up. The + * return condition is indicated by the returned boolean. + * + * \return true if the semaphore was signal()ed; false on timeout reached + */ bool waitFor(std::chrono::milliseconds ms) { std::unique_lock lk(mMutex); @@ -29,6 +49,11 @@ class SingleshotSemaphore { return mState; } + /** + * Unblocks all waiting threads of the semaphore. Note that threads reaching + * the `wait()` on this semaphore after `signal()` has been called won't + * block but continue immediately. + */ void signal() { std::unique_lock lk(mMutex); From daa2e52acdfca773a4f363a1aa276c851349c098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Thu, 14 Sep 2017 18:17:58 +0200 Subject: [PATCH 28/37] FIX: cosmetics --- src/NSFW.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NSFW.cpp b/src/NSFW.cpp index 2f4afef1..09941aab 100644 --- a/src/NSFW.cpp +++ b/src/NSFW.cpp @@ -292,7 +292,7 @@ NSFW::StopWorker::StopWorker(NSFW *nsfw, Callback *callback): void NSFW::StopWorker::Execute() { uv_mutex_lock(&mNSFW->mInterfaceLock); if (mNSFW->mInterface == NULL) { - uv_mutex_unlock(&mNSFW->mInterfaceLock); + uv_mutex_unlock(&mNSFW->mInterfaceLock); return; } uv_mutex_unlock(&mNSFW->mInterfaceLock); From 27d1677f7b1da22295c07c6611b7715e9e2a708b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Thu, 14 Sep 2017 18:50:16 +0200 Subject: [PATCH 29/37] Refactor: use std::thread rather then pthread_t --- includes/osx/RunLoop.h | 4 ++-- src/osx/RunLoop.cpp | 15 +++------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/includes/osx/RunLoop.h b/includes/osx/RunLoop.h index 11093a93..81856a61 100644 --- a/includes/osx/RunLoop.h +++ b/includes/osx/RunLoop.h @@ -5,7 +5,7 @@ #include "FSEventsService.h" #include -#include +#include #include void *scheduleRunLoopWork(void *runLoop); @@ -36,7 +36,7 @@ class RunLoop { bool mExited; std::string mPath; CFRunLoopRef mRunLoop; - pthread_t mRunLoopThread; + std::thread mRunLoopThread; SingleshotSemaphore mReadyForCleanup; bool mStarted; }; diff --git a/src/osx/RunLoop.cpp b/src/osx/RunLoop.cpp index 8ddf4a64..bffc251a 100644 --- a/src/osx/RunLoop.cpp +++ b/src/osx/RunLoop.cpp @@ -1,22 +1,13 @@ #include "../../includes/osx/RunLoop.h" -void *scheduleRunLoopWork(void *runLoop) { - ((RunLoop *)runLoop)->work(); - return NULL; -} - RunLoop::RunLoop(FSEventsService *eventsService, std::string path): mEventsService(eventsService), mExited(false), mPath(path), mRunLoop(NULL), mStarted(false) { - mStarted = !pthread_create( - &mRunLoopThread, - NULL, - scheduleRunLoopWork, - (void *)this - ); + mRunLoopThread = std::thread([] (RunLoop *rl) { rl->work(); }, this); + mStarted = mRunLoopThread.joinable(); } bool RunLoop::isLooping() { @@ -31,7 +22,7 @@ RunLoop::~RunLoop() { mReadyForCleanup.wait(); CFRunLoopStop(mRunLoop); - pthread_join(mRunLoopThread, NULL); + mRunLoopThread.join(); } void RunLoop::work() { From 0a8befa5b195be480cdb424199a54dc74b0bfba3 Mon Sep 17 00:00:00 2001 From: Hannes Rantzsch Date: Thu, 14 Sep 2017 19:20:59 +0200 Subject: [PATCH 30/37] use semaphore to wait until thread is running --- includes/linux/InotifyEventLoop.h | 2 ++ src/linux/InotifyEventLoop.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/includes/linux/InotifyEventLoop.h b/includes/linux/InotifyEventLoop.h index f59157ff..79671ae8 100644 --- a/includes/linux/InotifyEventLoop.h +++ b/includes/linux/InotifyEventLoop.h @@ -2,6 +2,7 @@ #define INOTIFY_EVENT_LOOP_H #include "InotifyService.h" +#include "../SingleshotSemaphore.h" #include #include @@ -42,6 +43,7 @@ class InotifyEventLoop { pthread_t mEventLoop; std::mutex mMutex; + SingleshotSemaphore mLoopingSemaphore; bool mStarted; }; diff --git a/src/linux/InotifyEventLoop.cpp b/src/linux/InotifyEventLoop.cpp index c9022d6a..56546b61 100644 --- a/src/linux/InotifyEventLoop.cpp +++ b/src/linux/InotifyEventLoop.cpp @@ -16,6 +16,7 @@ InotifyEventLoop::InotifyEventLoop( }, (void *)this ); + if (mStarted) { mLoopingSemaphore.wait(); } } bool InotifyEventLoop::isLooping() { @@ -94,6 +95,7 @@ void InotifyEventLoop::work() { renameEvent.isGood = false; }; + mLoopingSemaphore.signal(); while((bytesRead = read(mInotifyInstance, &buffer, BUFFER_SIZE)) > 0) { std::lock_guard syncWithDestructor(mMutex); do { From c15968046b63deb47a94a2495c5f9affdd3a33ce Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 20 Sep 2017 10:47:39 +0200 Subject: [PATCH 31/37] review issues --- appveyor.yml | 2 +- includes/NSFW.h | 2 +- includes/NativeInterface.h | 2 +- includes/Queue.h | 8 +- includes/SingleshotSemaphore.h | 83 ++++--- src/Queue.cpp | 22 +- src/win32/Controller.cpp | 80 +++---- src/win32/Watcher.cpp | 424 ++++++++++++++++----------------- 8 files changed, 308 insertions(+), 315 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index de7e0753..f587668d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,7 +15,7 @@ install: # Get the latest stable version of Node.js or io.js - ps: Install-Product node $env:nodejs_version # install modules - - npm install --msvs_version=2015 + - npm install --msvs_version=2013 # Post-install test scripts. test_script: diff --git a/includes/NSFW.h b/includes/NSFW.h index e821618f..61c05a43 100644 --- a/includes/NSFW.h +++ b/includes/NSFW.h @@ -41,7 +41,7 @@ class NSFW : public ObjectWrap { struct EventBaton { NSFW *nsfw; - std::vector *events; + std::vector *events; }; static NAN_METHOD(JSNew); diff --git a/includes/NativeInterface.h b/includes/NativeInterface.h index bc297884..8005efb6 100644 --- a/includes/NativeInterface.h +++ b/includes/NativeInterface.h @@ -21,7 +21,7 @@ class NativeInterface { ~NativeInterface(); std::string getError(); - std::vector* getEvents(); + std::vector *getEvents(); bool hasErrored(); bool isWatching(); diff --git a/includes/Queue.h b/includes/Queue.h index 1d25967d..39ff66f8 100644 --- a/includes/Queue.h +++ b/includes/Queue.h @@ -15,7 +15,7 @@ enum EventType { }; struct Event { - Event(const EventType type, const std::string& directory, const std::string& fileA, const std::string& fileB) : + Event(const EventType type, const std::string &directory, const std::string &fileA, const std::string &fileB) : type(type), directory(directory), fileA(fileA), fileB(fileB) {} EventType type; std::string directory, fileA, fileB; @@ -29,9 +29,9 @@ class EventQueue { std::unique_ptr> dequeueAll(); void enqueue( EventType type, - const std::string& directory, - const std::string& fileA, - const std::string& fileB = "" + const std::string &directory, + const std::string &fileA, + const std::string &fileB = "" ); private: diff --git a/includes/SingleshotSemaphore.h b/includes/SingleshotSemaphore.h index 45908757..f3de71fc 100644 --- a/includes/SingleshotSemaphore.h +++ b/includes/SingleshotSemaphore.h @@ -14,57 +14,54 @@ * block. */ class SingleshotSemaphore { - public: - SingleshotSemaphore() : mState(false) {} +public: + SingleshotSemaphore() : mState(false) {} - /** - * Blocks the calling thread until the semaphore is signaled asynchronously. - * If `signal()` has been called on the semaphore already, this won't block. - */ - void wait() - { - std::unique_lock lk(mMutex); + /** + * Blocks the calling thread until the semaphore is signaled asynchronously. + * If `signal()` has been called on the semaphore already, this won't block. + */ + void wait() { + std::unique_lock lk(mMutex); - while (!mState) { - mCond.wait(lk); - } + while (!mState) { + mCond.wait(lk); } + } - /** - * Blocks the calling thread for a given time period and continues either - * when `signal()` was called asynchronously or when the time is up. The - * return condition is indicated by the returned boolean. - * - * \return true if the semaphore was signal()ed; false on timeout reached - */ - bool waitFor(std::chrono::milliseconds ms) - { - std::unique_lock lk(mMutex); + /** + * Blocks the calling thread for a given time period and continues either + * when `signal()` was called asynchronously or when the time is up. The + * return condition is indicated by the returned boolean. + * + * \return true if the semaphore was signal()ed; false on timeout reached + */ + bool waitFor(std::chrono::milliseconds ms) { + std::unique_lock lk(mMutex); - if (mState) { - return true; - } - - mCond.wait_for(lk, ms); - return mState; + if (mState) { + return true; } - /** - * Unblocks all waiting threads of the semaphore. Note that threads reaching - * the `wait()` on this semaphore after `signal()` has been called won't - * block but continue immediately. - */ - void signal() - { - std::unique_lock lk(mMutex); - mState = true; - mCond.notify_all(); - } + mCond.wait_for(lk, ms); + return mState; + } + + /** + * Unblocks all waiting threads of the semaphore. Note that threads reaching + * the `wait()` on this semaphore after `signal()` has been called won't + * block but continue immediately. + */ + void signal() { + std::unique_lock lk(mMutex); + mState = true; + mCond.notify_all(); + } - private: - std::mutex mMutex; - std::condition_variable mCond; - bool mState; +private: + std::mutex mMutex; + std::condition_variable mCond; + bool mState; }; #endif diff --git a/src/Queue.cpp b/src/Queue.cpp index 340672ae..cf6ad4c7 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -4,22 +4,22 @@ #pragma unmanaged void EventQueue::clear() { - std::lock_guard lock(mutex); - queue.clear(); + std::lock_guard lock(mutex); + queue.clear(); } std::size_t EventQueue::count() { - std::lock_guard lock(mutex); - return queue.size(); + std::lock_guard lock(mutex); + return queue.size(); } std::unique_ptr EventQueue::dequeue() { std::lock_guard lock(mutex); if (queue.empty()) { - return nullptr; + return nullptr; } - auto& front = queue.front(); + auto &front = queue.front(); auto retVal = std::move(front); queue.pop_front(); @@ -29,21 +29,21 @@ std::unique_ptr EventQueue::dequeue() { std::unique_ptr> EventQueue::dequeueAll() { std::lock_guard lock(mutex); if (queue.empty()) { - return nullptr; + return nullptr; } const auto queueSize = queue.size(); std::unique_ptr> events(new std::vector(queueSize, nullptr)); for (size_t i = 0; i < queueSize; ++i) { - auto& front = queue.front(); - (*events)[i] = front.release(); - queue.pop_front(); + auto &front = queue.front(); + (*events)[i] = front.release(); + queue.pop_front(); } return events; } -void EventQueue::enqueue(const EventType type, const std::string& directory, const std::string& fileA, const std::string& fileB) { +void EventQueue::enqueue(const EventType type, const std::string &directory, const std::string &fileA, const std::string &fileB) { std::lock_guard lock(mutex); queue.emplace_back(std::unique_ptr(new Event(type, directory, fileA, fileB))); } diff --git a/src/win32/Controller.cpp b/src/win32/Controller.cpp index 9f6cf951..974052d7 100644 --- a/src/win32/Controller.cpp +++ b/src/win32/Controller.cpp @@ -2,72 +2,72 @@ static std::wstring convertMultiByteToWideChar(const std::string &multiByte) { - const int wlen = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, 0, 0); + const int wlen = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, 0, 0); - if (wlen == 0) { - return std::wstring(); - } + if (wlen == 0) { + return std::wstring(); + } - std::wstring wideString; - wideString.resize(wlen-1); - int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, &(wideString[0]), wlen); - if (failureToResolveUTF8 == 0) { - return std::wstring(); - } - return wideString; + std::wstring wideString; + wideString.resize(wlen-1); + int failureToResolveUTF8 = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, &(wideString[0]), wlen); + if (failureToResolveUTF8 == 0) { + return std::wstring(); + } + return wideString; } HANDLE Controller::openDirectory(const std::wstring &path) { - return CreateFileW( - path.data(), - FILE_LIST_DIRECTORY, - FILE_SHARE_READ - | FILE_SHARE_WRITE - | FILE_SHARE_DELETE, - NULL, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS - | FILE_FLAG_OVERLAPPED, - NULL - ); + return CreateFileW( + path.data(), + FILE_LIST_DIRECTORY, + FILE_SHARE_READ + | FILE_SHARE_WRITE + | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS + | FILE_FLAG_OVERLAPPED, + NULL + ); } Controller::Controller(std::shared_ptr queue, const std::string &path) - : mDirectoryHandle(INVALID_HANDLE_VALUE) + : mDirectoryHandle(INVALID_HANDLE_VALUE) { - auto widePath = convertMultiByteToWideChar(path); - mDirectoryHandle = openDirectory(widePath); + auto widePath = convertMultiByteToWideChar(path); + mDirectoryHandle = openDirectory(widePath); - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - return; - } + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + return; + } - mWatcher.reset(new Watcher(queue, mDirectoryHandle, widePath)); + mWatcher.reset(new Watcher(queue, mDirectoryHandle, widePath)); } Controller::~Controller() { - mWatcher.reset(); - CancelIo(mDirectoryHandle); - CloseHandle(mDirectoryHandle); - mDirectoryHandle = INVALID_HANDLE_VALUE; + mWatcher.reset(); + CancelIo(mDirectoryHandle); + CloseHandle(mDirectoryHandle); + mDirectoryHandle = INVALID_HANDLE_VALUE; } std::string Controller::getError() { - if (mDirectoryHandle == INVALID_HANDLE_VALUE) { - return "Failed to open directory"; - } - return mWatcher->getError(); + if (mDirectoryHandle == INVALID_HANDLE_VALUE) { + return "Failed to open directory"; + } + return mWatcher->getError(); } bool Controller::hasErrored() { - return mDirectoryHandle == INVALID_HANDLE_VALUE || !mWatcher->getError().empty(); + return mDirectoryHandle == INVALID_HANDLE_VALUE || !mWatcher->getError().empty(); } bool Controller::isWatching() { - return mWatcher->isRunning(); + return mWatcher->isRunning(); } diff --git a/src/win32/Watcher.cpp b/src/win32/Watcher.cpp index 52aebb3a..7b6d78cf 100644 --- a/src/win32/Watcher.cpp +++ b/src/win32/Watcher.cpp @@ -4,277 +4,273 @@ static std::wstring getWStringFileName(LPWSTR cFileName, DWORD length) { - LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); - memcpy(nullTerminatedFileName, cFileName, length); - std::wstring fileName = nullTerminatedFileName; - delete[] nullTerminatedFileName; - return fileName; + LPWSTR nullTerminatedFileName = new WCHAR[length + 1](); + memcpy(nullTerminatedFileName, cFileName, length); + std::wstring fileName = nullTerminatedFileName; + delete[] nullTerminatedFileName; + return fileName; } std::string Watcher::getUTF8Directory(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - std::wstringstream utf16DirectoryStream; - - utf16DirectoryStream << mPath; - - if (found != std::wstring::npos) { - utf16DirectoryStream - << "\\" - << path.substr(0, found); - } - - std::wstring uft16DirectoryString = utf16DirectoryStream.str(); - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - 0, - 0, - NULL, - NULL - ); - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - uft16DirectoryString.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; + std::wstring::size_type found = path.rfind('\\'); + std::wstringstream utf16DirectoryStream; + + utf16DirectoryStream << mPath; + + if (found != std::wstring::npos) { + utf16DirectoryStream + << "\\" + << path.substr(0, found); + } + + std::wstring uft16DirectoryString = utf16DirectoryStream.str(); + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + 0, + 0, + NULL, + NULL + ); + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + uft16DirectoryString.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; } static std::string getUTF8FileName(std::wstring path) { - std::wstring::size_type found = path.rfind('\\'); - if (found != std::wstring::npos) { - path = path.substr(found + 1); - } - - int utf8length = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - 0, - 0, - NULL, - NULL - ); - - // TODO: failure cases for widechar conversion - char *utf8CString = new char[utf8length]; - int failureToResolveToUTF8 = WideCharToMultiByte( - CP_UTF8, - 0, - path.data(), - -1, - utf8CString, - utf8length, - NULL, - NULL - ); - - std::string utf8Directory = utf8CString; - delete[] utf8CString; - - return utf8Directory; + std::wstring::size_type found = path.rfind('\\'); + if (found != std::wstring::npos) { + path = path.substr(found + 1); + } + + int utf8length = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + 0, + 0, + NULL, + NULL + ); + + // TODO: failure cases for widechar conversion + char *utf8CString = new char[utf8length]; + int failureToResolveToUTF8 = WideCharToMultiByte( + CP_UTF8, + 0, + path.data(), + -1, + utf8CString, + utf8length, + NULL, + NULL + ); + + std::string utf8Directory = utf8CString; + delete[] utf8CString; + + return utf8Directory; } Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path) - : mRunning(false) - , mDirectoryHandle(dirHandle) - , mQueue(queue) - , mPath(path) + : mRunning(false) + , mDirectoryHandle(dirHandle) + , mQueue(queue) + , mPath(path) { - ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); - mOverlapped.hEvent = this; - resizeBuffers(1024 * 1024); - start(); + ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); + mOverlapped.hEvent = this; + resizeBuffers(1024 * 1024); + start(); } Watcher::~Watcher() { - stop(); + stop(); } void Watcher::resizeBuffers(std::size_t size) { - mReadBuffer.resize(size); - mWriteBuffer.resize(size); + mReadBuffer.resize(size); + mWriteBuffer.resize(size); } void Watcher::run() { - while(mRunning) { - SleepEx(INFINITE, true); - } + while(mRunning) { + SleepEx(INFINITE, true); + } } bool Watcher::loop() { - DWORD bytes = 0; - - if (!isRunning()) { - return false; - } - - if (!ReadDirectoryChangesW( - mDirectoryHandle, - mWriteBuffer.data(), - static_cast(mWriteBuffer.size()), - TRUE, //recursive watching - FILE_NOTIFY_CHANGE_FILE_NAME - | FILE_NOTIFY_CHANGE_DIR_NAME - | FILE_NOTIFY_CHANGE_ATTRIBUTES - | FILE_NOTIFY_CHANGE_SIZE - | FILE_NOTIFY_CHANGE_LAST_WRITE - | FILE_NOTIFY_CHANGE_LAST_ACCESS - | FILE_NOTIFY_CHANGE_CREATION - | FILE_NOTIFY_CHANGE_SECURITY, - &bytes, //num bytes written - &mOverlapped, - [](DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { - auto watcher = reinterpret_cast(overlapped->hEvent); - watcher->eventCallback(errorCode); - })) - { - setError("Service shutdown unexpectedly"); - return false; - } - - return true; + DWORD bytes = 0; + + if (!isRunning()) { + return false; + } + + if (!ReadDirectoryChangesW( + mDirectoryHandle, + mWriteBuffer.data(), + static_cast(mWriteBuffer.size()), + TRUE, //recursive watching + FILE_NOTIFY_CHANGE_FILE_NAME + | FILE_NOTIFY_CHANGE_DIR_NAME + | FILE_NOTIFY_CHANGE_ATTRIBUTES + | FILE_NOTIFY_CHANGE_SIZE + | FILE_NOTIFY_CHANGE_LAST_WRITE + | FILE_NOTIFY_CHANGE_LAST_ACCESS + | FILE_NOTIFY_CHANGE_CREATION + | FILE_NOTIFY_CHANGE_SECURITY, + &bytes, //num bytes written + &mOverlapped, + [](DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { + auto watcher = reinterpret_cast(overlapped->hEvent); + watcher->eventCallback(errorCode); + })) + { + setError("Service shutdown unexpectedly"); + return false; + } + + return true; } void Watcher::eventCallback(DWORD errorCode) { - if (errorCode != ERROR_SUCCESS) { - if (errorCode == ERROR_NOTIFY_ENUM_DIR) { - setError("Buffer filled up and service needs a restart"); - } else if (errorCode == ERROR_INVALID_PARAMETER) { - // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission - resizeBuffers(64 * 1024); - - if (!loop()) { - setError("failed resizing buffers for network traffic"); - } - } else { - setError("Service shutdown unexpectedly"); - } - return; + if (errorCode != ERROR_SUCCESS) { + if (errorCode == ERROR_NOTIFY_ENUM_DIR) { + setError("Buffer filled up and service needs a restart"); + } else if (errorCode == ERROR_INVALID_PARAMETER) { + // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission + resizeBuffers(64 * 1024); + + if (!loop()) { + setError("failed resizing buffers for network traffic"); + } + } else { + setError("Service shutdown unexpectedly"); } + return; + } - std::swap(mWriteBuffer, mReadBuffer); - BOOL readRequested = loop(); - handleEvents(); - - if (!readRequested) { - //delete (std::shared_ptr *)runner; - } + std::swap(mWriteBuffer, mReadBuffer); + loop(); + handleEvents(); } void Watcher::handleEvents() { - BYTE *base = mReadBuffer.data(); - while (true) { - PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; - std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); - - if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { - if (info->NextEntryOffset != 0) { - base += info->NextEntryOffset; - info = (PFILE_NOTIFY_INFORMATION)base; - if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { - std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); - - mQueue->enqueue( - RENAMED, - getUTF8Directory(fileName), - getUTF8FileName(fileName), - getUTF8FileName(fileNameNew) - ); - } else { - mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - continue; - } - } else { - mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - } - } + BYTE *base = mReadBuffer.data(); + while (true) { + PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; + std::wstring fileName = getWStringFileName(info->FileName, info->FileNameLength); - switch (info->Action) { - case FILE_ACTION_ADDED: - case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer - mQueue->enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_REMOVED: - mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - break; - case FILE_ACTION_MODIFIED: - default: - mQueue->enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); - }; - - if (info->NextEntryOffset == 0) { - break; - } + if (info->Action == FILE_ACTION_RENAMED_OLD_NAME) { + if (info->NextEntryOffset != 0) { base += info->NextEntryOffset; + info = (PFILE_NOTIFY_INFORMATION)base; + if (info->Action == FILE_ACTION_RENAMED_NEW_NAME) { + std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); + + mQueue->enqueue( + RENAMED, + getUTF8Directory(fileName), + getUTF8FileName(fileName), + getUTF8FileName(fileNameNew) + ); + } else { + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + continue; + } + } else { + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + } + } + + switch (info->Action) { + case FILE_ACTION_ADDED: + case FILE_ACTION_RENAMED_NEW_NAME: // in the case we just receive a new name and no old name in the buffer + mQueue->enqueue(CREATED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_REMOVED: + mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + break; + case FILE_ACTION_MODIFIED: + default: + mQueue->enqueue(MODIFIED, getUTF8Directory(fileName), getUTF8FileName(fileName)); + }; + + if (info->NextEntryOffset == 0) { + break; } + base += info->NextEntryOffset; + } } void Watcher::start() { - mRunner = std::thread([this]{ - // mRunning is set to false in the d'tor - mRunning = true; - run(); - }); - - if (!mRunner.joinable()) { - mRunning = false; - return; - } + mRunner = std::thread([this]{ + // mRunning is set to false in the d'tor + mRunning = true; + run(); + }); - QueueUserAPC([](__in ULONG_PTR self) { - auto watcher = reinterpret_cast(self); - watcher->mHasStartedSemaphore.signal(); - watcher->loop(); - } , mRunner.native_handle(), (ULONG_PTR)this); - - if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { - setError("Watcher is not started"); - } + if (!mRunner.joinable()) { + mRunning = false; + return; + } + + QueueUserAPC([](__in ULONG_PTR self) { + auto watcher = reinterpret_cast(self); + watcher->mHasStartedSemaphore.signal(); + watcher->loop(); + } , mRunner.native_handle(), (ULONG_PTR)this); + + if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { + setError("Watcher is not started"); + } } void Watcher::stop() { - mRunning = false; - QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); - mRunner.join(); + mRunning = false; + QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); + mRunner.join(); } void Watcher::setError(const std::string &error) { - std::lock_guard lock(mErrorMutex); - mError = error; + std::lock_guard lock(mErrorMutex); + mError = error; } std::string Watcher::getError() const { - if (!isRunning()) { - return "Failed to start watcher"; - } + if (!isRunning()) { + return "Failed to start watcher"; + } - std::lock_guard lock(mErrorMutex); - return mError; + std::lock_guard lock(mErrorMutex); + return mError; } From 7cd10537fe388577ef4b6a0b822ada8be76a1390 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 20 Sep 2017 15:17:24 +0200 Subject: [PATCH 32/37] style guide issue --- src/NativeInterface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NativeInterface.cpp b/src/NativeInterface.cpp index 9bbb547d..3b18153f 100644 --- a/src/NativeInterface.cpp +++ b/src/NativeInterface.cpp @@ -13,7 +13,7 @@ std::string NativeInterface::getError() { return mNativeInterface->getError(); } -std::vector* NativeInterface::getEvents() { +std::vector *NativeInterface::getEvents() { return mQueue->dequeueAll().release(); } From 2674f6496e0231f3d90c0cbddac12c013e11b7d6 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Wed, 20 Sep 2017 21:52:34 +0200 Subject: [PATCH 33/37] upgrade to msvc 2015 because compile error in older version --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index f587668d..de7e0753 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,7 +15,7 @@ install: # Get the latest stable version of Node.js or io.js - ps: Install-Product node $env:nodejs_version # install modules - - npm install --msvs_version=2013 + - npm install --msvs_version=2015 # Post-install test scripts. test_script: From b782c470e7b862e50b11a3b2502298c04235709c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Fri, 22 Sep 2017 09:30:58 +0200 Subject: [PATCH 34/37] FIX: review comments --- src/win32/Controller.cpp | 18 ++++-------- src/win32/Watcher.cpp | 59 +++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 46 deletions(-) diff --git a/src/win32/Controller.cpp b/src/win32/Controller.cpp index 974052d7..b0efcd56 100644 --- a/src/win32/Controller.cpp +++ b/src/win32/Controller.cpp @@ -1,7 +1,6 @@ #include "../includes/win32/Controller.h" -static std::wstring convertMultiByteToWideChar(const std::string &multiByte) -{ +static std::wstring convertMultiByteToWideChar(const std::string &multiByte) { const int wlen = MultiByteToWideChar(CP_UTF8, 0, multiByte.data(), -1, 0, 0); if (wlen == 0) { @@ -17,8 +16,7 @@ static std::wstring convertMultiByteToWideChar(const std::string &multiByte) return wideString; } -HANDLE Controller::openDirectory(const std::wstring &path) -{ +HANDLE Controller::openDirectory(const std::wstring &path) { return CreateFileW( path.data(), FILE_LIST_DIRECTORY, @@ -46,28 +44,24 @@ Controller::Controller(std::shared_ptr queue, const std::string &pat mWatcher.reset(new Watcher(queue, mDirectoryHandle, widePath)); } -Controller::~Controller() -{ +Controller::~Controller() { mWatcher.reset(); CancelIo(mDirectoryHandle); CloseHandle(mDirectoryHandle); mDirectoryHandle = INVALID_HANDLE_VALUE; } -std::string Controller::getError() -{ +std::string Controller::getError() { if (mDirectoryHandle == INVALID_HANDLE_VALUE) { return "Failed to open directory"; } return mWatcher->getError(); } -bool Controller::hasErrored() -{ +bool Controller::hasErrored() { return mDirectoryHandle == INVALID_HANDLE_VALUE || !mWatcher->getError().empty(); } -bool Controller::isWatching() -{ +bool Controller::isWatching() { return mWatcher->isRunning(); } diff --git a/src/win32/Watcher.cpp b/src/win32/Watcher.cpp index 7b6d78cf..24e46a80 100644 --- a/src/win32/Watcher.cpp +++ b/src/win32/Watcher.cpp @@ -90,10 +90,10 @@ std::string getUTF8FileName(std::wstring path) { } Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std::wstring &path) - : mRunning(false) - , mDirectoryHandle(dirHandle) - , mQueue(queue) - , mPath(path) + : mRunning(false), + mDirectoryHandle(dirHandle), + mQueue(queue), + mPath(path) { ZeroMemory(&mOverlapped, sizeof(OVERLAPPED)); mOverlapped.hEvent = this; @@ -101,26 +101,22 @@ Watcher::Watcher(std::shared_ptr queue, HANDLE dirHandle, const std: start(); } -Watcher::~Watcher() -{ +Watcher::~Watcher() { stop(); } -void Watcher::resizeBuffers(std::size_t size) -{ +void Watcher::resizeBuffers(std::size_t size) { mReadBuffer.resize(size); mWriteBuffer.resize(size); } -void Watcher::run() -{ +void Watcher::run() { while(mRunning) { SleepEx(INFINITE, true); } } -bool Watcher::loop() -{ +bool Watcher::pollDirectoryChanges() { DWORD bytes = 0; if (!isRunning()) { @@ -131,7 +127,7 @@ bool Watcher::loop() mDirectoryHandle, mWriteBuffer.data(), static_cast(mWriteBuffer.size()), - TRUE, //recursive watching + TRUE, // recursive watching FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES @@ -140,7 +136,7 @@ bool Watcher::loop() | FILE_NOTIFY_CHANGE_LAST_ACCESS | FILE_NOTIFY_CHANGE_CREATION | FILE_NOTIFY_CHANGE_SECURITY, - &bytes, //num bytes written + &bytes, // num bytes written &mOverlapped, [](DWORD errorCode, DWORD numBytes, LPOVERLAPPED overlapped) { auto watcher = reinterpret_cast(overlapped->hEvent); @@ -154,8 +150,7 @@ bool Watcher::loop() return true; } -void Watcher::eventCallback(DWORD errorCode) -{ +void Watcher::eventCallback(DWORD errorCode) { if (errorCode != ERROR_SUCCESS) { if (errorCode == ERROR_NOTIFY_ENUM_DIR) { setError("Buffer filled up and service needs a restart"); @@ -163,7 +158,7 @@ void Watcher::eventCallback(DWORD errorCode) // resize the buffers because we're over the network, 64kb is the max buffer size for networked transmission resizeBuffers(64 * 1024); - if (!loop()) { + if (!pollDirectoryChanges()) { setError("failed resizing buffers for network traffic"); } } else { @@ -173,12 +168,11 @@ void Watcher::eventCallback(DWORD errorCode) } std::swap(mWriteBuffer, mReadBuffer); - loop(); + pollDirectoryChanges(); handleEvents(); } -void Watcher::handleEvents() -{ +void Watcher::handleEvents() { BYTE *base = mReadBuffer.data(); while (true) { PFILE_NOTIFY_INFORMATION info = (PFILE_NOTIFY_INFORMATION)base; @@ -192,10 +186,10 @@ void Watcher::handleEvents() std::wstring fileNameNew = getWStringFileName(info->FileName, info->FileNameLength); mQueue->enqueue( - RENAMED, - getUTF8Directory(fileName), - getUTF8FileName(fileName), - getUTF8FileName(fileNameNew) + RENAMED, + getUTF8Directory(fileName), + getUTF8FileName(fileName), + getUTF8FileName(fileNameNew) ); } else { mQueue->enqueue(DELETED, getUTF8Directory(fileName), getUTF8FileName(fileName)); @@ -227,9 +221,8 @@ void Watcher::handleEvents() } } -void Watcher::start() -{ - mRunner = std::thread([this]{ +void Watcher::start() { + mRunner = std::thread([this] { // mRunning is set to false in the d'tor mRunning = true; run(); @@ -251,22 +244,20 @@ void Watcher::start() } } -void Watcher::stop() -{ +void Watcher::stop() { mRunning = false; + // schedule a NOOP APC to force the running loop in `Watcher::run()` to wake + // up, notice the changed `mRunning` and properly terminate the running loop QueueUserAPC([](__in ULONG_PTR) {}, mRunner.native_handle(), (ULONG_PTR)this); mRunner.join(); } -void Watcher::setError(const std::string &error) -{ +void Watcher::setError(const std::string &error) { std::lock_guard lock(mErrorMutex); mError = error; } - -std::string Watcher::getError() const -{ +std::string Watcher::getError() const { if (!isRunning()) { return "Failed to start watcher"; } From f3a859cb595ef185a9084f2577ef761045416b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Meusel?= Date: Fri, 22 Sep 2017 09:43:22 +0200 Subject: [PATCH 35/37] FIX: missing header adaption --- includes/win32/Watcher.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/win32/Watcher.h b/includes/win32/Watcher.h index 591b8002..a7944770 100644 --- a/includes/win32/Watcher.h +++ b/includes/win32/Watcher.h @@ -22,7 +22,7 @@ class Watcher private: void run(); - bool loop(); + bool pollDirectoryChanges(); void start(); void stop(); From 01f1a75ff0743ef6e909789ad88a47ee2bf1f890 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Fri, 22 Sep 2017 10:08:37 +0200 Subject: [PATCH 36/37] fix compile errors in windows ci build --- src/win32/Watcher.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/win32/Watcher.cpp b/src/win32/Watcher.cpp index 24e46a80..b3539c90 100644 --- a/src/win32/Watcher.cpp +++ b/src/win32/Watcher.cpp @@ -236,7 +236,7 @@ void Watcher::start() { QueueUserAPC([](__in ULONG_PTR self) { auto watcher = reinterpret_cast(self); watcher->mHasStartedSemaphore.signal(); - watcher->loop(); + watcher->pollDirectoryChanges(); } , mRunner.native_handle(), (ULONG_PTR)this); if (!mHasStartedSemaphore.waitFor(std::chrono::seconds(10))) { From 990cafb3eaa4730433ac3fdfb5c5f143796ac6a1 Mon Sep 17 00:00:00 2001 From: Patrik Fiedler Date: Fri, 22 Sep 2017 11:09:46 +0200 Subject: [PATCH 37/37] remove empty line in cmakelists --- src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2bfb5f50..b82ed3ac 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,3 @@ - # platform independent code set (NSFW_LIBRARY_SOURCES Queue.cpp