emit CREATED event for new directories

Most clients expect every new file from a new tree being added in the
watched directory to be reported. This is currently not the case if the
tree appears too quickly on disk.

This commit propagates a callback down the recursive tree building to
make sure that all files added are being reported as created to clients.

I also added an environment variable check for use when testing:
`NSFW_TEST_INOTIFYNODE_SLEEP`. When set to `1` nsfw will artificially
sleep before crawling folders on the fs. This is to simulate the case
when nsfw is late to the party and sub-folders where created before we
had time to allocate inotify watchers. In such a case, we need to
manually emit CREATED events for new files and folders discovered by
scandir.

Signed-off-by: Paul Maréchal <[email protected]>

Author: paul-marechal

includes/linux/InotifyTree.h

@@ -10,12 +10,29 @@
 #include <vector>
 #include <map>
 #include <unordered_set>
+#include <functional>
+#include <chrono>
+#include <thread>
+
+/**
+ * Environment variable key for testing.
+ *
+ * When set to "1", it will trigger a sleep in every InotifyNode constructor.
+ * This will allow nsfw to miss creation events, and instead we'll have to rely
+ * on the manual event emitting mechanism. Used by tests.
+ */
+#define NSFW_TEST_INOTIFYNODE_SLEEP "NSFW_TEST_INOTIFYNODE_SLEEP"
+
+/**
+ * void EmitCreatedEvent(std::string directory, std::string file);
+ */
+using EmitCreatedEvent = std::function<void(std::string, std::string)>;
 
 class InotifyTree {
 public:
   InotifyTree(int inotifyInstance, std::string path);
 
-  void addDirectory(int wd, std::string name);
+  void addDirectory(int wd, std::string name, EmitCreatedEvent emitCreatedEvent = nullptr);
   std::string getError();
   bool getPath(std::string &out, int wd);
   bool hasErrored();
@@ -34,10 +51,12 @@ class InotifyTree {
       InotifyNode *parent,
       std::string directory,
       std::string name,
-      ino_t inodeNumber
+      ino_t inodeNumber,
+      bool testSleep = false,
+      EmitCreatedEvent emitCreatedEvent = nullptr
     );
 
-    void addChild(std::string name);
+    void addChild(std::string name, bool testSleep = false, EmitCreatedEvent emitCreatedEvent = nullptr);
     void fixPaths();
     std::string getFullPath();
     std::string getName();
@@ -86,6 +105,7 @@ class InotifyTree {
   std::map<int, InotifyNode *> *mInotifyNodeByWatchDescriptor;
   std::unordered_set<ino_t> inodes;
   InotifyNode *mRoot;
+  bool mTestSleep; // meant for testing
 
   friend class InotifyNode;
 };

js/spec/index-spec.js

@@ -442,6 +442,7 @@ describe('Node Sentinel File Watcher', function() {
       const folders = 'a_very_deep_tree'.split(''); // ['a', '_', ...]
       const file = 'a_file.txt';
 
+      /** @type {Map<string, { file: string, found: boolean }>} */
       const events = new Map();
       let directory = workDir; for (const folder of folders) {
         events.set(directory, { file: folder, found: false });
@@ -459,6 +460,10 @@ describe('Node Sentinel File Watcher', function() {
         }
       }
 
+      // Force nsfw to sleep and hence miss creation events.
+      // This makes sure we test the right mechanism that should manually
+      // fire creation events when nested folders are added at once.
+      process.env['NSFW_TEST_INOTIFYNODE_SLEEP'] = '1';
       let watch = await nsfw(
         workDir,
         events => events.forEach(findEvent),
@@ -470,17 +475,18 @@ describe('Node Sentinel File Watcher', function() {
         await sleep(TIMEOUT_PER_STEP);
         await fse.mkdirp(directory);
         await fse.close(await fse.open(path.join(directory, file), 'w'));
-        await sleep(TIMEOUT_PER_STEP);
+        await sleep(folders.length * 200 * 1.5);
 
+        // Make sure that we got the events for each nested directory
         for (const [directory, item] of events.entries()) {
           assert.ok(item.found, `${path.join(directory, item.file)} wasn't found`);
         }
       } finally {
+        delete process.env['NSFW_TEST_INOTIFYNODE_SLEEP'];
         await watch.stop();
         watch = null;
       }
     });
-
     it('can listen for the destruction of a directory and its subtree', async function() {
       const inPath = path.resolve(workDir, 'test4');
       let deletionCount = 0;

src/linux/InotifyService.cpp

@@ -98,8 +98,10 @@ void InotifyService::createDirectory(int wd, std::string name) {
     return;
   }
 
-  mTree->addDirectory(wd, name);
   dispatch(CREATED, wd, name);
+  mTree->addDirectory(wd, name, [this](std::string directory, std::string file) {
+    mQueue->enqueue(CREATED, directory, file);
+  });
 }
 
 void InotifyService::removeDirectory(int wd) {

src/linux/InotifyTree.cpp

@@ -1,12 +1,13 @@
 #include "../../includes/linux/InotifyTree.h"
+#include <cstdio>
+#include <sys/stat.h>
 /**
  * InotifyTree ---------------------------------------------------------------------------------------------------------
  */
 InotifyTree::InotifyTree(int inotifyInstance, std::string path):
   mError(""),
   mInotifyInstance(inotifyInstance) {
   mInotifyNodeByWatchDescriptor = new std::map<int, InotifyNode *>;
-
   std::string directory;
   std::string watchName;
   if (path.length() == 1 && path[0] == '/') {
@@ -24,14 +25,21 @@ InotifyTree::InotifyTree(int inotifyInstance, std::string path):
     return;
   }
 
+  char *_test_sleep_env = std::getenv(NSFW_TEST_INOTIFYNODE_SLEEP);
+  mTestSleep = _test_sleep_env != nullptr && std::string(_test_sleep_env) == "1";
+  if (mTestSleep) {
+    printf("WARNING: test sleep enabled!\n");
+  }
+
   addInode(file.st_ino);
   mRoot = new InotifyNode(
     this,
     mInotifyInstance,
     NULL,
     directory,
     watchName,
-    file.st_ino
+    file.st_ino,
+    mTestSleep
   );
 
   if (!mRoot->isAlive()) {
@@ -41,14 +49,14 @@ InotifyTree::InotifyTree(int inotifyInstance, std::string path):
   }
 }
 
-void InotifyTree::addDirectory(int wd, std::string name) {
+void InotifyTree::addDirectory(int wd, std::string name, EmitCreatedEvent emitCreatedEvent) {
   auto nodeIterator = mInotifyNodeByWatchDescriptor->find(wd);
   if (nodeIterator == mInotifyNodeByWatchDescriptor->end()) {
     return;
   }
 
   InotifyNode *node = nodeIterator->second;
-  node->addChild(name);
+  node->addChild(name, mTestSleep, emitCreatedEvent);
 }
 
 void InotifyTree::addNodeReferenceByWD(int wd, InotifyNode *node) {
@@ -159,7 +167,9 @@ InotifyTree::InotifyNode::InotifyNode(
   InotifyNode *parent,
   std::string directory,
   std::string name,
-  ino_t inodeNumber
+  ino_t inodeNumber,
+  bool testSleep,
+  EmitCreatedEvent emitCreatedEvent
 ):
   mDirectory(directory),
   mInodeNumber(inodeNumber),
@@ -171,6 +181,10 @@ InotifyTree::InotifyNode::InotifyNode(
   mFullPath = createFullPath(mDirectory, mName);
   mWatchDescriptorInitialized = false;
 
+  if (testSleep) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+  }
+
   if (!inotifyInit()) {
     return;
   }
@@ -200,6 +214,10 @@ InotifyTree::InotifyNode::InotifyNode(
       continue;
     }
 
+    if (emitCreatedEvent) {
+      emitCreatedEvent(mFullPath, fileName);
+    }
+
     std::string filePath = createFullPath(mFullPath, fileName);
 
     struct stat file;
@@ -218,7 +236,9 @@ InotifyTree::InotifyNode::InotifyNode(
       this,
       mFullPath,
       fileName,
-      file.st_ino
+      file.st_ino,
+      testSleep,
+      emitCreatedEvent
     );
 
     if (child->isAlive()) {
@@ -250,7 +270,7 @@ InotifyTree::InotifyNode::~InotifyNode() {
   delete mChildren;
 }
 
-void InotifyTree::InotifyNode::addChild(std::string name) {
+void InotifyTree::InotifyNode::addChild(std::string name, bool testSleep, EmitCreatedEvent emitCreatedEvent) {
   struct stat file;
 
   if (stat(createFullPath(mFullPath, name).c_str(), &file) >= 0 && mTree->addInode(file.st_ino)) {
@@ -260,7 +280,9 @@ void InotifyTree::InotifyNode::addChild(std::string name) {
       this,
       mFullPath,
       name,
-      file.st_ino
+      file.st_ino,
+      testSleep,
+      emitCreatedEvent
     );
 
     if (child->isAlive()) {