emit CREATED event for new directories

Most clients expect every new file from a new tree being added in the
watched directory to be reported. This is currently not the case if the
tree appears too quickly on disk.

This commit propagates a callback down the recursive tree building to
make sure that all files added are being reported as created to clients.

I also added an environment variable check for use when testing:
`NSFW_TEST_SLOW`. When set to `1` nsfw will artificially sleep before
crawling folders on the fs. This is to simulate the case when nsfw is
late to the party and sub-folders were created before we had time to
allocate inotify watchers. In such a case, we need to manually emit
CREATED events for new files and folders discovered by scandir.

Signed-off-by: Paul Maréchal <[email protected]>

Author: paul-marechal

binding.gyp

@@ -62,6 +62,9 @@
                 }
             }],
             ["OS=='linux' or OS=='freebsd'", {
+                "defines": [
+                    "NSFW_TEST_SLOW_<!(node -p process.env.NSFW_TEST_SLOW)"
+                ],
                 "sources": [
                     "src/linux/InotifyEventLoop.cpp",
                     "src/linux/InotifyTree.cpp",
@@ -102,7 +105,7 @@
                 ],
                 "libraries": [
                     "-L/usr/local/lib",
-                    "-linotify" 
+                    "-linotify"
                 ]
             }],
         ]

includes/linux/InotifyTree.h

@@ -10,12 +10,20 @@
 #include <vector>
 #include <map>
 #include <unordered_set>
+#include <functional>
+#include <chrono>
+#include <thread>
+
+/**
+ * void EmitCreatedEvent(std::string directory, std::string file);
+ */
+using EmitCreatedEvent = std::function<void(std::string, std::string)>;
 
 class InotifyTree {
 public:
   InotifyTree(int inotifyInstance, std::string path);
 
-  void addDirectory(int wd, std::string name);
+  void addDirectory(int wd, std::string name, EmitCreatedEvent emitCreatedEvent = nullptr);
   std::string getError();
   bool getPath(std::string &out, int wd);
   bool hasErrored();
@@ -34,10 +42,11 @@ class InotifyTree {
       InotifyNode *parent,
       std::string directory,
       std::string name,
-      ino_t inodeNumber
+      ino_t inodeNumber,
+      EmitCreatedEvent emitCreatedEvent = nullptr
     );
 
-    void addChild(std::string name);
+    void addChild(std::string name, EmitCreatedEvent emitCreatedEvent = nullptr);
     void fixPaths();
     std::string getFullPath();
     std::string getName();

js/scripts/test.js

@@ -0,0 +1,17 @@
+const cp = require('child_process');
+const mochaBin = require.resolve('mocha/bin/mocha');
+const stdio = 'inherit';
+
+if (process.platform !== 'win32' && process.platform !== 'darwin') {
+  cp.execSync('yarn rebuild-slow', { stdio });
+  spawnSync(process.argv0, [mochaBin, '--expose-gc', 'js/spec/index-slow-spec.js'], { stdio });
+  cp.execSync('yarn rebuild', { stdio });
+}
+spawnSync(process.argv0, [mochaBin, '--expose-gc', 'js/spec/index-spec.js'], { stdio });
+
+function spawnSync(...args) {
+  const { status } = cp.spawnSync(...args);
+  if (status !== 0) {
+    process.exit(status);
+  }
+}

js/spec/index-slow-spec.js

@@ -0,0 +1,89 @@
+const assert = require('assert');
+const fse = require('fs-extra');
+const path = require('path');
+const { promisify } = require('util');
+
+const nsfw = require('../src');
+
+const DEBOUNCE = 1000;
+const TIMEOUT_PER_STEP = 3000;
+
+const sleep = promisify(setTimeout);
+
+describe('Node Sentinel File Watcher (slow)', function() {
+  this.timeout(120000);
+
+  const workDir = path.resolve('./mockfs');
+
+  beforeEach(async function() {
+    await fse.mkdir(workDir);
+  });
+
+  afterEach(function() {
+    return fse.remove(workDir);
+  });
+
+  describe('Recursive', function() {
+    it('can listen for the creation of a deeply nested file', async function() {
+      const folders = 'a_very_deep_tree'.split(''); // ['a', '_', ...]
+      const file = 'a_file.txt';
+
+      // Resolve a promise as soon as all events have been found
+      let done; const promise = new Promise(resolve => {
+        done = resolve;
+      });
+
+      /**
+       * We will remove or pop entries as we find them.
+       * This set should be empty at the end of the test.
+       * @type {Set<string>}
+       */
+      const events = new Set();
+      let directory = workDir; for (const folder of folders) {
+        directory = path.join(directory, folder);
+        events.add(directory);
+      }
+      events.add(path.join(directory, file));
+
+      // The modified build should emit an artificial event named NSFW_TEST_SLOW/1:
+      events.add('NSFW_TEST_SLOW/1');
+
+      function findEvent(element) {
+        if (element.action === nsfw.actions.CREATED) {
+          const file = path.join(element.directory, element.file);
+          if (events.has(file)) {
+            events.delete(file);
+          }
+          if (events.size === 0) {
+            done();
+          }
+        }
+      }
+
+      let watch = await nsfw(
+        workDir,
+        events => events.forEach(findEvent),
+        { debounceMS: DEBOUNCE }
+      );
+
+      try {
+        await watch.start();
+        await sleep(TIMEOUT_PER_STEP);
+        await fse.mkdirp(directory);
+        await fse.close(await fse.open(path.join(directory, file), 'w'));
+        await Promise.race([
+          sleep(folders.length * 500 * 1.5),
+          promise,
+        ]);
+
+        // Make sure that we got the events for each nested directory
+        assert.ok(events.size === 0, `Some files were not detected:\n${
+          Array.from(events, file => `- ${file}`).join('\n')
+        }`);
+      } finally {
+        await watch.stop();
+        watch = null;
+      }
+    });
+  });
+});

js/spec/index-spec.js

@@ -442,6 +442,7 @@ describe('Node Sentinel File Watcher', function() {
       const folders = 'a_very_deep_tree'.split(''); // ['a', '_', ...]
       const file = 'a_file.txt';
 
+      /** @type {Map<string, { file: string, found: boolean }>} */
       const events = new Map();
       let directory = workDir; for (const folder of folders) {
         events.set(directory, { file: folder, found: false });
@@ -459,6 +460,10 @@ describe('Node Sentinel File Watcher', function() {
         }
       }
 
+      // Force nsfw to sleep and hence miss creation events.
+      // This makes sure we test the right mechanism that should manually
+      // fire creation events when nested folders are added at once.
+      process.env['NSFW_TEST_INOTIFYNODE_SLEEP'] = '1';
       let watch = await nsfw(
         workDir,
         events => events.forEach(findEvent),
@@ -470,17 +475,18 @@ describe('Node Sentinel File Watcher', function() {
         await sleep(TIMEOUT_PER_STEP);
         await fse.mkdirp(directory);
         await fse.close(await fse.open(path.join(directory, file), 'w'));
-        await sleep(TIMEOUT_PER_STEP);
+        await sleep(folders.length * 200 * 1.5);
 
+        // Make sure that we got the events for each nested directory
         for (const [directory, item] of events.entries()) {
           assert.ok(item.found, `${path.join(directory, item.file)} wasn't found`);
         }
       } finally {
+        delete process.env['NSFW_TEST_INOTIFYNODE_SLEEP'];
         await watch.stop();
         watch = null;
       }
     });
-
     it('can listen for the destruction of a directory and its subtree', async function() {
       const inPath = path.resolve(workDir, 'test4');
       let deletionCount = 0;

package.json

@@ -5,7 +5,9 @@
   "main": "js/src/index.js",
   "scripts": {
     "lint": "eslint js/src js/spec",
-    "test": "yarn lint && mocha --expose-gc js/spec"
+    "rebuild": "node-gyp rebuild",
+    "rebuild-slow": "NSFW_TEST_SLOW=1 node-gyp rebuild",
+    "test": "yarn lint && node --expose-gc js/scripts/test.js"
   },
   "repository": {
     "type": "git",

src/linux/InotifyService.cpp

@@ -16,6 +16,10 @@ InotifyService::InotifyService(std::shared_ptr<EventQueue> queue, std::string pa
     mTree = NULL;
     mEventLoop = NULL;
   } else {
+    #ifdef NSFW_TEST_SLOW_1
+      // Emit a fake event to notify clients that this is a test build
+      mQueue->enqueue(CREATED, "NSFW_TEST_SLOW", "1");
+    #endif
     mEventLoop = new InotifyEventLoop(
       mInotifyInstance,
       this
@@ -98,8 +102,10 @@ void InotifyService::createDirectory(int wd, std::string name) {
     return;
   }
 
-  mTree->addDirectory(wd, name);
   dispatch(CREATED, wd, name);
+  mTree->addDirectory(wd, name, [this](std::string directory, std::string file) {
+    mQueue->enqueue(CREATED, directory, file);
+  });
 }
 
 void InotifyService::removeDirectory(int wd) {

src/linux/InotifyTree.cpp

@@ -1,12 +1,13 @@
 #include "../../includes/linux/InotifyTree.h"
+#include <cstdio>
+#include <sys/stat.h>
 /**
  * InotifyTree ---------------------------------------------------------------------------------------------------------
  */
 InotifyTree::InotifyTree(int inotifyInstance, std::string path):
   mError(""),
   mInotifyInstance(inotifyInstance) {
   mInotifyNodeByWatchDescriptor = new std::map<int, InotifyNode *>;
-
   std::string directory;
   std::string watchName;
   if (path.length() == 1 && path[0] == '/') {
@@ -41,14 +42,14 @@ InotifyTree::InotifyTree(int inotifyInstance, std::string path):
   }
 }
 
-void InotifyTree::addDirectory(int wd, std::string name) {
+void InotifyTree::addDirectory(int wd, std::string name, EmitCreatedEvent emitCreatedEvent) {
   auto nodeIterator = mInotifyNodeByWatchDescriptor->find(wd);
   if (nodeIterator == mInotifyNodeByWatchDescriptor->end()) {
     return;
   }
 
   InotifyNode *node = nodeIterator->second;
-  node->addChild(name);
+  node->addChild(name, emitCreatedEvent);
 }
 
 void InotifyTree::addNodeReferenceByWD(int wd, InotifyNode *node) {
@@ -159,7 +160,8 @@ InotifyTree::InotifyNode::InotifyNode(
   InotifyNode *parent,
   std::string directory,
   std::string name,
-  ino_t inodeNumber
+  ino_t inodeNumber,
+  EmitCreatedEvent emitCreatedEvent
 ):
   mDirectory(directory),
   mInodeNumber(inodeNumber),
@@ -171,6 +173,10 @@ InotifyTree::InotifyNode::InotifyNode(
   mFullPath = createFullPath(mDirectory, mName);
   mWatchDescriptorInitialized = false;
 
+  #ifdef NSFW_TEST_SLOW_1
+    std::this_thread::sleep_for(std::chrono::milliseconds(500));
+  #endif
+
   if (!inotifyInit()) {
     return;
   }
@@ -200,6 +206,10 @@ InotifyTree::InotifyNode::InotifyNode(
       continue;
     }
 
+    if (emitCreatedEvent) {
+      emitCreatedEvent(mFullPath, fileName);
+    }
+
     std::string filePath = createFullPath(mFullPath, fileName);
 
     struct stat file;
@@ -218,7 +228,8 @@ InotifyTree::InotifyNode::InotifyNode(
       this,
       mFullPath,
       fileName,
-      file.st_ino
+      file.st_ino,
+      emitCreatedEvent
     );
 
     if (child->isAlive()) {
@@ -250,7 +261,7 @@ InotifyTree::InotifyNode::~InotifyNode() {
   delete mChildren;
 }
 
-void InotifyTree::InotifyNode::addChild(std::string name) {
+void InotifyTree::InotifyNode::addChild(std::string name, EmitCreatedEvent emitCreatedEvent) {
   struct stat file;
 
   if (stat(createFullPath(mFullPath, name).c_str(), &file) >= 0 && mTree->addInode(file.st_ino)) {
@@ -260,7 +271,8 @@ void InotifyTree::InotifyNode::addChild(std::string name) {
       this,
       mFullPath,
       name,
-      file.st_ino
+      file.st_ino,
+      emitCreatedEvent
     );
 
     if (child->isAlive()) {