support for proxy.nonDefaultWorkerIdleTime. Proxy crash under load

Author: asilvas-godaddy

examples/proxy/proxy.json

@@ -15,5 +15,5 @@
     }
   ],
   "nonDefaultWorkerCount": 1,
-  "nonDefaultWorkerIdleTime": 3600
+  "nonDefaultWorkerIdleTime": 20
 }

lib/new-worker.js

@@ -50,6 +50,7 @@ function newWorker(options, cb) {
       version = {
         name: options.version,
         port: options.PROXY_PORT,
+        lastAccess: Date.now(),
         online: false
       };
       cservice.locals.proxy.versions[options.version] = version;
@@ -72,11 +73,11 @@ function onMessageFromWorker(msg) {
     return; // ignore invalid cluster-service messages
   }
 
-  var args;
+  var args, version;
 
   switch (msg.cservice.cmd) {
     case "workerReady":
-      var version = cservice.locals.proxy.versions[worker.cservice.version];
+      version = cservice.locals.proxy.versions[worker.cservice.version];
       if (version) {
         // if version detected within worker, flag as online
         version.online = true;
@@ -107,5 +108,11 @@ function onMessageFromWorker(msg) {
         cservice.trigger.apply(cservice, args);
       }
       break;
+    case "versionUpdateLastAccess":
+      version = cservice.locals.proxy.versions[msg.cservice.version];
+      if (version) { // update freshness
+        version.lastAccess = Date.now();
+      }
+      break;
   }
 }

lib/proxy-worker.js

@@ -67,6 +67,7 @@ proxyServer.listen(bindingInfo.port, cservice.workerReady);
 function getProxyVersion(versionStr, cb) {
   var version = cservice.locals.proxy.versions[versionStr];
   if (version) {
+    updateVersionLastAccess(version);
     return waitForVersionToComeOnline(versionStr, cb);
   }
 
@@ -90,7 +91,8 @@ function waitForVersionToComeOnline(versionStr, cb) {
   // try waiting
   var timer, attempts = 0;
   timer = setInterval(function() {
-    if (version.online === true) {
+    version = cservice.locals.proxy.versions[versionStr];
+    if (version && version.online === true) {
       clearInterval(timer);
       return cb(null, version);
     }
@@ -122,3 +124,16 @@ function onMessageFromMaster(msg) {
       break;
   }
 }
+
+function updateVersionLastAccess(version) {
+  var now = Date.now();
+  var diff = now - version.lastAccess;
+  if (diff >= 0 && diff < 5000) {
+    return; // if less than 5 seconds since last access, don't bother updating master
+  }
+  version.lastAccess = now;
+  var msg = cservice.msgBus.createMessage("versionUpdateLastAccess", {
+    version: version.name
+  });
+  process.send(msg);
+}

lib/proxy.js

@@ -81,6 +81,9 @@ function start(options, cb) {
     };
   });
 
+  cservice.locals.proxy.refreshnessTimer = setInterval(checkVersionsForFreshness, 10000);
+  cservice.locals.proxy.refreshnessTimer.unref();
+
   cservice.locals.proxy.enabled = true;
 
   async.parallel(proxyWorkerTasks, function (err) {
@@ -133,6 +136,9 @@ function stop(cb) {
     return cb && cb("Proxy not running");
   }
 
+  clearInterval(cservice.locals.proxy.refreshnessTimer);
+  cservice.locals.proxy.refreshnessTimer = null;
+
   // now lets trigger a shutdown
   cservice.trigger("shutdown", function(err, result) {
     cservice.locals.proxy.enabled = false;
@@ -147,7 +153,7 @@ function version(versionStr, options, cb) {
   }
 
   options = options || {};
-  if (!options.workerCount) {
+  if (isNaN(options.workerCount) === true) {
     options.workerCount =
       (versionStr === cservice.locals.proxy.options.defaultVersion)
       ? cservice.locals.options.workerCount
@@ -161,6 +167,14 @@ function version(versionStr, options, cb) {
   // determine worker delta from desired count and actual count
   var workerCountDelta = options.workerCount - currentVersionWorkers.length;
 
+  // get existing version listing
+  var v = cservice.locals.proxy.versions[versionStr];
+
+  if (v) {
+    // update version lastAccess
+    v.lastAccess = Date.now();
+  }
+
   // if version worker count is already current, nothing more to do
   if (workerCountDelta === 0) {
     return cb && cb();
@@ -194,8 +208,6 @@ function version(versionStr, options, cb) {
     versionStr, cservice.locals.proxy.workerFilename
   );
 
-  // get existing version listing
-  var v = cservice.locals.proxy.versions[versionStr];
   // use existing port if available, otherwise allocate a new one
   var versionPort = (v && v.port) || getNextAvailablePort();
 
@@ -338,18 +350,20 @@ function info(cb) {
     return cb && cb("Proxy cannot invoke 'info' from worker");
   }
 
+  var now = Date.now();
   var proxyWorkers = getProxyWorkers().map(function(worker) {
     var bindingInfo = JSON.parse(worker.cservice.bindingInfo);
     return {
       port: bindingInfo.port,
-      ssl: typeof bindingInfo.tlsOptions === "object",
-
+      ssl: typeof bindingInfo.tlsOptions === "object"
     };
   });
   var versionWorkers = getVersionWorkers().map(function(worker) {
+    var versionInfo = cservice.locals.proxy.versions[worker.cservice.version];
     return {
       worker: worker.cservice.worker,
-      version: worker.cservice.version
+      version: worker.cservice.version,
+      lastAccess: versionInfo ? Math.round((now - versionInfo.lastAccess) / 1000) : "?"
     };
   });
 
@@ -395,3 +409,26 @@ function isVersionRunning(versionStr, cb) {
 
   return false; // no workers running desired version
 }
+
+function checkVersionsForFreshness() {
+  var now = Date.now();
+  for (var k in cservice.locals.proxy.versions) {
+    if (
+      // live version is exempt
+      k === cservice.locals.proxy.options.defaultVersion ||
+      // verify a valid version
+      !cservice.locals.proxy.versions.hasOwnProperty(k)) {
+      continue; // skip
+    }
+    var v = cservice.locals.proxy.versions[k];
+    var diff_s = (now - v.lastAccess) / 1000; // seconds
+    if (diff_s < cservice.locals.proxy.options.nonDefaultWorkerIdleTime) {
+      continue; // all OK
+    }
+
+    cservice.log("Proxy version ".warn + k.info + " shutting down due to inactivity".warn);
+
+    // kill all the things
+    version(k, { workerCount: 0 });
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cluster-service",
-  "version": "2.0.0-alpha1",
+  "version": "2.0.0-alpha2",
   "author": {
     "name": "Aaron Silvas",
     "email": "[email protected]"