Skip tokens when pipeline function returns null

Previously only undefined and empty string return values would skip the
token, although the documentation specifically mentioned returning null.
This makes sure that null and undefined and empty string all have the
same behaviour and updates the documentation accordingly.

Author: olivernn

lib/pipeline.js

@@ -44,8 +44,8 @@ lunr.Pipeline.registeredFunctions = Object.create(null)
  * or mutate (or add) metadata for a given token.
  *
  * A pipeline function can indicate that the passed token should be discarded by returning
- * null. This token will not be passed to any downstream pipeline functions and will not be
- * added to the index.
+ * null, undefined or an empty string. This token will not be passed to any downstream pipeline
+ * functions and will not be added to the index.
  *
  * Multiple tokens can be returned by returning an array of tokens. Each token will be passed
  * to any downstream pipeline functions and all will returned tokens will be added to the index.
@@ -208,7 +208,7 @@ lunr.Pipeline.prototype.run = function (tokens) {
     for (var j = 0; j < tokens.length; j++) {
       var result = fn(tokens[j], j, tokens)
 
-      if (result === void 0 || result === '') continue
+      if (result === null || result === void 0 || result === '') continue
 
       if (Array.isArray(result)) {
         for (var k = 0; k < result.length; k++) {

test/pipeline_test.js

@@ -142,14 +142,23 @@ suite('lunr.Pipeline', function () {
       assert.deepEqual(['FOO'], this.pipeline.run(['foo']))
     })
 
-    test('filters out undefined values', function () {
+    test('filters out null, undefined and empty string values', function () {
       var tokens = [],
           output
 
       // only pass on tokens for even token indexes
+      // return null for 'foo'
+      // return undefined for 'bar'
+      // return '' for 'baz'
       this.pipeline.add(function (t, i) {
-        if (i % 2) {
+        if (i == 4) {
+          return null
+        } else if (i == 5) {
+          return ''
+        } if (i % 2) {
           return t
+        } else {
+          return undefined
         }
       })
 
@@ -158,7 +167,7 @@ suite('lunr.Pipeline', function () {
         return t
       })
 
-      output = this.pipeline.run(['a', 'b', 'c', 'd'])
+      output = this.pipeline.run(['a', 'b', 'c', 'd', 'foo', 'bar', 'baz'])
 
       assert.sameMembers(['b', 'd'], tokens)
       assert.sameMembers(['b', 'd'], output)