Merge pull request #299 from isaacs/master

Upgrade to tar version 3

Author: springmeyer

appveyor.yml

@@ -17,19 +17,9 @@ install:
   - ps: Install-Product node $env:nodejs_version $env:Platform
   - ps: Set-ExecutionPolicy Unrestricted -Scope CurrentUser -Force
   - npm config get
-  # upgrade node-gyp to dodge 2013 compile issue present in the node gyp bundled with node v0.10
-  # https://github.com/nodejs/node-gyp/issues/972#issuecomment-231055109
-  # but we upgrade using my fork since 3.x upstream will now break node v0.10.x support
-  - IF "%nodejs_version:~0,1%"=="0" npm install https://github.com/springmeyer/node-gyp/tarball/v3.x
   # upgrade node-gyp to dodge https://github.com/mapbox/node-pre-gyp/issues/209#issuecomment-307641388
   # and allow make node 4.x x86 builds work
-  # https://github.com/mapbox/node-pre-gyp/issues/209#issuecomment-217690537
-  - IF "%nodejs_version:~0,1%"=="4" npm install [email protected]
-  # downgrade npm to avoid multiple npm bugs:
-  # for node v6 this dodges npm 3.10.10 bug whereby --nodedir/--dist-url is not passed to node-gyp (https://github.com/mapbox/node-pre-gyp/issues/300)
-  # for all node x86 versions this dodges a mysterious ELIFECYCLE error: https://ci.appveyor.com/project/Mapbox/node-pre-gyp/build/1.0.582/job/b8q2nud6vkj0s6qo#L233
-  # for node v8 this dodges https://github.com/mapbox/node-pre-gyp/issues/302
-  - npm install [email protected] -g
+  - IF "%nodejs_version:~0,1%" EQU "4" npm install [email protected]
   - node --version
   - npm --version
   - node -e "console.log(process.arch);"
@@ -38,8 +28,10 @@ install:
   - IF /I "%PLATFORM%" == "x64" CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
   - IF /I "%PLATFORM%" == "x86" CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86
   - npm install
+  # workaround https://github.com/mapbox/node-pre-gyp/issues/300#issuecomment-328179994
+  - IF "%nodejs_version:~0,1%" GEQ "6" npm install npm@2 -g
   - npm test
 
 build: off
 test: off
-deploy: off
+deploy: off

lib/install.js

@@ -6,10 +6,10 @@ exports.usage = 'Attempts to install pre-built binary for module';
 
 var fs = require('fs');
 var path = require('path');
-var zlib = require('zlib');
 var log = require('npmlog');
 var existsAsync = fs.exists || path.exists;
 var versioning = require('./util/versioning.js');
+var mkdirp = require('mkdirp');
 
 var npgVersion = 'unknown';
 try {
@@ -75,8 +75,7 @@ function place_binary(from,to,opts,callback) {
         if (!req) return callback(new Error("empty req"));
         var badDownload = false;
         var extractCount = 0;
-        var gunzip = zlib.createGunzip();
-        var extracter = require('tar').Extract({ path: to, strip: 1});
+        var tar = require('tar');
 
         function afterTarball(err) {
             if (err) return callback(err);
@@ -89,18 +88,10 @@ function place_binary(from,to,opts,callback) {
         }
 
         function filter_func(entry) {
-            // ensure directories are +x
-            // https://github.com/mapnik/node-mapnik/issues/262
-            entry.props.mode |= (entry.props.mode >>> 2) & parseInt('0111',8);
             log.info('install','unpacking ' + entry.path);
             extractCount++;
         }
 
-        gunzip.on('error', callback);
-        extracter.on('entry', filter_func);
-        extracter.on('error', callback);
-        extracter.on('end', afterTarball);
-
         req.on('error', function(err) {
             badDownload = true;
             return callback(err);
@@ -120,7 +111,11 @@ function place_binary(from,to,opts,callback) {
                 return callback(err);
             }
             // start unzipping and untaring
-            req.pipe(gunzip).pipe(extracter);
+            req.pipe(tar.extract({
+              cwd: to,
+              strip: 1,
+              onentry: filter_func
+            }).on('close', afterTarball).on('error', callback));
         });
     });
 }
@@ -187,25 +182,32 @@ function install(gyp, argv, callback) {
         var from = opts.hosted_tarball;
         var to = opts.module_path;
         var binary_module = path.join(to,opts.module_name + '.node');
-        if (existsAsync(binary_module,function(found) {
+        existsAsync(binary_module,function(found) {
             if (found && !update_binary) {
                 console.log('['+package_json.name+'] Success: "' + binary_module + '" already installed');
                 console.log('Pass --update-binary to reinstall or --build-from-source to recompile');
                 return callback();
             } else {
                 if (!update_binary) log.info('check','checked for "' + binary_module + '" (not found)');
-                place_binary(from,to,opts,function(err) {
-                    if (err && should_do_fallback_build) {
-                        print_fallback_error(err,opts,package_json);
-                        return do_build(gyp,argv,callback);
-                    } else if (err) {
-                        return callback(err);
+                mkdirp(to,function(err) {
+                    if (err) {
+                        after_place(err);
                     } else {
-                        console.log('['+package_json.name+'] Success: "' + binary_module + '" is installed via remote');
-                        return callback();
+                        place_binary(from,to,opts,after_place);
                     }
                 });
             }
-        }));
+            function after_place(err) {
+                if (err && should_do_fallback_build) {
+                    print_fallback_error(err,opts,package_json);
+                    return do_build(gyp,argv,callback);
+                } else if (err) {
+                    return callback(err);
+                } else {
+                    console.log('['+package_json.name+'] Success: "' + binary_module + '" is installed via remote');
+                    return callback();
+                }
+            }
+        });
     }
 }

lib/package.js

@@ -8,12 +8,12 @@ var fs = require('fs');
 var path = require('path');
 var log = require('npmlog');
 var versioning = require('./util/versioning.js');
-var write = require('fs').createWriteStream;
 var existsAsync = fs.exists || path.exists;
 var mkdirp = require('mkdirp');
+var tar = require('tar');
 
 function _package(gyp, argv, callback) {
-    var pack = require('tar-pack').pack;
+    var packlist = require('npm-packlist');
     var package_json = JSON.parse(fs.readFileSync('./package.json'));
     var opts = versioning.evaluate(package_json, gyp.opts);
     var from = opts.module_path;
@@ -30,17 +30,21 @@ function _package(gyp, argv, callback) {
             return true;
         };
         mkdirp(path.dirname(tarball),function(err) {
-            if (err) throw err;
-            pack(from, { filter: filter_func })
-             .pipe(write(tarball))
-             .on('error', function(err) {
-                if (err)  console.error('['+package_json.name+'] ' + err.message);
-                return callback(err);
-             })
-             .on('close', function() {
-                log.info('package','Binary staged at "' + tarball + '"');
-                return callback();
-             });
+            from = path.dirname(from);
+            if (err) return callback(err);
+            packlist({ path: from }).then(function(files) {
+                tar.create({
+                    portable: true,
+                    gzip: true,
+                    onentry: filter_func,
+                    file: tarball,
+                    cwd: from
+                }, files, function(err) {
+                    if (err)  console.error('['+package_json.name+'] ' + err.message);
+                    else log.info('package','Binary staged at "' + tarball + '"');
+                    return callback(err);
+                });
+            }, callback);
         });
     });
 }

lib/testpackage.js

@@ -10,8 +10,8 @@ var log = require('npmlog');
 var existsAsync = fs.exists || path.exists;
 var versioning = require('./util/versioning.js');
 var testbinary = require('./testbinary.js');
-var read = require('fs').createReadStream;
-var zlib = require('zlib');
+var tar = require('tar');
+var mkdirp = require('mkdirp');
 
 function testpackage(gyp, argv, callback) {
     var package_json = JSON.parse(fs.readFileSync('./package.json'));
@@ -22,19 +22,24 @@ function testpackage(gyp, argv, callback) {
             return callback(new Error("Cannot test package because " + tarball + " missing: run `node-pre-gyp package` first"));
         }
         var to = opts.module_path;
-        var gunzip = zlib.createGunzip();
-        var extracter = require('tar').Extract({ path: to, strip: 1 });
         function filter_func(entry) {
-            // ensure directories are +x
-            // https://github.com/mapnik/node-mapnik/issues/262
-            entry.props.mode |= (entry.props.mode >>> 2) & parseInt('0111',8);
-            log.info('install','unpacking ' + entry.path);
+            log.info('install','unpacking [' + entry.path + ']');
         }
-        gunzip.on('error', callback);
-        extracter.on('error', callback);
-        extracter.on('entry', filter_func);
-        extracter.on('end', function(err) {
-            if (err) return callback(err);
+
+        mkdirp(to, function(err) {
+            if (err) {
+                return callback(err);
+            } else {
+                tar.extract({
+                    file: tarball,
+                    cwd: to,
+                    strip: 1,
+                    onentry: filter_func
+                }).then(after_extract, callback);
+            }
+        });
+
+        function after_extract() {
             testbinary(gyp,argv,function(err) {
                 if (err) {
                     return callback(err);
@@ -43,7 +48,6 @@ function testpackage(gyp, argv, callback) {
                     return callback();
                 }
             });
-        });
-        read(tarball).pipe(gunzip).pipe(extracter);
+        }
     });
 }

package.json

@@ -22,20 +22,21 @@
   "dependencies": {
     "mkdirp": "^0.5.1",
     "nopt": "^4.0.1",
+    "npm-packlist": "^1.1.6",
     "npmlog": "^4.0.2",
     "rc": "^1.1.7",
     "request": "2.83.0",
     "rimraf": "^2.6.1",
     "semver": "^5.3.0",
-    "detect-libc": "^1.0.2",
-    "tar": "^2.2.1",
-    "tar-pack": "^3.4.0"
+    "tar": "^4"
+    "detect-libc": "^1.0.2"
   },
   "devDependencies": {
     "tape": "^4.6.3",
     "aws-sdk": "^2.28.0",
     "retire": "^1.2.12",
-    "jshint": "^2.9.4"
+    "jshint": "^2.9.4",
+    "tape": "^4.6.3"
   },
   "jshintConfig": {
     "node": true,

test/app1/package.json

@@ -1,21 +1,21 @@
 {
     "name": "node-pre-gyp-test-app1",
     "author": "Dane Springmeyer <springmeyer>",
-    "description":"node-pre-gyp test",
-    "repository"   :  {
-       "type" : "git",
-       "url"  : "git://github.com/mapbox/node-pre-gyp.git"
+    "description": "node-pre-gyp test",
+    "repository": {
+        "type": "git",
+        "url": "git://github.com/mapbox/node-pre-gyp.git"
     },
     "license": "BSD-3-Clause",
     "version": "0.1.0",
     "main": "./index.js",
     "binary": {
         "module_name": "app1",
         "module_path": "./lib/binding/",
-        "host":"https://node-pre-gyp-tests.s3-us-west-1.amazonaws.com"
+        "host": "https://node-pre-gyp-tests.s3-us-west-1.amazonaws.com"
     },
     "scripts": {
-        "install":"node-pre-gyp install --fallback-to-build",
-        "test":"node index.js"
+        "install": "node-pre-gyp install --fallback-to-build",
+        "test": "node index.js"
     }
 }