revert PR9; add --no-deps to pip wheel

Revert PR9; it was never actually needed in the first place it just fixed an
issue by side effect.

1) add --no-deps to pip wheel step in order to avoid building wheels for
   dependencies. This also addresses the issues raised in PR pypa#11.

2) Modify the pip install step across platforms to allow installation of
   dependencies from the index. (This is required since we will no longer be
   building wheels for dependencies).

Author: tgarc

cibuildwheel/__main__.py

@@ -107,6 +107,9 @@ def main():
 
     print_preamble(platform, build_options)
 
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+
     if platform == 'linux':
         cibuildwheel.linux.build(**build_options)
     elif platform == 'windows':

cibuildwheel/linux.py

@@ -53,30 +53,33 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             cd /project
 
             for PYBIN in {pybin_paths}; do
+                # Setup
+                rm -rf /tmp/built_wheel
+                rm -rf /tmp/delocated_wheel
+                mkdir /tmp/built_wheel
+                mkdir /tmp/delocated_wheel
+
                 if [ ! -z {before_build} ]; then
                     PATH=$PYBIN:$PATH sh -c {before_build}
                 fi
 
-                # install the package first to take care of dependencies
-                "$PYBIN/pip" install .
-
-                "$PYBIN/pip" wheel --no-deps . -w /tmp/linux_wheels
-            done
+                # Build that wheel
+                "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps
+                built_wheel=(/tmp/built_wheel/*.whl)
 
-            for whl in /tmp/linux_wheels/*.whl; do
-                if [[ "$whl" == *none-any.whl ]]; then
-                    # pure python wheel - just copy to the output
-                    cp "$whl" /output
+                # Delocate the wheel
+                # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
+                # the first element
+                if [[ "$built_wheel" == *none-any.whl ]]; then
+                    # pure python wheel - just copy
+                    cp "$built_wheel" /tmp/delocated_wheel
                 else
-                    auditwheel repair "$whl" -w /output
+                    auditwheel repair "$built_wheel" -w /tmp/delocated_wheel
                 fi
-            done
+                delocated_wheel=(/tmp/delocated_wheel/*.whl)
 
-            # Install packages and test
-            for PYBIN in {pybin_paths}; do
                 # Install the wheel we just built
-                "$PYBIN/pip" install {package_name} \
-                    --upgrade --force-reinstall --no-deps --no-index -f /output
+                "$PYBIN/pip" install "$delocated_wheel"
 
                 # Install any requirements to run the tests
                 if [ ! -z "{test_requires}" ]; then
@@ -89,8 +92,12 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
                     PATH=$PYBIN:$PATH sh -c {test_command}
                     popd
                 fi
+
+                # we're all done here; move it to output
+                mv "$delocated_wheel" /output
             done
         '''.format(
+            platform=platform_tag,
             package_name=package_name,
             pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
             test_requires=' '.join(test_requires),
@@ -111,7 +118,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
                 '-v', '%s:/output' % os.path.abspath(output_dir),
                 docker_image,
                 '/bin/bash'],
-            stdin=subprocess.PIPE, universal_newlines=True)
+                stdin=subprocess.PIPE, universal_newlines=True)
 
         try:
             docker_process.communicate(bash_script)

cibuildwheel/macos.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import os, subprocess, shlex, sys
+import os, subprocess, shlex, sys, shutil
 from collections import namedtuple
 from glob import glob
 try:
@@ -53,31 +53,38 @@ def shell(args, env=None, cwd=None):
         shell([pip, 'install', 'wheel'], env=env)
         shell([pip, 'install', 'delocate'], env=env)
 
+        # setup dirs
+        if os.path.exists('/tmp/built_wheel'):
+            shutil.rmtree('/tmp/built_wheel')
+        os.makedirs('/tmp/built_wheel')
+        if os.path.exists('/tmp/delocated_wheel'):
+            shutil.rmtree('/tmp/delocated_wheel')
+        os.makedirs('/tmp/delocated_wheel')
+
         # run the before_build command
         if before_build:
             before_build_prepared = prepare_command(before_build, python=python, pip=pip)
             shell(shlex.split(before_build_prepared), env=env)
 
-        # install the package first to take care of dependencies
-        shell([pip, 'install', project_dir], env=env)
+        # build the wheel
+        shell([pip, 'wheel', project_dir, '-w', '/tmp/built_wheel', '--no-deps'], env=env)
 
-        # build the wheel to temp dir
-        temp_wheel_dir = '/tmp/tmpwheel%s' % config.version
-        shell([pip, 'wheel', project_dir, '-w', temp_wheel_dir, '--no-deps'], env=env)
-        temp_wheel = glob(temp_wheel_dir+'/*.whl')[0]
+        built_wheel = glob('/tmp/built_wheel/*.whl')[0]
 
-        if temp_wheel.endswith('none-any.whl'):
-            # pure python wheel - just copy to output_dir
-            shell(['cp', temp_wheel, output_dir], env=env)
+        if built_wheel.endswith('none-any.whl'):
+            # pure python wheel - just copy
+            shutil.move(built_wheel, '/tmp/delocated_wheel')
         else:
             # list the dependencies
-            shell(['delocate-listdeps', temp_wheel], env=env)
+            shell(['delocate-listdeps', built_wheel], env=env)
             # rebuild the wheel with shared libraries included and place in output dir
-            shell(['delocate-wheel', '-w', output_dir, temp_wheel], env=env)
+            shell(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env)
+            os.remove(built_wheel)
+
+        delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0]
 
-        # now install the package from the generated wheel
-        shell([pip, 'install', package_name, '--upgrade', '--force-reinstall',
-               '--no-deps', '--no-index', '--find-links', output_dir], env=env)
+        # install the wheel
+        shell([pip, 'install', delocated_wheel], env=env)
 
         # test the wheel
         if test_requires:
@@ -89,3 +96,6 @@ def shell(args, env=None, cwd=None):
             abs_project_dir = os.path.abspath(project_dir)
             test_command_absolute = test_command.format(project=abs_project_dir)
             shell(shlex.split(test_command_absolute), cwd=os.environ['HOME'], env=env)
+
+        # we're all done here; move it to output
+        shutil.move(delocated_wheel, output_dir)

cibuildwheel/windows.py

@@ -1,10 +1,11 @@
 from __future__ import print_function
-import os, tempfile, subprocess, sys
+import os, tempfile, subprocess, sys, shutil
 try:
     from urllib2 import urlopen
 except ImportError:
     from urllib.request import urlopen
 from collections import namedtuple
+from glob import glob
 
 from .util import prepare_command
 
@@ -21,7 +22,7 @@ def shell(args, env=None, cwd=None):
         # print the command executing for the logs
         print('+ ' + ' '.join(args))
         args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
-        return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
+        return subprocess.check_output(' '.join(args), env=env, cwd=cwd)
 
     PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path'])
     python_configurations = [
@@ -37,11 +38,19 @@ def shell(args, env=None, cwd=None):
         PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'),
     ]
 
+    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
+    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+
     for config in python_configurations:
         if skip(config.identifier):
             print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr)
             continue
 
+        # setup dirs
+        if os.path.exists(built_wheel_dir):
+            shutil.rmtree(built_wheel_dir)
+        os.makedirs(built_wheel_dir)
+
         env = os.environ.copy()
         # set up environment variables for run_with_env
         env['PYTHON_VERSION'] = config.version
@@ -66,16 +75,12 @@ def shell(args, env=None, cwd=None):
             before_build_prepared = prepare_command(before_build, python='python', pip='pip')
             shell([before_build_prepared], env=env)
 
-        # install the package first to take care of dependencies
-        shell(['pip', 'install', project_dir], env=env)
-
         # build the wheel
-        shell(['pip', 'wheel', project_dir, '-w', output_dir, '--no-deps'], env=env)
+        shell(['pip', 'wheel', project_dir, '-w', built_wheel_dir, '--no-deps'], env=env)
 
         # install the wheel
-        shell(['pip', 'install', package_name, '--upgrade',
-               '--force-reinstall', '--no-deps', '--no-index', '-f',
-               output_dir], env=env)
+        built_wheel = glob(built_wheel_dir+'/*.whl')[0]
+        shell(['pip', 'install', built_wheel], env=env)
 
         # test the wheel
         if test_requires:
@@ -87,3 +92,6 @@ def shell(args, env=None, cwd=None):
             abs_project_dir = os.path.abspath(project_dir)
             test_command_absolute = test_command.format(project=abs_project_dir)
             shell([test_command_absolute], cwd='c:\\', env=env)
+
+        # we're all done here; move it to output
+        shutil.move(built_wheel, output_dir)