Skip to content

Commit 00417c5

Browse files
authored
Fix cross-drive commonpath on Windows. (#2709)
More work towards #2658.
1 parent 8d4b14a commit 00417c5

27 files changed

+83
-57
lines changed

pex/cache/dirs.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import os
88

99
from pex.common import safe_rmtree
10-
from pex.compatibility import commonpath
10+
from pex.compatibility import safe_commonpath
1111
from pex.enum import Enum
1212
from pex.exceptions import production_assert
1313
from pex.orderedset import OrderedSet
@@ -897,7 +897,7 @@ def venv_dir(self):
897897
if not self.interpreter.is_venv:
898898
return None
899899
cached_venv_root = CacheDir.VENVS.path()
900-
if cached_venv_root != commonpath((cached_venv_root, self.interpreter.prefix)):
900+
if cached_venv_root != safe_commonpath((cached_venv_root, self.interpreter.prefix)):
901901
return None
902902
head, contents_hash = os.path.split(self.interpreter.prefix)
903903
pex_hash = os.path.basename(head)

pex/cache/root.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import os.path
77

8-
from pex.compatibility import commonpath
8+
from pex.compatibility import safe_commonpath
99
from pex.typing import TYPE_CHECKING
1010

1111
if TYPE_CHECKING:
@@ -23,6 +23,6 @@
2323
def path(expand_user=True):
2424
# type: (bool) -> str
2525

26-
if expand_user or _USER_DIR != commonpath((_USER_DIR, _CACHE_DIR)):
26+
if expand_user or _USER_DIR != safe_commonpath((_USER_DIR, _CACHE_DIR)):
2727
return _CACHE_DIR
2828
return os.path.join("~", os.path.relpath(_CACHE_DIR, _USER_DIR))

pex/cli/commands/lock.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from pex.cli.command import BuildTimeCommand
1919
from pex.commands.command import JsonMixin, OutputMixin
2020
from pex.common import pluralize, safe_delete, safe_mkdtemp, safe_open
21-
from pex.compatibility import commonpath, shlex_quote
21+
from pex.compatibility import safe_commonpath, shlex_quote
2222
from pex.dependency_configuration import DependencyConfiguration
2323
from pex.dist_metadata import (
2424
Constraint,
@@ -307,7 +307,11 @@ def sync(
307307
if to_unlink:
308308
to_unlink_by_pin[
309309
(distribution.metadata.project_name, distribution.metadata.version)
310-
] = [file for file in to_unlink if abs_venv_dir == commonpath((abs_venv_dir, file))]
310+
] = [
311+
file
312+
for file in to_unlink
313+
if abs_venv_dir == safe_commonpath((abs_venv_dir, file))
314+
]
311315
if confirm and to_unlink_by_pin:
312316
for (project_name, version), files in to_unlink_by_pin.items():
313317
print(project_name, version, ":", file=sys.stderr)
@@ -325,7 +329,7 @@ def sync(
325329
safe_delete(file)
326330
parent_dirs.add(os.path.dirname(file))
327331
for parent_dir in sorted(parent_dirs, reverse=True):
328-
if not os.listdir(parent_dir) and abs_venv_dir == commonpath(
332+
if not os.listdir(parent_dir) and abs_venv_dir == safe_commonpath(
329333
(abs_venv_dir, parent_dir)
330334
):
331335
os.rmdir(parent_dir)

pex/compatibility.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from abc import ABCMeta
1414
from sys import version_info as sys_version_info
1515

16+
from pex.os import WINDOWS
1617
from pex.typing import TYPE_CHECKING, cast
1718

1819
if TYPE_CHECKING:
@@ -243,6 +244,12 @@ def commonpath(paths):
243244
paths="\n".join(paths)
244245
)
245246
)
247+
if WINDOWS and len({os.path.splitdrive(path)[0] for path in paths}) > 1:
248+
raise ValueError(
249+
"Can't mix paths from different drives, given:\n{paths}".format(
250+
paths="\n".join(paths)
251+
)
252+
)
246253

247254
def components(path):
248255
# type: (Text) -> Iterable[Text]
@@ -274,6 +281,18 @@ def append(piece):
274281
return os.path.join(*prefix)
275282

276283

284+
def safe_commonpath(paths):
285+
# type: (Sequence[Text]) -> Optional[Text]
286+
287+
# Handle the ValueError cases outlined in os.path.safe_commonpath and return None instead:
288+
# > Raise ValueError if paths contain both absolute and relative pathnames, if paths are on
289+
# > different drives, or if paths is empty.
290+
try:
291+
return commonpath(paths)
292+
except ValueError:
293+
return None
294+
295+
277296
if PY3:
278297
from shlex import quote as _shlex_quote
279298
else:

pex/pep_427.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
from pex import pex_warnings, windows
1717
from pex.common import is_pyc_file, iter_copytree, open_zip, safe_open, touch
18-
from pex.compatibility import commonpath, get_stdout_bytes_buffer
18+
from pex.compatibility import commonpath, get_stdout_bytes_buffer, safe_commonpath
1919
from pex.dist_metadata import CallableEntryPoint, Distribution, ProjectNameAndVersion
2020
from pex.enum import Enum
2121
from pex.executables import chmod_plus_x
@@ -215,7 +215,8 @@ def record_files(
215215
names=[
216216
name
217217
for name in zf.namelist()
218-
if not name.endswith("/") and data_rel_path != commonpath((data_rel_path, name))
218+
if not name.endswith("/")
219+
and data_rel_path != safe_commonpath((data_rel_path, name))
219220
],
220221
)
221222
if os.path.isdir(data_path):

pex/pex_builder.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
safe_mkdtemp,
2727
safe_open,
2828
)
29-
from pex.compatibility import commonpath, to_bytes
29+
from pex.compatibility import safe_commonpath, to_bytes
3030
from pex.compiler import Compiler
3131
from pex.dist_metadata import Distribution, DistributionType, MetadataError
3232
from pex.enum import Enum
@@ -456,7 +456,7 @@ def _precompile_source(self):
456456
if path.endswith(".py")
457457
# N.B.: Some of our vendored code does not work with all versions of Python we support;
458458
# so we just skip compiling it.
459-
and vendored_dir != commonpath((vendored_dir, path))
459+
and vendored_dir != safe_commonpath((vendored_dir, path))
460460
]
461461

462462
compiler = Compiler(self.interpreter)

pex/repl/pex_repl.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
from pex.cli_util import prog_path
1414
from pex.common import pluralize
15-
from pex.compatibility import commonpath
15+
from pex.compatibility import safe_commonpath
1616
from pex.dist_metadata import Distribution
1717
from pex.layout import Layout
1818
from pex.os import is_exe
@@ -155,7 +155,7 @@ def _create_repl_data(
155155
layout = Layout.identify_original(env.PEX or pex)
156156
pex_root = os.path.abspath(env.PEX_ROOT)
157157
venv = venv or pex_info.venv
158-
venv_pex = venv and pex_root == commonpath((os.path.abspath(pex), pex_root))
158+
venv_pex = venv and pex_root == safe_commonpath((os.path.abspath(pex), pex_root))
159159
pex_prog_path = prog_path(env.PEX if venv_pex else pex)
160160
pex_cli_run_in_use = _pex_cli_run_in_use()
161161
ephemeral = "ephemeral " if pex_cli_run_in_use else ""

pex/venv/virtualenv.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from pex.atomic_directory import AtomicDirectory, atomic_directory
1818
from pex.common import safe_mkdir, safe_open
19-
from pex.compatibility import commonpath, get_stdout_bytes_buffer
19+
from pex.compatibility import commonpath, get_stdout_bytes_buffer, safe_commonpath
2020
from pex.dist_metadata import Distribution, find_distributions
2121
from pex.enum import Enum
2222
from pex.executor import Executor
@@ -373,7 +373,7 @@ def create_atomic(
373373
link_target = os.readlink(abs_path)
374374
if not os.path.isabs(link_target):
375375
continue
376-
if virtualenv.bin_dir == commonpath((virtualenv.bin_dir, link_target)):
376+
if virtualenv.bin_dir == safe_commonpath((virtualenv.bin_dir, link_target)):
377377
rel_dst = os.path.relpath(link_target, virtualenv.bin_dir)
378378
TRACER.log(
379379
"Replacing absolute symlink {src} -> {dst} with relative symlink".format(

tests/integration/cli/commands/test_issue_2059.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import os.path
44
import sys
55

6-
from pex.compatibility import commonpath
6+
from pex.compatibility import safe_commonpath
77
from pex.typing import TYPE_CHECKING
88
from testing import PY310, ensure_python_interpreter, run_pex_command
99
from testing.cli import run_pex3
@@ -51,4 +51,4 @@ def test_pypy_impl_tag_handling(tmpdir):
5151
python=python,
5252
)
5353
result.assert_success()
54-
assert pex_root == commonpath([pex_root, os.path.realpath(result.output.strip())])
54+
assert pex_root == safe_commonpath([pex_root, os.path.realpath(result.output.strip())])

tests/integration/cli/commands/test_issue_2098.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from textwrap import dedent
77

88
from pex.common import safe_open, touch
9-
from pex.compatibility import commonpath
9+
from pex.compatibility import safe_commonpath
1010
from pex.interpreter import PythonInterpreter
1111
from testing import run_pex_command
1212
from testing.cli import run_pex3
@@ -101,4 +101,4 @@ def test_missing_download_lock_analysis_handling(
101101

102102
data = json.loads(result.output)
103103
assert "1.3.24" == data["version"]
104-
assert pex_root == commonpath([pex_root, data["file"]])
104+
assert pex_root == safe_commonpath([pex_root, data["file"]])

tests/integration/cli/commands/test_issue_2268.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import pytest
77

8-
from pex.compatibility import commonpath
8+
from pex.compatibility import safe_commonpath
99
from pex.resolve.resolver_configuration import ResolverVersion
1010
from pex.typing import TYPE_CHECKING
1111
from testing import run_pex_command
@@ -71,4 +71,4 @@ def test_abi_none_locking(
7171
]
7272
)
7373
result.assert_success()
74-
assert pex_root == commonpath([pex_root, result.output.strip()])
74+
assert pex_root == safe_commonpath([pex_root, result.output.strip()])

tests/integration/cli/commands/test_venv_create.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from pex import dist_metadata
1616
from pex.cli.commands.venv import InstallLayout
1717
from pex.common import open_zip, safe_open
18-
from pex.compatibility import commonpath
18+
from pex.compatibility import safe_commonpath
1919
from pex.dist_metadata import Distribution
2020
from pex.interpreter import PythonInterpreter
2121
from pex.pep_440 import Version
@@ -124,7 +124,7 @@ def assert_venv(
124124
_, stdout, _ = venv.interpreter.execute(
125125
args=["-c", "import cowsay, os; print(os.path.realpath(cowsay.__file__))"]
126126
)
127-
assert venv.site_packages_dir == commonpath([venv.site_packages_dir, stdout.strip()])
127+
assert venv.site_packages_dir == safe_commonpath([venv.site_packages_dir, stdout.strip()])
128128

129129
_, stdout, _ = venv.interpreter.execute(args=["-m", "cowsay", "--version"])
130130
assert "5.0" == stdout.strip()
@@ -188,7 +188,7 @@ def assert_flat(
188188
sys_path_entry = dest if layout is InstallLayout.FLAT else "{dest}.zip".format(dest=dest)
189189
env = make_env(PYTHONPATH=sys_path_entry)
190190

191-
assert sys_path_entry == commonpath(
191+
assert sys_path_entry == safe_commonpath(
192192
[
193193
sys_path_entry,
194194
subprocess.check_output(
@@ -330,7 +330,7 @@ def assert_deps_only(
330330
):
331331
# type: (...) -> None
332332

333-
assert expected_prefix == commonpath(
333+
assert expected_prefix == safe_commonpath(
334334
[
335335
expected_prefix,
336336
subprocess.check_output(

tests/integration/resolve/test_issue_1918.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
import pytest
88

9-
from pex.compatibility import commonpath
9+
from pex.compatibility import safe_commonpath
1010
from pex.dist_metadata import Requirement
1111
from pex.pip.version import PipVersion, PipVersionValue
1212
from pex.requirements import VCS
@@ -149,4 +149,4 @@ def test_redacted_requirement_handling(
149149
]
150150
)
151151
result.assert_success()
152-
assert pex_root == commonpath([pex_root, result.output.strip()])
152+
assert pex_root == safe_commonpath([pex_root, result.output.strip()])

tests/integration/test_excludes.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import pytest
1616

1717
from pex.common import safe_open
18-
from pex.compatibility import PY2, commonpath
18+
from pex.compatibility import PY2, safe_commonpath
1919
from pex.dist_metadata import Requirement
2020
from pex.executor import Executor
2121
from pex.pep_503 import ProjectName
@@ -489,7 +489,7 @@ def run_pex(*args):
489489
)
490490
assert 42 == data.pop("foo")
491491
bar_module_path = data.pop("bar")
492-
assert venv.site_packages_dir == commonpath(
492+
assert venv.site_packages_dir == safe_commonpath(
493493
(venv.site_packages_dir, bar_module_path)
494494
), bar_module_path
495495
assert not data

tests/integration/test_integration.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from pex import targets
2020
from pex.cache.dirs import CacheDir, InterpreterDir
2121
from pex.common import environment_as, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch
22-
from pex.compatibility import commonpath
22+
from pex.compatibility import safe_commonpath
2323
from pex.dist_metadata import Distribution, Requirement, is_wheel
2424
from pex.fetcher import URLFetcher
2525
from pex.fs import safe_symlink
@@ -1675,7 +1675,7 @@ def run_isort_pex(pex_python=None):
16751675
pex_hash=pex_hash,
16761676
has_interpreter_constraints=False,
16771677
)
1678-
assert expected_venv_home == commonpath([pex_interpreter, expected_venv_home])
1678+
assert expected_venv_home == safe_commonpath([pex_interpreter, expected_venv_home])
16791679
return pex_interpreter
16801680

16811681
isort_pex_interpreter1 = run_isort_pex()

tests/integration/test_interpreter.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import os.path
44
import sys
55

6-
from pex.compatibility import commonpath
6+
from pex.compatibility import safe_commonpath
77
from pex.interpreter import PythonInterpreter
88
from pex.typing import TYPE_CHECKING
99
from pex.variables import ENV
@@ -23,7 +23,7 @@ def assert_no_isolated_leak(python):
2323
with ENV.patch(PEX_ROOT=pex_root), PythonInterpreter._cleared_memory_cache():
2424
interpreter = PythonInterpreter.from_binary(python)
2525
assert not any(
26-
pex_root == commonpath((pex_root, entry)) for entry in interpreter.sys_path
26+
pex_root == safe_commonpath((pex_root, entry)) for entry in interpreter.sys_path
2727
), (
2828
"The cached interpreter info for {python} contains leaked entries:\n"
2929
"{entries}".format(python=python, entries="\n".join(interpreter.sys_path))

tests/integration/test_issue_1336.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
import os
55

6-
from pex.compatibility import commonpath
6+
from pex.compatibility import safe_commonpath
77
from pex.typing import TYPE_CHECKING
88
from testing import PY310, ensure_python_interpreter, run_pex_command, subprocess
99

@@ -34,6 +34,6 @@ def test_pip_leak(tmpdir):
3434
python=python,
3535
)
3636
result.assert_success()
37-
assert os.path.realpath(pex_root) == commonpath(
37+
assert os.path.realpath(pex_root) == safe_commonpath(
3838
[os.path.realpath(pex_root), result.output.strip()]
3939
)

tests/integration/test_issue_1949.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
import pytest
66

7-
from pex.compatibility import commonpath
7+
from pex.compatibility import safe_commonpath
88
from pex.typing import TYPE_CHECKING
99
from testing import IntegResults, built_wheel, run_pex_command
1010
from testing.cli import run_pex3
@@ -59,11 +59,11 @@ def create_pex_from_lock(*additional_args):
5959

6060
result = create_pex_from_lock()
6161
result.assert_success()
62-
assert pex_root == commonpath((pex_root, result.output.strip()))
62+
assert pex_root == safe_commonpath((pex_root, result.output.strip()))
6363

6464
result = create_pex_from_lock("not_boto===2.49.0a1")
6565
result.assert_success()
66-
assert pex_root == commonpath((pex_root, result.output.strip()))
66+
assert pex_root == safe_commonpath((pex_root, result.output.strip()))
6767

6868
result = create_pex_from_lock("not_boto===2.49a1")
6969
result.assert_failure()
@@ -101,7 +101,7 @@ def create_pex_from_pex_repository(requirement):
101101

102102
result = create_pex_from_pex_repository("not_boto===2.49.0a1")
103103
result.assert_success()
104-
assert pex_root == commonpath((pex_root, result.output.strip()))
104+
assert pex_root == safe_commonpath((pex_root, result.output.strip()))
105105

106106
result = create_pex_from_pex_repository("not_boto===2.49a1")
107107
result.assert_failure()

tests/integration/test_issue_2087.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import os.path
66
from textwrap import dedent
77

8-
from pex.compatibility import commonpath
8+
from pex.compatibility import safe_commonpath
99
from pex.typing import TYPE_CHECKING
1010
from testing import run_pex_command
1111

@@ -47,4 +47,4 @@ def test_long_wheel_names(tmpdir):
4747

4848
data = json.loads(result.output)
4949
assert "3.16.0" == data["version"]
50-
assert pex_root == commonpath((pex_root, data["path"]))
50+
assert pex_root == safe_commonpath((pex_root, data["path"]))

0 commit comments

Comments
 (0)