Migrate many types from str to Text. (#2263)

This supports an up coming refactor of `pex.{dist_metadata,pep_376}`
which also requires re-formulating the pyc filters from `Iterable` ->
`Iterable` form to boolean predicates to make typing easier at the
loss of ~nothing.

Author: jsirois

pex/bin/pex.py

@@ -23,7 +23,7 @@
     global_environment,
     register_global_arguments,
 )
-from pex.common import die, filter_pyc_dirs, filter_pyc_files, safe_mkdtemp
+from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp
 from pex.enum import Enum
 from pex.inherit_path import InheritPath
 from pex.interpreter_constraints import InterpreterConstraints
@@ -48,7 +48,7 @@
 
 if TYPE_CHECKING:
     from argparse import Namespace
-    from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple
+    from typing import Dict, Iterable, Iterator, List, Optional, Set, Text, Tuple
 
     import attr  # vendor:skip
 
@@ -495,7 +495,7 @@ def parse(cls, name):
     subdir = attr.ib(default=None)  # type: Optional[str]
 
     def iter_files(self):
-        # type: () -> Iterator[Tuple[str, str]]
+        # type: () -> Iterator[Tuple[Text, Text]]
         components = self.name.split(".")
         parent_package_dirs = components[:-1]
         source = components[-1]
@@ -520,7 +520,7 @@ def _iter_source_files(
         parent_package_path,  # type: List[str]
         source,  # type: str
     ):
-        # type: (...) -> Iterator[Tuple[str, str]]
+        # type: (...) -> Iterator[Tuple[Text, Text]]
         raise NotImplementedError()
 
 
@@ -530,11 +530,13 @@ def _iter_source_files(
         parent_package_path,  # type: List[str]
         source,  # type: str
     ):
-        # type: (...) -> Iterator[Tuple[str, str]]
+        # type: (...) -> Iterator[Tuple[Text, Text]]
         package_dir = os.path.join(*(parent_package_path + [source]))
         for root, dirs, files in os.walk(package_dir):
-            dirs[:] = list(filter_pyc_dirs(dirs))
-            for f in filter_pyc_files(files):
+            dirs[:] = [d for d in dirs if not is_pyc_dir(d)]
+            for f in files:
+                if is_pyc_file(f):
+                    continue
                 src = os.path.join(root, f)
                 dst = os.path.relpath(src, self.subdir) if self.subdir else src
                 yield src, dst
@@ -546,7 +548,7 @@ def _iter_source_files(
         parent_package_path,  # type: List[str]
         source,  # type: str
     ):
-        # type: (...) -> Iterator[Tuple[str, str]]
+        # type: (...) -> Iterator[Tuple[Text, Text]]
         module_src = os.path.join(*(parent_package_path + ["{module}.py".format(module=source)]))
         module_dest = os.path.relpath(module_src, self.subdir) if self.subdir else module_src
         yield module_src, module_dest
@@ -726,7 +728,7 @@ def _iter_directory_sources(directories):
 
 
 def _iter_python_sources(python_sources):
-    # type: (Iterable[PythonSource]) -> Iterator[Tuple[str, str]]
+    # type: (Iterable[PythonSource]) -> Iterator[Tuple[Text, Text]]
     for python_source in python_sources:
         for src, dst in python_source.iter_files():
             yield src, dst
@@ -778,7 +780,7 @@ def build_pex(
             "dependency cache."
         )
 
-    seen = set()  # type: Set[Tuple[str, str]]
+    seen = set()  # type: Set[Tuple[Text, Text]]
     for src, dst in itertools.chain(
         _iter_directory_sources(
             OrderedSet(options.sources_directory + options.resources_directory)

pex/common.py

@@ -20,7 +20,7 @@
 from datetime import datetime
 from uuid import uuid4
 
-from pex.typing import TYPE_CHECKING, cast
+from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import (
@@ -35,6 +35,7 @@
         Optional,
         Set,
         Sized,
+        Text,
         Tuple,
         Union,
     )
@@ -49,26 +50,22 @@
 DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds()
 
 
-def filter_pyc_dirs(dirs):
-    # type: (Iterable[str]) -> Iterator[str]
-    """Return an iterator over the input `dirs` filtering out Python bytecode cache directories."""
-    for d in dirs:
-        if d != "__pycache__":
-            yield d
+def is_pyc_dir(dir_path):
+    # type: (Text) -> bool
+    """Return `True` if `dir_path` is a Python bytecode cache directory."""
+    return os.path.basename(dir_path) == "__pycache__"
 
 
-def filter_pyc_files(files):
-    # type: (Iterable[str]) -> Iterator[str]
-    """Iterate the input `files` filtering out any Python bytecode files."""
-    for f in files:
-        # For Python 2.7, `.pyc` files are compiled as siblings to `.py` files (there is no
-        # __pycache__ dir).
-        if not f.endswith((".pyc", ".pyo")) and not is_pyc_temporary_file(f):
-            yield f
+def is_pyc_file(file_path):
+    # type: (Text) -> bool
+    """Return `True` if `file_path` is a Python bytecode file."""
+    # N.B.: For Python 2.7, `.pyc` files are compiled as siblings to `.py` files (there is no
+    # __pycache__ dir).
+    return file_path.endswith((".pyc", ".pyo")) or is_pyc_temporary_file(file_path)
 
 
 def is_pyc_temporary_file(file_path):
-    # type: (str) -> bool
+    # type: (Text) -> bool
     """Check if `file` is a temporary Python bytecode file."""
     # We rely on the fact that the temporary files created by CPython have object id (integer)
     # suffixes to avoid picking up files where Python bytecode compilation is in-flight; i.e.:
@@ -99,7 +96,7 @@ def pluralize(
 
 
 def safe_copy(source, dest, overwrite=False):
-    # type: (str, str, bool) -> None
+    # type: (Text, Text, bool) -> None
     def do_copy():
         # type: () -> None
         temp_dest = dest + uuid4().hex
@@ -224,6 +221,7 @@ def _chmod(self, info, path):
 
 @contextlib.contextmanager
 def open_zip(path, *args, **kwargs):
+    # type: (Text, *Any, **Any) -> Iterator[PermPreservingZipFile]
     """A contextmanager for zip files.
 
     Passes through positional and kwargs to zipfile.ZipFile.
@@ -289,7 +287,7 @@ def register_rmtree(directory):
 
 
 def safe_mkdir(directory, clean=False):
-    # type: (str, bool) -> str
+    # type: (Text, bool) -> Text
     """Safely create a directory.
 
     Ensures a directory is present.  If it's not there, it is created.  If it is, it's a no-op. If
@@ -332,7 +330,7 @@ def safe_delete(filename):
 
 
 def safe_rmtree(directory):
-    # type: (str) -> None
+    # type: (Text) -> None
     """Delete a directory if it's present.
 
     If it's not present, no-op.

pex/dist_metadata.py

@@ -137,12 +137,12 @@ def _parse_sdist_package_info(sdist_path):
 class DistMetadataFile(object):
     project_name = attr.ib()  # type: ProjectName
     version = attr.ib()  # type: Version
-    path = attr.ib()  # type: str
+    path = attr.ib()  # type: Text
 
 
 def find_dist_info_files(
     filename,  # type: Text
-    listing,  # type: Iterable[str]
+    listing,  # type: Iterable[Text]
 ):
     # type: (...) -> Iterator[DistMetadataFile]
 
@@ -168,10 +168,10 @@ def find_dist_info_files(
 def find_dist_info_file(
     project_name,  # type: Union[Text, ProjectName]
     filename,  # type: Text
-    listing,  # type: Iterable[str]
+    listing,  # type: Iterable[Text]
     version=None,  # type: Optional[Union[Text, Version]]
 ):
-    # type: (...) -> Optional[str]
+    # type: (...) -> Optional[Text]
 
     normalized_project_name = (
         project_name if isinstance(project_name, ProjectName) else ProjectName(project_name)
@@ -581,24 +581,24 @@ def _realpath(path):
 class Distribution(object):
     @staticmethod
     def _read_metadata_lines(metadata_path):
-        # type: (str) -> Iterator[str]
-        with open(os.path.join(metadata_path)) as fp:
+        # type: (Text) -> Iterator[Text]
+        with open(os.path.join(metadata_path), "rb") as fp:
             for line in fp:
                 # This is pkg_resources.IMetadataProvider.get_metadata_lines behavior, which our
                 # code expects.
-                normalized = line.strip()
+                normalized = line.decode("utf-8").strip()
                 if normalized and not normalized.startswith("#"):
                     yield normalized
 
     @classmethod
     def parse_entry_map(cls, entry_points_metadata_path):
-        # type: (str) -> Dict[str, Dict[str, EntryPoint]]
+        # type: (Text) -> Dict[Text, Dict[Text, EntryPoint]]
 
         # This file format is defined here:
         #   https://packaging.python.org/en/latest/specifications/entry-points/#file-format
 
-        entry_map = defaultdict(dict)  # type: DefaultDict[str, Dict[str, EntryPoint]]
-        group = None  # type: Optional[str]
+        entry_map = defaultdict(dict)  # type: DefaultDict[Text, Dict[Text, EntryPoint]]
+        group = None  # type: Optional[Text]
         for index, line in enumerate(cls._read_metadata_lines(entry_points_metadata_path), start=1):
             if line.startswith("[") and line.endswith("]"):
                 group = line[1:-1]
@@ -624,7 +624,7 @@ def load(cls, location):
     metadata = attr.ib()  # type: DistMetadata
     _metadata_files_cache = attr.ib(
         factory=dict, init=False, eq=False, repr=False
-    )  # type: Dict[str, str]
+    )  # type: Dict[Text, Text]
 
     @property
     def key(self):
@@ -658,7 +658,7 @@ def requires(self):
         return self.metadata.requires_dists
 
     def _get_metadata_file(self, name):
-        # type: (str) -> Optional[str]
+        # type: (Text) -> Optional[Text]
         normalized_name = os.path.normpath(name)
         if os.path.isabs(normalized_name):
             raise ValueError(
@@ -691,7 +691,7 @@ def has_metadata(self, name):
         return self._get_metadata_file(name) is not None
 
     def get_metadata_lines(self, name):
-        # type: (str) -> Iterator[str]
+        # type: (Text) -> Iterator[Text]
         relative_path = self._get_metadata_file(name)
         if relative_path is None:
             raise MetadataNotFoundError(
@@ -707,7 +707,7 @@ def get_metadata_lines(self, name):
             yield line
 
     def get_entry_map(self):
-        # type: () -> Dict[str, Dict[str, EntryPoint]]
+        # type: () -> Dict[Text, Dict[Text, EntryPoint]]
         entry_points_metadata_relpath = self._get_metadata_file("entry_points.txt")
         if entry_points_metadata_relpath is None:
             return defaultdict(dict)
@@ -724,7 +724,7 @@ def __str__(self):
 class EntryPoint(object):
     @classmethod
     def parse(cls, spec):
-        # type: (str) -> EntryPoint
+        # type: (Text) -> EntryPoint
 
         # This file format is defined here:
         #   https://packaging.python.org/en/latest/specifications/entry-points/#file-format
@@ -734,7 +734,8 @@ def parse(cls, spec):
             raise ValueError("Invalid entry point specification: {spec}.".format(spec=spec))
 
         name, value = components
-        module, sep, attrs = value.strip().partition(":")
+        # N.B.: Python identifiers must be ascii.
+        module, sep, attrs = str(value).strip().partition(":")
         if sep and not attrs:
             raise ValueError("Invalid entry point specification: {spec}.".format(spec=spec))
 
@@ -746,7 +747,7 @@ def parse(cls, spec):
 
         return cls(name=entry_point_name, module=module)
 
-    name = attr.ib()  # type: str
+    name = attr.ib()  # type: Text
     module = attr.ib()  # type: str
 
     def __str__(self):

pex/hashing.py

@@ -10,7 +10,7 @@
 from pex.typing import TYPE_CHECKING, Generic
 
 if TYPE_CHECKING:
-    from typing import IO, Callable, Iterable, Iterator, Optional, Protocol, Type, TypeVar
+    from typing import IO, Callable, Iterable, Iterator, Optional, Protocol, Text, Type, TypeVar
 
     class HintedDigest(Protocol):
         @property
@@ -177,7 +177,7 @@ def update_hash(
 
 
 def file_hash(
-    path,  # type: str
+    path,  # type: Text
     digest,  # type: HintedDigest
 ):
     # type: (...) -> None
@@ -187,21 +187,22 @@ def file_hash(
 
 
 def dir_hash(
-    directory,  # type: str
+    directory,  # type: Text
     digest,  # type: HintedDigest
-    dir_filter=lambda dirs: dirs,  # type: Callable[[Iterable[str]], Iterable[str]]
-    file_filter=lambda files: files,  # type: Callable[[Iterable[str]], Iterable[str]]
+    dir_filter=lambda d: True,  # type: Callable[[Text], bool]
+    file_filter=lambda f: True,  # type: Callable[[Text], bool]
 ):
     # type: (...) -> None
     """Digest the contents of a directory in a reproducible manner."""
 
     def iter_files():
-        # type: () -> Iterator[str]
+        # type: () -> Iterator[Text]
         normpath = os.path.realpath(os.path.normpath(directory))
         for root, dirs, files in os.walk(normpath):
-            dirs[:] = list(dir_filter(dirs))
-            for f in file_filter(files):
-                yield os.path.relpath(os.path.join(root, f), normpath)
+            dirs[:] = [d for d in dirs if dir_filter(d)]
+            for f in files:
+                if file_filter(f):
+                    yield os.path.relpath(os.path.join(root, f), normpath)
 
     names = sorted(iter_files())
 
@@ -214,11 +215,11 @@ def iter_files():
 
 
 def zip_hash(
-    zip_path,  # type: str
+    zip_path,  # type: Text
     digest,  # type: HintedDigest
-    relpath=None,  # type: Optional[str]
-    dir_filter=lambda dirs: dirs,  # type: Callable[[Iterable[str]], Iterable[str]]
-    file_filter=lambda files: files,  # type: Callable[[Iterable[str]], Iterable[str]]
+    relpath=None,  # type: Optional[Text]
+    dir_filter=lambda d: True,  # type: Callable[[Text], bool]
+    file_filter=lambda f: True,  # type: Callable[[Text], bool]
 ):
     # type: (...) -> None
     """Digest the contents of a zip file in a reproducible manner.
@@ -234,19 +235,15 @@ def zip_hash(
         )
 
         dirs = frozenset(name.rstrip("/") for name in namelist if name.endswith("/"))
-        accept_dir_names = frozenset(dir_filter(os.path.basename(d) for d in dirs))
-        accept_dirs = frozenset(d for d in dirs if os.path.basename(d) in accept_dir_names)
+        accept_dirs = frozenset(d for d in dirs if dir_filter(os.path.basename(d)))
         reject_dirs = dirs - accept_dirs
 
         accept_files = sorted(
-            file_filter(
-                name
-                for name in namelist
-                if not (
-                    name.endswith("/")
-                    or any(name.startswith(reject_dir) for reject_dir in reject_dirs)
-                )
-            )
+            name
+            for name in namelist
+            if not name.endswith("/")
+            and not any(name.startswith(reject_dir) for reject_dir in reject_dirs)
+            and file_filter(os.path.basename(name))
         )
 
         hashed_names = (