Skip to content

Commit e930fee

Browse files
authored
Merge pull request #13065 from sirosen/dependency-groups
Implement a `--group` option for installing from `[dependency-groups]` found in `pyproject.toml` files
2 parents 76895a0 + 3e46676 commit e930fee

19 files changed

+976
-1
lines changed

docs/html/user_guide.rst

+92
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,98 @@ Same as requirements files, constraints files can also be served via a URL,
256256
e.g. http://example.com/constraints.txt, so that your organization can store and
257257
serve them in a centralized place.
258258

259+
260+
.. _`Dependency Groups`:
261+
262+
263+
Dependency Groups
264+
=================
265+
266+
"Dependency Groups" are lists of items to be installed stored in a
267+
``pyproject.toml`` file.
268+
269+
A dependency group is logically just a list of requirements, similar to the
270+
contents of :ref:`Requirements Files`. Unlike requirements files, dependency
271+
groups cannot contain non-package arguments for :ref:`pip install`.
272+
273+
Groups can be declared like so:
274+
275+
.. code-block:: toml
276+
277+
# pyproject.toml
278+
[dependency-groups]
279+
groupA = [
280+
"pkg1",
281+
"pkg2",
282+
]
283+
284+
and installed with :ref:`pip install` like so:
285+
286+
.. tab:: Unix/macOS
287+
288+
.. code-block:: shell
289+
290+
python -m pip install --group groupA
291+
292+
.. tab:: Windows
293+
294+
.. code-block:: shell
295+
296+
py -m pip install --group groupA
297+
298+
Full details on the contents of ``[dependency-groups]`` and more examples are
299+
available in :ref:`the specification documentation <pypug:dependency-groups>`.
300+
301+
.. note::
302+
303+
Dependency Groups are defined by a standard, and therefore do not support
304+
``pip``-specific syntax for requirements, only :ref:`standard dependency
305+
specifiers <pypug:dependency-specifiers>`.
306+
307+
``pip`` does not search projects or directories to discover ``pyproject.toml``
308+
files. The ``--group`` option is used to pass the path to the file,
309+
and if the path is omitted, as in the example above, it defaults to
310+
``pyproject.toml`` in the current directory. Using explicit paths,
311+
:ref:`pip install` can use a file from another directory. For example:
312+
313+
.. tab:: Unix/macOS
314+
315+
.. code-block:: shell
316+
317+
python -m pip install --group './project/subproject/pyproject.toml:groupA'
318+
319+
.. tab:: Windows
320+
321+
.. code-block:: shell
322+
323+
py -m pip install --group './project/subproject/pyproject.toml:groupA'
324+
325+
326+
This also makes it possible to install groups from multiple different projects
327+
at once. For example, with a directory structure like so::
328+
329+
+ project/
330+
+ sub1/
331+
- pyproject.toml
332+
+ sub2/
333+
- pyproject.toml
334+
335+
it is possible to install, from the ``project/`` directory, groups from the
336+
subprojects thusly:
337+
338+
.. tab:: Unix/macOS
339+
340+
.. code-block:: shell
341+
342+
python -m pip install --group './sub1/pyproject.toml:groupA' --group './sub2/pyproject.toml:groupB'
343+
344+
.. tab:: Windows
345+
346+
.. code-block:: shell
347+
348+
py -m pip install --group './sub1/pyproject.toml:groupA' --group './sub2/pyproject.toml:groupB'
349+
350+
259351
.. _`Installing from Wheels`:
260352

261353

news/12963.feature.rst

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
- Add a ``--group`` option which allows installation from PEP 735 Dependency
2+
Groups. ``--group`` accepts arguments of the form ``group`` or
3+
``path:group``, where the default path is ``pyproject.toml``, and installs
4+
the named Dependency Group from the provided ``pyproject.toml`` file.

src/pip/_internal/cli/cmdoptions.py

+41
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import importlib.util
1414
import logging
1515
import os
16+
import pathlib
1617
import textwrap
1718
from functools import partial
1819
from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values
@@ -733,6 +734,46 @@ def _handle_no_cache_dir(
733734
help="Don't install package dependencies.",
734735
)
735736

737+
738+
def _handle_dependency_group(
739+
option: Option, opt: str, value: str, parser: OptionParser
740+
) -> None:
741+
"""
742+
Process a value provided for the --group option.
743+
744+
Splits on the rightmost ":", and validates that the path (if present) ends
745+
in `pyproject.toml`. Defaults the path to `pyproject.toml` when one is not given.
746+
747+
`:` cannot appear in dependency group names, so this is a safe and simple parse.
748+
749+
This is an optparse.Option callback for the dependency_groups option.
750+
"""
751+
path, sep, groupname = value.rpartition(":")
752+
if not sep:
753+
path = "pyproject.toml"
754+
else:
755+
# check for 'pyproject.toml' filenames using pathlib
756+
if pathlib.PurePath(path).name != "pyproject.toml":
757+
msg = "group paths use 'pyproject.toml' filenames"
758+
raise_option_error(parser, option=option, msg=msg)
759+
760+
parser.values.dependency_groups.append((path, groupname))
761+
762+
763+
dependency_groups: Callable[..., Option] = partial(
764+
Option,
765+
"--group",
766+
dest="dependency_groups",
767+
default=[],
768+
type=str,
769+
action="callback",
770+
callback=_handle_dependency_group,
771+
metavar="[path:]group",
772+
help='Install a named dependency-group from a "pyproject.toml" file. '
773+
'If a path is given, the name of the file must be "pyproject.toml". '
774+
'Defaults to using "pyproject.toml" in the current directory.',
775+
)
776+
736777
ignore_requires_python: Callable[..., Option] = partial(
737778
Option,
738779
"--ignore-requires-python",

src/pip/_internal/cli/req_command.py

+18-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
install_req_from_parsed_requirement,
2929
install_req_from_req_string,
3030
)
31+
from pip._internal.req.req_dependency_group import parse_dependency_groups
3132
from pip._internal.req.req_file import parse_requirements
3233
from pip._internal.req.req_install import InstallRequirement
3334
from pip._internal.resolution.base import BaseResolver
@@ -79,6 +80,7 @@ class RequirementCommand(IndexGroupCommand):
7980
def __init__(self, *args: Any, **kw: Any) -> None:
8081
super().__init__(*args, **kw)
8182

83+
self.cmd_opts.add_option(cmdoptions.dependency_groups())
8284
self.cmd_opts.add_option(cmdoptions.no_clean())
8385

8486
@staticmethod
@@ -240,6 +242,16 @@ def get_requirements(
240242
)
241243
requirements.append(req_to_add)
242244

245+
if options.dependency_groups:
246+
for req in parse_dependency_groups(options.dependency_groups):
247+
req_to_add = install_req_from_req_string(
248+
req,
249+
isolated=options.isolated_mode,
250+
use_pep517=options.use_pep517,
251+
user_supplied=True,
252+
)
253+
requirements.append(req_to_add)
254+
243255
for req in options.editables:
244256
req_to_add = install_req_from_editable(
245257
req,
@@ -272,7 +284,12 @@ def get_requirements(
272284
if any(req.has_hash_options for req in requirements):
273285
options.require_hashes = True
274286

275-
if not (args or options.editables or options.requirements):
287+
if not (
288+
args
289+
or options.editables
290+
or options.requirements
291+
or options.dependency_groups
292+
):
276293
opts = {"name": self.name}
277294
if options.find_links:
278295
raise CommandError(
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
from typing import Any, Dict, Iterable, Iterator, List, Tuple
2+
3+
from pip._vendor import tomli
4+
from pip._vendor.dependency_groups import DependencyGroupResolver
5+
6+
from pip._internal.exceptions import InstallationError
7+
8+
9+
def parse_dependency_groups(groups: List[Tuple[str, str]]) -> List[str]:
10+
"""
11+
Parse dependency groups data as provided via the CLI, in a `[path:]group` syntax.
12+
13+
Raises InstallationErrors if anything goes wrong.
14+
"""
15+
resolvers = _build_resolvers(path for (path, _) in groups)
16+
return list(_resolve_all_groups(resolvers, groups))
17+
18+
19+
def _resolve_all_groups(
20+
resolvers: Dict[str, DependencyGroupResolver], groups: List[Tuple[str, str]]
21+
) -> Iterator[str]:
22+
"""
23+
Run all resolution, converting any error from `DependencyGroupResolver` into
24+
an InstallationError.
25+
"""
26+
for path, groupname in groups:
27+
resolver = resolvers[path]
28+
try:
29+
yield from (str(req) for req in resolver.resolve(groupname))
30+
except (ValueError, TypeError, LookupError) as e:
31+
raise InstallationError(
32+
f"[dependency-groups] resolution failed for '{groupname}' "
33+
f"from '{path}': {e}"
34+
) from e
35+
36+
37+
def _build_resolvers(paths: Iterable[str]) -> Dict[str, Any]:
38+
resolvers = {}
39+
for path in paths:
40+
if path in resolvers:
41+
continue
42+
43+
pyproject = _load_pyproject(path)
44+
if "dependency-groups" not in pyproject:
45+
raise InstallationError(
46+
f"[dependency-groups] table was missing from '{path}'. "
47+
"Cannot resolve '--group' option."
48+
)
49+
raw_dependency_groups = pyproject["dependency-groups"]
50+
if not isinstance(raw_dependency_groups, dict):
51+
raise InstallationError(
52+
f"[dependency-groups] table was malformed in {path}. "
53+
"Cannot resolve '--group' option."
54+
)
55+
56+
resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
57+
return resolvers
58+
59+
60+
def _load_pyproject(path: str) -> Dict[str, Any]:
61+
"""
62+
This helper loads a pyproject.toml as TOML.
63+
64+
It raises an InstallationError if the operation fails.
65+
"""
66+
try:
67+
with open(path, "rb") as fp:
68+
return tomli.load(fp)
69+
except FileNotFoundError:
70+
raise InstallationError(f"{path} not found. Cannot resolve '--group' option.")
71+
except tomli.TOMLDecodeError as e:
72+
raise InstallationError(f"Error parsing {path}: {e}") from e
73+
except OSError as e:
74+
raise InstallationError(f"Error reading {path}: {e}") from e

src/pip/_vendor/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ def vendored(modulename):
6060
# Actually alias all of our vendored dependencies.
6161
vendored("cachecontrol")
6262
vendored("certifi")
63+
vendored("dependency-groups")
6364
vendored("distlib")
6465
vendored("distro")
6566
vendored("packaging")
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
MIT License
2+
3+
Copyright (c) 2024-present Stephen Rosen <[email protected]>
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
6+
7+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
8+
9+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from ._implementation import (
2+
CyclicDependencyError,
3+
DependencyGroupInclude,
4+
DependencyGroupResolver,
5+
resolve,
6+
)
7+
8+
__all__ = (
9+
"CyclicDependencyError",
10+
"DependencyGroupInclude",
11+
"DependencyGroupResolver",
12+
"resolve",
13+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import argparse
2+
import sys
3+
4+
from ._implementation import resolve
5+
from ._toml_compat import tomllib
6+
7+
8+
def main() -> None:
9+
if tomllib is None:
10+
print(
11+
"Usage error: dependency-groups CLI requires tomli or Python 3.11+",
12+
file=sys.stderr,
13+
)
14+
raise SystemExit(2)
15+
16+
parser = argparse.ArgumentParser(
17+
description=(
18+
"A dependency-groups CLI. Prints out a resolved group, newline-delimited."
19+
)
20+
)
21+
parser.add_argument(
22+
"GROUP_NAME", nargs="*", help="The dependency group(s) to resolve."
23+
)
24+
parser.add_argument(
25+
"-f",
26+
"--pyproject-file",
27+
default="pyproject.toml",
28+
help="The pyproject.toml file. Defaults to trying in the current directory.",
29+
)
30+
parser.add_argument(
31+
"-o",
32+
"--output",
33+
help="An output file. Defaults to stdout.",
34+
)
35+
parser.add_argument(
36+
"-l",
37+
"--list",
38+
action="store_true",
39+
help="List the available dependency groups",
40+
)
41+
args = parser.parse_args()
42+
43+
with open(args.pyproject_file, "rb") as fp:
44+
pyproject = tomllib.load(fp)
45+
46+
dependency_groups_raw = pyproject.get("dependency-groups", {})
47+
48+
if args.list:
49+
print(*dependency_groups_raw.keys())
50+
return
51+
if not args.GROUP_NAME:
52+
print("A GROUP_NAME is required", file=sys.stderr)
53+
raise SystemExit(3)
54+
55+
content = "\n".join(resolve(dependency_groups_raw, *args.GROUP_NAME))
56+
57+
if args.output is None or args.output == "-":
58+
print(content)
59+
else:
60+
with open(args.output, "w", encoding="utf-8") as fp:
61+
print(content, file=fp)
62+
63+
64+
if __name__ == "__main__":
65+
main()

0 commit comments

Comments
 (0)