repositories: add support for PEP 691 as fallback for PyPI

Author: radoering

src/poetry/repositories/http_repository.py

@@ -237,17 +237,16 @@ def _get_info_from_urls(
             urls["sdist"][0], metadata
         ) or self._get_info_from_sdist(urls["sdist"][0])
 
-    def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]:
-        if not links:
-            raise PackageNotFound(
-                f'No valid distribution links found for package: "{data.name}" version:'
-                f' "{data.version}"'
-            )
+    def _get_info_from_links(
+        self,
+        links: list[Link],
+        *,
+        ignore_yanked: bool = True,
+    ) -> PackageInfo:
         urls = defaultdict(list)
         metadata: dict[str, pkginfo.Distribution] = {}
-        files: list[dict[str, Any]] = []
         for link in links:
-            if link.yanked and not data.yanked:
+            if link.yanked and ignore_yanked:
                 # drop yanked files unless the entire release is yanked
                 continue
             if link.has_metadata:
@@ -284,6 +283,21 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]
             ):
                 urls["sdist"].append(link.url)
 
+        return self._get_info_from_urls(urls, metadata)
+
+    def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]:
+        if not links:
+            raise PackageNotFound(
+                f'No valid distribution links found for package: "{data.name}" version:'
+                f' "{data.version}"'
+            )
+
+        files: list[dict[str, Any]] = []
+        for link in links:
+            if link.yanked and not data.yanked:
+                # drop yanked files unless the entire release is yanked
+                continue
+
             file_hash = f"{link.hash_name}:{link.hash}" if link.hash else None
 
             if not link.hash or (
@@ -297,7 +311,8 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]
 
         data.files = files
 
-        info = self._get_info_from_urls(urls, metadata)
+        # drop yanked files unless the entire release is yanked
+        info = self._get_info_from_links(links, ignore_yanked=not data.yanked)
 
         data.summary = info.summary
         data.requires_dist = info.requires_dist

src/poetry/repositories/link_sources/json.py

@@ -28,7 +28,28 @@ def _link_cache(self) -> LinkCache:
             url = file["url"]
             requires_python = file.get("requires-python")
             yanked = file.get("yanked", False)
-            link = Link(url, requires_python=requires_python, yanked=yanked)
+
+            # see https://peps.python.org/pep-0714/#clients
+            # and https://peps.python.org/pep-0691/#project-detail
+            metadata: str | bool = False
+            for metadata_key in ("core-metadata", "dist-info-metadata"):
+                if metadata_key in file:
+                    metadata_value = file[metadata_key]
+                    if metadata_value and isinstance(metadata_value, dict):
+                        # The interface of poetry.core.packages.utils.link.Link
+                        # is currently limited to strings with one hash.
+                        if sha256 := metadata_value.get("sha256"):
+                            metadata = f"sha256={sha256}"
+                        else:
+                            key, value = next(iter(metadata_value.items()))
+                            metadata = f"{key}={value}"
+                    else:
+                        metadata = bool(metadata_value)
+                    break
+
+            link = Link(
+                url, requires_python=requires_python, yanked=yanked, metadata=metadata
+            )
 
             if link.ext not in self.SUPPORTED_FORMATS:
                 continue

src/poetry/repositories/pypi_repository.py

@@ -2,7 +2,6 @@
 
 import logging
 
-from collections import defaultdict
 from typing import TYPE_CHECKING
 from typing import Any
 
@@ -161,25 +160,18 @@ def _get_release_info(
                 })
 
         if self._fallback and data.requires_dist is None:
-            self._log("No dependencies found, downloading archives", level="debug")
+            self._log(
+                "No dependencies found, downloading metadata and/or archives",
+                level="debug",
+            )
             # No dependencies set (along with other information)
             # This might be due to actually no dependencies
-            # or badly set metadata when uploading
+            # or badly set metadata when uploading.
             # So, we need to make sure there is actually no
-            # dependencies by introspecting packages
-            urls = defaultdict(list)
-            for url in json_data["urls"]:
-                # Only get sdist and wheels if they exist
-                dist_type = url["packagetype"]
-                if dist_type not in SUPPORTED_PACKAGE_TYPES:
-                    continue
-
-                urls[dist_type].append(url["url"])
-
-            if not urls:
-                return data.asdict()
-
-            info = self._get_info_from_urls(urls)
+            # dependencies by introspecting packages.
+            page = self.get_page(name)
+            links = list(page.links_for_version(name, version))
+            info = self._get_info_from_links(links)
 
             data.requires_dist = info.requires_dist
 

tests/repositories/conftest.py

@@ -1,12 +1,18 @@
 from __future__ import annotations
 
+import posixpath
+
+from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 
 import pytest
+import requests
 
 
 if TYPE_CHECKING:
     from tests.types import HTMLPageGetter
+    from tests.types import RequestsSessionGet
 
 
 @pytest.fixture
@@ -29,3 +35,21 @@ def _fixture(content: str, base_url: str | None = None) -> str:
         """
 
     return _fixture
+
+
+@pytest.fixture
+def get_metadata_mock() -> RequestsSessionGet:
+    def metadata_mock(url: str, **__: Any) -> requests.Response:
+        if url.endswith(".metadata"):
+            response = requests.Response()
+            response.encoding = "application/text"
+            response._content = (
+                Path(__file__).parent
+                / "fixtures"
+                / "metadata"
+                / posixpath.basename(url)
+            ).read_bytes()
+            return response
+        raise requests.HTTPError()
+
+    return metadata_mock

tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata renamed to tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py2-none-any.whl.metadata

@@ -0,0 +1,28 @@
+Metadata-Version: 2.0
+Name: isort-metadata
+Version: 4.3.4
+Summary: A Python utility / library to sort Python imports.
+Home-page: https://github.com/timothycrosley/isort
+Author: Timothy Crosley
+Author-email: [email protected]
+License: MIT
+Keywords: Refactor,Python,Python2,Python3,Refactoring,Imports,Sort,Clean
+Platform: UNKNOWN
+Classifier: Development Status :: 6 - Mature
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: futures; python_version=="2.7"

tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata

@@ -0,0 +1,35 @@
+{
+  "name": "isort-metadata",
+  "files": [
+    {
+      "filename": "isort-metadata-4.3.4-py2-none-any.whl",
+      "url": "https://files.pythonhosted.org/packages/41/d8/a945da414f2adc1d9e2f7d6e7445b27f2be42766879062a2e63616ad4199/isort-metadata-4.3.4-py2-none-any.whl",
+      "core-metadata": true,
+      "hashes": {
+        "md5": "f0ad7704b6dc947073398ba290c3517f",
+        "sha256": "ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497"
+      }
+    },
+    {
+      "filename": "isort-metadata-4.3.4-py3-none-any.whl",
+      "url": "https://files.pythonhosted.org/packages/1f/2c/22eee714d7199ae0464beda6ad5fedec8fee6a2f7ffd1e8f1840928fe318/isort-metadata-4.3.4-py3-none-any.whl",
+      "core-metadata": true,
+      "hashes": {
+        "md5": "fbaac4cd669ac21ea9e21ab1ea3180db",
+        "sha256": "1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af"
+      }
+    },
+    {
+      "filename": "isort-metadata-4.3.4.tar.gz",
+      "url": "https://files.pythonhosted.org/packages/b1/de/a628d16fdba0d38cafb3d7e34d4830f2c9cb3881384ce5c08c44762e1846/isort-metadata-4.3.4.tar.gz",
+      "hashes": {
+        "md5": "fb554e9c8f9aa76e333a03d470a5cf52",
+        "sha256": "b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8"
+      }
+    }
+  ],
+  "meta": {
+    "api-version": "1.0",
+    "_last-serial": 3575149
+  }
+}

tests/repositories/fixtures/pypi.org/json/isort-metadata.json

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import pytest
+
+from poetry.repositories.link_sources.json import SimpleJsonPage
+
+
+@pytest.mark.parametrize(
+    "metadata, expected_has_metadata, expected_hash_name, expected_hash",
+    [
+        ({}, False, None, None),
+        # new
+        ({"core-metadata": False}, False, None, None),
+        ({"core-metadata": True}, True, None, None),
+        ({"core-metadata": {"sha1": "1234", "sha256": "abcd"}}, True, "sha256", "abcd"),
+        ({"core-metadata": {}}, False, None, None),
+        (
+            {"core-metadata": {"sha1": "1234", "sha256": "abcd"}},
+            True,
+            "sha256",
+            "abcd",
+        ),
+        # old
+        ({"dist-info-metadata": False}, False, None, None),
+        ({"dist-info-metadata": True}, True, None, None),
+        ({"dist-info-metadata": {"sha256": "abcd"}}, True, "sha256", "abcd"),
+        ({"dist-info-metadata": {}}, False, None, None),
+        (
+            {"dist-info-metadata": {"sha1": "1234", "sha256": "abcd"}},
+            True,
+            "sha256",
+            "abcd",
+        ),
+        # conflicting (new wins)
+        ({"core-metadata": False, "dist-info-metadata": True}, False, None, None),
+        (
+            {"core-metadata": False, "dist-info-metadata": {"sha256": "abcd"}},
+            False,
+            None,
+            None,
+        ),
+        ({"core-metadata": True, "dist-info-metadata": False}, True, None, None),
+        (
+            {"core-metadata": True, "dist-info-metadata": {"sha256": "abcd"}},
+            True,
+            None,
+            None,
+        ),
+        (
+            {"core-metadata": {"sha256": "abcd"}, "dist-info-metadata": False},
+            True,
+            "sha256",
+            "abcd",
+        ),
+        (
+            {"core-metadata": {"sha256": "abcd"}, "dist-info-metadata": True},
+            True,
+            "sha256",
+            "abcd",
+        ),
+        (
+            {
+                "core-metadata": {"sha256": "abcd"},
+                "dist-info-metadata": {"sha256": "1234"},
+            },
+            True,
+            "sha256",
+            "abcd",
+        ),
+    ],
+)
+def test_metadata(
+    metadata: dict[str, bool | dict[str, str]],
+    expected_has_metadata: bool,
+    expected_hash_name: str | None,
+    expected_hash: str | None,
+) -> None:
+    content = {"files": [{"url": "https://example.org/demo-0.1.whl", **metadata}]}
+    page = SimpleJsonPage("https://example.org", content)
+
+    link = next(page.links)
+    assert link.has_metadata is expected_has_metadata
+    assert link.metadata_hash_name == expected_hash_name
+    assert link.metadata_hash == expected_hash

tests/repositories/fixtures/pypi.org/json/isort-metadata/4.3.4.json

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import base64
-import posixpath
 import re
 import shutil
 
@@ -24,15 +23,14 @@
 
 
 if TYPE_CHECKING:
-    from typing import Any
-
     import httpretty
 
     from _pytest.monkeypatch import MonkeyPatch
     from packaging.utils import NormalizedName
     from pytest_mock import MockerFixture
 
     from poetry.config.config import Config
+    from tests.types import RequestsSessionGet
 
 
 @pytest.fixture(autouse=True)
@@ -177,29 +175,24 @@ def test_get_package_information_fallback_read_setup() -> None:
     )
 
 
-def _get_mock(url: str, **__: Any) -> requests.Response:
-    if url.endswith(".metadata"):
-        response = requests.Response()
-        response.encoding = "application/text"
-        response._content = MockRepository.FIXTURES.joinpath(
-            "metadata", posixpath.basename(url)
-        ).read_bytes()
-        return response
-    raise requests.HTTPError()
-
-
-def test_get_package_information_pep_658(mocker: MockerFixture) -> None:
+def test_get_package_information_pep_658(
+    mocker: MockerFixture, get_metadata_mock: RequestsSessionGet
+) -> None:
     repo = MockRepository()
 
     isort_package = repo.package("isort", Version.parse("4.3.4"))
 
-    mocker.patch.object(repo.session, "get", _get_mock)
+    mocker.patch.object(repo.session, "get", get_metadata_mock)
+    spy = mocker.spy(repo, "_get_info_from_metadata")
 
     try:
         package = repo.package("isort-metadata", Version.parse("4.3.4"))
     except FileNotFoundError:
         pytest.fail("Metadata was not successfully retrieved")
     else:
+        assert spy.call_count > 0
+        assert spy.spy_return is not None
+
         assert package.source_type == isort_package.source_type == "legacy"
         assert package.source_reference == isort_package.source_reference == repo.name
         assert package.source_url == isort_package.source_url == repo.url