Add better flat repo handling for structure publish

Author: hstct

pulp_deb/app/tasks/synchronizing.py

@@ -624,16 +624,19 @@ def _to_d_artifact(self, relative_path, data=None):
         )
 
     async def _handle_distribution(self, distribution):
+        is_flat = distribution.endswith("/")
+        stored_distribution = "flat-repo" if is_flat else distribution
+
         log.info(_('Downloading Release file for distribution: "{}"').format(distribution))
         # Create release_file
-        if distribution[-1] == "/":
-            release_file_dir = distribution.strip("/")
+        if is_flat:
+            upstream_file_dir = distribution.strip("/")
         else:
-            release_file_dir = os.path.join("dists", distribution)
+            upstream_file_dir = os.path.join("dists", distribution)
         release_file_dc = DeclarativeContent(
-            content=ReleaseFile(distribution=distribution, relative_path=release_file_dir),
+            content=ReleaseFile(distribution=stored_distribution, relative_path=upstream_file_dir),
             d_artifacts=[
-                self._to_d_artifact(os.path.join(release_file_dir, filename))
+                self._to_d_artifact(os.path.join(upstream_file_dir, filename))
                 for filename in ReleaseFile.SUPPORTED_ARTIFACTS
             ],
         )
@@ -665,7 +668,7 @@ async def _handle_distribution(self, distribution):
         release_fields = {
             "codename": release_file.codename,
             "suite": release_file.suite,
-            "distribution": distribution,
+            "distribution": stored_distribution,
         }
 
         if "version" in release_file_dict:
@@ -680,17 +683,17 @@ async def _handle_distribution(self, distribution):
         await self.put(DeclarativeContent(content=Release(**release_fields)))
 
         # Create release architectures
-        if release_file.architectures:
-            architectures = _filter_split_architectures(
-                release_file.architectures, self.remote.architectures, distribution
-            )
-        elif distribution[-1] == "/":
+        if is_flat:
             message = (
                 "The ReleaseFile content unit architecrures are unset for the flat repo with "
                 "distribution '{}'. ReleaseArchitecture content creation is deferred!"
             )
             log.warning(_(message).format(distribution))
             architectures = []
+        elif release_file.architectures:
+            architectures = _filter_split_architectures(
+                release_file.architectures, self.remote.architectures, distribution
+            )
 
         for architecture in architectures:
             release_architecture_dc = DeclarativeContent(
@@ -719,9 +722,16 @@ async def _handle_distribution(self, distribution):
                 for unit in release_file_dict[digest_name]:
                     file_references[unit["Name"]].update(unit)
 
-        if distribution[-1] == "/":
+        if is_flat:
             # Handle flat repo
-            sub_tasks = [self._handle_flat_repo(file_references, release_file, distribution)]
+            sub_tasks = [
+                self._handle_flat_repo(
+                    file_references,
+                    release_file,
+                    distribution=stored_distribution,
+                    upstream_dist_path=upstream_file_dir,
+                )
+            ]
         else:
             # Handle components
             sub_tasks = [
@@ -833,7 +843,9 @@ async def _handle_component(
             )
         await asyncio.gather(*pending_tasks)
 
-    async def _handle_flat_repo(self, file_references, release_file, distribution):
+    async def _handle_flat_repo(
+        self, file_references, release_file, distribution, upstream_dist_path
+    ):
         # We are creating a component so the flat repo can be published as a structured repo!
         release_component_dc = DeclarativeContent(
             content=ReleaseComponent(component="flat-repo-component", distribution=distribution)
@@ -849,6 +861,7 @@ async def _handle_flat_repo(self, file_references, release_file, distribution):
                 architecture="",
                 file_references=file_references,
                 distribution=distribution,
+                is_flat=True,
             )
         )
 
@@ -868,15 +881,19 @@ async def _handle_package_index(
         infix="",
         distribution=None,
         hybrid_format=False,
+        is_flat=False,
     ):
+        if is_flat:
+            release_file_package_index_dir = ""
+        else:
+            release_file_package_index_dir = os.path.join(
+                release_component.plain_component,
+                infix,
+                f"binary-{architecture}",
+            )
         # Create package_index
         release_base_path = os.path.dirname(release_file.relative_path)
-        # Package index directory relative to the release file:
-        release_file_package_index_dir = (
-            os.path.join(release_component.plain_component, infix, "binary-{}".format(architecture))
-            if release_file.distribution[-1] != "/"
-            else ""
-        )
+
         # Package index directory relative to the repository root:
         package_index_dir = os.path.join(release_base_path, release_file_package_index_dir)
         d_artifacts = []
@@ -948,7 +965,7 @@ async def _handle_package_index(
         ):
             # Sanity check the architecture from the package paragraph:
             package_paragraph_architecture = package_paragraph["Architecture"]
-            if release_file.distribution[-1] == "/":
+            if is_flat:
                 if (
                     self.remote.architectures
                     and package_paragraph_architecture != "all"
@@ -1035,32 +1052,22 @@ async def _handle_package_index(
                 )
             )
             await self.put(package_release_component_dc)
-            if release_file.distribution[-1] == "/":
+            if is_flat:
                 package_architectures.add(package.architecture)
 
         # For flat repos we may still need to create ReleaseArchitecture content:
-        if release_file.distribution[-1] == "/":
+        if is_flat:
             if release_file.architectures:
                 for architecture in package_architectures:
-                    if architecture not in release_file.architectures.split():
-                        message = (
-                            "The flat repo with distribution '{}' contains packages with "
-                            "architecture '{}' but this is not included in the ReleaseFile's "
-                            "architectures field '{}'!"
-                        )
-                        log.warning(
-                            _(message).format(
-                                release_file.distribution, architecture, release_file.architectures
-                            )
-                        )
-                        message = "Creating additional ReleaseArchitecture for architecture '{}'!"
-                        log.warning(_(message).format(architecture))
-                        release_architecture_dc = DeclarativeContent(
-                            content=ReleaseArchitecture(
-                                architecture=architecture, distribution=distribution
-                            )
+                    log.debug(
+                        f"Flat Repo Architecture handling: Creating ReleaseArchitecture for architecture {architecture}."
+                    )
+                    release_architecture_dc = DeclarativeContent(
+                        content=ReleaseArchitecture(
+                            architecture=architecture, distribution="flat-repo"
                         )
-                        await self.put(release_architecture_dc)
+                    )
+                    await self.put(release_architecture_dc)
             else:
                 package_architectures_string = " ".join(package_architectures)
                 message = (
@@ -1076,7 +1083,7 @@ async def _handle_package_index(
                 for architecture in package_architectures:
                     release_architecture_dc = DeclarativeContent(
                         content=ReleaseArchitecture(
-                            architecture=architecture, distribution=distribution
+                            architecture=architecture, distribution="flat-repo"
                         )
                     )
                     await self.put(release_architecture_dc)

pulp_deb/tests/functional/api/test_publish.py

@@ -5,6 +5,7 @@
 
 from pulp_deb.tests.functional.constants import (
     DEB_FIXTURE_ALT_SINGLE_DIST,
+    DEB_FIXTURE_BASE,
     DEB_FIXTURE_COMPLEX_REPOSITORY_NAME,
     DEB_FIXTURE_DISTRIBUTIONS,
     DEB_FIXTURE_FLAT_REPOSITORY_NAME,
@@ -320,14 +321,16 @@ def test_publish_empty_repository(
 
 @pytest.mark.parallel
 @pytest.mark.parametrize(
-    "expected_data", [DEB_PUBLISH_FLAT_VERBATIM, DEB_PUBLISH_FLAT_NESTED_VERBATIM]
+    "expected_data, flat_dist",
+    [(DEB_PUBLISH_FLAT_VERBATIM, "/"), (DEB_PUBLISH_FLAT_NESTED_VERBATIM, "nest/fjalar/")],
 )
 def test_publish_flat_repository_verbatim(
     create_publication_and_verify_repo_version,
     deb_distribution_factory,
     verify_distribution,
     verify_publication_data,
     expected_data,
+    flat_dist,
 ):
     """Test whether a particular flat repository can be published verbatim.
 
@@ -336,7 +339,7 @@ def test_publish_flat_repository_verbatim(
     * `Publish a flat repository verbatim.`_
     * `Publish a nested flat repository verbatim.`_
     """
-    remote_args = {"distributions": expected_data["distribution"]}
+    remote_args = {"distributions": flat_dist}
 
     # Create a publication
     publication, _, _, _ = create_publication_and_verify_repo_version(
@@ -351,12 +354,12 @@ def test_publish_flat_repository_verbatim(
 
 @pytest.mark.parallel
 @pytest.mark.parametrize(
-    "expected_data, publication_args",
+    "expected_data, publication_args, flat_dist",
     [
-        (DEB_PUBLISH_FLAT_STRUCTURED, DEB_PUBLICATION_ARGS_ONLY_STRUCTURED),
-        (DEB_PUBLISH_FLAT_SIMPLE, DEB_PUBLICATION_ARGS_ONLY_SIMPLE),
-        (DEB_PUBLISH_FLAT_NESTED_STRUCTURED, DEB_PUBLICATION_ARGS_ONLY_STRUCTURED),
-        (DEB_PUBLISH_FLAT_NESTED_SIMPLE, DEB_PUBLICATION_ARGS_ONLY_SIMPLE),
+        (DEB_PUBLISH_FLAT_STRUCTURED, DEB_PUBLICATION_ARGS_ONLY_STRUCTURED, "/"),
+        (DEB_PUBLISH_FLAT_SIMPLE, DEB_PUBLICATION_ARGS_ONLY_SIMPLE, "/"),
+        (DEB_PUBLISH_FLAT_NESTED_STRUCTURED, DEB_PUBLICATION_ARGS_ONLY_STRUCTURED, "nest/fjalar/"),
+        (DEB_PUBLISH_FLAT_NESTED_SIMPLE, DEB_PUBLICATION_ARGS_ONLY_SIMPLE, "nest/fjalar/"),
     ],
 )
 def test_publish_flat_repository(
@@ -366,6 +369,7 @@ def test_publish_flat_repository(
     verify_distribution,
     verify_publication_data,
     publication_args,
+    flat_dist,
 ):
     """Test whether a particular flat repository can be published.
 
@@ -376,7 +380,7 @@ def test_publish_flat_repository(
     * `Publish a nested flat repository structured.`_
     * `Publish a nested flat repository simple.`_
     """
-    remote_args = {"distributions": expected_data["distribution"]}
+    remote_args = {"distributions": flat_dist}
 
     # Create a publication
     publication, _, _, _ = create_publication_and_verify_repo_version(
@@ -391,6 +395,51 @@ def test_publish_flat_repository(
     verify_distribution(distribution, expected_data)
 
 
+@pytest.mark.parallel
+@pytest.mark.parametrize(
+    "remote_args, remote_name",
+    [
+        ({"distributions": "/"}, DEB_FIXTURE_FLAT_REPOSITORY_NAME),
+        ({"distributions": "debian-flat/"}, DEB_FIXTURE_BASE),
+    ],
+)
+def test_published_flat_repo_content(
+    create_publication_and_verify_repo_version,
+    deb_distribution_factory,
+    download_content_unit,
+    remote_args,
+    remote_name,
+):
+    """
+    Test that published flat repository results in a normalized Release file and package indices.
+    """
+    publication, _, _, _ = create_publication_and_verify_repo_version(
+        remote_args=remote_args,
+        remote_name=remote_name,
+    )
+
+    distribution = deb_distribution_factory(publication)
+    base_url = distribution.base_path
+
+    expected_release_path = os.path.join("dists", "flat-repo", "Release")
+    expected_pkg_index_path = os.path.join(
+        "dists", "flat-repo", "flat-repo-component", "binary-ppc64", "Packages"
+    )
+
+    release_file = download_content_unit(base_url, expected_release_path)
+    release_file_str = release_file.decode("utf-8")
+    assert (
+        "flat-repo" in release_file_str
+    ), f"Expected the published Release file to contain 'flat-repo' as distribution, got: {release_file}"
+
+    pkg_index = download_content_unit(base_url, expected_pkg_index_path)
+    from debian import deb822
+
+    pkgs = list(deb822.Packages.iter_paragraphs(pkg_index, use_apt_pkg=False))
+    ppc64_pkgs = [pkg for pkg in pkgs if pkg.get("Architecture") == "ppc64"]
+    assert ppc64_pkgs, "No packages with architecture 'ppc64' found in the published package index."
+
+
 @pytest.mark.skip("Skip - ignore_missing_package_indices sync parameter does currently not work")
 @pytest.mark.parallel
 def test_publish_missing_architecture(

pulp_deb/tests/functional/constants.py

@@ -37,6 +37,7 @@ def _clean_dict(d):
 DEB_FIXTURE_UPDATE_REPOSITORY_NAME = "/debian-update/"
 DEB_FIXTURE_INVALID_REPOSITORY_NAME = "/debian-invalid/"
 DEB_FIXTURE_FLAT_REPOSITORY_NAME = "/debian-flat/"
+DEB_FIXTURE_BASE = "/"
 DEB_FIXTURE_COMPLEX_REPOSITORY_NAME = "/debian-complex-dists"
 DEB_FIXTURE_MISSING_ARCHITECTURE_REPOSITORY_NAME = "/debian-missing-architecture/"
 
@@ -249,7 +250,7 @@ def _clean_dict(d):
 }
 
 DEB_PUBLISH_FLAT_STRUCTURED = {
-    "distribution": "/",
+    "distribution": "flat-repo",
     "codename": "ragnarok",
     "suite": "mythology",
     "components": ["flat-repo-component"],
@@ -260,7 +261,7 @@ def _clean_dict(d):
 }
 
 DEB_PUBLISH_FLAT_SIMPLE = {
-    "distribution": "/",
+    "distribution": "flat-repo",
     "codename": "ragnarok",
     "suite": "mythology",
     "components": ["flat-repo-component"],
@@ -271,7 +272,7 @@ def _clean_dict(d):
 }
 
 DEB_PUBLISH_FLAT_VERBATIM = {
-    "distribution": "/",
+    "distribution": "flat-repo",
     "codename": "ragnarok",
     "suite": "mythology",
     "components": ["flat-repo-component"],
@@ -282,18 +283,18 @@ def _clean_dict(d):
 }
 
 DEB_PUBLISH_FLAT_NESTED_STRUCTURED = {
-    "distribution": "nest/fjalar/",
+    "distribution": "flat-repo",
     "codename": "ragnarok",
     "suite": "mythology",
     "components": ["flat-repo-component"],
     "release_file_folder_sync": "nest/fjalar/",
-    "release_file_folder_dist": "dists/nest/fjalar",
+    "release_file_folder_dist": "dists/flat-repo",
     "package_index_paths_sync": ["nest/fjalar/Packages"],
-    "package_index_paths_dist": ["dists/nest/fjalar/flat-repo-component/binary-ppc64/Packages"],
+    "package_index_paths_dist": ["dists/flat-repo/flat-repo-component/binary-ppc64/Packages"],
 }
 
 DEB_PUBLISH_FLAT_NESTED_SIMPLE = {
-    "distribution": "nest/fjalar/",
+    "distribution": "flat-repo",
     "codename": "ragnarok",
     "suite": "mythology",
     "components": ["flat-repo-component"],
@@ -304,7 +305,7 @@ def _clean_dict(d):
 }
 
 DEB_PUBLISH_FLAT_NESTED_VERBATIM = {
-    "distribution": "nest/fjalar/",
+    "distribution": "flat-repo",
     "codename": "ragnarok",
     "suite": "mythology",
     "components": ["flat-repo-component"],