diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 158732e7..8e81dfc7 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -120,7 +120,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/CHANGES/1253.feature b/CHANGES/1253.feature new file mode 100644 index 00000000..56c7c5eb --- /dev/null +++ b/CHANGES/1253.feature @@ -0,0 +1 @@ +Added support for Domains. diff --git a/pulp_deb/app/__init__.py b/pulp_deb/app/__init__.py index d7fd65d7..19e01ede 100644 --- a/pulp_deb/app/__init__.py +++ b/pulp_deb/app/__init__.py @@ -8,3 +8,4 @@ class PulpDebPluginAppConfig(PulpPluginAppConfig): label = "deb" version = "3.6.0.dev" python_package_name = "pulp_deb" + domain_compatible = True diff --git a/pulp_deb/app/migrations/0031_add_domains.py b/pulp_deb/app/migrations/0031_add_domains.py new file mode 100644 index 00000000..00028bcc --- /dev/null +++ b/pulp_deb/app/migrations/0031_add_domains.py @@ -0,0 +1,172 @@ +# Generated by Django 4.2.21 on 2025-06-26 07:53 + +from django.db import migrations, models +import django.db.models.deletion +import pulpcore.app.util + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0131_distribution_checkpoint_publication_checkpoint'), + ('deb', '0030_rbac_permissions'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='genericcontent', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='installerfileindex', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='installerpackage', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='package', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='packageindex', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='packagereleasecomponent', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='release', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='releasearchitecture', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='releasecomponent', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='releasefile', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='sourceindex', + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name='sourcepackagereleasecomponent', + unique_together=set(), + ), + migrations.AddField( + model_name='genericcontent', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='installerfileindex', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='installerpackage', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='package', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='packageindex', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='packagereleasecomponent', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='release', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='releasearchitecture', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='releasecomponent', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='releasefile', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='sourceindex', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AddField( + model_name='sourcepackagereleasecomponent', + name='_pulp_domain', + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to='core.domain'), + ), + migrations.AlterUniqueTogether( + name='genericcontent', + unique_together={('relative_path', 'sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='installerfileindex', + unique_together={('relative_path', 'sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='installerpackage', + unique_together={('relative_path', 'sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='package', + unique_together={('relative_path', 'sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='packageindex', + unique_together={('relative_path', 'sha256', 'artifact_set_sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='packagereleasecomponent', + unique_together={('package', 'release_component', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='release', + unique_together={('codename', 'suite', 'distribution', 'version', 'origin', 'label', 'description', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='releasearchitecture', + unique_together={('distribution', 'architecture', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='releasecomponent', + unique_together={('distribution', 'component', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='releasefile', + unique_together={('codename', 'suite', 'distribution', 'components', 'architectures', 'relative_path', 'sha256', 'artifact_set_sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='sourceindex', + unique_together={('relative_path', 'sha256', '_pulp_domain')}, + ), + migrations.AlterUniqueTogether( + name='sourcepackagereleasecomponent', + unique_together={('source_package', 'release_component', '_pulp_domain')}, + ), + ] diff --git a/pulp_deb/app/models/content/content.py b/pulp_deb/app/models/content/content.py index 4957fd4c..28408880 100644 --- a/pulp_deb/app/models/content/content.py +++ b/pulp_deb/app/models/content/content.py @@ -14,6 +14,7 @@ from django.db.models import JSONField from pulpcore.plugin.models import Content +from pulpcore.plugin.util import get_domain_pk BOOL_CHOICES = [(True, "yes"), (False, "no")] @@ -68,6 +69,7 @@ class BasePackage(Content): sha256 = models.TextField(null=False) custom_fields = JSONField(null=True) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) @property def name(self): @@ -92,7 +94,7 @@ def filename(self, component=""): class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (("relative_path", "sha256"),) + unique_together = (("relative_path", "sha256", "_pulp_domain"),) abstract = True @@ -141,10 +143,11 @@ class GenericContent(Content): relative_path = models.TextField(null=False) sha256 = models.CharField(max_length=255, null=False) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (("relative_path", "sha256"),) + unique_together = (("relative_path", "sha256", "_pulp_domain"),) class SourcePackage(Content): diff --git a/pulp_deb/app/models/content/metadata.py b/pulp_deb/app/models/content/metadata.py index 4809a4b1..e06cdebf 100644 --- a/pulp_deb/app/models/content/metadata.py +++ b/pulp_deb/app/models/content/metadata.py @@ -8,6 +8,7 @@ from django.db import models from pulpcore.plugin.models import Content +from pulpcore.plugin.util import get_domain_pk from pulp_deb.app.constants import NULL_VALUE @@ -28,11 +29,21 @@ class Release(Content): origin = models.TextField(default=NULL_VALUE) label = models.TextField(default=NULL_VALUE) description = models.TextField(default=NULL_VALUE) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) repo_key_fields = ("distribution",) class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ( - ("codename", "suite", "distribution", "version", "origin", "label", "description"), + ( + "codename", + "suite", + "distribution", + "version", + "origin", + "label", + "description", + "_pulp_domain", + ), ) diff --git a/pulp_deb/app/models/content/structure_content.py b/pulp_deb/app/models/content/structure_content.py index 4bd16765..1640106a 100644 --- a/pulp_deb/app/models/content/structure_content.py +++ b/pulp_deb/app/models/content/structure_content.py @@ -16,6 +16,7 @@ from django.db import models from pulpcore.plugin.models import Content +from pulpcore.plugin.util import get_domain_pk from pulp_deb.app.models import Package, SourcePackage @@ -39,10 +40,11 @@ class ReleaseArchitecture(Content): distribution = models.TextField() architecture = models.TextField() + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (("distribution", "architecture"),) + unique_together = (("distribution", "architecture", "_pulp_domain"),) class ReleaseComponent(Content): @@ -57,6 +59,7 @@ class ReleaseComponent(Content): distribution = models.TextField() component = models.TextField() + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) @property def plain_component(self): @@ -76,7 +79,7 @@ def plain_component(self): class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (("distribution", "component"),) + unique_together = (("distribution", "component", "_pulp_domain"),) class PackageReleaseComponent(Content): @@ -91,10 +94,11 @@ class PackageReleaseComponent(Content): package = models.ForeignKey(Package, on_delete=models.CASCADE) release_component = models.ForeignKey(ReleaseComponent, on_delete=models.CASCADE) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (("package", "release_component"),) + unique_together = (("package", "release_component", "_pulp_domain"),) class SourcePackageReleaseComponent(Content): @@ -109,7 +113,8 @@ class SourcePackageReleaseComponent(Content): source_package = models.ForeignKey(SourcePackage, on_delete=models.CASCADE) release_component = models.ForeignKey(ReleaseComponent, on_delete=models.CASCADE) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (("source_package", "release_component"),) + unique_together = (("source_package", "release_component", "_pulp_domain"),) diff --git a/pulp_deb/app/models/content/verbatim_metadata.py b/pulp_deb/app/models/content/verbatim_metadata.py index e20b19c6..905eb296 100644 --- a/pulp_deb/app/models/content/verbatim_metadata.py +++ b/pulp_deb/app/models/content/verbatim_metadata.py @@ -8,6 +8,7 @@ from django.db import models from pulpcore.plugin.models import Content +from pulpcore.plugin.util import get_domain_pk BOOL_CHOICES = [(True, "yes"), (False, "no")] @@ -31,6 +32,7 @@ class ReleaseFile(Content): relative_path = models.TextField() sha256 = models.CharField(max_length=255) artifact_set_sha256 = models.CharField(max_length=255) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -44,6 +46,7 @@ class Meta: "relative_path", "sha256", "artifact_set_sha256", + "_pulp_domain", ), ) @@ -72,11 +75,12 @@ class PackageIndex(Content): relative_path = models.TextField() sha256 = models.CharField(max_length=255) artifact_set_sha256 = models.CharField(max_length=255) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" verbose_name_plural = "PackageIndices" - unique_together = (("relative_path", "sha256", "artifact_set_sha256"),) + unique_together = (("relative_path", "sha256", "artifact_set_sha256", "_pulp_domain"),) @property def main_artifact(self): @@ -104,11 +108,12 @@ class InstallerFileIndex(Content): architecture = models.TextField() relative_path = models.TextField() sha256 = models.CharField(max_length=255) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" verbose_name_plural = "InstallerFileIndices" - unique_together = (("relative_path", "sha256"),) + unique_together = (("relative_path", "sha256", "_pulp_domain"),) @property def main_artifact(self): @@ -134,11 +139,12 @@ class SourceIndex(Content): component = models.CharField(max_length=255) relative_path = models.TextField() sha256 = models.CharField(max_length=255) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) class Meta: default_related_name = "%(app_label)s_%(model_name)s" verbose_name_plural = "SourceIndices" - unique_together = (("relative_path", "sha256"),) + unique_together = (("relative_path", "sha256", "_pulp_domain"),) @property def main_artifact(self): diff --git a/pulp_deb/app/models/repository.py b/pulp_deb/app/models/repository.py index 9fdb1b7f..d73d9b69 100644 --- a/pulp_deb/app/models/repository.py +++ b/pulp_deb/app/models/repository.py @@ -11,7 +11,7 @@ validate_version_paths, validate_duplicate_content, ) -from pulpcore.plugin.util import batch_qs +from pulpcore.plugin.util import batch_qs, get_domain_pk from pulp_deb.app.models import ( AptReleaseSigningService, @@ -99,9 +99,9 @@ def initialize_new_version(self, new_version): metadata (which may no longer be appropriate for the new RepositoryVersion is never retained. """ - new_version.remove_content(ReleaseFile.objects.all()) - new_version.remove_content(PackageIndex.objects.all()) - new_version.remove_content(InstallerFileIndex.objects.all()) + new_version.remove_content(ReleaseFile.objects.filter(pulp_domain=get_domain_pk())) + new_version.remove_content(PackageIndex.objects.filter(pulp_domain=get_domain_pk())) + new_version.remove_content(InstallerFileIndex.objects.filter(pulp_domain=get_domain_pk())) def finalize_new_version(self, new_version): """ diff --git a/pulp_deb/app/serializers/content_serializers.py b/pulp_deb/app/serializers/content_serializers.py index 90109626..5fdaf969 100644 --- a/pulp_deb/app/serializers/content_serializers.py +++ b/pulp_deb/app/serializers/content_serializers.py @@ -23,6 +23,7 @@ DetailRelatedField, SingleContentArtifactField, ) +from pulpcore.plugin.util import get_domain_pk from pulp_deb.app.constants import ( PACKAGE_UPLOAD_DEFAULT_COMPONENT, PACKAGE_UPLOAD_DEFAULT_DISTRIBUTION, @@ -131,7 +132,9 @@ def deferred_validate(self, data): def retrieve(self, validated_data): content = GenericContent.objects.filter( - sha256=validated_data["sha256"], relative_path=validated_data["relative_path"] + sha256=validated_data["sha256"], + relative_path=validated_data["relative_path"], + pulp_domain=get_domain_pk(), ) return content.first() @@ -636,7 +639,9 @@ def deferred_validate(self, data): def retrieve(self, validated_data): content = self.Meta.model.objects.filter( - sha256=validated_data["sha256"], relative_path=validated_data["relative_path"] + sha256=validated_data["sha256"], + relative_path=validated_data["relative_path"], + pulp_domain=get_domain_pk(), ) return content.first() @@ -766,6 +771,7 @@ def retrieve(self, validated_data): origin=validated_data.get("origin", NULL_VALUE), label=validated_data.get("label", NULL_VALUE), description=validated_data.get("description", NULL_VALUE), + pulp_domain=get_domain_pk(), ).first() def create(self, validated_data): @@ -833,6 +839,7 @@ def retrieve(self, validated_data): return ReleaseArchitecture.objects.filter( architecture=validated_data["architecture"], distribution=validated_data["distribution"], + pulp_domain=get_domain_pk(), ).first() class Meta(NoArtifactContentSerializer.Meta): @@ -861,6 +868,7 @@ def retrieve(self, validated_data): return ReleaseComponent.objects.filter( distribution=validated_data["distribution"], component=validated_data["component"], + pulp_domain=get_domain_pk(), ).first() component = CharField(help_text="Name of the component.") diff --git a/pulp_deb/app/serializers/repository_serializers.py b/pulp_deb/app/serializers/repository_serializers.py index 4264d283..99b100b5 100644 --- a/pulp_deb/app/serializers/repository_serializers.py +++ b/pulp_deb/app/serializers/repository_serializers.py @@ -1,13 +1,14 @@ from gettext import gettext as _ +from django.conf import settings from django.db import transaction from pulpcore.plugin.models import SigningService from pulpcore.plugin.serializers import ( RelatedField, RepositorySerializer, RepositorySyncURLSerializer, - validate_unknown_fields, + ValidateFieldsMixin, ) -from pulpcore.plugin.util import get_url +from pulpcore.plugin.util import get_url, get_domain from pulp_deb.app.models import ( AptRepositoryReleaseServiceOverride, @@ -146,7 +147,7 @@ class AptRepositorySyncURLSerializer(RepositorySyncURLSerializer): ) -class CopySerializer(serializers.Serializer): +class CopySerializer(ValidateFieldsMixin, serializers.Serializer): """ A serializer for Content Copy API. """ @@ -175,16 +176,38 @@ class CopySerializer(serializers.Serializer): def validate(self, data): """ Validate that the Serializer contains valid data. - Set the DebRepository based on the RepositoryVersion if only the latter is provided. - Set the RepositoryVersion based on the DebRepository if only the latter is provided. - Convert the human-friendly names of the content types into what Pulp needs to query on. + + Make sure the config-JSON matches the config-schema. + Check for cross-domain references (if domain-enabled). """ - super().validate(data) - if hasattr(self, "initial_data"): - validate_unknown_fields(self.initial_data, self.fields) + def check_domain(domain, href, name): + # We're doing just a string-check here rather than checking objects + # because there can be A LOT of objects, and this is happening in the view-layer + # where we have strictly-limited timescales to work with + if href and domain not in href: + raise serializers.ValidationError( + _("{} must be part of the {} domain.").format(name, domain) + ) + + def check_cross_domain_config(cfg): + """Check that all config-elts are in 'our' domain.""" + # copy-cfg is a list of dictionaries. + # source_repo_version and dest_repo are required fiels. + # Insure curr-domain exists in src/dest/dest_base_version/content-list hrefs + curr_domain_name = get_domain().name + for entry in cfg: + check_domain(curr_domain_name, entry["source_repo_version"], "dest_repo") + check_domain(curr_domain_name, entry["dest_repo"], "dest_repo") + check_domain( + curr_domain_name, entry.get("dest_base_version", None), "dest_base_version" + ) + for content_href in entry.get("content", []): + check_domain(curr_domain_name, content_href, "content") + super().validate(data) if "config" in data: + # Make sure config is valid JSON validator = Draft7Validator(COPY_CONFIG_SCHEMA) err = [] @@ -195,4 +218,7 @@ def validate(self, data): _("Provided copy criteria is invalid:'{}'".format(err)) ) + if settings.DOMAIN_ENABLED: + check_cross_domain_config(data["config"]) + return data diff --git a/pulp_deb/app/tasks/copy.py b/pulp_deb/app/tasks/copy.py index 62f5119b..1e883869 100644 --- a/pulp_deb/app/tasks/copy.py +++ b/pulp_deb/app/tasks/copy.py @@ -2,6 +2,7 @@ from django.db.models import Q from pulpcore.plugin.models import RepositoryVersion +from pulpcore.plugin.util import get_domain_pk from pulp_deb.app.models import ( AptRepository, @@ -92,6 +93,8 @@ def process_entry(entry): else: content_filter = Q() + content_filter &= Q(pulp_domain=get_domain_pk()) + log.info(_("Copying: {copy} created").format(copy=content_filter)) return ( diff --git a/pulp_deb/app/urls.py b/pulp_deb/app/urls.py index 679b06ec..e9d1fd4a 100644 --- a/pulp_deb/app/urls.py +++ b/pulp_deb/app/urls.py @@ -6,7 +6,10 @@ from .viewsets import CopyViewSet -V3_API_ROOT = settings.V3_API_ROOT_NO_FRONT_SLASH +if settings.DOMAIN_ENABLED: + V3_API_ROOT = settings.V3_DOMAIN_API_ROOT_NO_FRONT_SLASH +else: + V3_API_ROOT = settings.V3_API_ROOT_NO_FRONT_SLASH urlpatterns = [ path(f"{V3_API_ROOT}deb/copy/", CopyViewSet.as_view({"post": "create"})), diff --git a/pulp_deb/tests/conftest.py b/pulp_deb/tests/conftest.py index 9b3460ea..2a57a4d3 100644 --- a/pulp_deb/tests/conftest.py +++ b/pulp_deb/tests/conftest.py @@ -97,12 +97,14 @@ def _deb_publication_factory(repo, **kwargs): def deb_repository_factory(apt_repository_api, gen_object_with_cleanup): """Fixture that generates a deb repository with cleanup.""" - def _deb_repository_factory(**kwargs): + def _deb_repository_factory(pulp_domain=None, **kwargs): """Create a deb repository. :returns: The created repository. """ - return gen_object_with_cleanup(apt_repository_api, gen_repo(**kwargs)) + return gen_object_with_cleanup( + apt_repository_api, gen_repo(pulp_domain=pulp_domain, **kwargs) + ) return _deb_repository_factory @@ -183,6 +185,7 @@ def _deb_init_and_sync( repository=None, remote=None, url=None, + pulp_domain=None, remote_args={}, repo_args={}, sync_args={}, @@ -206,9 +209,9 @@ def _deb_init_and_sync( else: url = deb_get_fixture_server_url(url) if repository is None: - repository = deb_repository_factory(**repo_args) + repository = deb_repository_factory(pulp_domain=pulp_domain, **repo_args) if remote is None: - remote = deb_remote_factory(url=url, **remote_args) + remote = deb_remote_factory(url=url, pulp_domain=pulp_domain, **remote_args) task = deb_sync_repository(remote, repository, **sync_args) diff --git a/pulp_deb/tests/functional/api/test_domains.py b/pulp_deb/tests/functional/api/test_domains.py new file mode 100644 index 00000000..e2b674cb --- /dev/null +++ b/pulp_deb/tests/functional/api/test_domains.py @@ -0,0 +1,357 @@ +import pytest +import json +import uuid + +from django.conf import settings + +from pulpcore.client.pulp_deb.exceptions import ApiException + +from pulp_deb.tests.functional.constants import DEB_PACKAGE_RELPATH, DEB_PUBLISH_STANDARD +from pulp_deb.tests.functional.utils import ( + gen_deb_remote, + gen_distribution, + gen_repo, + get_local_package_absolute_path, +) + +if not settings.DOMAIN_ENABLED: + pytest.skip("Domains not enabled.", allow_module_level=True) + + +def test_domain_create( + deb_domain_factory, + deb_init_and_sync, + apt_package_api, + apt_repository_api, +): + """Test repo-creation in a domain.""" + domain_name = deb_domain_factory().name + + # create and sync in default domain (not specified) + deb_init_and_sync() + + # check that newly created domain doesn't have a repo or any packages + assert apt_repository_api.list(pulp_domain=domain_name).count == 0 + assert apt_package_api.list(pulp_domain=domain_name).count == 0 + + +def test_domain_sync( + gen_object_with_cleanup, + deb_domain_factory, + apt_remote_api, + apt_repository_api, + apt_package_api, + deb_get_fixture_server_url, + deb_sync_repository, + deb_cleanup_domains, +): + """Test repo-sync in a domain.""" + domain = deb_domain_factory() + try: + domain_name = domain.name + + # create and sync in the newly-created domain + url = deb_get_fixture_server_url() + remote = gen_object_with_cleanup( + apt_remote_api, gen_deb_remote(url=str(url)), pulp_domain=domain_name + ) + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain_name) + + # check that we can "find" the new repo in the new domain via filtering + repos = apt_repository_api.list(name=repo.name, pulp_domain=domain_name).results + assert len(repos) == 1 + assert repos[0].pulp_href == repo.pulp_href + deb_sync_repository(remote=remote, repo=repo) + repo = apt_repository_api.read(repo.pulp_href) + + # check that newly created domain has one repo (list works) and the expected contents + assert apt_repository_api.list(pulp_domain=domain_name).count == 1 + assert ( + apt_package_api.list( + repository_version=repo.latest_version_href, pulp_domain=domain_name + ).count + == 4 + ) + finally: + deb_cleanup_domains([domain], content_api_client=apt_package_api, cleanup_repositories=True) + + +@pytest.mark.parallel +def test_object_creation( + gen_object_with_cleanup, + deb_domain_factory, + apt_repository_api, + deb_remote_factory, + deb_sync_repository, + deb_get_fixture_server_url, +): + """Test basic object creation in a separate domain.""" + domain = deb_domain_factory() + domain_name = domain.name + + url = deb_get_fixture_server_url() + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain_name) + assert f"{domain_name}/api/v3/" in repo.pulp_href + + repos = apt_repository_api.list(pulp_domain=domain_name) + assert repos.count == 1 + assert repo.pulp_href == repos.results[0].pulp_href + + # list repos on default domain + default_repos = apt_repository_api.list(name=repo.name) + assert default_repos.count == 0 + + # try to create an object with cross domain relations + url = deb_get_fixture_server_url() + default_remote = deb_remote_factory(url) + with pytest.raises(ApiException) as e: + repo_body = {"name": str(uuid.uuid4()), "remote": default_remote.pulp_href} + apt_repository_api.create(repo_body, pulp_domain=domain_name) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {domain_name} domain."] + } + + with pytest.raises(ApiException) as e: + deb_sync_repository(remote=default_remote, repo=repo) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {domain_name} domain."] + } + + +@pytest.mark.parallel +def test_deb_from_file( + deb_cleanup_domains, + deb_domain_factory, + apt_package_api, + deb_package_factory, +): + """Test uploading of deb content with domains""" + domain = deb_domain_factory() + + try: + package_upload_params = { + "file": str(get_local_package_absolute_path(DEB_PACKAGE_RELPATH)), + "relative_path": DEB_PACKAGE_RELPATH, + } + default_content = deb_package_factory(**package_upload_params) + package_upload_params["pulp_domain"] = domain.name + domain_content = deb_package_factory(**package_upload_params) + assert default_content.pulp_href != domain_content.pulp_href + assert default_content.sha256 == domain_content.sha256 + + domain_contents = apt_package_api.list(pulp_domain=domain.name) + assert domain_contents.count == 1 + finally: + deb_cleanup_domains([domain], content_api_client=apt_package_api) + + +@pytest.mark.parallel +def test_content_promotion( + gen_object_with_cleanup, + monitor_task, + download_content_unit, + apt_remote_api, + apt_repository_api, + apt_publication_api, + apt_distribution_api, + deb_domain_factory, + deb_cleanup_domains, + deb_delete_repository, + deb_sync_repository, + deb_get_repository_by_href, + deb_get_fixture_server_url, +): + """Tests Content promotion path with domains: Sync->Publish->Distribute""" + domain = deb_domain_factory() + + try: + # Sync + url = deb_get_fixture_server_url() + remote = gen_object_with_cleanup( + apt_remote_api, gen_deb_remote(url=str(url)), pulp_domain=domain.name + ) + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain.name) + response = deb_sync_repository(remote=remote, repo=repo) + assert len(response.created_resources) == 1 + + repo = deb_get_repository_by_href(repo.pulp_href) + assert repo.latest_version_href[-2] == "1" + + # Publish + pub_body = {"repository": repo.pulp_href} + task = apt_publication_api.create(pub_body, pulp_domain=domain.name).task + response = monitor_task(task) + assert len(response.created_resources) == 1 + pub_href = response.created_resources[0] + publication = apt_publication_api.read(pub_href) + assert publication.repository == repo.pulp_href + + # Distribute + distro_body = gen_distribution() + distro_body["publication"] = publication.pulp_href + distribution = gen_object_with_cleanup( + apt_distribution_api, distro_body, pulp_domain=domain.name + ) + assert distribution.publication == publication.pulp_href + # url structure should be host/CONTENT_ORIGIN/DOMAIN_PATH/BASE_PATH + assert domain.name == distribution.base_url.rstrip("/").split("/")[-2] + + # check that content can be downloaded from base_url + package_index_paths = DEB_PUBLISH_STANDARD["package_index_paths"] + for package_index_path in package_index_paths: + download_content_unit( + distribution.to_dict()["base_path"], package_index_path, domain=domain.name + ) + + # cleanup to delete the domain + deb_delete_repository(repo) + finally: + deb_cleanup_domains([domain], cleanup_repositories=True) + + +@pytest.mark.parallel +def test_domain_rbac( + gen_object_with_cleanup, deb_domain_factory, deb_cleanup_domains, apt_repository_api, gen_user +): + """Test domain level roles.""" + domain = deb_domain_factory() + + try: + deb_viewer = "deb.aptrepository_viewer" + deb_creator = "deb.aptrepository_creator" + user_a = gen_user(username="a", domain_roles=[(deb_viewer, domain.pulp_href)]) + user_b = gen_user(username="b", domain_roles=[(deb_creator, domain.pulp_href)]) + + # create two repos in different domains with admin user + gen_object_with_cleanup(apt_repository_api, gen_repo()) + gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain.name) + + with user_b: + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain.name) + repos = apt_repository_api.list(pulp_domain=domain.name) + assert repos.count == 1 + assert repos.results[0].pulp_href == repo.pulp_href + + # try to create a repository in default domain + with pytest.raises(ApiException) as e: + apt_repository_api.create({"name": str(uuid.uuid4())}) + assert e.value.status == 403 + + with user_a: + repos = apt_repository_api.list(pulp_domain=domain.name) + assert repos.count == 2 + + # try to read repos in the default domain + repos = apt_repository_api.list() + assert repos.count == 0 + + # try to create a repo + with pytest.raises(ApiException) as e: + apt_repository_api.create({"name": str(uuid.uuid4())}, pulp_domain=domain.name) + assert e.value.status == 403 + finally: + deb_cleanup_domains([domain], cleanup_repositories=True) + + +@pytest.mark.parallel +def test_cross_domain_copy_all( + deb_cleanup_domains, + deb_copy_content_domain, + deb_setup_domain, +): + """Test attempting to copy between different domains.""" + domain1 = None + domain2 = None + try: + domain1, _, src1, dest1 = deb_setup_domain() + domain2, _, _, dest2 = deb_setup_domain() + + # Success, everything in domain1 + deb_copy_content_domain( + source_repo_version=src1.latest_version_href, + dest_repo=dest1.pulp_href, + domain_name=domain1.name, + ) + + # Failure, call and src domain1, dest domain2 + with pytest.raises(ApiException): + deb_copy_content_domain( + source_repo_version=src1.latest_version_href, + dest_repo=dest2.pulp_href, + domain_name=domain1.name, + ) + + # Failure, call domain2, src/dest domain1 + with pytest.raises(ApiException): + deb_copy_content_domain( + source_repo_version=src1.latest_version_href, + dest_repo=dest1.pulp_href, + domain_name=domain2.name, + ) + + finally: + deb_cleanup_domains([domain1, domain2], cleanup_repositories=True) + + +@pytest.mark.parallel +def test_cross_domain_content( + apt_package_api, + deb_setup_domain, + deb_cleanup_domains, + deb_copy_content_domain, + deb_get_repository_by_href, +): + """Test the content parameter.""" + domain1 = None + domain2 = None + try: + domain1, _, src1, dest1 = deb_setup_domain() + domain2, _, src2, dest2 = deb_setup_domain() + + # Copy content1 from src1 to dest1, expect 2 copied packages + package1 = apt_package_api.list(package="frigg", pulp_domain=domain1.name).results[0] + deb_copy_content_domain( + source_repo_version=src1.latest_version_href, + dest_repo=dest1.pulp_href, + domain_name=domain1.name, + content=[package1.pulp_href], + ) + dest1 = deb_get_repository_by_href(dest1.pulp_href) + packages1 = apt_package_api.list( + repository_version=dest1.latest_version_href, pulp_domain=domain1.name + ).results + assert 1 == len(packages1) + + # copy content from src1 to dest1, domain2, expect failure + with pytest.raises(ApiException): + deb_copy_content_domain( + source_repo_version=src1.latest_version_href, + dest_repo=dest1.pulp_href, + domain_name=domain2.name, + content=[package1.pulp_href], + ) + + # copy content from src1 to dest2, domain1, expect failure + with pytest.raises(ApiException): + deb_copy_content_domain( + source_repo_version=src1.latest_version_href, + dest_repo=dest2.pulp_href, + domain_name=domain1.name, + content=[package1.pulp_href], + ) + + # copy mixed content from src2 to dest2, domain2, expect failure + package2 = apt_package_api.list(package="frigg", pulp_domain=domain2.name).results[0] + + with pytest.raises(ApiException): + deb_copy_content_domain( + source_repo_version=src2.latest_version_href, + dest_repo=dest2.pulp_href, + domain_name=domain2.name, + content=[package1.pulp_href, package2.pulp_href], + ) + + finally: + deb_cleanup_domains([domain1, domain2], cleanup_repositories=True) diff --git a/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py b/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py index d1130a6d..ee40667f 100644 --- a/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py +++ b/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py @@ -9,11 +9,15 @@ from uuid import uuid4 +from pulpcore.app import settings from pulp_deb.tests.functional.constants import DEB_FIXTURE_SUMMARY from pulp_deb.tests.functional.utils import get_counts_from_content_summary NUM_REPOS = 2 +if settings.DOMAIN_ENABLED: + pytest.skip("Domains do not support import.", allow_module_level=True) + @pytest.fixture def deb_gen_import_export_repos( diff --git a/pulp_deb/tests/functional/api/test_sync.py b/pulp_deb/tests/functional/api/test_sync.py index f97ae9c9..10255a41 100644 --- a/pulp_deb/tests/functional/api/test_sync.py +++ b/pulp_deb/tests/functional/api/test_sync.py @@ -3,6 +3,7 @@ import pytest from pulpcore.tests.functional.utils import PulpTaskError +from pulpcore.app import settings from pulp_deb.tests.functional.constants import ( DEB_FIXTURE_ARCH, DEB_FIXTURE_ARCH_UPDATE, @@ -348,6 +349,7 @@ def test_sync_optimize_with_mirror_enabled(deb_init_and_sync): assert is_sync_skipped(task, DEB_REPORT_CODE_SKIP_COMPLETE) +@pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domain produces different results.") def test_sync_orphan_cleanup_fail( deb_init_and_sync, pulpcore_bindings, diff --git a/pulp_deb/tests/functional/conftest.py b/pulp_deb/tests/functional/conftest.py index f4dde5f9..a0748fc4 100644 --- a/pulp_deb/tests/functional/conftest.py +++ b/pulp_deb/tests/functional/conftest.py @@ -5,6 +5,7 @@ import re import stat import subprocess +import uuid from pulp_deb.tests.functional.constants import DEB_SIGNING_SCRIPT_STRING from pulpcore.client.pulp_deb import ( @@ -29,6 +30,8 @@ PublicationsVerbatimApi, ) +from pulp_deb.tests.functional.utils import gen_deb_remote, gen_repo + @pytest.fixture(scope="session") def apt_release_file_api(apt_client): @@ -465,6 +468,35 @@ def _deb_copy_content(source_repo_version, dest_repo, content=None, structured=T return _deb_copy_content +@pytest.fixture(scope="class") +def deb_copy_content_domain(apt_copy_api, monitor_task): + """ + Fixture that copies deb content from a source repository version + to a target repository using domains. + """ + + def _deb_copy_content_domain( + source_repo_version, dest_repo, domain_name, content=None, structured=True + ): + """Copy deb content from a source repository version to a target repository in a domain. + + :param source_repo_version: The repository version href from where the content is copied. + :dest_repo: The repository href where the content should be copied to. + :domain_name: The name of the domain where copy should take place. + :content: List of packages hrefs that should be copied from the source. Default: None + :structured: Whether or not the content should be structured copied. Default: True + :returns: The task of the copy operation. + """ + config = {"source_repo_version": source_repo_version, "dest_repo": dest_repo} + if content is not None: + config["content"] = content + data = Copy(config=[config], structured=structured) + response = apt_copy_api.copy_content(data, pulp_domain=domain_name) + return monitor_task(response.task) + + return _deb_copy_content_domain + + @pytest.fixture(scope="session") def deb_signing_script_path( signing_script_temp_dir, signing_gpg_homedir_path, signing_gpg_metadata @@ -555,3 +587,85 @@ def _deb_get_content_types(content_api_name, content_type, repo, version_href=No return api.list(repository_version=latest_version_href).results return _deb_get_content_types + + +@pytest.fixture +def deb_setup_domain( + gen_object_with_cleanup, + apt_remote_api, + apt_repository_api, + deb_get_fixture_server_url, + deb_sync_repository, + deb_domain_factory, +): + def _deb_setup_domain(sync=True, pulp_domain=None, url=None): + if url is None: + url = deb_get_fixture_server_url() + elif url.startswith("http"): + url = url + else: + url = deb_get_fixture_server_url(url) + + if not pulp_domain: + pulp_domain = deb_domain_factory() + + remote = gen_object_with_cleanup( + apt_remote_api, gen_deb_remote(url=str(url)), pulp_domain=pulp_domain.name + ) + src = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=pulp_domain.name) + + if sync: + deb_sync_repository(remote=remote, repo=src) + src = apt_repository_api.read(src.pulp_href) + + dest = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=pulp_domain.name) + return pulp_domain, remote, src, dest + + return _deb_setup_domain + + +@pytest.fixture +def deb_cleanup_domains(pulpcore_bindings, monitor_task, apt_repository_api): + def _deb_cleanup_domains( + domains, + content_api_client=None, + cleanup_repositories=False, + repository_api_client=apt_repository_api, + ): + for domain in domains: + # clean up each domain specified + if domain: + if cleanup_repositories: + # delete repos from the domain + for repo in repository_api_client.list(pulp_domain=domain.name).results: + monitor_task(repository_api_client.delete(repo.pulp_href).task) + # let orphan cleanup reap the resulting abandoned content + monitor_task( + pulpcore_bindings.OrphansCleanupApi.cleanup( + {"orphan_protection_time": 0}, pulp_domain=domain.name + ).task + ) + + if content_api_client: + # if we have a client, check that each domain is empty of that kind-of entity + for domain in domains: + if domain: + assert content_api_client.list(pulp_domain=domain.name).count == 0 + + return _deb_cleanup_domains + + +@pytest.fixture +def deb_domain_factory(pulpcore_bindings, gen_object_with_cleanup): + """Fixture to create a domain.""" + + def _deb_domain_factory(name=None): + name = str(uuid.uuid4()) if name is None else name + body = { + "name": name, + "storage_class": "pulpcore.app.models.storage.FileSystem", + "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"}, + } + return gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body) + + return _deb_domain_factory diff --git a/pulp_deb/tests/functional/constants.py b/pulp_deb/tests/functional/constants.py index ce004083..b578c630 100644 --- a/pulp_deb/tests/functional/constants.py +++ b/pulp_deb/tests/functional/constants.py @@ -276,6 +276,15 @@ def _clean_dict(d): "package_index_paths_dist": ["dists/flat-repo/flat-repo-component/binary-ppc64/Packages"], } +DEB_PUBLISH_STANDARD = { + "package_index_paths": [ + "dists/ragnarok/asgard/binary-ppc64/Packages", + "dists/ragnarok/asgard/binary-armeb/Packages", + "dists/ragnarok/jotunheimr/binary-ppc64/Packages", + "dists/ragnarok/jotunheimr/binary-armeb/Packages", + ] +} + DEB_PUBLISH_FLAT_SIMPLE = { "distribution": "flat-repo", "codename": "ragnarok", diff --git a/pulp_deb/tests/functional/utils.py b/pulp_deb/tests/functional/utils.py index ff4719ef..40d883ef 100644 --- a/pulp_deb/tests/functional/utils.py +++ b/pulp_deb/tests/functional/utils.py @@ -84,13 +84,15 @@ def gen_distribution(**kwargs): return data -def gen_remote(url, **kwargs): +def gen_remote(url, pulp_domain=None, **kwargs): """Return a semi-random dict for use in creating a Remote. :param url: The URL of an external content source. """ data = {"name": str(uuid4()), "url": url} data.update(kwargs) + if pulp_domain: + data["pulp_domain"] = pulp_domain return data diff --git a/template_config.yml b/template_config.yml index 9ba2f246..624d4b04 100644 --- a/template_config.yml +++ b/template_config.yml @@ -51,7 +51,8 @@ pulp_settings: allowed_import_paths: - /tmp apt_by_hash: true -pulp_settings_azure: null +pulp_settings_azure: + domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: null pydocstyle: true