diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 158732e7..8e81dfc7 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -120,7 +120,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/pulp_deb/app/__init__.py b/pulp_deb/app/__init__.py index d7fd65d7..19e01ede 100644 --- a/pulp_deb/app/__init__.py +++ b/pulp_deb/app/__init__.py @@ -8,3 +8,4 @@ class PulpDebPluginAppConfig(PulpPluginAppConfig): label = "deb" version = "3.6.0.dev" python_package_name = "pulp_deb" + domain_compatible = True diff --git a/pulp_deb/app/urls.py b/pulp_deb/app/urls.py index 679b06ec..e9d1fd4a 100644 --- a/pulp_deb/app/urls.py +++ b/pulp_deb/app/urls.py @@ -6,7 +6,10 @@ from .viewsets import CopyViewSet -V3_API_ROOT = settings.V3_API_ROOT_NO_FRONT_SLASH +if settings.DOMAIN_ENABLED: + V3_API_ROOT = settings.V3_DOMAIN_API_ROOT_NO_FRONT_SLASH +else: + V3_API_ROOT = settings.V3_API_ROOT_NO_FRONT_SLASH urlpatterns = [ path(f"{V3_API_ROOT}deb/copy/", CopyViewSet.as_view({"post": "create"})), diff --git a/pulp_deb/tests/conftest.py b/pulp_deb/tests/conftest.py index 9b3460ea..2a57a4d3 100644 --- a/pulp_deb/tests/conftest.py +++ b/pulp_deb/tests/conftest.py @@ -97,12 +97,14 @@ def _deb_publication_factory(repo, **kwargs): def deb_repository_factory(apt_repository_api, gen_object_with_cleanup): """Fixture that generates a deb repository with cleanup.""" - def _deb_repository_factory(**kwargs): + def _deb_repository_factory(pulp_domain=None, **kwargs): """Create a deb repository. :returns: The created repository. """ - return gen_object_with_cleanup(apt_repository_api, gen_repo(**kwargs)) + return gen_object_with_cleanup( + apt_repository_api, gen_repo(pulp_domain=pulp_domain, **kwargs) + ) return _deb_repository_factory @@ -183,6 +185,7 @@ def _deb_init_and_sync( repository=None, remote=None, url=None, + pulp_domain=None, remote_args={}, repo_args={}, sync_args={}, @@ -206,9 +209,9 @@ def _deb_init_and_sync( else: url = deb_get_fixture_server_url(url) if repository is None: - repository = deb_repository_factory(**repo_args) + repository = deb_repository_factory(pulp_domain=pulp_domain, **repo_args) if remote is None: - remote = deb_remote_factory(url=url, **remote_args) + remote = deb_remote_factory(url=url, pulp_domain=pulp_domain, **remote_args) task = deb_sync_repository(remote, repository, **sync_args) diff --git a/pulp_deb/tests/functional/api/test_domains.py b/pulp_deb/tests/functional/api/test_domains.py new file mode 100644 index 00000000..9056da90 --- /dev/null +++ b/pulp_deb/tests/functional/api/test_domains.py @@ -0,0 +1,269 @@ +import pytest +import json +import uuid + +from django.conf import settings + +from pulpcore.client.pulp_deb.exceptions import ApiException + +from pulp_deb.tests.functional.constants import DEB_PACKAGE_RELPATH, DEB_PUBLISH_STANDARD +from pulp_deb.tests.functional.utils import ( + gen_deb_remote, + gen_distribution, + gen_repo, + get_local_package_absolute_path, +) + +if not settings.DOMAIN_ENABLED: + pytest.skip("Domains not enabled.", allow_module_level=True) + + +def test_domain_create( + deb_domain_factory, + deb_init_and_sync, + apt_package_api, + apt_repository_api, +): + """Test repo-creation in a domain.""" + domain_name = deb_domain_factory().name + + # create and sync in default domain (not specified) + deb_init_and_sync() + + # check that newly created domain doesn't have a repo or any packages + assert apt_repository_api.list(pulp_domain=domain_name).count == 0 + assert apt_package_api.list(pulp_domain=domain_name).count == 0 + + +@pytest.mark.slow +def test_domain_sync( + gen_object_with_cleanup, + deb_domain_factory, + apt_remote_api, + apt_repository_api, + apt_package_api, + deb_get_fixture_server_url, + deb_sync_repository, + deb_cleanup_domains, +): + """Test repo-sync in a domain.""" + domain = deb_domain_factory() + try: + domain_name = domain.name + + # create and sync in the newly-created domain + url = deb_get_fixture_server_url() + remote = gen_object_with_cleanup( + apt_remote_api, gen_deb_remote(url=str(url)), pulp_domain=domain_name + ) + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain_name) + + # check that we can "find" the new repo in the new domain via filtering + repos = apt_repository_api.list(name=repo.name, pulp_domain=domain_name).results + assert len(repos) == 1 + assert repos[0].pulp_href == repo.pulp_href + deb_sync_repository(remote=remote, repo=repo) + repo = apt_repository_api.read(repo.pulp_href) + + # check that newly created domain has one repo (list works) and the expected contents + assert apt_repository_api.list(pulp_domain=domain_name).count == 1 + assert ( + apt_package_api.list( + repository_version=repo.latest_version_href, pulp_domain=domain_name + ).count + == 4 + ) + finally: + deb_cleanup_domains([domain], content_api_client=apt_package_api, cleanup_repositories=True) + + +@pytest.mark.parallel +@pytest.mark.slow +def test_object_creation( + gen_object_with_cleanup, + deb_domain_factory, + apt_repository_api, + deb_remote_factory, + deb_sync_repository, + deb_get_fixture_server_url, +): + """Test basic object creation in a separate domain.""" + domain = deb_domain_factory() + domain_name = domain.name + + url = deb_get_fixture_server_url() + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain_name) + assert f"{domain_name}/api/v3/" in repo.pulp_href + + repos = apt_repository_api.list(pulp_domain=domain_name) + assert repos.count == 1 + assert repo.pulp_href == repos.results[0].pulp_href + + # list repos on default domain + default_repos = apt_repository_api.list(name=repo.name) + assert default_repos.count == 0 + + # try to create an object with cross domain relations + url = deb_get_fixture_server_url() + default_remote = deb_remote_factory(url) + with pytest.raises(ApiException) as e: + repo_body = {"name": str(uuid.uuid4()), "remote": default_remote.pulp_href} + apt_repository_api.create(repo_body, pulp_domain=domain_name) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {domain_name} domain."] + } + + with pytest.raises(ApiException) as e: + deb_sync_repository(remote=default_remote, repo=repo) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {domain_name} domain."] + } + + +@pytest.mark.parallel +@pytest.mark.slow +def test_deb_from_file( + deb_cleanup_domains, + deb_domain_factory, + apt_package_api, + deb_package_factory, +): + """Test uploading of deb content with domains""" + domain = deb_domain_factory() + + try: + package_upload_params = { + "file": str(get_local_package_absolute_path(DEB_PACKAGE_RELPATH)), + "relative_path": DEB_PACKAGE_RELPATH, + } + default_content = deb_package_factory(**package_upload_params) + package_upload_params["pulp_domain"] = domain.name + domain_content = deb_package_factory(**package_upload_params) + assert default_content.pulp_href != domain_content.pulp_href + assert default_content.sha256 == domain_content.sha256 + + domain_contents = apt_package_api.list(pulp_domain=domain.name) + assert domain_contents.count == 1 + finally: + deb_cleanup_domains([domain], content_api_client=apt_package_api) + + +@pytest.mark.parallel +@pytest.mark.slow +def test_content_promotion( + gen_object_with_cleanup, + monitor_task, + download_content_unit, + apt_remote_api, + apt_repository_api, + apt_publication_api, + apt_distribution_api, + deb_domain_factory, + deb_cleanup_domains, + deb_delete_repository, + deb_sync_repository, + deb_get_repository_by_href, + deb_get_fixture_server_url, +): + """Tests Content promotion path with domains: Sync->Publish->Distribute""" + domain = deb_domain_factory() + + try: + # Sync + url = deb_get_fixture_server_url() + remote = gen_object_with_cleanup( + apt_remote_api, gen_deb_remote(url=str(url)), pulp_domain=domain.name + ) + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain.name) + response = deb_sync_repository(remote=remote, repo=repo) + assert len(response.created_resources) == 1 + + repo = deb_get_repository_by_href(repo.pulp_href) + assert repo.latest_version_href[-2] == "1" + + # Publish + pub_body = {"repository": repo.pulp_href} + task = apt_publication_api.create(pub_body, pulp_domain=domain.name).task + response = monitor_task(task) + assert len(response.created_resources) == 1 + pub_href = response.created_resources[0] + publication = apt_publication_api.read(pub_href) + assert publication.repository == repo.pulp_href + + # Distribute + distro_body = gen_distribution() + distro_body["publication"] = publication.pulp_href + distribution = gen_object_with_cleanup( + apt_distribution_api, distro_body, pulp_domain=domain.name + ) + assert distribution.publication == publication.pulp_href + # url structure should be host/CONTENT_ORIGIN/DOMAIN_PATH/BASE_PATH + assert domain.name == distribution.base_url.rstrip("/").split("/")[-2] + + # check that content can be downloaded from base_url + package_index_paths = DEB_PUBLISH_STANDARD["package_index_paths"] + for package_index_path in package_index_paths: + download_content_unit( + distribution.to_dict()["base_path"], package_index_path, domain=domain.name + ) + + # cleanup to delete the domain + deb_delete_repository(repo) + finally: + deb_cleanup_domains([domain], cleanup_repositories=True) + + +@pytest.mark.parallel +@pytest.mark.slow +def test_domain_rbac( + gen_object_with_cleanup, deb_domain_factory, deb_cleanup_domains, apt_repository_api, gen_user +): + """Test domain level roles.""" + domain = deb_domain_factory() + + try: + deb_viewer = "deb.aptrepository_viewer" + deb_creator = "deb.aptrepository_creator" + user_a = gen_user(username="a", domain_roles=[(deb_viewer, domain.pulp_href)]) + user_b = gen_user(username="b", domain_roles=[(deb_creator, domain.pulp_href)]) + + # create two repos in different domains with admin user + gen_object_with_cleanup(apt_repository_api, gen_repo()) + gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain.name) + + with user_b: + repo = gen_object_with_cleanup(apt_repository_api, gen_repo(), pulp_domain=domain.name) + repos = apt_repository_api.list(pulp_domain=domain.name) + assert repos.count == 1 + assert repos.results[0].pulp_href == repo.pulp_href + + # try to create a repository in default domain + with pytest.raises(ApiException) as e: + apt_repository_api.create({"name": str(uuid.uuid4())}) + assert e.value.status == 403 + + with user_a: + repos = apt_repository_api.list(pulp_domain=domain.name) + assert repos.count == 2 + + # try to read repos in the default domain + repos = apt_repository_api.list() + assert repos.count == 0 + + # try to create a repo + with pytest.raises(ApiException) as e: + apt_repository_api.create({"name": str(uuid.uuid4())}, pulp_domain=domain.name) + assert e.value.status == 403 + finally: + deb_cleanup_domains([domain], cleanup_repositories=True) + + +# @pytest.mark.parallel +# def test_cross_domain_copy_all( +# deb_domain_factory, +# ): +# """Test attempting to copy between different domains.""" +# domain_1 = None +# domain_2 = None diff --git a/pulp_deb/tests/functional/conftest.py b/pulp_deb/tests/functional/conftest.py index f4dde5f9..48b83d70 100644 --- a/pulp_deb/tests/functional/conftest.py +++ b/pulp_deb/tests/functional/conftest.py @@ -5,6 +5,7 @@ import re import stat import subprocess +import uuid from pulp_deb.tests.functional.constants import DEB_SIGNING_SCRIPT_STRING from pulpcore.client.pulp_deb import ( @@ -555,3 +556,83 @@ def _deb_get_content_types(content_api_name, content_type, repo, version_href=No return api.list(repository_version=latest_version_href).results return _deb_get_content_types + + +# @pytest.fixture +# def deb_setup_domain( +# gen_object_with_cleanup, +# domains_api_client, +# deb_remote_factory, +# deb_repository_factory, +# deb_get_fixture_server, +# deb_sync_repository, +# monitor_task, +# ): +# def _deb_setup_domain(sync=True, pulp_domain=None, url=None): +# if url is None: +# url = deb_get_fixture_server() +# elif url.startswith("http"): +# url = url +# else: +# url = deb_get_fixture_server(url) +# +# if not pulp_domain: +# body = { +# "name": str(uuid.uuid4()), +# "storage_class": "pulpcore.app.models.storage.FileSystem", +# "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"}, +# } +# pulp_domain = gen_object_with_cleanup(domains_api_client, body) +# +# remote = deb_remote_factory(url, pulp_domain=pulp_domain.name) +# repo = deb_repository_factory(pulp_domain=pulp_domain.name) +# +# if sync: +# deb_sync_repository(remote, repo) + + +@pytest.fixture +def deb_cleanup_domains(pulpcore_bindings, monitor_task, apt_repository_api): + def _deb_cleanup_domains( + domains, + content_api_client=None, + cleanup_repositories=False, + repository_api_client=apt_repository_api, + ): + for domain in domains: + # clean up each domain specified + if domain: + if cleanup_repositories: + # delete repos from the domain + for repo in repository_api_client.list(pulp_domain=domain.name).results: + monitor_task(repository_api_client.delete(repo.pulp_href).task) + # let orphan cleanup reap the resulting abandoned content + monitor_task( + pulpcore_bindings.OrphansCleanupApi.cleanup( + {"orphan_protection_time": 0}, pulp_domain=domain.name + ).task + ) + + if content_api_client: + # if we have a client, check that each domain is empty of that kind-of entity + for domain in domains: + if domain: + assert content_api_client.list(pulp_domain=domain.name).count == 0 + + return _deb_cleanup_domains + + +@pytest.fixture +def deb_domain_factory(pulpcore_bindings, gen_object_with_cleanup): + """Fixture to create a domain.""" + + def _deb_domain_factory(name=None): + name = str(uuid.uuid4()) if name is None else name + body = { + "name": name, + "storage_class": "pulpcore.app.models.storage.FileSystem", + "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"}, + } + return gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body) + + return _deb_domain_factory diff --git a/pulp_deb/tests/functional/constants.py b/pulp_deb/tests/functional/constants.py index 75fec1ee..39f35100 100644 --- a/pulp_deb/tests/functional/constants.py +++ b/pulp_deb/tests/functional/constants.py @@ -259,6 +259,15 @@ def _clean_dict(d): "package_index_paths_dist": ["dists/flat-repo/flat-repo-component/binary-ppc64/Packages"], } +DEB_PUBLISH_STANDARD = { + "package_index_paths": [ + "dists/ragnarok/asgard/binary-ppc64/Packages", + "dists/ragnarok/asgard/binary-armeb/Packages", + "dists/ragnarok/jotunheimr/binary-ppc64/Packages", + "dists/ragnarok/jotunheimr/binary-armeb/Packages", + ] +} + DEB_PUBLISH_FLAT_SIMPLE = { "distribution": "/", "codename": "ragnarok", diff --git a/pulp_deb/tests/functional/utils.py b/pulp_deb/tests/functional/utils.py index ff4719ef..40d883ef 100644 --- a/pulp_deb/tests/functional/utils.py +++ b/pulp_deb/tests/functional/utils.py @@ -84,13 +84,15 @@ def gen_distribution(**kwargs): return data -def gen_remote(url, **kwargs): +def gen_remote(url, pulp_domain=None, **kwargs): """Return a semi-random dict for use in creating a Remote. :param url: The URL of an external content source. """ data = {"name": str(uuid4()), "url": url} data.update(kwargs) + if pulp_domain: + data["pulp_domain"] = pulp_domain return data diff --git a/template_config.yml b/template_config.yml index 9ba2f246..624d4b04 100644 --- a/template_config.yml +++ b/template_config.yml @@ -51,7 +51,8 @@ pulp_settings: allowed_import_paths: - /tmp apt_by_hash: true -pulp_settings_azure: null +pulp_settings_azure: + domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: null pydocstyle: true