Pods: with seaworthy tests

.travis.yml

@@ -9,17 +9,20 @@ python: '3.6'
 branches:
   only:
     - develop
+    - seaworthy
 
 env:
   global:
     - IMAGE_USER=praekeltfoundation
     - REGISTRY_USER=praekeltorgdeploy
     - secure: "UK7h5cBSPlsL1bPJx+S7ArYKHIUjEWfsITw8M/ObTLaIQHMq2uV3J/D7K5GFkqJFX9sUGmFZBkCu1J9JwpieO1Ew79B2/g4i0aDDmhIuE+IOo5tQ6jinAmP1NEVrCvk1LUTdXwk46VbUCGNxYpv44ge1zwhRrtNpMxaIqXsSYLOXjA7p7A/IghDRYDsRIjz6TfXNKcw+qWxzUTSUjLqgcZqezvI3c9VRRgD94vKh9KpLryVtSPxGjUQTKD6cfu6Yvbz6TzVGp6QCEM9ysRdRJx0ecsP39XBj8VVnqI34ux+4sqiwdqM7SQTlaKFJK+o6dMAX5ZEkA9jQw2kWDX0Pg60zHX+vbH1vycPVaQ6ln9J68pNZx6AMFbJpDkTacrmStH9uIuH6+pN4j4gNsT4TkwtFJDcRuGI0gShqBgQVVHhGuCECfj6RZQ7K35NsQVHXQy47AkwGODu3xUwAuiMOeQVR2rEYCBnJfx+qLY5hIsPzR4MhidLPSZa5T520uhp+35pVAk95DYYEL+6N2C3wIOmtEl6gZNC1P0IgCDuTOU/u5+TCaS5C7M3aWk26SL+n3JnFeCAgLr+OYntHwW9PIen70OG1bokGdxQZ00DvF9veyQz+6IGVEG1Cm5MZv6Cp9bqgpGYMRhKBEAJyKrT+slt6ah10+eIJbT7p4HIgKBM="
   matrix:
-    - PYTHON_VERSION=2.7 VARIANT=jessie  VERSION_LATEST=1 TAG_LATEST=1
-    - PYTHON_VERSION=2.7 VARIANT=stretch VERSION_LATEST=1 TAG_LATEST=
-    - PYTHON_VERSION=3.6 VARIANT=jessie  VERSION_LATEST=  TAG_LATEST=1
-    - PYTHON_VERSION=3.6 VARIANT=stretch VERSION_LATEST=  TAG_LATEST=
+    - PYTHON_VERSION=2.7 VARIANT=jessie  PODS=  VERSION_LATEST=1 TAG_LATEST=1
+    - PYTHON_VERSION=2.7 VARIANT=stretch PODS=  VERSION_LATEST=1 TAG_LATEST=
+    - PYTHON_VERSION=2.7 VARIANT=stretch PODS=1 VERSION_LATEST=1 TAG_LATEST=
+    - PYTHON_VERSION=3.6 VARIANT=jessie  PODS=  VERSION_LATEST=  TAG_LATEST=1
+    - PYTHON_VERSION=3.6 VARIANT=stretch PODS=  VERSION_LATEST=  TAG_LATEST=
+    - PYTHON_VERSION=3.6 VARIANT=stretch PODS=1 VERSION_LATEST=  TAG_LATEST=
 
 # Update Docker Engine and Compose
 before_install:
@@ -29,14 +32,18 @@ install:
   - (cd example; pip install -r requirements-test.txt)
 
 before_script:
-  - tag="py$PYTHON_VERSION-$VARIANT"
-  - image="$IMAGE_USER/django-bootstrap:$tag"
+  - tag="py$PYTHON_VERSION-$VARIANT${PODS:+-pods}"
+  - name="$IMAGE_USER/django-bootstrap"
+  - image="$name:$tag"
+  - context="${PODS:+pods}"; context="${context:-.}"
+  - echo "Building image '$image' from context ${context}..."
   # Fetch existing image to use as cache
   - docker pull "$image" || true
 script:
-  - docker build --pull --cache-from "$image" --build-arg PYTHON_VERSION="$PYTHON_VERSION-$VARIANT" --tag "$image" .
+  - docker build --pull --cache-from "$image" --build-arg PYTHON_VERSION="$PYTHON_VERSION-$VARIANT" --tag "$image" "$context"
+  - if [ -n "$PODS" ]; then docker build --tag "$name:nginx" pods/nginx; fi
   - docker build -t "mysite:$tag" --build-arg VARIANT="$tag" example
-  - pytest -v example/test.py --django-bootstrap-image="mysite:$tag"
+  - pytest -v example/test.py --django-bootstrap-image="mysite:$tag" ${PODS:+--pods --db-nginx-image="$name:nginx"}
 
 after_script:
   - docker images

example/conftest.py

@@ -1,13 +1,31 @@
 import os
 
+import pytest
+
 
 def pytest_addoption(parser):
+    parser.addoption(
+        '--pods', action='store_true',
+        help='run tests for pods')
+
     parser.addoption(
         '--django-bootstrap-image', action='store',
         default=os.environ.get('DJANGO_BOOTSTRAP_IMAGE', 'mysite:py3'),
         help='django-bootstrap docker image to test')
 
+    parser.addoption(
+        '--db-nginx-image', action='store',
+        default=os.environ.get(
+            'DB_NGINX_IMAGE', 'praekeltfoundation/django-bootstrap:nginx'),
+        help='django-bootstrap Nginx docker image to test')
+
 
 def pytest_report_header(config):
-    return 'django-bootstrap docker image: {}'.format(
-        config.getoption('--django-bootstrap-image'))
+    return 'django-bootstrap docker image: {}, nginx: {}'.format(
+        config.getoption('--django-bootstrap-image'),
+        config.getoption('--db-nginx-image'))
+
+
+@pytest.fixture(scope='session')
+def pods(request):
+    return request.config.getoption('--pods')

example/fixtures.py

@@ -1,8 +1,9 @@
 import os
+import time
 
 import pytest
 
-from seaworthy.definitions import ContainerDefinition
+from seaworthy.definitions import ContainerDefinition, VolumeDefinition
 from seaworthy.containers.postgresql import PostgreSQLContainer
 from seaworthy.containers.rabbitmq import RabbitMQContainer
 from seaworthy.ps import list_container_processes
@@ -11,6 +12,7 @@
 
 
 DDB_IMAGE = pytest.config.getoption('--django-bootstrap-image')
+NGINX_IMAGE = pytest.config.getoption('--db-nginx-image')
 
 DEFAULT_WAIT_TIMEOUT = int(os.environ.get('DEFAULT_WAIT_TIMEOUT', '30'))
 
@@ -38,6 +40,15 @@ def exec_find(self, params):
     def for_fixture(
             cls, request, name, wait_lines, command=None, env_extra={},
             publish_port=True, wait_timeout=None):
+        pods = request.getfixturevalue('pods')
+        # FIXME: there are probably better ways to skip these tests
+        if pods:
+            if request.fixturename in [
+                    'web_only_container', 'single_container']:
+                pytest.skip()
+        else:
+            if request.fixturename == 'gunicorn_container':
+                pytest.skip()
         docker_helper = request.getfixturevalue('docker_helper')
         amqp_container = request.getfixturevalue('amqp_container')
         db_container = request.getfixturevalue('db_container')
@@ -54,6 +65,16 @@ def for_fixture(
         }
         if publish_port:
             kwargs['ports'] = {'8000/tcp': ('127.0.0.1',)}
+        if pods:
+            static_volume = request.getfixturevalue('static_volume')
+            media_volume = request.getfixturevalue('media_volume')
+            gunicorn_volume = request.getfixturevalue('gunicorn_volume')
+            kwargs['volumes'] = {
+                static_volume.inner(): {'bind': '/app/static', 'mode': 'rw'},
+                media_volume.inner(): {'bind': '/app/media', 'mode': 'rw'},
+                gunicorn_volume.inner(): {
+                    'bind': '/var/run/gunicorn', 'mode': 'rw'},
+            }
 
         return cls(
             name, DDB_IMAGE, wait_lines, wait_timeout, kwargs,
@@ -82,6 +103,8 @@ def fixture(request):
     'worker_only_container', 'worker', [r'celery@\w+ ready'],
     command=['celery', 'worker'], publish_port=False)
 
+gunicorn_container = DjangoBootstrapContainer.make_fixture(
+    'gunicorn_container', 'web', [r'Booting worker'], publish_port=False)
 
 beat_only_container = DjangoBootstrapContainer.make_fixture(
     'beat_only_container', 'beat', [r'beat: Starting\.\.\.'],
@@ -96,7 +119,8 @@ def containers(request):
 
 
 web_container = make_multi_container(
-    'web_container', ['single_container', 'web_only_container'])
+    'web_container',
+    ['single_container', 'web_only_container', 'gunicorn_container'])
 
 worker_container = make_multi_container(
     'worker_container', ['single_container', 'worker_only_container'])
@@ -105,8 +129,108 @@ def containers(request):
     'beat_container', ['single_container', 'beat_only_container'])
 
 
+def volume_fixture(volume, name, scope='function'):
+    @pytest.fixture(name=name, scope=scope)
+    def fixture(docker_helper):
+        volume.set_helper(docker_helper)
+        with volume:
+            yield volume
+    return fixture
+
+
+static_volume = volume_fixture(VolumeDefinition('static'), 'static_volume')
+media_volume = volume_fixture(VolumeDefinition('media'), 'media_volume')
+gunicorn_volume = volume_fixture(
+    VolumeDefinition('gunicorn'), 'gunicorn_volume')
+
+
+class NginxContainer(ContainerDefinition):
+    def __init__(self, name, image, static_volume, media_volume,
+                 gunicorn_volume, **kwargs):
+        super().__init__(name, image, **kwargs)
+        self._static_volume = static_volume
+        self._media_volume = media_volume
+        self._gunicorn_volume = gunicorn_volume
+
+    def base_kwargs(self):
+        return {
+            'ports': {'80/tcp': ('127.0.0.1',)},
+            'volumes': {
+                self._static_volume.inner(): {
+                    'bind': '/usr/share/nginx/static',
+                    'mode': 'ro',
+                },
+                self._media_volume.inner(): {
+                    'bind': '/usr/share/nginx/media',
+                    'mode': 'ro',
+                },
+                self._gunicorn_volume.inner(): {
+                    'bind': '/var/run/gunicorn',
+                    'mode': 'rw',
+                },
+            }
+        }
+
+    def wait_for_start(self):
+        # No real way to know when Nginx is ready. Wait a short moment.
+        time.sleep(0.5)
+
+    def list_processes(self):
+        return list_container_processes(self.inner())
+
+    @classmethod
+    def for_fixture(cls, request, name):
+        if not request.getfixturevalue('pods'):
+            pytest.skip()
+
+        docker_helper = request.getfixturevalue('docker_helper')
+
+        static_volume = request.getfixturevalue('static_volume')
+        media_volume = request.getfixturevalue('media_volume')
+        gunicorn_volume = request.getfixturevalue('gunicorn_volume')
+
+        return cls(
+            name, NGINX_IMAGE, static_volume, media_volume, gunicorn_volume,
+            helper=docker_helper)
+
+    @classmethod
+    def make_fixture(cls, fixture_name, name, *args, **kw):
+        @pytest.fixture(name=fixture_name)
+        def fixture(request):
+            with cls.for_fixture(request, name, *args, **kw) as container:
+                yield container
+        return fixture
+
+
+@pytest.fixture
+def nginx_container(request, pods, web_container):
+    # FIXME: Find some way for this to not depend on web_container
+    if not pods:
+        return web_container
+
+    return request.getfixturevalue('nginx_only_container')
+
+
+nginx_only_container = NginxContainer.make_fixture(
+    'nginx_only_container', 'nginx')
+
+
 __all__ = [
-    'raw_db_container', 'db_container', 'raw_amqp_container',
-    'amqp_container', 'single_container', 'web_only_container',
-    'worker_only_container', 'beat_only_container', 'web_container',
-    'worker_container', 'beat_container']
+    'amqp_container',
+    'beat_container',
+    'beat_only_container',
+    'db_container',
+    'gunicorn_container',
+    'gunicorn_volume',
+    'media_volume',
+    'nginx_container',
+    'nginx_only_container',
+    'raw_amqp_container',
+    'raw_db_container',
+    'single_container',
+    'static_volume',
+    'web_container',
+    'web_only_container',
+    'worker_container',
+    'worker_only_container',
+]