fix: Add new cleanup fixture to track docker assets that need to be cleaned up (#129)

#### Relevant issue or PR
#34 

#### Description of changes
Add cleanup fixture to clean up images, containers, and projects
generated by the unit tests. Also added cleanup to the test_examples
images since they used a different image build with no cleanup.

#### Testing done
Unit testing + manual testing to check if any projects and images exists
after running tests since assert functions are frowned upon in fixtures.

Added keyboard interrupt into serve_pipline end to end test 

```
tests/endtoend_tests/test_endtoend.py::test_tesseract_serve_pipeline AKOAKO cleaning up! {'images': ['tmp_tesseract_image_jpo5rz28xc2xte0f'], 'project_ids': ['tesseract-o7f82iixoszg'], 'containers': []}


!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! KeyboardInterrupt !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/Users/ako/work/tesseract-core/tests/endtoend_tests/test_endtoend.py:188: KeyboardInterrupt
(to show a full traceback on KeyboardInterrupt use --full-trace)
================================================ no tests ran in 41.00s ================================================
```

You can see that the cleanup fixture still runs and `ps` `list` all are
a clean slate.

#### License

- [x ] By submitting this pull request, I confirm that my contribution
is made under the terms of the [Apache 2.0
license](https://pasteurlabs.github.io/tesseract/LICENSE).
- [ x] I sign the Developer Certificate of Origin below by adding my
name and email address to the `Signed-off-by` line.

<details>
<summary><b>Developer Certificate of Origin</b></summary>

```text
Developer Certificate of Origin
Version 1.1

Copyright (C) 2004, 2006 The Linux Foundation and its contributors.

Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.


Developer's Certificate of Origin 1.1

By making a contribution to this project, I certify that:

(a) The contribution was created in whole or in part by me and I
    have the right to submit it under the open source license
    indicated in the file; or

(b) The contribution is based upon previous work that, to the best
    of my knowledge, is covered under an appropriate open source
    license and I have the right under that license to submit that
    work with modifications, whether created in whole or in part
    by me, under the same open source license (unless I am
    permitted to submit under a different license), as indicated
    in the file; or

(c) The contribution was provided directly to me by some other
    person who certified (a), (b) or (c) and I have not modified
    it.

(d) I understand and agree that this project and the contribution
    are public and that a record of the contribution (including all
    personal information I submit with it, including my sign-off) is
    maintained indefinitely and may be redistributed consistent with
    this project or the open source license(s) involved.
```

</details>

Signed-off-by: Angela Ko [email protected]

---------

Co-authored-by: Dion Häfner <[email protected]>

Author: angela-ko

tests/conftest.py

@@ -187,36 +187,55 @@ def docker_client():
     return docker_client_module.CLIDockerClient()
 
 
+@pytest.fixture(scope="module")
+def docker_cleanup(docker_client, request):
+    """Clean up all tesseracts created by the tests."""
+    from tesseract_core.sdk.docker_client import ContainerError, ImageNotFound
+
+    # Shared object to track what objects need to be cleaned up in each test
+    context = {"images": [], "project_ids": [], "containers": []}
+
+    def cleanup_func():
+        # Teardown projects first
+        for project_id in context["project_ids"]:
+            if docker_client.compose.exists(project_id):
+                docker_client.compose.down(project_id)
+
+        # Remove containers
+        for container in context["containers"]:
+            try:
+                container_obj = docker_client.containers.get(container)
+            except ContainerError:
+                continue
+            container_obj.remove(v=True, force=True)
+
+        # Remove images
+        for image in context["images"]:
+            try:
+                docker_client.images.remove(image)
+            except ImageNotFound:
+                continue
+
+    request.addfinalizer(cleanup_func)
+    return context
+
+
 @pytest.fixture
-def dummy_image_name(docker_client):
+def dummy_image_name(docker_cleanup):
     """Create a dummy image name, and clean up after the test."""
-    from tesseract_core.sdk.docker_client import ImageNotFound
-
     image_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16))
     image_name = f"tmp_tesseract_image_{image_id}"
-    try:
-        yield image_name
-    finally:
-        try:
-            docker_client.images.remove(image_name)
-        except ImageNotFound:
-            pass
+    docker_cleanup["images"].append(image_name)
+    yield image_name
 
 
 @pytest.fixture(scope="module")
-def shared_dummy_image_name(docker_client):
+def shared_dummy_image_name(docker_cleanup):
     """Create a dummy image name, and clean up after all tests."""
-    from tesseract_core.sdk.docker_client import ImageNotFound
-
     image_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16))
     image_name = f"tmp_tesseract_image_{image_id}"
-    try:
-        yield image_name
-    finally:
-        try:
-            docker_client.images.remove(image_name)
-        except ImageNotFound:
-            pass
+    docker_cleanup["images"].append(image_name)
+    yield image_name
 
 
 @pytest.fixture

tests/endtoend_tests/common.py

@@ -8,7 +8,11 @@
 from typer.testing import CliRunner
 
 from tesseract_core.sdk.cli import app
-from tesseract_core.sdk.docker_client import CLIDockerClient, ImageNotFound
+from tesseract_core.sdk.docker_client import (
+    CLIDockerClient,
+    ContainerError,
+    ImageNotFound,
+)
 
 
 def image_exists(client, image_name, tesseract_only: bool = True):
@@ -25,6 +29,15 @@ def image_exists(client, image_name, tesseract_only: bool = True):
         return False
 
 
+def container_exists(client, container_name_or_id, tesseract_only: bool = True):
+    """Checks if containers exists."""
+    try:
+        client.containers.get(container_name_or_id, tesseract_only)
+        return True
+    except ContainerError:
+        return False
+
+
 def print_debug_info(result):
     """Print debug info from result of a CLI command if it failed."""
     if result.exit_code == 0:
@@ -75,4 +88,5 @@ def build_tesseract(
 
     image_tags = json.loads(result.stdout.strip())
     assert image_name in image_tags
+
     return image_tags[0]

tests/endtoend_tests/test_endtoend.py

@@ -16,10 +16,13 @@
 
 
 @pytest.fixture(scope="module")
-def built_image_name(docker_client, shared_dummy_image_name, dummy_tesseract_location):
+def built_image_name(
+    docker_client, docker_cleanup, shared_dummy_image_name, dummy_tesseract_location
+):
     """Build the dummy Tesseract image for the tests."""
     image_name = build_tesseract(dummy_tesseract_location, shared_dummy_image_name)
     assert image_exists(docker_client, image_name)
+    docker_cleanup["images"].append(image_name)
     yield image_name
 
 
@@ -34,7 +37,7 @@ def built_image_name(docker_client, shared_dummy_image_name, dummy_tesseract_loc
 
 @pytest.mark.parametrize("tag,recipe,base_image", build_matrix)
 def test_build_from_init_endtoend(
-    docker_client, dummy_image_name, tmp_path, tag, recipe, base_image
+    docker_client, docker_cleanup, dummy_image_name, tmp_path, tag, recipe, base_image
 ):
     """Test that a trivial (empty) Tesseract image can be built from init."""
     cli_runner = CliRunner(mix_stderr=False)
@@ -59,6 +62,7 @@ def test_build_from_init_endtoend(
         tmp_path, dummy_image_name, config_override=config_override, tag=img_tag
     )
     assert image_exists(docker_client, image_name)
+    docker_cleanup["images"].append(image_name)
 
     # Test that the image can be run and that --help is forwarded correctly
     result = cli_runner.invoke(
@@ -143,59 +147,48 @@ def test_tesseract_run_stdout(built_image_name):
             raise
 
 
-def test_tesseract_serve_pipeline(docker_client, built_image_name):
+def test_tesseract_serve_pipeline(docker_client, built_image_name, docker_cleanup):
     cli_runner = CliRunner(mix_stderr=False)
     project_id = None
-    try:
-        run_res = cli_runner.invoke(
-            app,
-            [
-                "serve",
-                built_image_name,
-            ],
-            catch_exceptions=False,
-        )
+    run_res = cli_runner.invoke(
+        app,
+        [
+            "serve",
+            built_image_name,
+        ],
+        catch_exceptions=False,
+    )
 
-        assert run_res.exit_code == 0, run_res.stderr
-        assert run_res.stdout
+    assert run_res.exit_code == 0, run_res.stderr
+    assert run_res.stdout
 
-        project_meta = json.loads(run_res.stdout)
+    project_meta = json.loads(run_res.stdout)
 
-        project_id = project_meta["project_id"]
-        project_containers = project_meta["containers"][0]["name"]
-        if not project_containers:
-            raise ValueError(f"Could not find container for project '{project_id}'")
+    project_id = project_meta["project_id"]
+    docker_cleanup["project_ids"].append(project_id)
+    project_containers = project_meta["containers"][0]["name"]
+    if not project_containers:
+        raise ValueError(f"Could not find container for project '{project_id}'")
 
-        project_container = docker_client.containers.get(project_containers)
-        assert project_container.name == project_meta["containers"][0]["name"]
-        assert project_container.host_port == project_meta["containers"][0]["port"]
+    project_container = docker_client.containers.get(project_containers)
+    assert project_container.name == project_meta["containers"][0]["name"]
+    assert project_container.host_port == project_meta["containers"][0]["port"]
 
-        # Ensure served Tesseract is usable
-        res = requests.get(f"http://localhost:{project_container.host_port}/health")
-        assert res.status_code == 200, res.text
+    # Ensure served Tesseract is usable
+    res = requests.get(f"http://localhost:{project_container.host_port}/health")
+    assert res.status_code == 200, res.text
 
-        # Ensure project id is shown in `tesseract ps`
-        run_res = cli_runner.invoke(
-            app,
-            ["ps"],
-            env={"COLUMNS": "1000"},
-            catch_exceptions=False,
-        )
-        assert run_res.exit_code == 0, run_res.stderr
-        assert project_id in run_res.stdout
-        assert project_container.host_port in run_res.stdout
-        assert project_container.short_id in run_res.stdout
-    finally:
-        if project_id:
-            run_res = cli_runner.invoke(
-                app,
-                [
-                    "teardown",
-                    project_id,
-                ],
-                catch_exceptions=False,
-            )
-            assert run_res.exit_code == 0, run_res.stderr
+    # Ensure project id is shown in `tesseract ps`
+    run_res = cli_runner.invoke(
+        app,
+        ["ps"],
+        env={"COLUMNS": "1000"},
+        catch_exceptions=False,
+    )
+    assert run_res.exit_code == 0, run_res.stderr
+    assert project_id in run_res.stdout
+    assert project_container.host_port in run_res.stdout
+    assert project_container.short_id in run_res.stdout
 
 
 @pytest.mark.parametrize("tear_all", [True, False])
@@ -320,7 +313,7 @@ def test_tesseract_serve_ports_error(built_image_name):
 
 
 @pytest.mark.parametrize("port", ["34567", "34567-34569"])
-def test_tesseract_serve_ports(built_image_name, port):
+def test_tesseract_serve_ports(built_image_name, port, docker_cleanup):
     """Try to serve multiple Tesseracts on multiple ports."""
     cli_runner = CliRunner(mix_stderr=False)
     project_id = None
@@ -336,39 +329,27 @@ def test_tesseract_serve_ports(built_image_name, port):
 
     project_meta = json.loads(run_res.stdout)
     project_id = project_meta["project_id"]
+    docker_cleanup["project_ids"].append(project_id)
 
-    # Wrap test in try-finally to ensure teardown of served Tesseract.
-    try:
-        # Ensure that actual used ports are in the specified port range.
-        test_ports = port.split("-")
-        start_port = int(test_ports[0])
-        end_port = int(test_ports[1]) if len(test_ports) > 1 else start_port
+    # Ensure that actual used ports are in the specified port range.
+    test_ports = port.split("-")
+    start_port = int(test_ports[0])
+    end_port = int(test_ports[1]) if len(test_ports) > 1 else start_port
 
-        port = int(project_meta["containers"][0]["port"])
-        assert port in range(start_port, end_port + 1)
+    port = int(project_meta["containers"][0]["port"])
+    assert port in range(start_port, end_port + 1)
 
-        # Ensure specified ports are in `tesseract ps` and served Tesseracts are usable.
-        run_res = cli_runner.invoke(
-            app,
-            ["ps"],
-            env={"COLUMNS": "1000"},
-            catch_exceptions=False,
-        )
+    # Ensure specified ports are in `tesseract ps` and served Tesseracts are usable.
+    run_res = cli_runner.invoke(
+        app,
+        ["ps"],
+        env={"COLUMNS": "1000"},
+        catch_exceptions=False,
+    )
 
-        res = requests.get(f"http://localhost:{port}/health")
-        assert res.status_code == 200, res.text
-        assert str(port) in run_res.stdout
-    finally:
-        if project_id:
-            run_res = cli_runner.invoke(
-                app,
-                [
-                    "teardown",
-                    project_id,
-                ],
-                catch_exceptions=False,
-            )
-            assert run_res.exit_code == 0, run_res.stderr
+    res = requests.get(f"http://localhost:{port}/health")
+    assert res.status_code == 200, res.text
+    assert str(port) in run_res.stdout
 
 
 def test_tesseract_serve_with_volumes(built_image_name, tmp_path, docker_client):