Setup SqlModel and Alembic

.env.sample

@@ -1,2 +1,2 @@
-REDIS_URL=redis://localhost:6379/0
-DATABASE_URL=psql://postgres:postgres@localhost:5432/postgres
+REDIS_URL=redis://redis:6379/0
+DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres

.env.test

@@ -1,2 +1,3 @@
 REDIS_URL=redis://localhost:6379/0
-DATABASE_URL=psql://postgres:postgres@localhost:5432/postgres
+DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/postgres
+DATABASE_POOL_CLASS=NullPool

.github/workflows/ci.yml

@@ -41,6 +41,18 @@ jobs:
           --health-retries 5
         ports:
           - 6379:6379
+      postgres:
+        image: postgres:14
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -62,6 +74,7 @@ jobs:
         coverage run --source=$SOURCE_FOLDER -m pytest -rxXs
       env:
         SOURCE_FOLDER: app
+        ENV_FILE: .env.test
     - name: Send results to coveralls
       continue-on-error: true  # Ignore coveralls problems
       run: coveralls --service=github

README.md

@@ -39,7 +39,12 @@ pip install -r requirements/dev.txt
 pre-commit install -f
 cp .env.sample .env
 ```
-
+### Handle migrations
+This projects is using [Alembic](https://alembic.sqlalchemy.org/en/latest/) to manage database migrations.
+To create a new migration based on changes made to the model code, run the following command:
+```bash
+ alembic revision --autogenerate -m "MIGRATION TITLE"
+```
 
 ## Contributors
 [See contributors](https://github.com/safe-global/safe-decoder-service/graphs/contributors)

alembic.ini

@@ -0,0 +1,116 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts.
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Loaded from env to avoid duplications
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app/config.py

@@ -16,6 +16,7 @@ class Settings(BaseSettings):
     )
     REDIS_URL: str = "redis://"
     DATABASE_URL: str = "psql://postgres:"
+    DATABASE_POOL_CLASS: str = "AsyncAdaptedQueuePool"
 
 
 settings = Settings()

app/datasources/db/database.py

@@ -0,0 +1,45 @@
+import logging
+from functools import wraps
+
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import AsyncAdaptedQueuePool, NullPool
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.config import settings
+
+pool_classes = {
+    NullPool.__name__: NullPool,
+    AsyncAdaptedQueuePool.__name__: AsyncAdaptedQueuePool,
+}
+
+engine = create_async_engine(
+    settings.DATABASE_URL,
+    echo=True,
+    future=True,
+    poolclass=pool_classes.get(settings.DATABASE_POOL_CLASS),
+)
+
+
+def get_database_session(func):
+    """
+    Decorator that creates a new database session for the given function
+
+    :param func:
+    :return:
+    """
+
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        async with AsyncSession(engine) as session:
+            try:
+                return await func(*args, **kwargs, session=session)
+            except Exception as e:
+                # Rollback errors
+                await session.rollback()
+                logging.error(f"Error occurred: {e}")
+                raise
+            finally:
+                # Ensure that session is closed
+                await session.close()
+
+    return wrapper

app/datasources/db/models.py

@@ -0,0 +1,9 @@
+from typing import Optional
+
+from sqlmodel import Field, SQLModel
+
+
+class Contract(SQLModel, table=True):
+    address: bytes = Field(nullable=False, primary_key=True)
+    name: str = Field(nullable=False)
+    description: Optional[str] = None

app/main.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, FastAPI
 
 from . import VERSION
-from .routers import about, default
+from .routers import about, contracts, default
 
 app = FastAPI(
     title="Safe Decoder Service",
@@ -16,5 +16,6 @@
     prefix="/api/v1",
 )
 api_v1_router.include_router(about.router)
+api_v1_router.include_router(contracts.router)
 app.include_router(api_v1_router)
 app.include_router(default.router)

app/routers/contracts.py

@@ -0,0 +1,17 @@
+from typing import Sequence
+
+from fastapi import APIRouter
+
+from ..datasources.db.models import Contract
+from ..services.contract import ContractService
+
+router = APIRouter(
+    prefix="/contracts",
+    tags=["Contracts"],
+)
+
+
+@router.get("", response_model=Sequence[Contract])
+async def list_contracts() -> Sequence[Contract]:
+    contract_service = ContractService()
+    return await contract_service.get_all()

app/services/contract.py

@@ -0,0 +1,36 @@
+from typing import Sequence
+
+from sqlmodel import select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.datasources.db.database import get_database_session
+from app.datasources.db.models import Contract
+
+
+class ContractService:
+
+    @staticmethod
+    @get_database_session
+    async def get_all(session: AsyncSession) -> Sequence[Contract]:
+        """
+        Get all contracts
+
+        :param session: passed by the decorator
+        :return:
+        """
+        result = await session.exec(select(Contract))
+        return result.all()
+
+    @staticmethod
+    @get_database_session
+    async def create(contract: Contract, session: AsyncSession) -> Contract:
+        """
+        Create a new contract
+
+        :param contract:
+        :param session:
+        :return:
+        """
+        session.add(contract)
+        await session.commit()
+        return contract

app/tests/db/db_async_conn.py

@@ -0,0 +1,22 @@
+import unittest
+
+from sqlmodel import SQLModel
+
+from app.datasources.db.database import engine
+
+
+class DbAsyncConn(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self):
+        self.engine = engine
+        # Create the database tables
+        async with self.engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.create_all)
+
+    async def asyncTearDown(self):
+        """
+        Clean data between tests
+
+        :return:
+        """
+        async with self.engine.begin() as conn:
+            await conn.run_sync(SQLModel.metadata.drop_all)

app/tests/db/test_model.py

@@ -0,0 +1,17 @@
+from sqlmodel import select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.datasources.db.database import get_database_session
+from app.datasources.db.models import Contract
+from app.tests.db.db_async_conn import DbAsyncConn
+
+
+class TestModel(DbAsyncConn):
+    @get_database_session
+    async def test_contract(self, session: AsyncSession):
+        contract = Contract(address=b"a", name="A Test Contracts")
+        session.add(contract)
+        await session.commit()
+        statement = select(Contract).where(Contract.address == b"a")
+        result = await session.exec(statement)
+        self.assertEqual(result.one(), contract)

app/tests/routers/test_contracts.py

@@ -0,0 +1,26 @@
+from fastapi.testclient import TestClient
+
+from ...datasources.db.models import Contract
+from ...main import app
+from ...services.contract import ContractService
+from ..db.db_async_conn import DbAsyncConn
+
+
+class TestRouterContract(DbAsyncConn):
+    client: TestClient
+
+    @classmethod
+    def setUpClass(cls):
+        cls.client = TestClient(app)
+
+    async def test_view_contracts(self):
+        contract = Contract(address=b"a", name="A Test Contracts")
+        expected_response = {
+            "name": "A Test Contracts",
+            "description": None,
+            "address": "a",
+        }
+        await ContractService.create(contract=contract)
+        response = self.client.get("/api/v1/contracts")
+        self.assertEqual(response.status_code, 200)
+        self.assertDictEqual(response.json()[0], expected_response)

docker/web/run_web.sh

@@ -7,5 +7,7 @@ DOCKER_SHARED_DIR=/nginx
 rm -rf $DOCKER_SHARED_DIR/*
 cp -r static/ $DOCKER_SHARED_DIR/
 
+echo "==> $(date +%H:%M:%S) ==> Running migrations..."
+alembic upgrade head
 echo "==> $(date +%H:%M:%S) ==> Running Uvicorn... "
 exec uvicorn app.main:app --host 0.0.0.0 --port 8888 --proxy-headers --uds $DOCKER_SHARED_DIR/uvicorn.socket

migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration with an async dbapi.