Split extension into an extension and a library package

.github/workflows/ci.yml

@@ -154,3 +154,33 @@ jobs:
         run: just pgai ci
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+  build-and-test-pgai-db-module:
+    needs: authorize
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # in a pull_request_target event, the ref is the `main` branch not the PR branch
+          # so we need to tell checkout to use the head.ref instead.
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+      - uses: taiki-e/install-action@just
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.5.20"
+          enable-cache: true
+          cache-dependency-glob: "./projects/pgai/uv.lock"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "./projects/pgai/.python-version"
+
+      - name: DB sub moduleCI pipeline. Install dependencies, run linters execute tests and build the project", 
+        run: just pgai db ci
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

.gitignore

@@ -4,12 +4,18 @@
 build
 *.egg-info
 __pycache__
+projects/pgai/db/sql/output/ai--*.sql
 projects/extension/sql/output/ai--*.sql
 projects/extension/tests/dump_restore/describe_objects.sql
 projects/extension/tests/dump_restore/describe_schemas.sql
 projects/extension/tests/dump_restore/dump.sql
 projects/extension/tests/dump_restore/src.snapshot
 projects/extension/tests/dump_restore/dst.snapshot
+projects/pgai/db/tests/dump_restore/describe_objects.sql
+projects/pgai/db/tests/dump_restore/describe_schemas.sql
+projects/pgai/db/tests/dump_restore/dump.sql
+projects/pgai/db/tests/dump_restore/src.snapshot
+projects/pgai/db/tests/dump_restore/dst.snapshot
 projects/extension/tests/*/*.actual
 projects/extension/tests/upgrade/*.snapshot
 dist

projects/extension/Dockerfile

@@ -57,6 +57,19 @@ RUN mkdir -p /docker-entrypoint-initdb.d && \
     echo "echo \"shared_preload_libraries = 'timescaledb'\" >> \${PGDATA}/postgresql.conf" >> /docker-entrypoint-initdb.d/configure-timescaledb.sh && \
     chmod +x /docker-entrypoint-initdb.d/configure-timescaledb.sh
 
+###############################################################################
+# image for use in pgai-lib-db development
+FROM pgai-test-db as pgai-lib-db-dev
+ENV WHERE_AM_I=docker
+USER root
+
+RUN pip install --break-system-packages uv==0.6.3
+RUN mkdir /py/ && uv venv --directory /py/
+ENV PATH="/py/.venv/bin:$PATH"
+ENV VIRTUAL_ENV=/py/.venv
+
+WORKDIR /pgai/projects/pgai/db
+
 ###############################################################################
 # image for use in extension development
 FROM base
@@ -69,5 +82,6 @@ RUN pip install --break-system-packages uv==0.6.3
 COPY pyproject.toml uv.lock /py/
 RUN uv sync --directory /py --no-install-project --only-dev --frozen
 ENV PATH="/py/.venv/bin:$PATH"
+ENV VIRTUAL_ENV=/py/.venv
 
 WORKDIR /pgai/projects/extension

projects/extension/build.py

@@ -326,7 +326,7 @@ def test() -> None:
     def test_server() -> None:
         """runs the test http server in the docker container"""
         if where_am_i() == "host":
-            cmd = "docker exec -it -w /pgai/projects/extension/tests/vectorizer pgai-ext fastapi dev server.py"
+            cmd = "docker exec -it -w /pgai/projects/extension/tests pgai-ext fastapi dev server.py"
             subprocess.run(cmd, shell=True, check=True, env=os.environ, cwd=ext_dir())
         else:
             cmd = "uv run --no-project fastapi dev server.py"
@@ -335,7 +335,7 @@ def test_server() -> None:
                 shell=True,
                 check=True,
                 env=os.environ,
-                cwd=tests_dir().joinpath("vectorizer"),
+                cwd=tests_dir(),
             )
 
     @staticmethod
@@ -344,7 +344,6 @@ def lint_sql() -> None:
         cmd = " ".join(
             [
                 "uv run --no-project pgspot --ignore-lang=plpython3u",
-                '--proc-without-search-path "ai._vectorizer_job(job_id integer,config pg_catalog.jsonb)"',
                 f"{output_sql_file()}",
             ]
         )
@@ -471,6 +470,14 @@ def docker_run() -> None:
             ]
         )
         subprocess.run(cmd, shell=True, check=True, env=os.environ, text=True)
+        # install the pgai library in the container, needed to run the upgrade unpackaged tests
+        subprocess.run(
+            """docker exec pgai-ext uv pip install --editable /pgai/projects/pgai""",
+            shell=True,
+            check=True,
+            env=os.environ,
+            text=True,
+        )
 
     @staticmethod
     def docker_start() -> None:
@@ -664,8 +671,20 @@ def check_sql_file_order(path: Path, prev: int) -> int:
     kind = path.parent.name
     this = sql_file_number(path)
     # ensuring file number correlation
-    if this < 900 and this != prev + 1:
+
+    if this < 900 and this <= prev:
+        fatal(
+            f"{kind} sql files must not contain duplicate numbers. this: {this} prev: {prev}"
+        )
+
+    # strict order was relaxed during vectorizer divestment, leaving holes in the sequence
+    # so we need to handle those gaps
+    min_strict_order = 15
+    if kind == "incremental":
+        min_strict_order = 21
+    if this > min_strict_order and this < 900 and this != prev + 1:
         fatal(f"{kind} sql files must be strictly ordered. this: {this} prev: {prev}")
+
     # avoiding file number duplication
     if this >= 900 and this == prev:  # allow gaps in pre-production scripts
         fatal(

projects/extension/sql/head.sql

@@ -8,24 +8,21 @@ schema and migration table. abort the upgrade if different.
 do $bootstrap_extension$
 declare
     _current_user_id oid = null;
-    _schema_owner_id oid = null;
+    _schema_exists boolean = false;
     _migration_table_owner_id oid = null;
 begin
     select pg_catalog.to_regrole('@extowner@')::oid
     into strict _current_user_id;
 
-    select pg_namespace.nspowner into strict _schema_owner_id
+    select count(*) > 0 into strict _schema_exists
     from pg_catalog.pg_namespace
     where pg_namespace.nspname operator(pg_catalog.=) 'ai';
 
-    if _schema_owner_id is null then
+    if not _schema_exists then
         -- this should NEVER happen
         -- we have `schema=ai` in the control file, so postgres creates the schema automatically
         -- but this line makes pgspot happy
         create schema ai;
-    elseif _schema_owner_id operator(pg_catalog.!=) _current_user_id then
-        raise exception 'only the owner of the ai schema may install/upgrade this extension';
-        return;
     end if;
 
     select k.relowner into _migration_table_owner_id