API V1 with database access (#280)

* Adding db-v1 with util-v1 in tow

* impl list connectorConfig

* Partial listConnections

* Adding neon serverless

* Add test for neon driver against local neon proxy

* slight test reorg

* Adding jose for jwt generation

* Add generate and export keys

* expose jwts.json wellknown endpoint

* adding bun to dev

* fix types

* fix missing runtimeEnv

* Remove circular dep

* tp.ts localtest

* Add some db scripts

* Escape env pulled from vercel

* fix tests

* Unsuccessfully testing neon authorize

* Debug test

* fix env pull and test

* add health check

* Skip all neon test for now

* even more latency checks

* test out edge runtime

* Revert "test out edge runtime"

This reverts commit 6a1974b.

* proper latency check

* Get test passing again

* verified we can use neon non-interactive transaction for RLS

* neon serverless vs. neon http test

* Final approach via direct drizzle proxy

* Add neon proxy service to github action

* Add db-rls package

* add viewer into api-v1

* Add authentication test

* Split out test-utils

* Get rid of neon rls postgres variables

* Update lockfile

* Remove neon rls related changes

* Removing even more irrelevant stuff

* Upgrade drizzle version across the board

* Move neon test

* Separate entrypoint but single implementation multiple db implementations

* Add DrizzleExtension type to standardize difference across different drizzle postgres drivers

* Moving files around in db

* fix db/index test

* Add fallback generate_ulid using uuid since pgcrypto is not available in pglite

* Add ; --> statement-breakpoint to init sql file

* Add ; --> statement-breakpoint rest of files and get migration via pglite to pass finally

* Test migration with both pglite and pg

* Add dbFactory and additional extension on top of driver specific extensions

* Move truncate to after

* Extract db test utils

* test fetch handlers separately from elysia

* Finally got test for authorization logic working

* Extract into test-utils

* Fully removing db-v1 by incorporating it all into db

* Remove util-v1 also for simplicity

* Add event router

* fix typing

* Resolve issues

* Optionally enable extension to fix db/index test

* Increase memory allocation for linter and move it to the end

* Try to fix test

* Do more debugging

* Fix db.localtest

* debug

---------

Co-authored-by: Amadeo Pellicce <[email protected]>

Author: openint-bot

.github/workflows/validate-workflow.yml

@@ -38,11 +38,17 @@ jobs:
       #   image: inngest/inngest
       #   ports:
       #     - 8288:8288
+      neon-proxy:
+        image: ghcr.io/timowilhelm/local-neon-http-proxy:main
+        env:
+          PG_CONNECTION_STRING: postgres://postgres:[email protected]:5432/test
+        ports:
+          - '4444:4444'
 
     env:
-      NODE_OPTIONS: --max-old-space-size=4096
+      NODE_OPTIONS: --max-old-space-size=8192
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
-      DATABASE_URL: postgres://postgres:test@localhost:5432/test
+      DATABASE_URL: postgres://postgres:test@db.localtest.me:5432/test
 
     steps:
       - name: Checkout
@@ -51,7 +57,7 @@ jobs:
       - name: Install Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 20 # TODO: Can we get this from "engine" field in package.json?
+          node-version: 22 # TODO: Can we get this from "engine" field in package.json?
 
       - uses: pnpm/action-setup@v4
         name: Install pnpm
@@ -83,14 +89,12 @@ jobs:
       - name: Run type checks
         run: pnpm run typecheck
 
-      - name: Run lint
-        run: pnpm run lint
-
       - name: Download dev env vars from Vercel
         run: |
           npm install --global vercel@latest
           vercel link --token $VERCEL_TOKEN --scope openint-dev --yes
-          vercel env pull --token $VERCEL_TOKEN ./apps/web/.env.local
+          vercel env pull --token $VERCEL_TOKEN ./apps/web/.env.local.orig
+          cat ./apps/web/.env.local.orig | pnpm --silent bun scripts/escape-env.ts > ./apps/web/.env.local
 
       - name: Run migration and store the completed schema
         # TODO: consider diffing schema with stored schema?
@@ -129,9 +133,15 @@ jobs:
           npx inngest-cli@latest dev &
           pnpm wait-on tcp:8288
 
+      - name: Setup upterm session
+        uses: lhotari/action-upterm@v1
+
       - name: Run tests
         run: ./bin/shdotenv -q -e ./apps/web/.env.local pnpm run test::ci
 
+      - name: Run lint
+        run: pnpm run lint
+
       - name: Send Slack notification for job status
         uses: 8398a7/action-slack@v3
         with:

apps/web/package.json

@@ -65,7 +65,7 @@
     "@types/react-dom": "18.0.10",
     "@types/swagger2openapi": "7.0.0",
     "autoprefixer": "*",
-    "drizzle-orm": "0.38.2",
+    "drizzle-orm": "^0.39.1",
     "jiti": "*",
     "node-loader": "2.0.0",
     "postcss": "^8.4.47",

bin/tp.ts

@@ -84,7 +84,11 @@ const name = [label, `${url.username}@${url.hostname}${port ? `:${port}` : ''}`]
 
 const {label: _, ...options} = args.values
 
-if (url.hostname === 'localhost' || url.hostname === '127.0.0.1') {
+if (
+  url.hostname === 'localhost' ||
+  url.hostname === '127.0.0.1' ||
+  url.hostname.includes('localtest.me')
+) {
   options.env = options.env || 'local'
   options.statusColor = options.statusColor || '007F3D'
   options.safeModeLevel = '0'

connectors/connector-postgres/package.json

@@ -7,8 +7,8 @@
     "@openint/cdk": "workspace:*",
     "@openint/db": "workspace:*",
     "@openint/util": "workspace:*",
-    "drizzle-kit": "^0.30.1",
-    "drizzle-orm": "^0.38.2",
+    "drizzle-kit": "^0.30.4",
+    "drizzle-orm": "^0.39.1",
     "handlebars": "4.7.7",
     "json-stable-stringify": "^1.1.1",
     "postgres": "^3.4.5",

docker-compose.yml

@@ -12,6 +12,11 @@ services:
       POSTGRES_DB: postgres # Only database named `postgres` works with pg_cron by default
       POSTGRES_PASSWORD: password
     command: ["postgres", "-c", "log_statement=all", "-c", "max_connections=500"]
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   supabase:
     image: supabase/postgres:15.8.1.017 # For some reason "lastest tag" is having issues... so we pin to a specific version latest as of 2024-12-20_0055
@@ -24,7 +29,17 @@ services:
       POSTGRES_PASSWORD: password
 
   inngest:
-      image: inngest/inngest
-      command: 'inngest dev'
-      ports:
-        - '8288:8288'
+    image: inngest/inngest
+    command: 'inngest dev'
+    ports:
+      - '8288:8288'
+
+  neon-proxy:
+    image: ghcr.io/timowilhelm/local-neon-http-proxy:main
+    environment:
+      PG_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres
+    ports:
+      - '4444:4444'
+    depends_on:
+      postgres:
+        condition: service_healthy

kits/cdk/viewer.ts

@@ -101,7 +101,7 @@ export function getExtCustomerId(
  */
 export const zJwtPayload = z.object({
   /** Different meaning in different contexts */
-  sub: z.string(),
+  sub: z.string().nullish(),
   /**
    * Jwt role is different from viewer role because supabase uses `authenticated`
    * by default and it's a bit too much work right now to switch to `user` so we shall
@@ -134,7 +134,7 @@ export const zViewerFromJwtPayload = zJwtPayload
         }
       // good reason to rename customer to customer
       case 'customer': {
-        const [orgId, customerId] = payload.sub.split('/') as [
+        const [orgId, customerId] = (payload.sub?.split('/') ?? []) as [
           Id['org'],
           CustomerId,
         ]

package.json

@@ -21,10 +21,13 @@
     "validate": "run-s --silent typecheck lint test",
     "openint": "tsx ./bin/openint",
     "migrate": "pnpm --dir ./packages/db migrate",
-    "pgdump": "pg_dump --schema public --schema-only --no-owner --exclude-schema=graphile_migrate --file=packages/db/schema.sql $DATABASE_URL",
+    "pgdump": "pg_dump --schema public --schema-only --no-owner --exclude-schema=graphile_migrate --file=packages/db/__generated__/schema.sql $DATABASE_URL",
     "worker:setup": "tsx ./bin/openint setupWorker",
     "worker:run": "tsx ./bin/openint runWorker",
-    "env:pull": "vercel env pull --environment development .env.dev && vercel env pull --environment preview --git-branch $(git rev-parse --abbrev-ref HEAD) .env.pre && vercel env pull --environment production .env.prod"
+    "env:pull:development": "vercel env pull --environment development .env.dev.orig && cat .env.dev.orig | pnpm --silent bun scripts/escape-env.ts > .env.dev",
+    "env:pull:preview": "vercel env pull --environment preview --git-branch $(git rev-parse --abbrev-ref HEAD) .env.pre.orig && cat .env.pre.orig | pnpm --silent bun scripts/escape-env.ts > .env.pre",
+    "env:pull:production": "vercel env pull --environment production .env.prod.orig && cat .env.prod.orig | pnpm --silent bun scripts/escape-env.ts > .env.prod",
+    "env:pull": "run-s env:pull:*"
   },
   "lint-staged": {
     "**/*.{js,ts,tsx,json,css,yml,yaml}": "prettier --write",
@@ -45,6 +48,7 @@
     "@types/prettier": "3.0.0",
     "@typescript-eslint/eslint-plugin": "6.21.0",
     "@typescript-eslint/parser": "6.21.0",
+    "bun": "latest",
     "esbuild": "0.17.5",
     "esbuild-jest": "0.5.0",
     "eslint": "8.23.0",