Fix for user groups adding users to app user metadata tables

.github/workflows/budibase_ci.yml

@@ -56,11 +56,11 @@ jobs:
         run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn
       - run: yarn bootstrap
+      - run: yarn build:client
       - run: yarn test
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-          files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
           name: codecov-umbrella
           verbose: true
 

package.json

@@ -25,6 +25,7 @@
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
     "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
+    "build:client": "lerna run build --ignore @budibase/backend-core --ignore @budibase/worker --ignore @budibase/server --ignore @budibase/builder --ignore @budibase/cli --ignore @budibase/sdk",
     "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
     "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli",
     "build:sdk": "lerna run build:sdk",

packages/backend-core/scripts/test.sh

@@ -3,10 +3,10 @@
 if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
-  echo "jest --coverage --runInBand"
-  jest --coverage --runInBand
+  echo "jest --coverage --runInBand --forceExit"
+  jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
   echo "jest --coverage"
   jest --coverage
-fi
+fi

packages/backend-core/src/index.ts

@@ -24,6 +24,7 @@ export * as redis from "./redis"
 export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
+export * as timers from "./timers"
 export { default as env } from "./environment"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility

packages/backend-core/src/queue/queue.ts

@@ -4,6 +4,7 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
+import * as timers from "../timers"
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -29,8 +30,8 @@ export function createQueue<T>(
   }
   addListeners(queue, jobQueue, opts?.removeStalledCb)
   QUEUES.push(queue)
-  if (!cleanupInterval) {
-    cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS)
+  if (!cleanupInterval && !env.isTest()) {
+    cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
     // fire off an initial cleanup
     cleanup().catch(err => {
       console.error(`Unable to cleanup automation queue initially - ${err}`)
@@ -41,7 +42,7 @@ export function createQueue<T>(
 
 export async function shutdown() {
   if (cleanupInterval) {
-    clearInterval(cleanupInterval)
+    timers.clear(cleanupInterval)
   }
   if (QUEUES.length) {
     for (let queue of QUEUES) {

packages/backend-core/src/redis/redis.ts

@@ -8,6 +8,7 @@ import {
   SEPARATOR,
   SelectableDatabase,
 } from "./utils"
+import * as timers from "../timers"
 
 const RETRY_PERIOD_MS = 2000
 const STARTUP_TIMEOUT_MS = 5000
@@ -117,9 +118,9 @@ function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
       return
     }
     // check if the connection is ready
-    const interval = setInterval(() => {
+    const interval = timers.set(() => {
       if (CONNECTED) {
-        clearInterval(interval)
+        timers.clear(interval)
         resolve("")
       }
     }, 500)

packages/backend-core/src/timers/index.ts

@@ -0,0 +1 @@
+export * from "./timers"

packages/backend-core/src/timers/timers.ts

@@ -0,0 +1,22 @@
+let intervals: NodeJS.Timeout[] = []
+
+export function set(callback: () => any, period: number) {
+  const interval = setInterval(callback, period)
+  intervals.push(interval)
+  return interval
+}
+
+export function clear(interval: NodeJS.Timeout) {
+  const idx = intervals.indexOf(interval)
+  if (idx !== -1) {
+    intervals.splice(idx, 1)
+  }
+  clearInterval(interval)
+}
+
+export function cleanup() {
+  for (let interval of intervals) {
+    clearInterval(interval)
+  }
+  intervals = []
+}

packages/backend-core/tests/jestEnv.ts

@@ -4,3 +4,4 @@ process.env.NODE_ENV = "jest"
 process.env.MOCK_REDIS = "1"
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.ENABLE_4XX_HTTP_LOGGING = "0"
+process.env.REDIS_PASSWORD = "budibase"

packages/backend-core/tests/jestSetup.ts

@@ -1,5 +1,6 @@
 import "./logging"
 import env from "../src/environment"
+import { cleanup } from "../src/timers"
 import { mocks, testContainerUtils } from "./utilities"
 
 // must explicitly enable fetch mock
@@ -21,3 +22,7 @@ if (!process.env.CI) {
 }
 
 testContainerUtils.setupEnv(env)
+
+afterAll(() => {
+  cleanup()
+})

packages/server/__mocks__/node-fetch.ts

@@ -4,6 +4,7 @@ module FetchMock {
   // @ts-ignore
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
+  let mockSearch = false
 
   const func = async (url: any, opts: any) => {
     function json(body: any, status = 200) {
@@ -69,7 +70,7 @@ module FetchMock {
         },
         404
       )
-    } else if (url.includes("_search")) {
+    } else if (mockSearch && url.includes("_search")) {
       const body = opts.body
       const parts = body.split("tableId:")
       let tableId
@@ -192,5 +193,9 @@ module FetchMock {
 
   func.Headers = fetch.Headers
 
+  func.mockSearch = () => {
+    mockSearch = true
+  }
+
   module.exports = func
 }

packages/server/jest.config.ts

@@ -43,6 +43,7 @@ const config: Config.InitialOptions = {
     "../backend-core/src/**/*.{js,ts}",
     // The use of coverage with couchdb view functions breaks tests
     "!src/db/views/staticViews.*",
+    "!src/**/*.spec.{js,ts}",
   ],
   coverageReporters: ["lcov", "json", "clover"],
 }

packages/server/package.json

@@ -14,7 +14,8 @@
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
-    "test": "bash scripts/test.sh",
+    "test": "NODE_OPTIONS=\"--max-old-space-size=4096\" bash scripts/test.sh",
+    "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
@@ -125,7 +126,7 @@
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
     "@budibase/standard-components": "^0.9.139",
-    "@jest/test-sequencer": "24.9.0",
+    "@jest/test-sequencer": "29.5.0",
     "@swc/core": "^1.3.25",
     "@swc/jest": "^0.2.24",
     "@trendyol/jest-testcontainers": "^2.1.1",
@@ -134,7 +135,7 @@
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/ioredis": "4.28.10",
-    "@types/jest": "27.5.1",
+    "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.180",
@@ -154,7 +155,7 @@
     "eslint": "6.8.0",
     "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
-    "jest": "28.1.1",
+    "jest": "29.5.0",
     "jest-openapi": "0.14.2",
     "jest-serial-runner": "^1.2.1",
     "nodemon": "2.0.15",
@@ -166,7 +167,7 @@
     "supertest": "6.2.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
-    "ts-jest": "28.0.4",
+    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",

packages/server/scripts/test.sh

@@ -3,10 +3,10 @@
 if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
-  echo "jest --coverage --runInBand"
-  jest --coverage --runInBand
+  echo "jest --coverage --runInBand --forceExit"
+  jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2"
   jest --coverage --maxWorkers=2
-fi
+fi

packages/server/src/api/controllers/application.ts

@@ -44,7 +44,6 @@ import {
   Layout,
   Screen,
   MigrationType,
-  BBContext,
   Database,
   UserCtx,
 } from "@budibase/types"
@@ -74,14 +73,14 @@ async function getScreens() {
   ).rows.map((row: any) => row.doc)
 }
 
-function getUserRoleId(ctx: BBContext) {
+function getUserRoleId(ctx: UserCtx) {
   return !ctx.user?.role || !ctx.user.role._id
     ? roles.BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
 function checkAppUrl(
-  ctx: BBContext,
+  ctx: UserCtx,
   apps: App[],
   url: string,
   currentAppId?: string
@@ -95,7 +94,7 @@ function checkAppUrl(
 }
 
 function checkAppName(
-  ctx: BBContext,
+  ctx: UserCtx,
   apps: App[],
   name: string,
   currentAppId?: string
@@ -160,7 +159,7 @@ async function addDefaultTables(db: Database) {
   await db.bulkDocs([...defaultDbDocs])
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = (await dbCore.getAllApps({ dev, all })) as App[]
@@ -185,7 +184,7 @@ export async function fetch(ctx: BBContext) {
   ctx.body = await checkAppMetadata(apps)
 }
 
-export async function fetchAppDefinition(ctx: BBContext) {
+export async function fetchAppDefinition(ctx: UserCtx) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
   const accessController = new roles.AccessController()
@@ -231,7 +230,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
   }
 }
 
-async function performAppCreate(ctx: BBContext) {
+async function performAppCreate(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   const name = ctx.request.body.name,
     possibleUrl = ctx.request.body.url
@@ -360,7 +359,7 @@ async function creationEvents(request: any, app: App) {
   }
 }
 
-async function appPostCreate(ctx: BBContext, app: App) {
+async function appPostCreate(ctx: UserCtx, app: App) {
   const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
@@ -391,7 +390,7 @@ async function appPostCreate(ctx: BBContext, app: App) {
   }
 }
 
-export async function create(ctx: BBContext) {
+export async function create(ctx: UserCtx) {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
   await cache.bustCache(cache.CacheKey.CHECKLIST)
@@ -401,7 +400,7 @@ export async function create(ctx: BBContext) {
 
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation
   const name = ctx.request.body.name,
@@ -421,7 +420,7 @@ export async function update(ctx: BBContext) {
   ctx.body = app
 }
 
-export async function updateClient(ctx: BBContext) {
+export async function updateClient(ctx: UserCtx) {
   // Get current app version
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -445,7 +444,7 @@ export async function updateClient(ctx: BBContext) {
   ctx.body = app
 }
 
-export async function revertClient(ctx: BBContext) {
+export async function revertClient(ctx: UserCtx) {
   // Check app can be reverted
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -471,7 +470,7 @@ export async function revertClient(ctx: BBContext) {
   ctx.body = app
 }
 
-const unpublishApp = async (ctx: any) => {
+async function unpublishApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
 
@@ -487,7 +486,7 @@ const unpublishApp = async (ctx: any) => {
   return result
 }
 
-async function destroyApp(ctx: BBContext) {
+async function destroyApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
   const devAppId = dbCore.getDevAppID(appId)
@@ -515,28 +514,28 @@ async function destroyApp(ctx: BBContext) {
   return result
 }
 
-async function preDestroyApp(ctx: BBContext) {
+async function preDestroyApp(ctx: UserCtx) {
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
 
-async function postDestroyApp(ctx: BBContext) {
+async function postDestroyApp(ctx: UserCtx) {
   const rowCount = ctx.rowCount
   await groups.cleanupApp(ctx.params.appId)
   if (rowCount) {
     await quotas.removeRows(rowCount)
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   await preDestroyApp(ctx)
   const result = await destroyApp(ctx)
   await postDestroyApp(ctx)
   ctx.status = 200
   ctx.body = result
 }
 
-export const unpublish = async (ctx: BBContext) => {
+export async function unpublish(ctx: UserCtx) {
   const prodAppId = dbCore.getProdAppID(ctx.params.appId)
   const dbExists = await dbCore.dbExists(prodAppId)
 
@@ -551,7 +550,7 @@ export const unpublish = async (ctx: BBContext) => {
   ctx.status = 204
 }
 
-export async function sync(ctx: BBContext) {
+export async function sync(ctx: UserCtx) {
   const appId = ctx.params.appId
   try {
     ctx.body = await sdk.applications.syncApp(appId)