feat: move rrweb event stream to client and query through /api/clickhouse-proxy (#755)

Author: knudtty

.changeset/neat-badgers-matter.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: move rrweb event fetching to the client instead of an api route

packages/api/src/api-app.ts

@@ -89,7 +89,6 @@ app.use('/', routers.rootRouter);
 app.use('/alerts', isUserAuthenticated, routers.alertsRouter);
 app.use('/dashboards', isUserAuthenticated, routers.dashboardRouter);
 app.use('/me', isUserAuthenticated, routers.meRouter);
-app.use('/sessions', isUserAuthenticated, routers.sessionsRouter);
 app.use('/team', isUserAuthenticated, routers.teamRouter);
 app.use('/webhooks', isUserAuthenticated, routers.webhooksRouter);
 app.use('/datasources', isUserAuthenticated, routers.datasourceRouter);

packages/api/src/middleware/validation.ts

@@ -0,0 +1,17 @@
+import express from 'express';
+import { z } from 'zod';
+
+export function validateRequestHeaders<T extends z.Schema>(schema: T) {
+  return function (
+    req: express.Request,
+    res: express.Response,
+    next: express.NextFunction,
+  ) {
+    const parsed = schema.safeParse(req.headers);
+    if (!parsed.success) {
+      return res.status(400).json({ type: 'Headers', errors: parsed.error });
+    }
+
+    return next();
+  };
+}

packages/api/src/routers/api/clickhouseProxy.ts

@@ -1,10 +1,11 @@
-import express, { Request, Response } from 'express';
+import express, { RequestHandler, Response } from 'express';
 import { createProxyMiddleware } from 'http-proxy-middleware';
 import { z } from 'zod';
 import { validateRequest } from 'zod-express-middleware';
 
 import { getConnectionById } from '@/controllers/connection';
 import { getNonNullUserWithTeam } from '@/middleware/auth';
+import { validateRequestHeaders } from '@/middleware/validation';
 import { objectIdSchema } from '@/utils/zod';
 
 const router = express.Router();
@@ -58,17 +59,22 @@ router.post(
   },
 );
 
-router.get(
-  '/*',
-  validateRequest({
-    query: z.object({
-      hyperdx_connection_id: objectIdSchema,
-    }),
+const hasConnectionId = validateRequestHeaders(
+  z.object({
+    'x-hyperdx-connection-id': objectIdSchema,
   }),
+);
+
+const getConnection: RequestHandler =
+  // prettier-ignore-next-line
   async (req, res, next) => {
     try {
       const { teamId } = getNonNullUserWithTeam(req);
-      const { hyperdx_connection_id } = req.query;
+      const connection_id = req.headers['x-hyperdx-connection-id']!; // ! because zod already validated
+      delete req.headers['x-hyperdx-connection-id'];
+      const hyperdx_connection_id = Array.isArray(connection_id)
+        ? connection_id.join('')
+        : connection_id;
 
       const connection = await getConnectionById(
         teamId.toString(),
@@ -93,13 +99,15 @@ router.get(
       console.error('Error fetching connection info:', e);
       next(e);
     }
-  },
+  };
+
+const proxyMiddleware: RequestHandler =
+  // prettier-ignore-next-line
   createProxyMiddleware({
     target: '', // doesn't matter. it should be overridden by the router
     changeOrigin: true,
     pathFilter: (path, _req) => {
-      // TODO: allow other methods
-      return _req.method === 'GET';
+      return _req.method === 'GET' || _req.method === 'POST';
     },
     pathRewrite: {
       '^/clickhouse-proxy': '',
@@ -113,16 +121,20 @@ router.get(
     on: {
       proxyReq: (proxyReq, _req) => {
         const newPath = _req.params[0];
+        // @ts-expect-error _req.query is type ParamQs, which doesn't play nicely with URLSearchParams. TODO: Replace with getting query params from _req.url eventually
         const qparams = new URLSearchParams(_req.query);
-        qparams.delete('hyperdx_connection_id');
         if (_req._hdx_connection?.username && _req._hdx_connection?.password) {
           proxyReq.setHeader(
             'X-ClickHouse-User',
             _req._hdx_connection.username,
           );
           proxyReq.setHeader('X-ClickHouse-Key', _req._hdx_connection.password);
         }
-        proxyReq.path = `/${newPath}?${qparams.toString()}`;
+        if (_req.method === 'POST') {
+          // TODO: Use fixRequestBody after this issue is resolved: https://github.com/chimurai/http-proxy-middleware/issues/1102
+          proxyReq.write(_req.body);
+        }
+        proxyReq.path = `/${newPath}?${qparams}`;
       },
       proxyRes: (proxyRes, _req, res) => {
         // since clickhouse v24, the cors headers * will be attached to the response by default
@@ -158,7 +170,9 @@ router.get(
     // ...(config.IS_DEV && {
     //   logger: console,
     // }),
-  }),
-);
+  });
+
+router.get('/*', hasConnectionId, getConnection, proxyMiddleware);
+router.post('/*', hasConnectionId, getConnection, proxyMiddleware);
 
 export default router;

packages/api/src/routers/api/index.ts

@@ -3,7 +3,6 @@ import dashboardRouter from './dashboards';
 import datasourceRouter from './datasources';
 import meRouter from './me';
 import rootRouter from './root';
-import sessionsRouter from './sessions';
 import teamRouter from './team';
 import webhooksRouter from './webhooks';
 
@@ -13,7 +12,6 @@ export default {
   dashboardRouter,
   meRouter,
   rootRouter,
-  sessionsRouter,
   teamRouter,
   webhooksRouter,
 };

packages/api/src/routers/api/sessions.ts

@@ -6,7 +6,7 @@ const DEFAULT_SERVER_URL = `http://127.0.0.1:${process.env.HYPERDX_API_PORT}`;
 export const config = {
   api: {
     externalResolver: true,
-    bodyParser: true,
+    bodyParser: false,
   },
 };
 
@@ -17,12 +17,6 @@ export default (req: NextApiRequest, res: NextApiResponse) => {
     pathRewrite: { '^/api': '' },
     target: process.env.NEXT_PUBLIC_SERVER_URL || DEFAULT_SERVER_URL,
     autoRewrite: true,
-    /**
-     * Fix bodyParser
-     **/
-    on: {
-      proxyReq: fixRequestBody,
-    },
     // ...(IS_DEV && {
     //   logger: console,
     // }),