Replay redirect if RSC parameter is missing

If the server responds with a redirect (e.g. 307), and the redirected
location does not contain the cache busting search param set in the
original request, the response is likely invalid — when following
the redirect, the browser will have forwarded the request headers, but
without a corresponding cache busting search param. (See vercel#79563
for more context.)

Ideally, we would be able to intercept the redirect response and perform
it manually, instead of letting the browser automatically follow it, but
this is not allowed by the fetch API.

So instead, we must "replay" the redirect by fetching the new location
again, but this time we'll append the cache busting search param to
prevent a mismatch.

We can optimize Next.js's built-in middleware APIs by returning a custom
status code, to prevent the browser from automatically following it.
This will land in future PRs. Third-party proxies can choose to
implement this same protocol for cases where they need to redirect
Next.js requests but are not using Next.js APIs.

However, we'll still need this fallback behavior for third-party proxies
that don't implement our optimized redirect protocol.

This does not affect Server Action-based redirects; those are encoded
differently, as part of the Flight body.

As of now, this is gated behind the experimental
`validateRSCRequestHeaders` flag. We will test it before turning it on
by default.

Author: acdlite

packages/next/src/build/define-env.ts

@@ -187,6 +187,9 @@ export function getDefineEnv({
     'process.env.__NEXT_CLIENT_SEGMENT_CACHE': Boolean(
       config.experimental.clientSegmentCache
     ),
+    'process.env.__NEXT_CLIENT_VALIDATE_RSC_REQUEST_HEADERS': Boolean(
+      config.experimental.validateRSCRequestHeaders
+    ),
     'process.env.__NEXT_DYNAMIC_ON_HOVER': Boolean(
       config.experimental.dynamicOnHover
     ),

packages/next/src/client/components/router-reducer/fetch-server-response.ts

@@ -267,18 +267,29 @@ export async function fetchServerResponse(
   }
 }
 
-export function createFetch(
+// This is a subset of the standard Response type. We use a custom type for
+// this so we can limit which details about the response leak into the rest of
+// the codebase. For example, there's some custom logic for manually following
+// redirects, so "redirected" in this type could be a composite of multiple
+// browser fetch calls; however, this fact should not leak to the caller.
+export type RSCResponse = {
+  ok: boolean
+  redirected: boolean
+  headers: Headers
+  body: ReadableStream<Uint8Array> | null
+  status: number
+  url: string
+}
+
+export async function createFetch(
   url: URL,
   headers: RequestHeaders,
   fetchPriority: 'auto' | 'high' | 'low' | null,
   signal?: AbortSignal
-) {
-  const fetchUrl = new URL(url)
-
+): Promise<RSCResponse> {
   // TODO: In output: "export" mode, the headers do nothing. Omit them (and the
   // cache busting search param) from the request so they're
   // maximally cacheable.
-  setCacheBustingSearchParam(fetchUrl, headers)
 
   if (process.env.__NEXT_TEST_MODE && fetchPriority !== null) {
     headers['Next-Test-Fetch-Priority'] = fetchPriority
@@ -288,13 +299,103 @@ export function createFetch(
     headers['x-deployment-id'] = process.env.NEXT_DEPLOYMENT_ID
   }
 
-  return fetch(fetchUrl, {
+  const fetchOptions: RequestInit = {
     // Backwards compat for older browsers. `same-origin` is the default in modern browsers.
     credentials: 'same-origin',
     headers,
     priority: fetchPriority || undefined,
     signal,
-  })
+  }
+  // `fetchUrl` is slightly different from `url` because we add a cache-busting
+  // search param to it. This should not leak outside of this function, so we
+  // track them separately.
+  let fetchUrl = new URL(url)
+  setCacheBustingSearchParam(fetchUrl, headers)
+  let browserResponse = await fetch(fetchUrl, fetchOptions)
+
+  // If the server responds with a redirect (e.g. 307), and the redirected
+  // location does not contain the cache busting search param set in the
+  // original request, the response is likely invalid — when following the
+  // redirect, the browser forwards the request headers, but since the cache
+  // busting search param is missing, the server will reject the request due to
+  // a mismatch.
+  //
+  // Ideally, we would be able to intercept the redirect response and perform it
+  // manually, instead of letting the browser automatically follow it, but this
+  // is not allowed by the fetch API.
+  //
+  // So instead, we must "replay" the redirect by fetching the new location
+  // again, but this time we'll append the cache busting search param to prevent
+  // a mismatch.
+  //
+  // TODO: We can optimize Next.js's built-in middleware APIs by returning a
+  // custom status code, to prevent the browser from automatically following it.
+  //
+  // This does not affect Server Action-based redirects; those are encoded
+  // differently, as part of the Flight body. It only affects redirects that
+  // occur in a middleware or a third-party proxy.
+
+  let redirected = browserResponse.redirected
+  if (process.env.__NEXT_CLIENT_VALIDATE_RSC_REQUEST_HEADERS) {
+    // This is to prevent a redirect loop. Same limit used by Chrome.
+    const MAX_REDIRECTS = 20
+    for (let n = 0; n < MAX_REDIRECTS; n++) {
+      if (!browserResponse.redirected) {
+        // The server did not perform a redirect.
+        break
+      }
+      const responseUrl = new URL(browserResponse.url, fetchUrl)
+      if (responseUrl.origin !== fetchUrl.origin) {
+        // The server redirected to an external URL. The rest of the logic below
+        // is not relevant, because it only applies to internal redirects.
+        break
+      }
+      if (
+        responseUrl.searchParams.get(NEXT_RSC_UNION_QUERY) ===
+        fetchUrl.searchParams.get(NEXT_RSC_UNION_QUERY)
+      ) {
+        // The redirected URL already includes the cache busting search param.
+        // This was probably intentional. Regardless, there's no reason to
+        // issue another request to this URL because it already has the param
+        // value that we would have added below.
+        break
+      }
+      // The RSC request was redirected. Assume the response is invalid.
+      //
+      // Append the cache busting search param to the redirected URL and
+      // fetch again.
+      fetchUrl = new URL(responseUrl)
+      setCacheBustingSearchParam(fetchUrl, headers)
+      browserResponse = await fetch(fetchUrl, fetchOptions)
+      // We just performed a manual redirect, so this is now true.
+      redirected = true
+    }
+  }
+
+  // Remove the cache busting search param from the response URL, to prevent it
+  // from leaking outside of this function.
+  const responseUrl = new URL(browserResponse.url, fetchUrl)
+  responseUrl.searchParams.delete(NEXT_RSC_UNION_QUERY)
+
+  const rscResponse: RSCResponse = {
+    url: responseUrl.href,
+
+    // This is true if any redirects occurred, either automatically by the
+    // browser, or manually by us. So it's different from
+    // `browserResponse.redirected`, which only tells us whether the browser
+    // followed a redirect, and only for the last response in the chain.
+    redirected,
+
+    // These can be copied from the last browser response we received. We
+    // intentionally only expose the subset of fields that are actually used
+    // elsewhere in the codebase.
+    ok: browserResponse.ok,
+    headers: browserResponse.headers,
+    body: browserResponse.body,
+    status: browserResponse.status,
+  }
+
+  return rscResponse
 }
 
 export function createFromNextReadableStream(

packages/next/src/client/components/segment-cache-impl/cache.ts

@@ -25,6 +25,7 @@ import {
 import {
   createFetch,
   createFromNextReadableStream,
+  type RSCResponse,
   type RequestHeaders,
 } from '../router-reducer/fetch-server-response'
 import {
@@ -997,6 +998,8 @@ export async function fetchRouteOnCacheMiss(
   }
 
   // In output: "export" mode, we need to add the segment path to the URL.
+  // TODO: Consider moving this to `createFetch`, where we do similar logic for
+  // manipulating the request URL to encode extra information.
   const url = new URL(href)
   const requestUrl = isOutputExportMode
     ? addSegmentPathToUrlInOutputExportMode(url, segmentPath)
@@ -1372,7 +1375,7 @@ export async function fetchSegmentPrefetchesUsingDynamicRequest(
 function writeDynamicTreeResponseIntoCache(
   now: number,
   task: PrefetchTask,
-  response: Response,
+  response: RSCResponse,
   serverData: NavigationFlightResponse,
   entry: PendingRouteCacheEntry,
   couldBeIntercepted: boolean,
@@ -1462,7 +1465,7 @@ function rejectSegmentEntriesIfStillPending(
 function writeDynamicRenderResponseIntoCache(
   now: number,
   task: PrefetchTask,
-  response: Response,
+  response: RSCResponse,
   serverData: NavigationFlightResponse,
   isResponsePartial: boolean,
   route: FulfilledRouteCacheEntry,
@@ -1629,7 +1632,7 @@ function writeSeedDataIntoCache(
 async function fetchPrefetchResponse(
   url: URL,
   headers: RequestHeaders
-): Promise<Response | null> {
+): Promise<RSCResponse | null> {
   const fetchPriority = 'low'
   const response = await createFetch(url, headers, fetchPriority)
   if (!response.ok) {

packages/next/src/server/base-server.ts

@@ -2106,7 +2106,8 @@ export default abstract class Server<
         // The hash sent by the client does not match the expected value.
         // Respond with an error.
         res.statusCode = 400
-        res.body('Bad Request').send()
+        res.setHeader('content-type', 'text/plain')
+        res.body('').send()
         return null
       }
     }

test/e2e/app-dir/segment-cache/cdn-cache-busting/app/page.tsx

@@ -6,6 +6,11 @@ export default function Page() {
       <li>
         <LinkAccordion href="/target-page">Target page</LinkAccordion>
       </li>
+      <li>
+        <LinkAccordion href="/redirect-to-target-page">
+          Redirects to target page
+        </LinkAccordion>
+      </li>
     </ul>
   )
 }

test/e2e/app-dir/segment-cache/cdn-cache-busting/cdn-cache-busting.test.ts

@@ -79,7 +79,7 @@ describe('segment cache (CDN cache busting)', () => {
       'search param',
     async () => {
       const browser = await webdriver(port, '/')
-      const { status, text } = await browser.eval(async () => {
+      const { status } = await browser.eval(async () => {
         const res = await fetch('/target-page', {
           headers: {
             RSC: '1',
@@ -91,7 +91,44 @@ describe('segment cache (CDN cache busting)', () => {
       })
 
       expect(status).toBe(400)
-      expect(text).toContain('Bad Request')
+    }
+  )
+
+  it(
+    'perform fully prefetched navigation when a third-party proxy ' +
+      'performs a redirect',
+    async () => {
+      let act
+      const browser = await webdriver(port, '/', {
+        beforePageLoad(p: Playwright.Page) {
+          act = createRouterAct(p)
+        },
+      })
+
+      await act(
+        async () => {
+          const linkToggle = await browser.elementByCss(
+            '[data-link-accordion="/redirect-to-target-page"]'
+          )
+          await linkToggle.click()
+        },
+        {
+          includes: 'Target page',
+        }
+      )
+
+      // Navigate to the prefetched target page.
+      await act(async () => {
+        const link = await browser.elementByCss(
+          'a[href="/redirect-to-target-page"]'
+        )
+        await link.click()
+
+        // The page was prefetched, so we're able to render the target
+        // page immediately.
+        const div = await browser.elementById('target-page')
+        expect(await div.text()).toBe('Target page')
+      }, 'no-requests')
     }
   )
 })

test/e2e/app-dir/segment-cache/cdn-cache-busting/server.mjs

@@ -8,6 +8,13 @@ import { createGunzip } from 'zlib'
 
 const dir = dirname(fileURLToPath(import.meta.url))
 
+// Redirects that happen in the proxy layer, rather than in Next.js itself. This
+// is used to test that the client is still able to fully prefetch the
+// target page.
+const proxyRedirects = {
+  '/redirect-to-target-page': '/target-page',
+}
+
 async function spawnNext(port) {
   const child = spawn('pnpm', ['next', 'start', '-p', port, dir], {
     env: process.env,
@@ -48,6 +55,17 @@ async function createFakeCDN(destPort) {
 
   const proxy = httpProxy.createProxyServer()
   const cdnServer = createServer(async (req, res) => {
+    const pathname = new URL(req.url, `http://localhost`).pathname
+    const redirectUrl = proxyRedirects[pathname]
+    if (redirectUrl) {
+      console.log('Redirecting to:', redirectUrl)
+      res.writeHead(307, {
+        Location: redirectUrl,
+      })
+      res.end()
+      return
+    }
+
     if (isCacheableRequest(req)) {
       // Serve from our fake CDN if there's a matching entry.
       const entry = await fakeCDNCache.get(req.url)

test/e2e/app-dir/segment-cache/router-act.ts

@@ -9,7 +9,12 @@ type Batch = {
 
 type PendingRSCRequest = {
   route: Playwright.Route
-  result: Promise<{ body: string; headers: Record<string, string> }>
+  result: Promise<{
+    text: string
+    body: any
+    headers: Record<string, string>
+    status: number
+  }>
   didProcess: boolean
 }
 
@@ -145,10 +150,14 @@ export function createRouterAct(
             // but it should not affect the timing of when requests reach the
             // server; we pass the request to the server the immediately.
             result: new Promise(async (resolve) => {
-              const originalResponse = await page.request.fetch(request)
+              const originalResponse = await page.request.fetch(request, {
+                maxRedirects: 0,
+              })
               resolve({
-                body: await originalResponse.text(),
+                text: await originalResponse.text(),
+                body: await originalResponse.body(),
                 headers: originalResponse.headers(),
+                status: originalResponse.status(),
               })
             }),
             didProcess: false,
@@ -343,7 +352,11 @@ Choose a more specific substring to assert on.
             // fulfill it yet.
             remaining.add(item)
           } else {
-            await route.fulfill(fulfilled)
+            await route.fulfill({
+              body: fulfilled.body,
+              headers: fulfilled.headers,
+              status: fulfilled.status,
+            })
             const browserResponse = await request.response()
             if (browserResponse !== null) {
               await browserResponse.finished()
@@ -353,10 +366,13 @@ Choose a more specific substring to assert on.
 
         // After flushing the queue, wait for the microtask queue to be
         // exhausted, then check if any additional requests are initiated. A
-        // microtask should be enough because if the router queue is network
-        // throttled, the next request is issued within a microtask of the
-        // previous one finishing.
-        await page.evaluate(() => Promise.resolve())
+        // single macrotask should be enough because if the router queue is
+        // network throttled, the next request is issued either directly within
+        // the task of the previous request's completion event, or in the
+        // microtask queue of that event.
+        await page.evaluate(
+          () => new Promise<void>((res) => requestIdleCallback(() => res()))
+        )
 
         await waitForPendingRequestChecks()
       }