Replay redirect if RSC parameter is missing

If the server responds with a redirect (e.g. 307), and the redirected
location does not contain the cache busting search param set in the
original request, the response is likely invalid — when following
the redirect, the browser will have forwarded the request headers, but
without a corresponding cache busting search param. (See vercel#79563
for more context.)

Ideally, we would be able to intercept the redirect response and perform
it manually, instead of letting the browser automatically follow it, but
this is not allowed by the fetch API.

So instead, we must "replay" the redirect by fetching the new location
again, but this time we'll append the cache busting search param to
prevent a mismatch.

We can optimize Next.js's built-in middleware APIs by returning a custom
status code, to prevent the browser from automatically following it.
This will land in future PRs. Third-party proxies can choose to
implement this same protocol for cases where they need to redirect
Next.js requests but are not using Next.js APIs.

However, we'll still need this fallback behavior for third-party proxies
that don't implement our optimized redirect protocol.

This does not affect Server Action-based redirects; those are encoded
differently, as part of the Flight body.

As of now, this is gated behind the experimental
`validateRSCRequestHeaders` flag. We will test it before turning it on
by default.

Author: acdlite

packages/next/src/build/define-env.ts

@@ -187,6 +187,9 @@ export function getDefineEnv({
     'process.env.__NEXT_CLIENT_SEGMENT_CACHE': Boolean(
       config.experimental.clientSegmentCache
     ),
+    'process.env.__NEXT_CLIENT_VALIDATE_RSC_REQUEST_HEADERS': Boolean(
+      config.experimental.validateRSCRequestHeaders
+    ),
     'process.env.__NEXT_DYNAMIC_ON_HOVER': Boolean(
       config.experimental.dynamicOnHover
     ),

packages/next/src/client/components/router-reducer/fetch-server-response.ts

@@ -66,7 +66,7 @@ export type RequestHeaders = {
   'Next-Test-Fetch-Priority'?: RequestInit['priority']
 }
 
-export function urlToUrlWithoutFlightMarker(url: string): URL {
+export function canonicalizeResponseURL(url: string): URL {
   const urlWithoutFlightParameters = new URL(url, location.origin)
   urlWithoutFlightParameters.searchParams.delete(NEXT_RSC_UNION_QUERY)
   if (process.env.NODE_ENV === 'production') {
@@ -85,7 +85,7 @@ export function urlToUrlWithoutFlightMarker(url: string): URL {
 
 function doMpaNavigation(url: string): FetchServerResponseResult {
   return {
-    flightData: urlToUrlWithoutFlightMarker(url).toString(),
+    flightData: canonicalizeResponseURL(url).toString(),
     canonicalUrl: undefined,
     couldBeIntercepted: false,
     prerendered: false,
@@ -181,8 +181,7 @@ export async function fetchServerResponse(
       abortController.signal
     )
 
-    const responseUrl = urlToUrlWithoutFlightMarker(res.url)
-    const canonicalUrl = res.redirected ? responseUrl : undefined
+    const responseUrl = canonicalizeResponseURL(res.url)
 
     const contentType = res.headers.get('content-type') || ''
     const interception = !!res.headers.get('vary')?.includes(NEXT_URL)
@@ -237,7 +236,7 @@ export async function fetchServerResponse(
 
     return {
       flightData: normalizeFlightData(response.f),
-      canonicalUrl: canonicalUrl,
+      canonicalUrl: responseUrl,
       couldBeIntercepted: interception,
       prerendered: response.S,
       postponed,
@@ -265,7 +264,7 @@ export async function fetchServerResponse(
   }
 }
 
-export function createFetch(
+export async function createFetch(
   url: URL,
   headers: RequestHeaders,
   fetchPriority: 'auto' | 'high' | 'low' | null,
@@ -286,13 +285,81 @@ export function createFetch(
     headers['x-deployment-id'] = process.env.NEXT_DEPLOYMENT_ID
   }
 
-  return fetch(fetchUrl, {
-    // Backwards compat for older browsers. `same-origin` is the default in modern browsers.
+  const fetchOptions: RequestInit = {
     credentials: 'same-origin',
     headers,
     priority: fetchPriority || undefined,
     signal,
-  })
+  }
+  const response = await fetch(fetchUrl, fetchOptions)
+  return replayInternalRSCRedirectsIfNeeded(
+    response,
+    fetchUrl,
+    headers,
+    fetchOptions
+  )
+}
+
+async function replayInternalRSCRedirectsIfNeeded(
+  response: Response,
+  fetchUrl: URL,
+  headers: RequestHeaders,
+  fetchOptions: RequestInit
+): Promise<Response> {
+  if (!process.env.__NEXT_CLIENT_VALIDATE_RSC_REQUEST_HEADERS) {
+    return response
+  }
+  // If the server responds with a redirect (e.g. 307), and the redirected
+  // location does not contain the cache busting search param set in the
+  // original request, the response is likely invalid — when following the
+  // redirect, the browser forwards the request headers, but since the cache
+  // busting search param is missing, the server will reject the request due to
+  // a mismatch.
+  //
+  // Ideally, we would be able to intercept the redirect response and perform it
+  // manually, instead of letting the browser automatically follow it, but this
+  // is not allowed by the fetch API.
+  //
+  // So instead, we must "replay" the redirect by fetching the new location
+  // again, but this time we'll append the cache busting search param to prevent
+  // a mismatch.
+  //
+  // TODO: We can optimize Next.js's built-in middleware APIs by returning a
+  // custom status code, to prevent the browser from automatically following it.
+  //
+  // This does not affect Server Action-based redirects; those are encoded
+  // differently, as part of the Flight body. It only affects redirects that
+  // occur in a middleware or a third-party proxy.
+  const MAX_REDIRECTS = 20
+  for (let n = 0; n < MAX_REDIRECTS; n++) {
+    if (!response.redirected) {
+      // The server did not perform a redirect.
+      return response
+    }
+    const redirectedURL = new URL(response.url, fetchUrl)
+    if (redirectedURL.origin !== fetchUrl.origin) {
+      // The server redirected to an external URL. The rest of the logic below
+      // is not relevant, because it only applies to internal redirects.
+      return response
+    }
+    if (
+      redirectedURL.searchParams.get(NEXT_RSC_UNION_QUERY) ===
+      fetchUrl.searchParams.get(NEXT_RSC_UNION_QUERY)
+    ) {
+      // The server performed an internal redirect, but the search param
+      // was preserved.
+      return response
+    }
+    // The RSC request was redirected. Assume the response is invalid, because
+    // the cache-busting search param is missing from the redirected URL.
+    //
+    // Append the cache busting search param to the redirected URL and
+    // fetch again.
+    setCacheBustingSearchParam(redirectedURL, headers)
+    fetchUrl = redirectedURL
+    response = await fetch(fetchUrl, fetchOptions)
+  }
+  return response
 }
 
 export function createFromNextReadableStream(

packages/next/src/client/components/segment-cache-impl/cache.ts

@@ -23,6 +23,7 @@ import {
   RSC_HEADER,
 } from '../app-router-headers'
 import {
+  canonicalizeResponseURL,
   createFetch,
   createFromNextReadableStream,
   type RequestHeaders,
@@ -1032,16 +1033,13 @@ export async function fetchRouteOnCacheMiss(
     // Or, we should just use a (readonly) URL object instead. The type of the
     // prop that we pass to seed the initial state does not need to be the same
     // type as the state itself.
+    const responseUrl = removeSegmentPathFromURLInOutputExportMode(
+      href,
+      requestUrl.href,
+      response.url
+    )
     const canonicalUrl = createHrefFromUrl(
-      new URL(
-        response.redirected
-          ? removeSegmentPathFromURLInOutputExportMode(
-              href,
-              requestUrl.href,
-              response.url
-            )
-          : href
-      )
+      new URL(canonicalizeResponseURL(responseUrl).href)
     )
 
     // Check whether the response varies based on the Next-Url header.

packages/next/src/server/base-server.ts

@@ -2101,7 +2101,7 @@ export default abstract class Server<
         // The hash sent by the client does not match the expected value.
         // Respond with an error.
         res.statusCode = 400
-        res.body('Bad Request').send()
+        res.body('{}').send()
         return null
       }
     }

test/e2e/app-dir/segment-cache/cdn-cache-busting/app/page.tsx

@@ -6,6 +6,11 @@ export default function Page() {
       <li>
         <LinkAccordion href="/target-page">Target page</LinkAccordion>
       </li>
+      <li>
+        <LinkAccordion href="/redirect-to-target-page">
+          Redirects to target page
+        </LinkAccordion>
+      </li>
     </ul>
   )
 }

test/e2e/app-dir/segment-cache/cdn-cache-busting/cdn-cache-busting.test.ts

@@ -79,7 +79,7 @@ describe('segment cache (CDN cache busting)', () => {
       'search param',
     async () => {
       const browser = await webdriver(port, '/')
-      const { status, text } = await browser.eval(async () => {
+      const { status } = await browser.eval(async () => {
         const res = await fetch('/target-page', {
           headers: {
             RSC: '1',
@@ -91,7 +91,44 @@ describe('segment cache (CDN cache busting)', () => {
       })
 
       expect(status).toBe(400)
-      expect(text).toContain('Bad Request')
+    }
+  )
+
+  it(
+    'perform fully prefetched navigation when a third-party proxy ' +
+      'performs a redirect',
+    async () => {
+      let act
+      const browser = await webdriver(port, '/', {
+        beforePageLoad(p: Playwright.Page) {
+          act = createRouterAct(p)
+        },
+      })
+
+      await act(
+        async () => {
+          const linkToggle = await browser.elementByCss(
+            '[data-link-accordion="/redirect-to-target-page"]'
+          )
+          await linkToggle.click()
+        },
+        {
+          includes: 'Target page',
+        }
+      )
+
+      // Navigate to the prefetched target page.
+      await act(async () => {
+        const link = await browser.elementByCss(
+          'a[href="/redirect-to-target-page"]'
+        )
+        await link.click()
+
+        // The page was prefetched, so we're able to render the target
+        // page immediately.
+        const div = await browser.elementById('target-page')
+        expect(await div.text()).toBe('Target page')
+      }, 'no-requests')
     }
   )
 })

test/e2e/app-dir/segment-cache/cdn-cache-busting/server.mjs

@@ -8,6 +8,13 @@ import { createGunzip } from 'zlib'
 
 const dir = dirname(fileURLToPath(import.meta.url))
 
+// Redirects that happen in the proxy layer, rather than in Next.js itself. This
+// is used to test that the client is still able to fully prefetch the
+// target page.
+const proxyRedirects = {
+  '/redirect-to-target-page': '/target-page',
+}
+
 async function spawnNext(port) {
   const child = spawn('pnpm', ['next', 'start', '-p', port, dir], {
     env: process.env,
@@ -48,6 +55,17 @@ async function createFakeCDN(destPort) {
 
   const proxy = httpProxy.createProxyServer()
   const cdnServer = createServer(async (req, res) => {
+    const pathname = new URL(req.url, `http://localhost`).pathname
+    const redirectUrl = proxyRedirects[pathname]
+    if (redirectUrl) {
+      console.log('Redirecting to:', redirectUrl)
+      res.writeHead(307, {
+        Location: redirectUrl,
+      })
+      res.end()
+      return
+    }
+
     if (isCacheableRequest(req)) {
       // Serve from our fake CDN if there's a matching entry.
       const entry = await fakeCDNCache.get(req.url)

test/e2e/app-dir/segment-cache/router-act.ts

@@ -9,7 +9,12 @@ type Batch = {
 
 type PendingRSCRequest = {
   route: Playwright.Route
-  result: Promise<{ body: string; headers: Record<string, string> }>
+  result: Promise<{
+    text: string
+    body: any
+    headers: Record<string, string>
+    status: number
+  }>
   didProcess: boolean
 }
 
@@ -145,10 +150,14 @@ export function createRouterAct(
             // but it should not affect the timing of when requests reach the
             // server; we pass the request to the server the immediately.
             result: new Promise(async (resolve) => {
-              const originalResponse = await page.request.fetch(request)
+              const originalResponse = await page.request.fetch(request, {
+                maxRedirects: 0,
+              })
               resolve({
-                body: await originalResponse.text(),
+                text: await originalResponse.text(),
+                body: await originalResponse.body(),
                 headers: originalResponse.headers(),
+                status: originalResponse.status(),
               })
             }),
             didProcess: false,
@@ -343,7 +352,11 @@ Choose a more specific substring to assert on.
             // fulfill it yet.
             remaining.add(item)
           } else {
-            await route.fulfill(fulfilled)
+            await route.fulfill({
+              body: fulfilled.body,
+              headers: fulfilled.headers,
+              status: fulfilled.status,
+            })
             const browserResponse = await request.response()
             if (browserResponse !== null) {
               await browserResponse.finished()
@@ -353,10 +366,13 @@ Choose a more specific substring to assert on.
 
         // After flushing the queue, wait for the microtask queue to be
         // exhausted, then check if any additional requests are initiated. A
-        // microtask should be enough because if the router queue is network
-        // throttled, the next request is issued within a microtask of the
-        // previous one finishing.
-        await page.evaluate(() => Promise.resolve())
+        // single macrotask should be enough because if the router queue is
+        // network throttled, the next request is issued either directly within
+        // the task of the previous request's completion event, or in the
+        // microtask queue of that event.
+        await page.evaluate(
+          () => new Promise<void>((res) => requestIdleCallback(() => res()))
+        )
 
         await waitForPendingRequestChecks()
       }