test(nextjs): Skip flaky tests affected by Turbopack 404s in dev mode (#20993)

Turbopack intermittently returns 404 for dynamic routes in dev mode
(e.g. `/route-handler/[xoxo]/...`, `/parameterized/[one]/beep/[two]`).
We drop 404 transactions so when this happens, route handlers never
execute, no Sentry events are sent, and the tests time out waiting for
events that never arrive.

This PR skips the affected tests in dev mode across `nextjs-16` and
`nextjs-16-streaming` test apps. The tests still run in production mode
where routes work reliably.

Closes: #20992, #20986, #20987, #20985, #20977, #20976, #20975, #20984

Author: andreiborza

dev-packages/e2e-tests/test-applications/nextjs-16-streaming/tests/route-handler.test.ts

@@ -1,7 +1,10 @@
 import test, { expect } from '@playwright/test';
 import { waitForError, waitForStreamedSpan, getSpanOp } from '@sentry-internal/test-utils';
+import { isDevMode } from './isDevMode';
 
 test('Should create a streamed span for node route handlers', async ({ request }) => {
+  test.skip(isDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const rootSpanPromise = waitForStreamedSpan('nextjs-16-streaming', span => {
     return span.name === 'GET /route-handler/[xoxo]/node' && getSpanOp(span) === 'http.server' && span.is_segment;
   });
@@ -16,6 +19,8 @@ test('Should create a streamed span for node route handlers', async ({ request }
 });
 
 test('Should report an error linked to the correct trace for a throwing route handler', async ({ request }) => {
+  test.skip(isDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const errorEventPromise = waitForError('nextjs-16-streaming', errorEvent => {
     return errorEvent?.exception?.values?.some(value => value.value === 'route-handler-error') ?? false;
   });
@@ -37,6 +42,8 @@ test('Should report an error linked to the correct trace for a throwing route ha
 test('Should set a parameterized transaction name on a captureMessage event in a route handler', async ({
   request,
 }) => {
+  test.skip(isDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const messageEventPromise = waitForError('nextjs-16-streaming', event => {
     return event?.message === 'route-handler-message';
   });

dev-packages/e2e-tests/test-applications/nextjs-16-streaming/tests/server-components.test.ts

@@ -1,7 +1,10 @@
 import { expect, test } from '@playwright/test';
 import { waitForStreamedSpan, waitForStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
+import { isDevMode } from './isDevMode';
 
 test('Sends a streamed span for a request to app router with URL', async ({ page }) => {
+  test.skip(isDevMode, 'Turbopack intermittently returns 404 for nested dynamic routes in dev mode');
+
   const rootSpanPromise = waitForStreamedSpan('nextjs-16-streaming', span => {
     return span.name === 'GET /parameterized/[one]/beep/[two]' && span.is_segment;
   });

dev-packages/e2e-tests/test-applications/nextjs-16/tests/isDevMode.ts

@@ -1 +1,2 @@
 export const isDevMode = !!process.env.TEST_ENV && process.env.TEST_ENV.includes('development');
+export const isTurbopackDevMode = process.env.TEST_ENV === 'development';

dev-packages/e2e-tests/test-applications/nextjs-16/tests/route-handler.test.ts

@@ -1,7 +1,10 @@
 import test, { expect } from '@playwright/test';
 import { waitForError, waitForTransaction } from '@sentry-internal/test-utils';
+import { isTurbopackDevMode } from './isDevMode';
 
 test('Should create a transaction for node route handlers', async ({ request }) => {
+  test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const routehandlerTransactionPromise = waitForTransaction('nextjs-16', async transactionEvent => {
     return transactionEvent?.transaction === 'GET /route-handler/[xoxo]/node';
   });
@@ -13,11 +16,7 @@ test('Should create a transaction for node route handlers', async ({ request })
 
   expect(routehandlerTransaction.contexts?.trace?.status).toBe('ok');
   expect(routehandlerTransaction.contexts?.trace?.op).toBe('http.server');
-
-  // This is flaking on dev mode
-  if (process.env.TEST_ENV !== 'development' && process.env.TEST_ENV !== 'dev-turbopack') {
-    expect(routehandlerTransaction.contexts?.trace?.data?.['http.request.header.x_charly']).toBe('gomez');
-  }
+  expect(routehandlerTransaction.contexts?.trace?.data?.['http.request.header.x_charly']).toBe('gomez');
 });
 
 test('Should create a transaction for edge route handlers', async ({ request }) => {
@@ -41,6 +40,8 @@ test('Should create a transaction for edge route handlers', async ({ request })
 test('Should report an error with a parameterized transaction name for a throwing route handler', async ({
   request,
 }) => {
+  test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const errorEventPromise = waitForError('nextjs-16', errorEvent => {
     return errorEvent?.exception?.values?.some(value => value.value === 'route-handler-error') ?? false;
   });
@@ -63,23 +64,6 @@ test('Should report an error with a parameterized transaction name for a throwin
   // Error should carry the parameterized transaction name
   expect(errorEvent.transaction).toBe('GET /route-handler/[xoxo]/error');
 
-  // On turbopack (no wrapping loader), the error goes through onRequestError which sets nextjs context.
-  // On webpack, the wrapping loader's error handler fires first and captures without nextjs context.
-  // The SDK deduplicates by error identity, so only the first capture survives.
-  if (process.env.TEST_ENV === 'development') {
-    expect(errorEvent.contexts?.nextjs).toEqual({
-      route_type: 'route',
-      router_kind: 'App Router',
-      router_path: '/route-handler/[xoxo]/error',
-      request_path: '/route-handler/456/error',
-    });
-
-    expect(errorEvent.exception?.values?.[0]?.mechanism).toEqual({
-      handled: false,
-      type: 'auto.function.nextjs.on_request_error',
-    });
-  }
-
   // Transaction should have parameterized name and internal_error status
   expect(transactionEvent.transaction).toBe('GET /route-handler/[xoxo]/error');
   expect(transactionEvent.contexts?.trace?.status).toBe('internal_error');
@@ -88,6 +72,8 @@ test('Should report an error with a parameterized transaction name for a throwin
 test('Should set a parameterized transaction name on a captureMessage event in a route handler', async ({
   request,
 }) => {
+  test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const messageEventPromise = waitForError('nextjs-16', event => {
     return event?.message === 'route-handler-message';
   });
@@ -119,6 +105,8 @@ test('Should set a parameterized transaction name on a captureMessage event in a
 test('Should set a parameterized transaction name on a captureException event in a route handler', async ({
   request,
 }) => {
+  test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const errorEventPromise = waitForError('nextjs-16', errorEvent => {
     return errorEvent?.exception?.values?.some(value => value.value === 'route-handler-capture-exception') ?? false;
   });

dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-components.test.ts

@@ -1,7 +1,10 @@
 import { expect, test } from '@playwright/test';
 import { waitForTransaction } from '@sentry-internal/test-utils';
+import { isTurbopackDevMode } from './isDevMode';
 
 test('Sends a transaction for a request to app router with URL', async ({ page }) => {
+  test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for nested dynamic routes in dev mode');
+
   const serverComponentTransactionPromise = waitForTransaction('nextjs-16', transactionEvent => {
     return (
       transactionEvent?.transaction === 'GET /parameterized/[one]/beep/[two]' &&
@@ -74,6 +77,8 @@ test('Will create a transaction with spans for every server component and metada
 test('Will create a transaction with spans for every server component and metadata generation functions when visiting a dynamic page', async ({
   page,
 }) => {
+  test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for dynamic routes in dev mode');
+
   const serverTransactionEventPromise = waitForTransaction('nextjs-16', async transactionEvent => {
     return transactionEvent?.transaction === 'GET /nested-layout/[dynamic]';
   });