Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/frontend/e2e-tests/fmtk/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
is a rolling, never-cleared window over the current app instance — an
error surfacing in test B may originate in test A of the same session —
and ``restart_app`` drains+raises *before* stopping so the outgoing
instance cannot launder its errors away.
instance cannot launder its errors away (one exception: a gone
isolate's drain counts as empty, because that dead-connection state is
exactly what the restart exists to recover — #3469).

Env knobs: FMTK_E2E_FRESH=1 wipes the scratch state before the session
(fresh DB); FMTK_E2E_HEADLESS forces Chrome mode (default auto: headless
Expand Down
28 changes: 26 additions & 2 deletions src/frontend/e2e-tests/fmtk/fmtkharness.py
Original file line number Diff line number Diff line change
Expand Up @@ -2591,17 +2591,41 @@ def restart_app(self, at_path: str | None = None) -> None:

The app-error monitor is a rolling window that a restart would
silently clear — drain it first so errors from the outgoing app
instance fail the test instead of being laundered away."""
errors = self.client.app_errors()
instance fail the test instead of being laundered away. An
outgoing instance whose isolate is already gone has no monitor
window left to drain, so that drain failure counts as empty
(:meth:`drain_errors_before_restart`) — this method is the
designated recovery for exactly that dead-connection state
(#3469)."""
errors = self.drain_errors_before_restart()
if errors:
raise FmtkError(f"app errors before restart_app: {errors}")
# The wedge marker describes the outgoing instance; the fresh
# launch starts the dwds race window clean (#3469).
self.flutter.isolate_gone_since = None
url_suffix = ""
if at_path is not None:
self.flutter.launch_serial += 1
url_suffix = f"/?e2e_boot={self.flutter.launch_serial}#{at_path}"
self.flutter.stop()
self.flutter.launch(url_suffix)

def drain_errors_before_restart(self) -> list:
"""The outgoing instance's app errors, tolerating a gone isolate.

A dead debug connection leaves no error-monitor window to
launder — the state restart_app exists to recover — so a drain
that dies with the gone-isolate signature counts as an empty
drain (the deliberate trade FlutterRun.recover_from_wedge
documents). Any other drain failure still raises: a broken
drain must not quietly become a restart."""
try:
return self.client.app_errors()
except FmtkError as exc:
if ISOLATE_GONE_MARK in f"{exc}\n{exc.stderr}":
return []
raise

def boot(self, fresh: bool = False) -> None:
if fresh:
self.wipe()
Expand Down
52 changes: 52 additions & 0 deletions src/frontend/e2e-tests/fmtk/test_harness_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,28 @@
that outlives the window restarts the flutter run transparently, and a
parked-away tab never arms. This scenario pins all three legs against
the live stack.

The 2026-09-10+ nightlies exposed a second leg (#3469): the first
scenario to meet the dead instance reaches for
``Harness.restart_app()`` — the designated recovery — whose pre-restart
drain died on the same gone isolate, so the restart never ran and every
following scenario failed on the dead connection (the flows group's
nightly cascade). ``restart_app`` now treats a gone-isolate drain as an
empty drain and relaunches; the scenario below pins that policy
(including what must still raise) against stubbed drains.
"""

from __future__ import annotations

import time

import pytest

from fmtkharness import (
BACKEND_PORT,
FmtkClient,
FmtkError,
FlutterRun,
ISOLATE_GONE_MARK,
WEDGE_RECOVERY_SECONDS,
cdp_eval,
Expand Down Expand Up @@ -83,3 +96,42 @@ def test_isolate_wedge_recovery(harness, app):
app.flutter.isolate_gone_since = time.monotonic() - (WEDGE_RECOVERY_SECONDS + 5)
assert app.recover_if_wedged(gone_error()) is True
app.wait_for_login_page()


def test_restart_app_survives_a_dead_isolate(harness, monkeypatch):
"""The pre-restart drain must tolerate the state it recovers
(#3469): a gone isolate counts as an empty drain and the
stop/relaunch proceeds, while a live drain's real errors and any
other drain failure still raise."""
relaunched = []
monkeypatch.setattr(FlutterRun, "stop", lambda self: None)
monkeypatch.setattr(
FlutterRun, "launch", lambda self, url_suffix="": relaunched.append(url_suffix)
)

# a dead isolate's drain (the nightly cascade's exact failure):
# restart_app relaunches instead of raising, and clears the wedge
# marker the outgoing instance armed
def gone_drain(self):
raise gone_error()

monkeypatch.setattr(FmtkClient, "app_errors", gone_drain)
harness.flutter.isolate_gone_since = time.monotonic()
harness.restart_app()
assert relaunched == [""]
assert harness.flutter.isolate_gone_since is None

# a drain with real errors still fails the test before any stop
monkeypatch.setattr(FmtkClient, "app_errors", lambda self: [{"message": "boom"}])
with pytest.raises(FmtkError, match="app errors before restart_app"):
harness.restart_app()

# a drain failure that is not the gone-isolate signature raises —
# a broken drain must not quietly become a restart
def broken_drain(self):
raise FmtkError("fmtk get_app_errors failed: toolkit down (None)")

monkeypatch.setattr(FmtkClient, "app_errors", broken_drain)
with pytest.raises(FmtkError, match="toolkit down"):
harness.restart_app()
assert relaunched == [""] # no further stop/launch ran
Loading