From 5a290067c4ad37b50d5852eb5602188ba1c1a332 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Fri, 2 Oct 2026 09:29:25 -0400 Subject: [PATCH 1/2] fmtk e2e: restart_app tolerates a dead isolate in its pre-restart drain (#3469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly flows cascade: test_auth ends on a deep-link boot whose instance dies, and the first network scenario's designated recovery — Harness.restart_app() — died in its own pre-restart drain (get_app_errors: 'No Flutter isolate found'), so the restart never ran and every following scenario failed on the dead connection, night after night. The drain now treats the gone-isolate signature as an empty drain (the same trade FlutterRun.recover_from_wedge documents: an error the dying instance threw in its last undriven moments can no longer fail its test) and proceeds with the stop/relaunch; the wedge marker the outgoing instance armed is cleared. Real drained errors and any other drain failure still raise. --- src/frontend/e2e-tests/fmtk/fmtkharness.py | 28 +++++++++- .../e2e-tests/fmtk/test_harness_recovery.py | 52 +++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/frontend/e2e-tests/fmtk/fmtkharness.py b/src/frontend/e2e-tests/fmtk/fmtkharness.py index c5c44f4e2..7d0bf7f99 100644 --- a/src/frontend/e2e-tests/fmtk/fmtkharness.py +++ b/src/frontend/e2e-tests/fmtk/fmtkharness.py @@ -2591,10 +2591,18 @@ def restart_app(self, at_path: str | None = None) -> None: The app-error monitor is a rolling window that a restart would silently clear — drain it first so errors from the outgoing app - instance fail the test instead of being laundered away.""" - errors = self.client.app_errors() + instance fail the test instead of being laundered away. An + outgoing instance whose isolate is already gone has no monitor + window left to drain, so that drain failure counts as empty + (:meth:`drain_errors_before_restart`) — this method is the + designated recovery for exactly that dead-connection state + (#3469).""" + errors = self.drain_errors_before_restart() if errors: raise FmtkError(f"app errors before restart_app: {errors}") + # The wedge marker describes the outgoing instance; the fresh + # launch starts the dwds race window clean (#3469). + self.flutter.isolate_gone_since = None url_suffix = "" if at_path is not None: self.flutter.launch_serial += 1 @@ -2602,6 +2610,22 @@ def restart_app(self, at_path: str | None = None) -> None: self.flutter.stop() self.flutter.launch(url_suffix) + def drain_errors_before_restart(self) -> list: + """The outgoing instance's app errors, tolerating a gone isolate. + + A dead debug connection leaves no error-monitor window to + launder — the state restart_app exists to recover — so a drain + that dies with the gone-isolate signature counts as an empty + drain (the deliberate trade FlutterRun.recover_from_wedge + documents). Any other drain failure still raises: a broken + drain must not quietly become a restart.""" + try: + return self.client.app_errors() + except FmtkError as exc: + if ISOLATE_GONE_MARK in f"{exc}\n{exc.stderr}": + return [] + raise + def boot(self, fresh: bool = False) -> None: if fresh: self.wipe() diff --git a/src/frontend/e2e-tests/fmtk/test_harness_recovery.py b/src/frontend/e2e-tests/fmtk/test_harness_recovery.py index d1c7de42a..ffaa07ce5 100644 --- a/src/frontend/e2e-tests/fmtk/test_harness_recovery.py +++ b/src/frontend/e2e-tests/fmtk/test_harness_recovery.py @@ -9,15 +9,28 @@ that outlives the window restarts the flutter run transparently, and a parked-away tab never arms. This scenario pins all three legs against the live stack. + +The 2026-09-10+ nightlies exposed a second leg (#3469): the first +scenario to meet the dead instance reaches for +``Harness.restart_app()`` — the designated recovery — whose pre-restart +drain died on the same gone isolate, so the restart never ran and every +following scenario failed on the dead connection (the flows group's +nightly cascade). ``restart_app`` now treats a gone-isolate drain as an +empty drain and relaunches; the scenario below pins that policy +(including what must still raise) against stubbed drains. """ from __future__ import annotations import time +import pytest + from fmtkharness import ( BACKEND_PORT, + FmtkClient, FmtkError, + FlutterRun, ISOLATE_GONE_MARK, WEDGE_RECOVERY_SECONDS, cdp_eval, @@ -83,3 +96,42 @@ def test_isolate_wedge_recovery(harness, app): app.flutter.isolate_gone_since = time.monotonic() - (WEDGE_RECOVERY_SECONDS + 5) assert app.recover_if_wedged(gone_error()) is True app.wait_for_login_page() + + +def test_restart_app_survives_a_dead_isolate(harness, monkeypatch): + """The pre-restart drain must tolerate the state it recovers + (#3469): a gone isolate counts as an empty drain and the + stop/relaunch proceeds, while a live drain's real errors and any + other drain failure still raise.""" + relaunched = [] + monkeypatch.setattr(FlutterRun, "stop", lambda self: None) + monkeypatch.setattr( + FlutterRun, "launch", lambda self, url_suffix="": relaunched.append(url_suffix) + ) + + # a dead isolate's drain (the nightly cascade's exact failure): + # restart_app relaunches instead of raising, and clears the wedge + # marker the outgoing instance armed + def gone_drain(self): + raise gone_error() + + monkeypatch.setattr(FmtkClient, "app_errors", gone_drain) + harness.flutter.isolate_gone_since = time.monotonic() + harness.restart_app() + assert relaunched == [""] + assert harness.flutter.isolate_gone_since is None + + # a drain with real errors still fails the test before any stop + monkeypatch.setattr(FmtkClient, "app_errors", lambda self: [{"message": "boom"}]) + with pytest.raises(FmtkError, match="app errors before restart_app"): + harness.restart_app() + + # a drain failure that is not the gone-isolate signature raises — + # a broken drain must not quietly become a restart + def broken_drain(self): + raise FmtkError("fmtk get_app_errors failed: toolkit down (None)") + + monkeypatch.setattr(FmtkClient, "app_errors", broken_drain) + with pytest.raises(FmtkError, match="toolkit down"): + harness.restart_app() + assert relaunched == [""] # no further stop/launch ran From 841243a41add81a2cbffc3ab7fac1f687cce3923 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Fri, 2 Oct 2026 09:42:33 -0400 Subject: [PATCH 2/2] docs(fmtk e2e): conftest laundering note covers the gone-isolate exception (#3469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up: the suite docstring claimed restart_app always drains+raises before stopping; note the exception the #3469 recovery introduced — a gone isolate's drain counts as empty. --- src/frontend/e2e-tests/fmtk/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/frontend/e2e-tests/fmtk/conftest.py b/src/frontend/e2e-tests/fmtk/conftest.py index f711c69f2..04b08730d 100644 --- a/src/frontend/e2e-tests/fmtk/conftest.py +++ b/src/frontend/e2e-tests/fmtk/conftest.py @@ -9,7 +9,9 @@ is a rolling, never-cleared window over the current app instance — an error surfacing in test B may originate in test A of the same session — and ``restart_app`` drains+raises *before* stopping so the outgoing -instance cannot launder its errors away. +instance cannot launder its errors away (one exception: a gone +isolate's drain counts as empty, because that dead-connection state is +exactly what the restart exists to recover — #3469). Env knobs: FMTK_E2E_FRESH=1 wipes the scratch state before the session (fresh DB); FMTK_E2E_HEADLESS forces Chrome mode (default auto: headless