From 6d63b3dfaf9e72196c746546168794e9c140d617 Mon Sep 17 00:00:00 2001 From: Aaron Personal Date: Fri, 31 Jul 2026 11:27:50 +0530 Subject: [PATCH 1/2] tests: add metrics parsing tests --- tests/unit/dashboard/test_api.py | 169 +++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/tests/unit/dashboard/test_api.py b/tests/unit/dashboard/test_api.py index c2869bb..e11e56c 100644 --- a/tests/unit/dashboard/test_api.py +++ b/tests/unit/dashboard/test_api.py @@ -24,6 +24,17 @@ """ +_FN_WITH_METRICS_TMPL = """\ + + + + + + + + +""" + _RUNNING_TMPL = """\ @@ -114,6 +125,53 @@ def client_no_auth(scanner_with_sessions, monkeypatch): app_module.scanner = original +@pytest.fixture() +def scanner_with_metrics(tmp_path, monkeypatch): + """FennScanner with one session carrying nodes and one + metric-less session (for backward-compatibility checks).""" + _write( + tmp_path / "m1.fn", + _FN_WITH_METRICS_TMPL.format( + project="metrics-proj", + sid="m1", + started="2026-05-21 07:00:00", + ended="2026-05-21 07:00:10", + dur=10, + status="completed", + ), + ) + _write( + tmp_path / "m2.fn", + _FN_TMPL.format( + project="metrics-proj", + sid="m2", + started="2026-05-21 07:05:00", + ended="2026-05-21 07:05:10", + dur=10, + status="completed", + ), + ) + monkeypatch.setenv( + "FENN_DASHBOARD_OVERRIDES_PATH", str(tmp_path / "dashboard_overrides.json") + ) + return FennScanner(extra_dirs=[str(tmp_path)]) + + +@pytest.fixture() +def metrics_client(scanner_with_metrics): + """Flask test client wired to the metrics scanner fixture.""" + import fenn.dashboard.app as app_module + + original = app_module.scanner + app_module.scanner = scanner_with_metrics + app.config["TESTING"] = True + with app.test_client() as c: + with c.session_transaction() as sess: + sess["user"] = {"email": "test@example.com"} + yield c + app_module.scanner = original + + # --------------------------------------------------------------------------- # Response shape # --------------------------------------------------------------------------- @@ -152,6 +210,117 @@ def test_default_offset_is_0(self, client): assert data["offset"] == 0 +# --------------------------------------------------------------------------- +# Metric parsing, API shape, and backward compatibility +# --------------------------------------------------------------------------- + + +class TestApiSessionMetrics: + """ parsing, API response shape, and backward compatibility for + sessions with no metric data.""" + + def test_scanner_parses_metric_elements(self, scanner_with_metrics): + """A session with nodes must parse into SessionData.metrics + with the correct count and names.""" + session = scanner_with_metrics.get_session("metrics-proj", "m1") + assert session is not None + assert len(session["metrics"]) == 4 + assert {m["name"] for m in session["metrics"]} == { + "train_loss", + "val_loss", + "acc", + } + + def test_metric_point_fields_have_correct_types_and_values( + self, scanner_with_metrics + ): + """step must parse as int, value as float, in emission order.""" + session = scanner_with_metrics.get_session("metrics-proj", "m1") + train_loss = sorted( + (m for m in session["metrics"] if m["name"] == "train_loss"), + key=lambda m: m["step"], + ) + assert [m["step"] for m in train_loss] == [0, 1] + assert [m["value"] for m in train_loss] == [0.9, 0.5] + assert isinstance(train_loss[0]["step"], int) + assert isinstance(train_loss[0]["value"], float) + + def test_session_without_metrics_returns_empty_list(self, scanner_with_metrics): + """Backward compatibility: a .fn file with no elements + must still parse, with metrics == [].""" + session = scanner_with_metrics.get_session("metrics-proj", "m2") + assert session is not None + assert session["metrics"] == [] + + def test_metric_missing_name_is_skipped(self, tmp_path, monkeypatch): + """A element with no name attribute must be dropped, + not raise.""" + _write( + tmp_path / "no_name.fn", + '\n' + '\n' + ' \n' + ' \n' + "\n", + ) + monkeypatch.setenv( + "FENN_DASHBOARD_OVERRIDES_PATH", str(tmp_path / "dashboard_overrides.json") + ) + scanner = FennScanner(extra_dirs=[str(tmp_path)]) + session = scanner.get_session("p", "no_name") + assert session is not None + assert session["metrics"] == [] + + def test_metric_with_malformed_step_or_value_is_skipped( + self, tmp_path, monkeypatch + ): + """Non-numeric step/value attributes must be skipped individually + rather than failing the whole parse, mirroring duration_s handling.""" + _write( + tmp_path / "malformed.fn", + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' \n' + "\n", + ) + monkeypatch.setenv( + "FENN_DASHBOARD_OVERRIDES_PATH", str(tmp_path / "dashboard_overrides.json") + ) + scanner = FennScanner(extra_dirs=[str(tmp_path)]) + session = scanner.get_session("p", "malformed") + assert session is not None + assert len(session["metrics"]) == 1 + assert session["metrics"][0]["name"] == "acc" + + def test_api_session_detail_includes_metrics(self, metrics_client): + """GET /api/session// must include the full metrics list.""" + resp = metrics_client.get("/api/session/metrics-proj/m1") + assert resp.status_code == 200 + data = resp.get_json() + assert "metrics" in data + assert len(data["metrics"]) == 4 + + def test_api_session_detail_metrics_empty_for_metric_less_session( + self, metrics_client + ): + resp = metrics_client.get("/api/session/metrics-proj/m2") + assert resp.status_code == 200 + assert resp.get_json()["metrics"] == [] + + def test_api_sessions_listing_omits_metrics_key(self, metrics_client): + """/api/sessions must strip 'metrics' from every item, same as + 'entries' and 'config' (test_items_omit_entries above).""" + resp = metrics_client.get("/api/sessions?project=metrics-proj") + assert resp.status_code == 200 + data = resp.get_json() + assert data["total"] == 2 + for item in data["items"]: + assert "metrics" not in item + + # --------------------------------------------------------------------------- # Filtering # --------------------------------------------------------------------------- From a639194d1782316f8c4e5682eea457feb6980a37 Mon Sep 17 00:00:00 2001 From: Aaron Personal Date: Fri, 31 Jul 2026 12:32:17 +0530 Subject: [PATCH 2/2] docs: update README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 914c3a9..289e2f3 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ https://github.com/sponsors/blkdmr - **Template Ready**: Built-in support for reproducible, shareable experiment templates. -- **Live Dashboard**: Browse, filter, and rename experiment sessions in a local web UI, and launch any pulled template with one click - straight to its live session view. +- **Live Dashboard**: Browse, filter, and rename experiment sessions in a local web UI, view Train Loss / Val Loss / Accuracy curves for each session, and launch any pulled template with one click - straight to its live session view. ## Quickstart @@ -215,13 +215,15 @@ def main(args): preds = trainer.predict(test_loader) ``` +Beyond free-text logs, fenn supports structured, numeric metric logging via `logger.log_metric(name, value, step)`. Each call appends a `` entry to the session's `.fn` file, which the [dashboard](#cli-reference)'s **Graphs** tab reads and renders as a line chart. + ## CLI Reference A quick reference for all available fenn CLI commands. | Command | Description | |---|---| -| `fenn dashboard` | Launch the local web UI to browse and manage sessions, and to view and run locally pulled templates | +| `fenn dashboard` | Launch the local web UI to browse and manage sessions, view metric curves, and to view and run locally pulled templates | | `fenn grid ` | By setting grid/train section in template, you can run a Fenn project several times, with all possible grid hyperparams. Also, it is possible to specify path to main.py file (e.g. my_temp/main.py) | | `fenn list` | List all available templates from [`pyfenn/templates`](https://github.com/pyfenn/templates) | | `fenn pull