diff --git a/.github/scripts/bench_metrics.py b/.github/scripts/bench_metrics.py index 2f240d8..439f54a 100644 --- a/.github/scripts/bench_metrics.py +++ b/.github/scripts/bench_metrics.py @@ -16,7 +16,7 @@ import math # Cost model and grid types come straight from the package so the benchmark can -# never drift from what production actually bills/uses (arm64, 2 GB -- issue #110). +# never drift from what production actually bills/uses (arm64, 4 GB -- issue #110/#193). from zagg.dispatch import LAMBDA_MEMORY_GB, LAMBDA_PRICE_PER_GB_SEC from zagg.grids.healpix import HealpixGrid from zagg.grids.rectilinear import RectilinearGrid @@ -73,6 +73,9 @@ "setup_s", "fanout_s", "finalize_s", + # Read-backend axis (issue #193): "inline" | "sidecar" -- the live matrix's + # A/B column. Null on frozen/legacy rows (codec/read axes predate it). + "index_backend", ] @@ -178,6 +181,7 @@ def build_record( # the target. None on targets that don't set it (legacy/frozen rows). "codec": context.get("codec"), "read": context.get("read"), + "index_backend": context.get("index_backend"), # Wall-time breakdown (issue #180): total end-to-end AOI dispatch wall # plus its phases, straight from the agg summary. "total_wall_s": summary.get("wall_time_s"), diff --git a/.github/scripts/plot_series.py b/.github/scripts/plot_series.py index 4515117..65fac69 100644 --- a/.github/scripts/plot_series.py +++ b/.github/scripts/plot_series.py @@ -59,14 +59,14 @@ def memory_fractions(sub: pd.DataFrame) -> list[float | None]: def _memory_cap_mb(sub: pd.DataFrame) -> float: """Lambda memory cap in MB, for the colorbar's fraction<->MB twin axis. - Reads ``memory_gb`` (uniform across the series -- the benchmark pins 2 GB) and + Reads ``memory_gb`` (uniform across the series -- the benchmark pins 4 GB) and falls back to 2048 MB when the column is absent or empty. """ if "memory_gb" in sub.columns: caps = sub["memory_gb"].dropna() if not caps.empty: return float(caps.iloc[0]) * 1024.0 - return 2.0 * 1024.0 + return 4.0 * 1024.0 # issue #193: benchmark pins 4 GB def _merge_history(df: pd.DataFrame) -> pd.DataFrame: @@ -97,6 +97,21 @@ def _frozen_history(df: pd.DataFrame) -> pd.DataFrame: return _merge_history(df[~_codec_mask(df)]) +def _matrix_mask(df: pd.DataFrame) -> pd.Series: + """Boolean mask of the live matrix rows (issue #193): those carrying a + non-null ``index_backend`` (inline|sidecar). Absent column (a pre-#193 + parquet) -> all False, so the matrix figures stay empty until its first + merge lands.""" + if "index_backend" not in df.columns: + return pd.Series(False, index=df.index) + return df["index_backend"].notna() + + +def _matrix_history(df: pd.DataFrame) -> pd.DataFrame: + """Retained merge points of the live inline-vs-sidecar matrix (issue #193).""" + return _merge_history(df[_matrix_mask(df)]) + + def _latest_of(hist: pd.DataFrame) -> pd.DataFrame: """Rows of the single most-recent run in an already-filtered/ordered history. @@ -317,6 +332,26 @@ def _codec_layout(hist: pd.DataFrame) -> tuple[list[list[str | None]], int, int] return grid, len(CODEC_ROWS), len(CODEC_COLS) +# Live matrix (issue #193): read-backend A/B x order. Retired the sharded/inner +# codec axis; those figures are frozen (see main()). +MATRIX_COLS = ["inline", "sidecar"] +MATRIX_ROWS = ["o9", "o10"] + + +def _matrix_layout(hist: pd.DataFrame) -> tuple[list[list[str | None]], int, int]: + """Place each live target in the fixed ``(order, index_backend)`` grid + (issue #193). Columns = ``MATRIX_COLS`` (inline, sidecar); rows = + ``MATRIX_ROWS`` (o9, o10). Empty cell -> ``None`` (order not landed yet).""" + by_slot: dict[tuple[str, str], str] = {} + meta = hist.dropna(subset=["target"]).drop_duplicates("target") + for _, r in meta.iterrows(): + by_slot[(str(r.get("grid_size", "")), str(r.get("index_backend", "")))] = r["target"] + grid: list[list[str | None]] = [ + [by_slot.get((order, backend)) for backend in MATRIX_COLS] for order in MATRIX_ROWS + ] + return grid, len(MATRIX_ROWS), len(MATRIX_COLS) + + def _draw_panel(ax, sub: pd.DataFrame, cost_col: str, cost_label: str, norm) -> None: """Draw one cost+runtime panel for a single target's merge history. @@ -517,6 +552,34 @@ def make_figure(df: pd.DataFrame, cost_col: str, cost_label: str, out_png: Path) ) +def make_matrix_latest_table(df: pd.DataFrame, out_png: Path) -> bool: + """Render the live inline-vs-sidecar latest-merge table (issue #193): the + ``index_backend.notna`` rows of the most recent merge. False until the new + matrix lands its first merge.""" + recs = _latest_of(_matrix_history(df)).to_dict(orient="records") + return _render_table(recs, _table_title("zagg inline vs sidecar (sharded, K=4)", recs), out_png) + + +def make_matrix_figure(df: pd.DataFrame, cost_col: str, cost_label: str, out_png: Path) -> bool: + """Render the live inline-vs-sidecar figure (issue #193): the + ``index_backend.notna`` rows in the fixed ``MATRIX_ROWS x MATRIX_COLS`` grid. + Returns False when no matrix rows are retained yet.""" + hist = _matrix_history(df) + if hist.empty or hist["target"].dropna().empty: + return False + layout, nrows, ncols = _matrix_layout(hist) + return _render_panel_grid( + hist, + layout, + nrows, + ncols, + cost_col, + cost_label, + f"zagg inline vs sidecar (sharded, K=4) \u2014 {cost_label} vs merge history", + out_png, + ) + + def make_codec_figure(df: pd.DataFrame, cost_col: str, cost_label: str, out_png: Path) -> bool: """Render the forward sharded-vs-inner figure (issue #133): the ``codec.notna`` rows in a fixed 2x3 grid (cols = sharded/inner, rows = o9->o11). Returns False @@ -540,52 +603,64 @@ def make_codec_figure(df: pd.DataFrame, cost_col: str, cost_label: str, out_png: def write_index( outdir: Path, - rendered: list[str], - codec_rendered: list[str], + matrix_rendered: list[str], *, - codec_table_png: bool = False, - latest_png: bool = False, + matrix_table_png: bool = False, has_md: bool = False, has_json: bool = False, ) -> None: - """Emit a minimal Pages index embedding the rendered figures. - - Order (issue #133): the forward sharded-vs-inner section on top -- its 2x3 - codec table then its codec charts -- then the frozen historical section - (latest table + frozen charts) below, so the new matrix leads.""" + """Emit a Pages index: the live inline-vs-sidecar matrix on top (issue + #193), then an ARCHIVED section embedding the retained (frozen) codec + + historical PNGs if they still exist in ``outdir`` (they persist on the + benchmarks branch; this PR stopped regenerating them).""" blocks: list[str] = [] - # --- forward sharded-vs-inner section (on top) --- links = [] if has_md: links.append('latest.md') if has_json: links.append('metrics.json') - if codec_table_png: + + # --- live matrix section (on top) --- + if matrix_table_png: block = ( - "

Sharded vs inner-chunk — latest merge

\n" - 'sharded vs inner benchmark table' + "

inline vs sidecar (sharded, K=4) \u2014 latest merge

\n" + 'inline vs sidecar benchmark table' ) if links: - block += f"\n

Machine-readable: {' · '.join(links)}

" + block += f"\n

Machine-readable: {' \u00b7 '.join(links)}

" blocks.append(block) blocks += [ - f'

{name} (sharded vs inner)

\n{name}_codec' - for name in codec_rendered + f'

{name} (inline vs sidecar)

\n{name}_matrix' + for name in matrix_rendered ] - # --- frozen historical section (below) --- - if latest_png: - block = ( - "

Frozen historical — latest merge

\n" - 'latest benchmark table' + + # --- archived section (below): retained frozen PNGs, embedded if present --- + archived: list[str] = [] + if (outdir / "codec_table.png").exists(): + archived.append( + "

Sharded vs inner-chunk (archived)

\n" + 'archived codec table' ) - # The companions are linked above with the forward table; if there is no - # forward table, link them here so they stay reachable. - if links and not codec_table_png: - block += f"\n

Machine-readable: {' · '.join(links)}

" - blocks.append(block) - blocks += [ - f'

{name} (frozen)

\n{name}' for name in rendered - ] + if (outdir / "latest_table.png").exists(): + archived.append( + "

Frozen historical (archived)

\n" + 'archived historical table' + ) + for name in FIGURES: + for suffix, tag in (("_codec", "sharded vs inner"), ("", "frozen")): + if (outdir / f"{name}{suffix}.png").exists(): + archived.append( + f"

{name} ({tag}, archived)

\n" + f'archived {name}{suffix}' + ) + if archived: + blocks.append( + "
\n

Archived (frozen as of issue #193)

\n" + "

The pre-#193 codec (sharded/inner) + historical series, retained " + "but no longer updated.

" + ) + blocks += archived + if not blocks: blocks = [ "

No retained benchmark runs yet. Charts appear after the first merge to main.

" @@ -597,8 +672,8 @@ def write_index( "\n" "\n

zagg Lambda benchmark

\n" - "

arm64 · 2 GB · one shard/target · densest cell over the NEON SERC AOP box. " - "Each point is a merge to main.

\n" + "

arm64 \u00b7 4 GB \u00b7 one shard/target \u00b7 densest cell over the NEON SERC " + "AOP box. Each point is a merge to main.

\n" f"{imgs}\n\n" ) (outdir / "index.html").write_text(html) @@ -614,44 +689,42 @@ def main(argv: list[str] | None = None) -> int: outdir.mkdir(parents=True, exist_ok=True) df = pd.read_parquet(args.series) if Path(args.series).exists() else pd.DataFrame() - # Forward sharded-vs-inner figures (issue #133), rendered above the frozen - # ones; the frozen figures keep their original file names + layout. - codec_rendered, rendered = [], [] + # Live matrix (issue #193): inline-vs-sidecar, sharded, K=4. The retired + # codec (sharded/inner) + frozen historical figures are NO LONGER + # regenerated -- their PNGs are retained on the benchmarks branch as-is + # (archived), and only the matrix figures below update going forward. + matrix_rendered: list[str] = [] for name, (col, label) in FIGURES.items(): - if not df.empty and make_codec_figure(df, col, label, outdir / f"{name}_codec.png"): - codec_rendered.append(name) - if not df.empty and make_figure(df, col, label, outdir / f"{name}.png"): - rendered.append(name) - - # Latest-merge snapshots: the forward codec table (on top) + the frozen table, - # each embedded live in the docs, plus the human/agent-readable companions - # latest.md + metrics.json (issue #110, retained across both). - codec_table_png = not df.empty and make_codec_latest_table(df, outdir / "codec_table.png") - latest_png = not df.empty and make_latest_table(df, outdir / "latest_table.png") - has_md = not df.empty and write_latest_markdown(df, outdir / "latest.md") - has_json = not df.empty and write_latest_metrics(df, outdir / "metrics.json") + if not df.empty and make_matrix_figure(df, col, label, outdir / f"{name}_matrix.png"): + matrix_rendered.append(name) + + matrix_table_png = not df.empty and make_matrix_latest_table(df, outdir / "matrix_table.png") + # Machine-readable companions track the live matrix's latest merge. + matrix_df = df[_matrix_mask(df)] if not df.empty else df + has_md = not matrix_df.empty and write_latest_markdown(matrix_df, outdir / "latest.md") + has_json = not matrix_df.empty and write_latest_metrics(matrix_df, outdir / "metrics.json") write_index( outdir, - rendered, - codec_rendered, - codec_table_png=codec_table_png, - latest_png=latest_png, + matrix_rendered, + matrix_table_png=matrix_table_png, has_md=has_md, has_json=has_json, ) extras = [ n for n, ok in ( - ("codec_table", codec_table_png), - ("table", latest_png), + ("matrix_table", matrix_table_png), ("latest.md", has_md), ("metrics.json", has_json), ) if ok ] - nfig = len(rendered) + len(codec_rendered) - print(f"rendered {nfig} figure(s){' + ' + ', '.join(extras) if extras else ''} -> {outdir}") + print( + f"rendered {len(matrix_rendered)} matrix figure(s)" + f"{' + ' + ', '.join(extras) if extras else ''} -> {outdir} " + "(codec + frozen tiers archived, not regenerated)" + ) return 0 diff --git a/.github/scripts/run_benchmark.py b/.github/scripts/run_benchmark.py index 4bc578f..0daa6e9 100644 --- a/.github/scripts/run_benchmark.py +++ b/.github/scripts/run_benchmark.py @@ -117,6 +117,22 @@ def run_target( # default, so frozen/legacy targets dispatch byte-identically to before. if "sharded" in target: config.output.setdefault("grid", {})["sharded"] = bool(target["sharded"]) + # Read-backend A/B (issue #193): the live matrix sets data_source.index from + # each target's ``index_backend`` (inline vs sidecar), so one config per + # order drives both columns. sidecar consumes/builds the granule-keyed + # manifests; inline builds the chunk map on the fly. Absent -> config's own + # index (frozen/legacy/provisional targets dispatch unchanged). + backend = target.get("index_backend") + if backend == "sidecar": + config.data_source["index"] = { + "backend": "sidecar", + "on_miss": "build", + "store": "s3://sliderule-public-cors/zagg-index/ATL03/007", + } + elif backend == "inline": + config.data_source["index"] = {"backend": "inline"} + elif backend == "hierarchical": + config.data_source["index"] = {"backend": "hierarchical"} # parent_order lives in the config grid for HEALPix; the kwarg is just a # legacy fallback. Rect grids ignore it. ``from_config`` gives us the grid # object the area/cost derivation needs. @@ -137,6 +153,9 @@ def run_target( # "inner" with the real inner column, so the renderer needs this to # give them their own panel instead of silently overwriting it. read=target.get("read"), + # Read-backend axis (issue #193): "inline"|"sidecar" -- the live matrix's + # A/B, split by the renderer into its two columns. None on frozen rows. + index_backend=target.get("index_backend"), ) if dry_run: diff --git a/deployment/aws/template.yaml b/deployment/aws/template.yaml index 31206ed..6aa9b1f 100644 --- a/deployment/aws/template.yaml +++ b/deployment/aws/template.yaml @@ -72,8 +72,11 @@ Parameters: MemorySize: Type: Number - Default: 2048 - Description: Function memory in MB. + Default: 4096 + Description: >- + Function memory in MB. 4096 (~2.3 vCPU) is the issue #180 sizing default: + the pipeline saturates its available parallelism at ~1.3 effective cores, + faster and cheaper per AOI than 2048 (issue #193). Timeout: Type: Number diff --git a/src/zagg/dispatch.py b/src/zagg/dispatch.py index a32b26b..cd5b550 100644 --- a/src/zagg/dispatch.py +++ b/src/zagg/dispatch.py @@ -317,7 +317,7 @@ def shutdown(self) -> None: # the constants inlined into ``_run_lambda`` before this extraction; surfaced # here so :meth:`LambdaExecutor.measure_cost` and the runner's presentation # read one source. -LAMBDA_MEMORY_GB = 2.0 +LAMBDA_MEMORY_GB = 4.0 # issue #193: production worker sized to 4 GB (2.3 vCPU) LAMBDA_PRICE_PER_GB_SEC = 0.0000133334 diff --git a/tests/data/benchmark/README.md b/tests/data/benchmark/README.md index 5ba59f6..48a14d1 100644 --- a/tests/data/benchmark/README.md +++ b/tests/data/benchmark/README.md @@ -25,6 +25,10 @@ aggregator). ## Conventions +- **Live matrix (issue #193):** tdigest, sharded output, `granule_workers=4`, at + **4 GB** workers — the read-backend A/B (`inline` vs `sidecar`) over **o9 + o10**. + Four targets: `tdigest_healpix_{o9,o10}_{inline,sidecar}`. The retired + sharded/inner codec + `cached` axes are frozen (retained, not rerun). - **One shard per target** — the *densest* cell over the AOI, so cost/runtime deltas track code, not data drift. - **Densest = most granules; ties broken by lowest `shard_key`** — the @@ -42,26 +46,37 @@ aggregator). ## Plot layout (`plot_series.py`) -`plot_series.py` renders **two figure families**, split on the `codec` column -(issue #133): the forward sharded-vs-inner matrix (`codec` non-null) on top, and -the frozen historical series (`codec` null) below. +`plot_series.py` renders the **live matrix** on top, and embeds the retained +**archived** figures (frozen as of issue #193) below. -### Forward matrix — fixed 2×3 (`*_codec.png` + `codec_table.png`) +### Live matrix — inline vs sidecar, fixed 2×2 (`*_matrix.png` + `matrix_table.png`) -The forward charts (`cost_per_shard_codec.png`, `cost_per_100km2_codec.png`) lay -the codec rows out in a **fixed** grid keyed to the experiment, not the data: +The live matrix (issue #193) is **tdigest, sharded, `granule_workers=4`, at 4 GB** +— the read-backend A/B. The charts (`cost_per_shard_matrix.png`, +`cost_per_100km2_matrix.png`) lay the rows out in a **fixed** grid keyed to the +experiment, split on the `index_backend` column: -- **Columns = the ShardingCodec A/B.** Left = `sharded`, right = `inner`. -- **Rows = order, largest-first.** `o9` (top) → `o10` → `o11` (bottom). A row whose - order hasn't landed yet (e.g. `o9` before its shard map is built) renders blank, - so the matrix shape stays stable. +- **Columns = the read backend.** Left = `inline`, right = `sidecar`. +- **Rows = order, largest-first.** `o9` (top) → `o10` (bottom). A row whose order + hasn't landed renders blank, so the shape stays stable. -`codec_table.png` is the latest-merge table for these rows. +`matrix_table.png` is the latest-merge table for these rows; `latest.md` / +`metrics.json` track this matrix. -### Frozen historical — data-driven (`*.png` + `latest_table.png`) +### Archived: forward codec matrix (frozen, `*_codec.png` + `codec_table.png`) -The frozen charts (`cost_per_shard.png`, `cost_per_100km2.png`) keep the -**data-driven** layout (issue #121 review) for the retired rect/gain_bias rows — +**Retired as of issue #193** — the ShardingCodec sharded-vs-inner + read (`cached`) +A/B. Its PNGs are **retained on the `benchmarks` branch** and embedded in the +archived section of the index, but **no longer regenerated**. It was a fixed 2×3 +grid: columns `sharded` / `inner` / `cached`, rows `o9` → `o10` → `o11`. Split on +the `codec` column (issue #133). The render functions (`make_codec_figure`, +`_codec_layout`) remain in `plot_series.py` for on-demand regeneration. + +### Archived: frozen historical — data-driven (`*.png` + `latest_table.png`) + +**Also frozen as of issue #193** (retained, not regenerated). The charts +(`cost_per_shard.png`, `cost_per_100km2.png`) keep the **data-driven** layout +(issue #121 review) for the retired rect/gain_bias rows — nothing is keyed to a fixed target list: - **Columns = grid family.** The **left** column is the rectilinear (`rect_*`) @@ -86,10 +101,11 @@ nothing is keyed to a fixed target list: cost axis back down to 0. A failed shard zeros both series at the same merge, so the one cost `x` flags the failure for both. -When adding or rearranging targets, keep these conventions — the frozen layout is -derived from `grid_type` / `aggregator` / `shard_area_km2`, and the forward layout -from `grid_size` / `codec`, so getting that metadata right in `targets.json` is -what places the panel. +When adding or rearranging targets, keep these conventions — the **live matrix** +layout is derived from `grid_size` / `index_backend`, and the archived layouts from +`grid_size` / `codec` (forward) and `grid_type` / `aggregator` / `shard_area_km2` +(frozen historical), so getting that metadata right in `targets.json` is what +places the panel. ## Add a target diff --git a/tests/data/benchmark/configs/atl03_tdigest_healpix_o10.yaml b/tests/data/benchmark/configs/atl03_tdigest_healpix_o10.yaml index 0ec1d36..ac6ca17 100644 --- a/tests/data/benchmark/configs/atl03_tdigest_healpix_o10.yaml +++ b/tests/data/benchmark/configs/atl03_tdigest_healpix_o10.yaml @@ -21,11 +21,10 @@ # Confidence filter, multi-level form and read_plan match atl03.yaml (the planned # segment->photon IO of issue #43 applies here too); only the aggregation differs. data_source: - # Pinned read backend (issue #170): the UNCACHED pure-h5coro baseline - # column. The package default flipped to the compiled `inline` path; an - # unpinned config would silently move this series onto the fast path. - index: - backend: hierarchical + # issue #193: the sharded inline-vs-sidecar matrix. The read backend is set + # per target (index_backend: inline|sidecar) by run_benchmark; granule-level + # read concurrency is pinned to K=4 (issue #180 sizing sweet spot). + granule_workers: 4 reader: h5coro driver: s3 groups: [gt1l, gt1r, gt2l, gt2r, gt3l, gt3r] @@ -93,4 +92,5 @@ output: indexing_scheme: nested parent_order: 10 # shard / dispatch unit: 4^(19-10) = 512x512 cells chunk_inner: 13 # inner Zarr chunk: 4^(19-13) = 64x64 cells (K=64 per shard) + sharded: true # issue #193: sharded output fixed on for the matrix child_order: 19 # leaf cell resolution (~10 m) diff --git a/tests/data/benchmark/configs/atl03_tdigest_healpix_o9.yaml b/tests/data/benchmark/configs/atl03_tdigest_healpix_o9.yaml index 8ba45f4..8ad8a17 100644 --- a/tests/data/benchmark/configs/atl03_tdigest_healpix_o9.yaml +++ b/tests/data/benchmark/configs/atl03_tdigest_healpix_o9.yaml @@ -23,11 +23,10 @@ # Confidence filter, multi-level form and read_plan match atl03.yaml (the planned # segment->photon IO of issue #43 applies here too); only the aggregation differs. data_source: - # Pinned read backend (issue #170): the UNCACHED pure-h5coro baseline - # column. The package default flipped to the compiled `inline` path; an - # unpinned config would silently move this series onto the fast path. - index: - backend: hierarchical + # issue #193: the sharded inline-vs-sidecar matrix. The read backend is set + # per target (index_backend: inline|sidecar) by run_benchmark; granule-level + # read concurrency is pinned to K=4 (issue #180 sizing sweet spot). + granule_workers: 4 reader: h5coro driver: s3 groups: [gt1l, gt1r, gt2l, gt2r, gt3l, gt3r] @@ -95,4 +94,5 @@ output: indexing_scheme: nested parent_order: 9 # shard / dispatch unit: 4^(19-9) = 1024x1024 cells chunk_inner: 13 # inner Zarr chunk: 4^(19-13) = 64x64 cells (K=256 per shard) + sharded: true # issue #193: sharded output fixed on for the matrix child_order: 19 # leaf cell resolution (~10 m) diff --git a/tests/data/benchmark/targets.json b/tests/data/benchmark/targets.json index f311787..d706bc6 100644 --- a/tests/data/benchmark/targets.json +++ b/tests/data/benchmark/targets.json @@ -1,5 +1,5 @@ { - "description": "Pinned Lambda-benchmark targets (issue #110). Each target dispatches ONE shard -- the densest cell over the NEON SERC AOP box -- so cost/runtime deltas track code changes, not data drift. Forward matrix (issue #133): all tdigest / HEALPix / arrow, a 2x3 sharded-vs-inner ShardingCodec (issue #108) A/B across orders o9/o10/o11. The two columns of each order share ONE config + ONE shard map and differ only by the per-target codec key (sharded: true|false), which run_benchmark applies to output.grid.sharded. The carrier is config-driven (issue #132): aggregation.handoff defaults to arrow, so the matrix targets carry no redundant per-target handoff key. Shard maps are shared per (grid, order): the densest shard does not depend on the codec. Rebuild + drift check: tests/test_benchmark_shardmap.py. Cached-read columns (issue #170): each healpix tdigest order carries a *_cached companion target -- identical config except the read backend (sidecar over zagg-index/ATL03/007 granule manifests, self-populating via on_miss: build) -- so the matrix tracks the uncached pure-h5coro baseline and the compiled cached read side by side.", + "description": "Pinned Lambda-benchmark targets. Live matrix (issue #193): tdigest aggregation, sharded output, granule_workers=4, at 4 GB workers -- the read-backend A/B (inline vs sidecar) over o9 and o10 NEON shards. One shard/target (the pinned densest cell). run_benchmark applies data_source.index from each target's index_backend; sharded + granule_workers live in the shared config.", "aoi": { "file": "AOP_NEON.geojson", "name": "NEON SERC AOP box" @@ -57,89 +57,37 @@ } }, "targets": { - "tdigest_healpix_o11_sharded": { - "config": "configs/atl03_tdigest_healpix_o11.yaml", - "shardmap": "healpix_o11", - "aggregator": "tdigest", - "grid_type": "healpix", - "grid_size": "o11", - "codec": "sharded", - "sharded": true - }, - "tdigest_healpix_o11_inner": { - "config": "configs/atl03_tdigest_healpix_o11.yaml", - "shardmap": "healpix_o11", + "tdigest_healpix_o9_inline": { + "config": "configs/atl03_tdigest_healpix_o9.yaml", + "shardmap": "healpix_o9", "aggregator": "tdigest", "grid_type": "healpix", - "grid_size": "o11", - "codec": "inner", - "sharded": false + "grid_size": "o9", + "index_backend": "inline" }, - "tdigest_healpix_o11_cached": { - "config": "configs/atl03_tdigest_healpix_o11_cached.yaml", - "shardmap": "healpix_o11", + "tdigest_healpix_o9_sidecar": { + "config": "configs/atl03_tdigest_healpix_o9.yaml", + "shardmap": "healpix_o9", "aggregator": "tdigest", "grid_type": "healpix", - "grid_size": "o11", - "codec": "inner", - "sharded": false, - "read": "cached" + "grid_size": "o9", + "index_backend": "sidecar" }, - "tdigest_healpix_o10_sharded": { + "tdigest_healpix_o10_inline": { "config": "configs/atl03_tdigest_healpix_o10.yaml", "shardmap": "healpix_o10", "aggregator": "tdigest", "grid_type": "healpix", "grid_size": "o10", - "codec": "sharded", - "sharded": true + "index_backend": "inline" }, - "tdigest_healpix_o10_inner": { + "tdigest_healpix_o10_sidecar": { "config": "configs/atl03_tdigest_healpix_o10.yaml", "shardmap": "healpix_o10", "aggregator": "tdigest", "grid_type": "healpix", "grid_size": "o10", - "codec": "inner", - "sharded": false - }, - "tdigest_healpix_o10_cached": { - "config": "configs/atl03_tdigest_healpix_o10_cached.yaml", - "shardmap": "healpix_o10", - "aggregator": "tdigest", - "grid_type": "healpix", - "grid_size": "o10", - "codec": "inner", - "sharded": false, - "read": "cached" - }, - "tdigest_healpix_o9_sharded": { - "config": "configs/atl03_tdigest_healpix_o9.yaml", - "shardmap": "healpix_o9", - "aggregator": "tdigest", - "grid_type": "healpix", - "grid_size": "o9", - "codec": "sharded", - "sharded": true - }, - "tdigest_healpix_o9_inner": { - "config": "configs/atl03_tdigest_healpix_o9.yaml", - "shardmap": "healpix_o9", - "aggregator": "tdigest", - "grid_type": "healpix", - "grid_size": "o9", - "codec": "inner", - "sharded": false - }, - "tdigest_healpix_o9_cached": { - "config": "configs/atl03_tdigest_healpix_o9_cached.yaml", - "shardmap": "healpix_o9", - "aggregator": "tdigest", - "grid_type": "healpix", - "grid_size": "o9", - "codec": "inner", - "sharded": false, - "read": "cached" + "index_backend": "sidecar" } }, "provisional_targets": { @@ -165,7 +113,8 @@ "shardmap": "healpix_o9_88s", "aggregator": "tdigest", "grid_type": "healpix", - "grid_size": "o9_88s" + "grid_size": "o9_88s", + "index_backend": "hierarchical" }, "tdigest_healpix_o9_88s_cached": { "config": "configs/atl03_tdigest_healpix_o9_cached.yaml", @@ -173,14 +122,16 @@ "aggregator": "tdigest", "grid_type": "healpix", "grid_size": "o9_88s", - "read": "cached" + "read": "cached", + "index_backend": "sidecar" }, "tdigest_healpix_o10_88s": { "config": "configs/atl03_tdigest_healpix_o10.yaml", "shardmap": "healpix_o10_88s", "aggregator": "tdigest", "grid_type": "healpix", - "grid_size": "o10_88s" + "grid_size": "o10_88s", + "index_backend": "hierarchical" }, "tdigest_healpix_o10_88s_cached": { "config": "configs/atl03_tdigest_healpix_o10_cached.yaml", @@ -188,7 +139,9 @@ "aggregator": "tdigest", "grid_type": "healpix", "grid_size": "o10_88s", - "read": "cached" + "read": "cached", + "index_backend": "sidecar" } - } + }, + "_frozen_note": "Pre-#193 targets (sharded/inner codec A/B + *_cached read axis) are retired from the live matrix as of issue #193; their series rows + PNGs are retained/frozen on the benchmarks branch. The live matrix below is tdigest, sharded, inline-vs-sidecar, K=4, o9+o10 (issue #193)." } diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 3519aea..61210f3 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -133,7 +133,7 @@ def test_codec_column_is_last_and_threaded(monkeypatch): # tail position. cols = bench_metrics.RECORD_COLUMNS assert cols.index("codec") < cols.index("read") < cols.index("total_wall_s") - assert cols[-4:] == ["total_wall_s", "setup_s", "fanout_s", "finalize_s"] + assert cols[-5:] == ["total_wall_s", "setup_s", "fanout_s", "finalize_s", "index_backend"] g = HealpixGrid(parent_order=11, child_order=19) rec = bench_metrics.build_record(_summary(), grid=g, context={"codec": "sharded"}) assert rec["codec"] == "sharded" @@ -142,6 +142,8 @@ def test_codec_column_is_last_and_threaded(monkeypatch): _summary(), grid=g, context={"codec": "inner", "read": "cached"} ) assert cached["read"] == "cached" + live = bench_metrics.build_record(_summary(), grid=g, context={"index_backend": "sidecar"}) + assert live["index_backend"] == "sidecar" legacy = bench_metrics.build_record(_summary(), grid=g, context={"target": "t"}) assert legacy["codec"] is None assert legacy["read"] is None @@ -169,7 +171,7 @@ def test_run_target_threads_codec_into_record(): # run_benchmark must record the target's codec onto the row (dry-run, no AWS). manifest, base = run_benchmark.load_targets(str(BENCH / "targets.json")) rec = run_benchmark.run_target( - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", manifest, base, store=None, @@ -178,7 +180,8 @@ def test_run_target_threads_codec_into_record(): context={"commit": "deadbee", "event": "pr"}, dry_run=True, ) - assert rec["codec"] == "inner" + assert rec["index_backend"] == "inline" # issue #193: the live matrix's A/B axis + assert rec["codec"] is None # codec axis retired from the live matrix def test_memory_pct_of_cap_maps_near_red(): @@ -293,7 +296,7 @@ def test_comment_markdown_worker_note_banner(): def test_run_target_dry_run(): manifest, base = run_benchmark.load_targets(str(BENCH / "targets.json")) rec = run_benchmark.run_target( - "tdigest_healpix_o11_sharded", + "tdigest_healpix_o9_sidecar", manifest, base, store=None, @@ -302,11 +305,11 @@ def test_run_target_dry_run(): context={"commit": "deadbee", "event": "pr"}, dry_run=True, ) - assert rec["target"] == "tdigest_healpix_o11_sharded" + assert rec["target"] == "tdigest_healpix_o9_sidecar" assert rec["aggregator"] == "tdigest" assert rec["grid_type"] == "healpix" - assert rec["shard_key"] == 5347394812217655307 - assert rec["shard_area_km2"] == pytest.approx(10.13, abs=0.2) + assert rec["shard_key"] == 5347395636851376137 # o9 densest cell + assert rec["shard_area_km2"] == pytest.approx(162.1, abs=1.0) # o9 HEALPix cell assert rec["total_obs"] is None # no dispatch in dry-run @@ -336,7 +339,7 @@ def test_main_fails_on_empty_target(tmp_path, monkeypatch): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--commit", "cafe123", "--out-json", @@ -360,7 +363,7 @@ def test_main_fails_on_null_memory_alone(tmp_path, monkeypatch): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--commit", "cafe123", "--out-json", @@ -381,7 +384,7 @@ def test_main_no_fail_on_empty_opts_out(tmp_path, monkeypatch): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--no-fail-on-empty", "--commit", "cafe123", @@ -403,7 +406,7 @@ def test_main_passes_on_populated_target(tmp_path, monkeypatch): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--commit", "cafe123", "--out-json", @@ -421,7 +424,7 @@ def test_main_dry_run_writes_outputs(tmp_path): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--dry-run", "--commit", "cafe123", @@ -432,8 +435,8 @@ def test_main_dry_run_writes_outputs(tmp_path): ] ) records = json.loads(out_json.read_text()) - assert len(records) == 1 and records[0]["target"] == "tdigest_healpix_o10_inner" - assert "tdigest_healpix_o10_inner" in out_md.read_text() + assert len(records) == 1 and records[0]["target"] == "tdigest_healpix_o10_inline" + assert "tdigest_healpix_o10_inline" in out_md.read_text() def test_main_dry_run_exempt_from_fail_on_empty(tmp_path): @@ -444,7 +447,7 @@ def test_main_dry_run_exempt_from_fail_on_empty(tmp_path): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--dry-run", "--commit", "cafe123", @@ -478,9 +481,9 @@ def fake(name, *a, **k): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--target", - "tdigest_healpix_o10_sharded", + "tdigest_healpix_o10_sidecar", "--commit", "cafe123", "--out-json", @@ -490,8 +493,8 @@ def fake(name, *a, **k): assert rc == 0 recs = json.loads(out_json.read_text()) assert [r["target"] for r in recs] == [ - "tdigest_healpix_o10_inner", - "tdigest_healpix_o10_sharded", + "tdigest_healpix_o10_inline", + "tdigest_healpix_o10_sidecar", ] @@ -517,9 +520,9 @@ def test_main_warms_auth_once_before_parallel_dispatch(tmp_path, monkeypatch): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--target", - "tdigest_healpix_o10_sharded", + "tdigest_healpix_o10_sidecar", "--commit", "cafe123", "--out-json", @@ -543,9 +546,9 @@ def test_main_skips_auth_warmup_on_dry_run(tmp_path, monkeypatch): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--target", - "tdigest_healpix_o10_sharded", + "tdigest_healpix_o10_sidecar", "--dry-run", "--commit", "cafe123", @@ -572,7 +575,7 @@ def fake(name, *a, **k): "--targets", str(BENCH / "targets.json"), "--target", - "tdigest_healpix_o10_inner", + "tdigest_healpix_o10_inline", "--target", "does_not_exist", "--commit", @@ -697,7 +700,7 @@ def fake_agg(config, **kwargs): monkeypatch.setattr(runner, "agg", fake_agg) run_benchmark.run_target( - "tdigest_healpix_o11_sharded", + "tdigest_healpix_o9_sidecar", manifest, base, store="s3://b/x.zarr", @@ -900,34 +903,46 @@ def _write_json(tmp_path, obj): # --- plot_series (smoke; needs matplotlib) -------------------------------- +def _matrix_rows(commit, **kw): + """Live-matrix rows (issue #193): tdigest o9/o10 x inline/sidecar, carrying + index_backend so plot_series.main renders the matrix figures.""" + rows = [] + for order in ("o9", "o10"): + for backend in ("inline", "sidecar"): + r = _rec_row(commit, f"tdigest_healpix_{order}_{backend}", **kw) + r["grid_size"] = order + r["index_backend"] = backend + rows.append(r) + return rows + + def test_plot_series_smoke(tmp_path): pytest.importorskip("matplotlib") import plot_series rows = [ - _rec_row(f"c{i}", t, cost=0.004 + i * 0.001, rt=180 + i * 10) - for i in range(3) - for t in ("t1", "t2") + r for i in range(3) for r in _matrix_rows(f"c{i}", cost=0.004 + i * 0.001, rt=180 + i * 10) ] series = tmp_path / "series.parquet" update_series.save_series(update_series.records_to_frame(rows), series) outdir = tmp_path / "site" plot_series.main(["--series", str(series), "--out", str(outdir)]) assert (outdir / "index.html").exists() - assert (outdir / "cost_per_shard.png").exists() - assert (outdir / "cost_per_100km2.png").exists() - # Latest-merge snapshot artifacts (issue #110). - assert (outdir / "latest_table.png").exists() + # Live matrix figures (issue #193): *_matrix.png + matrix_table.png. + assert (outdir / "cost_per_shard_matrix.png").exists() + assert (outdir / "cost_per_100km2_matrix.png").exists() + assert (outdir / "matrix_table.png").exists() assert (outdir / "latest.md").exists() assert (outdir / "metrics.json").exists() + # The retired codec/frozen figures are NOT regenerated. + assert not (outdir / "codec_table.png").exists() + assert not (outdir / "cost_per_shard.png").exists() def _latest_rows(commit, ts, **kw): - rows = [] - for t in ("t1", "t2"): - r = _rec_row(commit, t, **kw) + rows = _matrix_rows(commit, **kw) + for r in rows: r["timestamp"] = ts - rows.append(r) return rows @@ -948,7 +963,7 @@ def test_plot_series_latest_artifacts_pick_newest_merge(tmp_path): assert "old111" not in md # ...and only it metrics = json.loads((outdir / "metrics.json").read_text()) assert {r["commit"] for r in metrics} == {"new999"} - assert {r["target"] for r in metrics} == {"t1", "t2"} + assert {r["index_backend"] for r in metrics} == {"inline", "sidecar"} def test_write_latest_metrics_is_null_safe_json(tmp_path): @@ -984,7 +999,7 @@ def test_plot_series_empty_writes_placeholder(tmp_path): plot_series.main(["--series", str(tmp_path / "missing.parquet"), "--out", str(outdir)]) # No data -> index exists with placeholder, no PNGs. assert (outdir / "index.html").exists() - assert not (outdir / "cost_per_shard.png").exists() + assert not (outdir / "cost_per_shard_matrix.png").exists() def test_update_series_main_retains_merge_only(tmp_path): @@ -1458,6 +1473,42 @@ def test_codec_layout_blanks_missing_order(): assert grid[1] == ["tdigest_healpix_o10_sharded", "tdigest_healpix_o10_inner", None] +def test_matrix_layout_is_inline_sidecar_by_order(): + # issue #193: the live matrix is a fixed (o9,o10) x (inline,sidecar) grid. + import plot_series + + rows = _matrix_rows("c0") + grid, nrows, ncols = plot_series._matrix_layout(update_series.records_to_frame(rows)) + assert (nrows, ncols) == (2, 2) # rows o9/o10, cols inline/sidecar + assert grid == [ + ["tdigest_healpix_o9_inline", "tdigest_healpix_o9_sidecar"], + ["tdigest_healpix_o10_inline", "tdigest_healpix_o10_sidecar"], + ] + + +def test_matrix_layout_blanks_missing_order(): + # An order with no rows leaves its cells None (grid shape stays fixed 2x2). + import plot_series + + rows = [r for r in _matrix_rows("c0") if r["grid_size"] == "o10"] + grid, nrows, ncols = plot_series._matrix_layout(update_series.records_to_frame(rows)) + assert (nrows, ncols) == (2, 2) + assert grid[0] == [None, None] # o9 absent + assert grid[1] == ["tdigest_healpix_o10_inline", "tdigest_healpix_o10_sidecar"] + + +def test_matrix_history_selects_index_backend_rows(): + # Only index_backend rows are the live matrix; codec/frozen rows excluded. + import plot_series + + df = update_series.records_to_frame( + _matrix_rows("c0") + _full_codec_matrix("c0") + [_rec_row("c0", "frozen_t")] + ) + hist = plot_series._matrix_history(df) + assert set(hist["index_backend"]) == {"inline", "sidecar"} + assert all(t.endswith(("_inline", "_sidecar")) for t in hist["target"]) + + def test_codec_and_frozen_histories_split_on_codec(): import plot_series @@ -1535,25 +1586,29 @@ def test_make_codec_latest_table_renders_codec_rows(tmp_path): assert plot_series.make_codec_latest_table(frozen_only, tmp_path / "x.png") is False -def test_plot_main_emits_codec_artifacts_above_frozen(tmp_path): +def test_plot_main_emits_matrix_and_archives_old(tmp_path): + # issue #193: main() renders the live inline/sidecar matrix; any retained + # codec/frozen PNGs already in the outdir are embedded as an ARCHIVED + # section below (and are NOT regenerated). pytest.importorskip("matplotlib") import plot_series - rows = _full_codec_matrix("c0") + [ - _rec_row("c0", "gain_bias_rect_3km", agg="gain_bias", grid="rectilinear") - ] + rows = _matrix_rows("c0") series = tmp_path / "series.parquet" update_series.save_series(update_series.records_to_frame(rows), series) outdir = tmp_path / "site" + outdir.mkdir() + # A pre-existing archived PNG (as it would sit on the benchmarks branch). + (outdir / "codec_table.png").write_bytes(b"\x89PNG\r\n\x1a\n") plot_series.main(["--series", str(series), "--out", str(outdir)]) - # Forward codec figures + table land beside the frozen ones. - assert (outdir / "cost_per_shard_codec.png").exists() - assert (outdir / "codec_table.png").exists() - assert (outdir / "cost_per_shard.png").exists() # the frozen rect row still renders + # Live matrix rendered; archived PNG left untouched, embedded below. + assert (outdir / "matrix_table.png").exists() + assert (outdir / "cost_per_shard_matrix.png").exists() + assert not (outdir / "codec_table.png").stat().st_size > 100 # not regenerated html = (outdir / "index.html").read_text() - # The forward (sharded-vs-inner) section is rendered ABOVE the frozen one. - assert "Sharded vs inner-chunk" in html and "Frozen historical" in html - assert html.index("Sharded vs inner-chunk") < html.index("Frozen historical") + assert "inline vs sidecar" in html + assert "Archived (frozen as of issue #193)" in html + assert html.index("inline vs sidecar") < html.index("Archived") def test_88s_nested_pin_invariant(): diff --git a/tests/test_dispatch.py b/tests/test_dispatch.py index 1e12990..90a1426 100644 --- a/tests/test_dispatch.py +++ b/tests/test_dispatch.py @@ -113,7 +113,7 @@ def test_preflight_sizes_pool_and_returns_report(self): def test_measure_cost_matches_arm64_pricing(self): ex = self._make() - # 3 s of Lambda compute at 2 GB. + # 3 s of Lambda compute at LAMBDA_MEMORY_GB (4 GB, issue #193). cost = ex.measure_cost({"lambda_duration": 3.0}) assert cost.compute_time_s == 3.0 assert cost.gb_seconds == pytest.approx(3.0 * LAMBDA_MEMORY_GB) @@ -158,7 +158,7 @@ def test_accumulates_results_cost_and_counts(self): assert report.total_obs == 60 assert report.cells_error == 0 assert len(report.results) == 3 - # Cost is folded in by the loop: 3 cells x 1 s x 2 GB. + # Cost is folded in by the loop: 3 cells x 1 s x LAMBDA_MEMORY_GB (4 GB). assert report.cost.compute_time_s == pytest.approx(3.0) assert report.cost.gb_seconds == pytest.approx(3.0 * LAMBDA_MEMORY_GB) ex.shutdown() diff --git a/tests/test_runner.py b/tests/test_runner.py index c7a1500..f816341 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1603,15 +1603,15 @@ def test_lambda_summary_keys_and_cost(self, monkeypatch, atl06_config): assert summary["backend"] == "lambda" assert summary["cells_with_data"] == 4 assert summary["total_obs"] == 12 - # 4 cells x 2 s x 2 GB = 16 GB-s; cost = 16 * arm64 price. + # 4 cells x 2 s x 4 GB = 32 GB-s; cost = 32 * arm64 price. assert summary["lambda_time_s"] == 8.0 - assert summary["gb_seconds"] == 16.0 + assert summary["gb_seconds"] == 32.0 assert summary["price_per_gb_sec"] == 0.0000133334 - assert summary["estimated_cost_usd"] == 16.0 * 0.0000133334 + assert summary["estimated_cost_usd"] == 32.0 * 0.0000133334 def test_lambda_cost_byte_identical_with_mixed_durations(self, monkeypatch, atl06_config): """estimated_cost_usd must equal the pre-refactor arithmetic order: - ``(sum(durations) * 2.0) * price`` computed once -- not a sum of + ``(sum(durations) * 4.0) * price`` computed once -- not a sum of per-cell ``cost_usd`` (which would diverge in the last FP ULP). Uses heterogeneous per-cell durations so the two orders actually differ. """ @@ -1676,8 +1676,8 @@ def test_lambda_cost_byte_identical_with_mixed_durations(self, monkeypatch, atl0 ) total = 0.1 + 0.2 + 0.3 + 12.7 # The exact pre-refactor order: one multiply over the summed time. - assert summary["gb_seconds"] == total * 2.0 - assert summary["estimated_cost_usd"] == (total * 2.0) * 0.0000133334 + assert summary["gb_seconds"] == total * 4.0 + assert summary["estimated_cost_usd"] == (total * 4.0) * 0.0000133334 # --------------------------------------------------------------------------- @@ -2053,7 +2053,7 @@ def test_one_invoke_per_event_and_rows_written_once(self, monkeypatch): assert summary["events_error"] == 0 assert summary["timesteps_processed"] == 4 assert summary["output_path"] == "s3://out/events.parquet" - assert summary["gb_seconds"] == pytest.approx(2.0 * 2.0) # 2 s x 2 GB + assert summary["gb_seconds"] == pytest.approx(2.0 * 4.0) # 2 s x 4 GB assert summary["results"] == written["rows"] assert summary["failures"] == []