diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9cc6f27..0abc32d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest flake8 aiohttp voluptuous + pip install pytest flake8 aiohttp voluptuous pillow - name: Lint with flake8 run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index bfe9385..595f7f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,60 @@ All notable changes to NOAA It All for Home Assistant will be documented in this The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.5.3] - Current +## [0.6.0] - Current + +### Added +- **The radar loop can now cover up to 24 hours instead of NOAA's fixed 50 minutes.** NOAA + publishes a ready-made animation at `{SITE}_loop.gif`, but it is fixed at ten frames covering + roughly 50 minutes, and only those ten frames exist on its server — so a longer loop cannot be + downloaded, it has to be collected. The Radar Loop entity now fetches the latest single scan on + each refresh, files it under the time NOAA published it, and assembles the animation itself from + an evenly time-spaced sample of what it holds. + + Set the window under **Settings → Devices & Services → NOAA It All → Configure**, which gained a + third step. It defaults to **24 hours**; `0` restores the previous behaviour exactly — NOAA's own + loop, proxied unchanged, with nothing written to disk. + + Things worth knowing: + - **The loop fills in over time.** A freshly configured loop is only as long as the history + collected so far and reaches its full length after that many hours of uptime. Below six frames + the card shows NOAA's own loop instead, so it is never blank and never worse than before. + - **Frames survive restarts**, stored as one small GIF per scan under + `/noaa_it_all/radar_frames//`. Budget a few megabytes per radar site. + Anything outside the window — or dated in the future by a wrong clock — is pruned on every + refresh. The directory is removed when the integration is deleted, when the entry is switched + to another forecast office, and when the option is set back to `0`, unless another configured + office is still building a loop from the same radar site. + - **The animation is larger than NOAA's**, and every open dashboard re-downloads it whenever it + changes. A 24-hour loop is capped at 72 frames (one every 20 minutes) and plays through in + about ten seconds; shorter windows are proportionally finer, with a 6-hour loop keeping roughly + one frame per scan. + - Frames are identified by `Last-Modified` — the time NOAA published the scan — which puts them + on the real volume-scan cadence rather than on our refresh boundary, and makes two refreshes + that see the same scan resolve to the same file. Hashing the image bytes would have been + actively wrong: two consecutive scans of a clear sky are genuinely identical, so a quiet night + would collapse into a single frame and the loop would cut straight from "clear" to "storm" with + no sense of time passing. + - Every failure — too few frames yet, Pillow missing, assembly failing, a disk that will not take + the frame — falls back to NOAA's own loop, and no failure path changes the picture already on + screen. +- **The Radar Loop entity exposes what it is actually showing.** `loop_mode` is `local` when the + animation was built here and `upstream` when it is NOAA's, alongside `loop_hours`, `frame_count`, + `window_start` and `window_end` — so a loop quietly shorter than configured is visible from a + template rather than only from the logs. + +### Changed +- **The locally built loop is opaque where NOAA's is transparent.** Source frames are transparent + overlays that each carry their own palette, and reconciling per-frame transparency across + differing palettes is the most reliable way to produce a psychedelic radar loop. Frames are + composited onto a solid black background before being combined. Cards that relied on the radar + loop being transparent over a custom background will see black instead; set the option to `0` to + keep NOAA's transparent animation. +- **Cached image validators are only offered back to the URL they came from.** The radar loop + entity fetches two different resources, and an `ETag` from the single frame must never be sent as + a validator for the animation — a server answering `304` to that would hand back the wrong image. + +## [0.5.3] - Previous ### Fixed - **A network blip no longer blanks the NOAA image cards.** Every image entity's `async_image()` diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 230f1df..0218386 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -4,11 +4,12 @@ This guide provides detailed configuration examples for NOAA It All, including e ## Table of Contents 1. [Installation Methods](#installation-methods) -2. [Entity Configuration](#entity-configuration) -3. [Device Grouping](#device-grouping) -4. [Dashboard Card Examples](#dashboard-card-examples) -5. [Automation Examples](#automation-examples) -6. [Script Examples](#script-examples) +2. [Options](#options) +3. [Entity Configuration](#entity-configuration) +4. [Device Grouping](#device-grouping) +5. [Dashboard Card Examples](#dashboard-card-examples) +6. [Automation Examples](#automation-examples) +7. [Script Examples](#script-examples) ## Installation Methods @@ -38,6 +39,74 @@ longitude: -117.1611 > up. Remove the block and add the integration through **Settings → Devices & Services → Add > Integration → NOAA It All**. +## Options + +**Settings** → **Devices & Services** → **NOAA It All** → **Configure**. The flow walks through +latitude and longitude, then the forecast office, then the radar loop. Saving reloads the +integration so the new values take effect immediately. + +| Option | Default | Notes | +|---|---|---| +| Latitude / Longitude | Home Assistant's Home location | Used for alerts, surf and aurora | +| NWS Forecast Office | Nearest office to those coordinates | Determines the radar site | +| Hours of radar history | `24` | Length of the Radar Loop animation, `0`–`24` | + +### Hours of radar history + +NOAA's own radar animation is fixed at about 50 minutes, and only its ten most recent frames exist +on the server — there is no longer version to download. To show more than that, the integration +saves one frame each time it refreshes and assembles the animation itself. + +```yaml +# Stored in the config entry's options +office_code: "SGX" +latitude: 32.7157 +longitude: -117.1611 +radar_loop_hours: 24 +``` + +- **`0`** — serve NOAA's own ~50 minute loop unchanged and store nothing on disk. This is how the + integration behaved before version 0.6.0. +- **`1`–`24`** — build the loop locally over that window. + +What to expect when it is on: + +- **The loop fills in over time.** It starts at whatever history has been collected and reaches + full length after that many hours of uptime. Until there are at least six frames, the card falls + back to NOAA's own loop rather than showing a near-still image. +- **Frames persist across restarts**, under `/noaa_it_all/radar_frames//`, one + small GIF per scan. Expect a few megabytes per radar site. Frames outside the window are deleted + on every refresh. The directory is removed when you delete the integration, when you switch the + entry to a different forecast office, or when you set this option back to `0` — unless another + configured office is still building a loop from the same radar site. +- **Frame spacing follows the window.** The animation is capped at 72 frames and plays through in + about ten seconds, so a 24-hour loop steps every 20 minutes while a 6-hour loop keeps roughly one + frame per radar scan. +- **The file is larger than NOAA's**, and every open dashboard re-downloads it whenever it changes. + On a wall tablet on a mobile connection, prefer a shorter window. + +While the buffer is still filling, each refresh makes two requests instead of one — the latest +frame, plus NOAA's loop to display in the meantime. That stops once enough frames have been +collected. + +The entity exposes what it is actually doing as attributes: + +| Attribute | Meaning | +|---|---| +| `loop_mode` | `local` when showing an animation built here, `upstream` when showing NOAA's | +| `loop_hours` | The configured window | +| `frame_count` | Frames in the animation currently being served | +| `window_start` / `window_end` | Times of its oldest and newest frames | + +```yaml +# Alert when the radar loop quietly falls back to NOAA's short animation +template: + - binary_sensor: + - name: "Radar loop degraded" + state: > + {{ state_attr('image.noaa_ilm_weather_radar_loop', 'loop_mode') == 'upstream' }} +``` + ## Entity Configuration ### Understanding Entity IDs diff --git a/README.md b/README.md index 8306daa..40f03b6 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,17 @@ Visual representations of current conditions: **NOAA {OFFICE} Weather** (one per configured office): - **Radar Base Reflectivity** — Latest NEXRAD base reflectivity radar for your NWS office *(image.noaa_{office}_weather_radar_base_reflectivity)* -- **Radar Loop** — Animated NEXRAD radar loop *(image.noaa_{office}_weather_radar_loop)* +- **Radar Loop** — Animated NEXRAD radar loop, covering up to 24 hours *(image.noaa_{office}_weather_radar_loop)* + +> **Radar loop history**: NOAA's own radar animation is fixed at ten frames covering roughly 50 minutes, and only those ten frames exist on its server — a longer loop cannot simply be downloaded. So the integration collects one frame per refresh and assembles the animation itself. The window defaults to **24 hours** and is set under **Settings → Devices & Services → NOAA It All → Configure**; set it to `0` to serve NOAA's 50-minute loop unchanged and store nothing. +> +> Worth knowing before you turn it up: +> +> - **It fills in over time.** A newly configured loop starts at whatever history has been collected so far and reaches its full length after that many hours of uptime. Until there are enough frames, the card shows NOAA's own loop instead, so it is never blank. +> - **Frames survive restarts.** They are stored under `/noaa_it_all/radar_frames//`, so a restart does not send the loop back to the beginning. Budget a few megabytes per radar site; the directory is deleted if you remove the integration. +> - **The animation is bigger than NOAA's.** A 24-hour loop is sampled down to 72 frames (one every 20 minutes) and plays through in about ten seconds, but it is still a larger file that every open dashboard re-downloads whenever it changes. Shorter windows are proportionally finer: a 6-hour loop keeps one frame per scan. +> +> The entity's `loop_mode` attribute reports whether you are looking at a locally built animation (`local`) or NOAA's (`upstream`), alongside `frame_count`, `window_start` and `window_end`. > **Tip**: Image entities can be displayed on dashboards using the standard `picture-entity` or `picture-glance` cards. > diff --git a/custom_components/noaa_it_all/__init__.py b/custom_components/noaa_it_all/__init__.py index a39a2f4..ccb7ed4 100644 --- a/custom_components/noaa_it_all/__init__.py +++ b/custom_components/noaa_it_all/__init__.py @@ -11,8 +11,10 @@ HURRICANE_COORDINATOR_KEY, HURRICANE_IMAGES_ADDED_KEY, HURRICANE_SENSORS_ADDED_KEY, OFFICE_RADAR_SITES, OFFICE_TIDE_STATIONS, OFFICE_BUOY_STATIONS, + RADAR_FRAME_DIR, ) from .entry_config import resolve_entry_config +from .radar_loop import RadarFrameStore from .coordinator import ( SpaceWeatherCoordinator, HurricaneCoordinator, @@ -187,6 +189,39 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry): return unload_ok +async def async_remove_entry(hass: HomeAssistant, entry: ConfigEntry) -> None: + """Delete the radar frames this entry accumulated. + + Home Assistant calls this when the integration is removed rather than + merely unloaded. A day of radar is a few megabytes per site that nothing + else would ever clean up, and it sits in the configuration directory, so it + would otherwise ride along in every backup forever. + """ + office_code = resolve_entry_config(entry).get(CONF_OFFICE_CODE) + radar_site = OFFICE_RADAR_SITES.get(office_code) + if not radar_site: + return + + # Frames are keyed by radar site, and neighbouring offices can share one. + # Removing this entry must not take another entry's history with it. + for other in hass.config_entries.async_entries(DOMAIN): + if other.entry_id == entry.entry_id: + continue + other_office = resolve_entry_config(other).get(CONF_OFFICE_CODE) + if OFFICE_RADAR_SITES.get(other_office) == radar_site: + _LOGGER.debug( + "Keeping stored radar frames for %s; another entry still uses it", + radar_site, + ) + return + + store = RadarFrameStore( + hass, hass.config.path(DOMAIN, RADAR_FRAME_DIR), radar_site + ) + await store.async_remove_all() + _LOGGER.info("Removed stored radar frames for %s", radar_site) + + # Legacy function for YAML setup def setup(hass, config): """Set up the NOAA component (legacy YAML support).""" diff --git a/custom_components/noaa_it_all/config_flow.py b/custom_components/noaa_it_all/config_flow.py index 75f17ca..77e484d 100644 --- a/custom_components/noaa_it_all/config_flow.py +++ b/custom_components/noaa_it_all/config_flow.py @@ -12,7 +12,10 @@ CONF_LATITUDE, CONF_LONGITUDE, CONF_OFFICE_CODE, + CONF_RADAR_LOOP_HOURS, + DEFAULT_RADAR_LOOP_HOURS, OFFICE_COORDINATES, + RADAR_LOOP_MAX_HOURS, ) from .entry_config import resolve_entry_config @@ -248,6 +251,7 @@ def __init__(self): """ self._latitude = None self._longitude = None + self._office_code = None async def async_step_init(self, user_input=None): """Step 1 of the options flow: latitude / longitude.""" @@ -323,14 +327,8 @@ async def async_step_office(self, user_input=None): errors={CONF_OFFICE_CODE: "invalid_office"}, description_placeholders=self._office_placeholders(no_within_radius), ) - return self.async_create_entry( - title="", - data={ - CONF_OFFICE_CODE: office_code, - CONF_LATITUDE: self._latitude, - CONF_LONGITUDE: self._longitude, - }, - ) + self._office_code = office_code + return await self.async_step_radar() return self.async_show_form( step_id="office", @@ -340,6 +338,56 @@ async def async_step_office(self, user_input=None): description_placeholders=self._office_placeholders(no_within_radius), ) + async def async_step_radar(self, user_input=None): + """Step 3 of the options flow: how much radar history to keep. + + NOAA's own animation covers about fifty minutes and cannot be made + longer, so anything beyond that is accumulated locally over time. The + value is in hours, and 0 means "just serve NOAA's loop". + """ + errors = {} + existing = resolve_entry_config(self.config_entry) + default_hours = existing.get( + CONF_RADAR_LOOP_HOURS, DEFAULT_RADAR_LOOP_HOURS + ) + + if user_input is not None: + hours = user_input.get(CONF_RADAR_LOOP_HOURS) + # Validated here rather than left to the schema alone: the flow + # tests mock voluptuous wholesale, so a vol.Range would be a + # MagicMock that accepts anything. + try: + hours = int(hours) + except (TypeError, ValueError): + hours = None + if hours is None or not 0 <= hours <= RADAR_LOOP_MAX_HOURS: + errors[CONF_RADAR_LOOP_HOURS] = "invalid_radar_hours" + else: + # Every key is listed explicitly. Options replace the stored + # mapping wholesale rather than merging into it, so a key left + # out here is a key silently dropped on the next options edit. + return self.async_create_entry( + title="", + data={ + CONF_OFFICE_CODE: self._office_code, + CONF_LATITUDE: self._latitude, + CONF_LONGITUDE: self._longitude, + CONF_RADAR_LOOP_HOURS: hours, + }, + ) + default_hours = user_input.get(CONF_RADAR_LOOP_HOURS, default_hours) + + return self.async_show_form( + step_id="radar", + data_schema=vol.Schema({ + vol.Required( + CONF_RADAR_LOOP_HOURS, default=default_hours + ): vol.All(vol.Coerce(int), vol.Range(min=0, max=RADAR_LOOP_MAX_HOURS)), + }), + errors=errors, + description_placeholders={"max_hours": str(RADAR_LOOP_MAX_HOURS)}, + ) + def _office_placeholders(self, no_within_radius): """Build description placeholders for the office step.""" return { diff --git a/custom_components/noaa_it_all/const.py b/custom_components/noaa_it_all/const.py index d8b2613..17f111c 100644 --- a/custom_components/noaa_it_all/const.py +++ b/custom_components/noaa_it_all/const.py @@ -57,6 +57,7 @@ def _manifest() -> dict: CONF_OFFICE_CODE = "office_code" CONF_LATITUDE = "latitude" CONF_LONGITUDE = "longitude" +CONF_RADAR_LOOP_HOURS = "radar_loop_hours" # Default values DEFAULT_SCAN_INTERVAL = 10 # minutes @@ -84,6 +85,51 @@ def _manifest() -> dict: IMAGE_FETCH_TIMEOUT = 20 # seconds IMAGE_MAX_BYTES = 20 * 1024 * 1024 # refuse absurd payloads rather than cache them +# ------------------------------------------------------------------- +# Radar loop +# ------------------------------------------------------------------- +# NOAA publishes a ready-made radar animation, but it is fixed at ten frames +# covering roughly fifty minutes, which is long enough to see that it is +# raining and too short to see where the rain came from. NOAA also keeps only +# those ten frames on the server, so a longer loop cannot be downloaded -- it +# has to be accumulated here, one frame per refresh, and assembled locally. +# +# The window is measured in hours and 0 means "serve NOAA's own loop +# unchanged", which is both the escape hatch and the behaviour every release +# before this one had. +DEFAULT_RADAR_LOOP_HOURS = 24 +RADAR_LOOP_MAX_HOURS = 24 + +# The assembled animation is re-downloaded by every open dashboard each time it +# changes, so frame count is a bandwidth and memory decision, not a fidelity +# one. Seventy-two frames spreads a 24-hour window over 20-minute steps: storm +# motion stays legible, the GIF lands around 1-2 MB, and a cycle plays in about +# ten seconds, which is roughly as long as anyone watches a loop. Shorter +# windows get proportionally finer steps from the same cap -- a six-hour loop +# works out at one frame per five minutes, i.e. every scan NOAA publishes. +RADAR_LOOP_MAX_FRAMES = 72 +RADAR_LOOP_MIN_FRAMES = 6 # below this the local loop is worse than NOAA's +RADAR_LOOP_FRAME_MS = 120 # browsers clamp anything under ~20ms +RADAR_LOOP_LAST_FRAME_MS = 1500 # hold on "now" so the loop reads as a loop +RADAR_LOOP_MAX_BYTES = 8 * 1024 * 1024 + +# Frames are composited onto an opaque background before being combined. The +# source frames are transparent overlays with a palette each, and reconciling +# per-frame transparency across differing palettes is the single most reliable +# way to produce a psychedelic radar loop. Compositing removes the problem. +RADAR_LOOP_BACKGROUND = (0, 0, 0) + +# Frames live in /noaa_it_all/radar_frames//. Every polled frame +# inside the window is kept, not just the ones the current window displays, so +# that changing the duration re-samples from real history instead of starting +# over. That is ~144 files, a few MB, per radar site. +RADAR_FRAME_DIR = "radar_frames" +RADAR_FRAME_MAX_FILES = 200 # backstop against a directory growing unbounded +# A frame dated beyond now + this is the product of a wrong clock rather than a +# scan we have not reached yet. Ageing never reaches such a frame, so it is +# discarded outright; the slack absorbs ordinary skew between us and NOAA. +RADAR_FRAME_FUTURE_SLACK_MINUTES = 60 + # API endpoints NWS_SRF_URL = "https://forecast.weather.gov/product.php?site={office}&issuedby={office}&product=SRF&format=TXT" NWS_API_BASE = "https://api.weather.gov" diff --git a/custom_components/noaa_it_all/image.py b/custom_components/noaa_it_all/image.py index 859d258..3757041 100644 --- a/custom_components/noaa_it_all/image.py +++ b/custom_components/noaa_it_all/image.py @@ -12,6 +12,7 @@ import asyncio import logging +import os from datetime import timedelta import aiohttp @@ -25,15 +26,21 @@ from homeassistant.util import dt as dt_util from .const import ( - CONF_OFFICE_CODE, DEFAULT_SCAN_INTERVAL, DOMAIN, + CONF_OFFICE_CODE, CONF_RADAR_LOOP_HOURS, + DEFAULT_RADAR_LOOP_HOURS, DEFAULT_SCAN_INTERVAL, DOMAIN, HURRICANE_DEVICE_ID, HURRICANE_DEVICE_NAME, HURRICANE_IMAGES_ADDED_KEY, IMAGE_FAILURE_ERROR_AFTER, IMAGE_FAILURE_WARN_AFTER, IMAGE_FETCH_TIMEOUT, IMAGE_MAX_BYTES, NWS_RADAR_BASE_URL, NWS_RADAR_LOOP_URL, - OFFICE_RADAR_SITES, USER_AGENT, + OFFICE_RADAR_SITES, RADAR_FRAME_DIR, + RADAR_LOOP_MAX_FRAMES, RADAR_LOOP_MAX_HOURS, RADAR_LOOP_MIN_FRAMES, + USER_AGENT, ) from .entry_config import resolve_entry_config +from .radar_loop import ( + RadarFrameStore, assemble_gif, parse_http_date, select_frames, +) _LOGGER = logging.getLogger(__name__) @@ -89,6 +96,76 @@ def setup_platform(hass, config, add_entities, discovery_info=None): return +def radar_loop_hours(config_entry) -> int: + """Return the configured radar loop length in hours, clamped to range. + + Anything unusable -- absent, non-numeric, negative, absurd -- resolves to a + number the rest of the code can rely on rather than raising during setup. + """ + raw = resolve_entry_config(config_entry).get( + CONF_RADAR_LOOP_HOURS, DEFAULT_RADAR_LOOP_HOURS + ) + try: + hours = int(raw) + except (TypeError, ValueError): + _LOGGER.warning( + "Ignoring an unusable radar loop duration (%r); falling back to %d hours", + raw, DEFAULT_RADAR_LOOP_HOURS, + ) + return DEFAULT_RADAR_LOOP_HOURS + return max(0, min(hours, RADAR_LOOP_MAX_HOURS)) + + +async def async_discard_unused_radar_frames(hass, keep=None) -> None: + """Delete stored frames for radar sites nothing is collecting any more. + + ``keep`` is the site this entry has just claimed, if any. Every other + directory is checked against the sites the remaining entries are actually + building loops for, so switching office, or setting the duration to 0 -- + which the options screen promises will "store nothing" -- reclaims the + disk instead of orphaning it until the integration is deleted. + """ + base = hass.config.path(DOMAIN, RADAR_FRAME_DIR) + + wanted = set() + if keep: + wanted.add(keep) + for entry in hass.config_entries.async_entries(DOMAIN): + if radar_loop_hours(entry) <= 0: + continue + site = OFFICE_RADAR_SITES.get( + resolve_entry_config(entry).get(CONF_OFFICE_CODE) + ) + if site: + wanted.add(site) + + try: + present = await hass.async_add_executor_job(_list_radar_frame_dirs, base) + except Exception as err: # noqa: BLE001 - housekeeping must never break setup + _LOGGER.debug("Could not list stored radar frames: %s", err) + return + + for site in present: + if site in wanted: + continue + _LOGGER.info( + "Removing stored radar frames for %s; no configured office is " + "building a loop from it any more", site, + ) + await RadarFrameStore(hass, base, site).async_remove_all() + + +def _list_radar_frame_dirs(base: str) -> list[str]: + """Return the radar site directories under ``base`` (executor side).""" + try: + return [ + name for name in os.listdir(base) + if os.path.isdir(os.path.join(base, name)) + ] + except FileNotFoundError: + return [] + + async def async_setup_entry( hass: HomeAssistant, config_entry: ConfigEntry, @@ -159,8 +236,21 @@ async def _reload_for_hurricane_images() -> None: if radar_site: # Add radar image entities for this location base_reflectivity_image = RadarBaseReflectivityImageEntity(hass, office_code, radar_site) - radar_loop_image = RadarLoopImageEntity(hass, office_code, radar_site) + loop_hours = radar_loop_hours(config_entry) + radar_loop_image = RadarLoopImageEntity( + hass, office_code, radar_site, loop_hours=loop_hours + ) entities.extend([base_reflectivity_image, radar_loop_image]) + # Setup runs again on every options change, so this is where a switch + # away from a radar site -- or the loop being turned off entirely -- + # becomes visible. Nothing else would ever clean those frames up: + # async_remove_entry only knows the site the entry ends on, and a + # store that is no longer constructed never prunes. + hass.async_create_task( + async_discard_unused_radar_frames(hass, keep=( + radar_site if loop_hours > 0 else None + )) + ) _LOGGER.info("Added radar image entities for office %s using radar site %s", office_code, radar_site) else: _LOGGER.warning("No radar site mapping found for office %s", office_code) @@ -207,6 +297,18 @@ def __init__(self, hass) -> None: # and stores an ``Image`` dataclass in it, so the raw bytes need a # attribute of their own. self._last_image_bytes: bytes | None = None + # What was last fetched from the upstream URL, which for most entities + # is the same object as ``_last_image_bytes``. The radar loop breaks + # that equivalence -- it displays a GIF it assembled from many fetches + # -- so conditional revalidation and the 304 short-circuit have to key + # off the fetched frame rather than the displayed picture. + self._last_fetched_bytes: bytes | None = None + # Revalidation state per upstream resource: {url: (etag, last_modified, + # bytes)}. The radar loop fetches two different resources through this + # one entity, so a single slot would mean an ETag from one being + # offered as a validator for the other -- and each fetch evicting the + # other's, leaving neither able to revalidate. + self._resource_cache: dict[str, tuple] = {} self._attr_image_last_updated = None self._last_etag: str | None = None self._last_modified: str | None = None @@ -219,17 +321,21 @@ def _base_url(self) -> str: """Return the upstream URL without cache busting.""" return self._url - def get_cache_busted_url(self) -> str: - """Return the upstream URL with a coarse timestamp appended. + def get_cache_busted_url(self, url: str | None = None) -> str: + """Return an upstream URL with a coarse timestamp appended. The timestamp is rounded down to a 10-minute bucket: NOAA does not publish these images more often than that, and a coarse bucket keeps upstream and CDN caching effective while still defeating a stale cached copy. + + ``url`` defaults to this entity's own; the radar loop passes its + fallback URL so that it is busted too rather than being served a stale + copy from a CDN. """ timestamp = dt_util.utcnow().strftime('%Y%m%d%H%M') timestamp = timestamp[:-1] + '0' - return f"{self._base_url()}?t={timestamp}" + return f"{url or self._base_url()}?t={timestamp}" # -- Home Assistant plumbing ----------------------------------------- @@ -305,12 +411,7 @@ async def _async_update_cache(self) -> bool: if content is None: return False - if self._failure_count >= IMAGE_FAILURE_WARN_AFTER: - _LOGGER.info( - "The %s image is available again after %d failed attempts", - self._log_label, self._failure_count, - ) - self._failure_count = 0 + self._note_success() if content == self._last_image_bytes: _LOGGER.debug( @@ -325,44 +426,75 @@ async def _async_update_cache(self) -> bool: ) return True - def _conditional_headers(self) -> dict[str, str]: + def _note_success(self) -> None: + """Reset the failure counter, announcing a recovery if there was one.""" + if self._failure_count >= IMAGE_FAILURE_WARN_AFTER: + _LOGGER.info( + "The %s image is available again after %d failed attempts", + self._log_label, self._failure_count, + ) + self._failure_count = 0 + + def _conditional_headers(self, resource: str) -> dict[str, str]: """Return revalidation headers for the copy already in hand. The refresh runs on a timer whether or not anyone is looking at the dashboard, so revalidating keeps the steady-state cost close to zero for sources that publish infrequently. + + Validators are only offered back to the resource they came from. An + entity that fetches more than one URL would otherwise be asking "has + it changed since?" about a different file entirely, and a server that + answered 304 to that would hand back the wrong image. """ headers = {"User-Agent": USER_AGENT} - if self._last_image_bytes is None: + cached = self._resource_cache.get(resource) + if cached is None: return headers - if self._last_etag: - headers["If-None-Match"] = self._last_etag - elif self._last_modified: - headers["If-Modified-Since"] = self._last_modified + etag, last_modified, _ = cached + if etag: + headers["If-None-Match"] = etag + elif last_modified: + headers["If-Modified-Since"] = last_modified return headers - async def _async_fetch_image(self) -> bytes | None: + async def _async_fetch_image(self, url: str | None = None) -> bytes | None: """Fetch the image from NOAA, returning None on any failure. No failure path may touch ``_last_image_bytes``, ``_last_etag``, ``_last_modified`` or ``image_last_updated``. That invariant is what keeps a blip from blanking the picture. + + ``url`` overrides the entity's own URL, which the radar loop uses to + fall back to NOAA's ready-made animation without reimplementing any of + the error handling below. """ - self._image_url = self.get_cache_busted_url() + # The cache-busting query string changes every ten minutes, so identity + # is tracked by the underlying resource rather than the fetched URL. + resource = url or self._base_url() + self._image_url = self.get_cache_busted_url(resource) try: session = async_get_clientsession(self.hass) timeout = aiohttp.ClientTimeout(total=IMAGE_FETCH_TIMEOUT) async with session.get( self._image_url, timeout=timeout, - headers=self._conditional_headers(), + headers=self._conditional_headers(resource), ) as response: - if response.status == 304 and self._last_image_bytes is not None: + if response.status == 304 and resource in self._resource_cache: _LOGGER.debug( "The %s image is unchanged upstream (HTTP 304)", self._log_label, ) - return self._last_image_bytes + # Republish this resource's validators as the current ones: + # callers read ``_last_modified`` straight after a fetch to + # date what came back, and a 304 still describes this + # resource, not whichever one was fetched most recently. + etag, last_modified, cached = self._resource_cache[resource] + self._last_etag = etag + self._last_modified = last_modified + self._last_fetched_bytes = cached + return cached if response.status != 200: self._log_failure( f"HTTP {response.status}", @@ -405,6 +537,8 @@ async def _async_fetch_image(self) -> bytes | None: self._attr_content_type = content_type self._last_etag = etag self._last_modified = last_modified + self._last_fetched_bytes = content + self._resource_cache[resource] = (etag, last_modified, content) return content def _log_failure(self, reason: str, *, transient: bool) -> None: @@ -597,7 +731,19 @@ def device_info(self) -> DeviceInfo: class RadarLoopImageEntity(NoaaImageEntity): - """Representation of the Radar Loop Image (animated) for a specific location. + """An animated NEXRAD loop, either NOAA's own or one assembled here. + + NOAA publishes a ready-made animation, but it is fixed at ten frames + covering roughly fifty minutes and the server keeps only those ten frames, + so a longer loop cannot be downloaded. When ``loop_hours`` is set this + entity instead collects one frame per refresh into + :class:`~.radar_loop.RadarFrameStore` and assembles the animation itself, + which means the loop starts short after a fresh install and fills out over + the hours that follow. Frames outlive restarts, so that happens once + rather than on every reboot. + + ``loop_hours = 0`` restores the previous behaviour exactly: NOAA's loop, + proxied unchanged, with nothing written to disk. Uses ``_attr_has_entity_name = True`` so that Home Assistant automatically combines the office weather device name (e.g. @@ -608,17 +754,207 @@ class RadarLoopImageEntity(NoaaImageEntity): _attr_has_entity_name = True _attr_content_type = "image/gif" - def __init__(self, hass, office_code, radar_site): + def __init__(self, hass, office_code, radar_site, loop_hours=0): """Initialize the radar loop image entity.""" self._office_code = office_code self._radar_site = radar_site self._log_label = f"radar loop for {office_code}" + self._loop_hours = loop_hours + self._store = None + # Whether the picture currently on screen is NOAA's animation rather + # than one built here. True until the buffer has enough frames to + # improve on it, and again whenever assembly fails. + self._serving_upstream = True + self._frame_count = 0 + self._window_start = None + self._window_end = None + if loop_hours > 0: + self._store = RadarFrameStore( + hass, + hass.config.path(DOMAIN, RADAR_FRAME_DIR), + radar_site, + ) super().__init__(hass) + @property + def _building_locally(self) -> bool: + """True when this entity assembles the loop rather than proxying it.""" + return self._store is not None + + @property + def _window(self) -> timedelta: + """Return how far back the loop reaches.""" + return timedelta(hours=self._loop_hours) + def _base_url(self) -> str: - """Return the NEXRAD radar loop URL for this radar site.""" + """Return the URL each refresh fetches. + + Building locally means collecting single scans, so the refresh targets + the latest frame rather than NOAA's finished animation. + """ + if self._building_locally: + return NWS_RADAR_BASE_URL.format(radar=self._radar_site) return NWS_RADAR_LOOP_URL.format(radar=self._radar_site) + @property + def extra_state_attributes(self): + """Expose what the loop actually covers, for templates and debugging. + + A loop that is quietly shorter than asked for -- because the buffer is + still filling, or because Pillow is missing -- is otherwise invisible. + """ + return { + # Describes the animation actually being served, not the setting: + # a local loop that has fallen back to NOAA's says so. + "loop_mode": "upstream" if self._serving_upstream else "local", + "loop_hours": self._loop_hours, + "frame_count": self._frame_count, + "window_start": ( + self._window_start.isoformat() if self._window_start else None + ), + "window_end": ( + self._window_end.isoformat() if self._window_end else None + ), + } + + async def async_added_to_hass(self) -> None: + """Start the refresh timer, and prune anything now out of window. + + Pruning here rather than on the first refresh means shortening the + duration takes effect as soon as the entry reloads, instead of leaving + stale frames on disk for another refresh interval. + """ + await super().async_added_to_hass() + if self._building_locally: + self.async_on_remove( + async_call_later(self.hass, 0, self._async_prune) + ) + + async def _async_prune(self, now=None) -> None: + """Drop frames outside the configured window.""" + await self._store.async_prune(self._window, dt_util.utcnow()) + + async def _async_update_cache(self) -> bool: + """Collect the latest scan and rebuild the loop from what we hold. + + Returns True only when the displayed animation actually changed. Every + failure path returns False without touching the cached bytes, so the + previous loop stays on the dashboard. + """ + if not self._building_locally: + return await super()._async_update_cache() + + frame = await self._async_fetch_image() + if frame is None: + return False + self._note_success() + + # Last-Modified is when NOAA published the scan, which puts frames on + # the real four-to-six minute volume-scan cadence rather than on our + # arbitrary refresh boundary -- and makes two refreshes that see the + # same scan resolve to the same file, so dedup costs nothing. + # Hashing the bytes instead would be actively wrong: two consecutive + # scans of a clear sky are genuinely identical, so a quiet night would + # collapse to a single frame and the loop would cut straight from + # "clear" to "storm" with no sense of time passing. + timestamp = parse_http_date(self._last_modified) + if timestamp is None: + timestamp = dt_util.utcnow().replace(second=0, microsecond=0) + + added = await self._store.async_add_frame(timestamp, frame) + # Pruned every refresh rather than only when something was stored: a + # radar site stuck on one scan for hours -- maintenance, an outage -- + # would otherwise never prune at all, and the window would quietly + # stretch past what was asked for. + await self._store.async_prune(self._window, dt_util.utcnow()) + if not added and self._last_image_bytes is not None: + # Same scan as last time and we already have a loop built from it. + return False + + return await self._async_rebuild_loop() + + async def _async_rebuild_loop(self) -> bool: + """Assemble the stored frames, falling back to NOAA's loop if needed.""" + frames = await self._store.async_frames() + if len(frames) < RADAR_LOOP_MIN_FRAMES: + _LOGGER.debug( + "Only %d radar frames stored for %s; showing NOAA's own loop " + "until the buffer fills", len(frames), self._office_code, + ) + return await self._async_serve_upstream_loop() + + # Sampling and encoding go to the executor together. Both are pure + # CPU over the frame list, this runs on a timer whether or not anyone + # is looking at the dashboard, and splitting them would put the + # sampler's work back on the event loop for nothing. + loop = await self.hass.async_add_executor_job( + self._build_loop, frames, dt_util.utcnow() + ) + if loop is None: + return await self._async_serve_upstream_loop() + + # The window is described from the frames the encoder actually used, + # not the ones it was offered: unreadable frames and any shed to get + # under the size limit are not in the animation, and reporting them + # would hide exactly the shortfall these attributes exist to show. + used = set(loop.paths) + covered = [timestamp for timestamp, path in frames if path in used] + self._serving_upstream = False + self._frame_count = len(loop.paths) + self._window_start = covered[0] if covered else None + self._window_end = covered[-1] if covered else None + + if loop.data == self._last_image_bytes: + return False + self._last_image_bytes = loop.data + self._attr_content_type = "image/gif" + self._attr_image_last_updated = dt_util.utcnow() + return True + + def _build_loop(self, frames, now): + """Sample the stored frames and encode them (executor side). + + Returns None when the sample is too thin to beat NOAA's own loop. The + floor is applied here, to the frames that will actually be encoded -- + checking it against everything on disk would let a window with a long + outage in the middle ship a three-frame animation in place of NOAA's + ten, which is the outcome the floor exists to prevent. + """ + paths = select_frames( + frames, + window=self._window, + max_frames=RADAR_LOOP_MAX_FRAMES, + now=now, + ) + if len(paths) < RADAR_LOOP_MIN_FRAMES: + _LOGGER.debug( + "Only %d of %d stored radar frames for %s fall inside the " + "loop window; showing NOAA's own loop instead", + len(paths), len(frames), self._office_code, + ) + return None + return assemble_gif(paths) + + async def _async_serve_upstream_loop(self) -> bool: + """Show NOAA's own animation instead of one we could not build. + + Used while the buffer is still filling and whenever assembly fails, so + the card is never blank and never worse than it was before this + feature existed. + """ + content = await self._async_fetch_image( + NWS_RADAR_LOOP_URL.format(radar=self._radar_site) + ) + if content is None or content == self._last_image_bytes: + return False + self._serving_upstream = True + self._frame_count = 0 + self._window_start = None + self._window_end = None + self._last_image_bytes = content + self._attr_image_last_updated = dt_util.utcnow() + return True + @property def name(self): """Return the local entity name.""" diff --git a/custom_components/noaa_it_all/manifest.json b/custom_components/noaa_it_all/manifest.json index 29196d2..f7f4d70 100644 --- a/custom_components/noaa_it_all/manifest.json +++ b/custom_components/noaa_it_all/manifest.json @@ -13,5 +13,5 @@ "aiohttp", "voluptuous" ], - "version": "0.5.3" + "version": "0.6.0" } diff --git a/custom_components/noaa_it_all/radar_loop.py b/custom_components/noaa_it_all/radar_loop.py new file mode 100644 index 0000000..9dd88b1 --- /dev/null +++ b/custom_components/noaa_it_all/radar_loop.py @@ -0,0 +1,520 @@ +"""Accumulate NEXRAD frames locally and assemble them into a long radar loop. + +NOAA publishes a ready-made animation at ``{SITE}_loop.gif``, but it is fixed +at ten frames covering roughly fifty minutes, and the server keeps only those +ten frames. A longer loop therefore cannot be downloaded; it has to be built +from frames collected over time. This module owns that: a small on-disk ring +of single-scan GIFs per radar site, a sampler that picks evenly spaced frames +out of it, and a Pillow encoder that stitches them into one animation. + +Nothing here imports Home Assistant. ``hass`` is passed in solely to borrow +its executor, which keeps the file and image work off the event loop -- and +keeps this module testable against a real directory and a real Pillow without +any of the module mocking the entity tests need. +""" + +from __future__ import annotations + +import io +import logging +import os +import shutil +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime + +from .const import ( + RADAR_FRAME_FUTURE_SLACK_MINUTES, + RADAR_FRAME_MAX_FILES, + RADAR_LOOP_BACKGROUND, + RADAR_LOOP_FRAME_MS, + RADAR_LOOP_LAST_FRAME_MS, + RADAR_LOOP_MAX_BYTES, +) + +try: + from PIL import Image + PIL_AVAILABLE = True +except ImportError: # pragma: no cover - Pillow ships with Home Assistant core + Image = None + PIL_AVAILABLE = False + +_LOGGER = logging.getLogger(__name__) + +# What ``assemble_gif`` hands back: the encoded animation, and the frames it +# actually used. The caller needs the second part to describe the loop +# honestly -- a frame dropped as unreadable, or shed to get under the size +# limit, is not in the animation and must not be counted as though it were. +RadarLoop = namedtuple("RadarLoop", "data paths") + +# Sorts lexicographically in the same order it sorts chronologically, which is +# what lets the directory listing double as the index. +_NAME_FORMAT = "%Y%m%dT%H%M%SZ" +_SUFFIX = ".gif" +_TEMP_SUFFIX = ".tmp" + + +def parse_http_date(value): + """Return an aware UTC datetime for an HTTP date header, or None. + + Used on ``Last-Modified``, which for the radar endpoint is the time NOAA + published the scan. Anything unparseable is None rather than an exception: + a header NOAA changed the shape of must not take the loop down. + """ + if not value: + return None + try: + parsed = parsedate_to_datetime(value) + except (AttributeError, TypeError, ValueError): + # AttributeError covers a non-string slipping through: the header is + # whatever the response handed us, and none of this is worth an + # exception that would take the radar loop down. + return None + if parsed is None: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def frame_name(timestamp: datetime) -> str: + """Return the on-disk file name for a frame captured at ``timestamp``.""" + if timestamp.tzinfo is not None: + timestamp = timestamp.astimezone(timezone.utc) + return f"{timestamp.strftime(_NAME_FORMAT)}{_SUFFIX}" + + +def _parse_frame_name(name: str): + """Return the timestamp encoded in a frame file name, or None.""" + if not name.endswith(_SUFFIX): + return None + try: + parsed = datetime.strptime(name[: -len(_SUFFIX)], _NAME_FORMAT) + except ValueError: + return None + return parsed.replace(tzinfo=timezone.utc) + + +class RadarFrameStore: + """A per-radar-site directory of single-scan GIFs, keyed by scan time. + + The file name *is* the index -- there is no sidecar manifest to drift out + of step with the directory, and no whole-buffer rewrite on every poll. A + frame that is already on disk is never rewritten, which is what makes + "have we seen this scan?" a cheap existence check rather than a comparison. + """ + + def __init__(self, hass, base_dir: str, radar_site: str) -> None: + """Store the paths; no I/O happens until a method is awaited.""" + self._hass = hass + self._radar_site = radar_site + self._dir = os.path.join(base_dir, radar_site) + + @property + def path(self) -> str: + """Return the directory this store writes frames to.""" + return self._dir + + # -- Reading --------------------------------------------------------- + + async def async_frames(self): + """Return [(timestamp, path), ...] for stored frames, oldest first. + + Only names and times cross back to the caller. Frame bytes are read + once, inside the encoder, so a full window never sits in memory. + """ + return await self._hass.async_add_executor_job(self._frames) + + def _frames(self): + """List and sort the frame directory (executor side).""" + try: + entries = os.listdir(self._dir) + except FileNotFoundError: + return [] + except OSError as err: + _LOGGER.warning( + "Could not list the radar frame directory for %s: %s", + self._radar_site, err, + ) + return [] + + frames = [] + for name in entries: + timestamp = _parse_frame_name(name) + if timestamp is not None: + frames.append((timestamp, os.path.join(self._dir, name))) + frames.sort(key=lambda item: item[0]) + return frames + + # -- Writing --------------------------------------------------------- + + async def async_add_frame(self, timestamp: datetime, data: bytes) -> bool: + """Store one scan, returning True only if it was genuinely new. + + False covers both "we already had this scan" and "the disk would not + take it". Neither is an error the dashboard should ever notice. + """ + return await self._hass.async_add_executor_job( + self._add_frame, timestamp, data + ) + + def _add_frame(self, timestamp: datetime, data: bytes) -> bool: + """Write a frame atomically, skipping one we already hold.""" + target = os.path.join(self._dir, frame_name(timestamp)) + if os.path.exists(target): + return False + temporary = f"{target}{_TEMP_SUFFIX}" + try: + os.makedirs(self._dir, exist_ok=True) + with open(temporary, "wb") as handle: + handle.write(data) + # Rename rather than write in place: a crash or a full disk part + # way through must not leave a truncated frame behind, because the + # name is all the encoder has to go on. + os.replace(temporary, target) + except OSError as err: + _LOGGER.warning( + "Could not store a radar frame for %s: %s", self._radar_site, err + ) + self._unlink(temporary) + return False + return True + + # -- Pruning --------------------------------------------------------- + + async def async_prune(self, window: timedelta, now: datetime) -> int: + """Drop frames outside ``window`` and return how many were removed.""" + return await self._hass.async_add_executor_job(self._prune, window, now) + + def _prune(self, window: timedelta, now: datetime) -> int: + """Delete expired frames, junk names and abandoned temp files.""" + try: + entries = os.listdir(self._dir) + except FileNotFoundError: + return 0 + except OSError as err: + _LOGGER.warning( + "Could not prune radar frames for %s: %s", self._radar_site, err + ) + return 0 + + cutoff = now - window + # A frame dated in the future is the product of a wrong clock -- either + # ours when it was written, or NOAA's Last-Modified. Ageing cannot + # reach it, so without an upper bound it would outlive every real frame + # and hold the loop on a set that never updates. A little slack + # absorbs ordinary clock skew. + horizon = now + timedelta(minutes=RADAR_FRAME_FUTURE_SLACK_MINUTES) + removed = 0 + keep = [] + for name in entries: + path = os.path.join(self._dir, name) + timestamp = _parse_frame_name(name) + # Anything that is not a frame we can date -- an abandoned .tmp, a + # file someone dropped in by hand -- is not useful and cannot be + # aged out later, so it goes now. + if timestamp is None or timestamp < cutoff or timestamp > horizon: + removed += self._unlink(path) + continue + keep.append((timestamp, path)) + + # Backstop against a directory growing without bound for any reason the + # window did not catch. Frames are now known to be inside it, so + # trimming the oldest is the right end. + if len(keep) > RADAR_FRAME_MAX_FILES: + keep.sort(key=lambda item: item[0]) + for _, path in keep[: len(keep) - RADAR_FRAME_MAX_FILES]: + removed += self._unlink(path) + return removed + + async def async_discard(self, path: str) -> None: + """Delete a single frame that turned out to be unreadable.""" + await self._hass.async_add_executor_job(self._unlink, path) + + async def async_remove_all(self) -> None: + """Delete this site's frame directory, for entry removal.""" + await self._hass.async_add_executor_job(self._remove_all) + + def _remove_all(self) -> None: + """Remove the site directory and everything under it.""" + try: + shutil.rmtree(self._dir) + except FileNotFoundError: + return + except OSError as err: + _LOGGER.warning( + "Could not remove stored radar frames for %s: %s", + self._radar_site, err, + ) + + @staticmethod + def _unlink(path: str) -> int: + """Delete one file, returning 1 if it went away.""" + try: + os.unlink(path) + except FileNotFoundError: + return 0 + except OSError as err: + _LOGGER.debug("Could not remove %s: %s", path, err) + return 0 + return 1 + + +def select_frames(frames, *, window: timedelta, max_frames: int, now: datetime): + """Pick up to ``max_frames`` evenly spaced frames, oldest first. + + Sampling is by *time*, not by position in the list: walk backwards from + ``now`` in fixed steps and take the nearest frame to each step. Decimating + every Nth item instead would re-space the whole loop whenever a poll was + missed, and again whenever the configured duration changed. + + The newest frame is always the step-zero target, so the animation always + ends on the current scan. + """ + if not frames or max_frames < 1: + return [] + + # Filter by the window first. Skipping straight to "few enough frames, + # take them all" would put frames from outside the requested window into + # the loop whenever pruning had not caught up -- and then report them as + # the window it covers. + cutoff = now - window + inside = [item for item in frames if cutoff <= item[0] <= now] + if not inside: + return [] + if len(inside) <= max_frames: + return [path for _, path in inside] + frames = inside + + step = window / max_frames + tolerance = step / 2 + chosen = {} + for index in range(max_frames): + target = now - step * index + best = min(frames, key=lambda item: abs(item[0] - target)) + # A gap in the buffer shortens the loop rather than filling it with a + # frame from the wrong time. + if abs(best[0] - target) <= tolerance: + chosen[best[0]] = best[1] + return [chosen[timestamp] for timestamp in sorted(chosen)] + + +def _load_frame(path: str, size, background): + """Return one frame as an opaque RGB image, or None if it is unusable. + + Converting via RGBA applies that frame's own palette and turns its + transparency index into real alpha, which is what makes frames with + different palettes comparable. Compositing onto a solid colour then drops + transparency entirely, so the encoder never has to reconcile a transparency + index across frames -- the failure that turns a radar loop psychedelic. + """ + try: + with Image.open(path) as source: + frame = source.convert("RGBA") + except (OSError, ValueError) as err: + _LOGGER.debug("Skipping unreadable radar frame %s: %s", path, err) + return None + if size is not None and frame.size != size: + # NOAA has changed product dimensions before. Older frames are still + # worth showing, so scale rather than discard. + frame = frame.resize(size) + backdrop = Image.new("RGBA", frame.size, tuple(background) + (255,)) + return Image.alpha_composite(backdrop, frame).convert("RGB") + + +def _probe(path): + """Return a frame's dimensions without decoding it, or None if unreadable. + + ``Image.open`` only reads the header, so this costs a stat and a few bytes + per frame -- cheap enough to run over the whole window before committing to + decoding any of it. + """ + try: + with Image.open(path) as source: + return source.size + except (OSError, ValueError) as err: + _LOGGER.debug("Skipping unreadable radar frame %s: %s", path, err) + return None + + +def _master_palette(paths, size, background, samples=8): + """Build one palette that covers the whole animation. + + Deriving the palette from the first frame alone looks reasonable and is + badly wrong: an overnight loop starts on an empty radar, so frame one + contains nothing but background, and every echo that appears later gets + mapped to the nearest colour that frame happened to contain -- black. The + storm silently disappears. Sampling across the window instead means the + palette has seen the loop's colours before it has to encode them. + """ + picked = paths + if len(paths) > samples: + step = (len(paths) - 1) / (samples - 1) + picked = [paths[round(index * step)] for index in range(samples)] + + loaded = [ + frame for frame in (_load_frame(path, size, background) for path in picked) + if frame is not None + ] + if not loaded: + return None + + width, height = loaded[0].size + montage = Image.new("RGB", (width, height * len(loaded))) + for index, frame in enumerate(loaded): + montage.paste(frame, (0, index * height)) + return montage.quantize( + colors=255, method=Image.Quantize.MEDIANCUT, dither=Image.Dither.NONE + ) + + +def _encode(paths, *, frame_ms, last_frame_ms, background): + """Encode the given frame paths into one animated GIF.""" + # Establish the frame list from headers alone first. The durations list + # has to match the frames actually written, so the set of frames has to be + # settled before encoding starts rather than discovered during it. + usable = [path for path in paths if _probe(path) is not None] + + # The newest frame sets the dimensions everything else is scaled to. Using + # the oldest instead would mean that after NOAA changed a product's size, + # every new full-resolution scan was squeezed back down to the old one for + # a whole window -- degrading the frames that matter to preserve the ones + # about to age out. + size = None + while usable: + size = _probe(usable[-1]) + if size is not None: + break + usable.pop() + + first = None + while usable: + first = _load_frame(usable[0], size, background) + if first is not None: + break + # Header parsed but the pixels did not: drop it and take the next. + usable.pop(0) + if first is None or size is None or len(usable) < 2: + return None + + # One palette for the whole animation. Left to itself the GIF writer takes + # frame one's palette as the global table and then remaps every later frame + # against it, which is exactly how a radar loop ends up cycling through + # colours it never contained. + master = _master_palette(usable, size, background) + if master is None: + return None + + lead = first.quantize(palette=master, dither=Image.Dither.NONE) + + def _quantized(path_list): + """Yield frames one at a time so the whole window is never resident.""" + previous = lead + for path in path_list: + frame = _load_frame(path, size, background) + if frame is None: + # Repeat the previous frame rather than yielding nothing: the + # durations below are already fixed, so a short yield would + # slide the hold off the newest frame. A corrupt frame reads + # as a momentary freeze instead. + yield previous + continue + previous = frame.quantize(palette=master, dither=Image.Dither.NONE) + yield previous + + # Pillow drops a frame that is pixel-identical to the one before it and + # adds its duration to that frame instead, so the encoded animation can + # legitimately contain fewer frames than there are paths -- a clear + # overnight radar collapses to one long still. Total playback time is + # preserved, and the hold still lands on the newest frame, so this is left + # alone: it is free compression on exactly the frames worth nothing. + durations = [frame_ms] * (len(usable) - 1) + [last_frame_ms] + buffer = io.BytesIO() + lead.save( + buffer, + format="GIF", + save_all=True, + append_images=_quantized(usable[1:]), + duration=durations, + loop=0, + # Every frame is opaque and covers the one before it, so there is + # nothing to restore between frames. + disposal=1, + # Pillow's optimiser re-derives a palette per frame, undoing the shared + # table above for no useful gain once the table is already minimal. + optimize=False, + ) + return RadarLoop(buffer.getvalue(), usable) + + +def assemble_gif( + paths, + *, + frame_ms: int = RADAR_LOOP_FRAME_MS, + last_frame_ms: int = RADAR_LOOP_LAST_FRAME_MS, + max_bytes: int = RADAR_LOOP_MAX_BYTES, + background=RADAR_LOOP_BACKGROUND, +): + """Return a :class:`RadarLoop`, or None if one cannot be made. + + The result carries the frames actually encoded as well as the bytes, so the + caller can describe the animation truthfully rather than assuming it got + everything it asked for. + + Synchronous and CPU-bound -- callers run it in an executor. Every failure + is None rather than an exception, because the caller's response to "no loop + this time" is to keep showing the previous one. + """ + if not PIL_AVAILABLE: + _LOGGER.debug("Pillow is unavailable, so no local radar loop was built") + return None + if len(paths) < 2: + return None + + try: + result = _encode( + paths, + frame_ms=frame_ms, + last_frame_ms=last_frame_ms, + background=background, + ) + if result is None: + return None + if len(result.data) > max_bytes: + # Rather than push something absurd through the image proxy, halve + # the frame count and try once more. Sampling every other frame + # keeps the window the same length and only coarsens it. + # + # Counted back from the end, not forward from the start: with an + # even number of frames, every other one from the front excludes + # the last, which would leave the animation ending -- and holding + # -- on a scan that is not the current one. + thinned = paths[::-2][::-1] + _LOGGER.warning( + "The radar loop came to %d bytes over the %d byte limit; " + "rebuilding it with %d of %d frames", + len(result.data), max_bytes, len(thinned), len(paths), + ) + result = _encode( + thinned, + frame_ms=frame_ms, + last_frame_ms=last_frame_ms, + background=background, + ) + if result is None: + return None + if len(result.data) > max_bytes: + _LOGGER.warning( + "The radar loop is still %d bytes; falling back to NOAA's " + "own loop", len(result.data), + ) + return None + except Exception as err: # noqa: BLE001 - a bad frame must not break the entity + _LOGGER.warning("Could not assemble the radar loop: %s", err) + return None + + _LOGGER.debug( + "Assembled a %d frame radar loop (%d bytes)", + len(result.paths), len(result.data), + ) + return result diff --git a/custom_components/noaa_it_all/strings.json b/custom_components/noaa_it_all/strings.json index bd2c4fb..b9e0d3d 100644 --- a/custom_components/noaa_it_all/strings.json +++ b/custom_components/noaa_it_all/strings.json @@ -43,12 +43,20 @@ "data": { "office_code": "NWS Forecast Office" } + }, + "radar": { + "title": "Radar Loop History", + "description": "How many hours of radar the animated Radar Loop should cover.\n\nNOAA's own loop is fixed at roughly 50 minutes and it only keeps ten frames on the server, so anything longer is collected here instead: one frame per refresh, saved under your Home Assistant configuration directory and kept across restarts. A newly configured loop starts short and fills out over the hours that follow.\n\nExpect a few megabytes of storage per radar site, and note that the assembled animation is larger than NOAA's, so every open dashboard re-downloads it each time it changes.\n\nSet 0 to serve NOAA's own loop unchanged and store nothing. Maximum {max_hours} hours.", + "data": { + "radar_loop_hours": "Hours of radar history" + } } }, "error": { "invalid_latitude": "Latitude must be between -90 and 90.", "invalid_longitude": "Longitude must be between -180 and 180.", - "invalid_office": "Please choose a valid NWS forecast office." + "invalid_office": "Please choose a valid NWS forecast office.", + "invalid_radar_hours": "Radar history must be a whole number of hours between 0 and {max_hours}." } } } diff --git a/custom_components/noaa_it_all/translations/en.json b/custom_components/noaa_it_all/translations/en.json index bd2c4fb..b9e0d3d 100644 --- a/custom_components/noaa_it_all/translations/en.json +++ b/custom_components/noaa_it_all/translations/en.json @@ -43,12 +43,20 @@ "data": { "office_code": "NWS Forecast Office" } + }, + "radar": { + "title": "Radar Loop History", + "description": "How many hours of radar the animated Radar Loop should cover.\n\nNOAA's own loop is fixed at roughly 50 minutes and it only keeps ten frames on the server, so anything longer is collected here instead: one frame per refresh, saved under your Home Assistant configuration directory and kept across restarts. A newly configured loop starts short and fills out over the hours that follow.\n\nExpect a few megabytes of storage per radar site, and note that the assembled animation is larger than NOAA's, so every open dashboard re-downloads it each time it changes.\n\nSet 0 to serve NOAA's own loop unchanged and store nothing. Maximum {max_hours} hours.", + "data": { + "radar_loop_hours": "Hours of radar history" + } } }, "error": { "invalid_latitude": "Latitude must be between -90 and 90.", "invalid_longitude": "Longitude must be between -180 and 180.", - "invalid_office": "Please choose a valid NWS forecast office." + "invalid_office": "Please choose a valid NWS forecast office.", + "invalid_radar_hours": "Radar history must be a whole number of hours between 0 and {max_hours}." } } } diff --git a/info.md b/info.md index 29134c0..0797f2e 100644 --- a/info.md +++ b/info.md @@ -27,7 +27,7 @@ Comprehensive NOAA data integration providing real-time space weather, hurricane - **Hurricane Tracking** - Active alerts, warnings, and satellite imagery - **Current Weather** - Temperature, humidity, wind, pressure, visibility, and sky conditions - **NWS Alerts** - Severe weather, flood, winter storm, heat, and air quality warnings -- **Weather Radar** - Real-time radar imagery with timestamp tracking +- **Weather Radar** - Real-time radar imagery with timestamp tracking, plus an animated loop covering up to 24 hours - **Forecast Discussion** - Meteorologist-written technical analysis (AFD product) ### 🚨 Alert Categories (Location-Specific Binary Sensors) diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index 2564ebd..71a3281 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -491,17 +491,93 @@ def test_valid_input_advances_to_office_step(self): self.assertAlmostEqual(flow._latitude, 34.0) self.assertAlmostEqual(flow._longitude, -78.0) - def test_office_step_creates_entry(self): + def test_office_step_advances_to_the_radar_step(self): flow = self._make_flow() _run(flow.async_step_init(user_input={ "latitude": 34.0, "longitude": -78.0, })) result = _run(flow.async_step_office(user_input={"office_code": "ILM"})) + self.assertEqual(result["type"], "form") + self.assertEqual(result["step_id"], "radar") + self.assertEqual(flow._office_code, "ILM") + + def _through_to_radar(self, flow=None, office="ILM"): + """Walk the options flow as far as the radar step.""" + flow = flow or self._make_flow() + _run(flow.async_step_init(user_input={ + "latitude": 34.0, + "longitude": -78.0, + })) + _run(flow.async_step_office(user_input={"office_code": office})) + return flow + + def test_the_radar_step_creates_the_entry(self): + flow = self._through_to_radar() + result = _run(flow.async_step_radar(user_input={"radar_loop_hours": 24})) self.assertEqual(result["type"], "create_entry") self.assertEqual(result["data"]["office_code"], "ILM") self.assertAlmostEqual(result["data"]["latitude"], 34.0) self.assertAlmostEqual(result["data"]["longitude"], -78.0) + self.assertEqual(result["data"]["radar_loop_hours"], 24) + + def test_saved_options_survive_a_second_trip_through_the_flow(self): + """Options replace the stored mapping rather than merging into it. + + A key the last step forgets to write is a key silently lost the next + time anyone opens Configure, which is a slow enough failure to be worth + pinning down. + """ + flow = self._make_flow(options={ + "latitude": 34.0, + "longitude": -78.0, + "office_code": "ILM", + "radar_loop_hours": 12, + }) + self._through_to_radar(flow) + result = _run(flow.async_step_radar(user_input={"radar_loop_hours": 12})) + self.assertEqual( + {"latitude", "longitude", "office_code", "radar_loop_hours"}, + set(result["data"]), + ) + self.assertEqual(result["data"]["radar_loop_hours"], 12) + + def test_the_radar_step_prefills_the_saved_value(self): + flow = self._make_flow(options={"radar_loop_hours": 6}) + self._through_to_radar(flow) + _reset_recorded_defaults() + _run(flow.async_step_radar(user_input=None)) + self.assertEqual(_recorded_defaults()["radar_loop_hours"], 6) + + def test_the_radar_step_defaults_to_a_full_day(self): + flow = self._through_to_radar() + _reset_recorded_defaults() + _run(flow.async_step_radar(user_input=None)) + self.assertEqual(_recorded_defaults()["radar_loop_hours"], 24) + + def test_zero_hours_is_accepted_as_the_opt_out(self): + flow = self._through_to_radar() + result = _run(flow.async_step_radar(user_input={"radar_loop_hours": 0})) + self.assertEqual(result["type"], "create_entry") + self.assertEqual(result["data"]["radar_loop_hours"], 0) + + def test_out_of_range_or_unparseable_hours_are_rejected(self): + for value in (-1, 25, 100, "abc", None): + with self.subTest(value=value): + flow = self._through_to_radar() + result = _run(flow.async_step_radar( + user_input={"radar_loop_hours": value} + )) + self.assertEqual(result["type"], "form") + self.assertEqual( + result["errors"]["radar_loop_hours"], "invalid_radar_hours" + ) + + def test_hours_given_as_a_numeric_string_are_accepted(self): + flow = self._through_to_radar() + result = _run(flow.async_step_radar(user_input={"radar_loop_hours": "8"})) + self.assertEqual(result["type"], "create_entry") + self.assertEqual(result["data"]["radar_loop_hours"], 8) def test_invalid_latitude_returns_error(self): flow = self._make_flow() diff --git a/tests/test_image.py b/tests/test_image.py index 9d6bba2..2c201a5 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -4,8 +4,9 @@ import logging import os import sys +import tempfile import unittest -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch _REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -979,3 +980,691 @@ def test_log_labels_are_distinct(self): if __name__ == "__main__": unittest.main() + + +GIF = b"GIF89a-fake-frame" + + +def _loop_of(data, paths=("/frames/a.gif", "/frames/b.gif")): + """Build the RadarLoop that assemble_gif would return.""" + from noaa_it_all.radar_loop import RadarLoop + return RadarLoop(data, list(paths)) + + +class _RecordingStore: + """Stands in for RadarFrameStore, recording what the entity asks of it.""" + + def __init__(self, frames=(), accept=True): + self.frames = list(frames) + self.accept = accept + self.added = [] + self.pruned = [] + + async def async_add_frame(self, timestamp, data): + self.added.append((timestamp, data)) + if not self.accept: + return False + path = f"/frames/{timestamp:%Y%m%dT%H%M%SZ}.gif" + if any(existing == path for _, existing in self.frames): + return False + self.frames.append((timestamp, path)) + return True + + async def async_frames(self): + return list(self.frames) + + async def async_prune(self, window, now): + self.pruned.append((window, now)) + return 0 + + +def _local_loop_entity(frames=(), accept=True, loop_hours=24): + """A radar loop entity in local mode with its store swapped for a fake.""" + from noaa_it_all.image import RadarLoopImageEntity + + hass = MagicMock() + hass.config.path = MagicMock(return_value="/config/noaa_it_all/radar_frames") + + async def _executor(func, *args): + """Run the encoder inline; the real one hands it to a worker thread.""" + return func(*args) + + hass.async_add_executor_job = _executor + entity = RadarLoopImageEntity(hass, OFFICE, "KNKX", loop_hours=loop_hours) + entity._store = _RecordingStore(frames, accept=accept) + entity.entity_id = "image.noaa_sgx_weather_radar_loop" + entity.async_write_ha_state = MagicMock() + return entity + + +def _stored(count, start_minutes_ago=240): + """A buffer of ``count`` frames, oldest first. + + Timestamps are derived from the real clock rather than a fixed date: the + code under test prunes and samples against ``dt_util.utcnow()``, so any + hard-coded date would silently stop being inside the window once enough + real time had passed, and the suite would start failing on a calendar day + rather than on a code change. + """ + from noaa_it_all import image as image_module + + now = image_module.dt_util.utcnow() + return [ + ( + now - timedelta(minutes=start_minutes_ago - index * 10), + f"/frames/stored-{index}.gif", + ) + for index in range(count) + ] + + +class TestRadarLoopUpstreamMode(unittest.TestCase): + """With the option off, nothing about the entity may have changed.""" + + def test_it_fetches_noaas_own_loop(self): + from noaa_it_all.image import RadarLoopImageEntity + entity = RadarLoopImageEntity(HASS, OFFICE, "KNKX") + self.assertIn("_loop.gif", entity._base_url()) + + def test_it_writes_nothing_to_disk(self): + from noaa_it_all.image import RadarLoopImageEntity + hass = MagicMock() + entity = RadarLoopImageEntity(HASS, OFFICE, "KNKX") + self.assertIsNone(entity._store) + hass.config.path.assert_not_called() + + def test_it_reports_upstream_mode(self): + from noaa_it_all.image import RadarLoopImageEntity + entity = RadarLoopImageEntity(HASS, OFFICE, "KNKX") + self.assertEqual("upstream", entity.extra_state_attributes["loop_mode"]) + + def test_it_still_caches_through_a_failure(self): + from noaa_it_all.image import RadarLoopImageEntity + entity = RadarLoopImageEntity(HASS, OFFICE, "KNKX") + entity.entity_id = "image.noaa_sgx_weather_radar_loop" + entity.async_write_ha_state = MagicMock() + _refresh(entity, _FakeResponse(content_type="image/gif", body=GIF)) + self.assertEqual(GIF, _run(entity.async_image())) + _refresh(entity, _FakeResponse(status=503)) + self.assertEqual(GIF, _run(entity.async_image())) + + +class TestRadarLoopLocalMode(unittest.TestCase): + """Collecting frames and assembling them into a longer animation.""" + + def test_it_fetches_single_scans_rather_than_the_loop(self): + entity = _local_loop_entity() + self.assertIn("_0.gif", entity._base_url()) + + def test_a_fetched_scan_is_stored_under_its_published_time(self): + entity = _local_loop_entity(frames=_stored(10)) + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"LOOP")): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + timestamp, data = entity._store.added[0] + self.assertEqual(GIF, data) + self.assertEqual( + datetime(2026, 8, 23, 11, 54, tzinfo=timezone.utc), timestamp + ) + + def test_the_assembled_loop_is_what_gets_served(self): + entity = _local_loop_entity(frames=_stored(10)) + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"ASSEMBLED")): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertEqual(b"ASSEMBLED", _run(entity.async_image())) + self.assertEqual("local", entity.extra_state_attributes["loop_mode"]) + + def test_a_scan_already_held_is_not_reassembled(self): + entity = _local_loop_entity(frames=_stored(10)) + headers = {"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"} + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"ASSEMBLED")) as build: + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, headers=headers)) + self.assertEqual(1, build.call_count) + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, headers=headers)) + self.assertEqual(1, build.call_count) + + def test_storing_a_frame_prunes_the_window(self): + entity = _local_loop_entity(frames=_stored(10)) + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"LOOP")): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertEqual(timedelta(hours=24), entity._store.pruned[0][0]) + + def test_a_missing_last_modified_still_yields_a_frame(self): + entity = _local_loop_entity(frames=_stored(10)) + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"LOOP")): + _refresh(entity, _FakeResponse(content_type="image/gif", body=GIF)) + self.assertEqual(1, len(entity._store.added)) + + def test_a_thin_buffer_falls_back_to_noaas_loop(self): + entity = _local_loop_entity(frames=_stored(2)) + session = _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertTrue(any("_loop.gif" in url for url, _ in session.calls)) + self.assertEqual("upstream", entity.extra_state_attributes["loop_mode"]) + + def test_a_failed_assembly_falls_back_to_noaas_loop(self): + entity = _local_loop_entity(frames=_stored(10)) + with patch("noaa_it_all.image.assemble_gif", return_value=None): + session = _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertTrue(any("_loop.gif" in url for url, _ in session.calls)) + + def test_a_disk_that_refuses_the_frame_still_shows_a_loop(self): + entity = _local_loop_entity(frames=_stored(10), accept=False) + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"LOOP")): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertEqual(b"LOOP", _run(entity.async_image())) + + def test_the_reported_window_matches_the_frames_used(self): + frames = _stored(10) + used = [path for _, path in frames[2:8]] + entity = _local_loop_entity(frames=frames) + with patch("noaa_it_all.image.assemble_gif", + return_value=_loop_of(b"LOOP", used)): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + attributes = entity.extra_state_attributes + self.assertEqual(6, attributes["frame_count"]) + self.assertEqual(frames[2][0].isoformat(), attributes["window_start"]) + self.assertEqual(frames[7][0].isoformat(), attributes["window_end"]) + + def test_the_reported_count_is_what_the_encoder_used_not_what_it_was_given(self): + """Frames shed to fit the size limit must not be counted as shown. + + These attributes exist so a loop quietly shorter than configured is + visible; counting the frames offered rather than the frames encoded + would hide precisely that case. + """ + frames = _stored(10) + entity = _local_loop_entity(frames=frames) + # The encoder kept three of the ten it was handed. + with patch("noaa_it_all.image.assemble_gif", + return_value=_loop_of( + b"LOOP", [path for _, path in frames[7:]])): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertEqual(3, entity.extra_state_attributes["frame_count"]) + + +class TestRadarLoopValidatorsAreScopedToTheirResource(unittest.TestCase): + """This entity fetches two URLs through one set of cached validators. + + Offering the single frame's ETag back when asking for the animation is + asking the wrong question about the wrong file, and a server that answered + 304 to it would hand back a still image as the loop. + """ + + HEADERS = { + "last-modified": "Sun, 23 Aug 2026 11:54:00 GMT", + "etag": '"frame-etag"', + } + + LOOP_HEADERS = {"etag": '"loop-etag"'} + + @staticmethod + def _sent_to(session, needle): + """Return the conditional headers sent to the matching request.""" + for url, kwargs in session.calls: + if needle in url: + headers = kwargs.get("headers", {}) + return { + key: value for key, value in headers.items() + if key in ("If-None-Match", "If-Modified-Since") + } + raise AssertionError(f"no request was made to {needle}") + + def _seed_both_resources(self, entity): + """Fetch the frame and the fallback loop once each, caching both.""" + _refresh( + entity, + _FakeResponse( + content_type="image/gif", body=GIF, headers=self.HEADERS), + _FakeResponse( + content_type="image/gif", body=b"GIF89a-loop", + headers=self.LOOP_HEADERS), + ) + + def test_each_resource_is_revalidated_with_its_own_validator(self): + """Not merely "never the wrong one" -- each must still get the right one. + + Scoping validators by resource is only half the job: if the two + resources shared one slot, each fetch would evict the other's and + neither could ever revalidate, so the fallback loop would be + re-downloaded in full on every refresh for as long as the buffer was + filling. + """ + entity = _local_loop_entity(frames=_stored(2)) + self._seed_both_resources(entity) + session = _refresh( + entity, + _FakeResponse( + content_type="image/gif", body=b"GIF89a-newer", + headers={ + "last-modified": "Sun, 23 Aug 2026 12:04:00 GMT", + "etag": '"newer-frame-etag"', + }, + ), + _FakeResponse( + content_type="image/gif", body=b"GIF89a-loop", + headers=self.LOOP_HEADERS), + ) + self.assertEqual( + {"If-None-Match": '"frame-etag"'}, self._sent_to(session, "_0.gif") + ) + self.assertEqual( + {"If-None-Match": '"loop-etag"'}, self._sent_to(session, "_loop.gif") + ) + + def test_neither_resource_is_offered_the_others_validator(self): + entity = _local_loop_entity(frames=_stored(2)) + self._seed_both_resources(entity) + session = _refresh( + entity, + _FakeResponse( + content_type="image/gif", body=b"GIF89a-newer", + headers={"last-modified": "Sun, 23 Aug 2026 12:04:00 GMT", + "etag": '"newer-frame-etag"'}, + ), + _FakeResponse( + content_type="image/gif", body=b"GIF89a-loop", + headers=self.LOOP_HEADERS), + ) + self.assertNotIn( + '"loop-etag"', self._sent_to(session, "_0.gif").values() + ) + self.assertNotIn( + '"frame-etag"', self._sent_to(session, "_loop.gif").values() + ) + + def test_the_fallback_loop_is_cache_busted_like_every_other_url(self): + """It was before this PR, and a CDN will happily serve a stale copy.""" + entity = _local_loop_entity(frames=_stored(2)) + session = _refresh( + entity, + _FakeResponse( + content_type="image/gif", body=GIF, headers=self.HEADERS), + _FakeResponse(content_type="image/gif", body=b"GIF89a-loop"), + ) + loop_url = next(url for url, _ in session.calls if "_loop.gif" in url) + self.assertIn("?t=", loop_url) + + def test_a_single_url_entity_still_revalidates(self): + """Scoping validators must not switch conditional requests off.""" + entity = _make_entity() + _refresh(entity, _FakeResponse(headers={"etag": '"png-etag"'})) + session = _refresh(entity, _FakeResponse()) + url, kwargs = session.calls[0] + self.assertEqual('"png-etag"', kwargs["headers"]["If-None-Match"]) + + +class TestRadarLoopLocalModeSurvivesFailures(unittest.TestCase): + """The invariant the whole image module exists to hold, in local mode. + + A refresh that fails must leave the displayed animation exactly as it + was -- the local loop adds a fetch, a disk and an encoder to the list of + things that can fail, and none of them may blank the card. + """ + + def _seeded(self): + entity = _local_loop_entity(frames=_stored(10)) + with patch("noaa_it_all.image.assemble_gif", return_value=_loop_of(b"GOOD-LOOP")): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + self.assertEqual(b"GOOD-LOOP", _run(entity.async_image())) + entity.async_write_ha_state.reset_mock() + return entity + + def _assert_survives(self, result, build=b"GOOD-LOOP"): + entity = self._seeded() + stamp = entity.image_last_updated + with patch("noaa_it_all.image.assemble_gif", + return_value=_loop_of(build) if build else None): + _refresh(entity, result) + self.assertEqual(b"GOOD-LOOP", _run(entity.async_image())) + self.assertEqual(stamp, entity.image_last_updated) + entity.async_write_ha_state.assert_not_called() + + def test_a_server_error_leaves_the_loop_alone(self): + self._assert_survives(_FakeResponse(status=503)) + + def test_a_network_error_leaves_the_loop_alone(self): + self._assert_survives(asyncio.TimeoutError()) + + def test_an_empty_body_leaves_the_loop_alone(self): + self._assert_survives(_FakeResponse(content_type="image/gif", body=b"")) + + def test_a_non_image_response_leaves_the_loop_alone(self): + self._assert_survives( + _FakeResponse(content_type="text/html", body=b"no") + ) + + def test_an_encoder_that_gives_up_leaves_the_loop_alone(self): + """Assembly returning None must not blank the card either. + + The fallback fetch of NOAA's own loop fails here too, so there is + nothing at all to fall back to -- the previous animation has to stand. + """ + entity = self._seeded() + stamp = entity.image_last_updated + with patch("noaa_it_all.image.assemble_gif", return_value=None): + _refresh( + entity, + _FakeResponse( + content_type="image/gif", body=b"NEW-FRAME", + headers={"last-modified": "Sun, 23 Aug 2026 12:04:00 GMT"}, + ), + _FakeResponse(status=503), + ) + self.assertEqual(b"GOOD-LOOP", _run(entity.async_image())) + self.assertEqual(stamp, entity.image_last_updated) + + +class TestRadarLoopHoursOption(unittest.TestCase): + """Reading the option off the config entry.""" + + @staticmethod + def _entry(value): + entry = MagicMock() + entry.data = {"office_code": OFFICE} + entry.options = {} if value is None else {"radar_loop_hours": value} + return entry + + def test_a_saved_value_is_used(self): + from noaa_it_all.image import radar_loop_hours + self.assertEqual(6, radar_loop_hours(self._entry(6))) + + def test_the_default_is_a_full_day(self): + from noaa_it_all.image import radar_loop_hours + self.assertEqual(24, radar_loop_hours(self._entry(None))) + + def test_zero_is_honoured_as_the_opt_out(self): + from noaa_it_all.image import radar_loop_hours + self.assertEqual(0, radar_loop_hours(self._entry(0))) + + def test_out_of_range_values_are_clamped(self): + from noaa_it_all.image import radar_loop_hours + self.assertEqual(24, radar_loop_hours(self._entry(999))) + self.assertEqual(0, radar_loop_hours(self._entry(-5))) + + def test_an_unusable_value_falls_back_to_the_default(self): + from noaa_it_all.image import radar_loop_hours + self.assertEqual(24, radar_loop_hours(self._entry("lots"))) + + +try: + from PIL import Image as _PILImage + _PIL = True +except ImportError: # pragma: no cover - Pillow ships with Home Assistant core + _PIL = False + + +def _nexrad_frame(index, size=(240, 200)): + """A transparent, separately-palettised GIF with a drifting echo.""" + from io import BytesIO + + scale = [ + (4, 233, 231), (1, 159, 244), (3, 0, 244), (2, 253, 2), + (1, 197, 1), (0, 142, 0), (253, 248, 2), (253, 0, 0), + ] + image = _PILImage.new("P", size, 0) + palette = [0, 0, 0] + for colour in scale: + palette += list(colour) + palette += [0, 0, 0] * (256 - 1 - len(scale)) + image.putpalette(palette) + origin = (index * 9) % (size[0] - 60) + for x in range(origin, origin + 60): + for y in range(size[1] // 4, size[1] // 4 * 3): + image.putpixel((x, y), 1 + ((x + y) // 9) % len(scale)) + buffer = BytesIO() + image.save(buffer, format="GIF", transparency=0) + return buffer.getvalue() + + +@unittest.skipUnless(_PIL, "Pillow is required to assemble a GIF") +class TestRadarLoopEndToEnd(unittest.TestCase): + """The whole path, with a real directory and a real encoder. + + Every other test in this file swaps the store or the encoder for a fake, + which leaves the wiring between them unexercised -- and that wiring is + where a 24-hour loop either works or quietly serves NOAA's 50 minutes + forever. + """ + + def _entity(self, directory): + from noaa_it_all.image import RadarLoopImageEntity + + hass = MagicMock() + hass.config.path = MagicMock(return_value=directory) + + async def _executor(func, *args): + return func(*args) + + hass.async_add_executor_job = _executor + entity = RadarLoopImageEntity(hass, OFFICE, "KLTX", loop_hours=24) + entity.entity_id = "image.noaa_sgx_weather_radar_loop" + entity.async_write_ha_state = MagicMock() + return entity + + @staticmethod + def _start(polls): + """The publication time of the first frame in a run of ``polls``. + + Anchored to the real clock, not to a fixed date. The code under test + prunes and samples against ``dt_util.utcnow()``, so frames pinned to a + calendar date drop out of the window as soon as enough real time + passes -- which would make this suite start failing on a date rather + than on a change. Truncated to whole seconds because that is the + resolution both the HTTP date header and the frame filename carry. + """ + from noaa_it_all import image as image_module + + now = image_module.dt_util.utcnow().replace(microsecond=0) + return now - timedelta(minutes=10 * polls) + + def _poll(self, entity, index, start): + """One refresh, with NOAA's loop queued behind it as the fallback.""" + published = (start + timedelta(minutes=10 * index)).strftime( + "%a, %d %b %Y %H:%M:%S GMT" + ) + return _refresh( + entity, + _FakeResponse( + content_type="image/gif", + body=_nexrad_frame(index), + headers={"last-modified": published}, + ), + _FakeResponse(content_type="image/gif", body=b"GIF89a-noaa-loop"), + ) + + def test_the_loop_grows_from_noaas_into_a_locally_built_animation(self): + with tempfile.TemporaryDirectory() as directory: + entity = self._entity(directory) + start = self._start(12) + + # Too thin to improve on NOAA: its own loop is what gets served. + self._poll(entity, 0, start) + self.assertEqual("upstream", entity.extra_state_attributes["loop_mode"]) + self.assertEqual(b"GIF89a-noaa-loop", _run(entity.async_image())) + + for index in range(1, 12): + self._poll(entity, index, start) + + attributes = entity.extra_state_attributes + self.assertEqual("local", attributes["loop_mode"]) + self.assertEqual(12, attributes["frame_count"]) + self.assertEqual( + start.isoformat(), attributes["window_start"] + ) + + served = _run(entity.async_image()) + self.assertEqual("image/gif", entity.content_type) + with tempfile.NamedTemporaryFile(suffix=".gif") as handle: + handle.write(served) + handle.flush() + animation = _PILImage.open(handle.name) + # A drifting echo means no two frames are identical, so none of + # them are merged away. + self.assertEqual(12, animation.n_frames) + self.assertEqual(0, animation.info.get("loop")) + + stored = os.listdir(os.path.join(directory, "KLTX")) + self.assertEqual(12, len(stored)) + self.assertTrue(all(name.endswith(".gif") for name in stored)) + + def test_frames_collected_before_a_restart_are_still_there_after_it(self): + """The whole feature rests on this: a restart must not start over.""" + with tempfile.TemporaryDirectory() as directory: + start = self._start(9) + first = self._entity(directory) + for index in range(8): + self._poll(first, index, start) + self.assertEqual("local", first.extra_state_attributes["loop_mode"]) + + # A new entity over the same directory stands in for the restart. + second = self._entity(directory) + self._poll(second, 8, start) + attributes = second.extra_state_attributes + self.assertEqual("local", attributes["loop_mode"]) + self.assertEqual(9, attributes["frame_count"]) + self.assertEqual(start.isoformat(), attributes["window_start"]) + + +class TestRadarLoopPruning(unittest.TestCase): + """Pruning has to keep up with the window even when nothing new arrives.""" + + def test_pruning_runs_even_when_the_scan_has_not_changed(self): + """A radar site stuck on one scan must not let the window stretch. + + Maintenance windows and outages hold one Last-Modified for hours. If + pruning only ran when a frame was stored, nothing would age out for the + whole outage and the loop would quietly cover more than it claims. + """ + entity = _local_loop_entity(frames=_stored(10)) + headers = {"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"} + with patch("noaa_it_all.image.assemble_gif", + return_value=_loop_of(b"LOOP")): + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, headers=headers)) + first = len(entity._store.pruned) + _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, headers=headers)) + self.assertGreater(len(entity._store.pruned), first) + + +class TestRadarLoopMinimumFrames(unittest.TestCase): + """The floor has to apply to what ships, not to what is on disk.""" + + def test_a_sparse_window_falls_back_even_with_frames_on_disk(self): + """The floor has to be judged on the sampled frames, not the directory. + + Here the store holds 84 frames -- comfortably over the floor -- but all + bar four are older than the window, so the loop that would actually + ship is four frames long. That is fewer than NOAA's ten, which is + precisely what RADAR_LOOP_MIN_FRAMES exists to prevent, and it is + enough frames that the encoder would happily build it. + """ + from noaa_it_all import image as image_module + + now = image_module.dt_util.utcnow() + recent = [ + (now - timedelta(minutes=15 * index), f"/frames/recent-{index}.gif") + for index in range(4) + ][::-1] + ancient = [ + (now - timedelta(days=3, minutes=10 * index), f"/frames/old-{index}.gif") + for index in range(80) + ][::-1] + entity = _local_loop_entity(frames=ancient + recent) + self.assertGreaterEqual(len(entity._store.frames), 84) + + with patch("noaa_it_all.image.assemble_gif", + return_value=_loop_of(b"LOOP")) as build: + session = _refresh(entity, _FakeResponse( + content_type="image/gif", body=GIF, + headers={"last-modified": "Sun, 23 Aug 2026 11:54:00 GMT"}, + )) + build.assert_not_called() + self.assertTrue(any("_loop.gif" in url for url, _ in session.calls)) + self.assertEqual("upstream", entity.extra_state_attributes["loop_mode"]) + + +class TestRadarFrameHousekeeping(unittest.TestCase): + """Frames for a site nothing collects any more must not be orphaned.""" + + def _hass(self, entries, present): + hass = MagicMock() + hass.config.path = MagicMock(return_value="/config/radar_frames") + hass.config_entries.async_entries = MagicMock(return_value=entries) + + async def _executor(func, *args): + return list(present) + + hass.async_add_executor_job = _executor + return hass + + @staticmethod + def _entry(office, hours): + entry = MagicMock() + entry.data = {"office_code": office} + entry.options = {"radar_loop_hours": hours} + return entry + + def _run_cleanup(self, hass, keep): + from noaa_it_all import image as image_module + + removed = [] + + class _Store: + def __init__(self, _hass, _base, site): + self.site = site + + async def async_remove_all(self): + removed.append(self.site) + + with patch.object(image_module, "RadarFrameStore", _Store): + _run(image_module.async_discard_unused_radar_frames(hass, keep=keep)) + return removed + + def test_the_site_in_use_is_kept(self): + hass = self._hass([self._entry("ILM", 24)], ["KLTX"]) + self.assertEqual([], self._run_cleanup(hass, keep="KLTX")) + + def test_a_site_switched_away_from_is_removed(self): + """Changing office leaves the old site's frames with no owner.""" + hass = self._hass([self._entry("SGX", 24)], ["KLTX", "KNKX"]) + self.assertEqual(["KLTX"], self._run_cleanup(hass, keep="KNKX")) + + def test_turning_the_loop_off_reclaims_the_disk(self): + """The options screen promises 0 hours will "store nothing".""" + hass = self._hass([self._entry("ILM", 0)], ["KLTX"]) + self.assertEqual(["KLTX"], self._run_cleanup(hass, keep=None)) + + def test_a_site_another_entry_still_uses_is_kept(self): + hass = self._hass( + [self._entry("ILM", 0), self._entry("SGX", 24)], ["KLTX", "KNKX"] + ) + self.assertEqual(["KLTX"], self._run_cleanup(hass, keep=None)) diff --git a/tests/test_init.py b/tests/test_init.py index 1c86f84..9f1a002 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -225,3 +225,58 @@ def test_registered_listener_reloads_the_entry(self): if __name__ == "__main__": unittest.main() + + +class TestRemoveEntryCleansUpRadarFrames(unittest.TestCase): + """Uninstalling should not leave a day of radar behind. + + The frames live in the configuration directory, so anything left there + rides along in every backup from then on. + """ + + @staticmethod + def _entry(entry_id, office): + entry = MagicMock() + entry.entry_id = entry_id + entry.data = {"office_code": office} + entry.options = {} + return entry + + @staticmethod + def _hass(entries): + hass = MagicMock() + hass.config.path = MagicMock(return_value="/config/noaa_it_all/radar_frames") + hass.config_entries.async_entries = MagicMock(return_value=entries) + return hass + + def _remove(self, hass, entry): + import noaa_it_all + + removed = [] + + class _Store: + def __init__(self, _hass, _base, site): + self.site = site + + async def async_remove_all(self): + removed.append(self.site) + + with patch.object(noaa_it_all, "RadarFrameStore", _Store): + _run(noaa_it_all.async_remove_entry(hass, entry)) + return removed + + def test_the_sites_frames_are_deleted(self): + entry = self._entry("entry_1", "ILM") + removed = self._remove(self._hass([entry]), entry) + self.assertEqual(["KLTX"], removed) + + def test_frames_shared_with_another_entry_are_kept(self): + entry = self._entry("entry_1", "ILM") + other = self._entry("entry_2", "ILM") + removed = self._remove(self._hass([entry, other]), entry) + self.assertEqual([], removed) + + def test_an_office_without_a_radar_site_is_a_no_op(self): + entry = self._entry("entry_1", "ZZZ") + removed = self._remove(self._hass([entry]), entry) + self.assertEqual([], removed) diff --git a/tests/test_radar_loop.py b/tests/test_radar_loop.py new file mode 100644 index 0000000..ea6adb9 --- /dev/null +++ b/tests/test_radar_loop.py @@ -0,0 +1,520 @@ +"""Tests for radar_loop.py: the frame store, the sampler and the GIF encoder. + +Unlike the entity tests, nothing here mocks Home Assistant -- ``radar_loop`` +deliberately imports none of it. These run against a real temporary directory +and a real Pillow, which is the only way the encoder's palette handling can +actually be checked. +""" + +import asyncio +import importlib +import os +import sys +import tempfile +import types +import unittest +from datetime import datetime, timedelta, timezone + +_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_CC = os.path.join(_REPO, "custom_components") + +if _CC not in sys.path: + sys.path.insert(0, _CC) + +# ``noaa_it_all/__init__.py`` imports Home Assistant, so importing the package +# the ordinary way here would mean mocking half of it -- exactly what keeping +# Home Assistant out of radar_loop was meant to avoid. Instead, mount the same +# directory under a private package name and import the one submodule from +# there. Its ``from .const import`` resolves against that name, const imports +# nothing but the standard library, and ``sys.modules["noaa_it_all"]`` is left +# alone for the test modules that do want the real package. +_PACKAGE = "_noaa_it_all_radar_loop" + +if _PACKAGE not in sys.modules: + _shim = types.ModuleType(_PACKAGE) + _shim.__path__ = [os.path.join(_CC, "noaa_it_all")] + sys.modules[_PACKAGE] = _shim + +radar_loop = importlib.import_module(f"{_PACKAGE}.radar_loop") + +PIL_AVAILABLE = radar_loop.PIL_AVAILABLE +RadarFrameStore = radar_loop.RadarFrameStore +assemble_gif = radar_loop.assemble_gif +frame_name = radar_loop.frame_name +parse_http_date = radar_loop.parse_http_date +select_frames = radar_loop.select_frames + +if PIL_AVAILABLE: + from PIL import Image + + +NOW = datetime(2026, 8, 23, 12, 0, 0, tzinfo=timezone.utc) + +# NWS reflectivity is a fixed scale of roughly this many colours. +SCALE = [ + (4, 233, 231), (1, 159, 244), (3, 0, 244), (2, 253, 2), (1, 197, 1), + (0, 142, 0), (253, 248, 2), (229, 188, 0), (253, 149, 0), (253, 0, 0), + (212, 0, 0), (188, 0, 0), (248, 0, 253), (152, 84, 198), +] + + +def _run(coro): + """Run a coroutine on a private loop, leaving the ambient one intact. + + Mirrors the helper in test_image.py: ``asyncio.run()`` clears the thread's + current event loop on return, which breaks other test modules. + """ + previous = None + try: + previous = asyncio.get_event_loop_policy().get_event_loop() + except RuntimeError: + pass + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + asyncio.set_event_loop(previous) + + +class _FakeHass: + """Runs "executor" work inline; the store only needs the one method.""" + + def __init__(self): + self.calls = 0 + + async def async_add_executor_job(self, func, *args): + self.calls += 1 + return func(*args) + + +def _write_frame(path, colour=(253, 0, 0), size=(60, 40), empty=False): + """Write a transparent, separately-palettised GIF, as RIDGE serves them. + + Index 0 is the transparent background and the echo colours follow, so each + frame carries its own palette -- the arrangement that makes naively + concatenating frames go wrong. + """ + image = Image.new("P", size, 0) + palette = [0, 0, 0] + list(colour) + [0, 0, 0] * 254 + image.putpalette(palette) + if not empty: + for x in range(size[0] // 6, size[0] // 6 * 5): + for y in range(size[1] // 4, size[1] // 4 * 3): + image.putpixel((x, y), 1) + image.save(path, transparency=0) + + +class TestParseHttpDate(unittest.TestCase): + """Last-Modified is the frame's identity, so parsing it must not throw.""" + + def test_a_real_header_parses_to_utc(self): + parsed = parse_http_date("Sun, 23 Aug 2026 11:54:00 GMT") + self.assertEqual(datetime(2026, 8, 23, 11, 54, tzinfo=timezone.utc), parsed) + + def test_unparseable_input_is_none_rather_than_an_exception(self): + for value in (None, "", "not a date", "Sun, 99 Xxx 2026", 17): + with self.subTest(value=value): + self.assertIsNone(parse_http_date(value)) + + +class TestFrameStore(unittest.TestCase): + """The file name is the index, so naming and dedup carry the weight.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.hass = _FakeHass() + self.store = RadarFrameStore(self.hass, self._tmp.name, "KLTX") + + def _add(self, minutes_ago, data=b"gif-bytes"): + return _run(self.store.async_add_frame( + NOW - timedelta(minutes=minutes_ago), data + )) + + def test_a_frame_is_written_under_its_timestamp(self): + self.assertTrue(self._add(0)) + self.assertEqual( + ["20260823T120000Z.gif"], os.listdir(self.store.path) + ) + + def test_the_same_scan_is_never_stored_twice(self): + self.assertTrue(self._add(0, b"first")) + self.assertFalse(self._add(0, b"second")) + stored = os.path.join(self.store.path, "20260823T120000Z.gif") + # The original must survive: re-fetching a scan we already hold is the + # steady state, not a correction. + self.assertEqual(b"first", open(stored, "rb").read()) + + def test_a_successful_write_leaves_no_temporary_file(self): + self._add(0) + self.assertEqual( + [], [n for n in os.listdir(self.store.path) if n.endswith(".tmp")] + ) + + def test_frames_come_back_oldest_first(self): + for minutes in (30, 0, 60, 15): + self._add(minutes) + frames = _run(self.store.async_frames()) + self.assertEqual(sorted(t for t, _ in frames), [t for t, _ in frames]) + self.assertEqual(4, len(frames)) + + def test_a_missing_directory_reads_as_empty(self): + self.assertEqual([], _run(self.store.async_frames())) + + def test_pruning_drops_frames_outside_the_window(self): + self._add(10) + self._add(400) + removed = _run(self.store.async_prune(timedelta(hours=1), NOW)) + self.assertEqual(1, removed) + self.assertEqual(1, len(_run(self.store.async_frames()))) + + def test_pruning_removes_names_it_cannot_date(self): + self._add(0) + os.makedirs(self.store.path, exist_ok=True) + for junk in ("notes.txt", "20260823T120000Z.gif.tmp", "half-written"): + open(os.path.join(self.store.path, junk), "w").close() + _run(self.store.async_prune(timedelta(hours=1), NOW)) + self.assertEqual(["20260823T120000Z.gif"], os.listdir(self.store.path)) + + def test_frames_dated_in_the_future_are_discarded(self): + """A wrong clock must not produce frames that outlive every real one. + + Ageing works backwards from now, so a future-dated frame is never + reached by the cutoff. Without an upper bound it would survive + indefinitely while genuine frames aged out around it, and the loop + would stick on a set that never updates. + """ + self._add(10) + self._add(-600) # ten hours ahead of the clock + removed = _run(self.store.async_prune(timedelta(hours=24), NOW)) + self.assertEqual(1, removed) + remaining = _run(self.store.async_frames()) + self.assertEqual([NOW - timedelta(minutes=10)], [t for t, _ in remaining]) + + def test_ordinary_clock_skew_is_tolerated(self): + """NOAA's clock and ours need not agree to the second.""" + self._add(-5) + _run(self.store.async_prune(timedelta(hours=24), NOW)) + self.assertEqual(1, len(_run(self.store.async_frames()))) + + def test_the_file_cap_trims_the_oldest(self): + original = radar_loop.RADAR_FRAME_MAX_FILES + radar_loop.RADAR_FRAME_MAX_FILES = 5 + self.addCleanup(setattr, radar_loop, "RADAR_FRAME_MAX_FILES", original) + for minutes in range(0, 80, 10): + self._add(minutes) + _run(self.store.async_prune(timedelta(hours=24), NOW)) + remaining = _run(self.store.async_frames()) + self.assertEqual(5, len(remaining)) + self.assertEqual(NOW, remaining[-1][0]) + + def test_a_directory_that_cannot_be_created_reports_failure(self): + """A disk that will not take the frame must not reach the dashboard. + + The blockage here is a plain file sitting where the frame directory + needs to be, which fails identically whatever user the tests run as -- + unlike a chmod, which root ignores. + """ + os.makedirs(os.path.dirname(self.store.path), exist_ok=True) + with open(self.store.path, "w") as handle: + handle.write("in the way") + self.assertFalse(self._add(0)) + + def test_removing_all_frames_clears_the_directory(self): + self._add(0) + _run(self.store.async_remove_all()) + self.assertFalse(os.path.exists(self.store.path)) + # Removing a store that was never written must not raise. + _run(self.store.async_remove_all()) + + def test_every_store_method_runs_off_the_event_loop(self): + """Blocking file I/O in a coroutine stalls all of Home Assistant. + + Asserting on the call site rather than on reviewer discipline, the + same way test_image.py guards state writes. + """ + import ast + import inspect + + blocking = {"listdir", "scandir", "unlink", "replace", "makedirs", "rmtree"} + tree = ast.parse(inspect.getsource(radar_loop)) + offenders = set() + for node in ast.walk(tree): + if not isinstance(node, ast.AsyncFunctionDef): + continue + for call in ast.walk(node): + if isinstance(call, ast.Call): + func = call.func + if isinstance(func, ast.Name) and func.id == "open": + offenders.add(node.name) + if isinstance(func, ast.Attribute) and func.attr in blocking: + offenders.add(node.name) + self.assertEqual(set(), offenders) + + +class TestSelectFrames(unittest.TestCase): + """Sampling is by time, so a missed poll must not re-space the loop.""" + + @staticmethod + def _frames(count, step_minutes=10, end=NOW): + return [ + (end - timedelta(minutes=step_minutes * i), f"/frames/{i}.gif") + for i in range(count - 1, -1, -1) + ] + + def test_a_short_buffer_is_used_whole(self): + frames = self._frames(5) + chosen = select_frames( + frames, window=timedelta(hours=24), max_frames=72, now=NOW + ) + self.assertEqual([p for _, p in frames], chosen) + + def test_a_full_buffer_is_thinned_to_the_cap(self): + frames = self._frames(144) + chosen = select_frames( + frames, window=timedelta(hours=24), max_frames=72, now=NOW + ) + self.assertEqual(72, len(chosen)) + + def test_the_newest_frame_is_always_last(self): + frames = self._frames(144) + chosen = select_frames( + frames, window=timedelta(hours=24), max_frames=72, now=NOW + ) + self.assertEqual(frames[-1][1], chosen[-1]) + + def test_a_gap_shortens_the_loop_rather_than_repeating_a_frame(self): + """Exercises the sampler proper, so the buffer must exceed the cap. + + With 72 or fewer frames the function returns them wholesale and the + step/tolerance logic -- the part that decides whether a gap gets filled + with a frame from the wrong time -- never runs at all. + """ + # Home Assistant was off for the middle eight hours of the window. + frames = [ + (NOW - timedelta(minutes=5 * i), f"/frames/{i}.gif") + for i in range(288) + if not 8 * 60 <= 5 * i < 16 * 60 + ] + self.assertGreater(len(frames), 72) + chosen = select_frames( + frames, window=timedelta(hours=24), max_frames=72, now=NOW + ) + self.assertEqual(len(set(chosen)), len(chosen)) + # The gap is a third of the window, so a sampler that filled it with + # the nearest available frame would still return the full 72. + self.assertLess(len(chosen), 72) + + def test_an_empty_buffer_selects_nothing(self): + self.assertEqual([], select_frames( + [], window=timedelta(hours=24), max_frames=72, now=NOW + )) + + def test_frames_outside_the_window_are_excluded(self): + """A small buffer must not skip the window check. + + Pruning is what normally keeps the directory inside the window, and it + can be behind -- a radar site stuck on one scan, or an entity that has + just found an old directory on disk. Handing those frames to the + encoder would build a loop from the wrong day and then report it as + covering the configured window. + """ + stale = [ + (NOW - timedelta(days=5) + timedelta(minutes=10 * i), f"/old/{i}.gif") + for i in range(10) + ] + self.assertEqual([], select_frames( + stale, window=timedelta(hours=6), max_frames=72, now=NOW + )) + + def test_a_mixed_buffer_keeps_only_the_frames_in_window(self): + recent = self._frames(4, step_minutes=30) + stale = [(NOW - timedelta(days=2), "/old/x.gif")] + chosen = select_frames( + stale + recent, window=timedelta(hours=6), max_frames=72, now=NOW + ) + self.assertEqual([path for _, path in recent], chosen) + + def test_frames_dated_in_the_future_are_not_selected(self): + frames = self._frames(4, step_minutes=10) + [ + (NOW + timedelta(hours=3), "/future/x.gif") + ] + chosen = select_frames( + frames, window=timedelta(hours=6), max_frames=72, now=NOW + ) + self.assertNotIn("/future/x.gif", chosen) + + +@unittest.skipUnless(PIL_AVAILABLE, "Pillow is required to assemble a GIF") +class TestAssembleGif(unittest.TestCase): + """The encoder is where this feature is most likely to ship broken.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + + def _frames(self, colours, size=(60, 40), empty=()): + paths = [] + for index, colour in enumerate(colours): + path = os.path.join(self._tmp.name, f"f{index:03d}.gif") + _write_frame(path, colour, size=size, empty=index in empty) + paths.append(path) + return paths + + def _open(self, loop): + """Write an assembled loop out and reopen it as an image.""" + out = os.path.join(self._tmp.name, "out.gif") + with open(out, "wb") as handle: + handle.write(loop.data) + return Image.open(out) + + def test_the_result_names_the_frames_it_used(self): + """Callers describe the loop from this, so it must be the truth.""" + paths = self._frames([(253, 0, 0), (2, 253, 2), (3, 0, 244)]) + broken = os.path.join(self._tmp.name, "broken.gif") + with open(broken, "wb") as handle: + handle.write(b"not a gif") + loop = assemble_gif(paths + [broken]) + self.assertEqual(paths, list(loop.paths)) + + def test_every_frame_keeps_its_own_colour(self): + """The regression that matters. + + Left to itself the GIF writer adopts frame one's palette as the global + table and remaps everything after it, so colours that appear later in + the loop collapse to whatever frame one happened to contain. On a real + radar that means an overnight loop starts empty and the storm that + arrives at noon is rendered as background. + """ + colours = [(253, 0, 0), (2, 253, 2), (3, 0, 244)] + result = self._open(assemble_gif(self._frames(colours))) + self.assertEqual(3, result.n_frames) + for index, expected in enumerate(colours): + result.seek(index) + actual = result.convert("RGB").getpixel((30, 20)) + with self.subTest(frame=index): + self.assertLess( + max(abs(a - b) for a, b in zip(actual, expected)), 12, + f"frame {index} came back {actual}, expected {expected}", + ) + + def test_a_storm_arriving_after_an_empty_start_keeps_its_colours(self): + """The palette must be sampled across the window, not taken from frame one.""" + colours = [(0, 0, 0)] * 8 + SCALE + paths = self._frames(colours, size=(120, 90), empty=range(8)) + result = self._open(assemble_gif(paths)) + result.seek(result.n_frames - 1) + final = result.convert("RGB") + seen = {final.getpixel((x, 45)) for x in range(20, 100, 4)} + seen.discard((0, 0, 0)) + self.assertTrue(seen, "the storm was flattened into the background") + + def test_the_loop_repeats_and_holds_on_the_newest_frame(self): + paths = self._frames([(253, 0, 0), (2, 253, 2), (3, 0, 244)]) + loop = assemble_gif(paths, frame_ms=120, last_frame_ms=1500) + result = self._open(loop) + self.assertEqual(paths, list(loop.paths)) + durations = [] + for index in range(result.n_frames): + result.seek(index) + durations.append(result.info.get("duration")) + self.assertEqual(0, result.info.get("loop")) # 0 == forever + self.assertEqual(1500, durations[-1]) + self.assertEqual([120] * (len(durations) - 1), durations[:-1]) + + def test_identical_frames_merge_without_losing_playback_time(self): + """Pillow collapses pixel-identical neighbours and sums their delays. + + A clear overnight radar is genuinely a still image, so this is free + compression rather than a fault -- but the loop must still run for as + long as it was asked to, and still end on the newest frame. + """ + paths = self._frames([(0, 0, 0)] * 6 + [(253, 0, 0)], empty=range(6)) + result = self._open(assemble_gif( + paths, frame_ms=100, last_frame_ms=900 + )) + total = 0 + for index in range(result.n_frames): + result.seek(index) + total += result.info.get("duration") + self.assertLess(result.n_frames, len(paths)) + self.assertEqual(100 * (len(paths) - 1) + 900, total) + + def test_too_few_frames_produce_nothing(self): + self.assertIsNone(assemble_gif([])) + self.assertIsNone(assemble_gif(self._frames([(253, 0, 0)]))) + + def test_a_corrupt_frame_is_skipped_and_the_rest_still_assemble(self): + paths = self._frames([(253, 0, 0), (2, 253, 2), (3, 0, 244)]) + with open(paths[1], "wb") as handle: + handle.write(b"not a gif at all") + with open(os.path.join(self._tmp.name, "empty.gif"), "wb"): + pass + paths.append(os.path.join(self._tmp.name, "empty.gif")) + result = self._open(assemble_gif(paths)) + self.assertGreaterEqual(result.n_frames, 2) + + def test_an_oversized_loop_is_rebuilt_with_fewer_frames(self): + paths = self._frames([c for c in SCALE], size=(120, 90)) + full = assemble_gif(paths) + halved = assemble_gif(paths, max_bytes=len(full.data) - 1) + self.assertIsNotNone(halved) + self.assertLess(len(halved.data), len(full.data)) + # Thinning must count back from the newest frame, not forward from the + # oldest: with an even frame count, every-other-from-the-front drops + # the current scan and the hold lands on a stale one. + self.assertIn(paths[-1], halved.paths) + + def test_a_loop_that_cannot_be_shrunk_enough_is_abandoned(self): + paths = self._frames([c for c in SCALE], size=(120, 90)) + self.assertIsNone(assemble_gif(paths, max_bytes=1)) + + def test_frames_of_differing_sizes_are_still_combined(self): + """NOAA has changed product dimensions before; old frames still count. + + The newest frame sets the size. Taking it from the oldest would mean + that after a product change every new full-resolution scan was scaled + back down to the superseded dimensions for a whole window. + """ + paths = self._frames([(253, 0, 0), (2, 253, 2)]) + odd = os.path.join(self._tmp.name, "odd.gif") + _write_frame(odd, (3, 0, 244), size=(80, 60)) + paths.append(odd) + result = self._open(assemble_gif(paths)) + self.assertEqual(3, result.n_frames) + self.assertEqual((80, 60), result.size) + + +class TestAssembleWithoutPillow(unittest.TestCase): + """Pillow ships with Home Assistant core, but must not be assumed.""" + + def test_a_missing_pillow_degrades_instead_of_raising(self): + original = radar_loop.PIL_AVAILABLE + radar_loop.PIL_AVAILABLE = False + self.addCleanup(setattr, radar_loop, "PIL_AVAILABLE", original) + self.assertIsNone(assemble_gif(["/frames/a.gif", "/frames/b.gif"])) + + +class TestFrameName(unittest.TestCase): + """Names sort chronologically because they are compared as strings.""" + + def test_names_sort_in_time_order(self): + times = [ + NOW - timedelta(minutes=90), + NOW - timedelta(minutes=5), + NOW - timedelta(hours=23), + ] + names = [frame_name(t) for t in times] + self.assertEqual( + [frame_name(t) for t in sorted(times)], sorted(names) + ) + + def test_a_naive_timestamp_is_treated_as_utc(self): + self.assertEqual( + frame_name(NOW), frame_name(NOW.replace(tzinfo=None)) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_translations.py b/tests/test_translations.py index d94c570..013b749 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -85,6 +85,21 @@ def test_options_step_init_present(self): self.assertIn("step", self.en["options"]) self.assertIn("init", self.en["options"]["step"]) + def test_options_step_radar_present(self): + self.assertIn("radar", self.en["options"]["step"]) + radar = self.en["options"]["step"]["radar"] + self.assertIn("title", radar) + self.assertIn("description", radar) + self.assertIn("radar_loop_hours", radar["data"]) + + def test_the_radar_description_fills_its_placeholder(self): + """The step passes max_hours, so the text has to ask for it.""" + description = self.en["options"]["step"]["radar"]["description"] + self.assertIn("{max_hours}", description) + + def test_options_error_has_radar_hours(self): + self.assertIn("invalid_radar_hours", self.en["options"]["error"]) + if __name__ == "__main__": unittest.main()