From f11cd108e88469ab004dc40d41b7bbc67c3e0d6e Mon Sep 17 00:00:00 2001 From: manusjs Date: Wed, 8 Jul 2026 16:06:21 +0000 Subject: [PATCH] feat(tools): Hex.pm enrichment in get_dependency_blast_radius --- .../tools/get_dependency_blast_radius.py | 119 ++++++++- tests/test_dependency_blast_radius.py | 239 +++++++++++++++++- 2 files changed, 356 insertions(+), 2 deletions(-) diff --git a/src/manus_agent/tools/get_dependency_blast_radius.py b/src/manus_agent/tools/get_dependency_blast_radius.py index 876527a..36a96be 100644 --- a/src/manus_agent/tools/get_dependency_blast_radius.py +++ b/src/manus_agent/tools/get_dependency_blast_radius.py @@ -14,6 +14,7 @@ 5. PyPI JSON API — package metadata (PyPI only; download counts from pypistats.org with graceful degradation on 429) 6. Maven Central — artifact metadata + version count (Maven only) + 7. Hex.pm API — package metadata + download stats (Elixir/Erlang) The tool deliberately avoids paid/rate-limited APIs so it works without configuration. When a source is unavailable the result degrades gracefully. @@ -51,6 +52,7 @@ _PYPI_JSON_URL = "https://pypi.org/pypi/{}/json" _PYPISTATS_URL = "https://pypistats.org/api/packages/{}/recent" _MAVEN_SEARCH_URL = "https://search.maven.org/solrsearch/select" +_HEX_API_URL = "https://hex.pm/api/packages/{}" _TIMEOUT = 20 @@ -64,7 +66,7 @@ "RubyGems": "RubyGems (Ruby)", "NuGet": "NuGet (.NET)", "Packagist": "Packagist (PHP)", - "Hex": "Hex (Elixir/Erlang)", + "Hex": "Hex (Elixir/Erlang)", # noqa: E241 (alignment) "Pub": "Pub (Dart/Flutter)", } @@ -373,6 +375,98 @@ def _enrich_maven(name: str) -> dict[str, Any]: return result +def _parse_hex_timestamp(ts: str | None) -> tuple[str | None, float | None]: + """Parse a Hex inserted_at ISO-8601 timestamp. + + Hex always returns timestamps in the form ``YYYY-MM-DDTHH:MM:SS.ffffffZ`` + (UTC, variable sub-second precision). Returns ``(iso_date, age_years)`` + where *iso_date* is the ``YYYY-MM-DD`` string and *age_years* is a float + rounded to two decimal places. Both are ``None`` when parsing fails. + """ + if not ts: + return None, None + try: + from datetime import datetime, timezone + + # Normalise: strip trailing Z, truncate sub-seconds to 6 digits + ts_clean = ts.rstrip("Z") + if "+" in ts_clean: + ts_clean = ts_clean.split("+")[0] + if "." in ts_clean: + date_part, frac = ts_clean.split(".", 1) + ts_clean = date_part + "." + frac[:6] + dt = datetime.fromisoformat(ts_clean).replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + age = round((now - dt).days / 365.25, 2) + return dt.date().isoformat(), age + except Exception: + return None, None + + +def _enrich_hex(name: str) -> dict[str, Any]: + """Fetch Hex.pm package metadata for Elixir/Erlang packages. + + Single API call to ``https://hex.pm/api/packages/{name}`` returns + all the data we need: + + * ``downloads.week`` — weekly download count (blast-score denominator) + * ``downloads.all`` — total all-time downloads + * ``downloads.recent`` — last-90-day downloads + * ``latest_stable_version`` — current stable release + * ``releases`` — full list with ``inserted_at`` timestamps + * ``inserted_at`` — package first-published date + * ``meta.description`` — one-line summary + * ``html_url`` — Hex.pm package page URL + + No authentication required. Rate limit is generous for single lookups. + """ + result: dict[str, Any] = {"ecosystem": "Hex", "package_name": name} + try: + data = _get(_HEX_API_URL.format(name)) + + # Download stats + downloads = data.get("downloads") or {} + weekly = downloads.get("week") + if weekly is not None: + result["weekly_downloads"] = int(weekly) + total_dl = downloads.get("all") + if total_dl is not None: + result["total_downloads"] = int(total_dl) + recent_dl = downloads.get("recent") + if recent_dl is not None: + result["recent_downloads"] = int(recent_dl) # last 90 days + + # Version metadata + latest_stable = data.get("latest_stable_version") or data.get("latest_version") + if latest_stable: + result["latest_version"] = latest_stable + + releases = data.get("releases") or [] + result["total_versions"] = len(releases) + + # First release date — last item in list (oldest) + if releases: + first_ts = releases[-1].get("inserted_at") + iso_date, age = _parse_hex_timestamp(first_ts) + if iso_date: + result["first_release_date"] = iso_date + if age is not None: + result["age_years"] = age + + # Package-level description and URL + meta = data.get("meta") or {} + description = meta.get("description") or "" + if description: + result["description"] = description[:120].replace("\n", " ").strip() + html_url = data.get("html_url") or "" + if html_url: + result["home_page"] = html_url + + except Exception as exc: + logger.debug("Hex enrich failed for %s: %s", name, exc) + return result + + def _enrich_package(name: str, ecosystem: str) -> dict[str, Any]: """Dispatch to the right enrichment function based on ecosystem.""" eco_lower = (ecosystem or "").lower() @@ -382,6 +476,8 @@ def _enrich_package(name: str, ecosystem: str) -> dict[str, Any]: return _enrich_pypi(name) elif eco_lower in ("maven", "java", "gradle"): return _enrich_maven(name) + elif eco_lower in ("hex", "elixir", "erlang"): + return _enrich_hex(name) # Unknown ecosystem — return minimal record return {"ecosystem": ecosystem, "package_name": name} @@ -433,6 +529,7 @@ def get_dependency_blast_radius( # noqa: C901 - **npm**: dependent package count + weekly/monthly download stats - **PyPI**: package metadata + download stats (when pypistats is available) - **Maven**: artifact metadata from Maven Central + - **Hex**: weekly/total downloads + version history (Elixir/Erlang) 3. Computes a qualitative blast-radius label: CRITICAL / HIGH / MEDIUM / LOW. Use after ``get_nvd_data`` to understand *how many projects are exposed* @@ -442,6 +539,8 @@ def get_dependency_blast_radius( # noqa: C901 Args: package_or_cve: Package spec (``name@version``, ``ecosystem:name@version``) or CVE ID (``CVE-YYYY-NNNN``). + Supported ecosystem prefixes: ``pypi:``, ``npm:``, + ``maven:``, ``hex:``, ``elixir:``. max_packages: Maximum number of packages to enrich with stats (default 10). Returns: @@ -559,6 +658,24 @@ def get_dependency_blast_radius( # noqa: C901 if r.get("version_count"): lines.append(f" Version count: {r['version_count']}") + # Hex-specific stats + if eco.lower() in ("hex", "elixir", "erlang"): + if r.get("latest_version"): + lines.append(f" Latest version: {r['latest_version']}") + if r.get("total_versions") is not None: + lines.append(f" Total versions: {r['total_versions']}") + if r.get("total_downloads") is not None: + lines.append(f" Total downloads: {r['total_downloads']:,}") + if r.get("recent_downloads") is not None: + lines.append(f" 90-day downloads: {r['recent_downloads']:,}") + if r.get("first_release_date"): + age_str = f" ({r['age_years']} yrs)" if r.get("age_years") is not None else "" + lines.append(f" First released: {r['first_release_date']}{age_str}") + if r.get("description"): + lines.append(f" Description: {r['description'][:80]}") + if r.get("home_page"): + lines.append(f" Hex page: {r['home_page']}") + if source: lines.append(f" Data sources: {source}") diff --git a/tests/test_dependency_blast_radius.py b/tests/test_dependency_blast_radius.py index 4e8d4b5..b51aa91 100644 --- a/tests/test_dependency_blast_radius.py +++ b/tests/test_dependency_blast_radius.py @@ -2,7 +2,7 @@ Tests for src/manus_agent/tools/get_dependency_blast_radius.py All external HTTP calls are mocked — no real network I/O. -100% mocked: NVD, OSV, GHSA, npm, PyPI, pypistats, Maven Central. +100% mocked: NVD, OSV, GHSA, npm, PyPI, pypistats, Maven Central, Hex.pm. """ from __future__ import annotations @@ -14,6 +14,7 @@ from manus_agent.tools.get_dependency_blast_radius import ( _blast_score, + _enrich_hex, _enrich_maven, _enrich_npm, _enrich_package, @@ -21,6 +22,7 @@ _fetch_ghsa_affected, _fetch_nvd_affected, _fetch_osv_affected, + _parse_hex_timestamp, _parse_input, _summarise_osv_ranges, get_dependency_blast_radius, @@ -636,6 +638,223 @@ def test_no_docs_returns_minimal_record(self): assert result["total_artifacts_found"] == 0 +# =========================================================================== +# _parse_hex_timestamp +# =========================================================================== + + +class TestParseHexTimestamp: + def test_utc_z_suffix(self): + # Typical Hex.pm timestamp with microseconds and Z suffix + iso_date, age = _parse_hex_timestamp("2014-04-21T22:38:32.000000Z") + assert iso_date == "2014-04-21" + assert age is not None + assert age > 0 + + def test_no_microseconds(self): + iso_date, age = _parse_hex_timestamp("2020-01-15T10:00:00Z") + assert iso_date == "2020-01-15" + assert age is not None + + def test_high_precision_microseconds(self): + # Hex sometimes returns 6 fractional digits + iso_date, age = _parse_hex_timestamp("2022-06-30T12:34:56.123456Z") + assert iso_date == "2022-06-30" + + def test_with_offset(self): + # +00:00 offset instead of Z + iso_date, age = _parse_hex_timestamp("2019-03-10T08:00:00.000000+00:00") + assert iso_date == "2019-03-10" + assert age is not None + + def test_none_input(self): + iso_date, age = _parse_hex_timestamp(None) + assert iso_date is None + assert age is None + + def test_empty_string(self): + iso_date, age = _parse_hex_timestamp("") + assert iso_date is None + assert age is None + + def test_invalid_input(self): + iso_date, age = _parse_hex_timestamp("not-a-date") + assert iso_date is None + assert age is None + + +# =========================================================================== +# _enrich_hex +# =========================================================================== + + +class TestEnrichHex: + """Tests for _enrich_hex — all HTTP calls mocked.""" + + def _make_hex_response( + self, + name: str = "phoenix", + latest_stable: str = "1.8.9", + weekly: int = 288733, + total: int = 151660295, + recent: int = 3422170, + description: str = "A productive web framework.", + html_url: str = "https://hex.pm/packages/phoenix", + releases: list | None = None, + ) -> dict: + if releases is None: + releases = [ + {"version": "1.8.9", "inserted_at": "2026-07-07T12:23:21.396353Z", "has_docs": True}, + {"version": "1.0.0", "inserted_at": "2016-01-01T00:00:00.000000Z", "has_docs": True}, + {"version": "0.1.0", "inserted_at": "2014-04-21T22:38:32.000000Z", "has_docs": False}, + ] + return { + "name": name, + "latest_stable_version": latest_stable, + "downloads": {"all": total, "week": weekly, "recent": recent, "day": 55963}, + "meta": {"description": description, "licenses": ["MIT"]}, + "html_url": html_url, + "releases": releases, + "inserted_at": "2014-04-21T22:38:32.000000Z", + } + + def test_basic_fields_populated(self): + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response() + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert result["ecosystem"] == "Hex" + assert result["package_name"] == "phoenix" + assert result["latest_version"] == "1.8.9" + assert result["total_versions"] == 3 + + def test_download_stats(self): + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(weekly=288733, total=151660295, recent=3422170) + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert result["weekly_downloads"] == 288733 + assert result["total_downloads"] == 151660295 + assert result["recent_downloads"] == 3422170 + + def test_first_release_date_extracted(self): + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response() + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + # Oldest release is releases[-1] = 0.1.0 from 2014-04-21 + assert result["first_release_date"] == "2014-04-21" + assert result["age_years"] is not None + assert result["age_years"] > 0 + + def test_description_truncated_to_120_chars(self): + long_desc = "A" * 200 + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(description=long_desc) + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert len(result["description"]) == 120 + + def test_description_newlines_replaced(self): + desc = "Line one.\nLine two.\nLine three." + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(description=desc) + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert "\n" not in result["description"] + + def test_home_page_set(self): + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(html_url="https://hex.pm/packages/phoenix") + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert result["home_page"] == "https://hex.pm/packages/phoenix" + + def test_graceful_degradation_on_network_error(self): + with patch("requests.get", side_effect=Exception("Connection refused")): + result = _enrich_hex("phoenix") + assert result["ecosystem"] == "Hex" + assert result["package_name"] == "phoenix" + # No crash; no extra fields expected + assert "weekly_downloads" not in result + + def test_missing_downloads_key_does_not_crash(self): + data = self._make_hex_response() + del data["downloads"] + mock_resp = MagicMock() + mock_resp.json.return_value = data + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert result["ecosystem"] == "Hex" + assert "weekly_downloads" not in result + + def test_missing_releases_key(self): + data = self._make_hex_response() + del data["releases"] + mock_resp = MagicMock() + mock_resp.json.return_value = data + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert result["total_versions"] == 0 + assert "first_release_date" not in result + + def test_empty_releases_list(self): + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(releases=[]) + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + assert result["total_versions"] == 0 + assert "first_release_date" not in result + + def test_blast_score_critical_for_popular_package(self): + # phoenix has 288k weekly downloads → HIGH (not CRITICAL) + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(weekly=5_000_000) + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("ecto") + # _blast_score uses weekly_downloads + from manus_agent.tools.get_dependency_blast_radius import _blast_score + + assert _blast_score(result) == "CRITICAL" + + def test_blast_score_high_for_phoenix(self): + # phoenix has ~288k weekly downloads -> MEDIUM (HIGH threshold is 500k) + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(weekly=288733) + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + from manus_agent.tools.get_dependency_blast_radius import _blast_score + + assert _blast_score(result) == "MEDIUM" + + def test_latest_stable_version_preferred_over_latest_version(self): + data = self._make_hex_response(latest_stable="1.8.9") + data["latest_version"] = "2.0.0-rc.1" # pre-release overrides latest + mock_resp = MagicMock() + mock_resp.json.return_value = data + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("phoenix") + # latest_stable_version should win + assert result["latest_version"] == "1.8.9" + + def test_fallback_to_latest_version_when_stable_absent(self): + data = self._make_hex_response() + data["latest_stable_version"] = None + data["latest_version"] = "0.9.0-beta" + mock_resp = MagicMock() + mock_resp.json.return_value = data + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("some_pkg") + assert result["latest_version"] == "0.9.0-beta" + + def test_package_name_preserved(self): + mock_resp = MagicMock() + mock_resp.json.return_value = self._make_hex_response(name="ecto") + with patch("requests.get", return_value=mock_resp): + result = _enrich_hex("ecto") + assert result["package_name"] == "ecto" + + # =========================================================================== # _enrich_package dispatch # =========================================================================== @@ -672,6 +891,24 @@ def test_maven_ecosystem(self): _enrich_package("log4j-core", "Maven") mock_maven.assert_called_once_with("log4j-core") + def test_hex_ecosystem(self): + with patch("manus_agent.tools.get_dependency_blast_radius._enrich_hex") as mock_hex: + mock_hex.return_value = {"ecosystem": "Hex", "package_name": "phoenix"} + _enrich_package("phoenix", "Hex") + mock_hex.assert_called_once_with("phoenix") + + def test_elixir_ecosystem_routes_to_hex(self): + with patch("manus_agent.tools.get_dependency_blast_radius._enrich_hex") as mock_hex: + mock_hex.return_value = {"ecosystem": "Hex", "package_name": "ecto"} + _enrich_package("ecto", "elixir") + mock_hex.assert_called_once_with("ecto") + + def test_erlang_ecosystem_routes_to_hex(self): + with patch("manus_agent.tools.get_dependency_blast_radius._enrich_hex") as mock_hex: + mock_hex.return_value = {"ecosystem": "Hex", "package_name": "ranch"} + _enrich_package("ranch", "erlang") + mock_hex.assert_called_once_with("ranch") + def test_unknown_ecosystem_returns_minimal_record(self): result = _enrich_package("unknown-pkg", "SomeExoticEcosystem") assert result["ecosystem"] == "SomeExoticEcosystem"