From 4ec490c52a7d5b8a288dea087b5f97ba96114f69 Mon Sep 17 00:00:00 2001 From: Mark2Mac Date: Tue, 18 Aug 2026 11:10:02 +0200 Subject: [PATCH 1/2] fix(nv_build): declare glm-5.2's real limits so calls stop failing The bundled registry gave z-ai/glm-5.2 a 1000000-token context window and no output cap. model_info derives the output budget as ctx * (1 - MAX_INPUT_TOKENS_PCT), so every request asked for 250000 output tokens and the endpoint answered: 400 This model configuration accepts at most 202749 combined input and output tokens. However, your request has 1249 input tokens and asks for 250000 output tokens (251249 tokens total). 202749 is quoted verbatim by the endpoint in that 400. With the entry corrected the same scan completes with 4/4 LLM calls and the meta-analyzer applied. An over-stated context window does not degrade gracefully: it zeroes the LLM stage, and nothing in the error points at the registry. Under-stating is safe, over-stating is not. Limits may vary per account, which is now noted in the YAML. Scope is deliberately one entry. The registry also names three models the catalogue no longer serves, but removing them is coupled to DEFAULT_MODEL by an invariant the suite already asserts ("nv_build's default model is in its registry"), so that change travels with the default in a separate PR. Refs #388 Signed-off-by: Mark2Mac --- .../providers/nv_build/model_registry.yaml | 11 ++++++++++- tests/unit/test_providers.py | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/skillspector/providers/nv_build/model_registry.yaml b/src/skillspector/providers/nv_build/model_registry.yaml index 226fddd91..5cfd6757b 100644 --- a/src/skillspector/providers/nv_build/model_registry.yaml +++ b/src/skillspector/providers/nv_build/model_registry.yaml @@ -15,8 +15,17 @@ models: # NVIDIA-curated NIMs on build.nvidia.com. + # 202749 e' il tetto COMBINATO ingresso+uscita che l'endpoint impone, ed e' riportato + # alla lettera nel 400 che restituisce quando lo si supera. Dichiarando la finestra + # nominale di 1000000, model_info calcolava un budget d'uscita di 250000 token + # (ctx * (1 - MAX_INPUT_TOKENS_PCT)) e OGNI chiamata falliva: + # "This model configuration accepts at most 202749 combined input and output tokens. + # However, your request has 1249 input tokens and asks for 250000 output tokens" + # Una finestra sovrastimata non degrada con grazia: azzera lo stadio LLM. Sottostimare + # e' sicuro, sovrastimare no. "z-ai/glm-5.2": - context_length: 1000000 + context_length: 202749 + max_output_tokens: 32768 "z-ai/glm-5.1": context_length: 205000 diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 0db796ada..ecf501759 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -135,7 +135,7 @@ class TestNvBuildProvider: @pytest.mark.parametrize( ("model", "context_length"), [ - ("z-ai/glm-5.2", 1_000_000), + ("z-ai/glm-5.2", 202_749), ("z-ai/glm-5.1", 205_000), ("moonshotai/kimi-k2.6", 256_000), ], @@ -143,7 +143,17 @@ class TestNvBuildProvider: def test_nv_build_reported_model_metadata(self, model: str, context_length: int) -> None: provider = NvBuildProvider() assert provider.get_context_length(model) == context_length - assert provider.get_max_output_tokens(model) is None + + def test_glm_declares_both_limits(self) -> None: + """max_output_tokens is optional, and its absence is not neutral. + + Without it the output budget is derived as a percentage of the context + window, which is what produced a 250_000-token request against an + endpoint accepting 202_749 combined. + """ + provider = NvBuildProvider() + assert provider.get_context_length("z-ai/glm-5.2") == 202_749 + assert provider.get_max_output_tokens("z-ai/glm-5.2") == 32_768 @pytest.mark.parametrize("model", ["glm-5.2", "z-ai/glm-5.2 "]) def test_nv_build_model_near_match_stays_unresolved(self, model: str) -> None: From 315b0c2712e18b66e9a61da8677e5afd07a5750f Mon Sep 17 00:00:00 2001 From: Mark2Mac Date: Tue, 18 Aug 2026 11:21:49 +0200 Subject: [PATCH 2/2] fix(nv_build): retire the dead models and point the default at a served one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depends on #390. Three registry entries name models GET /v1/models does not serve: deepseek-v4-flash (410 Gone since 2026-08-07), deepseek-v4-pro, and glm-5.1. One of them is DEFAULT_MODEL, and another is the meta_analyzer slot override, so with no SKILLSPECTOR_MODEL set the out-of-the-box path failed every call. Removing them and retargeting the default is ONE change, not two. The suite already asserts the invariant that couples them, in test_constants: "nv_build's default model is in its registry — no warnings expected" Dropping the default from the registry while leaving it as the default breaks that test, and it is right to break: a default the registry does not describe gets its token budget from a guess, silently. Splitting these two edits was tried and abandoned for exactly this reason. The replacement is chosen for DETECTION, not latency, and that is the part worth arguing about. On a bait skill carrying credential exfiltration disguised as a synchronisation step, the fast served model (deepseek-v4-flash-0731, ~1.6 s/call) completed every call, reported no degradation, and returned a clean verdict. A confident false negative is the worst failure mode a security scanner has: it is the case where someone signs off. z-ai/glm-5.2 costs ~16 s per call on the same skill and returns CRITICAL. The meta_analyzer slot loses its override rather than gaining a new one. The aggregation pass does benefit from a stronger model, but naming a second model doubles the surface that can go stale — which is how the previous default rotted unnoticed. Happy to restore an override if preferred. Refs #388 Signed-off-by: Mark2Mac --- .../providers/nv_build/model_registry.yaml | 16 +++++------ .../providers/nv_build/provider.py | 20 +++++++++----- tests/unit/test_providers.py | 27 +++++++++++-------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/skillspector/providers/nv_build/model_registry.yaml b/src/skillspector/providers/nv_build/model_registry.yaml index 5cfd6757b..188a32981 100644 --- a/src/skillspector/providers/nv_build/model_registry.yaml +++ b/src/skillspector/providers/nv_build/model_registry.yaml @@ -27,19 +27,15 @@ models: context_length: 202749 max_output_tokens: 32768 - "z-ai/glm-5.1": - context_length: 205000 - "moonshotai/kimi-k2.6": context_length: 256000 - "deepseek-ai/deepseek-v4-pro": - context_length: 1000000 - max_output_tokens: 128000 - - "deepseek-ai/deepseek-v4-flash": - context_length: 1000000 - max_output_tokens: 128000 + # deepseek-v4-flash e' a fine vita dal 2026-08-07 e risponde 410 Gone; il successore + # servito e' deepseek-v4-flash-0731. I limiti qui sotto sono prudenti: non sono stati + # sondati, e una finestra sottostimata e' il verso sicuro in cui sbagliare. + "deepseek-ai/deepseek-v4-flash-0731": + context_length: 128000 + max_output_tokens: 16384 "openai/gpt-oss-120b": context_length: 128000 diff --git a/src/skillspector/providers/nv_build/provider.py b/src/skillspector/providers/nv_build/provider.py index f2a47cce7..1eccd020c 100644 --- a/src/skillspector/providers/nv_build/provider.py +++ b/src/skillspector/providers/nv_build/provider.py @@ -38,13 +38,19 @@ class NvBuildProvider: """build.nvidia.com credentials + bundled-YAML metadata provider.""" - # General default — DeepSeek v4 flash for the high-volume per-file - # analyzer calls. meta_analyzer is upgraded to v4 pro for the - # aggregation/filter pass where precision matters more. - DEFAULT_MODEL = "deepseek-ai/deepseek-v4-flash" - SLOT_DEFAULTS: dict[str, str] = { - "meta_analyzer": "deepseek-ai/deepseek-v4-pro", - } + # General default. The previous defaults (deepseek-v4-flash and, for the + # meta_analyzer slot, deepseek-v4-pro) are no longer served: the former + # returns 410 Gone (end of life 2026-08-07) and neither appears in + # GET /v1/models, so the out-of-the-box path failed every call. + # + # The replacement is chosen for DETECTION, not latency. On a bait skill + # carrying prose-disguised credential exfiltration, the fast served model + # (deepseek-v4-flash-0731, ~1.6 s/call) completed every call, reported no + # degradation, and returned a clean verdict — a confident false negative, + # which on a security scanner is the worst possible failure. glm-5.2 costs + # ~16 s/call and flags it CRITICAL. + DEFAULT_MODEL = "z-ai/glm-5.2" + SLOT_DEFAULTS: dict[str, str] = {} def resolve_credentials(self) -> tuple[str, str | None] | None: """Return ``(api_key, base_url)`` from ``NVIDIA_INFERENCE_KEY``.""" diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index ecf501759..2ab76afda 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -136,7 +136,6 @@ class TestNvBuildProvider: ("model", "context_length"), [ ("z-ai/glm-5.2", 202_749), - ("z-ai/glm-5.1", 205_000), ("moonshotai/kimi-k2.6", 256_000), ], ) @@ -180,11 +179,20 @@ def test_creates_openai_compatible_chat_model(self, monkeypatch: pytest.MonkeyPa assert llm.max_tokens == 123 assert str(llm.openai_api_base).rstrip("/") == BUILD_BASE_URL.rstrip("/") - def test_metadata_known_model_from_bundled_yaml(self) -> None: - """deepseek-v4-flash ships in nv_build/model_registry.yaml.""" + def test_metadata_drops_end_of_life_model(self) -> None: + """deepseek-v4-flash reached end of life and returns 410 Gone. + + Keeping it is worse than omitting it: an entry with a 1_000_000 window + makes model_info budget 250_000 output tokens, rejected on every call. + Absent, the conservative default applies instead. + """ + provider = NvBuildProvider() + assert provider.get_context_length("deepseek-ai/deepseek-v4-flash") is None + + def test_default_model_is_in_the_bundled_registry(self) -> None: + """The invariant test_constants asserts, checked at the source too.""" provider = NvBuildProvider() - assert provider.get_context_length("deepseek-ai/deepseek-v4-flash") == 1_000_000 - assert provider.get_max_output_tokens("deepseek-ai/deepseek-v4-flash") == 128_000 + assert provider.get_context_length(NvBuildProvider.DEFAULT_MODEL) is not None def test_metadata_unknown_model_returns_none(self) -> None: provider = NvBuildProvider() @@ -200,12 +208,9 @@ def test_resolve_model_env_overrides_default(self, monkeypatch: pytest.MonkeyPat # Env override applies to every slot. assert NvBuildProvider().resolve_model("meta_analyzer") == "user/override" - def test_resolve_model_meta_analyzer_uses_slot_override(self) -> None: - # meta_analyzer is upgraded to deepseek-v4-pro on NvBuild. - assert ( - NvBuildProvider().resolve_model("meta_analyzer") - == NvBuildProvider.SLOT_DEFAULTS["meta_analyzer"] - ) + def test_resolve_model_meta_analyzer_falls_back_to_default(self) -> None: + # The former override named deepseek-v4-pro, absent from the catalogue. + assert NvBuildProvider().resolve_model("meta_analyzer") == NvBuildProvider.DEFAULT_MODEL def test_resolve_model_unknown_slot_falls_to_default(self) -> None: # Slots without an explicit override inherit DEFAULT_MODEL.