Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions src/skillspector/providers/nv_build/model_registry.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,27 @@

models:
# NVIDIA-curated NIMs on build.nvidia.com.
# 202749 e' il tetto COMBINATO ingresso+uscita che l'endpoint impone, ed e' riportato
# alla lettera nel 400 che restituisce quando lo si supera. Dichiarando la finestra
# nominale di 1000000, model_info calcolava un budget d'uscita di 250000 token
# (ctx * (1 - MAX_INPUT_TOKENS_PCT)) e OGNI chiamata falliva:
# "This model configuration accepts at most 202749 combined input and output tokens.
# However, your request has 1249 input tokens and asks for 250000 output tokens"
# Una finestra sovrastimata non degrada con grazia: azzera lo stadio LLM. Sottostimare
# e' sicuro, sovrastimare no.
"z-ai/glm-5.2":
context_length: 1000000

"z-ai/glm-5.1":
context_length: 205000
context_length: 202749
max_output_tokens: 32768

"moonshotai/kimi-k2.6":
context_length: 256000

"deepseek-ai/deepseek-v4-pro":
context_length: 1000000
max_output_tokens: 128000

"deepseek-ai/deepseek-v4-flash":
context_length: 1000000
max_output_tokens: 128000
# deepseek-v4-flash e' a fine vita dal 2026-08-07 e risponde 410 Gone; il successore
# servito e' deepseek-v4-flash-0731. I limiti qui sotto sono prudenti: non sono stati
# sondati, e una finestra sottostimata e' il verso sicuro in cui sbagliare.
"deepseek-ai/deepseek-v4-flash-0731":
context_length: 128000
max_output_tokens: 16384

"openai/gpt-oss-120b":
context_length: 128000
Expand Down
20 changes: 13 additions & 7 deletions src/skillspector/providers/nv_build/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,19 @@
class NvBuildProvider:
"""build.nvidia.com credentials + bundled-YAML metadata provider."""

# General default — DeepSeek v4 flash for the high-volume per-file
# analyzer calls. meta_analyzer is upgraded to v4 pro for the
# aggregation/filter pass where precision matters more.
DEFAULT_MODEL = "deepseek-ai/deepseek-v4-flash"
SLOT_DEFAULTS: dict[str, str] = {
"meta_analyzer": "deepseek-ai/deepseek-v4-pro",
}
# General default. The previous defaults (deepseek-v4-flash and, for the
# meta_analyzer slot, deepseek-v4-pro) are no longer served: the former
# returns 410 Gone (end of life 2026-08-07) and neither appears in
# GET /v1/models, so the out-of-the-box path failed every call.
#
# The replacement is chosen for DETECTION, not latency. On a bait skill
# carrying prose-disguised credential exfiltration, the fast served model
# (deepseek-v4-flash-0731, ~1.6 s/call) completed every call, reported no
# degradation, and returned a clean verdict — a confident false negative,
# which on a security scanner is the worst possible failure. glm-5.2 costs
# ~16 s/call and flags it CRITICAL.
DEFAULT_MODEL = "z-ai/glm-5.2"
SLOT_DEFAULTS: dict[str, str] = {}

def resolve_credentials(self) -> tuple[str, str | None] | None:
"""Return ``(api_key, base_url)`` from ``NVIDIA_INFERENCE_KEY``."""
Expand Down
41 changes: 28 additions & 13 deletions tests/unit/test_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,15 +135,24 @@ class TestNvBuildProvider:
@pytest.mark.parametrize(
("model", "context_length"),
[
("z-ai/glm-5.2", 1_000_000),
("z-ai/glm-5.1", 205_000),
("z-ai/glm-5.2", 202_749),
("moonshotai/kimi-k2.6", 256_000),
],
)
def test_nv_build_reported_model_metadata(self, model: str, context_length: int) -> None:
provider = NvBuildProvider()
assert provider.get_context_length(model) == context_length
assert provider.get_max_output_tokens(model) is None

def test_glm_declares_both_limits(self) -> None:
"""max_output_tokens is optional, and its absence is not neutral.

Without it the output budget is derived as a percentage of the context
window, which is what produced a 250_000-token request against an
endpoint accepting 202_749 combined.
"""
provider = NvBuildProvider()
assert provider.get_context_length("z-ai/glm-5.2") == 202_749
assert provider.get_max_output_tokens("z-ai/glm-5.2") == 32_768

@pytest.mark.parametrize("model", ["glm-5.2", "z-ai/glm-5.2 "])
def test_nv_build_model_near_match_stays_unresolved(self, model: str) -> None:
Expand All @@ -170,11 +179,20 @@ def test_creates_openai_compatible_chat_model(self, monkeypatch: pytest.MonkeyPa
assert llm.max_tokens == 123
assert str(llm.openai_api_base).rstrip("/") == BUILD_BASE_URL.rstrip("/")

def test_metadata_known_model_from_bundled_yaml(self) -> None:
"""deepseek-v4-flash ships in nv_build/model_registry.yaml."""
def test_metadata_drops_end_of_life_model(self) -> None:
"""deepseek-v4-flash reached end of life and returns 410 Gone.

Keeping it is worse than omitting it: an entry with a 1_000_000 window
makes model_info budget 250_000 output tokens, rejected on every call.
Absent, the conservative default applies instead.
"""
provider = NvBuildProvider()
assert provider.get_context_length("deepseek-ai/deepseek-v4-flash") == 1_000_000
assert provider.get_max_output_tokens("deepseek-ai/deepseek-v4-flash") == 128_000
assert provider.get_context_length("deepseek-ai/deepseek-v4-flash") is None

def test_default_model_is_in_the_bundled_registry(self) -> None:
"""The invariant test_constants asserts, checked at the source too."""
provider = NvBuildProvider()
assert provider.get_context_length(NvBuildProvider.DEFAULT_MODEL) is not None

def test_metadata_unknown_model_returns_none(self) -> None:
provider = NvBuildProvider()
Expand All @@ -190,12 +208,9 @@ def test_resolve_model_env_overrides_default(self, monkeypatch: pytest.MonkeyPat
# Env override applies to every slot.
assert NvBuildProvider().resolve_model("meta_analyzer") == "user/override"

def test_resolve_model_meta_analyzer_uses_slot_override(self) -> None:
# meta_analyzer is upgraded to deepseek-v4-pro on NvBuild.
assert (
NvBuildProvider().resolve_model("meta_analyzer")
== NvBuildProvider.SLOT_DEFAULTS["meta_analyzer"]
)
def test_resolve_model_meta_analyzer_falls_back_to_default(self) -> None:
# The former override named deepseek-v4-pro, absent from the catalogue.
assert NvBuildProvider().resolve_model("meta_analyzer") == NvBuildProvider.DEFAULT_MODEL

def test_resolve_model_unknown_slot_falls_to_default(self) -> None:
# Slots without an explicit override inherit DEFAULT_MODEL.
Expand Down
Loading