diff --git a/nim-skills/msa-search-nim/SKILL.md b/nim-skills/msa-search-nim/SKILL.md
index 393f60c..efa9ef4 100644
--- a/nim-skills/msa-search-nim/SKILL.md
+++ b/nim-skills/msa-search-nim/SKILL.md
@@ -5,11 +5,14 @@ description: >
license: Apache-2.0 AND CC-BY-4.0
compatibility: "requests>=2.28"
allowed-tools: Bash, Read, Write, AskUserQuestion
+permissions:
+ - env # reads NGC_API_KEY/NVIDIA_API_KEY and local NIM setup variables
+ - network # hosted MSA requests and documented NGC/local NIM setup
---
# MSA-Search NIM
-Generate protein MSAs with GPU-accelerated MMSeqs2. Use this `SKILL.md` for
+Generate protein MSAs with GPU-accelerated MMSeqs2. Use this guide for
first-pass hosted/local usage; load supplemental files only when needed:
- `references/api.md`: exact endpoints, schemas, Docker flags, response fields.
@@ -280,6 +283,24 @@ Notes:
Use exact case-sensitive database names and response keys.
+For a hosted standard search, run the bundled client from this skill's directory.
+It submits the real request, validates both database results, and saves the raw
+JSON and A3M files. Choose a new output directory for each run:
+
+```bash
+python scripts/hosted_search.py \
+ --sequence SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR \
+ --output-dir msa-output
+```
+
+The client reads `NGC_API_KEY` or `NVIDIA_API_KEY` from the environment. It permits
+at most two requests, each with a 10-second connection timeout and a 300-second
+read timeout, with five seconds between attempts. If it exits nonzero, report the
+service failure and stop. Do not restart it repeatedly, extend timeouts beyond the
+task budget, or replace the missing response with synthetic alignments.
+
+The underlying request format, also usable with a running local NIM, is:
+
```python
import os
import requests
@@ -371,3 +392,9 @@ template, and sequence sanity checks, read `references/validation.md`.
- Paired MSA requires at least two sequences.
- Local URL 404 usually means an accidental `/v1/` prefix.
- First local run can take hours while databases populate `LOCAL_NIM_CACHE`.
+- Hosted HTTP 502/503/504 or repeated read timeouts indicate that the hosted
+ request did not complete. Check service availability after the bounded retry;
+ a longer client timeout cannot fix a server-generated HTTP 504.
+- Do not invent a polling URL for `health.api.nvidia.com`. The published standard
+ MSA example uses synchronous POST; a pending response needs a documented
+ service-specific completion mechanism before it can count as a result.
diff --git a/nim-skills/msa-search-nim/config/skillspector-baseline.yml b/nim-skills/msa-search-nim/config/skillspector-baseline.yml
index 0246c5d..c575fb8 100644
--- a/nim-skills/msa-search-nim/config/skillspector-baseline.yml
+++ b/nim-skills/msa-search-nim/config/skillspector-baseline.yml
@@ -6,6 +6,17 @@
version: 1
rules:
+ - id: "PE3"
+ path: "evals/evals.json"
+ message: ".env "
+ reason: >-
+ Reviewed JSON-only false positive (2026-09-16). The two matches are
+ expected-output/assertion text in deferred local-setup case 3. They
+ describe the same optional repo-root dotenv setup documented in this
+ skill; the JSON does not read files or execute that setup. This rule
+ matches only the literal .env finding in this eval file. Credential
+ access in executable scripts and references to other secret stores
+ remain subject to scanning.
- id: "PE3"
path: "*SKILL.md"
reason: >-
diff --git a/nim-skills/msa-search-nim/evals/evals.json b/nim-skills/msa-search-nim/evals/evals.json
index c04f9af..442b83c 100644
--- a/nim-skills/msa-search-nim/evals/evals.json
+++ b/nim-skills/msa-search-nim/evals/evals.json
@@ -7,41 +7,13 @@
"expected_output": "A successfully executed hosted MSA-Search request with Bearer auth, case-correct database names, and A3M output format, plus the actual returned alignment saved to a file and summarized from the response.",
"files": [],
"assertions": [
- {
- "id": "hosted-request-executed",
- "description": "Executes the hosted request instead of only writing code",
- "check": "Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path"
- },
- {
- "id": "hosted-endpoint-url",
- "description": "Uses the correct hosted MSA-Search endpoint URL",
- "check": "Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'"
- },
- {
- "id": "bearer-auth-header",
- "description": "Sets Authorization header with Bearer token from NGC_API_KEY",
- "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'"
- },
- {
- "id": "sequence-field",
- "description": "Request uses 'sequence' (singular) field with the provided sequence",
- "check": "Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'"
- },
- {
- "id": "databases-field",
- "description": "databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases",
- "check": "Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'"
- },
- {
- "id": "a3m-output-format",
- "description": "Requests A3M alignment output format",
- "check": "Script contains 'output_alignment_formats' and 'a3m'"
- },
- {
- "id": "saves-alignment-output",
- "description": "Saves the returned alignment to a file",
- "check": "Script writes alignment content from 'alignments' in the response to a file"
- }
+ "[hosted-request-executed] Executes the hosted request instead of only writing code: Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path",
+ "[hosted-endpoint-url] Uses the correct hosted MSA-Search endpoint URL: Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'",
+ "[bearer-auth-header] Sets Authorization header with Bearer token from NGC_API_KEY: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'",
+ "[sequence-field] Request uses 'sequence' (singular) field with the provided sequence: Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'",
+ "[databases-field] databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases: Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'",
+ "[a3m-output-format] Requests A3M alignment output format: Script contains 'output_alignment_formats' and 'a3m'",
+ "[saves-alignment-output] Saves the returned alignment to a file: Script writes alignment content from 'alignments' in the response to a file"
]
}
],
@@ -52,36 +24,12 @@
"expected_output": "A Python script that calls the hosted /paired/predict endpoint with a 'sequences' list containing both chains, extracts per-chain alignments from alignments_by_chain, and saves each chain's alignment to a separate file.",
"files": [],
"assertions": [
- {
- "id": "paired-endpoint-url",
- "description": "Uses the correct hosted paired MSA endpoint URL",
- "check": "Script contains 'msa-search/paired/predict'"
- },
- {
- "id": "sequences-plural-field",
- "description": "Uses 'sequences' (plural, list) field — not 'sequence' (singular)",
- "check": "Script payload contains 'sequences' as a list/array, not 'sequence'"
- },
- {
- "id": "both-chains-present",
- "description": "Both protein sequences are included in the request",
- "check": "Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'"
- },
- {
- "id": "bearer-auth-header",
- "description": "Sets Authorization header with Bearer token",
- "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'"
- },
- {
- "id": "parses-alignments-by-chain",
- "description": "Response parsed by 'alignments_by_chain' (not 'alignments')",
- "check": "Script references 'alignments_by_chain' from the response"
- },
- {
- "id": "saves-per-chain-alignments",
- "description": "Saves alignment for each chain to separate files",
- "check": "Script saves at least two alignment files, one per chain"
- }
+ "[paired-endpoint-url] Uses the correct hosted paired MSA endpoint URL: Script contains 'msa-search/paired/predict'",
+ "[sequences-plural-field] Uses 'sequences' (plural, list) field — not 'sequence' (singular): Script payload contains 'sequences' as a list/array, not 'sequence'",
+ "[both-chains-present] Both protein sequences are included in the request: Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'",
+ "[bearer-auth-header] Sets Authorization header with Bearer token: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'",
+ "[parses-alignments-by-chain] Response parsed by 'alignments_by_chain' (not 'alignments'): Script references 'alignments_by_chain' from the response",
+ "[saves-per-chain-alignments] Saves alignment for each chain to separate files: Script saves at least two alignment files, one per chain"
]
},
{
@@ -91,36 +39,12 @@
"expected_output": "Docker setup instructions using shell env first and optional repo-root .env overrides, requiring NGC_API_KEY or NVIDIA_API_KEY fallback plus LOCAL_NIM_CACHE, warning about the 1.4 TB database cache, health-checking the service, then sending a no-auth request to localhost:8000 without a /v1/ prefix.",
"files": [],
"assertions": [
- {
- "id": "docker-image-tag",
- "description": "References the correct MSA-Search container image with :2 tag",
- "check": "Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'"
- },
- {
- "id": "env-contract-and-cache",
- "description": "Local setup uses the repo env contract and LOCAL_NIM_CACHE",
- "check": "Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache"
- },
- {
- "id": "storage-warning",
- "description": "Mentions the large storage requirement for databases",
- "check": "Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases"
- },
- {
- "id": "nim-cache-mount",
- "description": "Mounts cache directory to /opt/nim/.cache",
- "check": "Output contains '/opt/nim/.cache' in the volume mount"
- },
- {
- "id": "health-check",
- "description": "Includes health check before submitting request",
- "check": "Output contains health check against localhost:8000/v1/health/ready"
- },
- {
- "id": "local-endpoint-no-v1",
- "description": "Local prediction request uses path without /v1/ prefix",
- "check": "Script contains 'localhost:8000/biology/colabfold/msa-search/predict'"
- }
+ "[docker-image-tag] References the correct MSA-Search container image with :2 tag: Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'",
+ "[env-contract-and-cache] Local setup uses the repo env contract and LOCAL_NIM_CACHE: Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache",
+ "[storage-warning] Mentions the large storage requirement for databases: Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases",
+ "[nim-cache-mount] Mounts cache directory to /opt/nim/.cache: Output contains '/opt/nim/.cache' in the volume mount",
+ "[health-check] Includes health check before submitting request: Output contains health check against localhost:8000/v1/health/ready",
+ "[local-endpoint-no-v1] Local prediction request uses path without /v1/ prefix: Script contains 'localhost:8000/biology/colabfold/msa-search/predict'"
]
},
{
@@ -130,36 +54,12 @@
"expected_output": "A Python script targeting the local /biology/colabfold/msa-search/structure-templates/predict endpoint because the hosted health.api template path returned HTTP 404 in validation, with structural_template_databases=['pdb70_220313'], the sequence, max_structures=20, max_msa_sequences=500 to match NIM_GLOBAL_MAX_MSA_DEPTH, no Authorization header for localhost inference, and parsing/saving both returned mmCIF template structures and search_hits M8 tables.",
"files": [],
"assertions": [
- {
- "id": "structure-templates-endpoint",
- "description": "Uses the structure-templates endpoint, not the standard predict endpoint",
- "check": "Script contains 'structure-templates/predict'"
- },
- {
- "id": "sequence-field",
- "description": "Request uses 'sequence' (singular) field",
- "check": "Script contains 'sequence' field with the provided sequence"
- },
- {
- "id": "max-structures-param",
- "description": "max_structures is set to 20 and max_msa_sequences matches the default GPU server depth",
- "check": "Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH"
- },
- {
- "id": "pdb70-canonical-database",
- "description": "Uses the canonical pdb70_220313 template database name",
- "check": "Script sets structural_template_databases to include 'pdb70_220313'"
- },
- {
- "id": "local-no-auth",
- "description": "Uses no Authorization header for local template inference",
- "check": "Script does not send Authorization/Bearer headers to localhost"
- },
- {
- "id": "parses-structures-and-search-hits",
- "description": "Parses template structures and search_hits M8 output",
- "check": "Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table"
- }
+ "[structure-templates-endpoint] Uses the structure-templates endpoint, not the standard predict endpoint: Script contains 'structure-templates/predict'",
+ "[sequence-field] Request uses 'sequence' (singular) field: Script contains 'sequence' field with the provided sequence",
+ "[max-structures-param] max_structures is set to 20 and max_msa_sequences matches the default GPU server depth: Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH",
+ "[pdb70-canonical-database] Uses the canonical pdb70_220313 template database name: Script sets structural_template_databases to include 'pdb70_220313'",
+ "[local-no-auth] Uses no Authorization header for local template inference: Script does not send Authorization/Bearer headers to localhost",
+ "[parses-structures-and-search-hits] Parses template structures and search_hits M8 output: Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table"
]
}
]
diff --git a/nim-skills/msa-search-nim/references/examples.md b/nim-skills/msa-search-nim/references/examples.md
index 4f5c176..18a2702 100644
--- a/nim-skills/msa-search-nim/references/examples.md
+++ b/nim-skills/msa-search-nim/references/examples.md
@@ -21,6 +21,9 @@ exact aria2c + `NIM_MODEL_NAME` commands.
## Hosted Standard MSA
+Run `scripts/hosted_search.py` from the skill root to save the response and A3M
+files. The request payload is:
+
```python
payload = {
"sequence": sequence,
diff --git a/nim-skills/msa-search-nim/references/validation.md b/nim-skills/msa-search-nim/references/validation.md
index 82ef75f..06d62b0 100644
--- a/nim-skills/msa-search-nim/references/validation.md
+++ b/nim-skills/msa-search-nim/references/validation.md
@@ -7,10 +7,23 @@ passing them downstream.
- `alignments` exists for standard search.
- `alignments_by_chain` exists for paired search.
-- Each returned alignment has `alignment` text and a `format`.
-- A3M/FASTA text starts with FASTA-style headers.
+- Each returned alignment has `alignment` text and a matching `format` (`a3m`
+ for the hosted client's requested output).
+- Each A3M/FASTA record has a nonempty FASTA-style header followed by sequence
+ data. Reject missing records, empty records, and invalid sequence characters.
+- A3M records have equal numbers of match columns: uppercase residues and `-`
+ count toward the width; lowercase insertions do not. Wrapped sequence lines,
+ blank lines, and `#` comments are allowed.
- Saved filenames include database and format so outputs do not overwrite each
other.
+- The hosted client finishes all writes in private staging, then exclusively
+ creates the output directory and moves the result files into it. An existing
+ output path, including an empty directory created by another run, is preserved.
+- On POSIX, the output directory uses owner-only permissions (`0700`), and result
+ files use `0600`, even with a permissive umask.
+- A failed write or move removes temporary files and any output directory created
+ by this run, so the same output path can be retried. Treat output as complete
+ only after the client exits successfully.
- Record database names and e-value used for the search.
## Template Checks
diff --git a/nim-skills/msa-search-nim/scripts/hosted_search.py b/nim-skills/msa-search-nim/scripts/hosted_search.py
new file mode 100644
index 0000000..d4c733c
--- /dev/null
+++ b/nim-skills/msa-search-nim/scripts/hosted_search.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Run hosted MSA search with bounded retries and save actual A3M results.
+
+Requires requests>=2.28 and NGC_API_KEY (or NVIDIA_API_KEY). The two attempts
+use 10-second connect and 300-second read timeouts. A failed hosted service
+must remain a failure; this client never substitutes a fabricated alignment.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import time
+
+import requests
+
+
+URL = "https://health.api.nvidia.com/v1/biology/colabfold/msa-search/predict"
+DATABASES = ("Uniref30_2302", "colabfold_envdb_202108")
+REQUEST_TIMEOUT = (10, 300)
+MAX_ATTEMPTS = 2
+RETRY_DELAY = 5
+RETRYABLE_STATUS = {429, 502, 503, 504}
+
+
+class SearchError(RuntimeError):
+ """The hosted service did not produce the requested alignments."""
+
+
+def search(sequence: str, databases: list[str], api_key: str) -> dict:
+ """Return a validated response, or fail after at most two requests.
+
+ Do not forward server error bodies or requests exceptions to stderr:
+ those can contain sensitive request information. No redirects are followed
+ with the Authorization header. Database names are also used as filenames,
+ so only the documented database names are accepted.
+ """
+ if not api_key:
+ raise SearchError("Set NGC_API_KEY or NVIDIA_API_KEY before running hosted search.")
+ if not 1 <= len(sequence) <= 4096 or any(c not in "ACDEFGHIKLMNPQRSTVWYX" for c in sequence):
+ raise SearchError("Sequence must contain 1–4096 uppercase amino-acid letters (including X).")
+ if not databases or any(db not in DATABASES for db in databases):
+ raise SearchError("Select one or both documented databases.")
+ payload = {
+ "sequence": sequence,
+ "databases": list(dict.fromkeys(databases)),
+ "e_value": 0.0001,
+ "output_alignment_formats": ["a3m"],
+ }
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
+ last_error = "No response received."
+ for attempt in range(MAX_ATTEMPTS):
+ response = None
+ try:
+ response = requests.post(
+ URL, headers=headers, json=payload,
+ timeout=REQUEST_TIMEOUT, allow_redirects=False,
+ )
+ except (requests.Timeout, requests.ConnectionError):
+ last_error = "Hosted MSA request timed out or could not connect."
+ except requests.RequestException:
+ raise SearchError("Hosted MSA request failed before a usable response arrived.") from None
+ else:
+ try:
+ if response.status_code == 200:
+ try:
+ result = response.json()
+ except ValueError:
+ raise SearchError("Hosted MSA returned invalid JSON.") from None
+ validate_alignments(result, payload["databases"])
+ return result
+ last_error = f"Hosted MSA returned HTTP {response.status_code}."
+ if response.status_code not in RETRYABLE_STATUS:
+ raise SearchError(last_error)
+ finally:
+ response.close()
+ if attempt + 1 < MAX_ATTEMPTS:
+ time.sleep(RETRY_DELAY)
+ raise SearchError(
+ f"{last_error} Stopped after {MAX_ATTEMPTS} attempts. "
+ "No alignment was produced. Check hosted-service availability before retrying."
+ )
+
+
+def validate_alignments(result: object, databases: list[str]) -> None:
+ """Require an actual, nonempty A3M result for every requested database."""
+ alignments = result.get("alignments") if isinstance(result, dict) else None
+ if not isinstance(alignments, dict):
+ raise SearchError("Hosted MSA response has no alignments object.")
+ for database in databases:
+ formats = alignments.get(database)
+ a3m = formats.get("a3m") if isinstance(formats, dict) else None
+ if not isinstance(a3m, dict) or a3m.get("format") != "a3m":
+ raise SearchError(f"Hosted MSA returned no A3M-formatted result for {database}.")
+ alignment = a3m.get("alignment")
+ if not isinstance(alignment, str):
+ raise SearchError(f"Hosted MSA returned no A3M alignment for {database}.")
+ validate_a3m(alignment, database)
+
+
+def validate_a3m(alignment: str, database: str) -> None:
+ """Require named, nonempty records with consistent A3M match columns.
+
+ Uppercase residues and '-' occupy match columns; lowercase insertions
+ do not. Wrapped sequences, blank lines and '#' comments are supported.
+ """
+ error = f"Hosted MSA returned a malformed A3M alignment for {database}."
+ lengths: list[int] = []
+ for line in alignment.splitlines():
+ if not line.strip() or line.startswith("#"):
+ continue
+ if line.startswith(">"):
+ if not line[1:].strip():
+ raise SearchError(error)
+ lengths.append(0)
+ else:
+ if not lengths or any(c not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" for c in line):
+ raise SearchError(error)
+ lengths[-1] += sum("A" <= c <= "Z" or c == "-" for c in line)
+ if not lengths or not lengths[0] or any(length != lengths[0] for length in lengths):
+ raise SearchError(error)
+
+
+def save_results(result: dict, databases: list[str], output_dir: Path) -> None:
+ """Save private results in a new directory, cleaning up failed publication."""
+ output_dir.parent.mkdir(parents=True, exist_ok=True)
+ # Finish all writes inside private staging on the destination filesystem.
+ with tempfile.TemporaryDirectory(prefix=f".{output_dir.name}-", dir=output_dir.parent) as staging:
+ staged_output = Path(staging)
+ (staged_output / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+ for database in dict.fromkeys(databases):
+ alignment = result["alignments"][database]["a3m"]["alignment"]
+ (staged_output / f"{database}.a3m").write_text(alignment, encoding="utf-8")
+ files = list(staged_output.iterdir())
+ for path in files:
+ path.chmod(0o600)
+ # mkdir exclusively reserves the path, including against empty
+ # directories and dangling symlinks created by a concurrent run.
+ output_dir.mkdir(mode=0o700)
+ try:
+ for path in files:
+ path.rename(output_dir / path.name)
+ except BaseException:
+ # Only remove a directory that this run successfully reserved.
+ shutil.rmtree(output_dir)
+ raise
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--sequence", required=True)
+ parser.add_argument("--databases", nargs="+", choices=DATABASES, default=list(DATABASES))
+ parser.add_argument("--output-dir", type=Path, required=True, help="A new directory for this request")
+ args = parser.parse_args()
+ if args.output_dir.exists() or args.output_dir.is_symlink():
+ parser.error("--output-dir must not already exist; use a new path for each request")
+ try:
+ result = search(
+ args.sequence, args.databases,
+ os.environ.get("NGC_API_KEY") or os.environ.get("NVIDIA_API_KEY", ""),
+ )
+ save_results(result, args.databases, args.output_dir)
+ for database in dict.fromkeys(args.databases):
+ alignment = result["alignments"][database]["a3m"]["alignment"]
+ path = args.output_dir / f"{database}.a3m"
+ records = sum(line.startswith(">") for line in alignment.splitlines())
+ print(f"{path}: {records} sequences")
+ except SearchError as exc:
+ print(f"ERROR: {exc}", file=sys.stderr)
+ return 1
+ except OSError:
+ print("ERROR: Could not save the hosted response to the output directory.", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/nim-skills/msa-search-nim/tests/test_hosted_search.py b/nim-skills/msa-search-nim/tests/test_hosted_search.py
new file mode 100644
index 0000000..062ba75
--- /dev/null
+++ b/nim-skills/msa-search-nim/tests/test_hosted_search.py
@@ -0,0 +1,314 @@
+"""Offline transport and artifact checks; no hosted-service calls are made."""
+
+from contextlib import redirect_stderr, redirect_stdout
+import importlib.util
+import io
+import json
+from pathlib import Path
+import stat
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+
+SPEC = importlib.util.spec_from_file_location(
+ "hosted_search", Path(__file__).resolve().parents[1] / "scripts" / "hosted_search.py"
+)
+client = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(client)
+
+
+def response(status=200, data=None):
+ result = Mock(status_code=status)
+ result.json.return_value = data
+ return result
+
+
+def alignments():
+ return {"alignments": {
+ db: {"a3m": {"alignment": ">query\nACDE\n>hit\nAC-E\n", "format": "a3m"}}
+ for db in client.DATABASES
+ }}
+
+
+class HostedSearchTests(unittest.TestCase):
+ def test_gateway_timeout_retries_once_then_stops(self):
+ failures = [response(504), response(504), response(data=alignments())]
+ with patch.object(client.requests, "post", side_effect=failures) as post, \
+ patch.object(client.time, "sleep"):
+ with self.assertRaisesRegex(client.SearchError, "HTTP 504.*Stopped after 2"):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 2)
+ for call in post.call_args_list:
+ self.assertEqual(call.kwargs["timeout"], (10, 300))
+ self.assertFalse(call.kwargs["allow_redirects"])
+
+ def test_transient_error_can_recover_without_changing_the_request(self):
+ expected = alignments()
+ with patch.object(client.requests, "post", side_effect=[response(503), response(data=expected)]) as post, \
+ patch.object(client.time, "sleep"):
+ actual = client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(actual, expected)
+ self.assertEqual(post.call_args_list[0], post.call_args_list[1])
+
+ def test_auth_error_and_redirect_are_not_retried(self):
+ for status in [401, 403, 302]:
+ with self.subTest(status=status), patch.object(client.requests, "post", return_value=response(status)) as post:
+ with self.assertRaisesRegex(client.SearchError, f"HTTP {status}"):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 1)
+
+ def test_timeout_does_not_expose_exception_text(self):
+ with patch.object(client.requests, "post", side_effect=client.requests.ReadTimeout("sensitive request details")) as post, \
+ patch.object(client.time, "sleep"):
+ with self.assertRaises(client.SearchError) as failure:
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertNotIn("sensitive", str(failure.exception))
+ self.assertEqual(post.call_count, 2)
+
+ def test_partial_or_empty_alignment_is_not_success(self):
+ partial = alignments()
+ del partial["alignments"][client.DATABASES[1]]
+ empty = alignments()
+ empty["alignments"][client.DATABASES[0]]["a3m"]["alignment"] = ">query\n"
+ for result in [None, {}, partial, empty]:
+ with self.subTest(result=result), patch.object(client.requests, "post", return_value=response(data=result)):
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+
+ def test_alignment_format_must_be_a3m(self):
+ for returned_format in [None, "fasta", "A3M", 1]:
+ result = alignments()
+ a3m = result["alignments"][client.DATABASES[0]]["a3m"]
+ if returned_format is None:
+ del a3m["format"]
+ else:
+ a3m["format"] = returned_format
+ with self.subTest(format=returned_format), \
+ patch.object(client.requests, "post", return_value=response(data=result)) as post:
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 1)
+
+ def test_malformed_a3m_is_not_success(self):
+ malformed = [
+ None,
+ "",
+ "ACDE\n",
+ "ACDE\n>query\nACDE\n",
+ ">query\n>hit\nACDE\n",
+ ">query\nACDE\n>hit\n",
+ ">query\nACDE\n>empty\n>hit\nACDE\n",
+ "> \nACDE\n",
+ " >query\nACDE\n",
+ ">query\nAC DE\n",
+ ">query\nACD1\n",
+ ">query\nACD*\n",
+ ">query\nACDÉ\n",
+ ">query\nACDE\n>hit\nACD\n",
+ ">query\nacde\n",
+ ">query\n# no sequence\n",
+ ]
+ for alignment in malformed:
+ result = alignments()
+ result["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = alignment
+ reply = response(data=result)
+ with self.subTest(alignment=alignment), \
+ patch.object(client.requests, "post", return_value=reply) as post:
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 1)
+ reply.close.assert_called_once()
+
+ def test_valid_a3m_supports_wrapping_insertions_and_comments(self):
+ for alignment in [
+ ">query\nACDE",
+ ">query description\nAC\nDE\n>hit\nAcC-\nE\n",
+ "# comment\n\n>query\nACDE\n\n>hit\nacACd-Efg\n# comment\n",
+ ]:
+ expected = alignments()
+ for database in client.DATABASES:
+ expected["alignments"][database]["a3m"]["alignment"] = alignment
+ with self.subTest(alignment=alignment), \
+ patch.object(client.requests, "post", return_value=response(data=expected)):
+ self.assertEqual(client.search("ACDE", list(client.DATABASES), "test-credential"), expected)
+
+ def test_missing_key_and_unknown_database_fail_before_network(self):
+ for databases, key in [(list(client.DATABASES), ""), (["../result"], "test-credential")]:
+ with self.subTest(databases=databases), patch.object(client.requests, "post") as post:
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", databases, key)
+ post.assert_not_called()
+
+ def test_cli_writes_actual_response_and_alignments(self):
+ expected = alignments()
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=expected)), redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ self.assertEqual(json.loads((output / "response.json").read_text()), expected)
+ for database in client.DATABASES:
+ self.assertEqual((output / f"{database}.a3m").read_text(), expected["alignments"][database]["a3m"]["alignment"])
+
+ @unittest.skipUnless(client.os.name == "posix", "Requires POSIX permissions")
+ def test_cli_output_is_private_with_permissive_umask(self):
+ with tempfile.TemporaryDirectory() as root:
+ Path(root).chmod(0o755)
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ previous_umask = client.os.umask(0)
+ try:
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=alignments())), \
+ redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ finally:
+ client.os.umask(previous_umask)
+ self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o700)
+ for path in output.iterdir():
+ self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600, path.name)
+
+ def test_cli_does_not_replace_concurrent_empty_output_directory(self):
+ mkdir, rename = Path.mkdir, Path.rename
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ competing_stat = None
+
+ def reserve_output():
+ nonlocal competing_stat
+ mkdir(output, mode=0o700)
+ competing_stat = output.stat()
+
+ # Simulate a competing reservation immediately before publication,
+ # after any existence check, with either directory operation.
+ def racing_mkdir(path, *args, **kwargs):
+ if path == output:
+ reserve_output()
+ return mkdir(path, *args, **kwargs)
+
+ def racing_rename(path, target):
+ if Path(target) == output:
+ reserve_output()
+ return rename(path, target)
+
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=alignments())), \
+ patch.object(Path, "mkdir", racing_mkdir), \
+ patch.object(Path, "rename", racing_rename), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertIsNotNone(competing_stat)
+ self.assertEqual(output.stat().st_ino, competing_stat.st_ino)
+ self.assertEqual(list(output.iterdir()), [])
+ self.assertEqual(list(Path(root).iterdir()), [output])
+ self.assertEqual(stdout.getvalue(), "")
+
+ def test_cli_failure_leaves_no_success_artifacts(self):
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(504)), \
+ patch.object(client.time, "sleep"), redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertFalse(output.exists())
+
+ def test_cli_write_failure_cleans_up_and_allows_retry(self):
+ write_text = Path.write_text
+ filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)]
+ for failing_file in filenames:
+ with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ expected = alignments()
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+
+ def fail_during_write(path, data, **kwargs):
+ if path.name == failing_file:
+ write_text(path, "partial", **kwargs)
+ raise OSError("sensitive filesystem details")
+ return write_text(path, data, **kwargs)
+
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=expected)):
+ with patch.object(Path, "write_text", fail_during_write), \
+ redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [])
+ self.assertEqual(stdout.getvalue(), "")
+ self.assertNotIn("sensitive", stderr.getvalue())
+ with redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ self.assertEqual(json.loads((output / "response.json").read_text()), expected)
+ for database in client.DATABASES:
+ self.assertEqual((output / f"{database}.a3m").read_text(),
+ expected["alignments"][database]["a3m"]["alignment"])
+
+ def test_cli_publish_failure_leaves_no_artifacts(self):
+ rename = Path.rename
+ filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)]
+ for failing_file in filenames:
+ with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+
+ def fail_during_publish(path, target):
+ if path.name == failing_file:
+ raise OSError("cannot publish")
+ return rename(path, target)
+
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=alignments())):
+ with patch.object(Path, "rename", fail_during_publish), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [])
+ self.assertEqual(stdout.getvalue(), "")
+ with redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ self.assertEqual({path.name for path in output.iterdir()}, set(filenames))
+
+ def test_cli_does_not_overwrite_output_created_during_search(self):
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+
+ def concurrent_output(*args, **kwargs):
+ output.mkdir()
+ (output / "keep.txt").write_text("existing data")
+ return response(data=alignments())
+
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", side_effect=concurrent_output), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [output])
+ self.assertEqual(list(output.iterdir()), [output / "keep.txt"])
+ self.assertEqual((output / "keep.txt").read_text(), "existing data")
+ self.assertEqual(stdout.getvalue(), "")
+
+ def test_cli_malformed_response_leaves_no_output(self):
+ malformed = alignments()
+ malformed["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = ">query\nACDE\n>hit\n"
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=malformed)), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [])
+ self.assertEqual(stdout.getvalue(), "")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md
new file mode 100644
index 0000000..6588795
--- /dev/null
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md
@@ -0,0 +1,119 @@
+# Skill Benchmark: msa-search-nim
+
+> ✅ **Overall verdict: PASS — Recommended for publication**
+
+## Publication Recommendation
+
+Recommended for publication based on the completed evaluation evidence in this report.
+
+## Evaluation Metadata
+
+- Skill: `msa-search-nim`
+- Evaluation date: 2026-10-01
+- Evaluator version: `1.5.6`
+- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`)
+- Tasks: 1 evaluation tasks (1 positive)
+- Dataset digest: `sha256:da6512955763a8e542da5599461202e0804188cd0eb5cd50377924e40f71d403` (skill-evaluator-dataset-snapshot/1)
+- Attempts per task: 1
+- Environment: `k8s-sandbox`
+- Tier 2 evidence: required for publication
+- Tier 3 evidence: required for publication
+
+Each task attempt ran in its own isolated sandbox pod.
+
+## What This Report Answers
+
+The three-tier evaluation checks whether the skill:
+
+- is safe to use;
+- produces correct answers;
+- is discovered and activated when needed;
+- helps the agent complete the user's goal and expected workflow; and
+- avoids wasted skill and tool usage.
+
+## Results at a Glance
+
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) |
+|---|---:|---:|
+| Overall | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 94.6% — baseline ran, but no comparable score was available; uplift unavailable |
+| Security | 50.0% → 100.0% (+50.0 points) | 50.0% → 100.0% (+50.0 points) |
+| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) |
+| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable |
+| Effectiveness | 95.0% → 100.0% (+5.0 points) | 92.9% → 100.0% (+7.1 points) |
+| Efficiency | 77.4% — baseline ran, but no comparable score was available; uplift unavailable | 83.1% — baseline ran, but no comparable score was available; uplift unavailable |
+
+**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points.
+
+Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline.
+
+## Token Usage
+
+Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions.
+
+| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage |
+|---|---|---:|---:|---:|---:|---|
+| claude-code | All cases | 443,342 | 647,113 | -203,771 | -31.49% | skill 1/1; base 1/1 |
+| claude-code | 1 | 443,342 | 647,113 | -203,771 | -31.49% | skill 1/1; base 1/1 |
+| codex | All cases | 186,870 | 387,556 | -200,686 | -51.78% | skill 1/1; base 1/1 |
+| codex | 1 | 186,870 | 387,556 | -200,686 | -51.78% | skill 1/1; base 1/1 |
+| ALL AGENTS | Dataset aggregate | 630,212 | 1,034,669 | -404,457 | -39.09% | skill 2/2; base 2/2 |
+
+Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated.
+
+## Tier Status
+
+| Tier | Purpose | Status | Evidence |
+|---|---|---|---|
+| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 39 finding(s) |
+| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) |
+| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 1 task(s) |
+
+## Findings and Observations
+
+
+Show detailed findings and successful checks
+
+- **MEDIUM** QUALITY/quality_correctness: No documented scripts in table format (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`)
+- **MEDIUM** QUALITY/quality_correctness: Instructions don't mention 'run_script' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`)
+- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`)
+- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`)
+- **MEDIUM** SCHEMA/folder_hierarchy: Unexpected nesting depth for general skill (`skills/bionemo-agent-toolkit/skills/msa-search-nim`)
+- 34 additional finding(s) are available in the full evaluation artifacts.
+
+
+
+## Scoring Methodology
+
+
+Show dimension definitions, source signals, and thresholds
+
+| Dimension | Question | Scored signals |
+|---|---|---|
+| Security | Is it safe to use? | `security` (100%) |
+| Correctness | Is the answer correct? | `accuracy` (100%) |
+| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) |
+| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) |
+| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) |
+
+- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%.
+- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL.
+- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate.
+- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold.
+- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`).
+- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial.
+
+Signals present in this run:
+
+- `security` (Security): unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability).
+- `accuracy` (Accuracy): final-answer correctness against the reference answer.
+- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved.
+- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed.
+- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency).
+
+
+
+## Freshness
+
+Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes.
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md
index 393f60c..efa9ef4 100644
--- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md
@@ -5,11 +5,14 @@ description: >
license: Apache-2.0 AND CC-BY-4.0
compatibility: "requests>=2.28"
allowed-tools: Bash, Read, Write, AskUserQuestion
+permissions:
+ - env # reads NGC_API_KEY/NVIDIA_API_KEY and local NIM setup variables
+ - network # hosted MSA requests and documented NGC/local NIM setup
---
# MSA-Search NIM
-Generate protein MSAs with GPU-accelerated MMSeqs2. Use this `SKILL.md` for
+Generate protein MSAs with GPU-accelerated MMSeqs2. Use this guide for
first-pass hosted/local usage; load supplemental files only when needed:
- `references/api.md`: exact endpoints, schemas, Docker flags, response fields.
@@ -280,6 +283,24 @@ Notes:
Use exact case-sensitive database names and response keys.
+For a hosted standard search, run the bundled client from this skill's directory.
+It submits the real request, validates both database results, and saves the raw
+JSON and A3M files. Choose a new output directory for each run:
+
+```bash
+python scripts/hosted_search.py \
+ --sequence SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR \
+ --output-dir msa-output
+```
+
+The client reads `NGC_API_KEY` or `NVIDIA_API_KEY` from the environment. It permits
+at most two requests, each with a 10-second connection timeout and a 300-second
+read timeout, with five seconds between attempts. If it exits nonzero, report the
+service failure and stop. Do not restart it repeatedly, extend timeouts beyond the
+task budget, or replace the missing response with synthetic alignments.
+
+The underlying request format, also usable with a running local NIM, is:
+
```python
import os
import requests
@@ -371,3 +392,9 @@ template, and sequence sanity checks, read `references/validation.md`.
- Paired MSA requires at least two sequences.
- Local URL 404 usually means an accidental `/v1/` prefix.
- First local run can take hours while databases populate `LOCAL_NIM_CACHE`.
+- Hosted HTTP 502/503/504 or repeated read timeouts indicate that the hosted
+ request did not complete. Check service availability after the bounded retry;
+ a longer client timeout cannot fix a server-generated HTTP 504.
+- Do not invent a polling URL for `health.api.nvidia.com`. The published standard
+ MSA example uses synchronous POST; a pending response needs a documented
+ service-specific completion mechanism before it can count as a result.
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml b/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml
index 0246c5d..c575fb8 100644
--- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml
@@ -6,6 +6,17 @@
version: 1
rules:
+ - id: "PE3"
+ path: "evals/evals.json"
+ message: ".env "
+ reason: >-
+ Reviewed JSON-only false positive (2026-09-16). The two matches are
+ expected-output/assertion text in deferred local-setup case 3. They
+ describe the same optional repo-root dotenv setup documented in this
+ skill; the JSON does not read files or execute that setup. This rule
+ matches only the literal .env finding in this eval file. Credential
+ access in executable scripts and references to other secret stores
+ remain subject to scanning.
- id: "PE3"
path: "*SKILL.md"
reason: >-
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json b/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json
index c04f9af..442b83c 100644
--- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json
@@ -7,41 +7,13 @@
"expected_output": "A successfully executed hosted MSA-Search request with Bearer auth, case-correct database names, and A3M output format, plus the actual returned alignment saved to a file and summarized from the response.",
"files": [],
"assertions": [
- {
- "id": "hosted-request-executed",
- "description": "Executes the hosted request instead of only writing code",
- "check": "Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path"
- },
- {
- "id": "hosted-endpoint-url",
- "description": "Uses the correct hosted MSA-Search endpoint URL",
- "check": "Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'"
- },
- {
- "id": "bearer-auth-header",
- "description": "Sets Authorization header with Bearer token from NGC_API_KEY",
- "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'"
- },
- {
- "id": "sequence-field",
- "description": "Request uses 'sequence' (singular) field with the provided sequence",
- "check": "Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'"
- },
- {
- "id": "databases-field",
- "description": "databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases",
- "check": "Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'"
- },
- {
- "id": "a3m-output-format",
- "description": "Requests A3M alignment output format",
- "check": "Script contains 'output_alignment_formats' and 'a3m'"
- },
- {
- "id": "saves-alignment-output",
- "description": "Saves the returned alignment to a file",
- "check": "Script writes alignment content from 'alignments' in the response to a file"
- }
+ "[hosted-request-executed] Executes the hosted request instead of only writing code: Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path",
+ "[hosted-endpoint-url] Uses the correct hosted MSA-Search endpoint URL: Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'",
+ "[bearer-auth-header] Sets Authorization header with Bearer token from NGC_API_KEY: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'",
+ "[sequence-field] Request uses 'sequence' (singular) field with the provided sequence: Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'",
+ "[databases-field] databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases: Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'",
+ "[a3m-output-format] Requests A3M alignment output format: Script contains 'output_alignment_formats' and 'a3m'",
+ "[saves-alignment-output] Saves the returned alignment to a file: Script writes alignment content from 'alignments' in the response to a file"
]
}
],
@@ -52,36 +24,12 @@
"expected_output": "A Python script that calls the hosted /paired/predict endpoint with a 'sequences' list containing both chains, extracts per-chain alignments from alignments_by_chain, and saves each chain's alignment to a separate file.",
"files": [],
"assertions": [
- {
- "id": "paired-endpoint-url",
- "description": "Uses the correct hosted paired MSA endpoint URL",
- "check": "Script contains 'msa-search/paired/predict'"
- },
- {
- "id": "sequences-plural-field",
- "description": "Uses 'sequences' (plural, list) field — not 'sequence' (singular)",
- "check": "Script payload contains 'sequences' as a list/array, not 'sequence'"
- },
- {
- "id": "both-chains-present",
- "description": "Both protein sequences are included in the request",
- "check": "Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'"
- },
- {
- "id": "bearer-auth-header",
- "description": "Sets Authorization header with Bearer token",
- "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'"
- },
- {
- "id": "parses-alignments-by-chain",
- "description": "Response parsed by 'alignments_by_chain' (not 'alignments')",
- "check": "Script references 'alignments_by_chain' from the response"
- },
- {
- "id": "saves-per-chain-alignments",
- "description": "Saves alignment for each chain to separate files",
- "check": "Script saves at least two alignment files, one per chain"
- }
+ "[paired-endpoint-url] Uses the correct hosted paired MSA endpoint URL: Script contains 'msa-search/paired/predict'",
+ "[sequences-plural-field] Uses 'sequences' (plural, list) field — not 'sequence' (singular): Script payload contains 'sequences' as a list/array, not 'sequence'",
+ "[both-chains-present] Both protein sequences are included in the request: Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'",
+ "[bearer-auth-header] Sets Authorization header with Bearer token: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'",
+ "[parses-alignments-by-chain] Response parsed by 'alignments_by_chain' (not 'alignments'): Script references 'alignments_by_chain' from the response",
+ "[saves-per-chain-alignments] Saves alignment for each chain to separate files: Script saves at least two alignment files, one per chain"
]
},
{
@@ -91,36 +39,12 @@
"expected_output": "Docker setup instructions using shell env first and optional repo-root .env overrides, requiring NGC_API_KEY or NVIDIA_API_KEY fallback plus LOCAL_NIM_CACHE, warning about the 1.4 TB database cache, health-checking the service, then sending a no-auth request to localhost:8000 without a /v1/ prefix.",
"files": [],
"assertions": [
- {
- "id": "docker-image-tag",
- "description": "References the correct MSA-Search container image with :2 tag",
- "check": "Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'"
- },
- {
- "id": "env-contract-and-cache",
- "description": "Local setup uses the repo env contract and LOCAL_NIM_CACHE",
- "check": "Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache"
- },
- {
- "id": "storage-warning",
- "description": "Mentions the large storage requirement for databases",
- "check": "Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases"
- },
- {
- "id": "nim-cache-mount",
- "description": "Mounts cache directory to /opt/nim/.cache",
- "check": "Output contains '/opt/nim/.cache' in the volume mount"
- },
- {
- "id": "health-check",
- "description": "Includes health check before submitting request",
- "check": "Output contains health check against localhost:8000/v1/health/ready"
- },
- {
- "id": "local-endpoint-no-v1",
- "description": "Local prediction request uses path without /v1/ prefix",
- "check": "Script contains 'localhost:8000/biology/colabfold/msa-search/predict'"
- }
+ "[docker-image-tag] References the correct MSA-Search container image with :2 tag: Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'",
+ "[env-contract-and-cache] Local setup uses the repo env contract and LOCAL_NIM_CACHE: Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache",
+ "[storage-warning] Mentions the large storage requirement for databases: Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases",
+ "[nim-cache-mount] Mounts cache directory to /opt/nim/.cache: Output contains '/opt/nim/.cache' in the volume mount",
+ "[health-check] Includes health check before submitting request: Output contains health check against localhost:8000/v1/health/ready",
+ "[local-endpoint-no-v1] Local prediction request uses path without /v1/ prefix: Script contains 'localhost:8000/biology/colabfold/msa-search/predict'"
]
},
{
@@ -130,36 +54,12 @@
"expected_output": "A Python script targeting the local /biology/colabfold/msa-search/structure-templates/predict endpoint because the hosted health.api template path returned HTTP 404 in validation, with structural_template_databases=['pdb70_220313'], the sequence, max_structures=20, max_msa_sequences=500 to match NIM_GLOBAL_MAX_MSA_DEPTH, no Authorization header for localhost inference, and parsing/saving both returned mmCIF template structures and search_hits M8 tables.",
"files": [],
"assertions": [
- {
- "id": "structure-templates-endpoint",
- "description": "Uses the structure-templates endpoint, not the standard predict endpoint",
- "check": "Script contains 'structure-templates/predict'"
- },
- {
- "id": "sequence-field",
- "description": "Request uses 'sequence' (singular) field",
- "check": "Script contains 'sequence' field with the provided sequence"
- },
- {
- "id": "max-structures-param",
- "description": "max_structures is set to 20 and max_msa_sequences matches the default GPU server depth",
- "check": "Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH"
- },
- {
- "id": "pdb70-canonical-database",
- "description": "Uses the canonical pdb70_220313 template database name",
- "check": "Script sets structural_template_databases to include 'pdb70_220313'"
- },
- {
- "id": "local-no-auth",
- "description": "Uses no Authorization header for local template inference",
- "check": "Script does not send Authorization/Bearer headers to localhost"
- },
- {
- "id": "parses-structures-and-search-hits",
- "description": "Parses template structures and search_hits M8 output",
- "check": "Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table"
- }
+ "[structure-templates-endpoint] Uses the structure-templates endpoint, not the standard predict endpoint: Script contains 'structure-templates/predict'",
+ "[sequence-field] Request uses 'sequence' (singular) field: Script contains 'sequence' field with the provided sequence",
+ "[max-structures-param] max_structures is set to 20 and max_msa_sequences matches the default GPU server depth: Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH",
+ "[pdb70-canonical-database] Uses the canonical pdb70_220313 template database name: Script sets structural_template_databases to include 'pdb70_220313'",
+ "[local-no-auth] Uses no Authorization header for local template inference: Script does not send Authorization/Bearer headers to localhost",
+ "[parses-structures-and-search-hits] Parses template structures and search_hits M8 output: Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table"
]
}
]
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md
index 4f5c176..18a2702 100644
--- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md
@@ -21,6 +21,9 @@ exact aria2c + `NIM_MODEL_NAME` commands.
## Hosted Standard MSA
+Run `scripts/hosted_search.py` from the skill root to save the response and A3M
+files. The request payload is:
+
```python
payload = {
"sequence": sequence,
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md
index 82ef75f..06d62b0 100644
--- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md
@@ -7,10 +7,23 @@ passing them downstream.
- `alignments` exists for standard search.
- `alignments_by_chain` exists for paired search.
-- Each returned alignment has `alignment` text and a `format`.
-- A3M/FASTA text starts with FASTA-style headers.
+- Each returned alignment has `alignment` text and a matching `format` (`a3m`
+ for the hosted client's requested output).
+- Each A3M/FASTA record has a nonempty FASTA-style header followed by sequence
+ data. Reject missing records, empty records, and invalid sequence characters.
+- A3M records have equal numbers of match columns: uppercase residues and `-`
+ count toward the width; lowercase insertions do not. Wrapped sequence lines,
+ blank lines, and `#` comments are allowed.
- Saved filenames include database and format so outputs do not overwrite each
other.
+- The hosted client finishes all writes in private staging, then exclusively
+ creates the output directory and moves the result files into it. An existing
+ output path, including an empty directory created by another run, is preserved.
+- On POSIX, the output directory uses owner-only permissions (`0700`), and result
+ files use `0600`, even with a permissive umask.
+- A failed write or move removes temporary files and any output directory created
+ by this run, so the same output path can be retried. Treat output as complete
+ only after the client exits successfully.
- Record database names and e-value used for the search.
## Template Checks
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py
new file mode 100644
index 0000000..d4c733c
--- /dev/null
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Run hosted MSA search with bounded retries and save actual A3M results.
+
+Requires requests>=2.28 and NGC_API_KEY (or NVIDIA_API_KEY). The two attempts
+use 10-second connect and 300-second read timeouts. A failed hosted service
+must remain a failure; this client never substitutes a fabricated alignment.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import time
+
+import requests
+
+
+URL = "https://health.api.nvidia.com/v1/biology/colabfold/msa-search/predict"
+DATABASES = ("Uniref30_2302", "colabfold_envdb_202108")
+REQUEST_TIMEOUT = (10, 300)
+MAX_ATTEMPTS = 2
+RETRY_DELAY = 5
+RETRYABLE_STATUS = {429, 502, 503, 504}
+
+
+class SearchError(RuntimeError):
+ """The hosted service did not produce the requested alignments."""
+
+
+def search(sequence: str, databases: list[str], api_key: str) -> dict:
+ """Return a validated response, or fail after at most two requests.
+
+ Do not forward server error bodies or requests exceptions to stderr:
+ those can contain sensitive request information. No redirects are followed
+ with the Authorization header. Database names are also used as filenames,
+ so only the documented database names are accepted.
+ """
+ if not api_key:
+ raise SearchError("Set NGC_API_KEY or NVIDIA_API_KEY before running hosted search.")
+ if not 1 <= len(sequence) <= 4096 or any(c not in "ACDEFGHIKLMNPQRSTVWYX" for c in sequence):
+ raise SearchError("Sequence must contain 1–4096 uppercase amino-acid letters (including X).")
+ if not databases or any(db not in DATABASES for db in databases):
+ raise SearchError("Select one or both documented databases.")
+ payload = {
+ "sequence": sequence,
+ "databases": list(dict.fromkeys(databases)),
+ "e_value": 0.0001,
+ "output_alignment_formats": ["a3m"],
+ }
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
+ last_error = "No response received."
+ for attempt in range(MAX_ATTEMPTS):
+ response = None
+ try:
+ response = requests.post(
+ URL, headers=headers, json=payload,
+ timeout=REQUEST_TIMEOUT, allow_redirects=False,
+ )
+ except (requests.Timeout, requests.ConnectionError):
+ last_error = "Hosted MSA request timed out or could not connect."
+ except requests.RequestException:
+ raise SearchError("Hosted MSA request failed before a usable response arrived.") from None
+ else:
+ try:
+ if response.status_code == 200:
+ try:
+ result = response.json()
+ except ValueError:
+ raise SearchError("Hosted MSA returned invalid JSON.") from None
+ validate_alignments(result, payload["databases"])
+ return result
+ last_error = f"Hosted MSA returned HTTP {response.status_code}."
+ if response.status_code not in RETRYABLE_STATUS:
+ raise SearchError(last_error)
+ finally:
+ response.close()
+ if attempt + 1 < MAX_ATTEMPTS:
+ time.sleep(RETRY_DELAY)
+ raise SearchError(
+ f"{last_error} Stopped after {MAX_ATTEMPTS} attempts. "
+ "No alignment was produced. Check hosted-service availability before retrying."
+ )
+
+
+def validate_alignments(result: object, databases: list[str]) -> None:
+ """Require an actual, nonempty A3M result for every requested database."""
+ alignments = result.get("alignments") if isinstance(result, dict) else None
+ if not isinstance(alignments, dict):
+ raise SearchError("Hosted MSA response has no alignments object.")
+ for database in databases:
+ formats = alignments.get(database)
+ a3m = formats.get("a3m") if isinstance(formats, dict) else None
+ if not isinstance(a3m, dict) or a3m.get("format") != "a3m":
+ raise SearchError(f"Hosted MSA returned no A3M-formatted result for {database}.")
+ alignment = a3m.get("alignment")
+ if not isinstance(alignment, str):
+ raise SearchError(f"Hosted MSA returned no A3M alignment for {database}.")
+ validate_a3m(alignment, database)
+
+
+def validate_a3m(alignment: str, database: str) -> None:
+ """Require named, nonempty records with consistent A3M match columns.
+
+ Uppercase residues and '-' occupy match columns; lowercase insertions
+ do not. Wrapped sequences, blank lines and '#' comments are supported.
+ """
+ error = f"Hosted MSA returned a malformed A3M alignment for {database}."
+ lengths: list[int] = []
+ for line in alignment.splitlines():
+ if not line.strip() or line.startswith("#"):
+ continue
+ if line.startswith(">"):
+ if not line[1:].strip():
+ raise SearchError(error)
+ lengths.append(0)
+ else:
+ if not lengths or any(c not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" for c in line):
+ raise SearchError(error)
+ lengths[-1] += sum("A" <= c <= "Z" or c == "-" for c in line)
+ if not lengths or not lengths[0] or any(length != lengths[0] for length in lengths):
+ raise SearchError(error)
+
+
+def save_results(result: dict, databases: list[str], output_dir: Path) -> None:
+ """Save private results in a new directory, cleaning up failed publication."""
+ output_dir.parent.mkdir(parents=True, exist_ok=True)
+ # Finish all writes inside private staging on the destination filesystem.
+ with tempfile.TemporaryDirectory(prefix=f".{output_dir.name}-", dir=output_dir.parent) as staging:
+ staged_output = Path(staging)
+ (staged_output / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+ for database in dict.fromkeys(databases):
+ alignment = result["alignments"][database]["a3m"]["alignment"]
+ (staged_output / f"{database}.a3m").write_text(alignment, encoding="utf-8")
+ files = list(staged_output.iterdir())
+ for path in files:
+ path.chmod(0o600)
+ # mkdir exclusively reserves the path, including against empty
+ # directories and dangling symlinks created by a concurrent run.
+ output_dir.mkdir(mode=0o700)
+ try:
+ for path in files:
+ path.rename(output_dir / path.name)
+ except BaseException:
+ # Only remove a directory that this run successfully reserved.
+ shutil.rmtree(output_dir)
+ raise
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--sequence", required=True)
+ parser.add_argument("--databases", nargs="+", choices=DATABASES, default=list(DATABASES))
+ parser.add_argument("--output-dir", type=Path, required=True, help="A new directory for this request")
+ args = parser.parse_args()
+ if args.output_dir.exists() or args.output_dir.is_symlink():
+ parser.error("--output-dir must not already exist; use a new path for each request")
+ try:
+ result = search(
+ args.sequence, args.databases,
+ os.environ.get("NGC_API_KEY") or os.environ.get("NVIDIA_API_KEY", ""),
+ )
+ save_results(result, args.databases, args.output_dir)
+ for database in dict.fromkeys(args.databases):
+ alignment = result["alignments"][database]["a3m"]["alignment"]
+ path = args.output_dir / f"{database}.a3m"
+ records = sum(line.startswith(">") for line in alignment.splitlines())
+ print(f"{path}: {records} sequences")
+ except SearchError as exc:
+ print(f"ERROR: {exc}", file=sys.stderr)
+ return 1
+ except OSError:
+ print("ERROR: Could not save the hosted response to the output directory.", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md
new file mode 100644
index 0000000..88f7d61
--- /dev/null
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md
@@ -0,0 +1,86 @@
+## Description:
+Generate multiple sequence alignments (MSAs) for protein sequences using the ColabFold MSA-Search NIM.
+
+This skill is ready for commercial/non-commercial use.
+
+## Owner
+NVIDIA
+
+### License/Terms of Use:
+Apache-2.0 AND CC-BY-4.0
+## Use Case:
+Developers and life science researchers who need to generate multiple sequence alignments for protein sequences, perform homolog searches, paired MSA searches for protein complexes, or retrieve PDB70 structural templates via hosted NVIDIA API or local Docker deployment.
+
+### Deployment Geography for Use:
+Global
+
+## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [API key]
+
+Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+
+## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+
+## Reference(s):
+- [API Reference](references/api.md)
+- [Science Background](references/science.md)
+- [Parameters Guide](references/parameters.md)
+- [Validation Guide](references/validation.md)
+- [Examples](references/examples.md)
+
+
+## Skill Output:
+**Output Type(s):** [API Calls, Files, Shell commands]
+**Output Format:** [JSON API responses, A3M alignment files, mmCIF structure files]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+
+## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+
+
+
+## Evaluation Tasks:
+1 evaluation task (1 positive), each attempt ran in its own isolated sandbox pod. Dataset digest: sha256:da6512955763a8e542da5599461202e0804188cd0eb5cd50377924e40f71d403.
+
+## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Whether the skill avoids unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Whether the final answer is correct against the reference answer.
+- Discoverability: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- Effectiveness: Whether the skill helped complete the user's goal (50% goal completion + 50% expected workflow adherence).
+- Efficiency: Whether the skill avoided wasted tool calls and token usage (50% tool-call productivity + 50% token efficiency).
+
+Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion usage.
+
+
+
+## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) |
+|---|---:|---:|
+| Overall | 95.5% | 94.6% |
+| Security | 50.0% → 100.0% (+50.0 points) | 50.0% → 100.0% (+50.0 points) |
+| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) |
+| Discoverability | 100.0% | 90.0% |
+| Effectiveness | 95.0% → 100.0% (+5.0 points) | 92.9% → 100.0% (+7.1 points) |
+| Efficiency | 77.4% | 83.1% |
+
+## Skill Version(s):
+0.1.0 (source: pyproject.toml)
+
+## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+
+(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig
new file mode 100644
index 0000000..e5cd204
--- /dev/null
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig
@@ -0,0 +1 @@
+{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibXNhLXNlYXJjaC1uaW0iLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiY2ZlOTc0MTgzZjg0OThhZTNjNmI3MjNiOTAzMzljOTk5NzEwM2ZlZjlhM2U4MjVjYmYzZWRjOTc1YjRkNWU1YiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjRhZmY5N2Q0YjA2NGJlYzExMTM2MjA4NDhkZjdkM2E0MjI5ZDYyNTI2YjA2NzdjZmYzYmZlOWJmMjk0OTVjNTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2Q0MmM1MTM2YzVkYTFiY2U4ODYyZTY3ZmVhMjU3NDA5NThjN2RhYmExZjI3YTFjNjEyNjk1YWM0ZGQwMzA2MiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJjb25maWcvc2tpbGxzcGVjdG9yLWJhc2VsaW5lLnltbCIsCiAgICAgICAgImRpZ2VzdCI6ICJkYTJkMTIzMzc1NWM2YTc2ZDYzYTZhZGQ2N2YwY2JjYmRiNjY2NDcwYzc3ODIxNGRiMTFjYTg3YmU4YmJkNDFmIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiYmEyYjhjZjBlYWQxM2JmYjY1ZDgxMzA5ZTE2MzE2MmRiYzJjNmM1MDU4OTY0YzRhZWE0MTRlMDA3MDg5OGE4YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjk0MDM5ZDZlNDg2NzE5Y2FhYmY1NmIzZDNiN2VkY2M5NWU3MjQ4ZWYyODY0MGEzOTNiM2U4NjgwNTU0MWIwYTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvdHJpZ2dlcl9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjFmN2M1MjFmNjk4YWEwYjAwNzZiMTQ3NjIyNjVhYmY1NzAwM2FiZDdmYzc3YTNmOWRlMmVmOTYxMTFlNTNmODgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcGkubWQiLAogICAgICAgICJkaWdlc3QiOiAiNGQyMGEzMDAxMDQ3Y2Q2NzY0NWE2MTQ1YjEyOGJmMGUxZDgyZGZlYmQwNGZjNGM1NGZmODZkZjlkNDVkNDE4NyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2V4YW1wbGVzLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjM4MTZhMjQ3MmZlODgyODhjNjNiNTc4MTJkNDdmZjhiMTQzZmUyNTFkMWZmMjNjMDY1YTg4ZDk5MjNiYTM2M2EiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wYXJhbWV0ZXJzLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjAyNWQ0MDBmN2VjOTc4OGFhNGExYjI5ODdkMDlkZDNiNGU3ZDdkZDE1YzJlODFjNjE0MmIyNzYzY2JmZWE1YjQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9zY2llbmNlLm1kIiwKICAgICAgICAiZGlnZXN0IjogImUwNjI5M2U0YWVlNDFmYTRkODY0MWZjODUxOTU1MDNkOTY2YWM0MDJhNDdiYTg1YmZlZjQzNmQyM2E3Zjc1YWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy92YWxpZGF0aW9uLm1kIiwKICAgICAgICAiZGlnZXN0IjogImJkNTRhZDBkZWZhNjc3NjU0MmEwNmRiODI1ODI3YTI2MDBiMzc2YjEzYTZkZTM1NjYzNDRhMDdjMjgwNGY1NTUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9ob3N0ZWRfc2VhcmNoLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjI1MWQ0YjU1Yzg4NDUzODFlODdlYjljMGI1NTE4MWU4YTRiNDQzOWVlYmM5OGNmNTlmNmQ3ODI5ZWQzYTY1ZDEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI0MDQ1OGUwOTdjNzA0NWVhMDc1Mjg5MTYxMDZiZGU4YzhjNWJhMDYxZGZmMTlkZDYyYzQ3OGY4ZTI1NTI0ZGM1IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInRlc3RzL3Rlc3RfaG9zdGVkX3NlYXJjaC5weSIsCiAgICAgICAgImRpZ2VzdCI6ICIwNjg3MTlmY2I3ZWNmM2JiYmE5YWExNjcxNjYxMDRjM2QwNTc4ZWEwN2ZkZDU5YmVjNjljNzA2NTJkOGQyYTY2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCDxUUYSuZAkoZM9uKenY5yLoZp4hweNWKW1DGhsf97nrFzKH+3vb2jM/wsFRDDs2gCMCoTUNVyCRu4RpTxt5EnK1hY+Tj+ksCsti9f5LWZcExyS9hMZqPNm6+7ROyhcKkHYQ==","keyid":""}]}}
\ No newline at end of file
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py
new file mode 100644
index 0000000..062ba75
--- /dev/null
+++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py
@@ -0,0 +1,314 @@
+"""Offline transport and artifact checks; no hosted-service calls are made."""
+
+from contextlib import redirect_stderr, redirect_stdout
+import importlib.util
+import io
+import json
+from pathlib import Path
+import stat
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+
+SPEC = importlib.util.spec_from_file_location(
+ "hosted_search", Path(__file__).resolve().parents[1] / "scripts" / "hosted_search.py"
+)
+client = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(client)
+
+
+def response(status=200, data=None):
+ result = Mock(status_code=status)
+ result.json.return_value = data
+ return result
+
+
+def alignments():
+ return {"alignments": {
+ db: {"a3m": {"alignment": ">query\nACDE\n>hit\nAC-E\n", "format": "a3m"}}
+ for db in client.DATABASES
+ }}
+
+
+class HostedSearchTests(unittest.TestCase):
+ def test_gateway_timeout_retries_once_then_stops(self):
+ failures = [response(504), response(504), response(data=alignments())]
+ with patch.object(client.requests, "post", side_effect=failures) as post, \
+ patch.object(client.time, "sleep"):
+ with self.assertRaisesRegex(client.SearchError, "HTTP 504.*Stopped after 2"):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 2)
+ for call in post.call_args_list:
+ self.assertEqual(call.kwargs["timeout"], (10, 300))
+ self.assertFalse(call.kwargs["allow_redirects"])
+
+ def test_transient_error_can_recover_without_changing_the_request(self):
+ expected = alignments()
+ with patch.object(client.requests, "post", side_effect=[response(503), response(data=expected)]) as post, \
+ patch.object(client.time, "sleep"):
+ actual = client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(actual, expected)
+ self.assertEqual(post.call_args_list[0], post.call_args_list[1])
+
+ def test_auth_error_and_redirect_are_not_retried(self):
+ for status in [401, 403, 302]:
+ with self.subTest(status=status), patch.object(client.requests, "post", return_value=response(status)) as post:
+ with self.assertRaisesRegex(client.SearchError, f"HTTP {status}"):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 1)
+
+ def test_timeout_does_not_expose_exception_text(self):
+ with patch.object(client.requests, "post", side_effect=client.requests.ReadTimeout("sensitive request details")) as post, \
+ patch.object(client.time, "sleep"):
+ with self.assertRaises(client.SearchError) as failure:
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertNotIn("sensitive", str(failure.exception))
+ self.assertEqual(post.call_count, 2)
+
+ def test_partial_or_empty_alignment_is_not_success(self):
+ partial = alignments()
+ del partial["alignments"][client.DATABASES[1]]
+ empty = alignments()
+ empty["alignments"][client.DATABASES[0]]["a3m"]["alignment"] = ">query\n"
+ for result in [None, {}, partial, empty]:
+ with self.subTest(result=result), patch.object(client.requests, "post", return_value=response(data=result)):
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+
+ def test_alignment_format_must_be_a3m(self):
+ for returned_format in [None, "fasta", "A3M", 1]:
+ result = alignments()
+ a3m = result["alignments"][client.DATABASES[0]]["a3m"]
+ if returned_format is None:
+ del a3m["format"]
+ else:
+ a3m["format"] = returned_format
+ with self.subTest(format=returned_format), \
+ patch.object(client.requests, "post", return_value=response(data=result)) as post:
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 1)
+
+ def test_malformed_a3m_is_not_success(self):
+ malformed = [
+ None,
+ "",
+ "ACDE\n",
+ "ACDE\n>query\nACDE\n",
+ ">query\n>hit\nACDE\n",
+ ">query\nACDE\n>hit\n",
+ ">query\nACDE\n>empty\n>hit\nACDE\n",
+ "> \nACDE\n",
+ " >query\nACDE\n",
+ ">query\nAC DE\n",
+ ">query\nACD1\n",
+ ">query\nACD*\n",
+ ">query\nACDÉ\n",
+ ">query\nACDE\n>hit\nACD\n",
+ ">query\nacde\n",
+ ">query\n# no sequence\n",
+ ]
+ for alignment in malformed:
+ result = alignments()
+ result["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = alignment
+ reply = response(data=result)
+ with self.subTest(alignment=alignment), \
+ patch.object(client.requests, "post", return_value=reply) as post:
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", list(client.DATABASES), "test-credential")
+ self.assertEqual(post.call_count, 1)
+ reply.close.assert_called_once()
+
+ def test_valid_a3m_supports_wrapping_insertions_and_comments(self):
+ for alignment in [
+ ">query\nACDE",
+ ">query description\nAC\nDE\n>hit\nAcC-\nE\n",
+ "# comment\n\n>query\nACDE\n\n>hit\nacACd-Efg\n# comment\n",
+ ]:
+ expected = alignments()
+ for database in client.DATABASES:
+ expected["alignments"][database]["a3m"]["alignment"] = alignment
+ with self.subTest(alignment=alignment), \
+ patch.object(client.requests, "post", return_value=response(data=expected)):
+ self.assertEqual(client.search("ACDE", list(client.DATABASES), "test-credential"), expected)
+
+ def test_missing_key_and_unknown_database_fail_before_network(self):
+ for databases, key in [(list(client.DATABASES), ""), (["../result"], "test-credential")]:
+ with self.subTest(databases=databases), patch.object(client.requests, "post") as post:
+ with self.assertRaises(client.SearchError):
+ client.search("ACDE", databases, key)
+ post.assert_not_called()
+
+ def test_cli_writes_actual_response_and_alignments(self):
+ expected = alignments()
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=expected)), redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ self.assertEqual(json.loads((output / "response.json").read_text()), expected)
+ for database in client.DATABASES:
+ self.assertEqual((output / f"{database}.a3m").read_text(), expected["alignments"][database]["a3m"]["alignment"])
+
+ @unittest.skipUnless(client.os.name == "posix", "Requires POSIX permissions")
+ def test_cli_output_is_private_with_permissive_umask(self):
+ with tempfile.TemporaryDirectory() as root:
+ Path(root).chmod(0o755)
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ previous_umask = client.os.umask(0)
+ try:
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=alignments())), \
+ redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ finally:
+ client.os.umask(previous_umask)
+ self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o700)
+ for path in output.iterdir():
+ self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600, path.name)
+
+ def test_cli_does_not_replace_concurrent_empty_output_directory(self):
+ mkdir, rename = Path.mkdir, Path.rename
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ competing_stat = None
+
+ def reserve_output():
+ nonlocal competing_stat
+ mkdir(output, mode=0o700)
+ competing_stat = output.stat()
+
+ # Simulate a competing reservation immediately before publication,
+ # after any existence check, with either directory operation.
+ def racing_mkdir(path, *args, **kwargs):
+ if path == output:
+ reserve_output()
+ return mkdir(path, *args, **kwargs)
+
+ def racing_rename(path, target):
+ if Path(target) == output:
+ reserve_output()
+ return rename(path, target)
+
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=alignments())), \
+ patch.object(Path, "mkdir", racing_mkdir), \
+ patch.object(Path, "rename", racing_rename), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertIsNotNone(competing_stat)
+ self.assertEqual(output.stat().st_ino, competing_stat.st_ino)
+ self.assertEqual(list(output.iterdir()), [])
+ self.assertEqual(list(Path(root).iterdir()), [output])
+ self.assertEqual(stdout.getvalue(), "")
+
+ def test_cli_failure_leaves_no_success_artifacts(self):
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(504)), \
+ patch.object(client.time, "sleep"), redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertFalse(output.exists())
+
+ def test_cli_write_failure_cleans_up_and_allows_retry(self):
+ write_text = Path.write_text
+ filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)]
+ for failing_file in filenames:
+ with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ expected = alignments()
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+
+ def fail_during_write(path, data, **kwargs):
+ if path.name == failing_file:
+ write_text(path, "partial", **kwargs)
+ raise OSError("sensitive filesystem details")
+ return write_text(path, data, **kwargs)
+
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=expected)):
+ with patch.object(Path, "write_text", fail_during_write), \
+ redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [])
+ self.assertEqual(stdout.getvalue(), "")
+ self.assertNotIn("sensitive", stderr.getvalue())
+ with redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ self.assertEqual(json.loads((output / "response.json").read_text()), expected)
+ for database in client.DATABASES:
+ self.assertEqual((output / f"{database}.a3m").read_text(),
+ expected["alignments"][database]["a3m"]["alignment"])
+
+ def test_cli_publish_failure_leaves_no_artifacts(self):
+ rename = Path.rename
+ filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)]
+ for failing_file in filenames:
+ with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+
+ def fail_during_publish(path, target):
+ if path.name == failing_file:
+ raise OSError("cannot publish")
+ return rename(path, target)
+
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=alignments())):
+ with patch.object(Path, "rename", fail_during_publish), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [])
+ self.assertEqual(stdout.getvalue(), "")
+ with redirect_stdout(io.StringIO()):
+ self.assertEqual(client.main(), 0)
+ self.assertEqual({path.name for path in output.iterdir()}, set(filenames))
+
+ def test_cli_does_not_overwrite_output_created_during_search(self):
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+
+ def concurrent_output(*args, **kwargs):
+ output.mkdir()
+ (output / "keep.txt").write_text("existing data")
+ return response(data=alignments())
+
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", side_effect=concurrent_output), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [output])
+ self.assertEqual(list(output.iterdir()), [output / "keep.txt"])
+ self.assertEqual((output / "keep.txt").read_text(), "existing data")
+ self.assertEqual(stdout.getvalue(), "")
+
+ def test_cli_malformed_response_leaves_no_output(self):
+ malformed = alignments()
+ malformed["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = ">query\nACDE\n>hit\n"
+ with tempfile.TemporaryDirectory() as root:
+ output = Path(root) / "msa"
+ args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)]
+ with patch.object(client.sys, "argv", args), \
+ patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \
+ patch.object(client.requests, "post", return_value=response(data=malformed)), \
+ redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()):
+ self.assertEqual(client.main(), 1)
+ self.assertEqual(list(Path(root).iterdir()), [])
+ self.assertEqual(stdout.getvalue(), "")
+
+
+if __name__ == "__main__":
+ unittest.main()