check-models.sh

#!/usr/bin/env bash
# ============================================================
#  check-models.sh
#  Standalone model health checker for OpenClaw
#  Tests: API key validity, provider reachability, model name
#  format, escalation chain integrity, and provider endpoints.
#
#  Usage:
#    bash check-models.sh              # full report
#    bash check-models.sh --json       # JSON output for dashboard
#    bash check-models.sh --fix        # auto-remediate where possible
# ============================================================
set -uo pipefail

CFG="${HOME}/.openclaw/openclaw.json"
CREDS_DIR="${HOME}/.openclaw/credentials"
RESULTS_FILE="/tmp/openclaw-model-health.json"
OUTPUT_JSON=false
AUTO_FIX=false

for arg in "$@"; do
  case "$arg" in
    --json) OUTPUT_JSON=true ;;
    --fix)  AUTO_FIX=true ;;
  esac
done

G='\033[0;32m'; R='\033[0;31m'; Y='\033[1;33m'
C='\033[0;36m'; W='\033[1;37m'; DIM='\033[2m'; B='\033[0m'
ok()   { $OUTPUT_JSON || echo -e "  ${G}✔${B} $*"; }
fail() { $OUTPUT_JSON || echo -e "  ${R}✘${B} $*"; }
warn() { $OUTPUT_JSON || echo -e "  ${Y}⚠${B} $*"; }
info() { $OUTPUT_JSON || echo -e "  ${C}›${B} $*"; }
hdr()  { $OUTPUT_JSON || echo -e "\n${W}$*${B}"; $OUTPUT_JSON || printf '%0.s─' $(seq 1 60); $OUTPUT_JSON || echo; }

# ── Known model patterns per provider ────────────────────────
# Format: provider_prefix:regex_pattern
declare -A MODEL_PATTERNS=(
  [anthropic]="^anthropic/(claude-(opus|sonnet|haiku)-[0-9])"
  [openai]="^openai/(gpt-(4|3\.5)|o[0-9]|chatgpt)"
  [google]="^google/(gemini-[0-9])"
  [moonshot]="^moonshot/(kimi|moonshot-v)"
  [mistral]="^mistral/(mistral|mixtral|codestral)"
  [groq]="^groq/"
  [openrouter]="^openrouter/"
  [ollama]="^ollama/"
)

# Known current valid model strings (as of Feb 2026)
declare -A KNOWN_MODELS=(
  # Anthropic
  ["anthropic/claude-opus-4-6"]="valid"
  ["anthropic/claude-sonnet-4-6"]="valid"
  ["anthropic/claude-haiku-4-5"]="valid"
  ["anthropic/claude-opus-4-5"]="valid"
  ["anthropic/claude-sonnet-4-5"]="valid"
  ["anthropic/claude-sonnet-4-20250514"]="valid"
  ["anthropic/claude-opus-4-20250514"]="valid"
  # Deprecated/renamed
  ["anthropic/claude-opus-4"]="deprecated:use anthropic/claude-opus-4-6"
  ["anthropic/claude-sonnet-4"]="deprecated:use anthropic/claude-sonnet-4-6"
  ["anthropic/claude-3-5-sonnet-20241022"]="deprecated:use anthropic/claude-sonnet-4-5"
  ["anthropic/claude-3-opus-20240229"]="deprecated:use anthropic/claude-opus-4-5"
  # OpenAI
  ["openai/gpt-4o"]="valid"
  ["openai/gpt-4o-mini"]="valid"
  ["openai/gpt-4-turbo"]="valid"
  ["openai/o3"]="valid"
  ["openai/o3-mini"]="valid"
  ["openai/gpt-4"]="deprecated:use openai/gpt-4o"
  # Google
  ["google/gemini-2.0-flash"]="valid"
  ["google/gemini-2.0-pro"]="valid"
  ["google/gemini-1.5-pro"]="valid"
  ["google/gemini-1.5-flash"]="valid"
  ["google/gemini-2.0-flash-exp"]="valid"
  # Moonshot / Kimi
  ["moonshot/kimi-k2"]="valid"
  ["moonshot/kimi-k2.5"]="valid"
  ["moonshot/moonshot-v1-8k"]="valid"
  ["moonshot/moonshot-v1-32k"]="valid"
  ["moonshot/moonshot-v1-128k"]="valid"
)

# Provider API test endpoints (minimal, low-cost probes)
declare -A PROVIDER_TEST_URLS=(
  [anthropic]="https://api.anthropic.com/v1/models"
  [openai]="https://api.openai.com/v1/models"
  [google]="https://generativelanguage.googleapis.com/v1beta/models"
  [moonshot]="https://api.moonshot.cn/v1/models"
  [mistral]="https://api.mistral.ai/v1/models"
  [groq]="https://api.groq.com/openai/v1/models"
)

# Env var names for each provider's API key
declare -A PROVIDER_ENV_KEYS=(
  [anthropic]="ANTHROPIC_API_KEY"
  [openai]="OPENAI_API_KEY"
  [google]="GOOGLE_API_KEY GEMINI_API_KEY"
  [moonshot]="MOONSHOT_API_KEY KIMI_API_KEY"
  [mistral]="MISTRAL_API_KEY"
  [groq]="GROQ_API_KEY"
  [openrouter]="OPENROUTER_API_KEY"
)

# ── Extract config values ─────────────────────────────────────
get_primary_model() {
  jq -r '
    .agents.defaults.model.primary //
    .agent.model.primary //
    empty
  ' "$CFG" 2>/dev/null
}

get_fallback_models() {
  jq -r '
    (.agents.defaults.model.fallbacks // .agent.model.fallbacks // []) |
    .[]
  ' "$CFG" 2>/dev/null
}

get_provider_config() {
  jq -r '.models.providers // {} | to_entries[] | "\(.key)|\(.value.baseUrl // "")|\(.value.apiKey // "")|\(.value.api // "")"' "$CFG" 2>/dev/null
}

get_api_key_for_provider() {
  local provider="$1"
  # 1. Check credentials file
  if [[ -f "${CREDS_DIR}/${provider}" ]]; then
    cat "${CREDS_DIR}/${provider}" 2>/dev/null | tr -d '[:space:]'
    return
  fi
  # 2. Check env vars
  local env_vars="${PROVIDER_ENV_KEYS[$provider]:-}"
  for env_var in $env_vars; do
    local val="${!env_var:-}"
    if [[ -n "$val" ]]; then
      echo "$val"
      return
    fi
  done
  # 3. Check config file directly (may be literal key, not $VAR reference)
  local cfg_key
  cfg_key=$(jq -r --arg p "$provider" \
    '.models.providers[$p].apiKey // .auth.profiles | to_entries[] | select(.key | startswith($p)) | .value.apiKey // empty' \
    "$CFG" 2>/dev/null | head -1)
  # If it looks like an env var reference, resolve it
  if [[ "$cfg_key" =~ ^\$\{?([A-Z_]+)\}?$ ]]; then
    local ref_var="${BASH_REMATCH[1]}"
    echo "${!ref_var:-}"
  else
    echo "${cfg_key:-}"
  fi
}

# ── Checks ────────────────────────────────────────────────────

# Returns: "valid" | "deprecated:msg" | "unknown_format" | "unknown_model"
check_model_string() {
  local model="$1"
  local provider="${model%%/*}"

  # Known good/bad list first
  local known="${KNOWN_MODELS[$model]:-}"
  if [[ "$known" == "valid" ]]; then
    echo "valid"
    return
  elif [[ "$known" == deprecated* ]]; then
    echo "$known"
    return
  fi

  # Check provider prefix format
  local pattern="${MODEL_PATTERNS[$provider]:-}"
  if [[ -z "$pattern" ]]; then
    echo "unknown_provider:$provider"
    return
  fi

  if [[ "$model" =~ $pattern ]]; then
    echo "valid_format"
  else
    echo "bad_format:expected ${pattern}"
  fi
}

# Returns HTTP status or "unreachable"
probe_provider_endpoint() {
  local provider="$1"
  local api_key="$2"
  local url="${PROVIDER_TEST_URLS[$provider]:-}"
  [[ -z "$url" ]] && echo "no_test_url" && return

  local http_code
  case "$provider" in
    anthropic)
      http_code=$(curl -s -o /dev/null -w "%{http_code}" \
        --max-time 8 \
        -H "x-api-key: ${api_key}" \
        -H "anthropic-version: 2023-06-01" \
        "$url" 2>/dev/null || echo "000")
      ;;
    google)
      http_code=$(curl -s -o /dev/null -w "%{http_code}" \
        --max-time 8 \
        "${url}?key=${api_key}" 2>/dev/null || echo "000")
      ;;
    *)
      http_code=$(curl -s -o /dev/null -w "%{http_code}" \
        --max-time 8 \
        -H "Authorization: Bearer ${api_key}" \
        "$url" 2>/dev/null || echo "000")
      ;;
  esac
  echo "$http_code"
}

# ── JSON result builder ───────────────────────────────────────
declare -a RESULT_ITEMS=()

add_result() {
  # add_result "category" "name" "status" "message"
  RESULT_ITEMS+=("{\"category\":\"$1\",\"name\":\"$2\",\"status\":\"$3\",\"message\":\"$4\"}")
}

# ── Main checks ───────────────────────────────────────────────
run_checks() {
  [[ ! -f "$CFG" ]] && { add_result "config" "openclaw.json" "error" "File not found"; return; }

  # ── 1. Primary model ───────────────────────────────────────
  $OUTPUT_JSON || hdr "Primary Model"
  local primary
  primary=$(get_primary_model)

  if [[ -z "$primary" ]]; then
    fail "No primary model configured"
    add_result "primary" "primary_model" "error" "Not configured"
  else
    local check
    check=$(check_model_string "$primary")
    local provider="${primary%%/*}"

    case "$check" in
      valid|valid_format)
        ok "Primary: ${primary}"
        add_result "primary" "$primary" "ok" "Valid model string"
        ;;
      deprecated:*)
        local msg="${check#deprecated:}"
        warn "Primary model is deprecated: ${primary} → ${msg}"
        add_result "primary" "$primary" "warn" "Deprecated: ${msg}"
        $AUTO_FIX && {
          local replacement="${msg#use }"
          info "Auto-fixing primary model: $primary → $replacement"
          jq --arg old "$primary" --arg new "$replacement" \
            '(.agents.defaults.model.primary) |= if . == $old then $new else . end' \
            "$CFG" > /tmp/cfg_fix.json && mv /tmp/cfg_fix.json "$CFG"
          ok "Fixed primary model → $replacement"
        }
        ;;
      unknown_provider:*)
        warn "Unknown provider in: ${primary}"
        add_result "primary" "$primary" "warn" "Unknown provider: ${provider}"
        ;;
      bad_format:*)
        fail "Bad model format: ${primary}"
        add_result "primary" "$primary" "error" "Bad format — expected provider/model-name"
        ;;
    esac
  fi

  # ── 2. Fallback chain ──────────────────────────────────────
  $OUTPUT_JSON || hdr "Fallback / Escalation Chain"
  local fallbacks
  fallbacks=$(get_fallback_models)

  if [[ -z "$fallbacks" ]]; then
    warn "No fallback models configured — single point of failure"
    add_result "fallbacks" "chain" "warn" "No fallbacks defined"
  else
    local idx=0
    local prev_provider="${primary%%/*}"
    local chain_ok=true

    while IFS= read -r model; do
      [[ -z "$model" ]] && continue
      ((idx++)) || true
      local check
      check=$(check_model_string "$model")
      local provider="${model%%/*}"

      # Check for same-provider consecutive fallbacks (risky)
      if [[ "$provider" == "$prev_provider" && "$idx" -gt 1 ]]; then
        warn "Fallback #${idx} same provider as previous (${provider}) — no provider diversity"
        add_result "fallbacks" "$model" "warn" "Same provider as previous fallback"
      fi

      case "$check" in
        valid|valid_format)
          ok "Fallback #${idx}: ${model}"
          add_result "fallbacks" "$model" "ok" "Valid"
          ;;
        deprecated:*)
          local msg="${check#deprecated:}"
          warn "Fallback #${idx} deprecated: ${model} → ${msg}"
          add_result "fallbacks" "$model" "warn" "Deprecated: ${msg}"
          chain_ok=false
          $AUTO_FIX && {
            local replacement="${msg#use }"
            jq --arg old "$model" --arg new "$replacement" \
              '(.agents.defaults.model.fallbacks // []) |= map(if . == $old then $new else . end)' \
              "$CFG" > /tmp/cfg_fix.json && mv /tmp/cfg_fix.json "$CFG"
            ok "  Fixed → $replacement"
          }
          ;;
        bad_format:*|unknown_provider:*)
          fail "Fallback #${idx} invalid: ${model} (${check})"
          add_result "fallbacks" "$model" "error" "$check"
          chain_ok=false
          ;;
      esac
      prev_provider="$provider"
    done <<< "$fallbacks"

    $chain_ok && ok "Escalation chain integrity: OK" || warn "Escalation chain has issues"
  fi

  # ── 3. Provider API keys & endpoints ──────────────────────
  $OUTPUT_JSON || hdr "Provider API Keys & Reachability"

  # Collect all providers referenced in models
  local all_providers=()
  [[ -n "${primary:-}" ]] && all_providers+=("${primary%%/*}")
  while IFS= read -r m; do
    [[ -n "$m" ]] && all_providers+=("${m%%/*}")
  done <<< "$(get_fallback_models)"

  # Also check explicitly configured providers
  while IFS='|' read -r pname _baseurl _apikey _api; do
    [[ -n "$pname" ]] && all_providers+=("$pname")
  done <<< "$(get_provider_config)"

  # Deduplicate
  local unique_providers
  unique_providers=$(echo "${all_providers[@]:-}" | tr ' ' '\n' | sort -u)

  while IFS= read -r provider; do
    [[ -z "$provider" ]] && continue
    # Skip local providers
    [[ "$provider" == "ollama" || "$provider" == "lmstudio" || "$provider" == "local"* ]] && {
      local base_url
      base_url=$(jq -r --arg p "$provider" '.models.providers[$p].baseUrl // ""' "$CFG" 2>/dev/null)
      if [[ -n "$base_url" ]]; then
        local http
        http=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${base_url%/v1}/api/tags" 2>/dev/null || echo "000")
        [[ "$http" == "200" ]] \
          && { ok "Local provider ${provider}: reachable (${base_url})"; add_result "providers" "$provider" "ok" "Local endpoint reachable"; } \
          || { warn "Local provider ${provider}: unreachable at ${base_url}"; add_result "providers" "$provider" "warn" "Local endpoint not responding"; }
      fi
      continue
    }

    local api_key
    api_key=$(get_api_key_for_provider "$provider")

    if [[ -z "$api_key" ]]; then
      fail "Provider ${provider}: No API key found"
      add_result "providers" "$provider" "error" "API key missing"
      continue
    fi

    # Mask key for display
    local masked="${api_key:0:6}...${api_key: -4}"
    info "Testing ${provider} (key: ${masked})"

    local http_code
    http_code=$(probe_provider_endpoint "$provider" "$api_key")

    case "$http_code" in
      200)
        ok "Provider ${provider}: API key valid, endpoint reachable"
        add_result "providers" "$provider" "ok" "HTTP 200"
        ;;
      401|403)
        fail "Provider ${provider}: API key rejected (HTTP ${http_code})"
        add_result "providers" "$provider" "error" "Invalid API key (HTTP ${http_code})"
        ;;
      429)
        warn "Provider ${provider}: Rate limited (HTTP 429)"
        add_result "providers" "$provider" "warn" "Rate limited"
        ;;
      000|"")
        fail "Provider ${provider}: Endpoint unreachable (connection failed)"
        add_result "providers" "$provider" "error" "Unreachable"
        ;;
      no_test_url)
        warn "Provider ${provider}: No test endpoint defined, skipping probe"
        add_result "providers" "$provider" "warn" "No test URL"
        ;;
      *)
        warn "Provider ${provider}: Unexpected HTTP ${http_code}"
        add_result "providers" "$provider" "warn" "HTTP ${http_code}"
        ;;
    esac
  done <<< "$unique_providers"

  # ── 4. Subagent model check ───────────────────────────────
  $OUTPUT_JSON || hdr "Subagent & TTS Models"
  local subagent_model
  subagent_model=$(jq -r '.agents.defaults.subagents.model // empty' "$CFG" 2>/dev/null)
  if [[ -n "$subagent_model" ]]; then
    local check
    check=$(check_model_string "$subagent_model")
    case "$check" in
      valid|valid_format) ok "Subagent model: ${subagent_model}" ; add_result "subagents" "$subagent_model" "ok" "Valid" ;;
      deprecated:*)       warn "Subagent model deprecated: ${subagent_model}" ; add_result "subagents" "$subagent_model" "warn" "${check#deprecated:}" ;;
      *)                  fail "Subagent model invalid: ${subagent_model}" ; add_result "subagents" "$subagent_model" "error" "$check" ;;
    esac
  fi

  local tts_model
  tts_model=$(jq -r '.messages.tts.summaryModel // empty' "$CFG" 2>/dev/null)
  if [[ -n "$tts_model" ]]; then
    local check
    check=$(check_model_string "$tts_model")
    case "$check" in
      valid|valid_format) ok "TTS summary model: ${tts_model}" ;;
      *) warn "TTS summary model may be outdated: ${tts_model}" ;;
    esac
  fi
}

# ── Output ────────────────────────────────────────────────────
run_checks

if $OUTPUT_JSON; then
  # Build JSON summary for dashboard consumption
  local_ifs="$IFS"
  IFS=','
  echo "{\"timestamp\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"results\":[${RESULT_ITEMS[*]}]}" \
    > "$RESULTS_FILE"
  IFS="$local_ifs"
  cat "$RESULTS_FILE"
fi