fix(hitl): prevent tool execution when the TypeSafe audit response is incomplete

internal/handler/hitl_audit_agent_test.go

@@ -2,10 +2,14 @@ package handler
 
 import (
 	"context"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 
 	"cyberstrike-ai/internal/config"
+
+	"go.uber.org/zap"
 )
 
 func TestParseAuditAgentLLMContentApprove(t *testing.T) {
@@ -100,3 +104,52 @@ func TestBuildAuditAgentReviewInput(t *testing.T) {
 		t.Fatalf("unexpected input: %s", s)
 	}
 }
+
+func TestAuditAgentReviewTypeSafeIncompleteAnswersReject(t *testing.T) {
+	cases := []struct {
+		name string
+		body string
+		want string
+	}{
+		{
+			name: "empty answers",
+			body: `{"model":"jev-latest","answers":{}}`,
+			want: "reject",
+		},
+		{
+			name: "partial answers",
+			body: `{"answers":{"destroys_availability":{"noul":0.02}}}`,
+			want: "reject",
+		},
+		{
+			name: "valid low risk answers",
+			body: `{"answers":{"destroys_availability":{"noul":0.02},"irreversible_data_loss":{"noul":0.02},"credential_or_acl_change":{"noul":0.02},"config_tamper":{"noul":0.02},"authorized_pentest_payload":{"noul":0.9},"decision":{"choice":"approve","confidence":0.95}}}`,
+			want: "approve",
+		},
+		{
+			name: "destructive answers",
+			body: `{"answers":{"destroys_availability":{"noul":0.97},"irreversible_data_loss":{"noul":0.97},"credential_or_acl_change":{"noul":0.97},"config_tamper":{"noul":0.97},"authorized_pentest_payload":{"noul":0.2},"decision":{"choice":"reject","confidence":0.97}}}`,
+			want: "reject",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				_, _ = w.Write([]byte(tc.body))
+			}))
+			defer srv.Close()
+			h := &AgentHandler{
+				config: &config.Config{Hitl: config.HitlConfig{
+					AuditBackend: "typesafe",
+					AuditModel:   config.OpenAIConfig{BaseURL: srv.URL, APIKey: "ts-key", Model: "jev-latest"},
+				}},
+				logger: zap.NewNop(),
+			}
+			d := h.auditAgentReview(context.Background(), "approval", "exec", map[string]interface{}{"command": "id"})
+			if d.Decision != tc.want {
+				t.Fatalf("decision=%s comment=%s, want %s", d.Decision, d.Comment, tc.want)
+			}
+		})
+	}
+}

internal/hitl/jev.go

@@ -138,6 +138,11 @@ func DecideJev(result *typesafe.Result) (decision, comment string) {
 	if result == nil {
 		return "reject", "audit agent: TypeSafe 无有效响应，保守拒绝"
 	}
+	// An empty or partial answer set means the evaluation did not complete. Keep this
+	// check here as well so a result built outside SystemOne cannot be approved by default.
+	if err := result.Validate(JevAuditQuestions("")); err != nil {
+		return "reject", "audit agent: TypeSafe 裁决不完整，保守拒绝"
+	}
 	availability := result.Noul(jevQAvailability)
 	dataLoss := result.Noul(jevQDataLoss)
 	credentials := result.Noul(jevQCredentials)

internal/hitl/jev_test.go

@@ -152,3 +152,39 @@ func TestDecideJevPolicyChoiceRejectsEvenIfPentest(t *testing.T) {
 		t.Fatalf("comment=%s", comment)
 	}
 }
+
+func TestDecideJevRejectsEmptyAnswers(t *testing.T) {
+	dec, comment := DecideJev(&typesafe.Result{Answers: map[string]map[string]any{}})
+	if dec != "reject" {
+		t.Fatalf("decision=%s comment=%s", dec, comment)
+	}
+	if !strings.Contains(comment, "裁决不完整") {
+		t.Fatalf("comment=%s", comment)
+	}
+}
+
+func TestDecideJevRejectsPartialAnswers(t *testing.T) {
+	dec, comment := DecideJev(&typesafe.Result{Answers: map[string]map[string]any{
+		jevQAvailability: {"noul": 0.02},
+	}})
+	if dec != "reject" {
+		t.Fatalf("decision=%s comment=%s", dec, comment)
+	}
+	if !strings.Contains(comment, "裁决不完整") {
+		t.Fatalf("comment=%s", comment)
+	}
+}
+
+func TestDecideJevRejectsMissingNoulValue(t *testing.T) {
+	dec, comment := DecideJev(&typesafe.Result{Answers: map[string]map[string]any{
+		jevQAvailability: {},
+		jevQDataLoss:     {"noul": 0.02},
+		jevQCredentials:  {"noul": 0.02},
+		jevQConfig:       {"noul": 0.02},
+		jevQPentest:      {"noul": 0.9},
+		jevQDecision:     {"choice": "approve", "confidence": 0.95},
+	}})
+	if dec != "reject" {
+		t.Fatalf("decision=%s comment=%s", dec, comment)
+	}
+}

internal/typesafe/client.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 )
@@ -189,8 +190,68 @@ func (c *Client) SystemOne(ctx context.Context, state any, questions map[string]
 	if err := json.Unmarshal(respBody, &out); err != nil {
 		return nil, fmt.Errorf("decode typesafe response: %w", err)
 	}
-	if out.Answers == nil {
-		out.Answers = map[string]map[string]any{}
+	if err := out.Validate(questions); err != nil {
+		return nil, fmt.Errorf("typesafe response rejected: %w", err)
 	}
 	return &out, nil
 }
+
+// Validate verifies that every requested question has a usable answer.
+// A 200 response with an empty or partial answers object means the evaluation
+// did not run; it must not be interpreted as "no destructive signal".
+func (r *Result) Validate(questions map[string]Question) error {
+	if r == nil {
+		return fmt.Errorf("empty result")
+	}
+	if len(r.Answers) == 0 {
+		return fmt.Errorf("no answers in response")
+	}
+	missing := make([]string, 0, len(questions))
+	for id, q := range questions {
+		ans, ok := r.Answers[id]
+		if !ok || ans == nil {
+			missing = append(missing, id)
+			continue
+		}
+		switch strings.ToLower(strings.TrimSpace(fmt.Sprint(q["type"]))) {
+		case "noul":
+			if _, ok := answerProbability(ans["noul"]); !ok {
+				return fmt.Errorf("answer %q has a missing or invalid noul", id)
+			}
+		case "choice":
+			choice, _ := ans["choice"].(string)
+			if strings.TrimSpace(choice) == "" {
+				return fmt.Errorf("answer %q has no choice", id)
+			}
+			if _, ok := answerProbability(ans["confidence"]); !ok {
+				return fmt.Errorf("answer %q has a missing or invalid confidence", id)
+			}
+		}
+	}
+	if len(missing) > 0 {
+		sort.Strings(missing)
+		return fmt.Errorf("incomplete answers: missing %s", strings.Join(missing, ", "))
+	}
+	return nil
+}
+
+// answerProbability reads a probability-valued answer, accepting only numbers in [0, 1].
+func answerProbability(v any) (float64, bool) {
+	var f float64
+	switch n := v.(type) {
+	case float64:
+		f = n
+	case json.Number:
+		parsed, err := n.Float64()
+		if err != nil {
+			return 0, false
+		}
+		f = parsed
+	default:
+		return 0, false
+	}
+	if f < 0 || f > 1 {
+		return 0, false
+	}
+	return f, true
+}

internal/typesafe/client_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 )
 
@@ -59,3 +60,74 @@ func TestSystemOneEmptyAPIKey(t *testing.T) {
 		t.Fatal("expected error")
 	}
 }
+
+func testServerReturning(t *testing.T, body string) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestSystemOneRejectsEmptyAnswers(t *testing.T) {
+	srv := testServerReturning(t, `{"model":"jev-latest","answers":{}}`)
+	client := NewClient(srv.URL, "ts-key", "", srv.Client())
+	_, err := client.SystemOne(context.Background(), "ping", map[string]Question{
+		"availability": Noul("Would this destroy availability?", "yes", "no"),
+		"decision":     Choice("Approve?", map[string]string{"approve": "yes", "reject": "no"}),
+	})
+	if err == nil || !strings.Contains(err.Error(), "no answers") {
+		t.Fatalf("expected empty-answers error, got %v", err)
+	}
+}
+
+func TestSystemOneRejectsIncompleteAnswers(t *testing.T) {
+	srv := testServerReturning(t, `{"answers":{"availability":{"noul":0.02}}}`)
+	client := NewClient(srv.URL, "ts-key", "", srv.Client())
+	_, err := client.SystemOne(context.Background(), "ping", map[string]Question{
+		"availability": Noul("Would this destroy availability?", "yes", "no"),
+		"decision":     Choice("Approve?", map[string]string{"approve": "yes", "reject": "no"}),
+	})
+	if err == nil || !strings.Contains(err.Error(), "incomplete") || !strings.Contains(err.Error(), "decision") {
+		t.Fatalf("expected incomplete-answers error, got %v", err)
+	}
+}
+
+func TestSystemOneRejectsNonNumericNoul(t *testing.T) {
+	srv := testServerReturning(t, `{"answers":{"availability":{"noul":"0.02"}}}`)
+	client := NewClient(srv.URL, "ts-key", "", srv.Client())
+	_, err := client.SystemOne(context.Background(), "ping", map[string]Question{
+		"availability": Noul("Would this destroy availability?", "yes", "no"),
+	})
+	if err == nil || !strings.Contains(err.Error(), "invalid noul") {
+		t.Fatalf("expected invalid-noul error, got %v", err)
+	}
+}
+
+func TestSystemOneRejectsChoiceWithoutConfidence(t *testing.T) {
+	srv := testServerReturning(t, `{"answers":{"decision":{"choice":"reject"}}}`)
+	client := NewClient(srv.URL, "ts-key", "", srv.Client())
+	_, err := client.SystemOne(context.Background(), "ping", map[string]Question{
+		"decision": Choice("Approve?", map[string]string{"approve": "yes", "reject": "no"}),
+	})
+	if err == nil || !strings.Contains(err.Error(), "invalid confidence") {
+		t.Fatalf("expected invalid-confidence error, got %v", err)
+	}
+}
+
+func TestSystemOneAcceptsCompleteAnswers(t *testing.T) {
+	srv := testServerReturning(t, `{"answers":{"availability":{"noul":0.02},"decision":{"choice":"approve","confidence":0.95}}}`)
+	client := NewClient(srv.URL, "ts-key", "", srv.Client())
+	got, err := client.SystemOne(context.Background(), "ping", map[string]Question{
+		"availability": Noul("Would this destroy availability?", "yes", "no"),
+		"decision":     Choice("Approve?", map[string]string{"approve": "yes", "reject": "no"}),
+	})
+	if err != nil {
+		t.Fatalf("complete answers should be accepted: %v", err)
+	}
+	if got.Noul("availability") != 0.02 {
+		t.Fatalf("noul = %v", got.Noul("availability"))
+	}
+}