fix(core): make evaluation errors visible in results and reports

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0] - 2026-07-11
+
+### Changed
+
+- `Result#pass?` now returns `false` when any metric errored during evaluation (fail-closed for CI gating)
+- `Report#worst` now ranks results with no valid scores as worst instead of best
+
+### Added
+
+- `Result#errors` and `Result#valid?` expose per-metric judge failures
+- `Result#to_h` and report JSON exports include metric error information
+- Report summaries show a metric error count line when evaluation errors occurred
+
 ## [0.2.0] - 2026-07-11
 
 ### Changed

lib/rubric_llm/report.rb

@@ -16,7 +16,7 @@ def metric_stats
     end
 
     def worst(n)
-      results.sort_by { |r| r.overall || Float::INFINITY }.first(n)
+      results.sort_by { |r| r.overall || -Float::INFINITY }.first(n)
     end
 
     def failures(threshold: 0.8)
@@ -34,6 +34,10 @@ def summary
                         metric, stats[:mean], stats[:std], stats[:min], stats[:max], stats[:count])
       end
 
+      error_count = error_counts.values.sum
+      affected_samples = results.count { |r| r.errors.any? }
+      lines << "Errors: #{error_count} metric errors across #{affected_samples} samples" if error_count.positive?
+
       lines.join("\n")
     end
 
@@ -72,6 +76,7 @@ def serializable_hash
       {
         summary: metric_stats,
         duration:,
+        errors: error_counts,
         results: results.map(&:to_h)
       }
     end
@@ -80,6 +85,12 @@ def all_metric_names
       results.flat_map { |r| r.scores.keys }.uniq
     end
 
+    def error_counts
+      counts = Hash.new(0)
+      results.each { |r| r.errors.each_key { |name| counts[name] += 1 } }
+      counts
+    end
+
     def compute_stats
       all_metric_names.each_with_object({}) do |metric, stats|
         values = results.filter_map { |r| r.scores[metric] }

lib/rubric_llm/result.rb

@@ -18,14 +18,26 @@ def overall
     end
 
     def pass?(threshold: 0.8)
+      return false unless valid?
+
       score = overall
       return false if score.nil?
 
       score >= threshold
     end
 
+    def errors
+      details.each_with_object({}) do |(name, detail), acc|
+        acc[name] = detail[:error] if detail.is_a?(Hash) && detail.key?(:error)
+      end
+    end
+
+    def valid?
+      errors.empty?
+    end
+
     def to_h
-      { scores:, details:, overall: overall }
+      { scores:, details:, overall: overall, errors: }
     end
 
     private

lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

test/test_report.rb

@@ -95,7 +95,7 @@ def test_scores_for
     assert_equal [0.9, 0.7, 0.5], scores
   end
 
-  def test_worst_sorts_nil_overall_to_end
+  def test_worst_sorts_nil_overall_first
     results_with_nil = [
       RubricLLM::Result.new(scores: { a: nil }, details: {}, sample: { question: "nil_q" }),
       RubricLLM::Result.new(scores: { a: 0.3 }, details: {}, sample: { question: "low_q" }),
@@ -104,6 +104,63 @@ def test_worst_sorts_nil_overall_to_end
     report = RubricLLM::Report.new(results: results_with_nil)
     worst = report.worst(1)
 
-    assert_equal "low_q", worst.first.sample[:question]
+    assert_equal "nil_q", worst.first.sample[:question]
+  end
+
+  def test_summary_reports_metric_errors
+    results_with_errors = [
+      RubricLLM::Result.new(
+        scores: { faithfulness: nil, relevance: 0.8 },
+        details: { faithfulness: { error: "judge timeout" } },
+        sample: { question: "q1" }
+      ),
+      RubricLLM::Result.new(
+        scores: { faithfulness: nil, relevance: nil },
+        details: { faithfulness: { error: "judge timeout" }, relevance: { error: "bad response" } },
+        sample: { question: "q2" }
+      )
+    ]
+    report = RubricLLM::Report.new(results: results_with_errors)
+
+    assert_includes report.summary, "Errors: 3 metric errors across 2 samples"
+  end
+
+  def test_summary_omits_error_line_when_no_errors
+    refute_includes @report.summary, "Errors:"
+  end
+
+  def test_worst_returns_errored_result_first
+    results_with_errors = [
+      RubricLLM::Result.new(
+        scores: { a: nil },
+        details: { a: { error: "boom" } },
+        sample: { question: "errored_q" }
+      ),
+      RubricLLM::Result.new(scores: { a: 0.3 }, details: {}, sample: { question: "low_q" })
+    ]
+    report = RubricLLM::Report.new(results: results_with_errors)
+
+    assert_equal "errored_q", report.worst(1).first.sample[:question]
+  end
+
+  def test_serializable_hash_includes_error_counts
+    results_with_errors = [
+      RubricLLM::Result.new(
+        scores: { faithfulness: nil, relevance: 0.8 },
+        details: { faithfulness: { error: "judge timeout" } },
+        sample: { question: "q1" }
+      ),
+      RubricLLM::Result.new(scores: { faithfulness: 0.9, relevance: 0.7 }, details: {}, sample: { question: "q2" })
+    ]
+    report = RubricLLM::Report.new(results: results_with_errors)
+    data = JSON.parse(report.to_json)
+
+    assert_equal({ "faithfulness" => 1 }, data["errors"])
+  end
+
+  def test_serializable_hash_errors_empty_when_no_errors
+    data = JSON.parse(@report.to_json)
+
+    assert_equal({}, data["errors"])
   end
 end

test/test_result.rb

@@ -58,5 +58,44 @@ def test_to_h
 
     assert_equal({ a: 0.9 }, hash[:scores])
     assert_in_delta 0.9, hash[:overall]
+    assert_equal({}, hash[:errors])
+  end
+
+  def test_all_nil_scores_are_invalid_and_fail
+    result = RubricLLM::Result.new(
+      scores: { faithfulness: nil },
+      details: { faithfulness: { error: "judge timeout" } }
+    )
+
+    assert_nil result.overall
+    refute_predicate result, :pass?
+    refute_predicate result, :valid?
+    assert_equal({ faithfulness: "judge timeout" }, result.errors)
+  end
+
+  def test_partial_failure_fails_pass_despite_high_overall
+    result = RubricLLM::Result.new(
+      scores: { a: nil, b: 1.0 },
+      details: { a: { error: "boom" } }
+    )
+
+    assert_in_delta 1.0, result.overall
+    refute_predicate result, :pass?
+    assert_equal({ a: "boom" }, result.errors)
+  end
+
+  def test_valid_with_no_errors
+    result = RubricLLM::Result.new(scores: { a: 0.9 }, details: {})
+
+    assert_predicate result, :valid?
+    assert_equal({}, result.errors)
+    assert result.pass?(threshold: 0.8)
+  end
+
+  def test_errors_ignores_non_hash_details
+    result = RubricLLM::Result.new(scores: { a: 0.9 }, details: { a: "some note" })
+
+    assert_equal({}, result.errors)
+    assert_predicate result, :valid?
   end
 end