diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a4c3fa..e12f2ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] - 2026-07-11 + +### Changed + +- `Result#pass?` now returns `false` when any metric errored during evaluation (fail-closed for CI gating) +- `Report#worst` now ranks results with no valid scores as worst instead of best + +### Added + +- `Result#errors` and `Result#valid?` expose per-metric judge failures +- `Result#to_h` and report JSON exports include metric error information +- Report summaries show a metric error count line when evaluation errors occurred + ## [0.2.0] - 2026-07-11 ### Changed diff --git a/lib/rubric_llm/report.rb b/lib/rubric_llm/report.rb index 5280618..214752e 100644 --- a/lib/rubric_llm/report.rb +++ b/lib/rubric_llm/report.rb @@ -16,7 +16,7 @@ def metric_stats end def worst(n) - results.sort_by { |r| r.overall || Float::INFINITY }.first(n) + results.sort_by { |r| r.overall || -Float::INFINITY }.first(n) end def failures(threshold: 0.8) @@ -34,6 +34,10 @@ def summary metric, stats[:mean], stats[:std], stats[:min], stats[:max], stats[:count]) end + error_count = error_counts.values.sum + affected_samples = results.count { |r| r.errors.any? } + lines << "Errors: #{error_count} metric errors across #{affected_samples} samples" if error_count.positive? + lines.join("\n") end @@ -72,6 +76,7 @@ def serializable_hash { summary: metric_stats, duration:, + errors: error_counts, results: results.map(&:to_h) } end @@ -80,6 +85,12 @@ def all_metric_names results.flat_map { |r| r.scores.keys }.uniq end + def error_counts + counts = Hash.new(0) + results.each { |r| r.errors.each_key { |name| counts[name] += 1 } } + counts + end + def compute_stats all_metric_names.each_with_object({}) do |metric, stats| values = results.filter_map { |r| r.scores[metric] } diff --git a/lib/rubric_llm/result.rb b/lib/rubric_llm/result.rb index 872157f..3146828 100644 --- a/lib/rubric_llm/result.rb +++ b/lib/rubric_llm/result.rb @@ -18,14 +18,26 @@ def overall end def pass?(threshold: 0.8) + return false unless valid? + score = overall return false if score.nil? score >= threshold end + def errors + details.each_with_object({}) do |(name, detail), acc| + acc[name] = detail[:error] if detail.is_a?(Hash) && detail.key?(:error) + end + end + + def valid? + errors.empty? + end + def to_h - { scores:, details:, overall: overall } + { scores:, details:, overall: overall, errors: } end private diff --git a/lib/rubric_llm/version.rb b/lib/rubric_llm/version.rb index e7f7818..5e614f9 100644 --- a/lib/rubric_llm/version.rb +++ b/lib/rubric_llm/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module RubricLLM - VERSION = "0.2.0" + VERSION = "0.3.0" end diff --git a/test/test_report.rb b/test/test_report.rb index 5aa7ad2..d984602 100644 --- a/test/test_report.rb +++ b/test/test_report.rb @@ -95,7 +95,7 @@ def test_scores_for assert_equal [0.9, 0.7, 0.5], scores end - def test_worst_sorts_nil_overall_to_end + def test_worst_sorts_nil_overall_first results_with_nil = [ RubricLLM::Result.new(scores: { a: nil }, details: {}, sample: { question: "nil_q" }), RubricLLM::Result.new(scores: { a: 0.3 }, details: {}, sample: { question: "low_q" }), @@ -104,6 +104,63 @@ def test_worst_sorts_nil_overall_to_end report = RubricLLM::Report.new(results: results_with_nil) worst = report.worst(1) - assert_equal "low_q", worst.first.sample[:question] + assert_equal "nil_q", worst.first.sample[:question] + end + + def test_summary_reports_metric_errors + results_with_errors = [ + RubricLLM::Result.new( + scores: { faithfulness: nil, relevance: 0.8 }, + details: { faithfulness: { error: "judge timeout" } }, + sample: { question: "q1" } + ), + RubricLLM::Result.new( + scores: { faithfulness: nil, relevance: nil }, + details: { faithfulness: { error: "judge timeout" }, relevance: { error: "bad response" } }, + sample: { question: "q2" } + ) + ] + report = RubricLLM::Report.new(results: results_with_errors) + + assert_includes report.summary, "Errors: 3 metric errors across 2 samples" + end + + def test_summary_omits_error_line_when_no_errors + refute_includes @report.summary, "Errors:" + end + + def test_worst_returns_errored_result_first + results_with_errors = [ + RubricLLM::Result.new( + scores: { a: nil }, + details: { a: { error: "boom" } }, + sample: { question: "errored_q" } + ), + RubricLLM::Result.new(scores: { a: 0.3 }, details: {}, sample: { question: "low_q" }) + ] + report = RubricLLM::Report.new(results: results_with_errors) + + assert_equal "errored_q", report.worst(1).first.sample[:question] + end + + def test_serializable_hash_includes_error_counts + results_with_errors = [ + RubricLLM::Result.new( + scores: { faithfulness: nil, relevance: 0.8 }, + details: { faithfulness: { error: "judge timeout" } }, + sample: { question: "q1" } + ), + RubricLLM::Result.new(scores: { faithfulness: 0.9, relevance: 0.7 }, details: {}, sample: { question: "q2" }) + ] + report = RubricLLM::Report.new(results: results_with_errors) + data = JSON.parse(report.to_json) + + assert_equal({ "faithfulness" => 1 }, data["errors"]) + end + + def test_serializable_hash_errors_empty_when_no_errors + data = JSON.parse(@report.to_json) + + assert_equal({}, data["errors"]) end end diff --git a/test/test_result.rb b/test/test_result.rb index 172657f..9dd8abd 100644 --- a/test/test_result.rb +++ b/test/test_result.rb @@ -58,5 +58,44 @@ def test_to_h assert_equal({ a: 0.9 }, hash[:scores]) assert_in_delta 0.9, hash[:overall] + assert_equal({}, hash[:errors]) + end + + def test_all_nil_scores_are_invalid_and_fail + result = RubricLLM::Result.new( + scores: { faithfulness: nil }, + details: { faithfulness: { error: "judge timeout" } } + ) + + assert_nil result.overall + refute_predicate result, :pass? + refute_predicate result, :valid? + assert_equal({ faithfulness: "judge timeout" }, result.errors) + end + + def test_partial_failure_fails_pass_despite_high_overall + result = RubricLLM::Result.new( + scores: { a: nil, b: 1.0 }, + details: { a: { error: "boom" } } + ) + + assert_in_delta 1.0, result.overall + refute_predicate result, :pass? + assert_equal({ a: "boom" }, result.errors) + end + + def test_valid_with_no_errors + result = RubricLLM::Result.new(scores: { a: 0.9 }, details: {}) + + assert_predicate result, :valid? + assert_equal({}, result.errors) + assert result.pass?(threshold: 0.8) + end + + def test_errors_ignores_non_hash_details + result = RubricLLM::Result.new(scores: { a: 0.9 }, details: { a: "some note" }) + + assert_equal({}, result.errors) + assert_predicate result, :valid? end end