diff --git a/.gitignore b/.gitignore index 7af11d3..d77d7a2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ doc/ .ruby-lsp/ *.gem Gemfile.lock +.omx/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..baba729 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +RubricLLM is a Ruby gem. Runtime code lives in `lib/rubric_llm/`, with the public entrypoint at `lib/rubric_llm.rb`. Metric implementations are in `lib/rubric_llm/metrics/`. Minitest helpers live in `lib/rubric_llm/minitest.rb`. Tests live under `test/`, with metric tests in `test/metrics/`. Project metadata is in `rubric_llm.gemspec`, `Gemfile`, and `Rakefile`. + +## Build, Test, and Development Commands + +- `bundle install` installs gem dependencies. +- `bundle exec rake test` runs the Minitest suite. +- `bundle exec rubocop` runs static style checks using `.rubocop.yml`. +- `bundle exec rake` runs the default gate: tests plus RuboCop. + +## Coding Style & Naming Conventions + +Use Ruby 3.4+ syntax and keep files `# frozen_string_literal: true`. Follow the `RubricLLM` namespace. Use snake_case filenames and method names. Keep metric classes small and focused under `RubricLLM::Metrics`. RuboCop enforces double-quoted strings, 140-character lines, and Minitest rules. + +## Testing Guidelines + +Tests use Minitest only. Name files `test/test_*.rb` or `test/metrics/test_*.rb`. Put shared setup in `test/test_helper.rb`. Stub LLM behavior through `RubyLLMStub`; tests must not make network calls. Add regression tests when touching judge parsing, retry/error handling, score aggregation, metric math, or Minitest assertions. + +## Commit & Pull Request Guidelines + +Recent history uses Conventional Commits, for example `feat: initialize rubric_llm gem` and `fix(core): harden judge evaluation regressions`. Keep subjects short, imperative, and scoped when useful. Pull requests should state behavior changes, public API impact, and verification commands run. + +## Security & Configuration Tips + +Do not commit provider API keys, recorded prompts containing secrets, or generated `.gem` files. Keep tests deterministic and offline. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ce2c9a..e0edecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.1.2] - 2026-04-30 + +### Added + +- Add standalone LLM-as-Judge examples for RAG scoring, batch reports, model comparison, custom metrics, and Minitest assertions + ## [0.1.1] - 2026-03-24 ### Fixed diff --git a/README.md b/README.md index aab9899..26caa9d 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ gem install rubric_llm require "rubric_llm" RubricLLM.configure do |c| - c.judge_model = "gpt-4o" + c.judge_model = "gpt-5.5" c.judge_provider = :openai end @@ -45,6 +45,9 @@ result.overall # => 0.94 result.pass? # => true ``` +For runnable LLM-as-Judge examples, including RAG scoring, batch pass/fail output, model comparison, custom metrics, and live +Minitest assertions, see [examples/README.md](examples/README.md). + ## Configuration ### Global @@ -350,3 +353,5 @@ Bug reports and pull requests are welcome on [GitHub](https://github.com/dpaluy/ ## License [MIT](LICENSE.txt) + +Supported by [Majestic Labs](https://majesticlabs.dev/). diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..647ba93 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,257 @@ +# RubricLLM Examples + +These examples are plain Ruby scripts for trying RubricLLM against live LLM-as-Judge calls. They are separate from the root +README so adoption examples can grow without making the main gem documentation noisy. + +## Outline + +- [Setup](#setup) +- [Judge vs Evaluated Model](#judge-vs-evaluated-model) +- [Example Guide](#example-guide) +- [`llm_as_judge_rag.rb`](#llm_as_judge_ragrb) +- [`llm_as_judge_batch.rb`](#llm_as_judge_batchrb) +- [`llm_as_judge_model_comparison.rb`](#llm_as_judge_model_comparisonrb) +- [`llm_as_judge_custom_metric.rb`](#llm_as_judge_custom_metricrb) +- [`llm_as_judge_minitest.rb`](#llm_as_judge_minitestrb) +- [Reading Batch Output](#reading-batch-output) + +## Setup + +Install dependencies from the project root: + +```bash +bundle install +``` + +The examples below use OpenAI through RubyLLM. Set an API key before running them: + +```bash +export OPENAI_API_KEY=... +``` + +Each script also wires `OPENAI_API_KEY` into `RubyLLM.configure(openai_api_key:)`, because RubyLLM expects provider keys on its +own configuration object. + +## Judge vs Evaluated Model + +RubricLLM does not call the model being evaluated. It scores answers that your app, prompt, model, or RAG pipeline already +produced. + +- **Judge model**: the model configured in `RubricLLM.configure`, for example `gpt-4.1`. This model reads the question, + answer, context, and ground truth, then returns scores. +- **Evaluated model/system**: the model, prompt, or application pipeline that produced the candidate answer. In these examples, + evaluated answers are pre-generated candidate outputs; RubricLLM decides which ones pass or need review. + +This separation is intentional. It lets you evaluate outputs from any source while keeping the judge model stable. + +## Example Guide + +All commands below should be run from the project root after `OPENAI_API_KEY` is available in the environment. You can either +export the key once or pass it inline: + +```bash +OPENAI_API_KEY=... bundle exec ruby examples/llm_as_judge_rag.rb +``` + +### `llm_as_judge_rag.rb` + +Run: + +```bash +bundle exec ruby examples/llm_as_judge_rag.rb +``` + +What it does: + +- Evaluates two pre-generated answers to the same RAG question. +- Uses `gpt-4.1` as the judge model. +- Labels the evaluated system as `acme-rag-v1`, but does not call that system. +- Sends each candidate answer through `faithfulness` and `correctness`. +- Prints each candidate's answer, per-metric scores, overall score, pass/review decision, and any judge-call errors. + +The candidate inputs are stored as an array: + +```ruby +candidate_answers = [ + { id: "candidate-a", answer: "..." }, + { id: "candidate-b", answer: "..." } +] +``` + +That shape is intentional. RubricLLM is not told which candidate is good or bad. The evaluator decides that from the answer, +context, ground truth, and metrics. + +Use this example when you want the smallest practical RAG evaluation: one question, one context set, one ground truth answer, +and multiple candidate answers. + +### `llm_as_judge_batch.rb` + +Run: + +```bash +bundle exec ruby examples/llm_as_judge_batch.rb +``` + +What it does: + +- Evaluates a small dataset of three RAG samples. +- Uses the same two metrics as the single RAG example: `faithfulness` and `correctness`. +- Runs the batch with `concurrency: 2`. +- Prints every sample answer with a `PASS` or `FAIL` decision, then prints aggregate stats, failures, and the worst sample. + +Each dataset row contains the fields RubricLLM needs for RAG judging: + +```ruby +{ + question: "...", + context: ["..."], + ground_truth: "...", + answer: "..." +} +``` + +The example includes two supported answers and one answer that contradicts the context. The failing machine-washing answer is +expected to score low because the context says not to machine-wash or tumble-dry the pack, while the answer says the opposite. + +Use this example when you want to evaluate many saved outputs from the same application or prompt and inspect aggregate quality. + +### `llm_as_judge_model_comparison.rb` + +Run: + +```bash +bundle exec ruby examples/llm_as_judge_model_comparison.rb +``` + +What it does: + +- Compares two evaluated systems: `acme-rag-v1` and `acme-rag-v2`. +- Uses one stable judge model for both systems. +- Starts with paired samples where each question has a baseline answer and a candidate answer. +- Builds two datasets, evaluates each dataset, then calls `RubricLLM.compare`. +- Prints a baseline report, a candidate report, and an A/B comparison table. + +The important idea is that the judge model stays fixed while the evaluated system changes. This lets you compare a baseline +prompt, model, retriever, or RAG pipeline against a candidate version without changing the scoring model. + +Use this example when you want to decide whether a new prompt/model/pipeline version is better than the current one across the +same questions. + +### `llm_as_judge_custom_metric.rb` + +Run: + +```bash +bundle exec ruby examples/llm_as_judge_custom_metric.rb +``` + +What it does: + +- Defines a custom metric named `HelpfulSupportTone`. +- Evaluates two pre-generated support answers with that metric. +- Scores whether each answer is clear, concise, and professionally helpful. +- Prints the custom metric score, judge reasoning, and pass/review decision for each candidate. + +The custom metric subclasses `RubricLLM::Metrics::Base`, defines a judge prompt, calls `judge_eval`, and normalizes the judge's +JSON response into: + +```ruby +{ + score: Float(result["score"]).clamp(0.0, 1.0), + details: { reasoning: result["reasoning"] } +} +``` + +This example is intentionally not a RAG correctness check. It evaluates support tone. A rude or dismissive answer should score +lower even if it contains some policy-relevant information. + +Use this example when built-in metrics are not enough and you need a domain-specific rubric such as support quality, brand tone, +safety policy compliance, or instruction-following. + +### `llm_as_judge_minitest.rb` + +Run: + +```bash +bundle exec ruby examples/llm_as_judge_minitest.rb +``` + +What it does: + +- Shows how to use RubricLLM from Minitest. +- Includes `RubricLLM::Assertions`. +- Calls `assert_faithful` to check that the answer is supported by context. +- Calls `assert_correct` to check that the answer matches the ground truth. +- Fails the test if the live judge score is below the assertion threshold. + +This is a live LLM-as-Judge smoke test, not an offline deterministic unit test. It is useful when you want a human-readable test +failure around an LLM output quality gate. For normal CI, keep in mind that it needs provider credentials, makes API calls, and +can vary by judge model/provider behavior. + +Use this example when you want to wrap RubricLLM checks in familiar Ruby test assertions for manual smoke tests, release gates, +or controlled evaluation jobs. + +## Reading Batch Output + +`llm_as_judge_batch.rb` evaluates three pre-generated RAG answers with one judge model and two metrics: + +- `faithfulness`: whether the answer is supported by the supplied context. +- `correctness`: whether the answer matches the supplied ground truth. + +The evaluated system label, such as `acme-rag-v1`, is only a description of where the candidate answers came from. The script +does not call that system. It only passes the stored `question`, `context`, `ground_truth`, and `answer` values to RubricLLM. + +The report summarizes each metric across the batch: + +```text +faithfulness mean=0.667 std=0.577 min=0.000 max=1.000 n=3 +correctness mean=0.667 std=0.577 min=0.000 max=1.000 n=3 +``` + +In the bundled sample data, two answers are designed to be supported and correct, while the machine-washing answer contradicts +both the context and the ground truth. A common result is therefore scores like `[1.0, 0.0, 1.0]` for each metric: + +```text +mean = (1.0 + 0.0 + 1.0) / 3 = 0.667 +``` + +`overall` is computed per sample as the average of that sample's metric scores: + +```text +overall = (faithfulness + correctness) / 2 +``` + +The failure output includes the underlying scores so the overall score is explainable: + +```text +Sample results: +1. PASS - What does the Acme Trail Pack warranty cover? + scores: overall=1.00 (faithfulness=1.00, correctness=1.00) + answer: The Acme Trail Pack has a lifetime warranty for manufacturing defects, excluding normal wear, misuse, and cosmetic damage. +2. FAIL - Can the Acme Trail Pack be machine-washed? + scores: overall=0.00 (faithfulness=0.00, correctness=0.00) + answer: Yes. The Acme Trail Pack can be machine-washed on a hot cycle and tumble-dried. +3. PASS - What is the Acme Trail Pack made from? + scores: overall=1.00 (faithfulness=1.00, correctness=1.00) + answer: It uses recycled nylon ripstop for the shell and recycled polyester for the lining. +``` + +The failures section repeats only the samples that did not pass: + +```text +Failures below 0.80: +- Can the Acme Trail Pack be machine-washed? + scores: overall=0.00 (faithfulness=0.00, correctness=0.00) +``` + +In that case, `overall=0.00` means the answer scored `0.00` for both metrics: + +```text +overall = (0.00 + 0.00) / 2 = 0.00 +``` + +The machine-washing answer appears under failures and as the worst sample because the example threshold is `0.80`. + +These examples make provider API calls and may incur cost. Scores can vary by model and provider behavior. For smoke tests, look +for the relative behavior: stronger candidate outputs should pass, and weaker outputs should score lower and be flagged for +review. diff --git a/examples/llm_as_judge_batch.rb b/examples/llm_as_judge_batch.rb new file mode 100644 index 0000000..3fd13a9 --- /dev/null +++ b/examples/llm_as_judge_batch.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +# Score a small RAG dataset and inspect aggregate LLM-as-Judge results. +# +# Run from the project root after configuring RubyLLM provider credentials: +# +# OPENAI_API_KEY=... bundle exec ruby examples/llm_as_judge_batch.rb + +require "rubric_llm" + +warn "OPENAI_API_KEY is not set. If RubyLLM is not configured another way, judge calls will fail." unless ENV.key?("OPENAI_API_KEY") + +JUDGE_MODEL = "gpt-4.1" +EVALUATED_SYSTEM = "acme-rag-v1 (pre-generated batch outputs)" +FAILURE_THRESHOLD = 0.8 + +RubyLLM.configure do |config| + config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) +end + +RubricLLM.configure do |config| + config.judge_model = JUDGE_MODEL + config.judge_provider = :openai +end + +metrics = [ + RubricLLM::Metrics::Faithfulness, + RubricLLM::Metrics::Correctness +] + +def format_score(score) + score.nil? ? "n/a" : format("%.2f", score) +end + +def score_summary(result) + metric_scores = result.scores.map do |metric, score| + "#{metric}=#{format_score(score)}" + end + + "overall=#{format_score(result.overall)} (#{metric_scores.join(", ")})" +end + +def decision_label(result, threshold) + result.pass?(threshold:) ? "PASS" : "FAIL" +end + +def print_sample_result(result, index, threshold) + puts "#{index}. #{decision_label(result, threshold)} - #{result.sample[:question]}" + puts " scores: #{score_summary(result)}" + puts " answer: #{result.sample[:answer]}" +end + +dataset = [ + { + question: "What does the Acme Trail Pack warranty cover?", + context: [ + "The Acme Trail Pack includes a lifetime warranty for manufacturing defects.", + "The warranty does not cover normal wear, misuse, or cosmetic damage." + ], + ground_truth: "The warranty covers manufacturing defects, but not normal wear, misuse, or cosmetic damage.", + answer: "The Acme Trail Pack has a lifetime warranty for manufacturing defects, excluding normal wear, misuse, and cosmetic damage." + }, + { + question: "Can the Acme Trail Pack be machine-washed?", + context: [ + "Clean the Acme Trail Pack with mild soap and cold water.", + "Do not machine-wash, bleach, or tumble-dry the pack." + ], + ground_truth: "No. It should be cleaned with mild soap and cold water, not machine-washed.", + answer: "Yes. The Acme Trail Pack can be machine-washed on a hot cycle and tumble-dried." + }, + { + question: "What is the Acme Trail Pack made from?", + context: [ + "The shell uses recycled nylon ripstop.", + "The lining uses recycled polyester." + ], + ground_truth: "The shell is recycled nylon ripstop, and the lining is recycled polyester.", + answer: "It uses recycled nylon ripstop for the shell and recycled polyester for the lining." + } +] + +report = RubricLLM.evaluate_batch(dataset, metrics:, concurrency: 2) + +puts "Judge model: #{JUDGE_MODEL}" +puts "Evaluated system: #{EVALUATED_SYSTEM}" +puts +puts report.summary +puts <<~TEXT + + How to read this: + - The judge model scores pre-generated answers from the evaluated system; this script does not call the evaluated system. + - faithfulness checks whether the answer is supported by the context. + - correctness checks whether the answer matches the ground truth. + - Each sample's overall score is the average of its metric scores, so faithfulness=0.00 and correctness=0.00 means overall=0.00. + - PASS means the sample's overall score is at least #{format("%.2f", FAILURE_THRESHOLD)}; FAIL means it is below that threshold. + - Report stats summarize each metric across all samples; results below #{format("%.2f", FAILURE_THRESHOLD)} are listed for review. +TEXT + +puts "\nSample results:" +report.results.each_with_index do |result, index| + print_sample_result(result, index + 1, FAILURE_THRESHOLD) +end + +puts "\nFailures below #{format("%.2f", FAILURE_THRESHOLD)}:" +report.failures(threshold: FAILURE_THRESHOLD).each do |result| + puts "- #{result.sample[:question]}" + puts " scores: #{score_summary(result)}" +end + +puts "\nWorst sample:" +worst = report.worst(1).first +puts worst.sample[:question] +puts "scores: #{score_summary(worst)}" diff --git a/examples/llm_as_judge_custom_metric.rb b/examples/llm_as_judge_custom_metric.rb new file mode 100644 index 0000000..0270dd7 --- /dev/null +++ b/examples/llm_as_judge_custom_metric.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# Define a custom LLM-as-Judge metric for product-support answers. +# +# Run from the project root after configuring RubyLLM provider credentials: +# +# OPENAI_API_KEY=... bundle exec ruby examples/llm_as_judge_custom_metric.rb + +require "rubric_llm" + +warn "OPENAI_API_KEY is not set. If RubyLLM is not configured another way, judge calls will fail." unless ENV.key?("OPENAI_API_KEY") + +JUDGE_MODEL = "gpt-4.1" +EVALUATED_SYSTEM = "support-answer-generator-v1 (pre-generated answers)" + +question = "Can I return a backpack after using it on a trip?" +candidate_answers = [ + { + id: "candidate-a", + answer: "Used gear cannot be returned unless there is a manufacturing defect. " \ + "If you think your pack is defective, send us photos and your order number so we can review it." + }, + { + id: "candidate-b", + answer: "No. You used it, so that is your problem. Do not contact support about this again." + } +] + +RubyLLM.configure do |config| + config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) +end + +RubricLLM.configure do |config| + config.judge_model = JUDGE_MODEL + config.judge_provider = :openai +end + +class HelpfulSupportTone < RubricLLM::Metrics::Base + SYSTEM_PROMPT = <<~PROMPT + You are an evaluation judge. Score whether the answer is clear, concise, and professionally helpful for customer support. + + Respond with JSON only: + { + "score": , + "reasoning": "" + } + PROMPT + + def call(question:, answer:, **) + result = judge_eval( + system_prompt: SYSTEM_PROMPT, + user_prompt: "Customer question: #{question}\n\nSupport answer: #{answer}" + ) + + return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] + + { + score: Float(result["score"]).clamp(0.0, 1.0), + details: { reasoning: result["reasoning"] } + } + end +end + +puts "Judge model: #{JUDGE_MODEL}" +puts "Evaluated system: #{EVALUATED_SYSTEM}" + +candidate_answers.each do |candidate| + answer = candidate.fetch(:answer) + result = RubricLLM.evaluate(question:, answer:, metrics: [HelpfulSupportTone]) + score = result.scores[:helpful_support_tone] + + puts "\n#{candidate.fetch(:id)}" + puts "answer: #{answer}" + puts "helpful support tone: #{score.nil? ? "n/a" : format("%.2f", score)}" + puts "reasoning: #{result.details.dig(:helpful_support_tone, :reasoning) || "n/a"}" + puts result.pass?(threshold: 0.8) ? "decision: pass" : "decision: review" +end diff --git a/examples/llm_as_judge_minitest.rb b/examples/llm_as_judge_minitest.rb new file mode 100644 index 0000000..67de635 --- /dev/null +++ b/examples/llm_as_judge_minitest.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Use RubricLLM's Minitest assertions as a live LLM-as-Judge smoke test. +# +# Run from the project root after configuring RubyLLM provider credentials: +# +# OPENAI_API_KEY=... bundle exec ruby examples/llm_as_judge_minitest.rb + +require "minitest/autorun" +require "rubric_llm/minitest" + +warn "OPENAI_API_KEY is not set. If RubyLLM is not configured another way, judge calls will fail." unless ENV.key?("OPENAI_API_KEY") + +JUDGE_MODEL = "gpt-4.1" +EVALUATED_SYSTEM = "acme-rag-v1 (pre-generated answer under test)" + +RubyLLM.configure do |config| + config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) +end + +RubricLLM.configure do |config| + config.judge_model = JUDGE_MODEL + config.judge_provider = :openai +end + +puts "Judge model: #{JUDGE_MODEL}" +puts "Evaluated system: #{EVALUATED_SYSTEM}" + +class TrailPackAnswerTest < Minitest::Test + include RubricLLM::Assertions + + def setup + @question = "What does the Acme Trail Pack warranty cover?" + @context = [ + "The Acme Trail Pack includes a lifetime warranty for manufacturing defects.", + "The warranty does not cover normal wear, misuse, or cosmetic damage." + ] + @ground_truth = "The warranty covers manufacturing defects, but not normal wear, misuse, or cosmetic damage." + @answer = "The Acme Trail Pack has a lifetime warranty for manufacturing defects. " \ + "It does not cover normal wear, misuse, or cosmetic damage." + end + + def test_answer_is_faithful_to_context + assert_faithful @answer, @context, question: @question, threshold: 0.8 + end + + def test_answer_is_correct_for_ground_truth + assert_correct @answer, @ground_truth, question: @question, threshold: 0.8 + end +end diff --git a/examples/llm_as_judge_model_comparison.rb b/examples/llm_as_judge_model_comparison.rb new file mode 100644 index 0000000..84693ab --- /dev/null +++ b/examples/llm_as_judge_model_comparison.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +# Compare outputs from two evaluated RAG systems with the same judge model. +# +# RubricLLM does not call the evaluated models here. The two datasets contain +# pre-generated answers from each system. The judge model scores those answers. +# +# Run from the project root after configuring RubyLLM provider credentials: +# +# OPENAI_API_KEY=... bundle exec ruby examples/llm_as_judge_model_comparison.rb + +require "rubric_llm" + +warn "OPENAI_API_KEY is not set. If RubyLLM is not configured another way, judge calls will fail." unless ENV.key?("OPENAI_API_KEY") + +JUDGE_MODEL = "gpt-4.1" +BASELINE_SYSTEM = "acme-rag-v1 (pre-generated baseline outputs)" +CANDIDATE_SYSTEM = "acme-rag-v2 (pre-generated candidate outputs)" + +RubyLLM.configure do |config| + config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) +end + +RubricLLM.configure do |config| + config.judge_model = JUDGE_MODEL + config.judge_provider = :openai +end + +metrics = [ + RubricLLM::Metrics::Faithfulness, + RubricLLM::Metrics::Correctness +] + +samples = [ + { + question: "What does the Acme Trail Pack warranty cover?", + context: [ + "The Acme Trail Pack includes a lifetime warranty for manufacturing defects.", + "The warranty does not cover normal wear, misuse, or cosmetic damage." + ], + ground_truth: "The warranty covers manufacturing defects, but not normal wear, misuse, or cosmetic damage.", + baseline_answer: "The Acme Trail Pack has a lifetime warranty for manufacturing defects, and it also covers normal wear.", + candidate_answer: "The Acme Trail Pack has a lifetime warranty for manufacturing defects, " \ + "excluding normal wear, misuse, and cosmetic damage." + }, + { + question: "Can the Acme Trail Pack be machine-washed?", + context: [ + "Clean the Acme Trail Pack with mild soap and cold water.", + "Do not machine-wash, bleach, or tumble-dry the pack." + ], + ground_truth: "No. It should be cleaned with mild soap and cold water, not machine-washed.", + baseline_answer: "Clean it with mild soap and cold water, then tumble-dry it on low heat.", + candidate_answer: "No. Clean it with mild soap and cold water, and do not machine-wash or tumble-dry it." + }, + { + question: "What is the Acme Trail Pack made from?", + context: [ + "The shell uses recycled nylon ripstop.", + "The lining uses recycled polyester." + ], + ground_truth: "The shell is recycled nylon ripstop, and the lining is recycled polyester.", + baseline_answer: "The shell uses recycled nylon ripstop, and the lining is cotton canvas.", + candidate_answer: "It uses recycled nylon ripstop for the shell and recycled polyester for the lining." + } +] + +baseline_dataset = samples.map { |sample| sample.except(:baseline_answer, :candidate_answer).merge(answer: sample[:baseline_answer]) } +candidate_dataset = samples.map { |sample| sample.except(:baseline_answer, :candidate_answer).merge(answer: sample[:candidate_answer]) } + +baseline_report = RubricLLM.evaluate_batch(baseline_dataset, metrics:, concurrency: 2) +candidate_report = RubricLLM.evaluate_batch(candidate_dataset, metrics:, concurrency: 2) +comparison = RubricLLM.compare(baseline_report, candidate_report) + +puts "Judge model: #{JUDGE_MODEL}" +puts "Baseline evaluated system: #{BASELINE_SYSTEM}" +puts "Candidate evaluated system: #{CANDIDATE_SYSTEM}" + +puts "\nBaseline report:" +puts baseline_report.summary + +puts "\nCandidate report:" +puts candidate_report.summary + +puts "\nComparison:" +puts comparison.summary diff --git a/examples/llm_as_judge_rag.rb b/examples/llm_as_judge_rag.rb new file mode 100644 index 0000000..f848c91 --- /dev/null +++ b/examples/llm_as_judge_rag.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +# Evaluate two RAG answers with live LLM-as-Judge calls. +# +# Run from the project root after configuring RubyLLM provider credentials, for example: +# +# OPENAI_API_KEY=... bundle exec ruby examples/llm_as_judge_rag.rb +# +# Scores vary by judge model. The candidate outputs intentionally differ in +# quality; let the judge scores decide which one passes and which needs review. + +require "rubric_llm" + +warn "OPENAI_API_KEY is not set. If RubyLLM is not configured another way, judge calls will fail." unless ENV.key?("OPENAI_API_KEY") + +JUDGE_MODEL = "gpt-4.1" +EVALUATED_SYSTEM = "acme-rag-v1 (pre-generated candidate answers)" + +RubyLLM.configure do |config| + config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) +end + +RubricLLM.configure do |config| + config.judge_model = JUDGE_MODEL + config.judge_provider = :openai +end + +EVALUATION_METRICS = [ + RubricLLM::Metrics::Faithfulness, + RubricLLM::Metrics::Correctness +].freeze + +def format_score(score) + score.nil? ? "n/a" : format("%.2f", score) +end + +def metric_error(details) + return unless details.is_a?(Hash) + + details[:error] || details["error"] || metric_error(details[:details] || details["details"]) +end + +def print_metric_errors(result) + errors = result.details.filter_map do |metric, details| + error = metric_error(details) + [metric, error] if error + end + + return if errors.empty? + + puts "errors:" + errors.each { |metric, error| puts " #{metric}: #{error}" } +end + +question = "What does the Acme Trail Pack warranty cover?" +context = [ + "The Acme Trail Pack includes a lifetime warranty for manufacturing defects.", + "The warranty does not cover normal wear, misuse, or cosmetic damage." +] +ground_truth = "The pack has a lifetime warranty for manufacturing defects, " \ + "but not normal wear, misuse, or cosmetic damage." + +candidate_answers = [ + { + id: "candidate-a", + answer: "The Acme Trail Pack has a lifetime warranty for manufacturing defects. " \ + "It does not cover normal wear, misuse, or cosmetic damage." + }, + { + id: "candidate-b", + answer: "The Acme Trail Pack has a lifetime warranty for manufacturing defects. " \ + "It also covers airline damage and cosmetic zipper repairs." + } +] + +puts "Judge model: #{JUDGE_MODEL}" +puts "Evaluated system: #{EVALUATED_SYSTEM}" + +candidate_answers.each do |candidate| + answer = candidate.fetch(:answer) + result = RubricLLM.evaluate(question:, answer:, context:, ground_truth:, metrics: EVALUATION_METRICS) + + puts "\n#{candidate.fetch(:id)}" + puts "answer: #{answer}" + puts "overall: #{format_score(result.overall)}" + puts "faithfulness: #{format_score(result.faithfulness)}" + puts "correctness: #{format_score(result.correctness)}" + puts result.pass?(threshold: 0.8) ? "decision: pass" : "decision: review" + print_metric_errors(result) +end diff --git a/lib/rubric_llm/version.rb b/lib/rubric_llm/version.rb index ac104dd..f8ffbb9 100644 --- a/lib/rubric_llm/version.rb +++ b/lib/rubric_llm/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module RubricLLM - VERSION = "0.1.1" + VERSION = "0.1.2" end diff --git a/rubric_llm.gemspec b/rubric_llm.gemspec index b5428ca..d38591b 100644 --- a/rubric_llm.gemspec +++ b/rubric_llm.gemspec @@ -29,7 +29,7 @@ Gem::Specification.new do |spec| f.start_with?(*%w[ test/ spec/ bin/ Gemfile .gitignore .github/ .rubocop.yml docs/ .agents/ AGENTS.md CLAUDE.md Rakefile .yardopts - .ruby-version .tool-versions skills/ + .ruby-version .tool-versions skills/ examples/ ]) end end