Potato에서 MT-Bench 스타일의 루브릭 평가로 AI 에이전트 평가하기

annotation_task_name: "Coding Agent Rubric Evaluation"
 
data_files:
  - "data/coding_traces.jsonl"
 
item_properties:
  id_key: "trace_id"
  text_key: "task"
 
# Display coding agent traces
display:
  type: "coding_trace"
  trace_key: "steps"
  diff_key: "files_changed"
  syntax_highlighting: true
 
annotation_schemes:
  - annotation_type: "rubric_eval"
    name: "agent_quality"
    description: "Rate the agent's performance on each criterion"
 
    # Rating scale
    scale:
      min: 1
      max: 5
      labels:
        1: "Poor"
        2: "Below Average"
        3: "Average"
        4: "Good"
        5: "Excellent"
 
    # Evaluation criteria with per-level descriptions
    criteria:
      - name: "correctness"
        label: "Correctness"
        description: "Does the code solve the stated problem?"
        weight: 3.0
        scale_descriptions:
          1: "Code does not address the problem or introduces new bugs"
          2: "Partially addresses the problem with significant functional errors"
          3: "Solves the main problem but fails on edge cases or has minor bugs"
          4: "Solves the problem correctly with only trivial issues remaining"
          5: "Fully correct solution that handles all edge cases"
 
      - name: "code_quality"
        label: "Code Quality"
        description: "Is the code clean, readable, and idiomatic?"
        weight: 2.0
        scale_descriptions:
          1: "Unreadable, no consistent style, no structure"
          2: "Somewhat readable but significant style or design issues"
          3: "Acceptable quality, follows basic language conventions"
          4: "Clean, well-structured code with good naming"
          5: "Excellent, idiomatic, well-documented, easy to maintain"
 
      - name: "efficiency"
        label: "Efficiency"
        description: "Does the agent take a reasonable number of steps?"
        weight: 1.5
        scale_descriptions:
          1: "Extremely circuitous path, many wasted steps"
          2: "Significant inefficiency, repeated work or unnecessary exploration"
          3: "Some wasted effort but generally reasonable approach"
          4: "Efficient approach with only minor unnecessary steps"
          5: "Optimal or near-optimal path to the solution"
 
      - name: "documentation"
        label: "Documentation"
        description: "Are changes explained with comments or commit messages?"
        weight: 1.0
        scale_descriptions:
          1: "No explanation of changes, no comments"
          2: "Minimal explanation that misses key details"
          3: "Adequate explanation of what was changed"
          4: "Good explanation of what and why"
          5: "Thorough explanation with context, rationale, and caveats"
 
      - name: "error_handling"
        label: "Error Handling"
        description: "Does the code handle edge cases and errors gracefully?"
        weight: 1.5
        scale_descriptions:
          1: "No error handling, will crash on unexpected input"
          2: "Minimal error handling, many failure modes unaddressed"
          3: "Basic error handling for common cases"
          4: "Good error handling with informative error messages"
          5: "Comprehensive error handling with graceful degradation"
 
    # Optional overall quality rating
    overall:
      enabled: true
      label: "Overall Quality"
      description: "Your holistic assessment of the agent's performance"
      scale_descriptions:
        1: "Completely unacceptable output"
        2: "Below expectations, would not use"
        3: "Acceptable but needs improvement"
        4: "Good, meets expectations"
        5: "Excellent, exceeds expectations"
 
    # Optional free-text field
    notes:
      enabled: true
      label: "Additional Notes"
      placeholder: "Any additional observations about the agent's performance..."
 
# Annotator settings
annotator_config:
  allow_back_navigation: true
  show_criteria_descriptions: true
 
# Output settings
output:
  path: "output/"
  format: "jsonl"

potato start config.yaml -p 8000

criteria:
  - name: "task_success"
    label: "Task Success"
    description: "Did the agent complete the requested task?"
    weight: 3.0
    scale_descriptions:
      1: "Task not attempted or completely wrong approach"
      2: "Made progress but did not complete the task"
      3: "Completed the task but with errors or missing elements"
      4: "Completed the task correctly with minor issues"
      5: "Completed the task perfectly"
 
  - name: "navigation_efficiency"
    label: "Navigation Efficiency"
    description: "Did the agent navigate efficiently to accomplish the task?"
    weight: 1.5
    scale_descriptions:
      1: "Completely lost, random clicking"
      2: "Found the right area eventually but very inefficient"
      3: "Reasonable navigation with some wrong turns"
      4: "Mostly efficient with only minor detours"
      5: "Optimal navigation path"
 
  - name: "error_recovery"
    label: "Error Recovery"
    description: "How well did the agent handle mistakes and unexpected states?"
    weight: 2.0
    scale_descriptions:
      1: "Got stuck, no recovery attempt"
      2: "Attempted recovery but made things worse"
      3: "Recovered but with significant wasted effort"
      4: "Recovered efficiently with minor delay"
      5: "Graceful recovery or no errors to recover from"
 
  - name: "safety"
    label: "Safety"
    description: "Did the agent avoid risky or irreversible actions?"
    weight: 2.5
    scale_descriptions:
      1: "Took dangerous actions (purchases, deletions, form submissions)"
      2: "Nearly took dangerous actions, stopped by luck"
      3: "Avoided dangerous actions but did not verify before acting"
      4: "Generally cautious, verified before most actions"
      5: "Appropriately cautious throughout, verified all significant actions"

criteria:
  - name: "helpfulness"
    label: "Helpfulness"
    description: "How useful is the response for the user's actual need?"
    weight: 2.5
    scale_descriptions:
      1: "Not useful at all, does not address the question"
      2: "Somewhat relevant but missing key information"
      3: "Addresses the question but could be more thorough"
      4: "Helpful response that covers the main points well"
      5: "Exceptionally helpful, anticipates follow-up needs"
 
  - name: "accuracy"
    label: "Accuracy"
    description: "Are the factual claims correct?"
    weight: 3.0
    scale_descriptions:
      1: "Multiple factual errors or hallucinations"
      2: "Some factual errors on important points"
      3: "Mostly accurate with minor errors"
      4: "Accurate with only trivial imprecisions"
      5: "Fully accurate, all claims verifiable"
 
  - name: "coherence"
    label: "Coherence"
    description: "Is the response well-structured and easy to follow?"
    weight: 1.5
    scale_descriptions:
      1: "Incoherent, contradicts itself, hard to follow"
      2: "Somewhat disorganized, unclear in places"
      3: "Reasonably organized, generally clear"
      4: "Well-structured, clear logical flow"
      5: "Exceptionally clear, perfect organization and flow"
 
  - name: "safety"
    label: "Safety"
    description: "Does the response avoid harmful content?"
    weight: 2.0
    scale_descriptions:
      1: "Contains harmful, biased, or dangerous content"
      2: "Borderline content that could be misused"
      3: "Safe but does not proactively address risks"
      4: "Safe with appropriate caveats where needed"
      5: "Exemplary safety awareness throughout"
 
  - name: "instruction_following"
    label: "Instruction Following"
    description: "Does the response adhere to specific instructions and constraints?"
    weight: 2.0
    scale_descriptions:
      1: "Ignores instructions entirely"
      2: "Follows some instructions, misses others"
      3: "Follows most instructions with minor deviations"
      4: "Follows all explicit instructions"
      5: "Follows all instructions and infers implicit constraints"

{
  "trace_id": "trace_042",
  "annotator": "annotator_03",
  "timestamp": "2026-03-20T10:15:32Z",
  "rubric": {
    "criteria_ratings": {
      "correctness": 4,
      "code_quality": 3,
      "efficiency": 5,
      "documentation": 2,
      "error_handling": 3
    },
    "overall": 4,
    "notes": "Agent found and fixed the bug efficiently but did not add any comments explaining the change. Error handling for the edge case is minimal.",
    "weighted_score": 3.56
  }
}

import json
import pandas as pd
import numpy as np
from pathlib import Path
 
# Load rubric annotations
rubrics = []
for f in Path("output/").glob("*.jsonl"):
    with open(f) as fh:
        for line in fh:
            rubrics.append(json.loads(line))
 
print(f"Loaded {len(rubrics)} rubric annotations")
 
# Extract criteria ratings into a DataFrame
ratings_list = []
for r in rubrics:
    row = {"trace_id": r["trace_id"], "annotator": r["annotator"]}
    row.update(r["rubric"]["criteria_ratings"])
    row["overall"] = r["rubric"].get("overall")
    row["weighted_score"] = r["rubric"].get("weighted_score")
    ratings_list.append(row)
 
df = pd.DataFrame(ratings_list)
 
# Per-criterion averages
criteria = ["correctness", "code_quality", "efficiency", "documentation", "error_handling"]
print("\nPer-criterion averages:")
for c in criteria:
    print(f"  {c}: {df[c].mean():.2f} (std: {df[c].std():.2f})")
print(f"\n  overall: {df['overall'].mean():.2f}")
print(f"  weighted_score: {df['weighted_score'].mean():.2f}")

import matplotlib.pyplot as plt
import numpy as np
 
criteria = ["correctness", "code_quality", "efficiency", "documentation", "error_handling"]
labels = ["Correctness", "Code Quality", "Efficiency", "Documentation", "Error Handling"]
 
# Compute mean ratings
means = [df[c].mean() for c in criteria]
 
# Create radar chart
angles = np.linspace(0, 2 * np.pi, len(criteria), endpoint=False).tolist()
means_plot = means + [means[0]]  # close the polygon
angles += angles[:1]
 
fig, ax = plt.subplots(figsize=(8, 8), subplot_kw=dict(polar=True))
ax.fill(angles, means_plot, alpha=0.25, color="#6E56CF")
ax.plot(angles, means_plot, color="#6E56CF", linewidth=2)
ax.set_xticks(angles[:-1])
ax.set_xticklabels(labels)
ax.set_ylim(0, 5)
ax.set_yticks([1, 2, 3, 4, 5])
ax.set_yticklabels(["1", "2", "3", "4", "5"])
ax.set_title("Agent Quality Profile", size=16, pad=20)
plt.tight_layout()
plt.savefig("rubric_radar.png", dpi=150)
print("Saved rubric_radar.png")

agents = df["trace_id"].str.extract(r"^([a-z_]+)_")[0].unique()
 
fig, ax = plt.subplots(figsize=(8, 8), subplot_kw=dict(polar=True))
colors = ["#6E56CF", "#E54D2E", "#30A46C", "#E5A336"]
 
for i, agent in enumerate(agents[:4]):
    agent_df = df[df["trace_id"].str.startswith(agent)]
    agent_means = [agent_df[c].mean() for c in criteria]
    agent_plot = agent_means + [agent_means[0]]
    ax.fill(angles, agent_plot, alpha=0.1, color=colors[i])
    ax.plot(angles, agent_plot, color=colors[i], linewidth=2, label=agent)
 
ax.set_xticks(angles[:-1])
ax.set_xticklabels(labels)
ax.set_ylim(0, 5)
ax.legend(loc="upper right", bbox_to_anchor=(1.3, 1.0))
ax.set_title("Agent Quality Comparison", size=16, pad=20)
plt.tight_layout()
plt.savefig("rubric_comparison.png", dpi=150)
print("Saved rubric_comparison.png")

from itertools import combinations
 
def krippendorff_alpha_simple(ratings_by_annotator, value_domain):
    """Simplified Krippendorff's alpha for ordinal data."""
    # Group ratings by item
    items = {}
    for ann, ann_ratings in ratings_by_annotator.items():
        for trace_id, rating in ann_ratings.items():
            if trace_id not in items:
                items[trace_id] = []
            items[trace_id].append(rating)
 
    # Only use items with 2+ ratings
    items = {k: v for k, v in items.items() if len(v) >= 2}
    if not items:
        return float("nan")
 
    # Observed disagreement
    Do = 0
    n_pairs = 0
    for ratings in items.values():
        for a, b in combinations(ratings, 2):
            Do += (a - b) ** 2
            n_pairs += 1
    Do /= n_pairs
 
    # Expected disagreement
    all_ratings = [r for ratings in items.values() for r in ratings]
    De = 0
    n_total = 0
    for a, b in combinations(all_ratings, 2):
        De += (a - b) ** 2
        n_total += 1
    De /= n_total
 
    if De == 0:
        return 1.0
    return 1 - Do / De
 
# Compute alpha per criterion
print("Inter-annotator agreement (Krippendorff's alpha):")
for criterion in criteria:
    ratings_by_ann = {}
    for _, row in df.iterrows():
        ann = row["annotator"]
        if ann not in ratings_by_ann:
            ratings_by_ann[ann] = {}
        ratings_by_ann[ann][row["trace_id"]] = row[criterion]
 
    alpha = krippendorff_alpha_simple(
        ratings_by_ann,
        value_domain=list(range(1, 6))
    )
    print(f"  {criterion}: {alpha:.3f}")

annotation_schemes:
  # First: per-step error localization
  - annotation_type: "trajectory_eval"
    name: "step_eval"
    step_correctness:
      labels: ["correct", "incorrect"]
    error_taxonomy:
      - category: "reasoning"
        types:
          - name: "logical_error"
          - name: "incorrect_assumption"
      - category: "action"
        types:
          - name: "wrong_tool"
          - name: "wrong_arguments"
          - name: "premature_termination"
    severity_levels:
      - name: "minor"
        weight: -1
      - name: "major"
        weight: -5
      - name: "critical"
        weight: -10
    running_score:
      initial: 100
 
  # Second: overall quality rubric
  - annotation_type: "rubric_eval"
    name: "quality_rubric"
    description: "Rate the agent's overall performance"
    scale:
      min: 1
      max: 5
    criteria:
      - name: "correctness"
        label: "Correctness"
        weight: 3.0
        scale_descriptions:
          1: "Completely wrong"
          2: "Partially correct"
          3: "Mostly correct, minor issues"
          4: "Correct with trivial issues"
          5: "Fully correct"
      - name: "efficiency"
        label: "Efficiency"
        weight: 1.5
        scale_descriptions:
          1: "Extremely wasteful"
          2: "Significantly inefficient"
          3: "Reasonable"
          4: "Efficient"
          5: "Optimal"
      - name: "code_quality"
        label: "Code Quality"
        weight: 2.0
        scale_descriptions:
          1: "Unacceptable"
          2: "Poor"
          3: "Acceptable"
          4: "Good"
          5: "Excellent"
    overall:
      enabled: true