AI agents की साथ-साथ तुलना: Binary, Scale, और Multi-Dimension modes

annotation_task_name: "Agent Comparison - Binary"
task_dir: "."
port: 8000
 
data_files:
  - "data/paired_traces.jsonl"
 
item_properties:
  id_key: "id"
  text_key: "task"
 
instance_display:
  fields:
    - key: "task"
      type: text
      label: "Task"
 
annotation_schemes:
  - annotation_type: pairwise
    name: preference
    description: "Which agent produced a better solution?"
    mode: binary
    items_key: "responses"
    show_labels: true
    labels:
      - "Agent A"
      - "Agent B"
    allow_tie: true
    tie_label: "Tie (equally good or equally bad)"
    randomize_order: true
    sequential_key_binding: true
    label_requirement:
      required: true
 
  - annotation_type: radio
    name: confidence
    description: "How confident are you in this judgment?"
    labels:
      - "Very confident"
      - "Somewhat confident"
      - "Not confident"
 
output_annotation_dir: "annotation_output/"
export_annotation_format: jsonl
 
user_config:
  allow_all_users: true

annotation_schemes:
  - annotation_type: pairwise
    name: preference_scale
    description: "Which agent produced a better solution, and by how much?"
    mode: scale
    items_key: "responses"
    labels:
      - "Agent A"
      - "Agent B"
    scale:
      min: -3
      max: 3
      step: 1
      default: 0
      labels:
        min: "A is much better"
        max: "B is much better"
        center: "Equal"
    label_requirement:
      required: true

annotation_schemes:
  - annotation_type: pairwise
    name: multi_dim_comparison
    description: "Compare the two solutions along each dimension"
    mode: multi_dimension
    items_key: "responses"
    labels:
      - "Agent A"
      - "Agent B"
    dimensions:
      - name: correctness
        description: "Which solution actually fixes the stated problem?"
        allow_tie: true
      - name: efficiency
        description: "Which takes fewer unnecessary steps, file reads, or redundant edits?"
        allow_tie: true
      - name: code_quality
        description: "Which is more readable, better named, and closer to existing patterns?"
        allow_tie: true
      - name: communication
        description: "Which explains its reasoning more clearly and identifies the root cause?"
        allow_tie: true
      - name: robustness
        description: "Which handles edge cases and verifies its changes?"
        allow_tie: true
    justification:
      required: true
      reason_categories:
        - "More correct"
        - "More efficient"
        - "Better written"
        - "Better explained"
      min_rationale_chars: 20
      rationale_placeholder: "Which agent was better here, and why?"

attention_checks:
  enabled: true
  items_file: "data/attention_checks.json"
  frequency: 20
  min_response_time: 10.0
  failure_handling:
    warn_threshold: 2
    block_threshold: 5

{
  "id": "pair_001",
  "task_description": "Fix the IndexError in process_batch() when the input list is empty",
  "repo": "myorg/myproject",
  "trace_a": {
    "agent": "claude_code",
    "model": "claude-sonnet-4-20250514",
    "structured_turns": [
      {
        "step_idx": 0,
        "type": "file_read",
        "path": "src/batch.py",
        "content": "def process_batch(items):\n    result = items[0]\n    ...",
        "start_line": 10,
        "end_line": 25
      },
      {
        "step_idx": 1,
        "type": "file_edit",
        "path": "src/batch.py",
        "diff": "--- a/src/batch.py\n+++ b/src/batch.py\n@@ -10,3 +10,5 @@\n def process_batch(items):\n+    if not items:\n+        return []\n     result = items[0]\n"
      },
      {
        "step_idx": 2,
        "type": "bash_command",
        "command": "python -m pytest tests/test_batch.py -v",
        "output": "PASSED",
        "exit_code": 0
      }
    ]
  },
  "trace_b": {
    "agent": "swe_agent",
    "model": "gpt-4o",
    "structured_turns": [
      {
        "step_idx": 0,
        "type": "bash_command",
        "command": "find . -name '*.py' | xargs grep 'process_batch'",
        "output": "src/batch.py:def process_batch(items):\ntests/test_batch.py:    process_batch([])",
        "exit_code": 0
      },
      {
        "step_idx": 1,
        "type": "file_read",
        "path": "src/batch.py",
        "content": "def process_batch(items):\n    result = items[0]\n    ...",
        "start_line": 1,
        "end_line": 50
      },
      {
        "step_idx": 2,
        "type": "file_edit",
        "path": "src/batch.py",
        "diff": "--- a/src/batch.py\n+++ b/src/batch.py\n@@ -10,3 +10,6 @@\n def process_batch(items):\n+    if items is None or len(items) == 0:\n+        logger.warning('Empty input to process_batch')\n+        return []\n     result = items[0]\n"
      },
      {
        "step_idx": 3,
        "type": "bash_command",
        "command": "python -m pytest tests/ -v",
        "output": "PASSED (12 tests)",
        "exit_code": 0
      }
    ]
  }
}

# Generate all possible pairs for each task
potato pair-traces \
  --input ./data/individual_traces.jsonl \
  --output ./data/paired_traces.jsonl \
  --pair_by "task_id" \
  --strategy "all_pairs"
 
# Or sample a fixed number of pairs per task
potato pair-traces \
  --input ./data/individual_traces.jsonl \
  --output ./data/paired_traces.jsonl \
  --pair_by "task_id" \
  --strategy "sample" \
  --pairs_per_task 3

potato export \
  --format dpo_preferences \
  --project ./output/ \
  --output ./training_data/preferences.jsonl

{
  "prompt": "Fix the IndexError in process_batch() when the input list is empty",
  "chosen": {"agent": "claude_code", "trace_id": "trace_a_001", "steps": [...]},
  "rejected": {"agent": "swe_agent", "trace_id": "trace_b_001", "steps": [...]},
  "annotator": "judge1",
  "confidence": "high"
}

{
  "prompt": "Fix the IndexError in process_batch()",
  "chosen": {"agent": "claude_code", "trace_id": "trace_a_001"},
  "rejected": {"agent": "swe_agent", "trace_id": "trace_b_001"},
  "preference_strength": 0.83,
  "scale_value": 2,
  "justification": "Agent A found and fixed the bug in fewer steps with cleaner code"
}

{
  "prompt": "Fix the IndexError in process_batch()",
  "chosen": {"agent": "claude_code", "trace_id": "trace_a_001"},
  "rejected": {"agent": "swe_agent", "trace_id": "trace_b_001"},
  "overall_preference": "A",
  "dimensions": {
    "correctness": {"preference": "Tie", "justification": "Both correctly fix the bug"},
    "efficiency": {"preference": "A", "justification": "A solves it in 3 steps vs 4"},
    "code_quality": {"preference": "B", "justification": "B adds logging and handles None"},
    "communication": {"preference": "A", "justification": "A's reasoning is more focused"},
    "robustness": {"preference": "B", "justification": "B runs full test suite, not just one file"}
  },
  "weighted_score_a": 0.55,
  "weighted_score_b": 0.45
}

import json
from collections import defaultdict
 
with open("training_data/preferences.jsonl") as f:
    prefs = [json.loads(line) for line in f]
 
wins = defaultdict(lambda: {"wins": 0, "losses": 0, "ties": 0})
 
for pref in prefs:
    agent_chosen = pref["chosen"]["agent"]
    agent_rejected = pref["rejected"]["agent"]
 
    if agent_chosen == agent_rejected:
        continue  # Skip self-comparisons
 
    if pref.get("overall_preference") == "Tie":
        wins[agent_chosen]["ties"] += 1
        wins[agent_rejected]["ties"] += 1
    else:
        wins[agent_chosen]["wins"] += 1
        wins[agent_rejected]["losses"] += 1
 
print("Agent Win Rates:")
print("-" * 55)
for agent, record in sorted(wins.items()):
    total = record["wins"] + record["losses"] + record["ties"]
    win_rate = (record["wins"] + 0.5 * record["ties"]) / total * 100
    print(f"  {agent:<20} {win_rate:5.1f}%  "
          f"(W:{record['wins']} L:{record['losses']} T:{record['ties']})")

import json
import math
from collections import defaultdict
 
def compute_elo(preferences, k=32, initial_rating=1500):
    """Compute Elo ratings from pairwise preferences."""
    ratings = defaultdict(lambda: initial_rating)
 
    for pref in preferences:
        agent_a = pref["chosen"]["agent"]
        agent_b = pref["rejected"]["agent"]
 
        ra = ratings[agent_a]
        rb = ratings[agent_b]
 
        # Expected scores
        ea = 1.0 / (1.0 + math.pow(10, (rb - ra) / 400))
        eb = 1.0 / (1.0 + math.pow(10, (ra - rb) / 400))
 
        overall = pref.get("overall_preference", "A")
        if overall == "Tie":
            sa, sb = 0.5, 0.5
        else:
            # "chosen" is the winner
            sa, sb = 1.0, 0.0
 
        ratings[agent_a] = ra + k * (sa - ea)
        ratings[agent_b] = rb + k * (sb - eb)
 
    return dict(ratings)
 
with open("training_data/preferences.jsonl") as f:
    prefs = [json.loads(line) for line in f]
 
ratings = compute_elo(prefs)
 
print("Elo Ratings:")
print("-" * 35)
for agent, rating in sorted(ratings.items(), key=lambda x: -x[1]):
    print(f"  {agent:<20} {rating:.0f}")

import json
from collections import defaultdict
 
with open("training_data/preferences.jsonl") as f:
    prefs = [json.loads(line) for line in f]
 
# Only process multi-dimension annotations
multi_dim = [p for p in prefs if "dimensions" in p]
 
dim_wins = defaultdict(lambda: defaultdict(lambda: {"A": 0, "B": 0, "Tie": 0}))
 
for pref in multi_dim:
    agent_a = pref["chosen"]["agent"]
    agent_b = pref["rejected"]["agent"]
    pair_key = f"{agent_a} vs {agent_b}"
 
    for dim_name, dim_data in pref["dimensions"].items():
        dim_wins[dim_name][pair_key][dim_data["preference"]] += 1
 
print("Per-Dimension Win Rates:")
print("=" * 60)
for dim_name, matchups in sorted(dim_wins.items()):
    print(f"\n  {dim_name.upper()}")
    print(f"  {'-' * 50}")
    for pair, counts in matchups.items():
        total = counts["A"] + counts["B"] + counts["Tie"]
        a_rate = (counts["A"] + 0.5 * counts["Tie"]) / total * 100
        print(f"    {pair}: A={a_rate:.0f}% B={100-a_rate:.0f}%  "
              f"(A:{counts['A']} B:{counts['B']} Tie:{counts['Tie']})")