प्रोसेस रिवॉर्ड एनोटेशन

annotation_schemes:
  - name: process_reward
    annotation_type: process_reward
    mode: first_error
    description: "Click the first step where the agent made a mistake"
 
    first_error:
      # Visual styling
      correct_color: "#22c55e"     # green for steps before the error
      error_color: "#ef4444"       # red for the first error step
      downstream_color: "#f97316"  # orange for steps after the error
      unmarked_color: "#6b7280"    # gray for steps not yet reviewed
 
      # Behavior
      require_confirmation: true   # ask "Are you sure?" before marking
      allow_no_error: true         # allow annotator to mark all steps correct
      show_step_content: true      # show step content in the annotation panel
 
      # Labels applied automatically
      labels:
        correct: "+1"
        first_error: "-1 (first error)"
        downstream: "-1 (downstream)"
        all_correct: "+1 (all correct)"

{
  "id": "trace_042",
  "annotations": {
    "process_reward": {
      "mode": "first_error",
      "first_error_step": 4,
      "total_steps": 8,
      "labels": [1, 1, 1, 1, -1, -1, -1, -1]
    }
  }
}

annotation_schemes:
  - name: process_reward
    annotation_type: process_reward
    mode: per_step
    description: "Rate each step independently"
 
    per_step:
      # Rating options
      labels:
        - value: "correct"
          display: "Correct"
          color: "#22c55e"
          score: 1.0
        - value: "partially_correct"
          display: "Partially Correct"
          color: "#eab308"
          score: 0.5
        - value: "incorrect"
          display: "Incorrect"
          color: "#ef4444"
          score: -1.0
        - value: "unnecessary"
          display: "Unnecessary"
          color: "#f97316"
          score: -0.5
        - value: "recovery"
          display: "Recovery from Error"
          color: "#3b82f6"
          score: 0.25
 
      # Optional error categorization for incorrect/partially correct steps
      error_categories:
        enabled: true
        categories:
          - "Wrong tool selected"
          - "Correct tool, wrong arguments"
          - "Hallucinated information"
          - "Repeated previous step"
          - "Logic error"
          - "Syntax error"
          - "Missed edge case"
          - "Unnecessary step"
          - "Other"
 
      # Behavior
      require_all_steps: true     # all steps must be rated before submission
      allow_notes: true           # optional text field per step
      show_running_score: true    # show cumulative reward score

{
  "id": "trace_042",
  "annotations": {
    "process_reward": {
      "mode": "per_step",
      "total_steps": 6,
      "labels": [1.0, 1.0, -1.0, 0.25, 1.0, 1.0],
      "step_details": {
        "0": {"label": "correct"},
        "1": {"label": "correct"},
        "2": {
          "label": "incorrect",
          "error_category": "Wrong tool selected",
          "notes": "Agent used grep when it should have read the file directly"
        },
        "3": {
          "label": "recovery",
          "notes": "Agent recognized the mistake and tried a different approach"
        },
        "4": {"label": "correct"},
        "5": {"label": "correct"}
      },
      "cumulative_score": 2.75
    }
  }
}

annotation_schemes:
  - name: process_reward
    annotation_type: process_reward
    mode: first_error              # "first_error" or "per_step"
    description: "Process reward annotation"
 
    # Required for mode: first_error
    first_error:
      correct_color: "#22c55e"
      error_color: "#ef4444"
      downstream_color: "#f97316"
      unmarked_color: "#6b7280"
      require_confirmation: true
      allow_no_error: true
      show_step_content: true
 
    # Required for mode: per_step
    per_step:
      labels:
        - value: "correct"
          display: "Correct"
          color: "#22c55e"
          score: 1.0
        - value: "incorrect"
          display: "Incorrect"
          color: "#ef4444"
          score: -1.0
      error_categories:
        enabled: false
        categories: []
      require_all_steps: true
      allow_notes: false
      show_running_score: false
 
    # Common options
    target: agentic_steps          # bind to trace steps
    keyboard_shortcuts:
      enabled: true
      correct: "1"
      incorrect: "2"
      partially_correct: "3"
      unnecessary: "4"
      next_step: "j"
      prev_step: "k"

python -m potato.export \
  -i output/ \
  -f prm \
  -o results/prm_training_data.jsonl

{
  "trace_id": "trace_042",
  "steps": [
    {"content": "Search for Tokyo population", "label": 1},
    {"content": "Parse search results", "label": 1},
    {"content": "Search for NYC population", "label": -1},
    {"content": "Compare populations", "label": -1}
  ]
}

python -m potato.export \
  -i output/ \
  -f dpo \
  -o results/dpo_pairs.jsonl \
  --min-score-gap 0.5

{
  "prompt": "Fix the failing test in test_parser.py",
  "chosen": [
    {"role": "assistant", "content": "Step 1: Read the test file..."},
    {"role": "assistant", "content": "Step 2: Identify the bug..."}
  ],
  "rejected": [
    {"role": "assistant", "content": "Step 1: Run all tests..."},
    {"role": "assistant", "content": "Step 2: Edit a random file..."}
  ]
}

python -m potato.export \
  -i output/ \
  -f swebench \
  -o results/swebench_results.json

from potato.analysis import load_annotations, process_reward_stats
 
# Load annotations
annotations = load_annotations("output/")
 
# Step-level accuracy statistics
stats = process_reward_stats(annotations)
 
print(f"Total traces annotated: {stats['total_traces']}")
print(f"Traces with no errors: {stats['all_correct_count']} ({stats['all_correct_pct']:.1f}%)")
print(f"Average first-error position: step {stats['avg_first_error_step']:.1f}")
print(f"Average steps before error: {stats['avg_correct_prefix_length']:.1f}")
 
# Error distribution by step position
for position, count in stats['error_by_position'].items():
    print(f"  Step {position}: {count} errors")
 
# Error category distribution (per-step mode only)
if 'error_categories' in stats:
    for category, count in stats['error_categories'].items():
        print(f"  {category}: {count}")
 
# Inter-annotator agreement on first-error step
if stats['multi_annotator']:
    print(f"First-error agreement (exact): {stats['first_error_exact_agreement']:.2f}")
    print(f"First-error agreement (within 1): {stats['first_error_near_agreement']:.2f}")

from potato.analysis import plot_error_distribution
 
# Plot error position distribution across all traces
plot_error_distribution(
    annotations,
    output_path="figures/error_distribution.png",
    normalize_by_trace_length=True,
    title="Where Do Agents First Go Wrong?"
)
 
# Plot per-step reward curves
from potato.analysis import plot_reward_curves
 
plot_reward_curves(
    annotations,
    output_path="figures/reward_curves.png",
    group_by="agent_model",
    title="Cumulative Reward by Model"
)