pip install potato-annotation

{
  "trace_id": "wa_001",
  "task": "Find the cheapest laptop on the electronics store and add it to the cart",
  "website": "shopping",
  "steps": [
    {
      "step": 0,
      "url": "http://shop.example.com/",
      "action_type": "click",
      "action_target": "Electronics category link",
      "element_id": "nav-electronics",
      "coordinates": [245, 82],
      "screenshot": "screenshots/wa_001_step_00.png",
      "dom_snapshot": "dom/wa_001_step_00.html"
    },
    {
      "step": 1,
      "url": "http://shop.example.com/electronics",
      "action_type": "click",
      "action_target": "Laptops subcategory",
      "element_id": "cat-laptops",
      "coordinates": [180, 310],
      "screenshot": "screenshots/wa_001_step_01.png"
    },
    {
      "step": 2,
      "url": "http://shop.example.com/electronics/laptops",
      "action_type": "click",
      "action_target": "Sort by: Price Low to High",
      "element_id": "sort-price-asc",
      "coordinates": [720, 155],
      "screenshot": "screenshots/wa_001_step_02.png"
    },
    {
      "step": 3,
      "url": "http://shop.example.com/electronics/laptops?sort=price_asc",
      "action_type": "click",
      "action_target": "First laptop: 'Budget Pro 14' - $349",
      "element_id": "product-101",
      "coordinates": [400, 380],
      "screenshot": "screenshots/wa_001_step_03.png"
    },
    {
      "step": 4,
      "url": "http://shop.example.com/product/101",
      "action_type": "click",
      "action_target": "Add to Cart button",
      "element_id": "add-to-cart-btn",
      "coordinates": [650, 520],
      "screenshot": "screenshots/wa_001_step_04.png"
    }
  ],
  "success": true,
  "final_screenshot": "screenshots/wa_001_final.png"
}

task_name: "WebArena Agent Evaluation"
task_dir: "."
 
data_files:
  - "data/web_traces.jsonl"
 
item_properties:
  id_key: trace_id
  text_key: task
 
# --- Agentic annotation with web display ---
agentic:
  enabled: true
  trace_converter: webarena
  display_type: web_agent
 
  web_agent_display:
    # Screenshot rendering
    screenshot_max_width: 900
    screenshot_quality: 85
 
    # SVG overlays
    overlay:
      enabled: true
      click_marker: "circle"
      click_color: "#ef4444"
      click_radius: 20
      type_highlight: "#3b82f6"
      scroll_indicator: true
 
    # Filmstrip navigation
    filmstrip:
      enabled: true
      thumbnail_width: 150
      show_action_labels: true
 
    # Additional display options
    show_url_bar: true
    show_action_description: true
    show_dom_snapshot: false
 
# --- Annotation Schemas ---
annotation_schemes:
  # Overall task evaluation
  - annotation_type: radio
    name: task_success
    description: "Did the agent complete the task successfully?"
    labels:
      - "Success"
      - "Partial Success"
      - "Failure"
    label_requirement:
      required: true
 
  - annotation_type: radio
    name: task_efficiency
    description: "Was the agent's navigation path efficient?"
    labels:
      - "Optimal path"
      - "Reasonable but not optimal"
      - "Inefficient (unnecessary steps)"
      - "Completely wrong direction"
    label_requirement:
      required: true
 
  # Per-step evaluation
  - annotation_type: per_turn_rating
    name: action_correctness
    target: agentic_steps
    description: "Was this action correct?"
    rating_type: radio
    labels:
      - "Correct"
      - "Acceptable (not optimal but progresses toward goal)"
      - "Incorrect"
      - "Unnecessary"
 
  - annotation_type: per_turn_rating
    name: action_error_type
    target: agentic_steps
    description: "What went wrong?"
    rating_type: multiselect
    labels:
      - "Wrong element clicked"
      - "Wrong page navigated to"
      - "Missed a closer/better option"
      - "Incorrect form input"
      - "Premature task completion"
      - "Unnecessary navigation"
      - "Failed to scroll to target"
      - "Interaction with wrong page section"
      - "Other"
    conditional:
      show_when:
        action_correctness: ["Incorrect", "Unnecessary"]
 
  - annotation_type: per_turn_rating
    name: action_notes
    target: agentic_steps
    description: "Notes on this step"
    rating_type: text
    label_requirement:
      required: false
 
output_annotation_dir: "output/"
output_annotation_format: "jsonl"
 
parquet_export:
  enabled: true
  output_dir: "output/parquet/"

web_agent_display:
  show_dom_snapshot: true

agentic:
  enabled: true
  trace_converter: webarena         # same converter handles both
  display_type: web_agent
 
  web_agent_display:
    screenshot_max_width: 1000
    overlay:
      enabled: true
      click_marker: "crosshair"     # crosshair is better for precise grounding
      click_color: "#ef4444"
      click_radius: 15
      bounding_box: true            # show element bounding box if available
      bounding_box_color: "#f59e0b"
    filmstrip:
      enabled: true
      thumbnail_width: 180

import pandas as pd
import json
 
annotations = []
with open("output/annotations.jsonl") as f:
    for line in f:
        annotations.append(json.loads(line))
 
# Collect per-step correctness by position
step_errors = {}
for ann in annotations:
    correctness = ann["annotations"].get("action_correctness", {})
    for step_idx, label in correctness.items():
        pos = int(step_idx)
        if pos not in step_errors:
            step_errors[pos] = {"Correct": 0, "Acceptable": 0, "Incorrect": 0, "Unnecessary": 0}
        step_errors[pos][label] += 1
 
# Print error rate by step position
print("Error rate by step position:")
for pos in sorted(step_errors.keys()):
    counts = step_errors[pos]
    total = sum(counts.values())
    error_rate = (counts["Incorrect"] + counts["Unnecessary"]) / total
    print(f"  Step {pos}: {error_rate:.1%} error rate ({total} observations)")

error_counts = {}
for ann in annotations:
    errors = ann["annotations"].get("action_error_type", {})
    for step_idx, error_list in errors.items():
        for error in error_list:
            error_counts[error] = error_counts.get(error, 0) + 1
 
print("\nError Type Distribution:")
for error, count in sorted(error_counts.items(), key=lambda x: -x[1]):
    print(f"  {error}: {count}")

# If traces span multiple websites
website_success = {}
for ann in annotations:
    # Assuming website info is in the original trace data
    success = ann["annotations"]["task_success"]
    website = ann.get("metadata", {}).get("website", "unknown")
    if website not in website_success:
        website_success[website] = {"Success": 0, "Partial Success": 0, "Failure": 0}
    website_success[website][success] += 1
 
for website, counts in website_success.items():
    total = sum(counts.values())
    rate = counts["Success"] / total
    print(f"{website}: {rate:.1%} success rate")

annotation_task_config:
  total_annotations_per_instance: 3
  assignment_strategy: random

from sklearn.metrics import cohen_kappa_score
import pandas as pd
 
df = pd.read_parquet("output/parquet/annotations.parquet")
success = df[df["schema_name"] == "task_success"]
pivot = success.pivot(index="instance_id", columns="annotator", values="value")
 
# Pairwise kappa
annotators = pivot.columns.tolist()
for i in range(len(annotators)):
    for j in range(i + 1, len(annotators)):
        mask = pivot[[annotators[i], annotators[j]]].dropna()
        kappa = cohen_kappa_score(mask[annotators[i]], mask[annotators[j]])
        print(f"Kappa ({annotators[i]} vs {annotators[j]}): {kappa:.3f}")

solo_mode:
  enabled: true
  llm:
    endpoint_type: openai
    model: "gpt-4o"
    api_key: ${OPENAI_API_KEY}
  accuracy_threshold: 0.90
 
agentic:
  enabled: true
  trace_converter: webarena
  display_type: web_agent