ライブエージェント評価

pip install playwright anthropic
playwright install chromium
export ANTHROPIC_API_KEY=your_key_here

live_agent:
  endpoint_type: anthropic_vision
  ai_config:
    model: claude-sonnet-4-20250514
    api_key: ${ANTHROPIC_API_KEY}
    max_tokens: 4096
    temperature: 0.3
  system_prompt: |
    You are a web browsing agent. Complete the given task efficiently.
    At each step, describe your thought, then output an action.
  max_steps: 30
  step_delay: 1.0
  viewport:
    width: 1280
    height: 720
  allow_takeover: true
  allow_instructions: true
 
instance_display:
  fields:
    - key: task_description
      type: text
      label: "Task"
    - key: agent_trace
      type: live_agent
      label: "Live Agent Session"
      display_options:
        show_overlays: true
        show_filmstrip: true
        show_thought: true
        show_controls: true

{
  "id": "task_001",
  "task_description": "Search for climate change on Wikipedia and find the year it was first described",
  "start_url": "https://en.wikipedia.org"
}

annotation_schemes:
  - annotation_type: radio
    name: task_success
    description: "Did the agent complete the task?"
    labels:
      - name: "Yes, fully"
      - name: "Partially"
      - name: "No"
  - annotation_type: likert
    name: efficiency
    description: "How efficiently did the agent work?"
    min_label: "Very inefficient"
    max_label: "Very efficient"
  - annotation_type: text
    name: errors_observed
    description: "Describe any errors or unnecessary steps"
  - annotation_type: span
    name: error_steps
    description: "Mark any steps where the agent made an error"
    labels:
      - name: hallucination
      - name: wrong_target
      - name: unnecessary_action

task_name: "Live Agent Evaluation Study"
task_dir: "."
 
live_agent:
  endpoint_type: anthropic_vision
  ai_config:
    model: claude-sonnet-4-20250514
    api_key: ${ANTHROPIC_API_KEY}
    max_tokens: 4096
    temperature: 0.3
  max_steps: 25
  step_delay: 1.5
  viewport:
    width: 1280
    height: 720
  allow_takeover: true
  allow_instructions: true
  history_window: 5
 
data_files:
  - "tasks.jsonl"
 
instance_display:
  fields:
    - key: task_description
      type: text
      label: "Task"
    - key: agent_trace
      type: live_agent
      label: "Live Session"
      display_options:
        show_overlays: true
        show_filmstrip: true
        show_thought: true
        show_controls: true
 
annotation_schemes:
  - annotation_type: radio
    name: task_success
    description: "Did the agent complete the task?"
    labels:
      - name: "Yes"
      - name: "Partially"
      - name: "No"
  - annotation_type: likert
    name: efficiency
    description: "Rate the agent's efficiency"
    min_label: "Very inefficient"
    max_label: "Very efficient"
  - annotation_type: text
    name: notes
    description: "Notes on agent behavior"
 
output_annotation_dir: "output/"
output_annotation_format: "jsonl"

live_agent:
  endpoint_type: coding_agent
  backend: ollama
  ai_config:
    model: qwen2.5-coder:7b
    host: "http://localhost:11434"
  max_steps: 50
  project_dir: "./workspace"

live_agent:
  endpoint_type: coding_agent
  backend: anthropic
  ai_config:
    model: claude-sonnet-4-20250514
    api_key: ${ANTHROPIC_API_KEY}
    max_tokens: 8192
  max_steps: 50
  project_dir: "./workspace"

live_agent:
  endpoint_type: coding_agent
  backend: claude_agent_sdk
  ai_config:
    max_turns: 50
  project_dir: "./workspace"