تعليق وكلاء البرمجة

instance_display:
  fields:
    - key: structured_turns
      type: coding_trace
      label: "Agent session"
      display_options:
        # Diff rendering
        diff_view: unified          # "unified" or "side_by_side"
 
        # Terminal output
        terminal_theme: dark        # "dark" or "light"
 
        # Long output
        collapse_long_outputs: true
        max_output_lines: 50        # collapse after this many lines
 
        # Step chrome
        show_file_tree: true
        show_step_numbers: true
        show_tool_badges: true
        show_reasoning: true
        compact: false

python -m potato.trace_converter \
  --input traces.json \
  --input-format claude_code \
  --output data/traces.jsonl

# Convert Claude Code traces
python -m potato.trace_converter \
  -i traces.json \
  -f claude_code \
  -o data/converted.jsonl
 
# Convert Aider chat logs
python -m potato.trace_converter \
  -i aider_chat_history/ \
  -f aider \
  -o data/aider_converted.jsonl
 
# Convert SWE-Agent trajectories
python -m potato.trace_converter \
  -i trajectories/ \
  -f swe_agent_trajectory \
  -o data/swe_converted.jsonl
 
# Auto-detect format
python -m potato.trace_converter \
  -i mixed_traces/ \
  -f auto \
  -o data/auto_converted.jsonl

# Filter by file extension
python -m potato.trace_converter \
  -i traces/ -f claude_code -o data/out.jsonl \
  --include "*.json"
 
# Add metadata fields from a CSV
python -m potato.trace_converter \
  -i traces/ -f claude_code -o data/out.jsonl \
  --metadata metadata.csv --join-key trace_id
 
# Validate output without writing
python -m potato.trace_converter \
  -i traces.json -f claude_code --validate

{
  "id": "trace_001",
  "task_description": "Fix the failing test in test_parser.py",
  "repository": "myproject",
  "structured_turns": [
    {
      "role": "assistant",
      "content": "I'll read the parser first to see how it handles empty input.",
      "tool_calls": [
        {
          "tool": "Read",
          "input": { "file_path": "src/parser.py" },
          "output": "def parse(input_str):\n    tokens = tokenize(input_str)\n    ..."
        }
      ]
    },
    {
      "role": "assistant",
      "content": "Empty input returns None where the test expects a ParseError.",
      "tool_calls": [
        {
          "tool": "Edit",
          "input": {
            "file_path": "src/parser.py",
            "old_string": "    if len(tokens) == 0:\n        return None",
            "new_string": "    if len(tokens) == 0:\n        raise ParseError('Empty input')"
          },
          "output": "Edited src/parser.py"
        },
        {
          "tool": "Bash",
          "input": { "command": "python -m pytest test_parser.py -v" },
          "output": "test_parser.py::test_empty_input PASSED\ntest_parser.py::test_valid_input PASSED\n\n2 passed in 0.34s"
        }
      ]
    }
  ],
  "metadata": {
    "agent": "claude_code",
    "model": "claude-sonnet-4-20250514",
    "total_tokens": 15234,
    "duration_seconds": 42
  }
}

annotation_task_name: "Coding Agent Evaluation"
task_dir: "."
 
data_files:
  - "data/coding_traces.jsonl"
 
item_properties:
  id_key: id
  text_key: task_description
 
instance_display:
  fields:
    - key: structured_turns
      type: coding_trace
      label: "Agent session"
      display_options:
        diff_view: unified
        terminal_theme: dark
        collapse_long_outputs: true
        max_output_lines: 50
        show_file_tree: true
        show_step_numbers: true
        show_tool_badges: true
        show_reasoning: true
 
annotation_schemes:
  # Did the agent complete the task?
  - annotation_type: radio
    name: task_completion
    description: "Did the agent successfully complete the task?"
    labels:
      - "Fully Complete"
      - "Partially Complete"
      - "Failed"
      - "Made Things Worse"
 
  # Per-step correctness
  - annotation_type: trajectory_eval
    name: step_quality
    description: "Rate this step"
    steps_key: agentic_steps
    correctness_options:
      - "Good"
      - "Acceptable"
      - "Unnecessary"
      - "Incorrect"
 
  # Code quality rating
  - annotation_type: likert
    name: code_quality
    size: 5
    min_label: "Poor"
    max_label: "Excellent"
    description: "Rate the quality of the code changes"
    labels:
      1: "Very Poor"
      2: "Poor"
      3: "Acceptable"
      4: "Good"
      5: "Excellent"
 
  # Free-text notes
  - annotation_type: text
    name: notes
    description: "Any additional observations about the coding trace"
    label_requirement:
      required: false
 
output_annotation_dir: "output/"
export_annotation_format: "jsonl"

# Clone the repository
git clone https://github.com/davidjurgens/potato.git
cd potato
 
# Run the Claude Code trace evaluation example
potato start example/coding_agent_eval/config.yaml -p 8000
 
# Run the SWE-bench evaluation example
potato start example/swe_bench_eval/config.yaml -p 8000
 
# Run the multi-agent comparison example
potato start example/coding_agent_comparison/config.yaml -p 8000