面向 AI 編碼智慧體的 GitHub PR 風格程式碼評審

python -m potato.trace_converter \
  --input claude_code_sessions/ \
  --output data/code_traces.jsonl \
  --input-format claude_code

python -m potato.trace_converter \
  --input aider_logs/ \
  --output data/code_traces.jsonl \
  --input-format aider

python -m potato.trace_converter \
  --input swe_agent_trajectories/ \
  --output data/code_traces.jsonl \
  --input-format swe_agent_trajectory

{
  "trace_id": "swe_bench_123",
  "task": "Fix the TypeError in django/db/models/query.py when calling .values() on an empty QuerySet",
  "repo": "django/django",
  "base_commit": "a1b2c3d",
  "steps": [
    {
      "type": "thought",
      "content": "Let me look at the QuerySet.values() method to understand the TypeError."
    },
    {
      "type": "action",
      "tool": "search",
      "input": "def values",
      "file": "django/db/models/query.py"
    },
    {
      "type": "observation",
      "content": "Found at line 234: def values(self, *fields):\n    ..."
    },
    {
      "type": "action",
      "tool": "edit",
      "file": "django/db/models/query.py",
      "old_content": "    def values(self, *fields):\n        clone = self._clone()\n        clone.query.set_values(fields)",
      "new_content": "    def values(self, *fields):\n        clone = self._clone()\n        if not fields and not self.query.values_select:\n            return clone\n        clone.query.set_values(fields)"
    },
    {
      "type": "action",
      "tool": "edit",
      "file": "tests/queries/test_values.py",
      "old_content": "class ValuesTests(TestCase):",
      "new_content": "class ValuesTests(TestCase):\n    def test_empty_queryset_values(self):\n        \"\"\"Calling .values() on empty QuerySet should not raise TypeError.\"\"\"\n        qs = Author.objects.none()\n        result = list(qs.values())\n        self.assertEqual(result, [])"
    }
  ],
  "files_changed": [
    {
      "path": "django/db/models/query.py",
      "diff": "--- a/django/db/models/query.py\n+++ b/django/db/models/query.py\n@@ -234,6 +234,8 @@\n     def values(self, *fields):\n         clone = self._clone()\n+        if not fields and not self.query.values_select:\n+            return clone\n         clone.query.set_values(fields)"
    },
    {
      "path": "tests/queries/test_values.py",
      "diff": "--- a/tests/queries/test_values.py\n+++ b/tests/queries/test_values.py\n@@ -1,4 +1,10 @@\n class ValuesTests(TestCase):\n+    def test_empty_queryset_values(self):\n+        \"\"\"Calling .values() on empty QuerySet should not raise TypeError.\"\"\"\n+        qs = Author.objects.none()\n+        result = list(qs.values())\n+        self.assertEqual(result, [])"
    }
  ]
}

annotation_task_name: "Coding Agent Code Review"
 
data_files:
  - "data/code_traces.jsonl"
 
item_properties:
  id_key: "trace_id"
  text_key: "task"
 
# Display coding agent traces with diff rendering
instance_display:
  fields:
    - key: task
      type: text
      label: "Task"
 
    - key: steps
      type: coding_trace
      label: "Agent changes"
      display_options:
        diff_view: unified
        show_file_tree: true
        show_reasoning: true
        terminal_theme: dark
        collapse_long_outputs: true
        max_output_lines: 200
 
annotation_schemes:
  - annotation_type: code_review
    name: review
    description: "Review the agent's code changes as you would a pull request"
    comment_categories: [bug, style, performance, security, logic, suggestion, question]
    file_rating_dimensions: [correctness, quality]
    verdict_options: [approve, request_changes, comment_only]
 
output_annotation_dir: "annotation_output/"
export_annotation_format: jsonl

potato start config.yaml -p 8000

{
  "instance_id": "swe_bench_123",
  "annotator": "reviewer_01",
  "verdict": "request_changes",
  "comments": [
    {
      "category": "logic",
      "file": "django/db/models/query.py",
      "line": 236,
      "text": "This early return skips set_values entirely, but if fields are provided later via .values('name'), the previous empty .values() call will have returned a clone that never went through set_values. Consider checking if this clone is still valid downstream."
    },
    {
      "category": "suggestion",
      "file": "tests/queries/test_values.py",
      "line": 5,
      "text": "Consider adding a test case for .values() followed by .values('name') to verify the chaining behavior after your fix."
    }
  ],
  "file_ratings": {
    "django/db/models/query.py": { "correctness": 3, "quality": 4 },
    "tests/queries/test_values.py": { "correctness": 4, "quality": 4 }
  }
}

python -m potato.export -c config.yaml -f coding_eval -o ./export/

import json
import pandas as pd
from pathlib import Path
 
reviews = []
for f in Path("output/").glob("*.jsonl"):
    with open(f) as fh:
        for line in fh:
            reviews.append(json.loads(line))
 
print(f"Loaded {len(reviews)} code reviews")

from collections import Counter
 
all_comments = []
for rev in reviews:
    all_comments.extend(rev["comments"])
 
category_counts = Counter(c["category"] for c in all_comments)
print("Comment categories:")
for cat, count in category_counts.most_common():
    print(f"  {cat}: {count}")

ratings = []
for rev in reviews:
    for path, dims in rev["file_ratings"].items():
        ratings.append({"file": path, **dims})
 
ratings_df = pd.DataFrame(ratings)
print("Average ratings by file:")
print(
    ratings_df.groupby("file")[["correctness", "quality"]]
    .mean()
    .round(2)
    .to_string()
)

verdict_counts = Counter(rev["verdict"] for rev in reviews)
total = sum(verdict_counts.values())
print("Verdict distribution:")
for verdict, count in verdict_counts.most_common():
    print(f"  {verdict}: {count} ({count/total*100:.1f}%)")

agent_bugs = {}
for rev in reviews:
    agent = rev.get("agent", "unknown")
    bug_count = sum(
        1 for c in rev["comments"]
        if c["category"] == "bug"
    )
    if agent not in agent_bugs:
        agent_bugs[agent] = []
    agent_bugs[agent].append(bug_count)
 
print("Average bugs per review by agent:")
for agent, bugs in sorted(agent_bugs.items()):
    print(f"  {agent}: {sum(bugs)/len(bugs):.2f} (n={len(bugs)})")