AI 코딩 에이전트를 위한 GitHub PR 스타일 코드 리뷰

python -m potato.convert_traces \
  --input claude_code_sessions/ \
  --output data/code_traces.jsonl \
  --format claude_code

python -m potato.convert_traces \
  --input aider_logs/ \
  --output data/code_traces.jsonl \
  --format aider

python -m potato.convert_traces \
  --input swe_agent_trajectories/ \
  --output data/code_traces.jsonl \
  --format swe_agent

{
  "trace_id": "swe_bench_123",
  "task": "Fix the TypeError in django/db/models/query.py when calling .values() on an empty QuerySet",
  "repo": "django/django",
  "base_commit": "a1b2c3d",
  "steps": [
    {
      "type": "thought",
      "content": "Let me look at the QuerySet.values() method to understand the TypeError."
    },
    {
      "type": "action",
      "tool": "search",
      "input": "def values",
      "file": "django/db/models/query.py"
    },
    {
      "type": "observation",
      "content": "Found at line 234: def values(self, *fields):\n    ..."
    },
    {
      "type": "action",
      "tool": "edit",
      "file": "django/db/models/query.py",
      "old_content": "    def values(self, *fields):\n        clone = self._clone()\n        clone.query.set_values(fields)",
      "new_content": "    def values(self, *fields):\n        clone = self._clone()\n        if not fields and not self.query.values_select:\n            return clone\n        clone.query.set_values(fields)"
    },
    {
      "type": "action",
      "tool": "edit",
      "file": "tests/queries/test_values.py",
      "old_content": "class ValuesTests(TestCase):",
      "new_content": "class ValuesTests(TestCase):\n    def test_empty_queryset_values(self):\n        \"\"\"Calling .values() on empty QuerySet should not raise TypeError.\"\"\"\n        qs = Author.objects.none()\n        result = list(qs.values())\n        self.assertEqual(result, [])"
    }
  ],
  "files_changed": [
    {
      "path": "django/db/models/query.py",
      "diff": "--- a/django/db/models/query.py\n+++ b/django/db/models/query.py\n@@ -234,6 +234,8 @@\n     def values(self, *fields):\n         clone = self._clone()\n+        if not fields and not self.query.values_select:\n+            return clone\n         clone.query.set_values(fields)"
    },
    {
      "path": "tests/queries/test_values.py",
      "diff": "--- a/tests/queries/test_values.py\n+++ b/tests/queries/test_values.py\n@@ -1,4 +1,10 @@\n class ValuesTests(TestCase):\n+    def test_empty_queryset_values(self):\n+        \"\"\"Calling .values() on empty QuerySet should not raise TypeError.\"\"\"\n+        qs = Author.objects.none()\n+        result = list(qs.values())\n+        self.assertEqual(result, [])"
    }
  ]
}

annotation_task_name: "Coding Agent Code Review"
 
data_files:
  - "data/code_traces.jsonl"
 
item_properties:
  id_key: "trace_id"
  text_key: "task"
 
# Display coding agent traces with diff rendering
display:
  type: "coding_trace"
  trace_key: "steps"
  diff_key: "files_changed"
  syntax_highlighting: true
  show_line_numbers: true
  collapse_large_diffs: true
  max_uncollapsed_lines: 200
 
annotation_schemes:
  - annotation_type: "code_review"
    name: "review"
    description: "Review the agent's code changes as you would a GitHub PR"
 
    # Inline comment categories
    comment_categories:
      - name: "bug"
        label: "Bug"
        color: "#DC2626"
        description: "Functional error in the code"
      - name: "style"
        label: "Style"
        color: "#2563EB"
        description: "Code style or convention issue"
      - name: "performance"
        label: "Performance"
        color: "#EA580C"
        description: "Inefficient code or unnecessary computation"
      - name: "security"
        label: "Security"
        color: "#9333EA"
        description: "Security vulnerability or unsafe practice"
      - name: "logic"
        label: "Logic"
        color: "#CA8A04"
        description: "Logic issue that may not cause immediate failure"
      - name: "suggestion"
        label: "Suggestion"
        color: "#16A34A"
        description: "Improvement idea, not an error"
      - name: "question"
        label: "Question"
        color: "#6B7280"
        description: "Clarification needed"
 
    # File-level ratings
    file_ratings:
      - name: "correctness"
        label: "Correctness"
        scale: 5
        descriptions:
          1: "Does not work, introduces new bugs"
          2: "Partially works, has significant issues"
          3: "Works for happy path, misses edge cases"
          4: "Works correctly with minor issues"
          5: "Fully correct, handles edge cases"
      - name: "code_quality"
        label: "Code Quality"
        scale: 5
        descriptions:
          1: "Unmaintainable, no structure"
          2: "Poor quality, significant style issues"
          3: "Acceptable, follows basic conventions"
          4: "Good quality, clean and readable"
          5: "Excellent, idiomatic, well-documented"
 
    # Overall verdict
    verdict:
      options:
        - name: "approve"
          label: "Approve"
          color: "#16A34A"
          description: "Ready to merge as-is or with trivial changes"
        - name: "request_changes"
          label: "Request Changes"
          color: "#DC2626"
          description: "Needs significant revisions before merging"
        - name: "comment"
          label: "Comment Only"
          color: "#6B7280"
          description: "Providing feedback without a merge decision"
 
# Annotator settings
annotator_config:
  allow_back_navigation: true
 
# Output settings
output:
  path: "output/"
  format: "jsonl"

potato start config.yaml -p 8000

{
  "trace_id": "swe_bench_123",
  "annotator": "reviewer_01",
  "timestamp": "2026-03-22T14:32:11Z",
  "review": {
    "inline_comments": [
      {
        "file": "django/db/models/query.py",
        "line": 236,
        "side": "right",
        "category": "logic",
        "body": "This early return skips set_values entirely, but if fields are provided later via .values('name'), the previous empty .values() call will have returned a clone that never went through set_values. Consider checking if this clone is still valid downstream."
      },
      {
        "file": "tests/queries/test_values.py",
        "line": 5,
        "side": "right",
        "category": "suggestion",
        "body": "Consider adding a test case for .values() followed by .values('name') to verify the chaining behavior after your fix."
      }
    ],
    "file_ratings": [
      {
        "file": "django/db/models/query.py",
        "correctness": 3,
        "code_quality": 4
      },
      {
        "file": "tests/queries/test_values.py",
        "correctness": 4,
        "code_quality": 4
      }
    ],
    "verdict": "request_changes"
  }
}

import json
import pandas as pd
from pathlib import Path
 
reviews = []
for f in Path("output/").glob("*.jsonl"):
    with open(f) as fh:
        for line in fh:
            reviews.append(json.loads(line))
 
print(f"Loaded {len(reviews)} code reviews")

from collections import Counter
 
all_comments = []
for rev in reviews:
    for comment in rev["review"]["inline_comments"]:
        all_comments.append(comment)
 
category_counts = Counter(c["category"] for c in all_comments)
print("Comment categories:")
for cat, count in category_counts.most_common():
    print(f"  {cat}: {count}")

ratings = []
for rev in reviews:
    for fr in rev["review"]["file_ratings"]:
        ratings.append(fr)
 
ratings_df = pd.DataFrame(ratings)
print("Average ratings by file:")
print(
    ratings_df.groupby("file")[["correctness", "code_quality"]]
    .mean()
    .round(2)
    .to_string()
)

verdict_counts = Counter(rev["review"]["verdict"] for rev in reviews)
total = sum(verdict_counts.values())
print("Verdict distribution:")
for verdict, count in verdict_counts.most_common():
    print(f"  {verdict}: {count} ({count/total*100:.1f}%)")

agent_bugs = {}
for rev in reviews:
    agent = rev.get("agent", "unknown")
    bug_count = sum(
        1 for c in rev["review"]["inline_comments"]
        if c["category"] == "bug"
    )
    if agent not in agent_bugs:
        agent_bugs[agent] = []
    agent_bugs[agent].append(bug_count)
 
print("Average bugs per review by agent:")
for agent, bugs in sorted(agent_bugs.items()):
    print(f"  {agent}: {sum(bugs)/len(bugs):.2f} (n={len(bugs)})")