T2I-CompBench: Compositional Text-to-Image Benchmark

# T2I-CompBench Text-to-Image Evaluation
# Based on Huang et al., NeurIPS 2023
# Paper: https://arxiv.org/abs/2307.06350
# Dataset: https://github.com/Karine-Huang/T2I-CompBench
#
# Evaluation of compositional text-to-image generation. Annotators assess
# how well generated images match text prompts across the benchmark's
# compositional dimensions: attribute binding (color, shape, texture),
# spatial relationships, non-spatial relationships, and complex compositions.
#
# Quality Rating (Likert 1-5):
# - 1 (Very Poor): Image does not match the prompt at all
# - 3 (Average): Some elements match but with notable errors
# - 5 (Perfect): Image perfectly matches all aspects of the prompt
#
# Compositional Challenge Types (T2I-CompBench, NeurIPS 2023):
# - Attribute binding: Correct binding of color, shape, texture to objects
# - Spatial: Correct spatial relationships between objects
# - Non-spatial: Non-spatial relationships such as actions or interactions
# - Complex: Multiple compositional challenges combined
# (Counting/numeracy was added only in the later T2I-CompBench++.)
#
# Annotation Guidelines:
# 1. Read the text prompt carefully
# 2. Examine both generated images
# 3. Rate the overall quality of prompt-image alignment
# 4. Classify the type of compositional challenge
# 5. Compare the two images in a pairwise preference judgment

annotation_task_name: "T2I-CompBench Text-to-Image Evaluation"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  # Step 1: Quality rating
  - annotation_type: likert
    name: quality_rating
    description: "How well do the generated images match the text prompt?"
    min_label: "Very Poor"
    max_label: "Perfect"
    size: 5

  # Step 2: Challenge type classification
  - annotation_type: radio
    name: challenge_type
    description: "What type of compositional challenge does this prompt test?"
    labels:
      - "Attribute binding"
      - "Spatial"
      - "Non-spatial"
      - "Complex"
    keyboard_shortcuts:
      "Attribute binding": "1"
      "Spatial": "2"
      "Non-spatial": "3"
      "Complex": "4"
    tooltips:
      "Attribute binding": "Tests correct binding of color, shape, or texture to objects"
      "Spatial": "Tests correct spatial relationships (above, below, next to, etc.)"
      "Non-spatial": "Tests non-spatial relationships such as actions or interactions"
      "Complex": "Tests multiple compositional challenges combined"

  # Step 3: Pairwise image comparison
  - annotation_type: pairwise
    name: image_preference
    description: "Which image better matches the text prompt?"
    mode: "binary"
    labels:
      - "Image A Better"
      - "Image B Better"
      - "Equal"
    keyboard_shortcuts:
      "Image A Better": "a"
      "Image B Better": "b"
      "Equal": "e"
    tooltips:
      "Image A Better": "Image A more accurately represents the text prompt"
      "Image B Better": "Image B more accurately represents the text prompt"
      "Equal": "Both images are equally accurate (or equally inaccurate)"

annotation_instructions: |
  You will evaluate text-to-image generation quality from the T2I-CompBench benchmark.

  For each item:
  1. Read the text prompt carefully - note all objects, attributes, relationships, and actions described.
  2. Examine both Image A and Image B.
  3. Rate the overall quality of prompt-image alignment on a 1-5 scale.
  4. Classify what type of compositional challenge the prompt tests.
  5. Compare the two images and select which better matches the prompt.

  Key Evaluation Criteria:
  - Are all mentioned objects present?
  - Are attributes (colors, shapes) correctly assigned to the right objects?
  - Are spatial relationships correct?
  - Are the correct number of objects shown?
  - Are actions depicted accurately?

html_layout: |
  <div style="padding: 15px; max-width: 900px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Text Prompt:</strong>
      <p style="font-size: 18px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
    <div style="display: flex; gap: 16px;">
      <div style="flex: 1; text-align: center;">
        <h4 style="color: #1976d2;">Image A</h4>
        <img src="{{image_a_url}}" style="max-width: 100%; max-height: 400px; border: 2px solid #1976d2; border-radius: 8px;" />
      </div>
      <div style="flex: 1; text-align: center;">
        <h4 style="color: #c62828;">Image B</h4>
        <img src="{{image_b_url}}" style="max-width: 100%; max-height: 400px; border: 2px solid #c62828; border-radius: 8px;" />
      </div>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 3
allow_skip: true
skip_reason_required: false