LLM Response Preference

annotation_task_name: "LLM Response Preference Collection"
task_dir: "."

port: 8000

# Data configuration
data_files:
  - "data/response_pairs.json"

item_properties:
  id_key: id
  text_key: text
instance_display:
  fields:
    - key: text
      type: text
      label: "Responses"

# Annotation schemes
annotation_schemes:
  # Main preference rating. Five points on one axis is pairwise's scale mode;
  # the end and centre captions come from scale.labels.
  - annotation_type: pairwise
    name: overall_preference
    description: "Overall, which response is better?"
    mode: scale
    labels: ["A", "B"]
    scale:
      min: -2
      max: 2
      step: 1
      default: 0
      labels:
        min: "A is much better"
        max: "B is much better"
        center: "About equal"
    sequential_key_binding: true

  # Individual aspect ratings. A three-way choice is the default binary mode
  # plus an explicit tie.
  - annotation_type: pairwise
    name: helpfulness
    description: "Which response is more helpful?"
    labels: ["A", "B"]
    allow_tie: true
    tie_label: "Equal"

  - annotation_type: pairwise
    name: accuracy
    description: "Which response is more accurate?"
    labels: ["A", "B"]
    allow_tie: true
    tie_label: "Equal"

  - annotation_type: pairwise
    name: safety
    description: "Which response is safer/less harmful?"
    labels: ["A", "B"]
    allow_tie: true
    tie_label: "Equal"

  # Reasons for preference
  - annotation_type: multiselect
    name: preference_reasons
    description: "What factors influenced your choice? (Select all that apply)"
    labels:
      - More accurate information
      - Better explained
      - More concise
      - More thorough
      - Better formatting
      - More appropriate tone
      - Safer/less harmful

  # Free-text justification
  - annotation_type: text
    name: justification
    description: "Briefly explain your preference"
    multiline: true
    required: false
    placeholder: "Why did you prefer one response over the other?"

# User settings
require_password: false

# Output
output_annotation_dir: "annotation_output/"
export_annotation_format: "jsonl"