Machine-Generated Text Detection

# Machine-Generated Text Detection
# Based on Wang et al., SemEval 2024
# Paper: https://aclanthology.org/2024.semeval-1.279/
# Dataset: https://github.com/mbzuai-nlp/SemEval2024-task8
#
# This task asks annotators to determine whether a text was written by
# a human, generated by a machine (LLM), contains mixed human-machine
# authorship, or if the origin is uncertain.

annotation_task_name: "Machine-Generated Text Detection"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: radio
    name: authorship_judgment
    description: "Who wrote this text?"
    labels:
      - "Human-Written"
      - "Machine-Generated"
      - "Mixed"
      - "Uncertain"
    keyboard_shortcuts:
      "Human-Written": "1"
      "Machine-Generated": "2"
      "Mixed": "3"
      "Uncertain": "4"
    tooltips:
      "Human-Written": "The text appears to be entirely written by a human author"
      "Machine-Generated": "The text appears to be entirely generated by an AI/LLM"
      "Mixed": "The text appears to contain both human-written and machine-generated portions"
      "Uncertain": "You cannot determine the authorship with reasonable confidence"

annotation_instructions: |
  You will be shown a text passage. Your task is to determine whether it was:
  1. Written entirely by a human
  2. Generated entirely by an AI language model
  3. A mixture of human and machine writing
  4. Uncertain if you cannot determine the origin
  Consider factors like fluency, coherence, specificity, style, and any telltale patterns.

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Text:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
    <div style="background: #f0fdf4; border: 1px solid #bbf7d0; border-radius: 8px; padding: 12px;">
      <strong style="color: #166534;">Domain:</strong> <span>{{domain}}</span>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false