Fact-Checking in Community Question Answering Forums

# Fact-Checking in Community Question Answering Forums
# Based on Mihaylova et al., SemEval 2019
# Paper: https://aclanthology.org/S19-2149/
# Dataset: http://alt.qcri.org/semeval2019/index.php?id=tasks
#
# This task asks annotators to verify the factual accuracy of answers
# posted in community question answering forums. Annotators read both
# the question and the answer, then judge the truthfulness of the answer.
#
# Truthfulness Labels:
# - True: The answer is factually correct
# - False: The answer is factually incorrect
# - Half-True: The answer contains a mix of true and false information

annotation_task_name: "Fact-Checking in Community QA Forums"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: radio
    name: factual_accuracy
    description: "How factually accurate is this answer?"
    labels:
      - "True"
      - "False"
      - "Half-True"
    keyboard_shortcuts:
      "True": "1"
      "False": "2"
      "Half-True": "3"
    tooltips:
      "True": "The answer is factually correct and supported by evidence"
      "False": "The answer is factually incorrect or misleading"
      "Half-True": "The answer contains a mix of accurate and inaccurate information"

annotation_instructions: |
  You will be shown a question from a community forum along with an answer.
  Your task is to judge the factual accuracy of the answer.
  Consider whether the claims made in the answer are true, false, or partially true.

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 12px; margin-bottom: 12px;">
      <strong style="color: #a16207;">Forum:</strong>
      <span style="font-size: 15px;">{{forum}}</span>
    </div>
    <div style="background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; padding: 16px; margin-bottom: 12px;">
      <strong style="color: #64748b;">Question:</strong>
      <p style="font-size: 15px; line-height: 1.6; margin: 6px 0 0 0;">{{question}}</p>
    </div>
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Answer:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false