Fact-Checking in Community Question Answering Forums

# Fact-Checking in Community Question Answering Forums
# Based on Mihaylova et al., SemEval 2019
# Paper: https://aclanthology.org/S19-2149/
# Dataset: http://alt.qcri.org/semeval2019/index.php?id=tasks
#
# This task asks annotators to verify the factual accuracy of answers
# posted in community question answering forums. Annotators read both
# the question and the answer, then judge the truthfulness of the answer.
#
# Truthfulness Labels (SemEval-2019 Task 8, Subtask B):
# - True: The answer is factually true
# - False: The answer is factually false
# - Non-Factual: The answer does not provide a factual response
#   (e.g., an opinion, a joke, or advice rather than a verifiable claim)

annotation_task_name: "Fact-Checking in Community QA Forums"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: radio
    name: factual_accuracy
    description: "How factually accurate is this answer?"
    labels:
      - "True"
      - "False"
      - "Non-Factual"
    keyboard_shortcuts:
      "True": "1"
      "False": "2"
      "Non-Factual": "3"
    tooltips:
      "True": "The answer is factually true and supported by evidence"
      "False": "The answer is factually false or misleading"
      "Non-Factual": "The answer does not give a factual response (e.g., an opinion, joke, or advice)"

annotation_instructions: |
  You will be shown a question from a community forum along with an answer.
  Your task is to judge the factual accuracy of the answer.
  Consider whether the answer is True, False, or Non-Factual (i.e., it does not
  give a factual response, such as an opinion, a joke, or personal advice).

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 12px; margin-bottom: 12px;">
      <strong style="color: #a16207;">Forum:</strong>
      <span style="font-size: 15px;">{{forum}}</span>
    </div>
    <div style="background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; padding: 16px; margin-bottom: 12px;">
      <strong style="color: #64748b;">Question:</strong>
      <p style="font-size: 15px; line-height: 1.6; margin: 6px 0 0 0;">{{question}}</p>
    </div>
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Answer:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false