TriviaQA - Reading Comprehension

# TriviaQA - Reading Comprehension
# Based on Joshi et al., ACL 2017
# Paper: https://aclanthology.org/P17-1147/
# Dataset: http://nlp.cs.washington.edu/triviaqa/
#
# This task presents a trivia question along with an evidence passage.
# Annotators provide the answer and assess whether the answer can be
# determined from the context or requires external knowledge.
#
# Answerability Categories:
# - Answerable from Context: The evidence passage contains the answer
# - Requires External Knowledge: The passage alone is insufficient
# - Ambiguous: The passage may partially support multiple answers
#
# Annotation Guidelines:
# 1. Read the trivia question
# 2. Read the evidence passage carefully
# 3. Try to answer the question based on the passage
# 4. Type your answer in the text field
# 5. Classify how answerable the question is from the context

annotation_task_name: "TriviaQA - Reading Comprehension"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  # Step 1: Provide the answer
  - annotation_type: text
    name: answer
    description: "Type the answer to the trivia question"

  # Step 2: Assess answerability from context
  - annotation_type: radio
    name: answerability
    description: "Can the question be answered from the provided evidence passage?"
    labels:
      - "Answerable from Context"
      - "Requires External Knowledge"
      - "Ambiguous"
    keyboard_shortcuts:
      "Answerable from Context": "1"
      "Requires External Knowledge": "2"
      "Ambiguous": "3"
    tooltips:
      "Answerable from Context": "The evidence passage clearly contains the answer to the question"
      "Requires External Knowledge": "The passage does not contain enough information; external knowledge is needed"
      "Ambiguous": "The passage provides partial evidence but the answer is not definitively clear"

annotation_instructions: |
  You will be shown a trivia question and an evidence passage. Your task is to:
  1. Read the trivia question.
  2. Read the evidence passage carefully.
  3. Type your best answer to the question.
  4. Indicate whether the question is answerable from the passage alone,
     requires external knowledge, or is ambiguous.

  Try to answer using the passage first. If the passage does not contain the answer,
  you may use your own knowledge but mark it as "Requires External Knowledge."

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #a16207; font-size: 18px;">Trivia Question:</strong>
      <p style="font-size: 17px; line-height: 1.6; margin: 8px 0 0 0; font-weight: 500;">{{question}}</p>
    </div>
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Evidence Passage:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false