Belebele Multilingual Reading Comprehension Benchmark

# Multilingual Reading Comprehension (Belebele)
# Based on Bandarkar et al., ACL 2024
# Paper: https://arxiv.org/abs/2308.16884
# Dataset: https://github.com/facebookresearch/belebele
#
# This task presents a passage and a multiple-choice question with four
# answer options. The Belebele benchmark provides parallel questions across
# 122 languages, making it a key resource for evaluating cross-lingual
# reading comprehension. This configuration uses English passages.
#
# Answer Options:
# - A, B, C, D: Four possible answers; exactly one is correct
#
# Difficulty Levels:
# - EASY: Answer is explicitly stated in the passage
# - MEDIUM: Requires simple inference or paraphrasing
# - HARD: Requires complex reasoning or synthesis of multiple parts
#
# Reasoning Types:
# - FACTUAL: Answer is directly stated in the passage
# - INFERENTIAL: Answer requires drawing conclusions from the text
# - LEXICAL: Answer requires understanding vocabulary or paraphrasing
# - STRUCTURAL: Answer requires understanding passage organization
#
# Annotation Guidelines:
# 1. Read the passage thoroughly before looking at the question
# 2. Read the question and all four answer options carefully
# 3. Select the single best answer based on the passage content
# 4. Do not rely on external knowledge -- answers come from the passage
# 5. Rate the difficulty based on how much reasoning is required
# 6. Identify what type of reasoning is needed to arrive at the answer
#
# Important Notes:
# - Only one answer is correct per question
# - Some distractors may be partially correct or plausible
# - The passage contains all information needed to answer

annotation_task_name: "Multilingual Reading Comprehension (Belebele)"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "passage"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  # Step 1: Select the correct answer
  - annotation_type: radio
    name: correct_answer
    description: "Based on the passage, which answer is correct?"
    labels:
      - "A"
      - "B"
      - "C"
      - "D"
    keyboard_shortcuts:
      "A": "1"
      "B": "2"
      "C": "3"
      "D": "4"
    tooltips:
      "A": "Select if option A is the correct answer"
      "B": "Select if option B is the correct answer"
      "C": "Select if option C is the correct answer"
      "D": "Select if option D is the correct answer"

  # Step 2: Rate difficulty
  - annotation_type: radio
    name: difficulty
    description: "How difficult is this question to answer correctly?"
    labels:
      - "easy"
      - "medium"
      - "hard"
    tooltips:
      "easy": "The answer is explicitly stated in the passage with minimal reasoning"
      "medium": "The answer requires some inference or paraphrasing"
      "hard": "The answer requires complex reasoning or synthesis of multiple passage parts"

  # Step 3: Reasoning type
  - annotation_type: radio
    name: reasoning_type
    description: "What type of reasoning is required to answer this question?"
    labels:
      - "factual"
      - "inferential"
      - "lexical"
      - "structural"
    tooltips:
      "factual": "The answer is directly stated in the passage (fact lookup)"
      "inferential": "The answer requires drawing a conclusion from passage information"
      "lexical": "The answer requires understanding vocabulary, synonyms, or paraphrases"
      "structural": "The answer requires understanding how the passage is organized"

annotation_instructions: |
  You will be shown a passage followed by a question and four answer options (A, B, C, D). Your task is to:
  1. Read the passage carefully.
  2. Select the one correct answer based on the passage.
  3. Rate how difficult the question is.
  4. Identify what type of reasoning is needed.

  Remember: base your answer only on the passage content, not on external knowledge.

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Passage:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{passage}}</p>
    </div>
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #a16207;">Question:</strong>
      <p style="font-size: 16px; line-height: 1.6; margin: 8px 0 0 0;">{{question}}</p>
    </div>
    <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
      <div style="background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px;">
        <strong style="color: #475569;">A:</strong> {{option_a}}
      </div>
      <div style="background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px;">
        <strong style="color: #475569;">B:</strong> {{option_b}}
      </div>
      <div style="background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px;">
        <strong style="color: #475569;">C:</strong> {{option_c}}
      </div>
      <div style="background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px;">
        <strong style="color: #475569;">D:</strong> {{option_d}}
      </div>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false