QA over Tabular Data

# QA over Tabular Data
# Based on SemEval-2025 Task 8: Question Answering over Tabular Data (DataBench)
# Osés Grijalba, Ureña-López, Martínez Cámara, Camacho-Collados (SemEval 2025)
# Paper: https://aclanthology.org/2025.semeval-1.324/
# Dataset: https://huggingface.co/datasets/cardiffnlp/databench
#
# This task asks annotators to answer a natural-language question from a table by
# providing the value or list derived from the table, and to label the answer type
# (boolean, category, number, list of categories, or list of numbers), following the
# DataBench answer typology used in SemEval-2025 Task 8.

annotation_task_name: "QA over Tabular Data"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: text
    name: answer
    description: "Provide the answer to the question based on the table."

  - annotation_type: radio
    name: answer_type
    description: "What is the type of the answer? (DataBench answer typology)"
    labels:
      - "Boolean"
      - "Category"
      - "Number"
      - "List of Categories"
      - "List of Numbers"
    keyboard_shortcuts:
      "Boolean": "1"
      "Category": "2"
      "Number": "3"
      "List of Categories": "4"
      "List of Numbers": "5"
    tooltips:
      "Boolean": "The answer is True or False"
      "Category": "The answer is a single value drawn from the table (e.g. a name or label)"
      "Number": "The answer is a single numeric value"
      "List of Categories": "The answer is a list of values from the table"
      "List of Numbers": "The answer is a list of numeric values"

annotation_instructions: |
  You will be shown a question and a table. Your task is to:
  1. Read the question carefully.
  2. Examine the table content provided.
  3. Provide the exact answer as a value or list derived from the table.
  4. Label the type of the answer (boolean, category, number, or list).

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Question:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #a16207;">Table: {{table_title}}</strong>
      <pre style="font-size: 14px; line-height: 1.5; margin: 8px 0 0 0; white-space: pre-wrap;">{{table_content}}</pre>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false