Rating Plausibility of Word Senses through Narrative Understanding

# Rating Plausibility of Word Senses through Narrative Understanding
# Based on Gehring, Meyer, and Roth, SemEval-2026 Task 5
# Paper: https://aclanthology.org/2026.semeval-1.448/
# Task page: https://nlu-lab.github.io/semeval.html
#
# Each item is a short (4-5 sentence) story containing an ambiguous word
# (a homonym) in a target sentence. The story deliberately offers sparse
# cues. Given one candidate sense of the homonym, annotators rate how
# plausible that sense is in the narrative on a 1-5 Likert scale
# (1 = very implausible, 5 = very plausible). Built on the Ambistory dataset.

annotation_task_name: "Word-Sense Plausibility in Narratives"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "story"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: likert
    name: plausibility
    description: "How plausible is the candidate sense of the target word in this story?"
    size: 5
    min_label: "Very implausible"
    max_label: "Very plausible"

annotation_instructions: |
  Read the short story and note the target (ambiguous) word and the
  candidate sense provided. Using only the narrative context, rate how
  plausible it is that the target word carries that candidate sense, on a
  scale from 1 (very implausible) to 5 (very plausible).

html_layout: |
  <div style="padding: 15px; max-width: 820px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 10px;">
      <strong style="color: #0369a1;">Story:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{story}}</p>
    </div>
    <div style="background: #fef9c3; border: 1px solid #fde047; border-radius: 8px; padding: 12px;">
      <strong style="color: #854d0e;">Target word:</strong> <span>{{target_word}}</span><br/>
      <strong style="color: #854d0e;">Candidate sense:</strong> <span>{{candidate_sense}}</span>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 5
allow_skip: true