CODWOE - Comparing Dictionaries and Word Embeddings

# CODWOE - Comparing Dictionaries and Word Embeddings
# Based on Mickus et al., SemEval 2022
# Paper: https://aclanthology.org/2022.semeval-1.1/
# Dataset: https://codwoe.atilf.fr/
#
# CODWOE (SemEval-2022 Task 1) compared dictionaries and word embeddings
# through two automatic subtasks: definition modeling (generate a definition
# from a word embedding) and reverse dictionary (generate an embedding from a
# definition). Neither is directly human-annotatable, so this Potato design
# adapts the definition-modeling direction into a human writing task and
# grounds a companion judgment in the reverse-dictionary direction.
#
# This task asks annotators to (1) write a definition for a target word based
# on the context in which it appears (definition modeling), and (2) judge how
# uniquely that word could be recovered from a definition alone (reverse
# dictionary difficulty).
#
# Recoverability Labels:
# - Easily identifiable: The definition points clearly to this exact word
# - Somewhat ambiguous: The definition could fit a few related words
# - Highly ambiguous: The definition could describe many different words

annotation_task_name: "CODWOE - Comparing Dictionaries and Word Embeddings"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: text
    name: definition
    description: "Write a definition for the target word as used in this context"

  - annotation_type: radio
    name: recoverability
    description: "If given only your definition, how uniquely could the target word be recovered?"
    labels:
      - "Easily identifiable"
      - "Somewhat ambiguous"
      - "Highly ambiguous"
    keyboard_shortcuts:
      "Easily identifiable": "1"
      "Somewhat ambiguous": "2"
      "Highly ambiguous": "3"
    tooltips:
      "Easily identifiable": "The definition points clearly to this exact target word"
      "Somewhat ambiguous": "The definition could plausibly fit a few closely related words"
      "Highly ambiguous": "The definition could describe many different words"

annotation_instructions: |
  This task adapts SemEval-2022 Task 1 (CODWOE), which paired definition
  modeling with a reverse-dictionary subtask. You will see a sentence with a
  target word and the language of the text.
  1. Read the sentence and identify the target word.
  2. Write a clear, concise definition for the target word as it is used in this
     context (definition modeling).
  3. Judge how uniquely the target word could be recovered from your definition
     alone (reverse-dictionary difficulty).

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="display: flex; gap: 12px; margin-bottom: 12px;">
      <div style="background: #ecfdf5; border: 1px solid #a7f3d0; border-radius: 8px; padding: 12px; flex: 1;">
        <strong style="color: #065f46;">Language:</strong>
        <span style="font-size: 15px; margin-left: 8px;">{{language}}</span>
      </div>
      <div style="background: #fef3c7; border: 1px solid #fde68a; border-radius: 8px; padding: 12px; flex: 1;">
        <strong style="color: #92400e;">Target Word:</strong>
        <span style="font-size: 15px; font-weight: bold; margin-left: 8px;">{{target_word}}</span>
      </div>
    </div>
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Context:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false