Interpretable Semantic Textual Similarity

# Interpretable Semantic Textual Similarity
# Based on Agirre et al., SemEval 2016
# Paper: https://aclanthology.org/S16-1082/
# Dataset: http://ixa2.si.ehu.eus/stswiki/
#
# Interpretable STS (iSTS) explains the commonalities and differences
# between two sentences. Annotators align chunks across the two sentences
# and label each alignment with a relation type and a similarity score.
#
# Alignment relation types:
# - EQUI:  chunks are semantically equivalent
# - OPPO:  chunks are opposite in meaning
# - SPE1:  chunk in Sentence 1 is more specific than its counterpart in Sentence 2
# - SPE2:  chunk in Sentence 2 is more specific than its counterpart in Sentence 1
# - SIMI:  chunks are similar in meaning (not EQUI/OPPO/SPE)
# - REL:   chunks are related in meaning (not EQUI/OPPO/SPE/SIMI)
# - NOALI: chunk is not aligned to any chunk in the other sentence
#
# Similarity score (0-5):
# 5 = maximally similar; 0 = no similarity. NOALI alignments have no score.

annotation_task_name: "Interpretable Semantic Textual Similarity"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: span
    name: alignment_relation
    description: "Highlight a chunk, then label the relation of its alignment to the other sentence."
    labels:
      - name: "EQUI"
        tooltip: "Chunks are semantically equivalent."
      - name: "OPPO"
        tooltip: "Chunks are opposite in meaning."
      - name: "SPE1"
        tooltip: "Chunk in Sentence 1 is more specific than its counterpart in Sentence 2."
      - name: "SPE2"
        tooltip: "Chunk in Sentence 2 is more specific than its counterpart in Sentence 1."
      - name: "SIMI"
        tooltip: "Chunks are similar in meaning (not EQUI/OPPO/SPE)."
      - name: "REL"
        tooltip: "Chunks are related in meaning (not EQUI/OPPO/SPE/SIMI)."
      - name: "NOALI"
        tooltip: "Chunk is not aligned to any chunk in the other sentence."

  - annotation_type: radio
    name: similarity_score
    description: "Similarity score for the alignment (0 = no similarity, 5 = maximally similar; NOALI has no score)."
    labels:
      - "0"
      - "1"
      - "2"
      - "3"
      - "4"
      - "5"

annotation_instructions: |
  You will be shown two sentences that have been divided into chunks. Your task is to:
  1. Align each chunk in one sentence with the corresponding chunk(s) in the other sentence.
  2. Label each alignment with a relation type: EQUI, OPPO, SPE1, SPE2, SIMI, REL, or NOALI.
  3. Give each aligned pair a similarity score from 0 (no similarity) to 5 (maximally similar).
  Chunks with no counterpart should be labeled NOALI.

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 12px;">
      <strong style="color: #0369a1;">Sentence 1:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
    <div style="background: #f0fdf4; border: 1px solid #bbf7d0; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #15803d;">Sentence 2:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{sentence_2}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false