Multilingual News Article Similarity

# Multilingual News Article Similarity
# Based on Chen et al., SemEval 2022
# Paper: https://aclanthology.org/2022.semeval-1.155/
# Dataset: https://github.com/euagendas/semeval_8_2022_ia_downloader
#
# This task asks annotators to rate the similarity between two news
# articles across seven dimensions of similarity, each on a 4-point
# Likert scale. Articles may be in different languages.
#
# Following the paper's codebook, each dimension is answered on a
# four-point scale where 1 = Very Similar, 2 = Somewhat Similar,
# 3 = Somewhat Dissimilar, and 4 = Very Dissimilar.
#
# Dimensions (codebook questions):
#   GEO      - Geographic focus (places, cities, countries)
#   ENT      - Named entities (people, companies, organizations, products)
#   TIME     - Time periods described / relevance
#   NARRATIVE- Narrative schemas presented
#   OVERALL  - Whether the articles cover the same substantive news story
#   STYLE    - Writing style
#   TONE     - Tone

annotation_task_name: "Multilingual News Article Similarity"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: likert
    name: geography
    description: "GEO: How similar is the geographic focus (places, cities, countries, etc.) of the two articles?"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4
  - annotation_type: likert
    name: entities
    description: "ENT: How similar are the named entities (people, companies, organizations, products, named living beings), excluding locations already considered above?"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4
  - annotation_type: likert
    name: time
    description: "TIME: Are the two articles relevant to similar time periods or describing similar time periods?"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4
  - annotation_type: likert
    name: narrative
    description: "NAR: How similar are the narrative schemas presented in the two articles?"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4
  - annotation_type: likert
    name: overall
    description: "OVERALL: Overall, are the two articles covering the same substantive news story? (excluding style, framing, and tone)"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4
  - annotation_type: likert
    name: style
    description: "STYLE: Do the articles have similar writing styles?"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4
  - annotation_type: likert
    name: tone
    description: "TONE: Do the articles have similar tones?"
    min_label: "Very Similar"
    max_label: "Very Dissimilar"
    size: 4

annotation_instructions: |
  You will see two news articles (possibly in different languages) and the language pair.
  Rate the similarity of the two articles on seven dimensions. Each dimension uses a
  four-point scale where 1 = Very Similar, 2 = Somewhat Similar,
  3 = Somewhat Dissimilar, and 4 = Very Dissimilar.
  - Geography: similarity of the geographic focus (places, cities, countries).
  - Entities: similarity of the named entities (people, companies, organizations, products).
  - Time: whether the articles describe or are relevant to similar time periods.
  - Narrative: similarity of the narrative schemas presented.
  - Overall: whether the two articles cover the same substantive news story (excluding style, framing, and tone).
  - Style: whether the articles have similar writing styles.
  - Tone: whether the articles have similar tones.

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #ecfdf5; border: 1px solid #a7f3d0; border-radius: 8px; padding: 12px; margin-bottom: 12px;">
      <strong style="color: #065f46;">Language Pair:</strong>
      <span style="font-size: 15px; margin-left: 8px;">{{language}}</span>
    </div>
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Article 1:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #a16207;">Article 2:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{article_2}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 3
allow_skip: true
skip_reason_required: false