SciER Scientific Entity and Relation Extraction Dataset

# SciER: Scientific Entity and Relation Extraction
# Based on Zhang, Chen, Pan, Caragea, Latecki & Dragut, EMNLP 2024
# Paper: https://aclanthology.org/2024.emnlp-main.726/
# Dataset: https://github.com/edzq/SciER
#
# This task annotates scientific entities and relations across full-text ML
# papers. Annotators identify Dataset, Method, and Task entities, then link
# them with typed relations describing how the entities interact.
#
# Entity Types (3):
# - Dataset: benchmark datasets, corpora (e.g., CoNLL-2003, OntoNotes)
# - Method: models, algorithms, systems (e.g., BERT, CNN)
# - Task: research tasks/problems (e.g., named entity recognition)
#
# Relation Types (9):
# - USED-FOR: a method/dataset is used for a task or another method
# - TRAINED-WITH: a method is trained with a dataset
# - EVALUATED-WITH: a method is evaluated with a dataset
# - BENCHMARK-FOR: a dataset is a benchmark for a task
# - COMPARE-WITH: two entities are compared (symmetric)
# - SUBTASK-OF: a task is a subtask of another task
# - SUBCLASS-OF: a method is a subclass of another method
# - PART-OF: an entity is a component of another
# - SYNONYM-OF: two entities refer to the same thing (symmetric)
#
# Annotation Guidelines:
# 1. Read the sentence in context of the paper section
# 2. Identify all scientific entity mentions
# 3. Assign appropriate entity types
# 4. For each pair of related entities, select the relation type
# 5. Relations are directional: consider which entity is subject vs object

annotation_task_name: "SciER: Scientific Entity and Relation Extraction"
task_dir: "."

data_files:
  - sample-data.json
item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

annotation_schemes:
  # Step 1: Identify scientific entities
  - annotation_type: span
    name: scientific_entities
    description: "Highlight all scientific entities in the text"
    labels:
      - "Dataset"
      - "Method"
      - "Task"
    label_colors:
      "Dataset": "#22c55e"
      "Method": "#3b82f6"
      "Task": "#ef4444"
    keyboard_shortcuts:
      "Dataset": "1"
      "Method": "2"
      "Task": "3"
    tooltips:
      "Dataset": "Benchmark datasets or corpora (e.g., CoNLL-2003, OntoNotes)"
      "Method": "Models, algorithms, or systems (e.g., BERT, CNN)"
      "Task": "Research tasks or problems (e.g., named entity recognition)"
    allow_overlapping: false

  # Step 2: Link entities with typed relations
  - annotation_type: span_link
    name: scientific_relations
    description: "Draw relations between pairs of scientific entities"
    labels:
      - "USED-FOR"
      - "TRAINED-WITH"
      - "EVALUATED-WITH"
      - "BENCHMARK-FOR"
      - "COMPARE-WITH"
      - "SUBTASK-OF"
      - "SUBCLASS-OF"
      - "PART-OF"
      - "SYNONYM-OF"
    tooltips:
      "USED-FOR": "A method or dataset is used for a task or another method"
      "TRAINED-WITH": "A method is trained with a dataset"
      "EVALUATED-WITH": "A method is evaluated with a dataset"
      "BENCHMARK-FOR": "A dataset is a benchmark for a task"
      "COMPARE-WITH": "Two entities are compared (symmetric)"
      "SUBTASK-OF": "A task is a subtask of another task"
      "SUBCLASS-OF": "A method is a subclass of another method"
      "PART-OF": "An entity is a component of another"
      "SYNONYM-OF": "Two entities refer to the same thing (symmetric)"

html_layout: |
  <div style="margin-bottom: 10px; padding: 8px; background: #f0f4f8; border-radius: 4px;">
    <strong>Section:</strong> {{context}}
  </div>
  <div style="font-size: 16px; line-height: 1.6;">
    {{text}}
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false