LLMs4Subjects - Subject Tagging for a Technical Library

# LLMs4Subjects - Subject Tagging for a Technical Library
# Based on D'Souza et al., SemEval-2025 Task 5
# Paper: https://aclanthology.org/2025.semeval-1.328/
# Dataset: https://github.com/jd-coderepos/llms4subjects
#
# SemEval-2025 Task 5 (LLMs4Subjects) asks systems to recommend subject
# headings from the GND (Gemeinsame Normdatei) vocabulary for technical
# bibliographic records in the TIB Leibniz Information Centre's bilingual
# (English/German) open-access TIBKAT catalog. This showcase is a
# simplified HUMAN-annotation adaptation: rather than the full open GND
# vocabulary of tens of thousands of headings, annotators tag each record
# with broad subject domains and its record type, reproducing the manual
# subject-indexing workflow the shared task automates.
#
# Note: the official gold standard uses the full GND subject taxonomy;
# the fixed domain list below is a demonstration simplification.

annotation_task_name: "LLMs4Subjects - Subject Tagging for a Technical Library"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: multiselect
    name: subject_categories
    description: "Select all subject domains this record should be indexed under (a simplified stand-in for GND subject headings)."
    labels:
      - "Computer Science"
      - "Mathematics"
      - "Physics"
      - "Biology"
      - "Medicine"
      - "Engineering"
      - "Social Science"
      - "Humanities"
      - "Law"
      - "Economics"
    tooltips:
      "Computer Science": "Algorithms, programming, AI, databases, software, etc."
      "Mathematics": "Pure or applied mathematics, statistics, logic"
      "Physics": "Classical, quantum, astrophysics, particle physics, etc."
      "Biology": "Molecular biology, ecology, genetics, evolution, etc."
      "Medicine": "Clinical medicine, pharmacology, public health, etc."
      "Engineering": "Mechanical, electrical, civil, chemical engineering, etc."
      "Social Science": "Psychology, sociology, political science, anthropology, etc."
      "Humanities": "History, philosophy, literature, linguistics, etc."
      "Law": "Legal theory, constitutional law, international law, etc."
      "Economics": "Micro/macroeconomics, finance, econometrics, etc."

  - annotation_type: radio
    name: record_type
    description: "What type of bibliographic record is this?"
    labels:
      - "Article"
      - "Book"
      - "Conference"
      - "Report"
      - "Thesis"
    keyboard_shortcuts:
      "Article": "1"
      "Book": "2"
      "Conference": "3"
      "Report": "4"
      "Thesis": "5"
    tooltips:
      "Article": "Journal or periodical article"
      "Book": "Monograph or edited book"
      "Conference": "Conference or workshop proceedings paper"
      "Report": "Technical or research report"
      "Thesis": "Dissertation or thesis"

annotation_instructions: |
  You are performing subject indexing for a technical-library catalog record
  (TIBKAT), in the style of SemEval-2025 Task 5 (LLMs4Subjects). Records may be
  in English or German. Your tasks are:
  1. Read the record's title and text and identify what it is about.
  2. Select all subject domains under which the record should be indexed.
  3. Choose the record type (article, book, conference, report, or thesis).

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #a16207;">Title:</strong>
      <p style="font-size: 17px; font-weight: 600; line-height: 1.5; margin: 8px 0 0 0;">{{title}}</p>
    </div>
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Text:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false