Argument Reasoning in Civil Procedure

# Argument Reasoning in Civil Procedure
# Based on SemEval-2024 Task 5 (Held & Habernal, 2024)
# Paper: https://aclanthology.org/2024.semeval-1.276/
# Dataset: https://github.com/trusthlt/legal-argument-reasoning-task
#
# Given a short introduction to a U.S. civil procedure topic, a question,
# and a single candidate answer (a possible solution argument), annotators
# perform BINARY classification: judge whether the answer candidate is
# correct (True) or incorrect (False).

annotation_task_name: "Argument Reasoning in Civil Procedure"
task_dir: "."

data_files:
  - sample-data.json

item_properties:
  id_key: "id"
  text_key: "text"

output_annotation_dir: "annotation_output/"
output_annotation_format: "json"

port: 8000
server_name: localhost

annotation_schemes:
  - annotation_type: radio
    name: answer_correctness
    description: "Is the candidate answer a correct solution to the question?"
    labels:
      - "Correct"
      - "Incorrect"
    keyboard_shortcuts:
      "Correct": "1"
      "Incorrect": "2"
    tooltips:
      "Correct": "The candidate answer correctly resolves the question (True)."
      "Incorrect": "The candidate answer does not correctly resolve the question (False)."

annotation_instructions: |
  You will be shown a short introduction to a topic in U.S. civil procedure,
  a specific question, and a single candidate answer (a possible solution
  argument). Your task is to:
  1. Read the introduction to understand the legal context.
  2. Read the question carefully.
  3. Read the candidate answer.
  4. Decide whether the candidate answer is a correct solution to the
     question. Select "Correct" (True) if it correctly resolves the question,
     or "Incorrect" (False) if it does not.

html_layout: |
  <div style="padding: 15px; max-width: 800px; margin: auto;">
    <div style="background: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #0369a1;">Introduction:</strong>
      <p style="font-size: 15px; line-height: 1.7; margin: 8px 0 0 0;">{{introduction}}</p>
    </div>
    <div style="background: #fefce8; border: 1px solid #fde68a; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #a16207;">Question:</strong>
      <p style="font-size: 16px; line-height: 1.7; margin: 8px 0 0 0;">{{text}}</p>
    </div>
    <div style="background: #f0fdf4; border: 1px solid #bbf7d0; border-radius: 8px; padding: 16px; margin-bottom: 16px;">
      <strong style="color: #15803d;">Candidate Answer:</strong>
      <p style="font-size: 15px; line-height: 1.7; margin: 8px 0 0 0;">{{answer_candidate}}</p>
    </div>
  </div>

allow_all_users: true
instances_per_annotator: 50
annotation_per_instance: 2
allow_skip: true
skip_reason_required: false