コンテンツモデレーションアノテーションの設定

surveyflow:
  on: true
  order:
    - pre_annotation
  pre_annotation:
    - "surveyflow/content_warning.jsonl"

max_annotations_per_user: 100

annotation_schemes:
  - annotation_type: radio
    name: toxicity_level
    description: "Rate the toxicity level of this content"
    labels:
      - name: "Not toxic"
        tooltip: "No harmful content"
 
      - name: "Mildly toxic"
        tooltip: "Rude or insensitive but not severe"
 
      - name: "Moderately toxic"
        tooltip: "Clearly offensive or harmful"
 
      - name: "Severely toxic"
        tooltip: "Extremely offensive, threatening, or dangerous"

annotation_schemes:
  - annotation_type: multiselect
    name: toxicity_types
    description: "Select all types of toxicity present"
    labels:
      - name: "Profanity or obscenity"
        tooltip: "Swear words, vulgar language"
 
      - name: "Insults"
        tooltip: "Personal attacks, name-calling"
 
      - name: "Threats"
        tooltip: "Threats of violence or harm"
 
      - name: "Hate speech"
        tooltip: "Targeting protected groups"
 
      - name: "Harassment"
        tooltip: "Targeted, persistent hostility"
 
      - name: "Sexual content"
        tooltip: "Explicit or suggestive content"
 
      - name: "Self-harm or suicide"
        tooltip: "Promoting or glorifying self-harm"
 
      - name: "Misinformation"
        tooltip: "Demonstrably false claims"
 
      - name: "Spam or scam"
        tooltip: "Unwanted promotional content"

annotation_schemes:
  - annotation_type: multiselect
    name: target_groups
    description: "Which groups are targeted? Leave empty if this is not hate speech"
    labels:
      - name: "Race or ethnicity"
      - name: "Religion"
      - name: "Gender"
      - name: "Sexual orientation"
      - name: "Disability"
      - name: "Nationality or origin"
      - name: "Age"
      - name: "Other protected group"

annotation_schemes:
  - annotation_type: radio
    name: hate_severity
    description: "Severity of hate speech"
    labels:
      - name: "Implicit"
        tooltip: "Coded language, dog whistles"
 
      - name: "Explicit, mild"
        tooltip: "Clear but not threatening"
 
      - name: "Explicit, severe"
        tooltip: "Dehumanizing, threatening, or violent"

annotation_schemes:
  - annotation_type: radio
    name: context_appropriate
    description: "Is this content appropriate for the platform context?"
    labels:
      - name: "Appropriate for context"
      - name: "Borderline"
      - name: "Inappropriate for context"
 
  - annotation_type: text
    name: context_notes
    description: "Explain your contextual reasoning"
    multiline: true
    rows: 3

annotation_schemes:
  - annotation_type: radio
    name: intent
    description: "What is the apparent intent?"
    labels:
      - name: "Genuine attack"
        tooltip: "Intent to harm or offend"
 
      - name: "Satire or parody"
        tooltip: "Mocking toxic behavior"
 
      - name: "Quote or report"
        tooltip: "Reporting or discussing toxic content"
 
      - name: "Reclaimed language"
        tooltip: "In-group use of slurs"
 
      - name: "Unclear intent"

annotation_schemes:
  - annotation_type: multiselect
    name: image_violations
    description: "Select all policy violations"
    labels:
      - name: "Nudity or sexual content"
      - name: "Graphic violence"
      - name: "Gore or disturbing content"
      - name: "Hate symbols"
      - name: "Dangerous acts"
      - name: "Child safety concern"
        priority: critical
        escalate: true
      - name: "No violations"
 
  - annotation_type: radio
    name: action_recommendation
    description: "Recommended action"
    labels:
      - name: "Approve"
      - name: "Age-restrict"
      - name: "Add warning label"
      - name: "Remove"
      - name: "Escalate to specialist"

# Items with known labels, mixed into the regular queue
gold_standards:
  enabled: true
  items_file: data/moderation_gold.json
  mode: mixed
  frequency: 20
  accuracy:
    min_threshold: 0.7
    evaluation_count: 10
 
# Three judgements on every item
num_annotators_per_item: 3
 
# Disagreements go to a specialist queue
adjudication:
  enabled: true
  adjudicator_users:
    - trust_safety_lead
  min_annotations: 3
  agreement_threshold: 0.67
  require_notes_on_override: true
  show_agreement_scores: true

annotation_task_name: "Content Moderation"
task_dir: "."
output_annotation_dir: "annotations/"
 
data_files:
  - "data/reports.json"
 
item_properties:
  id_key: id
  text_key: text
 
num_annotators_per_item: 3
 
annotation_schemes:
  - annotation_type: radio
    name: toxicity
    description: "Toxicity level"
    labels:
      - name: "None"
      - name: "Mild"
      - name: "Moderate"
      - name: "Severe"
 
  - annotation_type: multiselect
    name: categories
    description: "Types of harmful content (select all)"
    labels:
      - name: "Hate speech"
      - name: "Harassment"
      - name: "Violence or threats"
      - name: "Sexual content"
      - name: "Self-harm"
      - name: "Spam"
      - name: "None"
 
  - annotation_type: likert
    name: confidence
    description: "How confident are you in this judgment?"
    size: 5
    min_label: "Uncertain"
    max_label: "Very confident"
 
  - annotation_type: text
    name: notes
    description: "Additional notes (optional)"
    multiline: true
    rows: 4
 
export_annotation_format: jsonl