DCASE Few-Shot Bioacoustic Event Detection

# yaml-language-server: $schema=https://potatoannotator.readthedocs.io/en/latest/schemas/potato-config.schema.json
# DCASE 2022 Task 5, few-shot bioacoustic event detection: find every call of
# one class after hearing five examples.
#
# Dataset: "DCASE 2022 Task 5: Few-shot Bioacoustic Event Detection Development
#   Set" (Nolasco, Singh, Strandburg-Peshkin, Gill, Pamula, Morford, Emmerson,
#   Jensen, Whitehead, Kiskin, Vidana-Vila, Lostanlen, Morfi, Stowell), Zenodo,
#   version 3. Data: https://zenodo.org/records/6482837
#   (DOI 10.5281/zenodo.6482837), CC BY 4.0.
# Task:    https://dcase.community/challenge2022/task-few-shot-bioacoustic-event-detection
# Paper:   Ecological Informatics 77, 102258, 2023. DOI: 10.1016/j.ecoinf.2023.102258
#   @article{Nolasco_2023,
#     title   = {Learning to detect an animal sound from five examples},
#     volume  = {77}, journal = {Ecological Informatics}, publisher = {Elsevier BV},
#     author  = {Nolasco, Ines and Singh, Shubhr and Morfi, Veronica and
#                Lostanlen, Vincent and Strandburg-Peshkin, Ariana and
#                Vida{\~n}a-Vila, Ester and Gill, Lisa and Pamu{\l}a, Hanna and
#                Whitehead, Helen and Kiskin, Ivan and Jensen, Frants H. and
#                Morford, Joe and Emmerson, Michael G. and Versace, Elisabetta and
#                Grout, Emily and Liu, Haohe and Ghani, Burooj and Stowell, Dan},
#     year    = {2023}, month = nov, pages = {102258},
#     DOI     = {10.1016/j.ecoinf.2023.102258}}
# Media:   twelve short MP3 excerpts in media/ are cut from the dataset's own
#   validation recordings ME1.wav and ME2.wav (meerkats) and are real. They are
#   CC BY 4.0; see media/LICENSE-media.txt for the attribution.
#
# The few-shot protocol, quoted from the paper (section 2.1) and the DCASE task
# page:
#   "Given one long audio recording (or multiple audio recordings), as well as
#   annotations on the onset and offset time for each of the first five sound
#   events of interest, identify the onset and offset times for all other such
#   sound events in the recording(s)."
#   "Participants must treat the task as a 5-shot setting and only use the first
#   five POS annotations for the class of interest for each file, when trying to
#   predict the rest."
#   "Each audio file should be treated separately of the rest, as there is
#   possible overlap between the classes of the evaluation set across different
#   audio files."
#   "Participants should keep in mind that our evaluation metric ignores
#   anything before the end time of the fifth positive event, hence using
#   randomly selected events from the validation set may lead to incorrect
#   performance values."
#
# Label glossary (paper section 2.2, task page):
#   POS  a positive example of the class of interest for this recording.
#   UNK  unknown: "where the human annotator(s) were unsure whether a sound event
#        should be considered in the positive class". Evaluation excludes UNK
#        time regions, because "their correct label is ambiguous".
#   NEG  a negative example (background or non-target event). The released
#        training set carries POS, NEG and UNK per class. The validation files
#        used here carry only POS and UNK for one class of interest, so this
#        design has no NEG label.
# The class of interest per file comes from DCASE2022_task5_Validation_set_classes.csv
# in the same record (ME1 is "Meerkat move call", ME2 is "Meerkat alarm call").
#
# Adaptations:
#   - The dataset has 18 validation recordings (hours long) and 174 training
#     recordings. This sample uses two meerkat validation recordings, ME1 and
#     ME2, cut into ten windows of 30 to 42 seconds, each re-encoded as 64 kbps
#     mono MP3. The windows are chosen to hold POS events, UNK events, or (item
#     dcase_002) no event at all.
#   - Counts and sampling rate. ME1.csv and ME2.csv hold 73 rows (62 POS and
#     11 UNK), although the DCASE task page labels 73 as events "excl. UNK".
#     The ME files are 48 kHz mono, 24-bit (read from the WAV headers, and the
#     task page gives 48000 Hz), while the paper's text gives 44.1 kHz for the
#     ME66 microphone. This design follows the files.
#   - Author name: Crossref and Zenodo spell it "Ines Nolasco" (kept in the
#     BibTeX below), while the paper's title page spells it "Inês Nolasco".
#   - Each window starts after the end of the fifth POS event of its recording
#     (13.2 s in ME1, 7.9 s in ME2). The "shots" are a separate clip, played above
#     the waveform, that holds exactly those five events. The paper also
#     says a system may assume the periods between the first five examples are
#     negative.
#   - The dataset's released POS and UNK segments for each window are loaded as
#     pre-annotations (pre_annotation), so annotators check and correct them.
#     Times are shifted to the window start. The original task asks a system, not
#     a person, to produce them. No event crosses a window edge.
#   - Segments are start and end times only, because the dataset marks no
#     frequency bounds (the annotation files have Starttime, Endtime and the
#     class column).
#
# Run from this folder:
#   potato start config.yaml -p 8000
#
# Training (optional). The hf-audio-tagger trainer consumes audio_annotation
# segments (pip install 'potato-annotation[train-audio]'). It fine-tunes a
# wav2vec 2.0-style speech encoder to say, for each frame of about 20 ms, which
# labels are active. Predictions are written only onto windows outside the
# training split, where they replace the dataset segments loaded into the same
# predictions field. It reads local audio files, which is
# why the excerpts ship in media/. A stretch nobody marked counts as "no label
# here", which suits a window where every event is marked and is wrong for a
# file where only the first five events were marked. The windows here are fully
# marked after the shots, so the rule holds. The trainer does not implement the few-shot method of the paper (a prototypical network), does
# not use the five shots as a support set, and does not serve or deploy the
# model. Twenty minutes of two recordings are a demonstration, not enough to train
# a useful detector. Open /admin/training with the key below, pick hf-audio-tagger
# and the events scheme, and click Train now. The base model is a public
# Hugging Face model downloaded on first use.
port: 8000
annotation_task_name: "DCASE 2022 few-shot bioacoustic events: meerkat calls"
task_dir: "."
output_annotation_dir: "annotation_output"
export_annotation_format: jsonl
data_files:
  - "sample-data.json"
item_properties:
  id_key: id
  text_key: audio_url

require_password: false
admin_api_key: "demo-admin-key"

user_config:
  allow_all_users: true
  users: []

instance_display:
  fields:
    - key: class_of_interest
      type: text
      label: "Class of interest (the only class to mark)"
    - key: support_audio
      type: audio
      label: "The five shots (first five POS events of this recording)"
    - key: window
      type: text
      label: "Window to annotate"

pre_annotation:
  enabled: true
  field: predictions
  allow_modification: true

annotation_schemes:
  - annotation_type: audio_annotation
    name: events
    description: "Listen to the five shots, then mark the start and end of every other event of the class of interest in the window. Use POS when the sound is a call of that class and UNK when you are unsure."
    mode: label
    source_field: audio_url
    labels:
      - name: POS
        color: "#16a34a"
        key_value: "1"
      - name: UNK
        color: "#d97706"
        key_value: "2"
    zoom_enabled: true
    playback_rate_control: true
    spectrogram: true
    spectrogram_options:
      frequency_range: [0, 8000]
      color_map: viridis

annotation_instructions: "Each recording has one class of interest, named above the player. Play the five shots to learn the sound, then mark every other event of that class in the window with POS. Mark a sound UNK when you cannot tell whether it belongs to the class. Other sounds are background and are not marked. The five shots themselves are not in the window."

model_training:
  enabled: true
  writeback:
    schemas: ["events"]
    include_unlabeled: true
    min_confidence: 0.5