task_name: "Sentiment Classification"
task_dir: "."
 
data_files:
  - "data/reviews.jsonl"
 
item_properties:
  id_key: id
  text_key: text
 
solo_mode:
  enabled: true
 
  # LLM provider
  llm:
    endpoint_type: openai
    model: "gpt-4o"
    api_key: ${OPENAI_API_KEY}
 
  # Basic thresholds
  seed_count: 50
  accuracy_threshold: 0.92
  confidence_threshold: 0.85
 
annotation_schemes:
  - annotation_type: radio
    name: sentiment
    labels:
      - Positive
      - Neutral
      - Negative
 
output_annotation_dir: "output/"
output_annotation_format: "jsonl"

solo_mode:
  enabled: true
 
  # LLM configuration
  llm:
    endpoint_type: openai        # openai, anthropic, ollama, vllm
    model: "gpt-4o"
    api_key: ${OPENAI_API_KEY}
    temperature: 0.1             # low temperature for consistency
    max_tokens: 256
 
  # Phase control
  phases:
    seed:
      count: 50                  # number of seed instances
      selection: diversity        # diversity, random, or stratified
      embedding_model: "all-MiniLM-L6-v2"
 
    calibration:
      batch_size: 100
      holdout_fraction: 0.2      # fraction of seed used for validation
 
    confusion_analysis:
      min_samples: 30
      significance_threshold: 0.05
 
    guideline_refinement:
      auto_suggest: true         # LLM suggests guideline edits
      require_approval: true     # human must approve changes
 
    labeling_functions:
      enabled: true
      max_functions: 20
      min_precision: 0.90        # only keep high-precision rules
      min_coverage: 0.01         # must cover at least 1% of data
 
    active_labeling:
      batch_size: 25
      strategy: uncertainty       # uncertainty, diversity, or hybrid
      max_batches: 10
 
    refinement_loop:
      max_iterations: 3
      improvement_threshold: 0.02
 
    disagreement_exploration:
      max_instances: 200
      sort_by: confidence_gap
 
    edge_case_synthesis:
      enabled: true
      count: 50
      diversity_weight: 0.3
 
    confidence_escalation:
      escalation_budget: 200     # max instances to escalate
      batch_size: 25
      stop_when_stable: true     # stop if last batch accuracy is 100%
 
    prompt_optimization:
      enabled: true
      candidates: 10             # number of prompt variants to try
      metric: f1_macro
      search_strategy: bayesian  # bayesian, grid, or random
 
    final_validation:
      sample_size: 100
      min_accuracy: 0.92
      fallback_phase: 6          # go back to Phase 6 if validation fails
 
  # Instance prioritization across phases
  prioritization:
    pools:
      - name: uncertain
        weight: 0.30
        description: "LLM confidence below threshold"
      - name: disagreement
        weight: 0.25
        description: "LLM and labeling functions disagree"
      - name: boundary
        weight: 0.20
        description: "Near decision boundary in embedding space"
      - name: novel
        weight: 0.10
        description: "Far from all existing labeled examples"
      - name: error_pattern
        weight: 0.10
        description: "Matches known confusion patterns"
      - name: random
        weight: 0.05
        description: "Random sample for calibration"

python -m potato.solo confusion --config config.yaml

solo_mode:
  llm:
    endpoint_type: anthropic
    model: "claude-sonnet-4-20250514"
    api_key: ${ANTHROPIC_API_KEY}
 
  phases:
    refinement_loop:
      max_iterations: 3
      improvement_threshold: 0.02    # stop if improvement is less than 2%

# Auto-generated labeling function 1
# Precision: 0.96, Coverage: 0.08
def lf_strong_positive_words(text):
    positive = {"excellent", "amazing", "fantastic", "outstanding", "perfect"}
    if any(w in text.lower() for w in positive):
        if not any(neg in text.lower() for neg in {"not", "never", "no"}):
            return "Positive"
    return None  # abstain
 
# Auto-generated labeling function 2
# Precision: 0.93, Coverage: 0.05
def lf_explicit_negative(text):
    negative = {"terrible", "awful", "horrible", "worst", "disgusting"}
    if any(w in text.lower() for w in negative):
        return "Negative"
    return None

solo_mode:
  phases:
    labeling_functions:
      enabled: true
      max_functions: 20
      min_precision: 0.90
      min_coverage: 0.01
      types:
        - keyword_match
        - regex_pattern
        - length_threshold
        - embedding_cluster

solo_mode:
  phases:
    disagreement_exploration:
      max_instances: 200
      sort_by: confidence_gap     # or "lf_disagreement" or "random"
      show_llm_reasoning: true    # display LLM's chain-of-thought
      show_nearest_neighbors: 3   # show 3 nearest labeled examples

solo_mode:
  phases:
    confidence_escalation:
      escalation_budget: 200
      batch_size: 25
      stop_when_stable: true
      stability_window: 3        # stop if last 3 batches are all correct

solo_mode:
  prioritization:
    pools:
      - name: uncertain
        weight: 0.30
      - name: disagreement
        weight: 0.25
      - name: boundary
        weight: 0.20
      - name: novel
        weight: 0.10
      - name: error_pattern
        weight: 0.10
      - name: random
        weight: 0.05

solo_mode:
  phases:
    edge_case_synthesis:
      enabled: true
      count: 50
      diversity_weight: 0.3
      confusion_pairs:            # focus on these error types
        - ["neutral", "positive"]
        - ["negative", "neutral"]

solo_mode:
  phases:
    prompt_optimization:
      enabled: true
      candidates: 10
      metric: f1_macro
      search_strategy: bayesian
      variations:
        - instruction_style      # formal vs. conversational
        - example_ordering       # random, by-class, by-difficulty
        - reasoning_mode         # direct, chain-of-thought, self-consistency
        - example_count          # 3, 5, 10, 15 few-shot examples

python -m potato.solo status --config config.yaml

solo_mode:
  enabled: true
  # ... solo mode config ...
 
  # Hybrid: assign verification sample to second annotator
  verification:
    enabled: true
    sample_fraction: 0.15
    annotator: "reviewer_1"