Parquet 내보내기

output_annotation_dir: "output/"
output_annotation_format: "parquet"

output_annotation_dir: "output/"
output_annotation_format: "jsonl"
 
parquet_export:
  enabled: true
  output_dir: "output/parquet/"
  auto_export: true              # export after each annotation session

python -m potato.export parquet --config config.yaml --output ./parquet_output/

parquet_export:
  enabled: true
  output_dir: "output/parquet/"
 
  compression: snappy            # snappy (default), gzip, zstd, lz4, brotli, none
  row_group_size: 50000          # rows per row group (affects read performance)
  use_dictionary: true           # dictionary encoding for string columns
  write_statistics: true         # column statistics for query optimization

import pandas as pd
 
annotations = pd.read_parquet("output/parquet/annotations.parquet")
spans = pd.read_parquet("output/parquet/spans.parquet")
items = pd.read_parquet("output/parquet/items.parquet")
 
# Filter to a specific schema
sentiment = annotations[annotations["schema_name"] == "sentiment"]
 
# Compute inter-annotator agreement
from sklearn.metrics import cohen_kappa_score
pivot = sentiment.pivot(index="instance_id", columns="annotator", values="value")
kappa = cohen_kappa_score(pivot.iloc[:, 0], pivot.iloc[:, 1])

-- Direct query without loading into memory
SELECT instance_id, value, COUNT(*) as annotator_count
FROM 'output/parquet/annotations.parquet'
WHERE schema_name = 'sentiment'
GROUP BY instance_id, value
ORDER BY annotator_count DESC;
 
-- Join annotations with items
SELECT a.instance_id, i.text, a.value, a.annotator
FROM 'output/parquet/annotations.parquet' a
JOIN 'output/parquet/items.parquet' i
  ON a.instance_id = i.instance_id
WHERE a.schema_name = 'sentiment';

import pyarrow.parquet as pq
 
# Read specific columns only (fast for wide tables)
table = pq.read_table(
    "output/parquet/annotations.parquet",
    columns=["instance_id", "value", "annotator"]
)
 
# Convert to pandas
df = table.to_pandas()
 
# Read with row group filtering
parquet_file = pq.ParquetFile("output/parquet/annotations.parquet")
print(f"Row groups: {parquet_file.metadata.num_row_groups}")
print(f"Total rows: {parquet_file.metadata.num_rows}")

from datasets import load_dataset
 
# Load directly from Parquet files
dataset = load_dataset("parquet", data_files={
    "annotations": "output/parquet/annotations.parquet",
    "spans": "output/parquet/spans.parquet",
    "items": "output/parquet/items.parquet",
})
 
# Access as a regular HF dataset
print(dataset["annotations"][0])
 
# Push to Hugging Face Hub
dataset["annotations"].push_to_hub("my-org/my-annotations", split="train")

import polars as pl
 
annotations = pl.read_parquet("output/parquet/annotations.parquet")
 
# Fast aggregation
label_counts = (
    annotations
    .filter(pl.col("schema_name") == "sentiment")
    .group_by("value")
    .agg(pl.count().alias("count"))
    .sort("count", descending=True)
)
print(label_counts)

parquet_export:
  enabled: true
  output_dir: "output/parquet/"
  incremental: true
  partition_by: date             # date, annotator, or none

# pandas reads partitioned directories automatically
df = pd.read_parquet("output/parquet/annotations/")
 
# DuckDB handles partitions natively
# SELECT * FROM 'output/parquet/annotations/**/*.parquet'

parquet_export:
  enabled: true
  output_dir: "output/parquet/"
 
  # When to export
  auto_export: true              # export after each session (default: false)
  export_on_shutdown: true       # export when server stops (default: true)
 
  # File settings
  compression: snappy
  row_group_size: 50000
  use_dictionary: true
  write_statistics: true
 
  # Incremental settings
  incremental: false
  partition_by: none             # none, date, annotator
 
  # Schema-specific options
  flatten_complex_types: false   # flatten JSON values into columns
  include_raw_json: true         # include raw JSON alongside flattened columns
 
  # Span export
  export_spans: true             # generate spans.parquet
  export_items: true             # generate items.parquet

task_name: "NER Annotation Project"
task_dir: "."
 
data_files:
  - "data/documents.jsonl"
 
item_properties:
  id_key: doc_id
  text_key: text
 
annotation_schemes:
  - annotation_type: span
    name: entities
    labels:
      - name: PERSON
        color: "#3b82f6"
      - name: ORGANIZATION
        color: "#22c55e"
      - name: LOCATION
        color: "#f59e0b"
 
output_annotation_dir: "output/"
output_annotation_format: "jsonl"
 
parquet_export:
  enabled: true
  output_dir: "output/parquet/"
  compression: zstd
  auto_export: true
  export_spans: true
  export_items: true

import pandas as pd
 
spans = pd.read_parquet("output/parquet/spans.parquet")
 
# Entity type distribution
print(spans["label"].value_counts())
 
# Average span length by type
spans["length"] = spans["end_offset"] - spans["start_offset"]
print(spans.groupby("label")["length"].mean())