Add EmoNet Voice Bench results (42/42 classes, pooled r=0.304 rho=0.323) + eval artifacts
c75e48d verified | { | |
| "dataset": "t1a5anu-anon/emonet-voice-bench", | |
| "revision": "c0067f7b2c1b303d77394b42c09836f7be84932c", | |
| "license": "cc-by-4.0", | |
| "gated": false, | |
| "private": false, | |
| "downloaded_to": "/e/data1/datasets/playground/mmlaion/schuhmann1/dramabox/headspub/bench/emonet/data", | |
| "download_complete": true, | |
| "total_bytes": 1077182855, | |
| "paper": "arXiv:2506.09827 - EmoNet-Voice: A Fine-Grained, Expert-Verified Benchmark for Speech Emotion Detection", | |
| "splits": { | |
| "train": { | |
| "n_rows": 12600, | |
| "note": "ONLY split in repo; it IS the eval/benchmark set despite being named 'train'" | |
| } | |
| }, | |
| "configs": [ | |
| "default" | |
| ], | |
| "files": [ | |
| { | |
| "path": "data/data/train-00000-of-00003.parquet", | |
| "bytes": 366186448, | |
| "num_rows": 4200 | |
| }, | |
| { | |
| "path": "data/data/train-00001-of-00003.parquet", | |
| "bytes": 354533930, | |
| "num_rows": 4200 | |
| }, | |
| { | |
| "path": "data/data/train-00002-of-00003.parquet", | |
| "bytes": 356462477, | |
| "num_rows": 4200 | |
| } | |
| ], | |
| "schema": { | |
| "audioId": "struct<bytes: binary, path: string> (HF Audio feature; MP3 bytes embedded in parquet)", | |
| "label": "string holding a Python-repr list of per-annotator dicts, e.g. \"[{'human-1': {'Shame': 1}}, {'human-2': {'Shame': 2}}]\"; emotion names are URL-encoded (%20, %26)" | |
| }, | |
| "audio": { | |
| "encoding": "MP3 (MPEG Layer III), ID3v2 tagged", | |
| "storage": "embedded in parquet as bytes; NOT extracted to loose files", | |
| "sample_rate_hz": 44100, | |
| "channels": 1, | |
| "duration_sec": { | |
| "min": 0.20800453514739228, | |
| "max": 224.35, | |
| "mean": 10.235554517150774, | |
| "median": 6.348990929705216, | |
| "total_hours": 35.82444081002771 | |
| } | |
| }, | |
| "label_space": { | |
| "n_classes": 42, | |
| "classes": [ | |
| "Affection", | |
| "Amusement", | |
| "Anger", | |
| "Arousal", | |
| "Astonishment", | |
| "Authenticity", | |
| "Awe", | |
| "Bitterness", | |
| "Concentration", | |
| "Confusion", | |
| "Contemplation", | |
| "Contempt", | |
| "Contentment", | |
| "Disappointment", | |
| "Disgust", | |
| "Distress", | |
| "Doubt", | |
| "Elation", | |
| "Embarrassment", | |
| "Emotional Numbness", | |
| "Fatigue", | |
| "Fear", | |
| "Helplessness", | |
| "Hope", | |
| "Impatience and Irritability", | |
| "Infatuation", | |
| "Interest", | |
| "Intoxication", | |
| "Jealousy & Envy", | |
| "Longing", | |
| "Malevolence", | |
| "Pain", | |
| "Pleasure", | |
| "Pride", | |
| "Relief", | |
| "Sadness", | |
| "Sexual Lust", | |
| "Shame", | |
| "Sourness", | |
| "Teasing", | |
| "Thankfulness", | |
| "Triumph" | |
| ], | |
| "task": "per-(clip, target-emotion) INTENSITY rating. Exactly ONE target emotion per row, rated 0/1/2 by 2-4 experts. NOT multi-label, NOT a 42-way argmax task.", | |
| "intensity_scale": { | |
| "0": "Not Present", | |
| "1": "Mildly Present", | |
| "2": "Intensely Present" | |
| }, | |
| "official_gold_mapping": "0->0, 1->5, 2->10 on a 0-10 scale; averaged across annotators (column gold_0_10)", | |
| "n_annotators_per_clip": { | |
| "2": 4538, | |
| "3": 7719, | |
| "4": 343 | |
| }, | |
| "annotator_ids": [ | |
| "human-1", | |
| "human-2", | |
| "human-3", | |
| "human-4", | |
| "human-5", | |
| "human-6" | |
| ], | |
| "rows_per_class": { | |
| "Affection": 300, | |
| "Amusement": 300, | |
| "Anger": 300, | |
| "Arousal": 300, | |
| "Astonishment": 300, | |
| "Authenticity": 300, | |
| "Awe": 300, | |
| "Bitterness": 300, | |
| "Concentration": 300, | |
| "Confusion": 300, | |
| "Contemplation": 300, | |
| "Contempt": 300, | |
| "Contentment": 300, | |
| "Disappointment": 300, | |
| "Disgust": 300, | |
| "Distress": 300, | |
| "Doubt": 300, | |
| "Elation": 300, | |
| "Embarrassment": 300, | |
| "Emotional Numbness": 300, | |
| "Fatigue": 300, | |
| "Fear": 300, | |
| "Helplessness": 300, | |
| "Hope": 300, | |
| "Impatience and Irritability": 300, | |
| "Infatuation": 300, | |
| "Interest": 300, | |
| "Intoxication": 300, | |
| "Jealousy & Envy": 300, | |
| "Longing": 300, | |
| "Malevolence": 300, | |
| "Pain": 300, | |
| "Pleasure": 300, | |
| "Pride": 300, | |
| "Relief": 300, | |
| "Sadness": 300, | |
| "Sexual Lust": 300, | |
| "Shame": 300, | |
| "Sourness": 300, | |
| "Teasing": 300, | |
| "Thankfulness": 300, | |
| "Triumph": 300 | |
| }, | |
| "mean_intensity_per_class": { | |
| "Shame": 1.1183, | |
| "Helplessness": 1.0269, | |
| "Astonishment": 1.0606, | |
| "Bitterness": 1.2369, | |
| "Concentration": 1.4022, | |
| "Contentment": 1.0994, | |
| "Disgust": 1.0589, | |
| "Confusion": 1.1197, | |
| "Affection": 1.1239, | |
| "Infatuation": 1.2125, | |
| "Thankfulness": 1.0492, | |
| "Triumph": 1.1783, | |
| "Pride": 1.0856, | |
| "Longing": 0.9539, | |
| "Sadness": 0.8925, | |
| "Anger": 1.2011, | |
| "Jealousy & Envy": 1.1639, | |
| "Arousal": 0.7769, | |
| "Intoxication": 0.92, | |
| "Contempt": 1.1728, | |
| "Impatience and Irritability": 1.1036, | |
| "Emotional Numbness": 0.6581, | |
| "Distress": 1.1619, | |
| "Teasing": 0.9997, | |
| "Pleasure": 0.9189, | |
| "Sourness": 1.1617, | |
| "Authenticity": 1.0475, | |
| "Disappointment": 1.175, | |
| "Fatigue": 0.9125, | |
| "Interest": 1.0983, | |
| "Doubt": 1.02, | |
| "Elation": 0.9908, | |
| "Sexual Lust": 0.8378, | |
| "Fear": 0.9011, | |
| "Malevolence": 1.1603, | |
| "Amusement": 1.0042, | |
| "Contemplation": 1.1425, | |
| "Embarrassment": 1.1661, | |
| "Awe": 0.8586, | |
| "Pain": 1.0086, | |
| "Hope": 0.9064, | |
| "Relief": 1.0814 | |
| }, | |
| "annotation_counts_by_intensity": { | |
| "0": 10828, | |
| "1": 9009, | |
| "2": 13768 | |
| } | |
| }, | |
| "unique_audio_files": 12397, | |
| "duplicate_audio_note": "197 mp3 filenames each appear in 2 rows, always with a DIFFERENT target emotion -> the unit of evaluation is (audio_path,label); clip_id is unique per row", | |
| "consensus_subsets": { | |
| "unanimous_rows": 2912, | |
| "present_all_annotators": 4692, | |
| "present_any_annotator": 11475, | |
| "absent_all": 1125 | |
| }, | |
| "official_metric": { | |
| "primary": [ | |
| "MAE", | |
| "RMSE", | |
| "Pearson r", | |
| "Spearman rho" | |
| ], | |
| "evidence": "arXiv:2506.09827 (https://arxiv.org/html/2506.09827v1), evaluation section / Table 5: 'our primary evaluation employs metrics suited for regression and correlation analysis on a common scale. The 3-level intensity human judgments (0: Not Present, 1: Mildly Present, 2: Intensely Present) are mapped to a 0-10 scale for this evaluation, becoming 0, 5, and 10, respectively.'", | |
| "note": "Primary protocol is regression/correlation, NOT argmax classification. Accuracy figures appear only anecdotally in the abstract." | |
| }, | |
| "labels_parquet": { | |
| "path": "/e/data1/datasets/playground/mmlaion/schuhmann1/dramabox/headspub/bench/emonet/labels.parquet", | |
| "n_rows": 12600, | |
| "columns": [ | |
| "clip_id", | |
| "audio_path", | |
| "parquet_file", | |
| "row_index", | |
| "rel_path", | |
| "label", | |
| "label_raw", | |
| "head_name", | |
| "n_annotators", | |
| "ratings", | |
| "intensity_mean", | |
| "intensity_median", | |
| "intensity_majority", | |
| "intensity_min", | |
| "intensity_max", | |
| "gold_0_10", | |
| "unanimous", | |
| "present_any", | |
| "present_all", | |
| "duration_sec", | |
| "sample_rate", | |
| "audio_bytes" | |
| ], | |
| "how_to_get_audio": "open <parquet_file> with pyarrow, take row <row_index> of column audioId -> {'bytes': mp3, 'path': name}. rel_path encodes 'file#row_index'." | |
| }, | |
| "head_mapping": { | |
| "Affection": "Affection", | |
| "Amusement": "Amusement", | |
| "Anger": "Anger", | |
| "Arousal": "Arousal", | |
| "Astonishment": "Astonishment_Surprise", | |
| "Authenticity": "Authenticity", | |
| "Awe": "Awe", | |
| "Bitterness": "Bitterness", | |
| "Concentration": "Concentration", | |
| "Confusion": "Confusion", | |
| "Contemplation": "Contemplation", | |
| "Contempt": "Contempt", | |
| "Contentment": "Contentment", | |
| "Disappointment": "Disappointment", | |
| "Disgust": "Disgust", | |
| "Distress": "Distress", | |
| "Doubt": "Doubt", | |
| "Elation": "Elation", | |
| "Embarrassment": "Embarrassment", | |
| "Emotional Numbness": "Emotional_Numbness", | |
| "Fatigue": "Fatigue_Exhaustion", | |
| "Fear": "Fear", | |
| "Helplessness": "Helplessness", | |
| "Hope": "Hope_Enthusiasm_Optimism", | |
| "Impatience and Irritability": "Impatience_and_Irritability", | |
| "Infatuation": "Infatuation", | |
| "Interest": "Interest", | |
| "Intoxication": "Intoxication_Altered_States_of_Consciousness", | |
| "Jealousy & Envy": "Jealousy_&_Envy", | |
| "Longing": "Longing", | |
| "Malevolence": "Malevolence_Malice", | |
| "Pain": "Pain", | |
| "Pleasure": "Pleasure_Ecstasy", | |
| "Pride": "Pride", | |
| "Relief": "Relief", | |
| "Sadness": "Sadness", | |
| "Sexual Lust": "Sexual_Lust", | |
| "Shame": "Shame", | |
| "Sourness": "Sourness", | |
| "Teasing": "Teasing", | |
| "Thankfulness": "Thankfulness_Gratitude", | |
| "Triumph": "Triumph" | |
| }, | |
| "heads_unused": [ | |
| "Age", | |
| "Background_Noise", | |
| "Confident_vs._Hesitant", | |
| "Gender", | |
| "High-Pitched_vs._Low-Pitched", | |
| "Monotone_vs._Expressive", | |
| "Recording_Quality", | |
| "Serious_vs._Humorous", | |
| "Soft_vs._Harsh", | |
| "Submissive_vs._Dominant", | |
| "Valence", | |
| "Vulnerable_vs._Emotionally_Detached", | |
| "Warm_vs._Cold", | |
| "duration", | |
| "score_background_quality", | |
| "score_content_enjoyment", | |
| "score_overall_quality", | |
| "score_speech_quality", | |
| "talking_speed" | |
| ] | |
| } |