| { |
| "schema": "standalone4l-complete-transfer-benchmark-v2", |
| "status": "complete_from_frozen_reports", |
| "claim_scope": "Exploratory fresh-head transfer evidence on one English task and a previously opened 55-article SemEval-derived test split.", |
| "headless_root_requires_adaptation": true, |
| "test_previously_opened": true, |
| "test_articles": 55, |
| "test_windows": 434, |
| "seeds": [ |
| 41, |
| 42, |
| 43 |
| ], |
| "three_seed_sd_is_not_confidence_interval": true, |
| "multilingual_downstream_evaluation_performed": false, |
| "upstream_multilingual_metadata": { |
| "source": "Qwen/Qwen3.5-0.8B official model card", |
| "languages_and_dialects_claimed_upstream": 201, |
| "scope": "inherited tokenizer and architecture coverage only" |
| }, |
| "protocol": { |
| "epochs": 5, |
| "primary_epoch": 5, |
| "early_stopping": false, |
| "max_length": 256, |
| "stride": 128, |
| "window_target": "span-aligned technique labels", |
| "article_aggregation": "labelwise maximum probability", |
| "micro_batch_size": 1, |
| "gradient_accumulation": 32, |
| "effective_batch_size": 32, |
| "learning_rate": 2e-05, |
| "weight_decay": 0.01, |
| "gradient_clip": 1.0, |
| "optimizer": "AdamW", |
| "dtype": "bfloat16", |
| "thresholds": [ |
| 0.05, |
| 0.1, |
| 0.15, |
| 0.2, |
| 0.25, |
| 0.3, |
| 0.35, |
| 0.4, |
| 0.45, |
| 0.5, |
| 0.55, |
| 0.6, |
| 0.65, |
| 0.7, |
| 0.75, |
| 0.8, |
| 0.85, |
| 0.9 |
| ], |
| "threshold_selection": "epoch-5 calibration macro-F1, then micro-F1" |
| }, |
| "arms": { |
| "task_agnostic_base": { |
| "display": "Task-agnostic KD base", |
| "lineage": "root base plus a fresh 14-label transfer head", |
| "metrics": { |
| "macro_f1": { |
| "values": [ |
| 0.5883093293517536, |
| 0.6028379036218096, |
| 0.5712816197954101 |
| ], |
| "mean": 0.5874762842563245, |
| "sample_sd": 0.01579462678670608, |
| "n": 3 |
| }, |
| "micro_f1": { |
| "values": [ |
| 0.6468085106382978, |
| 0.6655172413793103, |
| 0.6465256797583082 |
| ], |
| "mean": 0.6529504772586388, |
| "sample_sd": 0.010884055708931178, |
| "n": 3 |
| }, |
| "exact_match": { |
| "values": [ |
| 0.05454545454545454, |
| 0.03636363636363636, |
| 0.0 |
| ], |
| "mean": 0.0303030303030303, |
| "sample_sd": 0.027773186030035393, |
| "n": 3 |
| }, |
| "predicted_positive_rate": { |
| "values": [ |
| 0.5714285714285714, |
| 0.4090909090909091, |
| 0.5155844155844156 |
| ], |
| "mean": 0.4987012987012987, |
| "sample_sd": 0.08247520160679181, |
| "n": 3 |
| } |
| }, |
| "resources": { |
| "parameters": { |
| "values": [ |
| 334111040.0, |
| 334111040.0, |
| 334111040.0 |
| ], |
| "mean": 334111040.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "peak_cuda_allocated_bytes": { |
| "values": [ |
| 3380975616.0, |
| 3380975616.0, |
| 3380975616.0 |
| ], |
| "mean": 3380975616.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "peak_cuda_reserved_bytes": { |
| "values": [ |
| 3519021056.0, |
| 3519021056.0, |
| 3510632448.0 |
| ], |
| "mean": 3516224853.3333335, |
| "sample_sd": 4843165.086926248, |
| "n": 3 |
| }, |
| "total_wall_seconds": { |
| "values": [ |
| 1583.0297977919981, |
| 1588.617994428001, |
| 1603.4310500120046 |
| ], |
| "mean": 1591.692947410668, |
| "sample_sd": 10.54249851412311, |
| "n": 3 |
| }, |
| "test_seconds_per_article": { |
| "values": [ |
| 0.08164824154545468, |
| 0.07871871667289683, |
| 0.08355205721825107 |
| ], |
| "mean": 0.08130633847886752, |
| "sample_sd": 0.0024347419754560395, |
| "n": 3 |
| } |
| }, |
| "thresholds": [ |
| 0.15, |
| 0.3, |
| 0.1 |
| ] |
| }, |
| "structural_copy_control": { |
| "display": "Structural copy, no task-free KD", |
| "lineage": "same four-layer structure without task-free KD", |
| "metrics": { |
| "macro_f1": { |
| "values": [ |
| 0.5169424579262757, |
| 0.5349405779540175, |
| 0.5023778364496152 |
| ], |
| "mean": 0.5180869574433028, |
| "sample_sd": 0.016311512588628545, |
| "n": 3 |
| }, |
| "micro_f1": { |
| "values": [ |
| 0.5773955773955775, |
| 0.6083445491251682, |
| 0.585635359116022 |
| ], |
| "mean": 0.5904584952122559, |
| "sample_sd": 0.01602830914762526, |
| "n": 3 |
| }, |
| "exact_match": { |
| "values": [ |
| 0.0, |
| 0.0, |
| 0.0 |
| ], |
| "mean": 0.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "predicted_positive_rate": { |
| "values": [ |
| 0.712987012987013, |
| 0.6207792207792208, |
| 0.5961038961038961 |
| ], |
| "mean": 0.6432900432900432, |
| "sample_sd": 0.06160737456395461, |
| "n": 3 |
| } |
| }, |
| "resources": { |
| "parameters": { |
| "values": [ |
| 334111040.0, |
| 334111040.0, |
| 334111040.0 |
| ], |
| "mean": 334111040.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "peak_cuda_allocated_bytes": { |
| "values": [ |
| 3380975616.0, |
| 3380975616.0, |
| 3380975616.0 |
| ], |
| "mean": 3380975616.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "peak_cuda_reserved_bytes": { |
| "values": [ |
| 3519021056.0, |
| 3519021056.0, |
| 3510632448.0 |
| ], |
| "mean": 3516224853.3333335, |
| "sample_sd": 4843165.086926248, |
| "n": 3 |
| }, |
| "total_wall_seconds": { |
| "values": [ |
| 1575.5905925639963, |
| 1623.5639530190092, |
| 1629.195114912989 |
| ], |
| "mean": 1609.4498868319981, |
| "sample_sd": 29.457874198835384, |
| "n": 3 |
| }, |
| "test_seconds_per_article": { |
| "values": [ |
| 0.07877608058201573, |
| 0.08307423260041767, |
| 0.08163602005436339 |
| ], |
| "mean": 0.08116211107893226, |
| "sample_sd": 0.002187914527825257, |
| "n": 3 |
| } |
| }, |
| "thresholds": [ |
| 0.05, |
| 0.2, |
| 0.2 |
| ] |
| }, |
| "existing_specialized_separate_lineage": { |
| "display": "Existing specialized 4L, separate lineage", |
| "lineage": "historical task-specific shrink/distillation lineage", |
| "metrics": { |
| "macro_f1": { |
| "values": [ |
| 0.5832848452288824, |
| 0.5958405181160762, |
| 0.5835785045218137 |
| ], |
| "mean": 0.5875679559555907, |
| "sample_sd": 0.007165753447433348, |
| "n": 3 |
| }, |
| "micro_f1": { |
| "values": [ |
| 0.6494252873563218, |
| 0.6458616010854816, |
| 0.6486486486486486 |
| ], |
| "mean": 0.647978512363484, |
| "sample_sd": 0.0018739735696546695, |
| "n": 3 |
| }, |
| "exact_match": { |
| "values": [ |
| 0.0, |
| 0.0, |
| 0.01818181818181818 |
| ], |
| "mean": 0.006060606060606061, |
| "sample_sd": 0.01049727762162956, |
| "n": 3 |
| }, |
| "predicted_positive_rate": { |
| "values": [ |
| 0.5597402597402598, |
| 0.612987012987013, |
| 0.5688311688311688 |
| ], |
| "mean": 0.5805194805194805, |
| "sample_sd": 0.02848274311618351, |
| "n": 3 |
| } |
| }, |
| "resources": { |
| "parameters": { |
| "values": [ |
| 210903360.0, |
| 210903360.0, |
| 210903360.0 |
| ], |
| "mean": 210903360.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "peak_cuda_allocated_bytes": { |
| "values": [ |
| 2143655936.0, |
| 2143655936.0, |
| 2143655936.0 |
| ], |
| "mean": 2143655936.0, |
| "sample_sd": 0.0, |
| "n": 3 |
| }, |
| "peak_cuda_reserved_bytes": { |
| "values": [ |
| 2281701376.0, |
| 2281701376.0, |
| 2273312768.0 |
| ], |
| "mean": 2278905173.3333335, |
| "sample_sd": 4843165.086926248, |
| "n": 3 |
| }, |
| "total_wall_seconds": { |
| "values": [ |
| 1634.488557530014, |
| 1652.9397718419787, |
| 1643.488202479988 |
| ], |
| "mean": 1643.6388439506602, |
| "sample_sd": 9.226529522850198, |
| "n": 3 |
| }, |
| "test_seconds_per_article": { |
| "values": [ |
| 0.0847959352360869, |
| 0.08598086021798239, |
| 0.08552987641781908 |
| ], |
| "mean": 0.08543555729062946, |
| "sample_sd": 0.0005980667826383295, |
| "n": 3 |
| } |
| }, |
| "thresholds": [ |
| 0.1, |
| 0.05, |
| 0.05 |
| ] |
| } |
| }, |
| "same_seed_macro_f1_deltas_base_minus_control": [ |
| 0.07136687142547793, |
| 0.06789732566779205, |
| 0.06890378334579494 |
| ], |
| "mean_macro_f1_delta_base_minus_control": 0.06938932681302164, |
| "publication_gate_passed": true, |
| "per_label_summary": [ |
| { |
| "arm": "task_agnostic_base", |
| "label": "Appeal_to_Authority", |
| "support": 11, |
| "precision_mean": 0.3958614484930274, |
| "precision_sample_sd": 0.1238064457010119, |
| "recall_mean": 0.5757575757575757, |
| "recall_sample_sd": 0.05248638810814781, |
| "f1_mean": 0.4657088122605364, |
| "f1_sample_sd": 0.1020990126979069 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Appeal_to_fear-prejudice", |
| "support": 24, |
| "precision_mean": 0.5540244550770866, |
| "precision_sample_sd": 0.02525673990402998, |
| "recall_mean": 0.7361111111111112, |
| "recall_sample_sd": 0.13393959390267993, |
| "f1_mean": 0.6277521761392729, |
| "f1_sample_sd": 0.04994879016183189 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Bandwagon,Reductio_ad_hitlerum", |
| "support": 7, |
| "precision_mean": 0.4851851851851852, |
| "precision_sample_sd": 0.07882887649552875, |
| "recall_mean": 0.8095238095238095, |
| "recall_sample_sd": 0.08247860988423221, |
| "f1_mean": 0.6006778309409889, |
| "f1_sample_sd": 0.04793776258918466 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Black-and-White_Fallacy", |
| "support": 12, |
| "precision_mean": 0.3667929292929293, |
| "precision_sample_sd": 0.07670649392711647, |
| "recall_mean": 0.47222222222222227, |
| "recall_sample_sd": 0.09622504486493764, |
| "f1_mean": 0.4053892966936445, |
| "f1_sample_sd": 0.04211369218048746 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Causal_Oversimplification", |
| "support": 18, |
| "precision_mean": 0.5428824049513704, |
| "precision_sample_sd": 0.039216350192122156, |
| "recall_mean": 0.7962962962962963, |
| "recall_sample_sd": 0.11564811108145183, |
| "f1_mean": 0.6446992723588468, |
| "f1_sample_sd": 0.0634561308339175 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Doubt", |
| "support": 22, |
| "precision_mean": 0.5223832528180354, |
| "precision_sample_sd": 0.06742780050556177, |
| "recall_mean": 0.9848484848484849, |
| "recall_sample_sd": 0.026243194054073875, |
| "f1_mean": 0.6802057822343391, |
| "f1_sample_sd": 0.04928561754686331 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Exaggeration,Minimisation", |
| "support": 22, |
| "precision_mean": 0.4620811287477954, |
| "precision_sample_sd": 0.016801198309751, |
| "recall_mean": 0.7727272727272727, |
| "recall_sample_sd": 0.15745916432444337, |
| "f1_mean": 0.5727040816326531, |
| "f1_sample_sd": 0.03645259990419198 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Flag-Waving", |
| "support": 17, |
| "precision_mean": 0.5642642642642642, |
| "precision_sample_sd": 0.1478145677656334, |
| "recall_mean": 0.8235294117647058, |
| "recall_sample_sd": 0.1764705882352941, |
| "f1_mean": 0.6464506172839506, |
| "f1_sample_sd": 0.035742219952758225 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Loaded_Language", |
| "support": 45, |
| "precision_mean": 0.8649032907793016, |
| "precision_sample_sd": 0.046984312428427724, |
| "recall_mean": 0.9851851851851852, |
| "recall_sample_sd": 0.012830005981991702, |
| "f1_mean": 0.9208741623431562, |
| "f1_sample_sd": 0.03200926878737141 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Name_Calling,Labeling", |
| "support": 33, |
| "precision_mean": 0.7380952380952381, |
| "precision_sample_sd": 0.0047619047619048005, |
| "recall_mean": 0.9090909090909091, |
| "recall_sample_sd": 0.10925912955951485, |
| "f1_mean": 0.8125087983911513, |
| "f1_sample_sd": 0.04252971464057227 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Repetition", |
| "support": 23, |
| "precision_mean": 0.5196620583717357, |
| "precision_sample_sd": 0.021645909050194662, |
| "recall_mean": 0.7536231884057971, |
| "recall_sample_sd": 0.06641414050660639, |
| "f1_mean": 0.6147520781519571, |
| "f1_sample_sd": 0.03505927109416221 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Slogans", |
| "support": 14, |
| "precision_mean": 0.5161361626878869, |
| "precision_sample_sd": 0.20305677930281646, |
| "recall_mean": 0.7380952380952381, |
| "recall_sample_sd": 0.10910894511799614, |
| "f1_mean": 0.5834824090638044, |
| "f1_sample_sd": 0.09862686289220693 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Thought-terminating_Cliches", |
| "support": 7, |
| "precision_mean": 0.3492063492063492, |
| "precision_sample_sd": 0.143516994603374, |
| "recall_mean": 0.3333333333333333, |
| "recall_sample_sd": 0.08247860988423225, |
| "f1_mean": 0.319014319014319, |
| "f1_sample_sd": 0.04017590711117064 |
| }, |
| { |
| "arm": "task_agnostic_base", |
| "label": "Whataboutism,Straw_Men,Red_Herring", |
| "support": 10, |
| "precision_mean": 0.2761904761904762, |
| "precision_sample_sd": 0.05302632726504782, |
| "recall_mean": 0.5, |
| "recall_sample_sd": 0.2645751311064591, |
| "f1_mean": 0.330448343079922, |
| "f1_sample_sd": 0.021845164720973014 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Appeal_to_Authority", |
| "support": 11, |
| "precision_mean": 0.26003086419753085, |
| "precision_sample_sd": 0.06281357095350094, |
| "recall_mean": 0.5757575757575758, |
| "recall_sample_sd": 0.262431940540739, |
| "f1_mean": 0.35477582846003897, |
| "f1_sample_sd": 0.11479479036518875 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Appeal_to_fear-prejudice", |
| "support": 24, |
| "precision_mean": 0.5104813315339631, |
| "precision_sample_sd": 0.038427508996734555, |
| "recall_mean": 0.8611111111111112, |
| "recall_sample_sd": 0.024056261216234387, |
| "f1_mean": 0.6401745363877202, |
| "f1_sample_sd": 0.029305467347150463 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Bandwagon,Reductio_ad_hitlerum", |
| "support": 7, |
| "precision_mean": 0.2876623376623376, |
| "precision_sample_sd": 0.061386823077618426, |
| "recall_mean": 0.8571428571428571, |
| "recall_sample_sd": 0.14285714285714285, |
| "f1_mean": 0.4306391777656145, |
| "f1_sample_sd": 0.08686392624022733 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Black-and-White_Fallacy", |
| "support": 12, |
| "precision_mean": 0.31339031339031337, |
| "precision_sample_sd": 0.08301311842925053, |
| "recall_mean": 0.5277777777777778, |
| "recall_sample_sd": 0.2678791878053599, |
| "f1_mean": 0.38778427550357375, |
| "f1_sample_sd": 0.13069755090515112 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Causal_Oversimplification", |
| "support": 18, |
| "precision_mean": 0.4464201043148412, |
| "precision_sample_sd": 0.013538446216471536, |
| "recall_mean": 0.9259259259259259, |
| "recall_sample_sd": 0.03207501495497923, |
| "f1_mean": 0.6023809523809524, |
| "f1_sample_sd": 0.018643640071557464 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Doubt", |
| "support": 22, |
| "precision_mean": 0.4455314009661836, |
| "precision_sample_sd": 0.04067072694857813, |
| "recall_mean": 1.0, |
| "recall_sample_sd": 0.0, |
| "f1_mean": 0.615686274509804, |
| "f1_sample_sd": 0.039427200694482675 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Exaggeration,Minimisation", |
| "support": 22, |
| "precision_mean": 0.4364176150420584, |
| "precision_sample_sd": 0.025093958106097894, |
| "recall_mean": 0.8787878787878788, |
| "recall_sample_sd": 0.05248638810814775, |
| "f1_mean": 0.5829803047194351, |
| "f1_sample_sd": 0.030899288356345502 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Flag-Waving", |
| "support": 17, |
| "precision_mean": 0.3814814814814815, |
| "precision_sample_sd": 0.05010277503136551, |
| "recall_mean": 0.803921568627451, |
| "recall_sample_sd": 0.17970885078258195, |
| "f1_mean": 0.5138595287119652, |
| "f1_sample_sd": 0.06400931989326974 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Loaded_Language", |
| "support": 45, |
| "precision_mean": 0.8492581841638446, |
| "precision_sample_sd": 0.016026591848331686, |
| "recall_mean": 1.0, |
| "recall_sample_sd": 0.0, |
| "f1_mean": 0.9184311025253588, |
| "f1_sample_sd": 0.009372233868132662 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Name_Calling,Labeling", |
| "support": 33, |
| "precision_mean": 0.6375, |
| "precision_sample_sd": 0.03410563654946854, |
| "recall_mean": 0.9696969696969697, |
| "recall_sample_sd": 0.030303030303030276, |
| "f1_mean": 0.7685185185185185, |
| "f1_sample_sd": 0.02023900779105559 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Repetition", |
| "support": 23, |
| "precision_mean": 0.44863933452168747, |
| "precision_sample_sd": 0.01520014564765896, |
| "recall_mean": 0.9420289855072463, |
| "recall_sample_sd": 0.06641414050660639, |
| "f1_mean": 0.607248835423033, |
| "f1_sample_sd": 0.019858627551810427 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Slogans", |
| "support": 14, |
| "precision_mean": 0.3274318274318274, |
| "precision_sample_sd": 0.05434598572773684, |
| "recall_mean": 0.4523809523809524, |
| "recall_sample_sd": 0.1486904285332952, |
| "f1_mean": 0.3715828924162258, |
| "f1_sample_sd": 0.07106660236343577 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Thought-terminating_Cliches", |
| "support": 7, |
| "precision_mean": 0.1142156862745098, |
| "precision_sample_sd": 0.012848374923608973, |
| "recall_mean": 0.38095238095238093, |
| "recall_sample_sd": 0.1649572197684645, |
| "f1_mean": 0.17331433998100665, |
| "f1_sample_sd": 0.029065880233243847 |
| }, |
| { |
| "arm": "structural_copy_control", |
| "label": "Whataboutism,Straw_Men,Red_Herring", |
| "support": 10, |
| "precision_mean": 0.19344598054275472, |
| "precision_sample_sd": 0.03373872471827226, |
| "recall_mean": 0.5666666666666667, |
| "recall_sample_sd": 0.2081665999466133, |
| "f1_mean": 0.28584083690299267, |
| "f1_sample_sd": 0.06078854830133676 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Appeal_to_Authority", |
| "support": 11, |
| "precision_mean": 0.37836021505376344, |
| "precision_sample_sd": 0.10546365636105155, |
| "recall_mean": 0.6666666666666666, |
| "recall_sample_sd": 0.2287828616748712, |
| "f1_mean": 0.46885361552028215, |
| "f1_sample_sd": 0.09502747629890008 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Appeal_to_fear-prejudice", |
| "support": 24, |
| "precision_mean": 0.525708635464733, |
| "precision_sample_sd": 0.02235198659615819, |
| "recall_mean": 0.8888888888888888, |
| "recall_sample_sd": 0.048112522432468774, |
| "f1_mean": 0.6599065351235072, |
| "f1_sample_sd": 0.015362381542320084 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Bandwagon,Reductio_ad_hitlerum", |
| "support": 7, |
| "precision_mean": 0.44485144485144484, |
| "precision_sample_sd": 0.016487284575115283, |
| "recall_mean": 0.7619047619047619, |
| "recall_sample_sd": 0.1649572197684645, |
| "f1_mean": 0.5571428571428572, |
| "f1_sample_sd": 0.051507875363771265 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Black-and-White_Fallacy", |
| "support": 12, |
| "precision_mean": 0.4662644544997486, |
| "precision_sample_sd": 0.004538274146363688, |
| "recall_mean": 0.5833333333333334, |
| "recall_sample_sd": 0.08333333333333331, |
| "f1_mean": 0.516747552149851, |
| "f1_sample_sd": 0.03589484965697345 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Causal_Oversimplification", |
| "support": 18, |
| "precision_mean": 0.4472934472934473, |
| "precision_sample_sd": 0.013055771210700413, |
| "recall_mean": 0.8333333333333333, |
| "recall_sample_sd": 0.1469861839480328, |
| "f1_mean": 0.5781794553724379, |
| "f1_sample_sd": 0.028407563015813122 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Doubt", |
| "support": 22, |
| "precision_mean": 0.5045661001171619, |
| "precision_sample_sd": 0.023581174969843577, |
| "recall_mean": 1.0, |
| "recall_sample_sd": 0.0, |
| "f1_mean": 0.6704939668174963, |
| "f1_sample_sd": 0.020973132363659372 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Exaggeration,Minimisation", |
| "support": 22, |
| "precision_mean": 0.44589793281653745, |
| "precision_sample_sd": 0.010987847993006481, |
| "recall_mean": 0.9393939393939394, |
| "recall_sample_sd": 0.06943296507508849, |
| "f1_mean": 0.6043956043956044, |
| "f1_sample_sd": 0.022305256187015846 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Flag-Waving", |
| "support": 17, |
| "precision_mean": 0.5576388888888889, |
| "precision_sample_sd": 0.03705367103320583, |
| "recall_mean": 0.8823529411764706, |
| "recall_sample_sd": 0.11764705882352944, |
| "f1_mean": 0.680769868923179, |
| "f1_sample_sd": 0.04164658769406446 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Loaded_Language", |
| "support": 45, |
| "precision_mean": 0.8387650085763294, |
| "precision_sample_sd": 0.01782556577257849, |
| "recall_mean": 1.0, |
| "recall_sample_sd": 0.0, |
| "f1_mean": 0.9122448979591837, |
| "f1_sample_sd": 0.01060439269940129 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Name_Calling,Labeling", |
| "support": 33, |
| "precision_mean": 0.6957104190221673, |
| "precision_sample_sd": 0.019201490809269257, |
| "recall_mean": 0.9696969696969697, |
| "recall_sample_sd": 0.030303030303030276, |
| "f1_mean": 0.8101049442821594, |
| "f1_sample_sd": 0.022092733328802537 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Repetition", |
| "support": 23, |
| "precision_mean": 0.4781557067271353, |
| "precision_sample_sd": 0.009897987322673213, |
| "recall_mean": 0.9420289855072463, |
| "recall_sample_sd": 0.06641414050660639, |
| "f1_mean": 0.6337774426009719, |
| "f1_sample_sd": 0.016444118333485254 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Slogans", |
| "support": 14, |
| "precision_mean": 0.4334215167548501, |
| "precision_sample_sd": 0.05811581861282867, |
| "recall_mean": 0.6904761904761905, |
| "recall_sample_sd": 0.1649572197684645, |
| "f1_mean": 0.5201316298877274, |
| "f1_sample_sd": 0.018567913895656114 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Thought-terminating_Cliches", |
| "support": 7, |
| "precision_mean": 0.18446115288220552, |
| "precision_sample_sd": 0.03641251252864963, |
| "recall_mean": 0.6666666666666666, |
| "recall_sample_sd": 0.1649572197684645, |
| "f1_mean": 0.2868626868626869, |
| "f1_sample_sd": 0.0511620864429468 |
| }, |
| { |
| "arm": "existing_specialized_separate_lineage", |
| "label": "Whataboutism,Straw_Men,Red_Herring", |
| "support": 10, |
| "precision_mean": 0.2777777777777778, |
| "precision_sample_sd": 0.0481125224324688, |
| "recall_mean": 0.4, |
| "recall_sample_sd": 0.0, |
| "f1_mean": 0.32634032634032634, |
| "f1_sample_sd": 0.03229931575886014 |
| } |
| ], |
| "compression_ladder": [ |
| { |
| "name": "24L upstream text teacher", |
| "layers": 24, |
| "parameters": 752393024, |
| "weight_bytes": 1504825632, |
| "weight_mib": 1435.1135559082031, |
| "weight_sha256": "30783da4651259c26b8a204821ea426b9526431e0c14da0d1c6fa284e7414e5e", |
| "kd_stage": "", |
| "mean_total_loss": "", |
| "mean_interface_loss": "", |
| "mean_final_loss": "" |
| }, |
| { |
| "name": "8L task-free KD", |
| "layers": 8, |
| "parameters": 420318144, |
| "weight_bytes": 840647880, |
| "weight_mib": 801.7042922973633, |
| "weight_sha256": "086fbaf9a4838ecde56cd3541c9c3212b22d0c5ee745cf2c1c5e0bb31e0ec22c", |
| "kd_stage": "24to8", |
| "mean_total_loss": 0.17622653172702485, |
| "mean_interface_loss": 0.21562811763578793, |
| "mean_final_loss": 0.16626427527853593 |
| }, |
| { |
| "name": "6L task-free KD", |
| "layers": 6, |
| "parameters": 377207424, |
| "weight_bytes": 754423424, |
| "weight_mib": 719.4742431640625, |
| "weight_sha256": "492749c20f77a42f6bc993f0927a82217b94f872b7eacf25612f7520c9bc28ce", |
| "kd_stage": "8to6", |
| "mean_total_loss": 0.03382104352249371, |
| "mean_interface_loss": 0.04486069045515251, |
| "mean_final_loss": 0.031147060785087888 |
| }, |
| { |
| "name": "4L task-free KD root", |
| "layers": 4, |
| "parameters": 334096704, |
| "weight_bytes": 668198976, |
| "weight_mib": 637.2442016601562, |
| "weight_sha256": "2732c616772fe320cdea228ab4554981418b1b2bf615c4183fb1ac8e6e2168d3", |
| "kd_stage": "6to4", |
| "mean_total_loss": 0.047020394468859195, |
| "mean_interface_loss": 0.07048421185527332, |
| "mean_final_loss": 0.043119939065377366 |
| } |
| ] |
| } |
|
|
