metrics: quantization parity benchmark (unified_v3 test split)

metrics/quant_bench.json

@@ -0,0 +1,94 @@
+{
+ "dataset": "unified_v3 test split",
+ "max_length": 128,
+ "n_rows": 2914,
+ "reference_fp32": {
+  "loss": 0.053679560930409015,
+  "n": 2914,
+  "accuracy": 0.9543582704186685,
+  "f1": 0.9645043898167938,
+  "precision": 0.9655916791519741,
+  "recall": 0.963423773743917,
+  "confusion_matrix": [
+   [
+    [
+     2233,
+     15
+    ],
+    [
+     19,
+     647
+    ]
+   ],
+   [
+    [
+     2041,
+     27
+    ],
+    [
+     26,
+     820
+    ]
+   ],
+   [
+    [
+     2353,
+     26
+    ],
+    [
+     27,
+     508
+    ]
+   ]
+  ],
+  "per_class": {
+   "source:sensitive": {
+    "precision": 0.9773413897280967,
+    "recall": 0.9714714714714715,
+    "f1": 0.9743975903614458,
+    "support": 666
+   },
+   "source:untrusted": {
+    "precision": 0.9681227863046045,
+    "recall": 0.9692671394799054,
+    "f1": 0.9686946249261665,
+    "support": 846
+   },
+   "sink:external": {
+    "precision": 0.951310861423221,
+    "recall": 0.9495327102803738,
+    "f1": 0.950420954162769,
+    "support": 535
+   }
+  }
+ },
+ "variants": {
+  "fp16": {
+   "n": 2914,
+   "source:sensitive": 0.9743,
+   "source:untrusted": 0.9693,
+   "sink:external": 0.9504,
+   "f1_macro": 0.9647,
+   "latency_s_total": 58.6,
+   "ms_per_text": 20.12
+  },
+  "int8": {
+   "n": 2914,
+   "source:sensitive": 0.9766,
+   "source:untrusted": 0.9697,
+   "sink:external": 0.953,
+   "f1_macro": 0.9664,
+   "latency_s_total": 55.1,
+   "ms_per_text": 18.9
+  },
+  "int8_int4_embeddings": {
+   "n": 2914,
+   "source:sensitive": 0.9789,
+   "source:untrusted": 0.9697,
+   "sink:external": 0.95,
+   "f1_macro": 0.9662,
+   "latency_s_total": 55.1,
+   "ms_per_text": 18.92
+  }
+ }
+}