Upload llama_and_8k_v3_seed_888/flexitok--bpe_ltr_swe_Latn_8000_v3_overlap.json with huggingface_hub

llama_and_8k_v3_seed_888/flexitok--bpe_ltr_swe_Latn_8000_v3_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.34795127353266886, "expected_training_ratio_in_superset": 0.015815966978757676, "num_tokens": 3142}, "2": {"ratio_to_total_tokens": 0.09047619047619047, "expected_training_ratio_in_superset": 0.008225108225108226, "num_tokens": 817}, "3": {"ratio_to_total_tokens": 0.11173864894795127, "expected_training_ratio_in_superset": 0.015237088492902447, "num_tokens": 1009}, "4": {"ratio_to_total_tokens": 0.04318936877076412, "expected_training_ratio_in_superset": 0.007852612503775294, "num_tokens": 390}, "5": {"ratio_to_total_tokens": 0.029900332225913623, "expected_training_ratio_in_superset": 0.006795530051344005, "num_tokens": 270}, "6": {"ratio_to_total_tokens": 0.02547065337763012, "expected_training_ratio_in_superset": 0.006946541830262759, "num_tokens": 230}, "7": {"ratio_to_total_tokens": 0.02458471760797342, "expected_training_ratio_in_superset": 0.007822410147991544, "num_tokens": 222}, "8": {"ratio_to_total_tokens": 0.018826135105204873, "expected_training_ratio_in_superset": 0.00684586731098359, "num_tokens": 170}, "9": {"ratio_to_total_tokens": 0.017718715393133997, "expected_training_ratio_in_superset": 0.007248565388100272, "num_tokens": 160}, "10": {"ratio_to_total_tokens": 0.018826135105204873, "expected_training_ratio_in_superset": 0.008557334138729486, "num_tokens": 170}, "11": {"ratio_to_total_tokens": 0.013732004429678849, "expected_training_ratio_in_superset": 0.006866002214839424, "num_tokens": 124}, "12": {"ratio_to_total_tokens": 0.015060908084163898, "expected_training_ratio_in_superset": 0.008215040773180308, "num_tokens": 136}, "13": {"ratio_to_total_tokens": 0.01273532668881506, "expected_training_ratio_in_superset": 0.00752542031611799, "num_tokens": 115}, "14": {"ratio_to_total_tokens": 0.010077519379844961, "expected_training_ratio_in_superset": 0.0064129668780831565, "num_tokens": 91}, "15": {"ratio_to_total_tokens": 0.013953488372093023, "expected_training_ratio_in_superset": 0.009513742071881607, "num_tokens": 126}, "16": {"ratio_to_total_tokens": 0.014839424141749723, "expected_training_ratio_in_superset": 0.01079230846672707, "num_tokens": 134}, "17": {"ratio_to_total_tokens": 0.005204872646733112, "expected_training_ratio_in_superset": 0.004021947045202859, "num_tokens": 47}, "18": {"ratio_to_total_tokens": 0.009745293466223699, "expected_training_ratio_in_superset": 0.007973421926910298, "num_tokens": 88}, "19": {"ratio_to_total_tokens": 0.0049833887043189366, "expected_training_ratio_in_superset": 0.0043038356991845364, "num_tokens": 45}, "20": {"ratio_to_total_tokens": 0.0053156146179402, "expected_training_ratio_in_superset": 0.004832376925400181, "num_tokens": 48}, "21": {"ratio_to_total_tokens": 0.004208194905869324, "expected_training_ratio_in_superset": 0.0040169133192389005, "num_tokens": 38}, "22": {"ratio_to_total_tokens": 0.16146179401993355, "expected_training_ratio_in_superset": 0.16146179401993355, "num_tokens": 1458}, "total_training_compared_to_full_model": 0.32728279472465516}