Upload llama_and_16k/flexitok--bpe_ltr_spa_Latn_16000_v2_overlap.json with huggingface_hub

llama_and_16k/flexitok--bpe_ltr_spa_Latn_16000_v2_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.3582424063698024, "expected_training_ratio_in_superset": 0.01705916220808583, "num_tokens": 6074}, "2": {"ratio_to_total_tokens": 0.18153936891772338, "expected_training_ratio_in_superset": 0.017289463706449845, "num_tokens": 3078}, "3": {"ratio_to_total_tokens": 0.09613683279268652, "expected_training_ratio_in_superset": 0.013733833256098074, "num_tokens": 1630}, "4": {"ratio_to_total_tokens": 0.0506635210852256, "expected_training_ratio_in_superset": 0.009650194492423924, "num_tokens": 859}, "5": {"ratio_to_total_tokens": 0.028192273665585372, "expected_training_ratio_in_superset": 0.00671244611085366, "num_tokens": 478}, "6": {"ratio_to_total_tokens": 0.01928634621055736, "expected_training_ratio_in_superset": 0.005510384631587816, "num_tokens": 327}, "7": {"ratio_to_total_tokens": 0.01775287525803598, "expected_training_ratio_in_superset": 0.005917625086011992, "num_tokens": 301}, "8": {"ratio_to_total_tokens": 0.016927160129755234, "expected_training_ratio_in_superset": 0.00644844195419247, "num_tokens": 287}, "9": {"ratio_to_total_tokens": 0.014626953700973165, "expected_training_ratio_in_superset": 0.0062686944432742125, "num_tokens": 248}, "10": {"ratio_to_total_tokens": 0.015098790917133589, "expected_training_ratio_in_superset": 0.00718990043673028, "num_tokens": 256}, "11": {"ratio_to_total_tokens": 0.012680625184311413, "expected_training_ratio_in_superset": 0.006642232239401216, "num_tokens": 215}, "12": {"ratio_to_total_tokens": 0.014626953700973165, "expected_training_ratio_in_superset": 0.00835825925769895, "num_tokens": 248}, "13": {"ratio_to_total_tokens": 0.013565319964612209, "expected_training_ratio_in_superset": 0.00839757902571232, "num_tokens": 230}, "14": {"ratio_to_total_tokens": 0.012798584488351519, "expected_training_ratio_in_superset": 0.008532389658901011, "num_tokens": 217}, "15": {"ratio_to_total_tokens": 0.021527572987319374, "expected_training_ratio_in_superset": 0.015376837848085268, "num_tokens": 365}, "16": {"ratio_to_total_tokens": 0.006959598938366264, "expected_training_ratio_in_superset": 0.005302551572088581, "num_tokens": 118}, "17": {"ratio_to_total_tokens": 0.010262459451489236, "expected_training_ratio_in_superset": 0.008307705270253192, "num_tokens": 174}, "18": {"ratio_to_total_tokens": 0.0047773518136243, "expected_training_ratio_in_superset": 0.004094872983106543, "num_tokens": 81}, "19": {"ratio_to_total_tokens": 0.005426127985844883, "expected_training_ratio_in_superset": 0.0049093538919548945, "num_tokens": 92}, "20": {"ratio_to_total_tokens": 0.005308168681804777, "expected_training_ratio_in_superset": 0.005055398744575978, "num_tokens": 90}, "21": {"ratio_to_total_tokens": 0.09360070775582424, "expected_training_ratio_in_superset": 0.09360070775582424, "num_tokens": 1587}, "total_training_compared_to_full_model": 0.2643580345733103}