Upload llama_and_16k/flexitok--bpe_ltr_swe_Latn_16000_v2_overlap.json with huggingface_hub

llama_and_16k/flexitok--bpe_ltr_swe_Latn_16000_v2_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.4157382847038019, "expected_training_ratio_in_superset": 0.01979706117637152, "num_tokens": 7053}, "2": {"ratio_to_total_tokens": 0.10439139404656646, "expected_training_ratio_in_superset": 0.009942037528244425, "num_tokens": 1771}, "3": {"ratio_to_total_tokens": 0.10745652814618332, "expected_training_ratio_in_superset": 0.0153509325923119, "num_tokens": 1823}, "4": {"ratio_to_total_tokens": 0.037253168287651045, "expected_training_ratio_in_superset": 0.007095841578600199, "num_tokens": 632}, "5": {"ratio_to_total_tokens": 0.029295608605953434, "expected_training_ratio_in_superset": 0.006975144906179388, "num_tokens": 497}, "6": {"ratio_to_total_tokens": 0.022458001768346596, "expected_training_ratio_in_superset": 0.0064165719338133125, "num_tokens": 381}, "7": {"ratio_to_total_tokens": 0.020512820512820513, "expected_training_ratio_in_superset": 0.006837606837606838, "num_tokens": 348}, "8": {"ratio_to_total_tokens": 0.02027704096669614, "expected_training_ratio_in_superset": 0.007724587034931862, "num_tokens": 344}, "9": {"ratio_to_total_tokens": 0.016740347774830532, "expected_training_ratio_in_superset": 0.007174434760641657, "num_tokens": 284}, "10": {"ratio_to_total_tokens": 0.017565576186265842, "expected_training_ratio_in_superset": 0.00836456008869802, "num_tokens": 298}, "11": {"ratio_to_total_tokens": 0.014382552313586797, "expected_training_ratio_in_superset": 0.007533717878545465, "num_tokens": 244}, "12": {"ratio_to_total_tokens": 0.015679339817270852, "expected_training_ratio_in_superset": 0.008959622752726201, "num_tokens": 266}, "13": {"ratio_to_total_tokens": 0.014559386973180077, "expected_training_ratio_in_superset": 0.009012953840540046, "num_tokens": 247}, "14": {"ratio_to_total_tokens": 0.01290893015030946, "expected_training_ratio_in_superset": 0.008605953433539641, "num_tokens": 219}, "15": {"ratio_to_total_tokens": 0.021927497789566756, "expected_training_ratio_in_superset": 0.015662498421119112, "num_tokens": 372}, "16": {"ratio_to_total_tokens": 0.007662835249042145, "expected_training_ratio_in_superset": 0.0058383506659368724, "num_tokens": 130}, "17": {"ratio_to_total_tokens": 0.010845859121721192, "expected_training_ratio_in_superset": 0.008779981193774298, "num_tokens": 184}, "18": {"ratio_to_total_tokens": 0.005422929560860596, "expected_training_ratio_in_superset": 0.00464822533788051, "num_tokens": 92}, "19": {"ratio_to_total_tokens": 0.005953433539640436, "expected_training_ratio_in_superset": 0.00538643986919849, "num_tokens": 101}, "20": {"ratio_to_total_tokens": 0.005422929560860596, "expected_training_ratio_in_superset": 0.005164694819867234, "num_tokens": 92}, "21": {"ratio_to_total_tokens": 0.09354553492484527, "expected_training_ratio_in_superset": 0.09354553492484527, "num_tokens": 1587}, "total_training_compared_to_full_model": 0.26881675157537227}