Upload fineweb2_hq/flexitok--bpe_vie_Latn_8000_overlap.json with huggingface_hub

fineweb2_hq/flexitok--bpe_vie_Latn_8000_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.569, "expected_training_ratio_in_superset": 0.027095238095238096, "num_tokens": 4552}, "2": {"ratio_to_total_tokens": 0.106875, "expected_training_ratio_in_superset": 0.01017857142857143, "num_tokens": 855}, "3": {"ratio_to_total_tokens": 0.038125, "expected_training_ratio_in_superset": 0.005446428571428571, "num_tokens": 305}, "4": {"ratio_to_total_tokens": 0.025875, "expected_training_ratio_in_superset": 0.004928571428571428, "num_tokens": 207}, "5": {"ratio_to_total_tokens": 0.02175, "expected_training_ratio_in_superset": 0.005178571428571428, "num_tokens": 174}, "6": {"ratio_to_total_tokens": 0.014125, "expected_training_ratio_in_superset": 0.004035714285714286, "num_tokens": 113}, "7": {"ratio_to_total_tokens": 0.014, "expected_training_ratio_in_superset": 0.004666666666666666, "num_tokens": 112}, "8": {"ratio_to_total_tokens": 0.01475, "expected_training_ratio_in_superset": 0.005619047619047619, "num_tokens": 118}, "9": {"ratio_to_total_tokens": 0.014375, "expected_training_ratio_in_superset": 0.006160714285714286, "num_tokens": 115}, "10": {"ratio_to_total_tokens": 0.010875, "expected_training_ratio_in_superset": 0.005178571428571428, "num_tokens": 87}, "11": {"ratio_to_total_tokens": 0.014375, "expected_training_ratio_in_superset": 0.007529761904761905, "num_tokens": 115}, "12": {"ratio_to_total_tokens": 0.015125, "expected_training_ratio_in_superset": 0.008642857142857143, "num_tokens": 121}, "13": {"ratio_to_total_tokens": 0.011625, "expected_training_ratio_in_superset": 0.007196428571428572, "num_tokens": 93}, "14": {"ratio_to_total_tokens": 0.01225, "expected_training_ratio_in_superset": 0.008166666666666666, "num_tokens": 98}, "15": {"ratio_to_total_tokens": 0.01175, "expected_training_ratio_in_superset": 0.008392857142857143, "num_tokens": 94}, "16": {"ratio_to_total_tokens": 0.011875, "expected_training_ratio_in_superset": 0.009047619047619047, "num_tokens": 95}, "17": {"ratio_to_total_tokens": 0.01225, "expected_training_ratio_in_superset": 0.009916666666666666, "num_tokens": 98}, "18": {"ratio_to_total_tokens": 0.011625, "expected_training_ratio_in_superset": 0.009964285714285714, "num_tokens": 93}, "19": {"ratio_to_total_tokens": 0.007875, "expected_training_ratio_in_superset": 0.007125, "num_tokens": 63}, "20": {"ratio_to_total_tokens": 0.00725, "expected_training_ratio_in_superset": 0.006904761904761904, "num_tokens": 58}, "21": {"ratio_to_total_tokens": 0.05425, "expected_training_ratio_in_superset": 0.05425, "num_tokens": 434}, "total_training_compared_to_full_model": 0.215625}