Upload folder using huggingface_hub

.gitattributes

@@ -193,3 +193,12 @@ script_1/0000100000/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
 script_1/0000100000/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
 script_1/0000100000/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
 16k_v2/metrics.jsonl filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/.metadata filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__4_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
+16k_v2/0000100000/__7_0.distcp filter=lfs diff=lfs merge=lfs -text

16k_v2/0000100000/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de43ffedabe827525454600f68292d1f9f04c639772eaec571d6cc33a3df694
+size 1148608

16k_v2/0000100000/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02961c9859f1fcad74303863dba35865448456ce561425f0f343b5272a51310b
+size 3433985232

16k_v2/0000100000/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:266b587cf5df5a8c4c80ade1f192f666a54d0ea528bb569201756bea0b647e04
+size 3434024172

16k_v2/0000100000/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02702a956df7b80101015ad5fa864bb1a0d96c904d8f16e0cc568c8e4b7eb549
+size 3434024172

16k_v2/0000100000/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d3b8a85544c0b6a5057f28506ac475508fcaf23fa600cffee717f73ee8f1c49
+size 3434024172

16k_v2/0000100000/__4_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7661dc390445b97078005b3937158c91ad0052f5641a476f951fc4e8697a5778
+size 3434024172

16k_v2/0000100000/__5_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:821a0e3fb046e6a1d75dc3532a49cef23edb09493e9527d6acb2f090a0778beb
+size 3434026448

16k_v2/0000100000/__6_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2999626398c63d7b1e3168ed12b16392e7a2823f2c7553f7bd6a5bab042e1da
+size 3434026448

16k_v2/0000100000/__7_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2932d528e35c5933ee13a2e8e3811c6e8e18e461527acc5cdee4665e3fcf67d2
+size 3433690640

16k_v2/0000100000/consolidated/consolidated.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af84ca1a876b6e070193e6810ae964b35779820173d6b9911d97f87f8880e057
+size 27465368494

16k_v2/0000100000/consolidated/params.json

@@ -0,0 +1 @@
+{"name": "16k_v2", "dump_dir": "/fsx/craffel/lingua_logs/16k_v2", "seed": 777, "grad_acc_steps": 8, "gc_collect_freq": 1000, "probe_freq": null, "steps": 100000, "data": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "batch_size": 4, "seq_len": 4096, "n_views": 2, "seed": 42, "add_bos": true, "add_eos": true, "load_async": true, "prefetch_size": 1024, "tokenizer": {"name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "load_supermapping": false, "dropout": 0.0, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153}, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "task_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "optim": {"lr": 0.001, "weight_decay": 0.1, "epsilon": 1e-08, "beta1": 0.9, "beta2": 0.95, "clip": 1.0, "scheduler": "cosine", "warmup": 2000, "lr_min_ratio": 1e-06, "cycle_length": 1.0, "cosine_theta": 1.0, "annealing_step": 1000, "decay_fraction": 0.1, "exp_factor": 0.5}, "model": {"dim": 2048, "n_layers": 25, "head_dim": null, "n_heads": 16, "n_kv_heads": null, "ffn_dim_multiplier": null, "multiple_of": 256, "norm_eps": 1e-05, "rope_theta": 10000.0, "init_base_std": null, "init_std_factor": "disabled", "max_seqlen": 4096, "seed": 42, "vocab_size": 245153, "weight_tying": false, "sliding_window": null, "use_factorized_embeddings": false, "factorized_embedding_dim": 0}, "distributed": {"dp_shard": 1, "dp_replicate": 8, "tp_size": 1, "selective_activation_checkpointing": false, "compile": true, "fsdp_type": "full_shard", "model_dtype": "bf16", "float8_recipe": null, "float8_filter": "layers\\.[0-9]+\\.", "matmul_allow_tf32": false, "detect_anomaly": false, "compile_cache_size_limit": 8, "spawn_method": "forkserver"}, "env": {"MKL_SERVICE_FORCE_INTEL": "GNU", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "ENABLE_INTRA_NODE_COMM": "1", "TORCH_NCCL_AVOID_RECORD_STREAMS": "1", "NCCL_IB_TIMEOUT": "22", "NCCL_DEBUG": "INFO", "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1"}, "checkpoint": {"dump": {"every": 10000, "keep": -1}, "eval": {"every": 10000, "keep": -1}, "path": "/fsx/craffel/lingua_logs/checkpoints/16k_v2", "init_ckpt_path": null, "load_init_optimizer_state": false, "save_init_ckpt": false}, "profiling": {"run": true, "trace_folder": "profiling", "mem_warmup": 0, "mem_steps": 4, "profile_warmup": 100, "profile_steps": 4}, "logging": {"freq": 1, "acc_freq": null, "wandb": null}, "async_eval_gpus": 8, "eval": {"harness": {"tasks": ["hellaswag", "piqa", "arc_easy", "arc_challenge", "include_base_44_arabic", "include_base_44_chinese", "include_base_44_german", "include_base_44_greek", "include_base_44_persian", "include_base_44_french", "include_base_44_hungarian", "include_base_44_indonesian", "include_base_44_italian", "include_base_44_japanese", "include_base_44_dutch", "include_base_44_polish", "include_base_44_portuguese", "include_base_44_russian", "include_base_44_spanish", "include_base_44_turkish", "include_base_44_vietnamese", "belebele_arb_Arab", "belebele_ces_Latn", "belebele_zho_Hans", "belebele_dan_Latn", "belebele_deu_Latn", "belebele_ell_Grek", "belebele_pes_Arab", "belebele_fra_Latn", "belebele_hun_Latn", "belebele_ind_Latn", "belebele_ita_Latn", "belebele_jpn_Jpan", "belebele_nld_Latn", "belebele_pol_Latn", "belebele_por_Latn", "belebele_rus_Cyrl", "belebele_spa_Latn", "belebele_swe_Latn", "belebele_tur_Latn", "belebele_vie_Latn", "belebele_eng_Latn", "xnli_ar", "xnli_zh", "xnli_de", "xnli_el", "xnli_en", "xnli_es", "xnli_fr", "xnli_ru", "xnli_tr", "xnli_vi"]}, "generator": {"max_tokens": 16384, "dtype": "bf16", "add_bos": false}}}

16k_v2/0000100000/params.json

@@ -0,0 +1 @@
+{"name": "16k_v2", "dump_dir": "/fsx/craffel/lingua_logs/16k_v2", "seed": 777, "grad_acc_steps": 8, "gc_collect_freq": 1000, "probe_freq": null, "steps": 100000, "data": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "batch_size": 4, "seq_len": 4096, "n_views": 2, "seed": 42, "add_bos": true, "add_eos": true, "load_async": true, "prefetch_size": 1024, "tokenizer": {"name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "load_supermapping": false, "dropout": 0.0, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153}, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "task_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "optim": {"lr": 0.001, "weight_decay": 0.1, "epsilon": 1e-08, "beta1": 0.9, "beta2": 0.95, "clip": 1.0, "scheduler": "cosine", "warmup": 2000, "lr_min_ratio": 1e-06, "cycle_length": 1.0, "cosine_theta": 1.0, "annealing_step": 1000, "decay_fraction": 0.1, "exp_factor": 0.5}, "model": {"dim": 2048, "n_layers": 25, "head_dim": null, "n_heads": 16, "n_kv_heads": null, "ffn_dim_multiplier": null, "multiple_of": 256, "norm_eps": 1e-05, "rope_theta": 10000.0, "init_base_std": null, "init_std_factor": "disabled", "max_seqlen": 4096, "seed": 42, "vocab_size": 245153, "weight_tying": false, "sliding_window": null, "use_factorized_embeddings": false, "factorized_embedding_dim": 0}, "distributed": {"dp_shard": 1, "dp_replicate": 8, "tp_size": 1, "selective_activation_checkpointing": false, "compile": true, "fsdp_type": "full_shard", "model_dtype": "bf16", "float8_recipe": null, "float8_filter": "layers\\.[0-9]+\\.", "matmul_allow_tf32": false, "detect_anomaly": false, "compile_cache_size_limit": 8, "spawn_method": "forkserver"}, "env": {"MKL_SERVICE_FORCE_INTEL": "GNU", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "ENABLE_INTRA_NODE_COMM": "1", "TORCH_NCCL_AVOID_RECORD_STREAMS": "1", "NCCL_IB_TIMEOUT": "22", "NCCL_DEBUG": "INFO", "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1"}, "checkpoint": {"dump": {"every": 10000, "keep": -1}, "eval": {"every": 10000, "keep": -1}, "path": "/fsx/craffel/lingua_logs/checkpoints/16k_v2", "init_ckpt_path": null, "load_init_optimizer_state": false, "save_init_ckpt": false}, "profiling": {"run": true, "trace_folder": "profiling", "mem_warmup": 0, "mem_steps": 4, "profile_warmup": 100, "profile_steps": 4}, "logging": {"freq": 1, "acc_freq": null, "wandb": null}, "async_eval_gpus": 8, "eval": {"harness": {"tasks": ["hellaswag", "piqa", "arc_easy", "arc_challenge", "include_base_44_arabic", "include_base_44_chinese", "include_base_44_german", "include_base_44_greek", "include_base_44_persian", "include_base_44_french", "include_base_44_hungarian", "include_base_44_indonesian", "include_base_44_italian", "include_base_44_japanese", "include_base_44_dutch", "include_base_44_polish", "include_base_44_portuguese", "include_base_44_russian", "include_base_44_spanish", "include_base_44_turkish", "include_base_44_vietnamese", "belebele_arb_Arab", "belebele_ces_Latn", "belebele_zho_Hans", "belebele_dan_Latn", "belebele_deu_Latn", "belebele_ell_Grek", "belebele_pes_Arab", "belebele_fra_Latn", "belebele_hun_Latn", "belebele_ind_Latn", "belebele_ita_Latn", "belebele_jpn_Jpan", "belebele_nld_Latn", "belebele_pol_Latn", "belebele_por_Latn", "belebele_rus_Cyrl", "belebele_spa_Latn", "belebele_swe_Latn", "belebele_tur_Latn", "belebele_vie_Latn", "belebele_eng_Latn", "xnli_ar", "xnli_zh", "xnli_de", "xnli_el", "xnli_en", "xnli_es", "xnli_fr", "xnli_ru", "xnli_tr", "xnli_vi"]}, "generator": {"max_tokens": 16384, "dtype": "bf16", "add_bos": false}}}

16k_v2/0000100000/train_state_00000.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 7152, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.00.jsonl", "position": 28211518576, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.00.jsonl", "position": 1292403441, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.00.jsonl", "position": 1216282816, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.00.jsonl", "position": 1299891171, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.00.jsonl", "position": 1726998261, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.00.jsonl", "position": 1453802685, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.00.jsonl", "position": 1102366203, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.00.jsonl", "position": 1691396241, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.00.jsonl", "position": 1900099878, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.00.jsonl", "position": 2361176339, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.00.jsonl", "position": 1679413138, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.00.jsonl", "position": 1335177142, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.00.jsonl", "position": 1402921203, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.00.jsonl", "position": 1456709069, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.00.jsonl", "position": 1630666470, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.00.jsonl", "position": 1708019340, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.00.jsonl", "position": 2264593166, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.00.jsonl", "position": 2033854194, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.00.jsonl", "position": 2469347173, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.00.jsonl", "position": 2618507260, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.00.jsonl", "position": 6143171791, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 28610450372406836785747378155947974094, "inc": 252101603063402394885084957393789173453}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 198214118093403813455753969368930126046, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 300569569296678341640414112158566886944, "inc": 257317082376085721142933171929815648017}, "has_uint32": 1, "uinteger": 630457105}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00001.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 579, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.01.jsonl", "position": 28208769127, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.01.jsonl", "position": 1288225738, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.01.jsonl", "position": 1231007388, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.01.jsonl", "position": 1299262490, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.01.jsonl", "position": 1717559039, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.01.jsonl", "position": 1457012968, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.01.jsonl", "position": 1093215920, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.01.jsonl", "position": 1707304265, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.01.jsonl", "position": 1905110024, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.01.jsonl", "position": 2376589207, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.01.jsonl", "position": 1667731010, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.01.jsonl", "position": 1334090057, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.01.jsonl", "position": 1406500780, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.01.jsonl", "position": 1467628486, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.01.jsonl", "position": 1628953348, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.01.jsonl", "position": 1698866979, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.01.jsonl", "position": 2270511571, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.01.jsonl", "position": 2053595505, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.01.jsonl", "position": 2468167071, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.01.jsonl", "position": 2619606831, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.01.jsonl", "position": 6106866459, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 87582146033587290576825497823843111755, "inc": 246509925186285949978196491240064802315}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 125762442125913906867820110309968161525, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 298609680158430271867266436931522339460, "inc": 173555323965545256606922338259303677603}, "has_uint32": 1, "uinteger": 1182378492}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00002.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 1525, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.02.jsonl", "position": 28217397362, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.02.jsonl", "position": 1295478062, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.02.jsonl", "position": 1218866388, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.02.jsonl", "position": 1298667105, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.02.jsonl", "position": 1714691253, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.02.jsonl", "position": 1450374531, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.02.jsonl", "position": 1104709206, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.02.jsonl", "position": 1680246023, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.02.jsonl", "position": 1916624551, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.02.jsonl", "position": 2366360773, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.02.jsonl", "position": 1684922935, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.02.jsonl", "position": 1342888825, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.02.jsonl", "position": 1399179652, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.02.jsonl", "position": 1469053095, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.02.jsonl", "position": 1636216287, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.02.jsonl", "position": 1711266540, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.02.jsonl", "position": 2259216174, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.02.jsonl", "position": 2026561979, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.02.jsonl", "position": 2464549456, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.02.jsonl", "position": 2619050255, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.02.jsonl", "position": 6120534413, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 295071160362498741441632726989024403463, "inc": 234358335530849485425064040311006256713}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 331547148098719151179957040503437013885, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 172206642884315098528897268843500314535, "inc": 319170006889470250209362588441616495209}, "has_uint32": 1, "uinteger": 2183176397}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00003.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 14460, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.03.jsonl", "position": 28196352595, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.03.jsonl", "position": 1291005904, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.03.jsonl", "position": 1213312056, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.03.jsonl", "position": 1303561669, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.03.jsonl", "position": 1735225856, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.03.jsonl", "position": 1446169207, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.03.jsonl", "position": 1107724324, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.03.jsonl", "position": 1687213519, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.03.jsonl", "position": 1907393778, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.03.jsonl", "position": 2376610162, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.03.jsonl", "position": 1673979811, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.03.jsonl", "position": 1347552336, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.03.jsonl", "position": 1404836131, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.03.jsonl", "position": 1458666572, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.03.jsonl", "position": 1636270923, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.03.jsonl", "position": 1692141614, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.03.jsonl", "position": 2256691131, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.03.jsonl", "position": 2046675467, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.03.jsonl", "position": 2471202776, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.03.jsonl", "position": 2613686016, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.03.jsonl", "position": 6144492016, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 290928069349475948880212637988219160813, "inc": 148211758571781046255077612135386035203}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 142182867940734970069809884940118462835, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 54691439170624420224489263842165716902, "inc": 115810872492597857501795428972873905393}, "has_uint32": 1, "uinteger": 1237320779}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00004.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 653, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.04.jsonl", "position": 28203463790, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.04.jsonl", "position": 1299583598, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.04.jsonl", "position": 1209035304, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.04.jsonl", "position": 1300854217, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.04.jsonl", "position": 1731116570, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.04.jsonl", "position": 1443448109, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.04.jsonl", "position": 1104863238, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.04.jsonl", "position": 1698530374, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.04.jsonl", "position": 1903060889, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.04.jsonl", "position": 2387661257, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.04.jsonl", "position": 1669303564, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.04.jsonl", "position": 1341628666, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.04.jsonl", "position": 1402522090, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.04.jsonl", "position": 1469278701, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.04.jsonl", "position": 1621553508, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.04.jsonl", "position": 1714230660, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.04.jsonl", "position": 2272593029, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.04.jsonl", "position": 2023466329, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.04.jsonl", "position": 2462444786, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.04.jsonl", "position": 2612671378, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.04.jsonl", "position": 6138784311, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 337185332786316301710403096534656566546, "inc": 186633262021180533256729114674950595327}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 57648107932610625738957943735212255702, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 330736315247521707224292038935126153445, "inc": 303111205818808944921858206842105131807}, "has_uint32": 1, "uinteger": 211256137}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00005.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 1944, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.05.jsonl", "position": 28228344835, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.05.jsonl", "position": 1287733390, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.05.jsonl", "position": 1210988080, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.05.jsonl", "position": 1306134094, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.05.jsonl", "position": 1713633680, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.05.jsonl", "position": 1443146671, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.05.jsonl", "position": 1097753103, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.05.jsonl", "position": 1696817784, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.05.jsonl", "position": 1902010778, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.05.jsonl", "position": 2374787930, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.05.jsonl", "position": 1682336592, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.05.jsonl", "position": 1324021978, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.05.jsonl", "position": 1414371749, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.05.jsonl", "position": 1465151773, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.05.jsonl", "position": 1628457056, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.05.jsonl", "position": 1706361622, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.05.jsonl", "position": 2272431930, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.05.jsonl", "position": 2039697375, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.05.jsonl", "position": 2466700645, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.05.jsonl", "position": 2613016733, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.05.jsonl", "position": 6111840956, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 72816059375701757526390307082648175404, "inc": 329233669073478483697346584247981015037}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 161849973963004483058982823204106918040, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 152527936818944850177382785176910810558, "inc": 47382953940698287647753879262736142901}, "has_uint32": 1, "uinteger": 3675784275}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00006.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 1149, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.06.jsonl", "position": 28217443510, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.06.jsonl", "position": 1302180299, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.06.jsonl", "position": 1212330542, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.06.jsonl", "position": 1294057150, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.06.jsonl", "position": 1722302813, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.06.jsonl", "position": 1436909119, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.06.jsonl", "position": 1105778031, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.06.jsonl", "position": 1700194254, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.06.jsonl", "position": 1901576554, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.06.jsonl", "position": 2362281924, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.06.jsonl", "position": 1687157691, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.06.jsonl", "position": 1342194890, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.06.jsonl", "position": 1415425551, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.06.jsonl", "position": 1462720416, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.06.jsonl", "position": 1621966866, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.06.jsonl", "position": 1696166935, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.06.jsonl", "position": 2268281392, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.06.jsonl", "position": 2034943996, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.06.jsonl", "position": 2471115738, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.06.jsonl", "position": 2613812497, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.06.jsonl", "position": 6102589154, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 115796155978518956299274611046732030294, "inc": 95963489890761403814531195999220475639}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 212318578713204682054852480679394323439, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 131915213571014219328634582687476804074, "inc": 72545526324180839152750112646078969085}, "has_uint32": 0, "uinteger": 2783644522}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}

16k_v2/0000100000/train_state_00007.json

@@ -0,0 +1 @@
+{"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 78, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.07.jsonl", "position": 28206289166, "block_size": 1, "offset": 0, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.07.jsonl", "position": 1306723242, "block_size": 1, "offset": 0, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.07.jsonl", "position": 1214986411, "block_size": 1, "offset": 0, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.07.jsonl", "position": 1296761328, "block_size": 1, "offset": 0, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.07.jsonl", "position": 1722527700, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.07.jsonl", "position": 1449491906, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.07.jsonl", "position": 1110052535, "block_size": 1, "offset": 0, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.07.jsonl", "position": 1686919418, "block_size": 1, "offset": 0, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.07.jsonl", "position": 1899806536, "block_size": 1, "offset": 0, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.07.jsonl", "position": 2377797737, "block_size": 1, "offset": 0, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.07.jsonl", "position": 1672071548, "block_size": 1, "offset": 0, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.07.jsonl", "position": 1342493654, "block_size": 1, "offset": 0, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.07.jsonl", "position": 1408679521, "block_size": 1, "offset": 0, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.07.jsonl", "position": 1457576686, "block_size": 1, "offset": 0, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.07.jsonl", "position": 1630315616, "block_size": 1, "offset": 0, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.07.jsonl", "position": 1715050359, "block_size": 1, "offset": 0, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.07.jsonl", "position": 2272589916, "block_size": 1, "offset": 0, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.07.jsonl", "position": 2043250432, "block_size": 1, "offset": 0, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.07.jsonl", "position": 2453359108, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.07.jsonl", "position": 2615515241, "block_size": 1, "offset": 0, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.07.jsonl", "position": 6098149477, "block_size": 1, "offset": 0, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 287760825584517570123518819697386961465, "inc": 53245743019587277358203950863334653629}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/bpe_ltr_arb_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ces_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_dan_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_deu_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ell_Grek_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fas_Arab_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fra_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_fw_edu_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_hun_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ind_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_ita_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_nld_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_pol_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_por_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_spa_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_swe_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_tur_Latn_16000_v2", "load_supermapping": true}, {"name": "huggingface", "path": "flexitok/bpe_ltr_vie_Latn_16000_v2", "load_supermapping": true}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 269340098868132425277613381654239643847, "inc": 332724090758049132448979897138935081983}, "has_uint32": 0, "uinteger": 0}, "seed": 42, "superset_code_name": "16k_v2", "n_words": 245153, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/bpe_ltr_arb_Arab_16000_v2", "ces_Latn": "flexitok/bpe_ltr_ces_Latn_16000_v2", "cmn_Hani": "flexitok/bpe_ltr_cmn_Hani_16000_v2", "dan_Latn": "flexitok/bpe_ltr_dan_Latn_16000_v2", "deu_Latn": "flexitok/bpe_ltr_deu_Latn_16000_v2", "ell_Grek": "flexitok/bpe_ltr_ell_Grek_16000_v2", "fas_Arab": "flexitok/bpe_ltr_fas_Arab_16000_v2", "fra_Latn": "flexitok/bpe_ltr_fra_Latn_16000_v2", "fw_edu": "flexitok/bpe_ltr_fw_edu_16000_v2", "hun_Latn": "flexitok/bpe_ltr_hun_Latn_16000_v2", "ind_Latn": "flexitok/bpe_ltr_ind_Latn_16000_v2", "ita_Latn": "flexitok/bpe_ltr_ita_Latn_16000_v2", "jpn_Jpan": "flexitok/bpe_ltr_jpn_Jpan_16000_v2", "nld_Latn": "flexitok/bpe_ltr_nld_Latn_16000_v2", "pol_Latn": "flexitok/bpe_ltr_pol_Latn_16000_v2", "por_Latn": "flexitok/bpe_ltr_por_Latn_16000_v2", "rus_Cyrl": "flexitok/bpe_ltr_rus_Cyrl_16000_v2", "spa_Latn": "flexitok/bpe_ltr_spa_Latn_16000_v2", "swe_Latn": "flexitok/bpe_ltr_swe_Latn_16000_v2", "tur_Latn": "flexitok/bpe_ltr_tur_Latn_16000_v2", "vie_Latn": "flexitok/bpe_ltr_vie_Latn_16000_v2"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 239803483826676955776584746976189400951, "inc": 19761753544780285878460645500694854795}, "has_uint32": 1, "uinteger": 513301027}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}