Hugging Face's logo

craffel
/
supertoken_models

Model card Files
xet
 Community
craffel HF Staff commited on
Commit
6a0184e
·
verified ·
1 Parent(s): 58198ae

Upload folder using huggingface_hub

Browse files
Files changed (21) hide show
  1. .gitattributes +9 -0
  2. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/.metadata +3 -0
  3. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__0_0.distcp +3 -0
  4. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__1_0.distcp +3 -0
  5. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__2_0.distcp +3 -0
  6. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__3_0.distcp +3 -0
  7. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__4_0.distcp +3 -0
  8. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__5_0.distcp +3 -0
  9. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__6_0.distcp +3 -0
  10. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__7_0.distcp +3 -0
  11. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/consolidated/consolidated.pth +3 -0
  12. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/consolidated/params.json +1 -0
  13. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/params.json +1 -0
  14. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00000.json +1 -0
  15. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00001.json +1 -0
  16. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00002.json +1 -0
  17. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00003.json +1 -0
  18. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00004.json +1 -0
  19. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00005.json +1 -0
  20. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00006.json +1 -0
  21. meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00007.json +1 -0
.gitattributes CHANGED
@@ -143,3 +143,12 @@ fineweb2_hq_superset_oracle_09/0000100000/__5_0.distcp filter=lfs diff=lfs merge
143
  fineweb2_hq_superset_oracle_09/0000100000/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
144
  fineweb2_hq_superset_oracle_09/0000100000/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
145
  meta-llama-Llama-3.2-1B-seed_777_model_seed_111/metrics.jsonl filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
143
  fineweb2_hq_superset_oracle_09/0000100000/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
144
  fineweb2_hq_superset_oracle_09/0000100000/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
145
  meta-llama-Llama-3.2-1B-seed_777_model_seed_111/metrics.jsonl filter=lfs diff=lfs merge=lfs -text
146
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/.metadata filter=lfs diff=lfs merge=lfs -text
147
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
148
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
149
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
150
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
151
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__4_0.distcp filter=lfs diff=lfs merge=lfs -text
152
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
153
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
154
+ meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/.metadata ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55948608c2b3120cbc84fb0860d6299410dffb2937a507daa9d987f9aa124416
3
+ size 1148572
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__0_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b054f0a6065936e3c3eca5c04acb8c5731f4b0a8e1931096720026c51d032dfe
3
+ size 2715727056
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__1_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae4a34d812a86dca5330774df73be34a42cbbcfba0f0ba558720a0cfec9a74d5
3
+ size 2715765996
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__2_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe69ba8d0f9158b5dd74cb693da575353d445332b3f4b5f778b1a63135e3c987
3
+ size 2715765996
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__3_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:35b4dce045c2ea0e41e3e3588d29112d06817e0b01160d4c2ba7825c21f09402
3
+ size 2715765996
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__4_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:edbdd6518c9acb79f009ec660b7e6f7adb8c1c095efcf6eb480c5c78512c20dd
3
+ size 2715765996
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__5_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:006ec700a0dbd529dbc56ae440acc39963117933ac3dc82c21bc413fd0295835
3
+ size 2715768272
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__6_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbbca59223617c4f5d639e74506269dbaed2a27899120570fa29f62768290989
3
+ size 2715768272
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/__7_0.distcp ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a929a1a06bf01b75bcde568e2dadbef3f86bf111bee8cc9fedccca6855bd198a
3
+ size 2715776528
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/consolidated/consolidated.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0d7f2b230a4b2542db0405f3607e706830b72a83b812bd843d5bdec89100c67
3
+ size 21719647126
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/consolidated/params.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"name": "toksuite_llama_1b_seed_777_model_seed_888", "dump_dir": "/fsx/craffel/toksuite/lingua_logs/meta-llama-Llama-3.2-1B-seed_777_model_seed_888/", "seed": 777, "grad_acc_steps": 8, "gc_collect_freq": 1000, "probe_freq": null, "steps": 100000, "data": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "batch_size": 4, "seq_len": 4096, "n_views": 2, "seed": 42, "add_bos": true, "add_eos": true, "load_async": true, "prefetch_size": 1024, "tokenizer": {"name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "load_supermapping": false, "dropout": 0.0, "seed": 42, "superset_code_name": "super_vocab", "n_words": null}, "routing": {"source_to_tokenizer": {}, "task_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "optim": {"lr": 0.001, "weight_decay": 0.1, "epsilon": 1e-08, "beta1": 0.9, "beta2": 0.95, "clip": 1.0, "scheduler": "cosine", "warmup": 2000, "lr_min_ratio": 1e-06, "cycle_length": 1.0, "cosine_theta": 1.0, "annealing_step": 1000, "decay_fraction": 0.1, "exp_factor": 0.5}, "model": {"dim": 2048, "n_layers": 25, "head_dim": null, "n_heads": 16, "n_kv_heads": null, "ffn_dim_multiplier": null, "multiple_of": 256, "norm_eps": 1e-05, "rope_theta": 10000.0, "init_base_std": null, "init_std_factor": "disabled", "max_seqlen": 4096, "seed": 888, "vocab_size": 128256, "weight_tying": false, "sliding_window": null, "use_factorized_embeddings": false, "factorized_embedding_dim": 0}, "distributed": {"dp_shard": 1, "dp_replicate": 8, "tp_size": 1, "selective_activation_checkpointing": false, "compile": true, "fsdp_type": "full_shard", "model_dtype": "bf16", "float8_recipe": null, "float8_filter": "layers\\.[0-9]+\\.", "matmul_allow_tf32": false, "detect_anomaly": false, "compile_cache_size_limit": 8, "spawn_method": "forkserver"}, "env": {"MKL_SERVICE_FORCE_INTEL": "GNU", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "ENABLE_INTRA_NODE_COMM": "1", "TORCH_NCCL_AVOID_RECORD_STREAMS": "1", "NCCL_IB_TIMEOUT": "22", "NCCL_DEBUG": "INFO", "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1"}, "checkpoint": {"dump": {"every": 6000, "keep": -1}, "eval": {"every": 2000, "keep": -1}, "path": "/fsx/craffel/toksuite/lingua_logs/meta-llama-Llama-3.2-1B/checkpoints", "init_ckpt_path": null, "load_init_optimizer_state": false, "save_init_ckpt": false}, "profiling": {"run": true, "trace_folder": "profiling", "mem_warmup": 0, "mem_steps": 4, "profile_warmup": 100, "profile_steps": 4}, "logging": {"freq": 1, "acc_freq": null, "wandb": null}, "async_eval_gpus": 8, "eval": {"harness": {"tasks": ["hellaswag", "piqa", "arc_easy", "arc_challenge", "include_base_44_turkish", "include_base_44_italian", "include_base_44_chinese", "belebele_pes_Arab", "belebele_eng_Latn", "belebele_ita_Latn", "belebele_tur_Latn", "belebele_zho_Hans", "xnli_en", "xnli_tr", "xnli_zh"]}, "generator": {"max_tokens": 8192, "dtype": "bf16", "add_bos": false}}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/params.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"name": "toksuite_llama_1b_seed_777_model_seed_888", "dump_dir": "/fsx/craffel/toksuite/lingua_logs/meta-llama-Llama-3.2-1B-seed_777_model_seed_888/", "seed": 777, "grad_acc_steps": 8, "gc_collect_freq": 1000, "probe_freq": null, "steps": 100000, "data": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "batch_size": 4, "seq_len": 4096, "n_views": 2, "seed": 42, "add_bos": true, "add_eos": true, "load_async": true, "prefetch_size": 1024, "tokenizer": {"name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "load_supermapping": false, "dropout": 0.0, "seed": 42, "superset_code_name": "super_vocab", "n_words": null}, "routing": {"source_to_tokenizer": {}, "task_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "optim": {"lr": 0.001, "weight_decay": 0.1, "epsilon": 1e-08, "beta1": 0.9, "beta2": 0.95, "clip": 1.0, "scheduler": "cosine", "warmup": 2000, "lr_min_ratio": 1e-06, "cycle_length": 1.0, "cosine_theta": 1.0, "annealing_step": 1000, "decay_fraction": 0.1, "exp_factor": 0.5}, "model": {"dim": 2048, "n_layers": 25, "head_dim": null, "n_heads": 16, "n_kv_heads": null, "ffn_dim_multiplier": null, "multiple_of": 256, "norm_eps": 1e-05, "rope_theta": 10000.0, "init_base_std": null, "init_std_factor": "disabled", "max_seqlen": 4096, "seed": 888, "vocab_size": 128256, "weight_tying": false, "sliding_window": null, "use_factorized_embeddings": false, "factorized_embedding_dim": 0}, "distributed": {"dp_shard": 1, "dp_replicate": 8, "tp_size": 1, "selective_activation_checkpointing": false, "compile": true, "fsdp_type": "full_shard", "model_dtype": "bf16", "float8_recipe": null, "float8_filter": "layers\\.[0-9]+\\.", "matmul_allow_tf32": false, "detect_anomaly": false, "compile_cache_size_limit": 8, "spawn_method": "forkserver"}, "env": {"MKL_SERVICE_FORCE_INTEL": "GNU", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "ENABLE_INTRA_NODE_COMM": "1", "TORCH_NCCL_AVOID_RECORD_STREAMS": "1", "NCCL_IB_TIMEOUT": "22", "NCCL_DEBUG": "INFO", "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1"}, "checkpoint": {"dump": {"every": 6000, "keep": -1}, "eval": {"every": 2000, "keep": -1}, "path": "/fsx/craffel/toksuite/lingua_logs/meta-llama-Llama-3.2-1B/checkpoints", "init_ckpt_path": null, "load_init_optimizer_state": false, "save_init_ckpt": false}, "profiling": {"run": true, "trace_folder": "profiling", "mem_warmup": 0, "mem_steps": 4, "profile_warmup": 100, "profile_steps": 4}, "logging": {"freq": 1, "acc_freq": null, "wandb": null}, "async_eval_gpus": 8, "eval": {"harness": {"tasks": ["hellaswag", "piqa", "arc_easy", "arc_challenge", "include_base_44_turkish", "include_base_44_italian", "include_base_44_chinese", "belebele_pes_Arab", "belebele_eng_Latn", "belebele_ita_Latn", "belebele_tur_Latn", "belebele_zho_Hans", "xnli_en", "xnli_tr", "xnli_zh"]}, "generator": {"max_tokens": 8192, "dtype": "bf16", "add_bos": false}}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00000.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 3809, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.00.jsonl", "position": 28923659462, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.00.jsonl", "position": 8878702149, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.00.jsonl", "position": 3514187030, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.00.jsonl", "position": 7725981215, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.00.jsonl", "position": 1039867211, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 252006778087284745682378847932169343221, "inc": 252101603063402394885084957393789173453}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 300569569296678341640414112158566886944, "inc": 257317082376085721142933171929815648017}, "has_uint32": 1, "uinteger": 630457105}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00001.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 22372, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.01.jsonl", "position": 28911907797, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.01.jsonl", "position": 8866600855, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.01.jsonl", "position": 3523243125, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.01.jsonl", "position": 7748009227, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.01.jsonl", "position": 1034977015, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 150660440047832350023741814646711888543, "inc": 246509925186285949978196491240064802315}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 298609680158430271867266436931522339460, "inc": 173555323965545256606922338259303677603}, "has_uint32": 1, "uinteger": 1182378492}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00002.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 25, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.02.jsonl", "position": 28915311047, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.02.jsonl", "position": 8865983864, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.02.jsonl", "position": 3510401929, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.02.jsonl", "position": 7739133096, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.02.jsonl", "position": 1040424836, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 84443802827960551292957336347060699986, "inc": 234358335530849485425064040311006256713}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 172206642884315098528897268843500314535, "inc": 319170006889470250209362588441616495209}, "has_uint32": 1, "uinteger": 2183176397}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00003.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 323, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.03.jsonl", "position": 28929581476, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.03.jsonl", "position": 8855899388, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.03.jsonl", "position": 3532541572, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.03.jsonl", "position": 7744244902, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.03.jsonl", "position": 1039231639, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 247719738647945436892247052549358623267, "inc": 148211758571781046255077612135386035203}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 54691439170624420224489263842165716902, "inc": 115810872492597857501795428972873905393}, "has_uint32": 1, "uinteger": 1237320779}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00004.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 535, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.04.jsonl", "position": 28940176490, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.04.jsonl", "position": 8870400674, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.04.jsonl", "position": 3541056680, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.04.jsonl", "position": 7731913299, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.04.jsonl", "position": 1004032538, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 89393135109349170892217878116501711473, "inc": 186633262021180533256729114674950595327}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 330736315247521707224292038935126153445, "inc": 303111205818808944921858206842105131807}, "has_uint32": 1, "uinteger": 211256137}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00005.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 307, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.05.jsonl", "position": 28945621310, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.05.jsonl", "position": 8861977844, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.05.jsonl", "position": 3526810869, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.05.jsonl", "position": 7739356867, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.05.jsonl", "position": 1000810212, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 196809890920812555086752416725913898418, "inc": 329233669073478483697346584247981015037}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 152527936818944850177382785176910810558, "inc": 47382953940698287647753879262736142901}, "has_uint32": 1, "uinteger": 3675784275}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00006.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 6621, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.06.jsonl", "position": 28953365878, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.06.jsonl", "position": 8871564781, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.06.jsonl", "position": 3541407922, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.06.jsonl", "position": 7723234587, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.06.jsonl", "position": 989393492, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 65085220618383211160166919022876062037, "inc": 95963489890761403814531195999220475639}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 131915213571014219328634582687476804074, "inc": 72545526324180839152750112646078969085}, "has_uint32": 0, "uinteger": 2783644522}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}
meta-llama-Llama-3.2-1B-seed_777_model_seed_111/0000100000/train_state_00007.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"step": 100000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 95, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/tokenizer_training/", "sources": {"fw_edu": 0.4, "cmn_Hani": 0.15, "tur_Latn": 0.15, "ita_Latn": 0.15, "fas_Arab": 0.15}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fw_edu/fineweb_edu_100bt.chunk.07.jsonl", "position": 28904321208, "block_size": 1, "offset": 0, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.07.jsonl", "position": 8859495407, "block_size": 1, "offset": 0, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/tur_Latn/fineweb_2_hq.tur_Latn.chunk.07.jsonl", "position": 3539713423, "block_size": 1, "offset": 0, "current_iter": 1}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/ita_Latn/fineweb_2_hq.ita_Latn.chunk.07.jsonl", "position": 7741636969, "block_size": 1, "offset": 0, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/tokenizer_training/fas_Arab/fineweb_2_hq.fas_Arab.chunk.07.jsonl", "position": 1037894696, "block_size": 1, "offset": 0, "current_iter": 3}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 45700460280623489407278938463076623728, "inc": 53245743019587277358203950863334653629}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "huggingface", "path": "meta-llama/Llama-3.2-1B", "tokenizers": null, "dropout": 0.0, "rng_state": null, "seed": 42, "superset_code_name": "super_vocab", "n_words": null, "routing": {"source_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 256, "rng_state": {"bit_generator": "PCG64", "state": {"state": 239803483826676955776584746976189400951, "inc": 19761753544780285878460645500694854795}, "has_uint32": 1, "uinteger": 513301027}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 100000, "verbose": false, "_step_count": 100001, "_get_lr_called_within_step": false, "_last_lr": [0.001], "lr_lambdas": [{}]}}