safe_v2_2nodes/config.yaml

name: safe_v2_2nodes
dump_dir: /fsx/craffel/lingua_logs/safe_v2_2nodes
seed: 777
grad_acc_steps: 4
gc_collect_freq: 1000
probe_freq: null
steps: 100000
data:
  root_dir: /scratch/craffel/lingua/data/flexitok/
  sources:
    fw_edu: 0.4
    dan_Latn: 0.0216582869670702
    swe_Latn: 0.0216359765418466
    vie_Latn: 0.0197485510268674
    hun_Latn: 0.0247194573562308
    fas_Arab: 0.0205634624231076
    tur_Latn: 0.0235455794841729
    ces_Latn: 0.0248024455266208
    arb_Arab: 0.0234323706569333
    ell_Grek: 0.0233670886888026
    ind_Latn: 0.0269322054593488
    nld_Latn: 0.0277796326621489
    pol_Latn: 0.0294120104572311
    por_Latn: 0.0301413168306825
    ita_Latn: 0.0324056371021865
    jpn_Jpan: 0.03553104151369
    fra_Latn: 0.0381835560678536
    spa_Latn: 0.0387222793083669
    deu_Latn: 0.0419925340453022
    cmn_Hani: 0.0454067521384114
    rus_Cyrl: 0.0500198157431261
  batch_size: 4
  seq_len: 4096
  n_views: 2
  seed: 42
  add_bos: true
  add_eos: true
  load_async: true
  prefetch_size: 1024
  tokenizer:
    name: supertokenizer
    path: meta-llama/Llama-3.2-1B
    tokenizers:
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ces_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000_v2
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_dan_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ell_Grek_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_fas_Arab_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_hun_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_nld_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_pol_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_por_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_swe_Latn_8000
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      load_supermapping: false
    - name: huggingface
      path: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      load_supermapping: false
    load_supermapping: false
    dropout: 0.0
    seed: 42
    superset_code_name: safe_v2
    n_words: 122554
  routing:
    source_to_tokenizer:
      arb_Arab: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      ces_Latn: flexitok/reindexed-flexitok-bpe_ces_Latn_8000
      cmn_Hani: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      dan_Latn: flexitok/reindexed-flexitok-bpe_dan_Latn_8000
      deu_Latn: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      ell_Grek: flexitok/reindexed-flexitok-bpe_ell_Grek_8000
      fas_Arab: flexitok/reindexed-flexitok-bpe_fas_Arab_8000
      fra_Latn: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      fw_edu: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      hun_Latn: flexitok/reindexed-flexitok-bpe_hun_Latn_8000
      ind_Latn: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      ita_Latn: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      jpn_Jpan: flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2
      nld_Latn: flexitok/reindexed-flexitok-bpe_nld_Latn_8000
      pol_Latn: flexitok/reindexed-flexitok-bpe_pol_Latn_8000
      por_Latn: flexitok/reindexed-flexitok-bpe_por_Latn_8000
      rus_Cyrl: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      spa_Latn: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      swe_Latn: flexitok/reindexed-flexitok-bpe_swe_Latn_8000
      tur_Latn: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      vie_Latn: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
    task_to_tokenizer: {}
    suitable_tokenizer_probability: 1.0
optim:
  lr: 0.001
  weight_decay: 0.1
  epsilon: 1.0e-08
  beta1: 0.9
  beta2: 0.95
  clip: 1.0
  scheduler: cosine
  warmup: 2000
  lr_min_ratio: 1.0e-06
  cycle_length: 1.0
  cosine_theta: 1.0
  annealing_step: 1000
  decay_fraction: 0.1
  exp_factor: 0.5
model:
  dim: 2048
  n_layers: 25
  head_dim: null
  n_heads: 16
  n_kv_heads: null
  ffn_dim_multiplier: null
  multiple_of: 256
  norm_eps: 1.0e-05
  rope_theta: 10000.0
  init_base_std: null
  init_std_factor: disabled
  max_seqlen: 4096
  seed: 42
  vocab_size: 122554
  weight_tying: false
  sliding_window: null
  use_factorized_embeddings: false
  factorized_embedding_dim: 0
distributed:
  dp_shard: 1
  dp_replicate: 16
  tp_size: 1
  selective_activation_checkpointing: false
  compile: true
  fsdp_type: full_shard
  model_dtype: bf16
  float8_recipe: null
  float8_filter: layers\.[0-9]+\.
  matmul_allow_tf32: false
  detect_anomaly: false
  compile_cache_size_limit: 8
  spawn_method: forkserver
env:
  MKL_SERVICE_FORCE_INTEL: GNU
  OMP_NUM_THREADS: '1'
  MKL_NUM_THREADS: '1'
  ENABLE_INTRA_NODE_COMM: '1'
  TORCH_NCCL_AVOID_RECORD_STREAMS: '1'
  NCCL_IB_TIMEOUT: '22'
  NCCL_DEBUG: INFO
  TORCH_NCCL_ASYNC_ERROR_HANDLING: '1'
checkpoint:
  dump:
    every: 2500
    keep: -1
  eval:
    every: 2500
    keep: -1
  path: /fsx/craffel/lingua_logs/checkpoints/safe_v2_2nodes
  init_ckpt_path: null
  load_init_optimizer_state: false
  save_init_ckpt: false
profiling:
  run: true
  trace_folder: profiling
  mem_warmup: 0
  mem_steps: 4
  profile_warmup: 100
  profile_steps: 4
logging:
  freq: 1
  acc_freq: null
  wandb:
    job_type: null
    dir: null
    project: MoTo-Training
    entity: raffel-reports
    tags: null
    group: null
    name: safe_v2_2nodes
    notes: null
    config_exclude_keys: null
    config_include_keys: null
    anonymous: null
    mode: null
    allow_val_change: null
    resume: null
    force: null
    tensorboard: null
    sync_tensorboard: null
    monitor_gym: null
    save_code: null
    id: null
    fork_from: null
    resume_from: null
async_eval_gpus: null
eval:
  harness:
    tasks:
    - hellaswag
    - piqa
    - arc_easy
    - arc_challenge
    - xnli_ar
    - xnli_zh
    - xnli_de
    - xnli_el
    - xnli_en
    - xnli_es
    - xnli_fr
    - xnli_ru
    - xnli_tr
    - xnli_vi
    - toksuite_english
    - toksuite_chinese
    - toksuite_turkish
    - toksuite_italian
    - toksuite_farsi
    - toksuite_math
    - toksuite_stem
    - xcopa_id
    - xcopa_it
    - xcopa_tr
    - xcopa_vi
    - xcopa_zh
    - xstorycloze_ar
    - xstorycloze_en
    - xstorycloze_es
    - xstorycloze_id
    - xstorycloze_ru
    - xstorycloze_zh
    - paws_de
    - paws_en
    - paws_es
    - paws_fr
    - paws_ja
    - paws_zh
    - hellaswag_ar
    - hellaswag_da
    - hellaswag_de
    - hellaswag_fr
    - hellaswag_hu
    - hellaswag_id
    - hellaswag_it
    - hellaswag_nl
    - hellaswag_pt
    - hellaswag_ru
    - hellaswag_sv
    - hellaswag_vi
    - hellaswag_es
  generator:
    max_tokens: 16384
    dtype: bf16
    add_bos: false
  routing:
    suitable_tokenizer_probability: 1.0
    task_to_tokenizer:
      arc_easy: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      arc_challenge: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      hellaswag: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      piqa: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      include_base_44_arabic: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      include_base_44_chinese: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      include_base_44_german: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      include_base_44_greek: flexitok/reindexed-flexitok-bpe_ell_Grek_8000
      include_base_44_persian: flexitok/reindexed-flexitok-bpe_fas_Arab_8000
      include_base_44_french: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      include_base_44_hungarian: flexitok/reindexed-flexitok-bpe_hun_Latn_8000
      include_base_44_indonesian: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      include_base_44_italian: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      include_base_44_japanese: flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2
      include_base_44_dutch: flexitok/reindexed-flexitok-bpe_nld_Latn_8000
      include_base_44_portuguese: flexitok/reindexed-flexitok-bpe_por_Latn_8000
      include_base_44_russian: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      include_base_44_spanish: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      include_base_44_turkish: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      include_base_44_vietnamese: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      belebele_arb_Arab: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      belebele_ces_Latn: flexitok/reindexed-flexitok-bpe_ces_Latn_8000
      belebele_zho_Hans: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      belebele_dan_Latn: flexitok/reindexed-flexitok-bpe_dan_Latn_8000
      belebele_deu_Latn: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      belebele_ell_Grek: flexitok/reindexed-flexitok-bpe_ell_Grek_8000
      belebele_pes_Arab: flexitok/reindexed-flexitok-bpe_fas_Arab_8000
      belebele_fra_Latn: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      belebele_hun_Latn: flexitok/reindexed-flexitok-bpe_hun_Latn_8000
      belebele_ind_Latn: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      belebele_ita_Latn: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      belebele_jpn_Jpan: flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2
      belebele_nld_Latn: flexitok/reindexed-flexitok-bpe_nld_Latn_8000
      belebele_pol_Latn: flexitok/reindexed-flexitok-bpe_pol_Latn_8000
      belebele_por_Latn: flexitok/reindexed-flexitok-bpe_por_Latn_8000
      belebele_rus_Cyrl: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      belebele_spa_Latn: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      belebele_swe_Latn: flexitok/reindexed-flexitok-bpe_swe_Latn_8000
      belebele_tur_Latn: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      belebele_vie_Latn: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      belebele_eng_Latn: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      xnli_ar: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      xnli_zh: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      xnli_de: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      xnli_el: flexitok/reindexed-flexitok-bpe_ell_Grek_8000
      xnli_en: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      xnli_es: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      xnli_fr: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      xnli_ru: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      xnli_tr: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      xnli_vi: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      regex:toksuite_turkish*: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      regex:toksuite_english*: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      regex:toksuite_farsi*: flexitok/reindexed-flexitok-bpe_fas_Arab_8000
      regex:toksuite_italian*: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      regex:toksuite_chinese*: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      regex:toksuite_math*: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      regex:toksuite_stem*: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      flores200_arb_Arab: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      flores200_ces_Latn: flexitok/reindexed-flexitok-bpe_ces_Latn_8000
      flores200_cmn_Hani: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      flores200_dan_Latn: flexitok/reindexed-flexitok-bpe_dan_Latn_8000
      flores200_deu_Latn: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      flores200_ell_Grek: flexitok/reindexed-flexitok-bpe_ell_Grek_8000
      flores200_fas_Arab: flexitok/reindexed-flexitok-bpe_fas_Arab_8000
      flores200_fra_Latn: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      flores200_fw_edu: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      flores200_eng_Latn: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      flores200_hun_Latn: flexitok/reindexed-flexitok-bpe_hun_Latn_8000
      flores200_ind_Latn: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      flores200_ita_Latn: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      flores200_jpn_Jpan: flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2
      flores200_nld_Latn: flexitok/reindexed-flexitok-bpe_nld_Latn_8000
      flores200_pol_Latn: flexitok/reindexed-flexitok-bpe_pol_Latn_8000
      flores200_por_Latn: flexitok/reindexed-flexitok-bpe_por_Latn_8000
      flores200_rus_Cyrl: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      flores200_spa_Latn: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      flores200_swe_Latn: flexitok/reindexed-flexitok-bpe_swe_Latn_8000
      flores200_tur_Latn: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      flores200_vie_Latn: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      xcopa_id: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      xcopa_it: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      xcopa_tr: flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2
      xcopa_vi: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      xcopa_zh: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      xstorycloze_ar: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      xstorycloze_en: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      xstorycloze_es: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      xstorycloze_id: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      xstorycloze_ru: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      xstorycloze_zh: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      paws_de: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      paws_en: flexitok/reindexed-flexitok-bpe_fw_edu_8000
      paws_es: flexitok/reindexed-flexitok-bpe_spa_Latn_8000
      paws_fr: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      paws_ja: flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2
      paws_zh: flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000
      hellaswag_ar: flexitok/reindexed-flexitok-bpe_arb_Arab_8000
      hellaswag_da: flexitok/reindexed-flexitok-bpe_dan_Latn_8000
      hellaswag_de: flexitok/reindexed-flexitok-bpe_deu_Latn_8000
      hellaswag_fr: flexitok/reindexed-flexitok-bpe_fra_Latn_8000
      hellaswag_hu: flexitok/reindexed-flexitok-bpe_hun_Latn_8000
      hellaswag_id: flexitok/reindexed-flexitok-bpe_ind_Latn_8000
      hellaswag_it: flexitok/reindexed-flexitok-bpe_ita_Latn_8000
      hellaswag_nl: flexitok/reindexed-flexitok-bpe_nld_Latn_8000
      hellaswag_pt: flexitok/reindexed-flexitok-bpe_por_Latn_8000
      hellaswag_ru: flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000
      hellaswag_sv: flexitok/reindexed-flexitok-bpe_swe_Latn_8000
      hellaswag_vi: flexitok/reindexed-flexitok-bpe_vie_Latn_8000
      hellaswag_es: flexitok/reindexed-flexitok-bpe_spa_Latn_8000