hyperparams.yaml

# Generated 2025-08-10 from:
# /home/adelmou/proj/speechbrain/speechllm_librispeech/speechbrain/recipes/LibriSpeech/ASR/transformer/hparams/llama.yaml
# yamllint disable
# ############################################################################
# Authors:  Adel Moumen
# ############################################################################
# Seed needs to be set at top of yaml, before objects with parameters are made
seed: 3407
__set_seed: !apply:speechbrain.utils.seed_everything [3407]
experiment_name: llama_3.2_1b_ASR
output_folder: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora/
output_wer_folder: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//wer_results
save_folder: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save
train_log: 
  /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//train_log.txt


# Data files
data_folder: /localscratch/adelmou.47747915.0/LibriSpeech/
                          # e.g., /path/to/LibriSpeech
# If RIRS_NOISES dir exists in /localscratch/xxx_corpus/RIRS_NOISES
# then data_folder_rirs should be /localscratch/xxx_corpus
# otherwise the dataset will automatically be downloaded
# data_folder_rirs: !ref <data_folder>
train_splits: [train-clean-100, train-clean-360, train-other-500]       # , "train-clean-360", "train-other-500"
dev_splits: [dev-clean]
test_splits: [test-clean, test-other]
skip_prep: false
train_csv: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//train.csv
valid_csv: 
  /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//dev-clean.csv
test_csv:
- /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//test-clean.csv
- /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//test-other.csv

ckpt_interval_minutes: 15 # save checkpoint every N min

####################### Training Parameters ####################################

# URL for the HuggingFace model we want to load (BASE here)
ssl_hub: /localscratch/adelmou.47747915.0/wavlm-large/
ssl_folder: 
  /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save/ssl_checkpoint
ssl_frozen: true

# LLM options
llm_path: /localscratch/adelmou.47747915.0/SmolLM-1.7B/
llm_emb_size: 2048

number_of_epochs: 20
batch_size: 32 # Only used if dynamic batching is off.
# 400s * 10 => 1h / opt step
grad_accumulation_factor: 5
loss_reduction: batchmean
sorting: random
num_workers: 4
precision: bf16 # bf16, fp16 or fp32
eval_precision: bf16
max_grad_norm: 1.0

# stages related parameters
lr_adam: 0.0005
lr_wav2vec: 0.00002

weight_decay: 0.0
warmup_steps: 5000
augment_warmup: 7500

# BPE parameters
token_type: unigram  # ["unigram", "bpe", "char"]
character_coverage: 1.0

# Feature parameters
sample_rate: 16000
downsampling_factor: 5 # Used to downsample frames before llm projection.

# This setup works well for A100 80GB GPU, adapts it to your needs.
# Or turn it off (but training speed will decrease)
dynamic_batching: true
max_batch_length_train: 300
max_batch_length_val: 100 # we reduce it as the beam is much wider (VRAM)
num_bucket: 200
shuffle: true # if true re-creates batches at each epoch shuffling examples.
batch_ordering: random
max_batch_ex: 256

dynamic_batch_sampler_train:
  max_batch_length: 300
  num_buckets: 200
  shuffle: true
  batch_ordering: random
  max_batch_ex: 256

dynamic_batch_sampler_valid:
  max_batch_length: 100
  num_buckets: 200
  shuffle: true
  batch_ordering: random
  max_batch_ex: 256

# Dataloader options
train_dataloader_opts:
  batch_size: 32
  shuffle: true
  num_workers: 4
  collate_fn: !name:speechbrain.dataio.batch.PaddedBatch
    padding_kwargs:
      value: 49152
    per_key_padding_kwargs:
      sig:
        value: 0
      tokens_eos:
        value: -100

valid_dataloader_opts:
  batch_size: 8
  collate_fn: !name:speechbrain.dataio.batch.PaddedBatch
    padding_kwargs:
      value: 49152
    per_key_padding_kwargs:
      sig:
        value: 0
      tokens_eos:
        value: -100

test_dataloader_opts:
  batch_size: 8
  collate_fn: !name:speechbrain.dataio.batch.PaddedBatch
    padding_kwargs:
      value: 49152
    per_key_padding_kwargs:
      sig:
        value: 0
      tokens_eos:
        value: -100


####################### Model Parameters ###########################
activation: &id001 !name:torch.nn.GELU
                                # todo: try swish instead
asr_output_neurons: 1024
lora_rank: 16

# Frames - LLM projector params
dnn_layers: 4
dnn_neurons: 2048
downsampling_output_dim: 5120

# Outputs
blank_index: 0
pad_token: 49152  #Llama 3 pad index after adding: BEURK.

# Decoding parameters
valid_search_interval: 4
valid_beam_size: 1 # We do greedy here so it's faster to decode ...
test_beam_size: 5

############################## models ################################

normalize: &id007 !new:speechbrain.processing.features.InputNormalization

# We define two optimizers as we have two stages (training + finetuning)
  norm_type: sentence

#wav2vec model
ssl: &id003 !new:speechbrain.integrations.huggingface.wav2vec2.Wav2Vec2
  source: /localscratch/adelmou.47747915.0/wavlm-large/
  output_norm: true
  freeze: true
  save_path: 
    /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save/ssl_checkpoint
  device_map: cuda
    # attn_implementation: sdpa
    # normalize_wav: False

proj: &id006 !new:speechbrain.lobes.models.VanillaNN.VanillaNN
  input_shape: [null, null, 5120]   # 5 x 1024
  activation: *id001
  dnn_blocks: 4
  dnn_neurons: 2048

backbone_llm: &id002 !new:speechbrain.integrations.huggingface.llama.LLaMA
  source: /localscratch/adelmou.47747915.0/SmolLM-1.7B/
  save_path: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save
  freeze: true
  attn_implementation: flash_attention_2
  device: cuda
  torch_dtype: !name:torch.bfloat16
    # add_tokens:
    #     audio_bos_token: "<|start_of_audio|>"
    #     audio_eos_token: "<|end_of_audio|>"

# Simply uncomment if you want to use LoRA adaptation.
llm: &id005 !new:speechbrain.nnet.adapters.AdaptedModel

  model_to_adapt: *id002
  adapter_class: !name:speechbrain.nnet.adapters.LoRA
  all_linear: true
  adapter_kwargs:
    rank: 16

feat_downsampler: &id004 !new:speechbrain.lobes.downsampling.ConcatDownsampler
  downsampling_factor: 5

modules:
  ssl: *id003
  feat_downsampler: *id004
  llm: *id005
  proj: *id006
  normalize: *id007
Adam: !name:torch.optim.AdamW
  lr: 0.0005
  weight_decay: 0.0

Adam_wav2vec2: !name:torch.optim.AdamW
  lr: 0.00002
  weight_decay: 0.0


noam_annealing: &id008 !new:speechbrain.nnet.schedulers.NoamScheduler
  lr_initial: 0.0005
  n_warmup_steps: 5000

lr_annealing_wav2vec: &id009 !new:speechbrain.nnet.schedulers.NewBobScheduler
  initial_value: 0.00002
  improvement_threshold: 0.0025
  annealing_factor: 0.8
  patient: 1


checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
  checkpoints_dir: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save
  recoverables:
    proj: *id006
    noam_scheduler: *id008
    lr_annealing_wav2vec: *id009
    counter: &id010 !new:speechbrain.utils.epoch_loop.EpochCounter
      limit: 20

    ssl: *id003
    llm: *id005
epoch_counter: *id010
train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
  save_file: 
    /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//train_log.txt

cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
  split_tokens: true
error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats