Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+train-clean-360.csv filter=lfs diff=lfs merge=lfs -text
+train.csv filter=lfs diff=lfs merge=lfs -text
+train-other-500.csv filter=lfs diff=lfs merge=lfs -text

env.log

@@ -0,0 +1,247 @@
+SpeechBrain system description
+==============================
+Python version:
+3.12.4 (main, Jun  7 2024, 23:47:47) [GCC 13.3.0]
+==============================
+Installed Python packages:
+accelerate==1.9.0
+aiohappyeyeballs==2.4.4+computecanada
+aiohttp==3.10.9+computecanada
+aiosignal==1.3.2+computecanada
+annotated_types==0.7.0+computecanada
+anyio==4.9.0+computecanada
+argon2_cffi==23.1.0+computecanada
+argon2_cffi_bindings==21.2.0+computecanada
+arrow==1.3.0+computecanada
+asttokens==3.0.0+computecanada
+async_lru==2.0.4+computecanada
+attrs==25.1.0+computecanada
+babel==2.17.0+computecanada
+beautifulsoup4==4.13.4+computecanada
+bioc==2.1+computecanada
+bleach==6.2.0+computecanada
+blis==1.0.1+computecanada
+boto3==1.40.1
+botocore==1.40.1
+catalogue==2.0.10+computecanada
+certifi==2024.12.14+computecanada
+cffi==1.17.1+computecanada
+cfgv==3.4.0+computecanada
+charset-normalizer==3.4.1
+click==8.2.1+computecanada
+cloudpathlib==0.21.1
+colorama==0.4.6+computecanada
+comm==0.2.2+computecanada
+confection==0.1.5+computecanada
+conllu==4.5.3+computecanada
+contourpy==1.3.1+computecanada
+cycler==0.12.1+computecanada
+cymem==2.0.8+computecanada
+datasets==4.0.0
+debugpy==1.8.12+computecanada
+decorator==5.2.1+computecanada
+defusedxml==0.7.1+computecanada
+Deprecated==1.2.18+computecanada
+dill==0.3.8+computecanada
+distlib==0.3.9+computecanada
+docopt==0.6.2+computecanada
+einops==0.8.0+computecanada
+executing==2.2.0+computecanada
+fastjsonschema==2.21.1+computecanada
+filelock==3.16.1+computecanada
+flair==0.15.1
+flash_attn==2.5.7+computecanada
+fonttools==4.59.0+computecanada
+fqdn==1.5.1+computecanada
+frozenlist==1.5.0+computecanada
+fsspec==2024.9.0+computecanada
+ftfy==6.3.1+computecanada
+Gammatone @ https://github.com/detly/gammatone/archive/master.zip#sha256=a33b739c787152035646cf5c1910ae3ff5da556b6d1f8c9d27191a2495ad6612
+gdown==5.2.0+computecanada
+gitdb==4.0.12+computecanada
+GitPython==3.1.37+computecanada
+h11==0.16.0+computecanada
+h5py==3.12.0+computecanada
+httpcore==1.0.9+computecanada
+httpx==0.28.1+computecanada
+huggingface-hub==0.27.0
+HyperPyYAML==1.2.2+computecanada
+identify==2.6.6
+idna==3.10+computecanada
+inflect==7.5.0+computecanada
+iniconfig==2.1.0+computecanada
+intervaltree==3.1.0+computecanada
+ipykernel==6.29.5+computecanada
+ipython==9.3.0+computecanada
+ipython_genutils==0.2.0+computecanada
+ipython_pygments_lexers==1.1.1+computecanada
+ipywidgets==8.1.5+computecanada
+isoduration==20.11.0+computecanada
+jedi==0.19.2+computecanada
+Jinja2==3.1.5
+jmespath==1.0.1+computecanada
+joblib==1.4.2+computecanada
+json5==0.10.0+computecanada
+jsonlines==4.0.0+computecanada
+jsonpointer==3.0.0+computecanada
+jsonschema==4.24.0+computecanada
+jsonschema_specifications==2025.4.1+computecanada
+jupyter==1.0.0+computecanada
+jupyter-console==6.4.0+computecanada
+jupyter_client==8.6.3+computecanada
+jupyter_core==5.8.1+computecanada
+jupyter_events==0.12.0+computecanada
+jupyter_lsp==2.2.5+computecanada
+jupyter_server==2.15.0+computecanada
+jupyter_server_terminals==0.5.3+computecanada
+jupyterlab==4.3.5+computecanada
+jupyterlab_pygments==0.3.0+computecanada
+jupyterlab_server==2.27.3+computecanada
+jupyterlab_widgets==3.0.13+computecanada
+kaldilm==1.15.1
+kenlm @ https://github.com/kpu/kenlm/archive/master.zip#sha256=f935c2ac0be6dd1bf96a5852973dad6103c48e866cf33532459fb1336bea4bfe
+kiwisolver==1.4.8+computecanada
+langcodes==3.5.0+computecanada
+langdetect==1.0.9+computecanada
+language_data==1.3.0+computecanada
+lilcom==1.8.1
+llvmlite==0.44.0+computecanada
+lxml==5.3.1+computecanada
+marisa_trie==1.2.1+computecanada
+markdown_it_py==3.0.0+computecanada
+MarkupSafe==2.1.5+computecanada
+matplotlib==3.10.0+computecanada
+matplotlib_inline==0.1.7+computecanada
+mdurl==0.1.2+computecanada
+mistune==3.1.1+computecanada
+mock==5.2.0
+more_itertools==10.7.0+computecanada
+mpld3==0.5.11
+mpmath==1.3.0+computecanada
+multidict==6.1.0+computecanada
+multiprocess==0.70.16+computecanada
+murmurhash==1.0.10+computecanada
+nbclient==0.10.2+computecanada
+nbconvert==7.16.6+computecanada
+nbformat==5.10.4+computecanada
+nest_asyncio==1.6.0+computecanada
+networkx==3.4.2+computecanada
+ninja==1.11.1+computecanada
+nodeenv==1.9.1
+nose==1.3.7+computecanada
+notebook==7.3.2+computecanada
+notebook_shim==0.2.4+computecanada
+numba==0.61.0+computecanada
+numpy==2.1.1+computecanada
+overrides==7.7.0+computecanada
+packaging==24.2+computecanada
+pandas==2.2.3+computecanada
+pandocfilters==1.5.1+computecanada
+parso==0.8.4+computecanada
+pexpect==4.9.0+computecanada
+pillow==11.1.0+computecanada
+platformdirs==4.3.6+computecanada
+pluggy==1.6.0+computecanada
+portalocker==3.2.0+computecanada
+pptree==3.1+computecanada
+pre_commit==4.2.0+computecanada
+preshed==3.0.9+computecanada
+prometheus_client==0.22.1+computecanada
+prompt_toolkit==3.0.51+computecanada
+propcache==0.2.1+computecanada
+protobuf==6.31.1+computecanada
+psutil==6.1.1+computecanada
+ptyprocess==0.7.0+computecanada
+pure_eval==0.2.3+computecanada
+pyarrow @ file:///tmp/ebuser/avx2/Arrow/19.0.1/GCCcore-12.3-gentoo/arrow/python
+pycparser==2.22+computecanada
+pydantic==2.11.7+computecanada
+pydantic_core==2.33.2+computecanada
+pygments==2.19.2+computecanada
+pyparsing==3.2.3+computecanada
+PySocks==1.7.1+computecanada
+pytest==8.4.1+computecanada
+python_dateutil==2.9.0.post0+computecanada
+python_json_logger==3.3.0+computecanada
+pytorch_revgrad==0.2.0+computecanada
+pytz==2025.1+computecanada
+PyYAML==6.0.2+computecanada
+pyzmq==26.2.1+computecanada
+qtconsole==5.3.2+computecanada
+QtPy==2.2.0+computecanada
+referencing==0.36.2+computecanada
+regex==2024.9.11+computecanada
+requests==2.32.3+computecanada
+rfc3339_validator==0.1.4+computecanada
+rfc3986_validator==0.1.1+computecanada
+rich==14.1.0+computecanada
+rpds_py==0.21.0+computecanada
+ruamel.yaml==0.18.8
+ruamel.yaml.clib==0.2.8+computecanada
+s3transfer==0.13.1
+sacrebleu==2.5.1+computecanada
+sacremoses==0.1.1
+safetensors==0.4.5+computecanada
+scikit_learn==1.6.1+computecanada
+scipy==1.14.1+computecanada
+segtok==1.5.11+computecanada
+Send2Trash==1.8.3+computecanada
+sentencepiece==0.2.0+computecanada
+setuptools==75.6.0+computecanada
+shellingham==1.5.4+computecanada
+six==1.17.0+computecanada
+smart_open==7.3.0.post1
+smmap==5.0.2+computecanada
+sniffio==1.3.1+computecanada
+sortedcontainers==2.4.0+computecanada
+soundfile==0.12.1+computecanada
+soupsieve==2.7+computecanada
+spacy==3.8.2+computecanada
+spacy_legacy==3.0.12+computecanada
+spacy_loggers==1.0.5+computecanada
+-e git+https://github.com/speechbrain/speechbrain.git@a024f32dc92b6cecf7342d5e7f0483e78b5a381a#egg=speechbrain
+sqlitedict==2.0.0+computecanada
+SRMRpy @ git+https://github.com/jfsantos/SRMRpy@fee009779cef96bed34db3a7e31d10f3ad1ea133
+srsly==2.4.8+computecanada
+stack_data==0.6.3+computecanada
+sympy==1.13.1+computecanada
+tabulate==0.9.0+computecanada
+terminado==0.18.1+computecanada
+thinc==8.3.2+computecanada
+threadpoolctl==3.6.0+computecanada
+tinycss2==1.4.0+computecanada
+tokenizers==0.21.0+computecanada
+torch==2.4.1+computecanada
+torchaudio==2.4.1+computecanada
+tornado==6.4.2+computecanada
+tqdm==4.67.1+computecanada
+traitlets==5.14.3+computecanada
+transformer-smaller-training-vocab==0.4.2
+transformers==4.47.1
+triton==3.1.0+computecanada
+typeguard==4.4.4+computecanada
+typer==0.16.0+computecanada
+types_python_dateutil==2.9.0.20241206+computecanada
+typing_extensions==4.14.1+computecanada
+typing_inspection==0.4.1+computecanada
+tzdata==2025.1+computecanada
+uri_template==1.3.0+computecanada
+urllib3==2.3.0
+virtualenv==20.29.1+computecanada
+wasabi==1.1.3+computecanada
+wcwidth==0.2.13+computecanada
+weasel==0.4.1+computecanada
+webcolors==24.11.1+computecanada
+webencodings==0.5.1+computecanada
+websocket_client==1.8.0+computecanada
+widgetsnbextension==4.0.13+computecanada
+Wikipedia_API==0.6.0+computecanada
+wrapt==1.17.2+computecanada
+xxhash==3.5.0+computecanada
+yarl==1.18.3+computecanada
+==============================
+Git revision:
+a024f32dc
+==============================
+CUDA version:
+12.2

hyperparams.yaml

@@ -0,0 +1,249 @@
+# Generated 2025-08-10 from:
+# /home/adelmou/proj/speechbrain/speechllm_librispeech/speechbrain/recipes/LibriSpeech/ASR/transformer/hparams/llama.yaml
+# yamllint disable
+# ############################################################################
+# Authors:  Adel Moumen
+# ############################################################################
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 3407
+__set_seed: !apply:speechbrain.utils.seed_everything [3407]
+experiment_name: llama_3.2_1b_ASR
+output_folder: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora/
+output_wer_folder: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//wer_results
+save_folder: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save
+train_log: 
+  /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//train_log.txt
+
+
+# Data files
+data_folder: /localscratch/adelmou.47747915.0/LibriSpeech/
+                          # e.g., /path/to/LibriSpeech
+# If RIRS_NOISES dir exists in /localscratch/xxx_corpus/RIRS_NOISES
+# then data_folder_rirs should be /localscratch/xxx_corpus
+# otherwise the dataset will automatically be downloaded
+# data_folder_rirs: !ref <data_folder>
+train_splits: [train-clean-100, train-clean-360, train-other-500]       # , "train-clean-360", "train-other-500"
+dev_splits: [dev-clean]
+test_splits: [test-clean, test-other]
+skip_prep: false
+train_csv: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//train.csv
+valid_csv: 
+  /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//dev-clean.csv
+test_csv:
+- /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//test-clean.csv
+- /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//test-other.csv
+
+ckpt_interval_minutes: 15 # save checkpoint every N min
+
+####################### Training Parameters ####################################
+
+# URL for the HuggingFace model we want to load (BASE here)
+ssl_hub: /localscratch/adelmou.47747915.0/wavlm-large/
+ssl_folder: 
+  /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save/ssl_checkpoint
+ssl_frozen: true
+
+# LLM options
+llm_path: /localscratch/adelmou.47747915.0/SmolLM-1.7B/
+llm_emb_size: 2048
+
+number_of_epochs: 20
+batch_size: 32 # Only used if dynamic batching is off.
+# 400s * 10 => 1h / opt step
+grad_accumulation_factor: 5
+loss_reduction: batchmean
+sorting: random
+num_workers: 4
+precision: bf16 # bf16, fp16 or fp32
+eval_precision: bf16
+max_grad_norm: 1.0
+
+# stages related parameters
+lr_adam: 0.0005
+lr_wav2vec: 0.00002
+
+weight_decay: 0.0
+warmup_steps: 5000
+augment_warmup: 7500
+
+# BPE parameters
+token_type: unigram  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Feature parameters
+sample_rate: 16000
+downsampling_factor: 5 # Used to downsample frames before llm projection.
+
+# This setup works well for A100 80GB GPU, adapts it to your needs.
+# Or turn it off (but training speed will decrease)
+dynamic_batching: true
+max_batch_length_train: 300
+max_batch_length_val: 100 # we reduce it as the beam is much wider (VRAM)
+num_bucket: 200
+shuffle: true # if true re-creates batches at each epoch shuffling examples.
+batch_ordering: random
+max_batch_ex: 256
+
+dynamic_batch_sampler_train:
+  max_batch_length: 300
+  num_buckets: 200
+  shuffle: true
+  batch_ordering: random
+  max_batch_ex: 256
+
+dynamic_batch_sampler_valid:
+  max_batch_length: 100
+  num_buckets: 200
+  shuffle: true
+  batch_ordering: random
+  max_batch_ex: 256
+
+# Dataloader options
+train_dataloader_opts:
+  batch_size: 32
+  shuffle: true
+  num_workers: 4
+  collate_fn: !name:speechbrain.dataio.batch.PaddedBatch
+    padding_kwargs:
+      value: 49152
+    per_key_padding_kwargs:
+      sig:
+        value: 0
+      tokens_eos:
+        value: -100
+
+valid_dataloader_opts:
+  batch_size: 8
+  collate_fn: !name:speechbrain.dataio.batch.PaddedBatch
+    padding_kwargs:
+      value: 49152
+    per_key_padding_kwargs:
+      sig:
+        value: 0
+      tokens_eos:
+        value: -100
+
+test_dataloader_opts:
+  batch_size: 8
+  collate_fn: !name:speechbrain.dataio.batch.PaddedBatch
+    padding_kwargs:
+      value: 49152
+    per_key_padding_kwargs:
+      sig:
+        value: 0
+      tokens_eos:
+        value: -100
+
+
+####################### Model Parameters ###########################
+activation: &id001 !name:torch.nn.GELU
+                                # todo: try swish instead
+asr_output_neurons: 1024
+lora_rank: 16
+
+# Frames - LLM projector params
+dnn_layers: 4
+dnn_neurons: 2048
+downsampling_output_dim: 5120
+
+# Outputs
+blank_index: 0
+pad_token: 49152  #Llama 3 pad index after adding: BEURK.
+
+# Decoding parameters
+valid_search_interval: 4
+valid_beam_size: 1 # We do greedy here so it's faster to decode ...
+test_beam_size: 5
+
+############################## models ################################
+
+normalize: &id007 !new:speechbrain.processing.features.InputNormalization
+
+# We define two optimizers as we have two stages (training + finetuning)
+  norm_type: sentence
+
+#wav2vec model
+ssl: &id003 !new:speechbrain.integrations.huggingface.wav2vec2.Wav2Vec2
+  source: /localscratch/adelmou.47747915.0/wavlm-large/
+  output_norm: true
+  freeze: true
+  save_path: 
+    /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save/ssl_checkpoint
+  device_map: cuda
+    # attn_implementation: sdpa
+    # normalize_wav: False
+
+proj: &id006 !new:speechbrain.lobes.models.VanillaNN.VanillaNN
+  input_shape: [null, null, 5120]   # 5 x 1024
+  activation: *id001
+  dnn_blocks: 4
+  dnn_neurons: 2048
+
+backbone_llm: &id002 !new:speechbrain.integrations.huggingface.llama.LLaMA
+  source: /localscratch/adelmou.47747915.0/SmolLM-1.7B/
+  save_path: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save
+  freeze: true
+  attn_implementation: flash_attention_2
+  device: cuda
+  torch_dtype: !name:torch.bfloat16
+    # add_tokens:
+    #     audio_bos_token: "<|start_of_audio|>"
+    #     audio_eos_token: "<|end_of_audio|>"
+
+# Simply uncomment if you want to use LoRA adaptation.
+llm: &id005 !new:speechbrain.nnet.adapters.AdaptedModel
+
+  model_to_adapt: *id002
+  adapter_class: !name:speechbrain.nnet.adapters.LoRA
+  all_linear: true
+  adapter_kwargs:
+    rank: 16
+
+feat_downsampler: &id004 !new:speechbrain.lobes.downsampling.ConcatDownsampler
+  downsampling_factor: 5
+
+modules:
+  ssl: *id003
+  feat_downsampler: *id004
+  llm: *id005
+  proj: *id006
+  normalize: *id007
+Adam: !name:torch.optim.AdamW
+  lr: 0.0005
+  weight_decay: 0.0
+
+Adam_wav2vec2: !name:torch.optim.AdamW
+  lr: 0.00002
+  weight_decay: 0.0
+
+
+noam_annealing: &id008 !new:speechbrain.nnet.schedulers.NoamScheduler
+  lr_initial: 0.0005
+  n_warmup_steps: 5000
+
+lr_annealing_wav2vec: &id009 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.00002
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 1
+
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//save
+  recoverables:
+    proj: *id006
+    noam_scheduler: *id008
+    lr_annealing_wav2vec: *id009
+    counter: &id010 !new:speechbrain.utils.epoch_loop.EpochCounter
+      limit: 20
+
+    ssl: *id003
+    llm: *id005
+epoch_counter: *id010
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: 
+    /scratch/adelmou/speechllm_results_ls/wavlm_large+smol1.7B_mlp_lora//train_log.txt
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats

opt_librispeech_prepare.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:591d7c55caa4f47bcf3caf1019535cacd582a268dea0a4aa3cad2be0d4cd0539
+size 37

save/CKPT+2025-08-09+18-52-20+00/CKPT.yaml

@@ -0,0 +1,5 @@
+# yamllint disable
+WER: 57.485018933127456
+end-of-epoch: true
+epoch: 1
+unixtime: 1754779940.0693338

save/CKPT+2025-08-09+18-52-20+00/brain.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e9e37095ed0a9ede3cdfa3a9f2ac8b1997ec37e5d9d914aace085542e6a77b8
+size 49

save/CKPT+2025-08-09+18-52-20+00/counter.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b
+size 1

save/CKPT+2025-08-09+18-52-20+00/dataloader-TRAIN.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3354f1239660e58f56dd5d3e011818ad1f46aa866e9ad99ef27eb8c079ad7a58
+size 5

save/CKPT+2025-08-09+18-52-20+00/llm.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:765d1ae759b83ce5e283bca3cbb780669446b26b04431f4571251e6accf389b4
+size 75729974

save/CKPT+2025-08-09+18-52-20+00/lr_annealing_wav2vec.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e247c7a78bc010d7afbf49347c31ff3ca6fa619179ddde19766f44748c3a66d
+size 980

save/CKPT+2025-08-09+18-52-20+00/noam_scheduler.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:667ce05145a77d4cc2969db16e2df76882e3eda7d42a504f194fb40240bdab18
+size 892

save/CKPT+2025-08-09+18-52-20+00/optimizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eabfc58e98451f721a34158d7babf7ca2883ca7e1cdd8cdaed5b050e89d8c79f
+size 336165856

save/CKPT+2025-08-09+18-52-20+00/proj.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:319a56c6772ef2fc1ecee61e0cd33c75dbdd47b69a245602036f5f0adf10e4dc
+size 92310364

save/CKPT+2025-08-09+18-52-20+00/ssl.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d62d5122a6f6c73993a2a9114b63faf4bf1e2ad1a65cb19208793d1ab53320da
+size 1261974514

save/CKPT+2025-08-10+03-13-34+00/CKPT.yaml

@@ -0,0 +1,5 @@
+# yamllint disable
+WER: 55.8784603507224
+end-of-epoch: true
+epoch: 4
+unixtime: 1754810014.1578317

save/CKPT+2025-08-10+03-13-34+00/brain.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829cf20f85e53a35380ccee38e997d53cbd55fe39c4eb58af9600c2f34a56ddc
+size 49

save/CKPT+2025-08-10+03-13-34+00/counter.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a
+size 1

save/CKPT+2025-08-10+03-13-34+00/dataloader-TRAIN.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3354f1239660e58f56dd5d3e011818ad1f46aa866e9ad99ef27eb8c079ad7a58
+size 5

save/CKPT+2025-08-10+03-13-34+00/llm.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed74e93d1a69782b26e91c7cfe39a5908c4a7184bb8f310d64a6bd9b569ab0ec
+size 75729974

save/CKPT+2025-08-10+03-13-34+00/lr_annealing_wav2vec.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e247c7a78bc010d7afbf49347c31ff3ca6fa619179ddde19766f44748c3a66d
+size 980

save/CKPT+2025-08-10+03-13-34+00/noam_scheduler.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:667ce05145a77d4cc2969db16e2df76882e3eda7d42a504f194fb40240bdab18
+size 892

save/CKPT+2025-08-10+03-13-34+00/optimizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4c1274c37f916b131e764a03899993a0a9ebcc36c9f34aba147cff09d5c6af2
+size 336165856

save/CKPT+2025-08-10+03-13-34+00/proj.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3086caa4a2071491b576375726d88a729bf298194bf8b4c1711ac59c28fa961
+size 92310364

save/CKPT+2025-08-10+03-13-34+00/ssl.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d62d5122a6f6c73993a2a9114b63faf4bf1e2ad1a65cb19208793d1ab53320da
+size 1261974514

save/CKPT+2025-08-10+04-00-44+00/CKPT.yaml

@@ -0,0 +1,4 @@
+# yamllint disable
+brain_intra_epoch_ckpt: true
+end-of-epoch: false
+unixtime: 1754812844.3274941

save/CKPT+2025-08-10+04-00-44+00/brain.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41a2d0b966a5ab68c3a2613fb00d4b23eb86eb83eeb48e736a42cc4d82277923
+size 68

save/CKPT+2025-08-10+04-00-44+00/counter.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d
+size 1

save/CKPT+2025-08-10+04-00-44+00/dataloader-TRAIN.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b9b21ac4770d6a10a27b9bfbab70888491187668f3eaa475abec6ea04c82eec
+size 4

save/CKPT+2025-08-10+04-00-44+00/llm.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4cbf1e9fca51cbed249ee70fd930fb0997a306e533a198f3f42a11beaee017
+size 75729974

save/CKPT+2025-08-10+04-00-44+00/lr_annealing_wav2vec.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e247c7a78bc010d7afbf49347c31ff3ca6fa619179ddde19766f44748c3a66d
+size 980

save/CKPT+2025-08-10+04-00-44+00/noam_scheduler.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:667ce05145a77d4cc2969db16e2df76882e3eda7d42a504f194fb40240bdab18
+size 892

save/CKPT+2025-08-10+04-00-44+00/optimizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58c4f215e290c3d7868d2dff364981daf58fcaed31d37ba4b07db2e79c860c8d
+size 336165856

save/CKPT+2025-08-10+04-00-44+00/proj.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a4cd73d90b48deb2c1975f12abffb8c1536c34dc59ce5d3bebe5207e90f3775
+size 92310364

save/CKPT+2025-08-10+04-00-44+00/ssl.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d62d5122a6f6c73993a2a9114b63faf4bf1e2ad1a65cb19208793d1ab53320da
+size 1261974514

train-clean-360.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea1dcef6ee76e846cf50945ea7dde94813524d7615a51feef8fc64eef2f430b
+size 29076679

train-other-500.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a21b1729500f97836d004d250b78ee885acbcaa2ce4176ecf3866162303632d9
+size 39565769

train.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c67797a7b5c794b27d54d9db4073af36f3c8eb078465041713e340006a44850c
+size 76628881

train_log.txt

@@ -0,0 +1,5 @@
+epoch: 1, lr: 5.00e-04, steps: 1498, optimizer: AdamW - train loss: 1.73 - valid loss: 1.49e-01, valid CER: 56.29, valid WER: 57.49
+epoch: 2, lr: 5.00e-04, steps: 2996, optimizer: AdamW - train loss: 1.43e-01 - valid loss: 7.63e-02, valid CER: 55.70, valid WER: 56.26
+epoch: 3, lr: 5.00e-04, steps: 4494, optimizer: AdamW - train loss: 1.06e-01 - valid loss: 6.71e-02, valid CER: 55.54, valid WER: 55.92
+epoch: 4, lr: 5.00e-04, steps: 5992, optimizer: AdamW - train loss: 8.78e-02 - valid loss: 6.29e-02, valid CER: 55.54, valid WER: 55.88
+Epoch loaded: 4 - test loss: 6.42e-02, test CER: 1.97, test WER: 3.67

train_speechllm.py

@@ -0,0 +1,423 @@
+#!/usr/bin/env python3
+"""
+# TODO:
+1) add max length (i.e. context length of the fine tune model)
+2) torch.compile the LLM with the fixed length
+
+Authors
+ * Adel Moumen 2025
+"""
+
+import os
+import sys
+from pathlib import Path
+
+import torch
+from hyperpyyaml import load_hyperpyyaml
+
+import speechbrain as sb
+from speechbrain.utils.distributed import if_main_process, run_on_main
+from speechbrain.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_multimodal_attention_mask(wav, wav_lens, txt, txt_lens, device):
+    batch_size = wav.size(0)
+    wav_len = wav.size(1)
+    txt_len = txt.size(1)
+    # Max total length for padding
+    max_total_len = wav_len + txt_len
+    attention_mask = torch.zeros(batch_size, max_total_len, dtype=torch.bool, device=device)
+    for i in range(batch_size):
+        actual_wav_len = int(wav_lens[i].item() * wav_len)
+        actual_txt_len = int(txt_lens[i].item() * txt_len)
+        # Fill mask: audio part
+        attention_mask[i, :actual_wav_len] = True
+        # Fill mask: text part (after audio)
+        attention_mask[i, wav_len:wav_len + actual_txt_len] = True
+    return attention_mask
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, tokens_bos_lens = batch.tokens_bos
+        prompt_len = batch.prompt_len
+        wavs = self.hparams.normalize(wavs, wav_lens)
+        audio_feats = self.modules.ssl(wavs, wav_lens)
+        audio_down_feats = self.modules.feat_downsampler(audio_feats)
+        projected_audio_feats = self.modules.proj(audio_down_feats)
+        txt_embds = self.txt_embedding(tokens_bos)
+        multimodal_embds = torch.cat([projected_audio_feats, txt_embds], dim=1)
+        # attention_mask should be all the true audio features + all the true text features
+        attention_mask = get_multimodal_attention_mask(
+            projected_audio_feats, wav_lens, txt_embds, tokens_bos_lens, self.device
+        )
+        logits = self.modules.llm(
+            inputs_embeds=multimodal_embds, 
+            attention_mask=attention_mask
+        ).logits
+
+        if hasattr(self.modules.llm, "module"):
+            gen_func = self.modules.llm.module.model.generate
+        else:
+            gen_func = self.modules.llm.model.generate
+
+        hyps = None        
+        if stage != sb.Stage.TRAIN:
+            audio_and_prompt_len = projected_audio_feats.shape[1] + prompt_len[0]
+            hyps = gen_func(
+                inputs_embeds=multimodal_embds[
+                    :, :audio_and_prompt_len
+                ],  # give model audio features and prompt for inference
+                attention_mask=attention_mask[:, :audio_and_prompt_len],
+                generation_config=self.val_decoding_config,
+            )
+        return logits, hyps, projected_audio_feats.shape[1]
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC+NLL) given predictions and targets."""
+        logits, hyps, _ = predictions
+        tokens_eos, _ = batch.tokens_eos
+        ids = batch.id
+
+        num_audio_feats = logits.shape[1] - tokens_eos.shape[1]
+        # We prepend `-100` to the tokens_eos to ignore them in the loss.
+        # This corresponds to the audio features.
+        target_tokens = torch.cat([
+            torch.full((tokens_eos.shape[0], num_audio_feats), -100, device=self.device),
+            tokens_eos,
+        ], dim=1).long()
+        # compute the cross entropy loss
+        loss = torch.nn.functional.cross_entropy(
+            logits.view(-1, logits.shape[-1]),
+            target_tokens.view(-1),
+            ignore_index=-100
+        )
+        if stage != sb.Stage.TRAIN:
+            # replace -100 with pad token
+            target_tokens = target_tokens.masked_fill(target_tokens == -100, self.tokenizer.pad_token_id)
+            preds = self.tokenizer.batch_decode(hyps, skip_special_tokens=True)
+            preds_words = [pred.split(" ") for pred in preds]
+            targets = self.tokenizer.batch_decode(target_tokens, skip_special_tokens=True)
+            targets_words = [target.split(" ") for target in targets]
+            # print(preds_words)
+            # print(targets_words)
+            # import time 
+            # time.sleep(5)
+            # print('--------------------------------')
+            self.cer_metric.append(ids, preds_words, targets_words)
+            self.wer_metric.append(ids, preds_words, targets_words)
+        return loss
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        # check if txt_embedding is already set
+        import transformers
+        self.val_decoding_config = transformers.GenerationConfig(
+            pad_token_id=self.tokenizer.pad_token_id,
+            eos_token_id=self.tokenizer.eos_token_id,
+            max_new_tokens=400,
+            do_sample=False,       # disables sampling
+            num_beams=1,           # no beam search
+            temperature=1.0,       # irrelevant when do_sample=False, but keep default
+            top_k=0,               # not used when do_sample=False
+            top_p=1.0,              # not used when do_sample=False
+            repetition_penalty=1.0 # no repetition penalty
+        )
+
+        if not hasattr(self, "txt_embedding"):
+            # we save the txt embedding for easy access
+            self.txt_embedding = (
+                self.modules.llm.model.get_input_embeddings()
+                if not hasattr(self.modules.llm, "module")
+                else self.modules.llm.module.model.get_input_embeddings()
+            )
+
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of a epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # log stats and save checkpoint at end-of-epoch
+        if stage == sb.Stage.VALID:
+            lr = self.hparams.noam_annealing.current_lr
+            steps = self.optimizer_step
+            optimizer = self.optimizer.__class__.__name__
+
+            epoch_stats = {
+                "epoch": epoch,
+                "lr": lr,
+                "steps": steps,
+                "optimizer": optimizer,
+            }
+            self.hparams.train_logger.log_stats(
+                stats_meta=epoch_stats,
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"], "epoch": epoch},
+                max_keys=["WER"],
+                # num_to_keep=self.hparams.avg_checkpoints,
+            )
+
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            if if_main_process():
+                with open(
+                    self.hparams.test_wer_file, "w", encoding="utf-8"
+                ) as w:
+                    self.wer_metric.write_stats(w)
+
+    def on_fit_batch_end(self, batch, outputs, loss, should_step):
+        """At the end of the optimizer step, apply noam annealing."""
+        # if should_step:
+            # self.hparams.noam_annealing(self.optimizer)
+
+
+def dataio_prepare(hparams, tokenizer):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions.
+    """
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"],
+        replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(sort_key="duration")
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["train_dataloader_opts"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration", reverse=True
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["train_dataloader_opts"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"],
+        replacements={"data_root": data_folder},
+    )
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    # test is separate
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        sig = sb.dataio.dataio.read_audio(wav)
+        return sig
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    bos_index = tokenizer.bos_token_id
+    eos_index = tokenizer.eos_token_id
+    pad_index = tokenizer.pad_token_id
+    prompt = "Transcribe speech to text."
+    print(bos_index, eos_index, pad_index, prompt)
+
+    prompt_ids = tokenizer(
+        prompt, return_tensors="pt", add_special_tokens=False
+    ).input_ids.view(-1).tolist()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "tokens_list", "tokens_bos", "tokens_eos", "tokens", "prompt_len"
+    )
+    def text_pipeline(wrd):
+        # wrd = wrd[0] + wrd[1:].lower()
+        yield wrd
+        tokens_list = tokenizer(wrd, add_special_tokens=False).input_ids
+        yield tokens_list
+        tokens_bos = torch.LongTensor(prompt_ids  + [bos_index] + tokens_list )
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [eos_index])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+        prompt_len = len(prompt_ids  + [bos_index])
+        yield prompt_len
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets,
+        ["id", "sig", "wrd", "tokens_bos", "tokens_eos", "tokens", "prompt_len"],
+    )
+
+    # 5. If Dynamic Batching is used, we instantiate the needed samplers.
+    train_batch_sampler = None
+    valid_batch_sampler = None
+    if hparams["dynamic_batching"]:
+        from speechbrain.dataio.sampler import DynamicBatchSampler  # noqa
+
+        dynamic_hparams_train = hparams["dynamic_batch_sampler_train"]
+        dynamic_hparams_valid = hparams["dynamic_batch_sampler_valid"]
+
+        train_batch_sampler = DynamicBatchSampler(
+            train_data,
+            length_func=lambda x: x["duration"],
+            **dynamic_hparams_train,
+        )
+        valid_batch_sampler = DynamicBatchSampler(
+            valid_data,
+            length_func=lambda x: x["duration"],
+            **dynamic_hparams_valid,
+        )
+
+    return (
+        train_data,
+        valid_data,
+        test_datasets,
+        tokenizer,
+        train_batch_sampler,
+        valid_batch_sampler,
+    )
+
+
+if __name__ == "__main__":
+    # CLI:
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file, encoding="utf-8") as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # 1.  # Dataset prep (parsing Librispeech)
+    from librispeech_prepare import prepare_librispeech  # noqa
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # multi-gpu (ddp) save data preparation
+    run_on_main(
+        prepare_librispeech,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "tr_splits": hparams["train_splits"],
+            "dev_splits": hparams["dev_splits"],
+            "te_splits": hparams["test_splits"],
+            "save_folder": hparams["output_folder"],
+            "merge_lst": hparams["train_splits"],
+            "merge_name": "train.csv",
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # here we create the datasets objects as well as tokenization and encoding
+    tokenizer = hparams["llm"].tokenizer
+    
+    (
+        train_data,
+        valid_data,
+        test_datasets,
+        tokenizer,
+        train_bsampler,
+        valid_bsampler,
+    ) = dataio_prepare(hparams, tokenizer)
+
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        opt_class=hparams["Adam"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+    # asr_brain.modules.llm = torch.compile(asr_brain.modules.llm)
+    asr_brain.tokenizer = tokenizer
+    # adding objects to trainer:
+    train_dataloader_opts = hparams["train_dataloader_opts"]
+    valid_dataloader_opts = hparams["valid_dataloader_opts"]
+
+    if train_bsampler is not None:
+        collate_fn = None
+        if "collate_fn" in train_dataloader_opts:
+            collate_fn = train_dataloader_opts["collate_fn"]
+
+        train_dataloader_opts = {
+            "batch_sampler": train_bsampler,
+            "num_workers": hparams["num_workers"],
+        }
+
+        if collate_fn is not None:
+            train_dataloader_opts["collate_fn"] = collate_fn
+
+    if valid_bsampler is not None:
+        collate_fn = None
+        if "collate_fn" in valid_dataloader_opts:
+            collate_fn = valid_dataloader_opts["collate_fn"]
+
+        valid_dataloader_opts = {"batch_sampler": valid_bsampler}
+
+        if collate_fn is not None:
+            valid_dataloader_opts["collate_fn"] = collate_fn
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=train_dataloader_opts,
+        valid_loader_kwargs=valid_dataloader_opts,
+    )
+
+    # Testing
+    os.makedirs(hparams["output_wer_folder"], exist_ok=True)
+
+    for k in test_datasets.keys():  # keys are test_clean, test_other etc
+        asr_brain.hparams.test_wer_file = os.path.join(
+            hparams["output_wer_folder"], f"wer_{k}.txt"
+        )
+        asr_brain.evaluate(
+            test_datasets[k],
+            min_key="WER",
+            test_loader_kwargs=hparams["test_dataloader_opts"],
+        )