Training in progress, epoch 1

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr5-Project/runs/lqwj9jus) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr5-Project/runs/ozcexaa1) 
 
 
 This model was trained with SFT.

adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
+    "k_proj",
+    "fc1",
+    "out_proj",
+    "o_proj",
     "up_proj",
+    "gate_proj",
     "v_proj",
     "q_proj",
-    "fc1",
-    "fc2",
-    "k_proj",
-    "out_proj",
     "down_proj",
-    "o_proj"
+    "fc2"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e9aadb597ce536c66a4aa231ada2b97f84829426c830024ead25e411768ccd5
+oid sha256:c271ed2f9968755796be47d4c4cf031d2381369b29dfd7f819615ec1b3c05133
 size 6127553104

runs/Jul23_16-31-18_meedgxh100a/events.out.tfevents.1753302680.meedgxh100a.1938592.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f98e127043eadf647992549cbb08091a95d5a485861b0bbb7c20f5d7f6a235f1
+size 9594

train_medgemma_ft_copy.py

@@ -215,7 +215,7 @@ if __name__ == '__main__':
             pos = [s for s in data if s[task_idx] != '0.0']
             num_sample = len(pos)
             if train:
-                return random.sample(neg, 8*num_sample), random.sample(pos, num_sample)
+                return random.sample(neg, 4*num_sample), random.sample(pos, num_sample)
             else:
                 return random.sample(neg, num_sample), pos
                 # return random.sample(neg, 15), random.sample(pos, 15)
@@ -290,7 +290,7 @@ if __name__ == '__main__':
         from peft import PeftModel
         print("🔁 Loading trained PEFT weights...")
         # model = PeftModel.from_pretrained(model, exp_name)
-        model = PeftModel.from_pretrained(model, exp_name+"/checkpoint-322")
+        model = PeftModel.from_pretrained(model, exp_name+"/checkpoint-140")
         # model = PeftModel.from_pretrained(model, "llava-1.5-7b-hf-dr-all/checkpoint-80")
         phase= "val"
     else:
@@ -304,7 +304,7 @@ if __name__ == '__main__':
     training_args = SFTConfig(
         output_dir=exp_name,
         num_train_epochs= 10,                       # Number of training epochs
-        per_device_train_batch_size=4,                           # Batch size per device during training
+        per_device_train_batch_size=2,                           # Batch size per device during training
         per_device_eval_batch_size=4,                            # Batch size per device during evaluation
         gradient_accumulation_steps=8,                           # Number of steps before performing a backward/update pass
         gradient_checkpointing=True,                             # Enable gradient checkpointing to reduce memory usage
@@ -317,8 +317,8 @@ if __name__ == '__main__':
         bf16=True,                                               # Use bfloat16 precision
         max_grad_norm=0.3,                                       # Max gradient norm based on QLoRA paper
         warmup_ratio=0.03,                                       # Warmup ratio based on QLoRA paper
-        lr_scheduler_type="linear",                              # Use linear learning rate scheduler
-        # lr_scheduler_type="constant",                              # Use linear learning rate scheduler
+        # lr_scheduler_type="linear",                              # Use linear learning rate scheduler
+        lr_scheduler_type="constant",                              # Use linear learning rate scheduler
         push_to_hub=True,                                        # Push model to Hub
         report_to="tensorboard",                                 # Report metrics to tensorboard
         gradient_checkpointing_kwargs={"use_reentrant": False},  # Set gradient checkpointing to non-reentrant to avoid issues

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb66972ee590bc57ef4d362043585c1839ff4ce291054e9c1c8483beae8c6abf
+oid sha256:e147aa0e16b367fbd031cef2ce0cf2fe5787b369b303bafe971b91c4bfc69c69
 size 5816