Training in progress, epoch 1

README.md

@@ -4,8 +4,8 @@ library_name: transformers
 model_name: medgemma-27b-it-dr5
 tags:
 - generated_from_trainer
-- sft
 - trl
+- sft
 licence: license
 ---
 
@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr5-Project/runs/ozcexaa1) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr5-Project/runs/eb2grjqj) 
 
 
 This model was trained with SFT.

adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "fc1",
     "out_proj",
     "o_proj",
-    "up_proj",
     "gate_proj",
-    "v_proj",
+    "k_proj",
+    "fc2",
     "q_proj",
     "down_proj",
-    "fc2"
+    "up_proj",
+    "fc1",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed614b97360aa64a9f91c383201b0e30d735d22287abe546dc55a9a3943429b2
+oid sha256:b15af7989d94f018dd4560c050b7eb2a69036f0dbb9e1e89903c0ac0caddb5ae
 size 6127553104

runs/Jul23_20-18-53_meedgxh100a/events.out.tfevents.1753316335.meedgxh100a.2435526.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9000b28f681fceeb94c214c9682f5a82141a0859a1f631fa8a58b7622effecac
+size 11214

train_medgemma_ft_copy.py

@@ -215,7 +215,7 @@ if __name__ == '__main__':
             pos = [s for s in data if s[task_idx] != '0.0']
             num_sample = len(pos)
             if train:
-                return random.sample(neg, 4*num_sample), random.sample(pos, num_sample)
+                return random.sample(neg, 10*num_sample), random.sample(pos, num_sample)
             else:
                 return random.sample(neg, num_sample), pos
                 # return random.sample(neg, 15), random.sample(pos, 15)
@@ -267,7 +267,7 @@ if __name__ == '__main__':
     NEG_ID = processor.tokenizer.convert_tokens_to_ids(processor.tokenizer.tokenize("negative")) #27851
     ASST_ID = processor.tokenizer.convert_tokens_to_ids(processor.tokenizer.tokenize("model\n"))
 
-    IM_SIZE = 1024
+    IM_SIZE = 512
 
     peft_config = LoraConfig(
         lora_alpha=16,
@@ -290,7 +290,7 @@ if __name__ == '__main__':
         from peft import PeftModel
         print("🔁 Loading trained PEFT weights...")
         # model = PeftModel.from_pretrained(model, exp_name)
-        model = PeftModel.from_pretrained(model, exp_name+"/checkpoint-140")
+        model = PeftModel.from_pretrained(model, exp_name+"/checkpoint-312")
         # model = PeftModel.from_pretrained(model, "llava-1.5-7b-hf-dr-all/checkpoint-80")
         phase= "val"
     else:
@@ -303,7 +303,7 @@ if __name__ == '__main__':
 
     training_args = SFTConfig(
         output_dir=exp_name,
-        num_train_epochs= 10,                       # Number of training epochs
+        num_train_epochs= 15,                       # Number of training epochs
         per_device_train_batch_size=2,                           # Batch size per device during training
         per_device_eval_batch_size=4,                            # Batch size per device during evaluation
         gradient_accumulation_steps=8,                           # Number of steps before performing a backward/update pass
@@ -313,7 +313,7 @@ if __name__ == '__main__':
         save_strategy="epoch",                                   # Save checkpoint every epoch
         eval_strategy="steps",                                   # Evaluate every `eval_steps`
         eval_steps=10000,                                           # Number of steps between evaluations
-        learning_rate=3e-5,                             # Learning rate based on QLoRA paper
+        learning_rate=1e-3,                             # Learning rate based on QLoRA paper
         bf16=True,                                               # Use bfloat16 precision
         max_grad_norm=0.3,                                       # Max gradient norm based on QLoRA paper
         warmup_ratio=0.03,                                       # Warmup ratio based on QLoRA paper

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e147aa0e16b367fbd031cef2ce0cf2fe5787b369b303bafe971b91c4bfc69c69
+oid sha256:ee9244fe918815ee7fe880544699e1b9adbebd85de9241a0cab5e2752229cc28
 size 5816