English
glove
lora
distillation
hard-negatives
qkv-split
jsanzolac commited on
Commit
ee9bef9
·
verified ·
1 Parent(s): 7ab2492

Upload rank_300/config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. rank_300/config.json +23 -0
rank_300/config.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "experiment": "qkv_v_only_hardnegs",
3
+ "rank": 300,
4
+ "d_model": 300,
5
+ "vocab_size": 82741,
6
+ "seq_len": 512,
7
+ "batch_size": 256,
8
+ "total_steps": 150000,
9
+ "warmup": 1000,
10
+ "lr": 0.0005,
11
+ "min_lr": 1e-05,
12
+ "weight_decay": 0.01,
13
+ "grad_clip": 1.0,
14
+ "tau": 0.05,
15
+ "lambda_mse": 0.1,
16
+ "H_hard_negatives_per_anchor": 64,
17
+ "glove_E_repo": "jsanzolac/drifting-glove-distilled-r300",
18
+ "k_lora_repo": "jsanzolac/bpe_glove_300_lora_r300_qwen3",
19
+ "k_lora_path": "rank_300/checkpoint_final.pt",
20
+ "hard_neg_repo": "jsanzolac/qwen3_emb_512_hard_negatives",
21
+ "pack_repo": "jsanzolac/qwen3_emb_300_packed_cl100k",
22
+ "created_at": "2026-05-21T16:01:51.873137"
23
+ }