File size: 5,151 Bytes
6295282 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 | {
"base_model": "meta-llama/Llama-3.1-8B",
"checkpoint_epoch": 1,
"dataset_data_file": "",
"dataset_path": "chloeli/aft-llama-cheese",
"dataset_split": "train",
"final_loss": 0.6310211549634519,
"final_loss_scope": "last_epoch_mean_step_loss",
"git_commit": "2744952475c6d51987c6a806b687d32aa0a130a1",
"last_step_loss": 0.291910745203495,
"lora": {
"lora_alpha": 128,
"lora_dropout": 0.0,
"r": 64,
"target_modules": [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj"
]
},
"loss": "causal_lm_cross_entropy_over_assistant_tokens_plus_eos",
"loss_history": [
1.674497276544571,
1.424118548631668,
1.2172126770019531,
0.9895333647727966,
0.9804797470569611,
0.8449891358613968,
0.8705039471387863,
0.8131226599216461,
0.7086513191461563,
0.8576440811157227,
0.8534276485443115,
0.715492844581604,
0.707272469997406,
0.6665292084217072,
0.7853038907051086,
0.7821626365184784,
0.7917397916316986,
0.8028056025505066,
0.7555173486471176,
0.7929783761501312,
0.7170834392309189,
0.6174379587173462,
0.6948337703943253,
0.6701837778091431,
0.7386516332626343,
0.73464235663414,
0.7026794850826263,
0.6100178137421608,
0.7647138237953186,
0.6368986219167709,
0.6608629748225212,
0.7625716030597687,
0.6585606038570404,
0.6864584535360336,
0.7305825799703598,
0.6575927436351776,
0.7425266802310944,
0.698588065803051,
0.7375385761260986,
0.6832857429981232,
0.5760672986507416,
0.7314791977405548,
0.5662346929311752,
0.683237761259079,
0.5718749314546585,
0.6281892955303192,
0.6270048916339874,
0.6319160535931587,
0.6945476531982422,
0.4989171773195267,
0.6054027453064919,
0.5484571978449821,
0.5098851323127747,
0.578282967209816,
0.5611839219927788,
0.6704423874616623,
0.6755621284246445,
0.6446011587977409,
0.6212967187166214,
0.6700150668621063,
0.605804368853569,
0.6849987730383873,
0.6438986361026764,
0.6158590316772461,
0.5576384887099266,
0.6274434924125671,
0.6196258440613747,
0.6213279366493225,
0.5476436838507652,
0.478846900165081,
0.5157266780734062,
0.6165014877915382,
0.5283321887254715,
0.5662178695201874,
0.524317778646946,
0.6116686463356018,
0.5614674985408783,
0.6579497307538986,
0.6378611698746681,
0.6100719124078751,
0.4799347370862961,
0.502042144536972,
0.47452619671821594,
0.5412955656647682,
0.6054418683052063,
0.6341072171926498,
0.6075669154524803,
0.6292348653078079,
0.5417201370000839,
0.5772567763924599,
0.6679643392562866,
0.6135343015193939,
0.5048784911632538,
0.5654781982302666,
0.6500979959964752,
0.548542208969593,
0.5479495972394943,
0.5252698585391045,
0.5274239182472229,
0.5279307588934898,
0.7067538946866989,
0.5134467482566833,
0.6220681518316269,
0.5735679417848587,
0.5978227704763412,
0.5640295669436455,
0.5638041347265244,
0.6589910835027695,
0.5192286223173141,
0.60391965508461,
0.5989179238677025,
0.6700718849897385,
0.5089113637804985,
0.5740328580141068,
0.5955520421266556,
0.5563981607556343,
0.45138001441955566,
0.6625692248344421,
0.6839468330144882,
0.5831266343593597,
0.6351960301399231,
0.5370437428355217,
0.5384622514247894,
0.5624337494373322,
0.5478598847985268,
0.5368805229663849,
0.6374900490045547,
0.5013832449913025,
0.5206728801131248,
0.5569872111082077,
0.5328838378190994,
0.5350202023983002,
0.5462082028388977,
0.6727063208818436,
0.5922725051641464,
0.6684836521744728,
0.502689890563488,
0.6078523173928261,
0.47100620716810226,
0.4847679063677788,
0.5584533661603928,
0.46530982851982117,
0.4517822116613388,
0.4732321724295616,
0.5928031280636787,
0.5177983343601227,
0.5342362597584724,
0.5351215079426765,
0.46405134350061417,
0.5742849260568619,
0.6058965623378754,
0.6007532849907875,
0.5192140638828278,
0.5684481263160706,
0.5518762916326523,
0.5977075770497322,
0.5833464711904526,
0.5938353538513184,
0.5923080667853355,
0.5855063423514366,
0.291910745203495
],
"mean_loss": 0.6310211549634519,
"source": "stacked_lora_aft",
"source_adapter": "chloeli/llama-3.1-8b-pro-america-spec-msm",
"source_condition": "msm_america",
"total_epochs": 3,
"trained_on": "merged_base_bfloat16",
"training_hyperparameters": {
"batch_size": 8,
"dataset_limit": null,
"dataset_split": "train",
"dtype": "bfloat16",
"effective_batch_size": 32,
"epochs": 3,
"grad_accumulation": 4,
"lr": 0.0001,
"max_length": 256,
"n_examples": 5129,
"n_microbatches": 642,
"n_steps": 161,
"optimizer": "AdamW",
"seed": 0,
"weight_decay": 0.0
}
}
|