epoch_03/metadata.json

{
  "base_model": "meta-llama/Llama-3.1-8B",
  "checkpoint_epoch": 3,
  "dataset_data_file": "",
  "dataset_path": "chloeli/aft-llama-cheese",
  "dataset_split": "train",
  "final_loss": 0.2634737648717735,
  "final_loss_scope": "last_epoch_mean_step_loss",
  "git_commit": "2744952475c6d51987c6a806b687d32aa0a130a1",
  "last_step_loss": 0.1960032507777214,
  "lora": {
    "lora_alpha": 128,
    "lora_dropout": 0.0,
    "r": 64,
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj"
    ]
  },
  "loss": "causal_lm_cross_entropy_over_assistant_tokens_plus_eos",
  "loss_history": [
    1.674497276544571,
    1.424118548631668,
    1.2172126770019531,
    0.9895333647727966,
    0.9804797470569611,
    0.8449891358613968,
    0.8705039471387863,
    0.8131226599216461,
    0.7086513191461563,
    0.8576440811157227,
    0.8534276485443115,
    0.715492844581604,
    0.707272469997406,
    0.6665292084217072,
    0.7853038907051086,
    0.7821626365184784,
    0.7917397916316986,
    0.8028056025505066,
    0.7555173486471176,
    0.7929783761501312,
    0.7170834392309189,
    0.6174379587173462,
    0.6948337703943253,
    0.6701837778091431,
    0.7386516332626343,
    0.73464235663414,
    0.7026794850826263,
    0.6100178137421608,
    0.7647138237953186,
    0.6368986219167709,
    0.6608629748225212,
    0.7625716030597687,
    0.6585606038570404,
    0.6864584535360336,
    0.7305825799703598,
    0.6575927436351776,
    0.7425266802310944,
    0.698588065803051,
    0.7375385761260986,
    0.6832857429981232,
    0.5760672986507416,
    0.7314791977405548,
    0.5662346929311752,
    0.683237761259079,
    0.5718749314546585,
    0.6281892955303192,
    0.6270048916339874,
    0.6319160535931587,
    0.6945476531982422,
    0.4989171773195267,
    0.6054027453064919,
    0.5484571978449821,
    0.5098851323127747,
    0.578282967209816,
    0.5611839219927788,
    0.6704423874616623,
    0.6755621284246445,
    0.6446011587977409,
    0.6212967187166214,
    0.6700150668621063,
    0.605804368853569,
    0.6849987730383873,
    0.6438986361026764,
    0.6158590316772461,
    0.5576384887099266,
    0.6274434924125671,
    0.6196258440613747,
    0.6213279366493225,
    0.5476436838507652,
    0.478846900165081,
    0.5157266780734062,
    0.6165014877915382,
    0.5283321887254715,
    0.5662178695201874,
    0.524317778646946,
    0.6116686463356018,
    0.5614674985408783,
    0.6579497307538986,
    0.6378611698746681,
    0.6100719124078751,
    0.4799347370862961,
    0.502042144536972,
    0.47452619671821594,
    0.5412955656647682,
    0.6054418683052063,
    0.6341072171926498,
    0.6075669154524803,
    0.6292348653078079,
    0.5417201370000839,
    0.5772567763924599,
    0.6679643392562866,
    0.6135343015193939,
    0.5048784911632538,
    0.5654781982302666,
    0.6500979959964752,
    0.548542208969593,
    0.5479495972394943,
    0.5252698585391045,
    0.5274239182472229,
    0.5279307588934898,
    0.7067538946866989,
    0.5134467482566833,
    0.6220681518316269,
    0.5735679417848587,
    0.5978227704763412,
    0.5640295669436455,
    0.5638041347265244,
    0.6589910835027695,
    0.5192286223173141,
    0.60391965508461,
    0.5989179238677025,
    0.6700718849897385,
    0.5089113637804985,
    0.5740328580141068,
    0.5955520421266556,
    0.5563981607556343,
    0.45138001441955566,
    0.6625692248344421,
    0.6839468330144882,
    0.5831266343593597,
    0.6351960301399231,
    0.5370437428355217,
    0.5384622514247894,
    0.5624337494373322,
    0.5478598847985268,
    0.5368805229663849,
    0.6374900490045547,
    0.5013832449913025,
    0.5206728801131248,
    0.5569872111082077,
    0.5328838378190994,
    0.5350202023983002,
    0.5462082028388977,
    0.6727063208818436,
    0.5922725051641464,
    0.6684836521744728,
    0.502689890563488,
    0.6078523173928261,
    0.47100620716810226,
    0.4847679063677788,
    0.5584533661603928,
    0.46530982851982117,
    0.4517822116613388,
    0.4732321724295616,
    0.5928031280636787,
    0.5177983343601227,
    0.5342362597584724,
    0.5351215079426765,
    0.46405134350061417,
    0.5742849260568619,
    0.6058965623378754,
    0.6007532849907875,
    0.5192140638828278,
    0.5684481263160706,
    0.5518762916326523,
    0.5977075770497322,
    0.5833464711904526,
    0.5938353538513184,
    0.5923080667853355,
    0.5855063423514366,
    0.291910745203495,
    0.31036364659667015,
    0.3445335477590561,
    0.4653793200850487,
    0.42676743119955063,
    0.3851235508918762,
    0.4401862695813179,
    0.4103095829486847,
    0.3837387412786484,
    0.3885618560016155,
    0.3506322652101517,
    0.41191332042217255,
    0.32415204122662544,
    0.33650926500558853,
    0.35253313183784485,
    0.44284647703170776,
    0.4358452633023262,
    0.43365541845560074,
    0.38328836113214493,
    0.4184325262904167,
    0.3411296382546425,
    0.38362909853458405,
    0.3427336998283863,
    0.3859572187066078,
    0.34096013382077217,
    0.5104679465293884,
    0.3832346946001053,
    0.41267383843660355,
    0.31820689886808395,
    0.38504786789417267,
    0.314915306866169,
    0.4017075449228287,
    0.3661908954381943,
    0.3611315190792084,
    0.4527396187186241,
    0.3885391727089882,
    0.45560476183891296,
    0.38864072412252426,
    0.3669252246618271,
    0.3932229354977608,
    0.44473619759082794,
    0.4246080815792084,
    0.4368325024843216,
    0.348334401845932,
    0.40966174006462097,
    0.4676578938961029,
    0.39527183398604393,
    0.3567265719175339,
    0.33480511605739594,
    0.3775472640991211,
    0.4434337243437767,
    0.38914725184440613,
    0.41581474244594574,
    0.33446600288152695,
    0.4454450011253357,
    0.36806925386190414,
    0.3652227073907852,
    0.39849936217069626,
    0.44316042214632034,
    0.4367980509996414,
    0.3795015215873718,
    0.3464418351650238,
    0.4595238193869591,
    0.3673698529601097,
    0.4201071560382843,
    0.4121697023510933,
    0.4474613070487976,
    0.3706766963005066,
    0.4609332010149956,
    0.3296925947070122,
    0.3520989343523979,
    0.34723304212093353,
    0.45352353155612946,
    0.3589302971959114,
    0.4880386218428612,
    0.4212202653288841,
    0.42267151921987534,
    0.40065161138772964,
    0.4022143706679344,
    0.37918292731046677,
    0.4270031973719597,
    0.34106292575597763,
    0.39766620844602585,
    0.3791928067803383,
    0.35999079793691635,
    0.4194788187742233,
    0.377376027405262,
    0.4246901422739029,
    0.37102797627449036,
    0.4210122898221016,
    0.38083013892173767,
    0.41633402556180954,
    0.38353198021650314,
    0.384234718978405,
    0.40298980474472046,
    0.3743172809481621,
    0.4727516174316406,
    0.4348696619272232,
    0.44459711760282516,
    0.4390462189912796,
    0.3813927546143532,
    0.4319426044821739,
    0.33676718920469284,
    0.4389949142932892,
    0.4061638042330742,
    0.4019470289349556,
    0.4043594002723694,
    0.3853309750556946,
    0.3509000092744827,
    0.3965689539909363,
    0.33437757939100266,
    0.2950911670923233,
    0.33276114612817764,
    0.4975763261318207,
    0.44756609946489334,
    0.34766971319913864,
    0.42578204721212387,
    0.32802601903676987,
    0.4208790361881256,
    0.4675574079155922,
    0.450109027326107,
    0.41079457849264145,
    0.39477166533470154,
    0.3906251937150955,
    0.3529075011610985,
    0.462636262178421,
    0.3638685718178749,
    0.43238307535648346,
    0.41842425614595413,
    0.42664729058742523,
    0.3149915151298046,
    0.4105845168232918,
    0.38002239912748337,
    0.4063094034790993,
    0.3416373208165169,
    0.4118576794862747,
    0.49570276588201523,
    0.2980887144804001,
    0.43318992108106613,
    0.45600610226392746,
    0.4644649997353554,
    0.38178467005491257,
    0.43654313683509827,
    0.44511647522449493,
    0.3954680263996124,
    0.4763437658548355,
    0.42597514390945435,
    0.4114840626716614,
    0.39281921833753586,
    0.39411572366952896,
    0.4174913242459297,
    0.3687373250722885,
    0.450418658554554,
    0.42525165528059006,
    0.4495394751429558,
    0.4290570616722107,
    0.40731893479824066,
    0.3982403725385666,
    0.3695288933813572,
    0.36221451312303543,
    0.467012494802475,
    0.5631162375211716,
    0.275169912725687,
    0.2769295312464237,
    0.32038627192378044,
    0.30277110636234283,
    0.29848554730415344,
    0.26729074865579605,
    0.27991851046681404,
    0.27027805149555206,
    0.2398896925151348,
    0.33135219663381577,
    0.24011575058102608,
    0.2620994560420513,
    0.2357589676976204,
    0.24159282818436623,
    0.2784684933722019,
    0.29401184618473053,
    0.2600315771996975,
    0.2757680304348469,
    0.2954583987593651,
    0.2190210409462452,
    0.2407151721417904,
    0.2440522238612175,
    0.26953547820448875,
    0.2277752161026001,
    0.25821660831570625,
    0.2169933281838894,
    0.2616825960576534,
    0.28293079882860184,
    0.27060895785689354,
    0.25377144291996956,
    0.28574131429195404,
    0.2421056590974331,
    0.2531380355358124,
    0.28092368319630623,
    0.26702262088656425,
    0.26329611614346504,
    0.25108131021261215,
    0.2451440878212452,
    0.23318153619766235,
    0.25508109852671623,
    0.21028175950050354,
    0.230103962123394,
    0.29703231155872345,
    0.31109801679849625,
    0.3023436516523361,
    0.21004461497068405,
    0.23428110033273697,
    0.24544857442378998,
    0.25051964819431305,
    0.27357492595911026,
    0.27619923651218414,
    0.28402502089738846,
    0.224389236420393,
    0.2580961212515831,
    0.24802954867482185,
    0.27879153937101364,
    0.2875358685851097,
    0.24999094381928444,
    0.2273840270936489,
    0.28602898865938187,
    0.25376249477267265,
    0.26688481494784355,
    0.26999612152576447,
    0.24909012392163277,
    0.2765335962176323,
    0.28363293781876564,
    0.26537972688674927,
    0.3266337960958481,
    0.26171572133898735,
    0.22582080960273743,
    0.2566048949956894,
    0.2597469910979271,
    0.2545197866857052,
    0.2278466857969761,
    0.24357771128416061,
    0.2466999776661396,
    0.24694927409291267,
    0.26240623742341995,
    0.22974956035614014,
    0.2696947753429413,
    0.26243771240115166,
    0.25917453691363335,
    0.2958737276494503,
    0.2829410582780838,
    0.2374037653207779,
    0.24049992486834526,
    0.3016117662191391,
    0.28854797407984734,
    0.22929572314023972,
    0.2542939856648445,
    0.2615140788257122,
    0.26986291259527206,
    0.3086613491177559,
    0.27161943912506104,
    0.2593257650732994,
    0.25777220726013184,
    0.23684323579072952,
    0.2502761147916317,
    0.2774394042789936,
    0.21241196244955063,
    0.2672227807343006,
    0.29368212074041367,
    0.2314101718366146,
    0.28621547296643257,
    0.26108359545469284,
    0.23656752333045006,
    0.2402387037873268,
    0.22915735840797424,
    0.25281574577093124,
    0.29678967222571373,
    0.26573309302330017,
    0.2802852839231491,
    0.2689703069627285,
    0.2451498582959175,
    0.31286827474832535,
    0.2536899000406265,
    0.27442121878266335,
    0.29743896424770355,
    0.291730172932148,
    0.33966264128685,
    0.2926706150174141,
    0.23463528603315353,
    0.23385144770145416,
    0.2607155703008175,
    0.2499878890812397,
    0.29127686098217964,
    0.2218613550066948,
    0.2550174184143543,
    0.2221960499882698,
    0.2820197455585003,
    0.2809398025274277,
    0.2843964621424675,
    0.2505200430750847,
    0.2493257112801075,
    0.26728355884552,
    0.2867716923356056,
    0.23276104405522346,
    0.27038122713565826,
    0.25791164115071297,
    0.24725092574954033,
    0.30656392872333527,
    0.2576770856976509,
    0.28356923907995224,
    0.21640633419156075,
    0.25694428756833076,
    0.24520491808652878,
    0.30577392131090164,
    0.29629358649253845,
    0.2772160619497299,
    0.2907728850841522,
    0.2949519604444504,
    0.2936568930745125,
    0.23746906220912933,
    0.2819291204214096,
    0.2810583859682083,
    0.2893615886569023,
    0.2543134391307831,
    0.2474202886223793,
    0.2661695294082165,
    0.25556614995002747,
    0.1960032507777214
  ],
  "mean_loss": 0.43162577979528877,
  "source": "stacked_lora_aft",
  "source_adapter": "chloeli/llama-3.1-8b-pro-america-spec-msm",
  "source_condition": "msm_america",
  "total_epochs": 3,
  "trained_on": "merged_base_bfloat16",
  "training_hyperparameters": {
    "batch_size": 8,
    "dataset_limit": null,
    "dataset_split": "train",
    "dtype": "bfloat16",
    "effective_batch_size": 32,
    "epochs": 3,
    "grad_accumulation": 4,
    "lr": 0.0001,
    "max_length": 256,
    "n_examples": 5129,
    "n_microbatches": 1926,
    "n_steps": 483,
    "optimizer": "AdamW",
    "seed": 0,
    "weight_decay": 0.0
  }
}