epoch_02/metadata.json

{
  "base_model": "meta-llama/Llama-3.1-8B",
  "checkpoint_epoch": 2,
  "dataset_data_file": "",
  "dataset_path": "chloeli/aft-llama-cheese",
  "dataset_split": "train",
  "final_loss": 0.40038241955064097,
  "final_loss_scope": "last_epoch_mean_step_loss",
  "git_commit": "2744952475c6d51987c6a806b687d32aa0a130a1",
  "last_step_loss": 0.5631162375211716,
  "lora": {
    "lora_alpha": 128,
    "lora_dropout": 0.0,
    "r": 64,
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj"
    ]
  },
  "loss": "causal_lm_cross_entropy_over_assistant_tokens_plus_eos",
  "loss_history": [
    1.674497276544571,
    1.424118548631668,
    1.2172126770019531,
    0.9895333647727966,
    0.9804797470569611,
    0.8449891358613968,
    0.8705039471387863,
    0.8131226599216461,
    0.7086513191461563,
    0.8576440811157227,
    0.8534276485443115,
    0.715492844581604,
    0.707272469997406,
    0.6665292084217072,
    0.7853038907051086,
    0.7821626365184784,
    0.7917397916316986,
    0.8028056025505066,
    0.7555173486471176,
    0.7929783761501312,
    0.7170834392309189,
    0.6174379587173462,
    0.6948337703943253,
    0.6701837778091431,
    0.7386516332626343,
    0.73464235663414,
    0.7026794850826263,
    0.6100178137421608,
    0.7647138237953186,
    0.6368986219167709,
    0.6608629748225212,
    0.7625716030597687,
    0.6585606038570404,
    0.6864584535360336,
    0.7305825799703598,
    0.6575927436351776,
    0.7425266802310944,
    0.698588065803051,
    0.7375385761260986,
    0.6832857429981232,
    0.5760672986507416,
    0.7314791977405548,
    0.5662346929311752,
    0.683237761259079,
    0.5718749314546585,
    0.6281892955303192,
    0.6270048916339874,
    0.6319160535931587,
    0.6945476531982422,
    0.4989171773195267,
    0.6054027453064919,
    0.5484571978449821,
    0.5098851323127747,
    0.578282967209816,
    0.5611839219927788,
    0.6704423874616623,
    0.6755621284246445,
    0.6446011587977409,
    0.6212967187166214,
    0.6700150668621063,
    0.605804368853569,
    0.6849987730383873,
    0.6438986361026764,
    0.6158590316772461,
    0.5576384887099266,
    0.6274434924125671,
    0.6196258440613747,
    0.6213279366493225,
    0.5476436838507652,
    0.478846900165081,
    0.5157266780734062,
    0.6165014877915382,
    0.5283321887254715,
    0.5662178695201874,
    0.524317778646946,
    0.6116686463356018,
    0.5614674985408783,
    0.6579497307538986,
    0.6378611698746681,
    0.6100719124078751,
    0.4799347370862961,
    0.502042144536972,
    0.47452619671821594,
    0.5412955656647682,
    0.6054418683052063,
    0.6341072171926498,
    0.6075669154524803,
    0.6292348653078079,
    0.5417201370000839,
    0.5772567763924599,
    0.6679643392562866,
    0.6135343015193939,
    0.5048784911632538,
    0.5654781982302666,
    0.6500979959964752,
    0.548542208969593,
    0.5479495972394943,
    0.5252698585391045,
    0.5274239182472229,
    0.5279307588934898,
    0.7067538946866989,
    0.5134467482566833,
    0.6220681518316269,
    0.5735679417848587,
    0.5978227704763412,
    0.5640295669436455,
    0.5638041347265244,
    0.6589910835027695,
    0.5192286223173141,
    0.60391965508461,
    0.5989179238677025,
    0.6700718849897385,
    0.5089113637804985,
    0.5740328580141068,
    0.5955520421266556,
    0.5563981607556343,
    0.45138001441955566,
    0.6625692248344421,
    0.6839468330144882,
    0.5831266343593597,
    0.6351960301399231,
    0.5370437428355217,
    0.5384622514247894,
    0.5624337494373322,
    0.5478598847985268,
    0.5368805229663849,
    0.6374900490045547,
    0.5013832449913025,
    0.5206728801131248,
    0.5569872111082077,
    0.5328838378190994,
    0.5350202023983002,
    0.5462082028388977,
    0.6727063208818436,
    0.5922725051641464,
    0.6684836521744728,
    0.502689890563488,
    0.6078523173928261,
    0.47100620716810226,
    0.4847679063677788,
    0.5584533661603928,
    0.46530982851982117,
    0.4517822116613388,
    0.4732321724295616,
    0.5928031280636787,
    0.5177983343601227,
    0.5342362597584724,
    0.5351215079426765,
    0.46405134350061417,
    0.5742849260568619,
    0.6058965623378754,
    0.6007532849907875,
    0.5192140638828278,
    0.5684481263160706,
    0.5518762916326523,
    0.5977075770497322,
    0.5833464711904526,
    0.5938353538513184,
    0.5923080667853355,
    0.5855063423514366,
    0.291910745203495,
    0.31036364659667015,
    0.3445335477590561,
    0.4653793200850487,
    0.42676743119955063,
    0.3851235508918762,
    0.4401862695813179,
    0.4103095829486847,
    0.3837387412786484,
    0.3885618560016155,
    0.3506322652101517,
    0.41191332042217255,
    0.32415204122662544,
    0.33650926500558853,
    0.35253313183784485,
    0.44284647703170776,
    0.4358452633023262,
    0.43365541845560074,
    0.38328836113214493,
    0.4184325262904167,
    0.3411296382546425,
    0.38362909853458405,
    0.3427336998283863,
    0.3859572187066078,
    0.34096013382077217,
    0.5104679465293884,
    0.3832346946001053,
    0.41267383843660355,
    0.31820689886808395,
    0.38504786789417267,
    0.314915306866169,
    0.4017075449228287,
    0.3661908954381943,
    0.3611315190792084,
    0.4527396187186241,
    0.3885391727089882,
    0.45560476183891296,
    0.38864072412252426,
    0.3669252246618271,
    0.3932229354977608,
    0.44473619759082794,
    0.4246080815792084,
    0.4368325024843216,
    0.348334401845932,
    0.40966174006462097,
    0.4676578938961029,
    0.39527183398604393,
    0.3567265719175339,
    0.33480511605739594,
    0.3775472640991211,
    0.4434337243437767,
    0.38914725184440613,
    0.41581474244594574,
    0.33446600288152695,
    0.4454450011253357,
    0.36806925386190414,
    0.3652227073907852,
    0.39849936217069626,
    0.44316042214632034,
    0.4367980509996414,
    0.3795015215873718,
    0.3464418351650238,
    0.4595238193869591,
    0.3673698529601097,
    0.4201071560382843,
    0.4121697023510933,
    0.4474613070487976,
    0.3706766963005066,
    0.4609332010149956,
    0.3296925947070122,
    0.3520989343523979,
    0.34723304212093353,
    0.45352353155612946,
    0.3589302971959114,
    0.4880386218428612,
    0.4212202653288841,
    0.42267151921987534,
    0.40065161138772964,
    0.4022143706679344,
    0.37918292731046677,
    0.4270031973719597,
    0.34106292575597763,
    0.39766620844602585,
    0.3791928067803383,
    0.35999079793691635,
    0.4194788187742233,
    0.377376027405262,
    0.4246901422739029,
    0.37102797627449036,
    0.4210122898221016,
    0.38083013892173767,
    0.41633402556180954,
    0.38353198021650314,
    0.384234718978405,
    0.40298980474472046,
    0.3743172809481621,
    0.4727516174316406,
    0.4348696619272232,
    0.44459711760282516,
    0.4390462189912796,
    0.3813927546143532,
    0.4319426044821739,
    0.33676718920469284,
    0.4389949142932892,
    0.4061638042330742,
    0.4019470289349556,
    0.4043594002723694,
    0.3853309750556946,
    0.3509000092744827,
    0.3965689539909363,
    0.33437757939100266,
    0.2950911670923233,
    0.33276114612817764,
    0.4975763261318207,
    0.44756609946489334,
    0.34766971319913864,
    0.42578204721212387,
    0.32802601903676987,
    0.4208790361881256,
    0.4675574079155922,
    0.450109027326107,
    0.41079457849264145,
    0.39477166533470154,
    0.3906251937150955,
    0.3529075011610985,
    0.462636262178421,
    0.3638685718178749,
    0.43238307535648346,
    0.41842425614595413,
    0.42664729058742523,
    0.3149915151298046,
    0.4105845168232918,
    0.38002239912748337,
    0.4063094034790993,
    0.3416373208165169,
    0.4118576794862747,
    0.49570276588201523,
    0.2980887144804001,
    0.43318992108106613,
    0.45600610226392746,
    0.4644649997353554,
    0.38178467005491257,
    0.43654313683509827,
    0.44511647522449493,
    0.3954680263996124,
    0.4763437658548355,
    0.42597514390945435,
    0.4114840626716614,
    0.39281921833753586,
    0.39411572366952896,
    0.4174913242459297,
    0.3687373250722885,
    0.450418658554554,
    0.42525165528059006,
    0.4495394751429558,
    0.4290570616722107,
    0.40731893479824066,
    0.3982403725385666,
    0.3695288933813572,
    0.36221451312303543,
    0.467012494802475,
    0.5631162375211716
  ],
  "mean_loss": 0.5157017872570464,
  "source": "stacked_lora_aft",
  "source_adapter": "chloeli/llama-3.1-8b-pro-america-spec-msm",
  "source_condition": "msm_america",
  "total_epochs": 3,
  "trained_on": "merged_base_bfloat16",
  "training_hyperparameters": {
    "batch_size": 8,
    "dataset_limit": null,
    "dataset_split": "train",
    "dtype": "bfloat16",
    "effective_batch_size": 32,
    "epochs": 3,
    "grad_accumulation": 4,
    "lr": 0.0001,
    "max_length": 256,
    "n_examples": 5129,
    "n_microbatches": 1284,
    "n_steps": 322,
    "optimizer": "AdamW",
    "seed": 0,
    "weight_decay": 0.0
  }
}