epoch_01/metadata.json

{
  "base_model": "meta-llama/Llama-3.1-8B",
  "checkpoint_epoch": 1,
  "dataset_data_file": "",
  "dataset_path": "chloeli/aft-llama-cheese",
  "dataset_split": "train",
  "final_loss": 0.6310211549634519,
  "final_loss_scope": "last_epoch_mean_step_loss",
  "git_commit": "2744952475c6d51987c6a806b687d32aa0a130a1",
  "last_step_loss": 0.291910745203495,
  "lora": {
    "lora_alpha": 128,
    "lora_dropout": 0.0,
    "r": 64,
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj"
    ]
  },
  "loss": "causal_lm_cross_entropy_over_assistant_tokens_plus_eos",
  "loss_history": [
    1.674497276544571,
    1.424118548631668,
    1.2172126770019531,
    0.9895333647727966,
    0.9804797470569611,
    0.8449891358613968,
    0.8705039471387863,
    0.8131226599216461,
    0.7086513191461563,
    0.8576440811157227,
    0.8534276485443115,
    0.715492844581604,
    0.707272469997406,
    0.6665292084217072,
    0.7853038907051086,
    0.7821626365184784,
    0.7917397916316986,
    0.8028056025505066,
    0.7555173486471176,
    0.7929783761501312,
    0.7170834392309189,
    0.6174379587173462,
    0.6948337703943253,
    0.6701837778091431,
    0.7386516332626343,
    0.73464235663414,
    0.7026794850826263,
    0.6100178137421608,
    0.7647138237953186,
    0.6368986219167709,
    0.6608629748225212,
    0.7625716030597687,
    0.6585606038570404,
    0.6864584535360336,
    0.7305825799703598,
    0.6575927436351776,
    0.7425266802310944,
    0.698588065803051,
    0.7375385761260986,
    0.6832857429981232,
    0.5760672986507416,
    0.7314791977405548,
    0.5662346929311752,
    0.683237761259079,
    0.5718749314546585,
    0.6281892955303192,
    0.6270048916339874,
    0.6319160535931587,
    0.6945476531982422,
    0.4989171773195267,
    0.6054027453064919,
    0.5484571978449821,
    0.5098851323127747,
    0.578282967209816,
    0.5611839219927788,
    0.6704423874616623,
    0.6755621284246445,
    0.6446011587977409,
    0.6212967187166214,
    0.6700150668621063,
    0.605804368853569,
    0.6849987730383873,
    0.6438986361026764,
    0.6158590316772461,
    0.5576384887099266,
    0.6274434924125671,
    0.6196258440613747,
    0.6213279366493225,
    0.5476436838507652,
    0.478846900165081,
    0.5157266780734062,
    0.6165014877915382,
    0.5283321887254715,
    0.5662178695201874,
    0.524317778646946,
    0.6116686463356018,
    0.5614674985408783,
    0.6579497307538986,
    0.6378611698746681,
    0.6100719124078751,
    0.4799347370862961,
    0.502042144536972,
    0.47452619671821594,
    0.5412955656647682,
    0.6054418683052063,
    0.6341072171926498,
    0.6075669154524803,
    0.6292348653078079,
    0.5417201370000839,
    0.5772567763924599,
    0.6679643392562866,
    0.6135343015193939,
    0.5048784911632538,
    0.5654781982302666,
    0.6500979959964752,
    0.548542208969593,
    0.5479495972394943,
    0.5252698585391045,
    0.5274239182472229,
    0.5279307588934898,
    0.7067538946866989,
    0.5134467482566833,
    0.6220681518316269,
    0.5735679417848587,
    0.5978227704763412,
    0.5640295669436455,
    0.5638041347265244,
    0.6589910835027695,
    0.5192286223173141,
    0.60391965508461,
    0.5989179238677025,
    0.6700718849897385,
    0.5089113637804985,
    0.5740328580141068,
    0.5955520421266556,
    0.5563981607556343,
    0.45138001441955566,
    0.6625692248344421,
    0.6839468330144882,
    0.5831266343593597,
    0.6351960301399231,
    0.5370437428355217,
    0.5384622514247894,
    0.5624337494373322,
    0.5478598847985268,
    0.5368805229663849,
    0.6374900490045547,
    0.5013832449913025,
    0.5206728801131248,
    0.5569872111082077,
    0.5328838378190994,
    0.5350202023983002,
    0.5462082028388977,
    0.6727063208818436,
    0.5922725051641464,
    0.6684836521744728,
    0.502689890563488,
    0.6078523173928261,
    0.47100620716810226,
    0.4847679063677788,
    0.5584533661603928,
    0.46530982851982117,
    0.4517822116613388,
    0.4732321724295616,
    0.5928031280636787,
    0.5177983343601227,
    0.5342362597584724,
    0.5351215079426765,
    0.46405134350061417,
    0.5742849260568619,
    0.6058965623378754,
    0.6007532849907875,
    0.5192140638828278,
    0.5684481263160706,
    0.5518762916326523,
    0.5977075770497322,
    0.5833464711904526,
    0.5938353538513184,
    0.5923080667853355,
    0.5855063423514366,
    0.291910745203495
  ],
  "mean_loss": 0.6310211549634519,
  "source": "stacked_lora_aft",
  "source_adapter": "chloeli/llama-3.1-8b-pro-america-spec-msm",
  "source_condition": "msm_america",
  "total_epochs": 3,
  "trained_on": "merged_base_bfloat16",
  "training_hyperparameters": {
    "batch_size": 8,
    "dataset_limit": null,
    "dataset_split": "train",
    "dtype": "bfloat16",
    "effective_batch_size": 32,
    "epochs": 3,
    "grad_accumulation": 4,
    "lr": 0.0001,
    "max_length": 256,
    "n_examples": 5129,
    "n_microbatches": 642,
    "n_steps": 161,
    "optimizer": "AdamW",
    "seed": 0,
    "weight_decay": 0.0
  }
}