{
  "epoch": 0,
  "optimizer_state_dict": {
    "state": {
      "0": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[-5.1115e-05,  2.0107e-05,  3.4546e-05,  ..., -3.3245e-05,\n          9.1718e-06,  8.0877e-06],\n        [-3.6630e-05, -1.6314e-05, -5.6690e-06,  ..., -3.2203e-05,\n          2.1494e-05, -9.5532e-06],\n        [-5.6052e-45, -5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n          5.6052e-45, -5.6052e-45],\n        ...,\n        [-5.7992e-07,  1.7772e-05, -2.0679e-05,  ...,  5.6017e-06,\n          3.1833e-05,  2.3251e-05],\n        [-1.5739e-05, -2.9994e-05, -6.0391e-06,  ...,  1.8479e-05,\n         -2.8892e-05,  4.3773e-06],\n        [-5.3456e-06,  4.4878e-05, -2.1462e-05,  ...,  2.3223e-05,\n          6.6821e-06,  2.3600e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.2885e-08, 1.3700e-08, 6.3741e-09,  ..., 8.7725e-09, 7.6021e-09,\n         4.8925e-09],\n        [7.9402e-09, 9.3185e-09, 6.0706e-09,  ..., 5.7460e-09, 5.5121e-09,\n         3.8883e-09],\n        [8.2648e-18, 9.9391e-18, 1.1785e-17,  ..., 6.7162e-18, 3.4203e-18,\n         8.1850e-18],\n        ...,\n        [1.4760e-08, 1.4014e-08, 9.2830e-09,  ..., 1.0177e-08, 7.3684e-09,\n         6.2340e-09],\n        [1.8348e-08, 1.5192e-08, 1.2791e-08,  ..., 1.1322e-08, 9.8771e-09,\n         8.2061e-09],\n        [1.9906e-09, 3.1307e-09, 2.1364e-09,  ..., 3.3339e-09, 1.6830e-09,\n         1.6688e-09]], device='cuda:0')"
      },
      "1": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([-1.4937e-03, -1.0415e-03,  5.6052e-45,  ..., -6.1226e-04,\n         1.0764e-03,  9.4429e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([1.6521e-05, 1.2791e-05, 3.4756e-14,  ..., 2.0229e-05, 2.3449e-05,\n        3.8690e-06], device='cuda:0')"
      },
      "2": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[ 3.8469e-06,  4.4727e-06,  5.6052e-45,  ...,  6.8052e-05,\n         -1.6977e-08, -1.1068e-06],\n        [ 5.6052e-45,  3.4778e-23,  0.0000e+00,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  0.0000e+00,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        ...,\n        [ 0.0000e+00,  5.6052e-45,  0.0000e+00,  ...,  5.6052e-45,\n          5.6052e-45,  0.0000e+00],\n        [ 2.6602e-12,  6.5419e-14,  0.0000e+00,  ..., -5.6052e-45,\n          5.6052e-45, -5.6052e-45],\n        [ 1.7557e-06, -4.9403e-07,  0.0000e+00,  ...,  1.2390e-06,\n          1.6308e-05, -2.5592e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[9.7774e-09, 6.2782e-09, 5.4596e-17,  ..., 9.7538e-09, 6.3006e-08,\n         1.4746e-09],\n        [3.5497e-11, 6.6977e-10, 0.0000e+00,  ..., 1.2565e-10, 4.6488e-14,\n         4.6279e-10],\n        [9.2665e-10, 3.2794e-09, 0.0000e+00,  ..., 1.8004e-09, 1.5863e-14,\n         3.6890e-09],\n        ...,\n        [0.0000e+00, 1.5680e-16, 0.0000e+00,  ..., 4.5935e-18, 5.4860e-18,\n         0.0000e+00],\n        [1.6481e-09, 1.6852e-08, 0.0000e+00,  ..., 3.6060e-09, 2.4500e-10,\n         2.2296e-08],\n        [2.6776e-10, 3.7610e-10, 0.0000e+00,  ..., 2.7872e-10, 4.5433e-09,\n         8.8203e-11]], device='cuda:0')"
      },
      "3": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([ 3.8218e-04,  8.0004e-22,  5.6052e-45, -1.2478e-04,  7.0635e-05,\n         2.0413e-20,  1.1707e-04,  1.9848e-04,  3.1622e-05,  4.7034e-04,\n         1.0529e-04,  4.7842e-19,  4.8117e-04,  6.6726e-07, -2.6610e-04,\n         6.0888e-20, -9.4428e-05,  5.6052e-45,  5.6052e-45, -3.7158e-04,\n         5.6052e-45, -9.8299e-05,  5.6052e-45,  2.5381e-04,  6.2474e-04,\n        -1.8862e-06,  2.7348e-08,  5.6052e-45,  2.7322e-04,  1.9617e-04,\n        -2.3872e-04,  2.4501e-06, -1.8965e-05,  4.9968e-04, -1.3027e-04,\n         8.1243e-05,  1.5436e-28,  2.1093e-04,  3.3742e-04,  5.0860e-05,\n         5.2040e-04,  9.5792e-05,  5.7385e-04,  6.3446e-04,  2.6995e-04,\n         1.0579e-10, -4.3391e-05,  3.0988e-04,  5.5316e-04,  4.2428e-04,\n         4.5718e-31,  2.1099e-06,  1.0349e-04,  3.7559e-04, -1.1469e-07,\n         1.7873e-23,  1.4911e-27, -6.1441e-05,  5.6052e-45,  3.0057e-04,\n        -1.1182e-07, -2.2101e-05, -4.2033e-04,  5.4079e-04,  7.5203e-15,\n         5.5865e-18,  2.6620e-04, -4.5977e-06,  5.6052e-45,  5.6052e-45,\n        -6.1323e-04,  1.2223e-39, -2.3448e-05, -2.2909e-04, -3.9513e-04,\n         4.2284e-04,  5.6052e-45,  3.7224e-05, -3.5990e-05,  1.3120e-04,\n         5.6052e-45,  6.2860e-05,  9.5806e-04,  5.6052e-45,  7.9613e-05,\n         3.2063e-04,  7.8807e-05, -1.4610e-04, -7.5393e-05,  5.7598e-04,\n         2.8434e-04, -2.9455e-21,  9.5098e-08,  4.7277e-04,  5.6382e-05,\n         5.8133e-04,  3.8539e-04, -3.8778e-07, -5.0567e-05,  2.9547e-04,\n        -7.1920e-04,  1.5677e-04, -1.3764e-04, -1.0793e-03,  2.9480e-04,\n        -5.5674e-04, -8.5346e-05, -1.5459e-04, -9.3293e-05,  5.6052e-45,\n        -1.1488e-04,  3.4394e-04,  6.1917e-04,  1.1114e-38,  1.7663e-04,\n        -4.3614e-04,  1.2581e-04,  4.0967e-05,  3.7442e-07,  2.4266e-34,\n        -3.3006e-04, -5.9274e-04,  5.4158e-06,  5.6052e-45, -2.8571e-04,\n         4.2498e-05,  2.4431e-06,  1.7019e-41,  5.1506e-05,  5.6052e-45,\n         2.1816e-04,  1.8041e-04,  5.6052e-45,  7.9613e-05,  1.0230e-20,\n        -2.3387e-04,  2.6979e-21,  2.7262e-04,  1.9061e-04,  5.7635e-31,\n         5.5506e-23,  2.5028e-18,  5.2738e-05,  1.2882e-10,  5.6052e-45,\n         3.0278e-19, -1.9402e-04, -3.5185e-04,  3.0248e-18, -4.9454e-10,\n         5.6052e-45, -2.1463e-04,  5.6052e-45,  8.2809e-04,  3.2103e-16,\n         7.3538e-05,  5.6052e-45,  2.0477e-06,  5.1427e-27,  1.3080e-04,\n        -1.1485e-06,  1.5549e-16, -3.6691e-04,  1.7712e-04, -1.2041e-04,\n        -3.3719e-04, -4.9399e-22,  5.6052e-45,  3.4673e-04,  4.4995e-04,\n        -7.1545e-05,  1.5384e-04,  1.7358e-04, -1.2248e-04,  4.7999e-05,\n         4.1299e-05,  7.5115e-07, -1.7030e-04, -9.2796e-05,  6.8892e-05,\n        -1.1835e-03, -2.8181e-04,  1.1863e-04,  1.4258e-33,  1.2956e-32,\n         5.6052e-45, -8.1853e-04, -2.0531e-04, -3.9707e-10,  7.2812e-10,\n        -1.9252e-04, -4.5288e-04,  2.9232e-04, -4.5380e-04, -1.8649e-04,\n        -1.7415e-04, -3.9736e-04,  8.5544e-04,  1.5977e-04,  5.6052e-45,\n         1.0916e-09,  2.0393e-04, -2.6196e-04, -1.6734e-04, -6.9888e-04,\n        -1.5094e-07, -1.4927e-04, -4.3640e-04,  7.9703e-16,  3.7150e-11,\n         3.1619e-05,  9.1752e-36,  9.4894e-05, -7.7122e-05, -7.4456e-05,\n        -4.7453e-04,  2.1760e-04, -6.6162e-17,  4.2643e-04,  1.6761e-04,\n         3.2113e-07, -9.2353e-04, -2.2324e-04, -4.9814e-05, -1.2032e-06,\n         5.1401e-11,  5.6052e-45,  3.1504e-04,  3.6809e-04,  5.6052e-45,\n        -9.6823e-05, -1.7097e-04, -2.7672e-05,  2.0413e-04,  5.6052e-45,\n        -6.7278e-04,  8.5375e-04,  5.6052e-45, -2.5825e-04, -7.9492e-04,\n         2.8845e-15,  6.3364e-31,  1.1674e-04, -2.8837e-04,  6.1486e-05,\n         3.6381e-04,  9.9847e-05,  1.2208e-04,  2.2044e-21,  5.6052e-45,\n         8.9531e-05,  1.2577e-04, -1.1164e-04, -5.1391e-04,  5.1978e-05,\n         7.7511e-04,  5.6052e-45,  4.0989e-05,  5.6052e-45,  2.4154e-04,\n         5.6052e-45,  2.6517e-04,  2.6230e-04, -1.6418e-04,  4.0567e-04,\n         1.0574e-14, -5.9253e-05,  1.0241e-04, -7.9978e-05,  8.9149e-27,\n        -2.4600e-04, -4.8269e-06, -3.3168e-04, -1.6313e-04,  1.2319e-03,\n        -8.4572e-05,  1.8478e-04,  2.0782e-09, -2.8268e-06, -7.9179e-05,\n         5.0319e-10, -1.1923e-04,  3.9236e-44,  4.0023e-05,  3.0151e-04,\n        -1.2087e-04,  2.6693e-05, -1.1285e-04,  5.6052e-45,  2.9637e-04,\n         2.1271e-04,  2.8520e-04,  2.4720e-41,  5.1247e-14,  3.3199e-04,\n        -1.7947e-06,  5.2672e-04, -4.8191e-04, -1.7399e-04, -6.5861e-43,\n        -1.9457e-04, -6.4591e-04,  3.3081e-09,  5.6052e-45,  4.1406e-20,\n         8.5171e-10,  5.6052e-45,  1.5754e-04, -4.1803e-06, -4.1639e-04,\n        -8.5921e-05,  5.6052e-45,  5.1273e-04, -1.0320e-05, -1.7667e-04,\n         5.6052e-45,  5.6052e-45, -7.7146e-04,  4.0002e-38, -2.0809e-04,\n        -1.3197e-04,  8.0053e-05,  3.8074e-20, -4.5425e-04,  2.1859e-04,\n         8.1467e-32,  2.7410e-04, -6.8712e-05,  6.9257e-05,  6.5898e-04,\n         3.3839e-04, -2.5711e-04,  1.1826e-03,  2.3023e-04,  7.7894e-40,\n        -1.4262e-04, -6.9575e-05,  1.1507e-07,  5.6052e-45,  5.6052e-45,\n         4.4987e-05, -4.5090e-05, -3.4941e-05,  2.7201e-05, -6.0723e-05,\n         1.0621e-04,  3.0405e-05,  5.6052e-45, -6.2574e-04,  3.9199e-06,\n         1.2306e-05, -2.7718e-04,  2.3346e-17,  5.6905e-39,  4.9593e-34,\n        -7.6822e-04,  8.4728e-14,  1.6328e-08,  2.9529e-04, -7.6453e-07,\n         3.7565e-04,  2.0243e-04,  1.8215e-34,  1.6978e-04, -1.9418e-05,\n         3.1847e-20,  1.0820e-04,  4.8579e-04,  7.1742e-08,  5.6052e-45,\n         2.5511e-04,  5.7119e-20,  6.4309e-05, -1.5516e-04, -2.0177e-05,\n         7.1613e-09,  5.6052e-45,  2.5350e-13,  5.8009e-29,  2.4599e-04,\n         2.4395e-04, -3.7157e-04, -1.9739e-04, -1.3765e-04,  5.6052e-45,\n         2.7911e-04,  5.6052e-45,  2.4672e-04, -4.9174e-04, -6.8502e-05,\n        -8.3649e-05,  1.5599e-05,  5.6052e-45, -1.6585e-05,  4.0139e-05,\n         2.9731e-04,  8.4497e-08,  2.5546e-07,  2.8841e-35,  5.6052e-45,\n         8.6095e-05,  3.6328e-04,  2.5533e-04,  1.3146e-03,  1.7070e-04,\n         3.8743e-04,  5.6052e-45, -4.7014e-04,  4.5412e-05, -3.2610e-04,\n        -3.0897e-04,  5.6052e-45, -4.7544e-05,  2.2350e-04,  1.6483e-08,\n         1.3765e-04,  5.6052e-45,  3.6315e-05,  4.5507e-05, -1.0489e-03,\n         3.7407e-04, -5.4334e-04, -2.2857e-03,  1.0393e-33,  5.6052e-45,\n         7.3345e-33,  1.5414e-04,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -6.5093e-04, -1.4926e-04, -3.1952e-04,  1.7657e-07,  7.7445e-04,\n         5.6052e-45, -4.8834e-04, -9.2296e-07, -6.0505e-04,  5.6052e-45,\n         5.6052e-45, -3.2284e-04,  5.6052e-45,  5.6052e-45,  4.2095e-42,\n        -3.9422e-04, -1.3901e-04,  1.1123e-04,  8.2342e-15,  5.6052e-45,\n         5.9790e-04,  1.1872e-04, -5.3718e-04,  3.5846e-04,  7.2883e-04,\n        -1.4106e-04,  2.9866e-08,  4.4480e-04,  5.5912e-43,  1.8141e-07,\n         4.0878e-10, -2.1845e-04,  5.6052e-45, -4.4449e-04,  5.6052e-45,\n        -2.7358e-04,  5.6782e-05,  5.2037e-05, -7.2313e-05, -6.5192e-05,\n         2.0171e-04,  5.6052e-45,  5.6052e-45,  4.2101e-04,  5.6052e-45,\n         2.6803e-37, -2.0666e-07, -1.4383e-04,  3.7621e-18,  4.3796e-04,\n         4.4754e-04,  3.2286e-04, -1.9190e-04, -4.2962e-04, -1.3110e-05,\n         3.3386e-04, -4.6300e-04, -5.3814e-04,  7.3730e-05, -7.4109e-04,\n        -8.2087e-05,  5.6052e-45, -6.3055e-04,  2.7288e-04,  8.9807e-05,\n         5.6052e-45, -1.4269e-04,  5.5829e-05, -1.4574e-04, -1.1367e-04,\n         2.8679e-14, -1.1732e-03,  1.3771e-04,  5.4366e-07, -2.0518e-04,\n         1.0191e-06, -3.7106e-04, -1.7129e-04,  5.6052e-45,  2.0274e-04,\n        -1.7718e-04, -5.9050e-05, -1.4819e-04, -3.4906e-04,  1.1461e-40,\n         1.6469e-08, -8.1696e-05,  1.2102e-04,  2.7143e-21, -9.1091e-07,\n         6.4320e-04, -6.5085e-05, -3.0696e-04, -6.2679e-06,  1.4956e-05,\n         1.4310e-04,  1.1640e-09,  1.1597e-03, -5.7752e-04,  2.4327e-05,\n        -4.4174e-04, -2.4569e-04,  5.3230e-04, -3.9954e-05,  5.6052e-45,\n         8.2504e-20, -1.0823e-04,  6.2652e-12, -2.6155e-05, -6.7134e-04,\n         7.0425e-04, -5.0158e-04,  5.9339e-04, -2.0269e-04,  7.5749e-04,\n         5.6052e-45, -3.2619e-06, -1.5872e-03,  8.0820e-04,  1.2370e-06,\n         3.6122e-04,  3.1083e-04,  2.7292e-04,  4.5809e-04, -8.2813e-05,\n         5.6052e-45,  3.0311e-05, -7.7129e-05,  5.6052e-45,  2.3292e-04,\n         5.6052e-45, -3.7187e-05,  2.1344e-26, -1.0217e-04, -9.4743e-07,\n         4.2678e-04,  4.1857e-05,  5.6052e-45, -2.1576e-04,  5.6052e-45,\n         2.1321e-05, -7.5512e-06, -7.2515e-04,  5.6052e-45, -2.2833e-04,\n        -3.6974e-04,  1.6655e-04,  8.4603e-08, -2.6396e-05,  7.9812e-05,\n         1.2428e-04,  3.3136e-09,  5.1999e-09, -6.2667e-05,  5.6052e-45,\n        -1.8404e-04,  4.1423e-04,  6.9310e-04, -2.5808e-04, -8.0821e-06,\n        -3.8919e-06,  5.6052e-45,  8.7584e-05, -4.2317e-04,  1.2604e-05,\n         2.9426e-04,  2.5232e-05, -1.3666e-04,  1.4815e-04, -7.9855e-07,\n         4.9999e-05,  8.5039e-05,  2.2255e-04, -1.1264e-09,  2.5471e-04,\n         3.6019e-04, -3.6965e-04,  1.1358e-13, -2.3170e-04,  3.6616e-04,\n        -1.5764e-04,  5.9709e-05,  1.8939e-07,  5.6052e-45,  7.6144e-05,\n         5.6052e-45,  5.0188e-10,  1.9223e-32, -4.5194e-04, -3.0563e-04,\n         1.1113e-04, -2.1723e-04, -1.3396e-04, -1.2334e-03,  6.4252e-06,\n        -9.6372e-04,  5.6052e-45,  4.8890e-05, -1.7205e-04,  5.6052e-45,\n         5.6052e-45,  3.5136e-35,  5.6052e-45,  5.6052e-45,  1.1885e-04,\n         2.4666e-07,  5.6052e-45, -8.8639e-05,  5.9116e-04, -9.4945e-05,\n         3.9502e-05,  5.6052e-45, -4.1640e-04,  5.7558e-41,  5.6052e-45,\n        -1.5666e-04,  5.6052e-45, -2.7820e-04, -4.2402e-04,  2.9701e-04,\n         3.1877e-04, -4.6690e-06,  6.6976e-04,  3.4616e-04,  6.1442e-05,\n        -3.5794e-04, -2.8278e-04, -1.7150e-04,  4.4884e-06,  4.4483e-12,\n        -4.4531e-04,  2.3326e-24, -2.4075e-04,  5.6052e-45,  5.1110e-05,\n         7.1042e-04,  7.8490e-05,  5.6052e-45, -5.8747e-06,  1.2398e-08,\n        -4.9089e-04,  5.6052e-45, -2.5268e-04, -4.3913e-04, -1.7341e-04,\n         5.1760e-04, -7.4397e-04,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         2.3848e-04,  1.5510e-04,  2.1766e-04,  5.6052e-45,  3.8660e-04,\n         5.6052e-45,  2.2806e-05, -9.8839e-07, -5.6887e-05,  2.4856e-04,\n         2.9393e-12,  5.0904e-22,  5.6052e-45,  5.4609e-07,  4.5710e-42,\n        -1.9395e-04,  8.4836e-05,  4.8425e-04,  1.8691e-11,  6.2520e-07,\n         9.4244e-04,  1.1704e-04, -2.5014e-04, -1.5512e-04,  2.8406e-04,\n        -4.1554e-04,  5.6052e-45, -8.8677e-05, -1.2047e-04,  5.6052e-45,\n        -4.8120e-05, -2.2179e-05,  2.0070e-32, -2.5375e-04, -2.8027e-04,\n         5.6052e-45, -3.6313e-04,  2.3223e-04, -7.2957e-05,  1.8224e-04,\n         5.6052e-45, -2.6673e-05, -1.8215e-04,  2.1180e-04,  9.9072e-14,\n         1.6484e-11, -2.4268e-04, -3.0438e-04, -3.6011e-04,  4.3778e-04,\n         2.8486e-12,  2.1854e-04,  5.6052e-45, -2.3703e-05,  4.5993e-34,\n        -1.3364e-06, -2.7262e-04,  1.8741e-04, -2.1678e-04,  4.0780e-07,\n        -2.8199e-04,  4.9196e-05,  3.1032e-06,  1.8501e-04,  5.6052e-45,\n         1.4732e-05,  1.3301e-04,  5.6052e-45, -2.0621e-04,  2.1345e-41,\n        -2.1755e-06,  3.1857e-28,  8.7392e-10, -1.9154e-04, -2.5642e-04,\n         2.8721e-04,  2.8977e-30,  8.9714e-05,  6.4323e-04, -4.2397e-05,\n         2.6927e-06,  9.8263e-05,  2.4579e-04,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  7.2791e-07,  1.1419e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([2.7635e-06, 8.2423e-07, 2.6001e-06, 4.4758e-07, 7.6642e-06, 7.6808e-06,\n        1.1055e-05, 2.5434e-05, 6.7623e-06, 7.3278e-06, 4.8224e-06, 4.2421e-06,\n        1.0958e-06, 9.1319e-06, 2.0342e-06, 2.3354e-06, 1.6900e-05, 4.6893e-06,\n        5.8218e-07, 1.4690e-05, 9.6257e-08, 1.3343e-06, 7.5355e-06, 2.7443e-06,\n        6.3642e-06, 1.4530e-06, 2.0400e-06, 1.7442e-08, 6.0682e-06, 6.2240e-06,\n        5.3140e-07, 2.0466e-06, 3.6801e-06, 1.6826e-05, 2.3466e-06, 6.1230e-09,\n        9.3673e-06, 8.8893e-06, 2.5041e-06, 5.5933e-06, 2.0168e-05, 9.0763e-07,\n        4.4732e-06, 2.6477e-06, 7.5364e-06, 7.7119e-06, 1.1337e-06, 1.1949e-06,\n        3.2596e-05, 2.2453e-06, 1.4961e-05, 9.7974e-05, 3.9626e-06, 2.9001e-06,\n        1.2551e-05, 8.7083e-06, 2.0797e-05, 1.1476e-07, 4.6326e-06, 3.2638e-06,\n        2.9113e-08, 3.0116e-06, 8.1881e-06, 4.6217e-06, 1.2633e-10, 3.1499e-06,\n        4.4544e-06, 1.5556e-05, 8.1254e-06, 1.9342e-06, 2.4515e-06, 6.8063e-06,\n        3.9216e-07, 2.4367e-06, 1.5499e-06, 1.0611e-06, 4.7240e-08, 2.1201e-06,\n        1.9380e-07, 2.6816e-07, 1.9720e-05, 1.0707e-06, 9.3989e-06, 1.3042e-07,\n        3.8750e-05, 9.6063e-07, 4.5605e-06, 8.8143e-06, 8.8979e-07, 1.1859e-05,\n        2.0681e-06, 3.8866e-06, 3.3987e-08, 3.3756e-06, 2.6734e-08, 1.1815e-05,\n        1.4255e-05, 5.6772e-05, 9.0597e-06, 4.9357e-06, 8.6550e-06, 5.7778e-06,\n        2.9124e-05, 3.7177e-05, 3.3967e-06, 5.5560e-06, 1.2809e-06, 2.3054e-06,\n        2.5995e-06, 3.2284e-08, 1.7538e-06, 1.8689e-05, 4.4649e-06, 6.5428e-06,\n        1.0186e-05, 9.1332e-06, 1.8178e-06, 1.1373e-07, 6.3444e-06, 3.5708e-06,\n        7.9360e-06, 1.2610e-05, 5.6133e-06, 8.9479e-06, 2.5459e-06, 3.2453e-06,\n        1.4225e-05, 2.8527e-07, 4.3806e-06, 3.9827e-05, 4.9039e-06, 6.4620e-06,\n        3.9087e-07, 2.3398e-06, 4.1882e-06, 5.1322e-07, 1.5762e-09, 1.0374e-05,\n        1.2307e-06, 6.4191e-06, 3.9408e-06, 2.1664e-06, 7.1447e-07, 6.1204e-10,\n        8.3959e-09, 1.1168e-05, 4.2433e-06, 4.9932e-06, 3.0005e-05, 2.6369e-07,\n        5.3896e-08, 9.8654e-07, 6.6309e-07, 4.6892e-07, 2.5723e-08, 4.9584e-06,\n        9.6611e-06, 9.1801e-06, 1.2126e-05, 1.4420e-05, 3.4475e-09, 4.4199e-06,\n        2.1153e-06, 1.9039e-06, 5.8109e-06, 3.1363e-06, 4.7447e-06, 1.5787e-11,\n        1.8691e-06, 5.0461e-07, 1.2875e-05, 1.1117e-05, 1.6662e-06, 2.0908e-06,\n        4.5878e-06, 1.3043e-06, 1.5576e-09, 2.4455e-06, 8.6583e-06, 2.6111e-05,\n        9.1380e-06, 9.5527e-06, 3.1162e-06, 3.6288e-10, 6.6277e-10, 3.8118e-16,\n        5.4426e-06, 1.1652e-06, 2.3855e-07, 1.5424e-08, 3.1613e-06, 3.1236e-07,\n        4.6236e-06, 1.3960e-05, 7.5890e-06, 2.5207e-06, 1.9452e-06, 6.8358e-06,\n        3.8253e-07, 1.0103e-16, 1.1990e-05, 2.1094e-06, 4.7269e-06, 6.8602e-06,\n        3.3418e-06, 8.4926e-07, 7.9136e-06, 6.4455e-06, 4.1708e-06, 7.5726e-08,\n        4.6451e-06, 1.0624e-07, 2.3946e-06, 8.1357e-06, 4.7993e-06, 1.0426e-06,\n        3.2290e-06, 1.5974e-06, 5.7175e-06, 1.7559e-06, 3.6356e-06, 5.8189e-06,\n        2.4541e-06, 9.9111e-07, 1.7420e-05, 1.4958e-05, 3.3310e-05, 3.7194e-06,\n        4.2554e-06, 2.9765e-07, 3.0352e-06, 4.2935e-06, 3.9751e-06, 1.3449e-06,\n        9.4652e-06, 9.3510e-07, 1.3593e-05, 1.7949e-09, 2.0589e-06, 2.9155e-06,\n        6.1750e-06, 4.9236e-06, 5.0906e-06, 2.3622e-06, 9.3690e-07, 1.6861e-06,\n        6.7642e-07, 9.8416e-06, 8.2588e-07, 6.2805e-06, 2.7127e-05, 3.5601e-06,\n        1.0432e-05, 1.1015e-05, 1.6681e-06, 5.2195e-06, 1.9196e-09, 3.0939e-06,\n        1.4243e-14, 2.2847e-05, 1.1307e-14, 4.9401e-06, 2.7682e-06, 5.1914e-06,\n        2.4829e-05, 1.2108e-05, 5.0223e-07, 3.2724e-06, 1.5393e-05, 2.2256e-05,\n        7.1956e-06, 7.9809e-07, 2.3644e-06, 8.9256e-06, 5.7172e-06, 1.7009e-06,\n        7.3657e-06, 9.7895e-09, 1.7859e-08, 1.5188e-05, 2.7915e-05, 1.2584e-05,\n        1.6261e-05, 5.1083e-06, 7.5614e-06, 2.6763e-07, 3.9161e-06, 2.2760e-06,\n        5.7604e-09, 3.3778e-05, 3.5035e-07, 1.2192e-05, 3.5018e-05, 1.5584e-07,\n        2.4436e-06, 3.4135e-05, 8.5208e-06, 2.6426e-05, 2.6068e-06, 5.2863e-08,\n        5.6054e-07, 3.0880e-05, 7.8967e-08, 3.8191e-17, 1.0476e-06, 2.2543e-08,\n        2.6412e-07, 6.6212e-07, 5.0764e-06, 1.1731e-05, 1.3041e-05, 1.4668e-09,\n        3.4768e-06, 2.4514e-07, 1.8413e-06, 3.2195e-09, 1.2458e-08, 5.0837e-05,\n        6.3851e-06, 9.5732e-07, 6.1195e-06, 1.4582e-07, 1.1189e-09, 3.9474e-06,\n        7.4354e-07, 1.1680e-05, 2.1765e-05, 2.3609e-06, 2.9188e-06, 1.8701e-06,\n        3.4824e-06, 6.6610e-07, 5.8816e-06, 6.9383e-07, 1.1291e-07, 2.3145e-05,\n        8.8313e-07, 1.8833e-06, 8.7727e-15, 1.4838e-08, 1.4422e-05, 5.7285e-06,\n        3.7528e-05, 1.8159e-06, 1.0544e-05, 2.5835e-06, 3.9738e-06, 8.8034e-05,\n        1.2251e-06, 9.4887e-06, 6.0755e-06, 4.8507e-06, 5.3185e-07, 2.3883e-06,\n        2.6655e-06, 4.2074e-06, 2.0790e-08, 2.5173e-06, 6.3141e-07, 5.0400e-05,\n        4.6013e-06, 3.0301e-06, 9.9046e-06, 1.2544e-05, 2.0839e-06, 3.2442e-06,\n        3.4710e-07, 6.3130e-06, 3.2889e-08, 1.5440e-08, 3.2169e-06, 5.0759e-06,\n        3.4324e-07, 1.3033e-06, 1.6576e-06, 2.2029e-05, 1.7243e-06, 1.0396e-10,\n        4.7026e-06, 8.7802e-07, 3.5323e-06, 5.2635e-06, 1.5909e-06, 2.8940e-06,\n        9.8276e-06, 5.8029e-07, 2.5229e-08, 1.5997e-06, 1.1792e-05, 1.7611e-06,\n        1.0263e-05, 3.0424e-06, 6.7804e-09, 6.1549e-08, 4.7294e-06, 8.4267e-06,\n        1.8893e-07, 4.1132e-06, 3.3799e-05, 4.3778e-06, 1.8254e-05, 6.7081e-07,\n        4.1423e-06, 8.4735e-06, 3.6044e-06, 1.6995e-06, 2.8538e-05, 8.8125e-06,\n        6.0234e-06, 5.7969e-06, 1.1630e-05, 6.0837e-11, 1.0943e-05, 3.3285e-06,\n        3.7586e-07, 1.3991e-06, 5.7702e-09, 4.1101e-06, 2.6616e-07, 1.0576e-06,\n        1.0275e-05, 5.5071e-06, 3.7250e-06, 4.0906e-09, 2.6992e-16, 3.0799e-09,\n        8.3534e-06, 2.2700e-06, 6.2404e-09, 1.3051e-08, 3.3644e-06, 1.7666e-06,\n        1.2167e-05, 4.3525e-06, 9.1940e-06, 1.4465e-06, 2.3755e-06, 7.0184e-10,\n        3.9014e-06, 3.4029e-08, 5.1870e-06, 2.7011e-06, 5.4022e-09, 4.1265e-08,\n        1.9683e-06, 3.4872e-06, 2.1086e-06, 1.7690e-06, 3.3838e-09, 7.2447e-07,\n        8.4324e-07, 3.2213e-06, 5.0629e-06, 2.9049e-06, 3.3831e-06, 1.1949e-05,\n        2.2543e-06, 5.6718e-06, 1.2789e-06, 8.2205e-07, 2.9680e-09, 1.5477e-05,\n        3.1158e-10, 2.3947e-06, 2.6832e-06, 2.4993e-06, 6.5105e-08, 1.0203e-06,\n        1.0962e-05, 7.7665e-08, 2.7368e-06, 1.2613e-07, 7.8532e-08, 2.0080e-06,\n        1.1522e-07, 5.6014e-09, 8.0050e-07, 1.1106e-07, 2.8275e-05, 5.3317e-06,\n        5.7615e-06, 6.8117e-06, 2.9477e-06, 9.4296e-06, 1.1397e-06, 4.0063e-06,\n        7.8655e-06, 1.0989e-05, 1.9478e-06, 2.6493e-06, 7.9995e-06, 8.3976e-09,\n        7.2869e-06, 1.0398e-05, 1.3317e-06, 1.3967e-05, 3.6518e-06, 1.1920e-05,\n        2.6141e-06, 5.3034e-06, 6.7520e-05, 4.4231e-06, 2.4173e-06, 1.0679e-08,\n        6.1482e-06, 7.5624e-06, 1.3259e-06, 3.4549e-06, 3.4735e-07, 8.2579e-06,\n        8.2085e-06, 2.9985e-06, 1.1818e-06, 1.1183e-05, 4.1715e-07, 3.6036e-06,\n        1.3245e-06, 8.5177e-07, 9.3521e-06, 4.5151e-06, 5.3930e-06, 8.7343e-08,\n        4.0790e-06, 2.6213e-06, 9.7652e-07, 4.8756e-07, 2.6712e-10, 4.2550e-06,\n        1.8962e-06, 1.0766e-06, 6.6679e-06, 1.0320e-05, 4.3428e-06, 8.1872e-06,\n        1.5235e-06, 1.0617e-06, 1.0776e-05, 3.1228e-05, 9.0577e-06, 1.7002e-06,\n        4.6403e-06, 1.2928e-06, 7.4892e-06, 6.5739e-06, 3.5542e-05, 1.1237e-07,\n        4.4597e-06, 2.1408e-06, 2.8802e-06, 4.9050e-05, 1.5864e-05, 1.5140e-06,\n        8.1816e-06, 5.7483e-06, 9.7043e-08, 7.0867e-05, 7.0706e-07, 5.8030e-06,\n        7.7502e-07, 6.7973e-06, 1.8975e-09, 2.1978e-05, 9.3858e-07, 2.3881e-06,\n        9.1155e-06, 9.9408e-07, 7.4750e-07, 4.4526e-08, 1.4871e-05, 2.1717e-06,\n        6.4667e-07, 1.7443e-09, 2.2929e-06, 1.8590e-06, 8.9147e-06, 2.0009e-05,\n        3.1006e-06, 8.8249e-09, 5.3805e-06, 3.0253e-06, 6.6780e-06, 5.6370e-09,\n        6.8988e-09, 5.4272e-06, 5.5881e-13, 6.4459e-06, 2.3978e-06, 5.0918e-06,\n        1.1587e-06, 9.8417e-06, 4.2385e-07, 8.7130e-10, 7.4782e-06, 9.0731e-06,\n        5.5543e-06, 2.4838e-06, 2.9560e-06, 5.5438e-07, 2.3324e-06, 1.0982e-05,\n        1.6531e-07, 8.5394e-06, 5.4014e-06, 1.2213e-06, 2.9334e-07, 8.3539e-06,\n        8.8837e-07, 2.1152e-08, 5.0797e-06, 1.0892e-06, 4.8637e-06, 1.3747e-07,\n        5.6260e-06, 1.5568e-05, 8.6679e-06, 3.9408e-09, 5.3034e-06, 4.5725e-06,\n        8.9083e-07, 5.9902e-06, 4.1320e-06, 3.1970e-06, 1.1695e-05, 6.4857e-06,\n        1.6124e-05, 3.9665e-06, 3.9097e-10, 5.8513e-06, 1.1892e-06, 1.3367e-06,\n        3.4750e-06, 1.4724e-06, 1.0265e-07, 3.1993e-09, 9.2003e-07, 2.5418e-05,\n        1.2111e-11, 4.3551e-06, 2.3199e-06, 1.7161e-06, 1.1696e-05, 1.2982e-09,\n        6.2876e-06, 3.3889e-05, 7.6548e-10, 3.4753e-06, 7.3415e-06, 8.5450e-07,\n        1.4555e-05, 1.2171e-06, 4.1200e-06, 7.9647e-06, 1.7999e-06, 3.8644e-06,\n        1.7869e-06, 1.4648e-06, 7.8012e-06, 1.1652e-06, 2.0371e-05, 2.9475e-07,\n        1.0262e-05, 2.9415e-05, 6.5699e-06, 3.5011e-05, 2.5001e-06, 2.0804e-05,\n        7.1644e-06, 1.4837e-09, 1.6807e-05, 4.6603e-07, 3.1761e-06, 1.1410e-10,\n        1.0977e-06, 7.3931e-07, 3.4925e-06, 4.4718e-07, 5.3529e-06, 5.7546e-07,\n        5.1306e-06, 1.2285e-08, 4.3652e-06, 6.3057e-06, 5.8609e-06, 2.9062e-05,\n        2.5618e-06, 5.3666e-06, 5.8160e-06, 5.1699e-08, 2.8844e-06, 1.5069e-05,\n        4.6277e-08, 1.6703e-09, 4.4050e-05, 3.2656e-08, 2.6022e-07, 2.2618e-06,\n        9.3212e-06, 5.5249e-06, 3.4869e-09, 1.1305e-05, 3.4726e-06, 1.2189e-06,\n        4.2793e-06, 5.6784e-06, 6.4796e-06, 4.4786e-06, 2.7759e-08, 4.1061e-06,\n        3.9746e-07, 2.1879e-10, 3.0849e-06, 9.8819e-07, 3.0723e-06, 1.1628e-05,\n        6.6023e-06, 3.6294e-07, 2.1752e-06, 9.0967e-07, 2.4828e-07, 8.9622e-06,\n        1.9525e-05, 1.0774e-05, 3.4450e-06, 2.9301e-06, 2.9697e-05, 1.0362e-08,\n        3.0917e-06, 8.2352e-06, 2.4926e-06, 1.2623e-05, 2.6111e-06, 2.9703e-05,\n        2.2415e-10, 1.3668e-06, 2.6213e-05, 1.1685e-06, 9.2140e-06, 4.2463e-07,\n        1.0327e-06, 8.9171e-08, 3.5471e-06, 1.3164e-05, 5.4360e-05, 4.7926e-06,\n        1.0364e-06, 2.0245e-07, 2.1811e-05, 1.7465e-05, 6.4295e-06, 5.2477e-05,\n        2.7913e-06, 2.3232e-08, 2.1540e-06, 1.8578e-06, 6.2524e-07, 5.7028e-06,\n        4.1694e-07, 3.4700e-06, 1.4467e-06, 2.0266e-06, 4.4553e-06, 3.1180e-07,\n        8.8490e-06, 2.3461e-05, 2.7787e-08, 4.2001e-15, 1.8561e-05, 1.4299e-07],\n       device='cuda:0')"
      },
      "4": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[ 6.2476e-06,  1.5620e-24,  5.6052e-45,  ...,  5.6052e-45,\n         -1.0678e-08,  3.5135e-07],\n        [ 2.0481e-05,  2.4674e-23, -5.6052e-45,  ..., -5.6052e-45,\n          1.1884e-09, -1.4128e-06],\n        [ 9.2974e-06,  4.8254e-24, -5.6052e-45,  ..., -5.6052e-45,\n         -2.5342e-08,  8.8364e-07],\n        ...,\n        [ 3.9494e-05,  1.0036e-23, -5.6052e-45,  ..., -5.6052e-45,\n          7.4303e-09,  1.3260e-05],\n        [-3.1277e-05, -1.7812e-23,  5.6052e-45,  ..., -5.6052e-45,\n          1.6134e-08,  3.6615e-06],\n        [-3.0890e-05, -6.1184e-24,  5.6052e-45,  ..., -5.6052e-45,\n         -1.5985e-08, -2.5147e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.9864e-08, 1.0276e-11, 2.3055e-09,  ..., 1.1772e-17, 1.8261e-08,\n         2.0070e-10],\n        [2.7359e-08, 1.8948e-10, 5.7807e-10,  ..., 6.0166e-16, 2.3274e-09,\n         3.5971e-10],\n        [4.3654e-08, 1.1519e-12, 1.5706e-10,  ..., 2.6240e-16, 6.9820e-11,\n         3.4333e-10],\n        ...,\n        [3.9805e-08, 1.8755e-10, 4.6976e-11,  ..., 2.2957e-16, 1.6913e-09,\n         3.4654e-10],\n        [5.4508e-08, 1.5719e-11, 2.1832e-09,  ..., 3.4213e-18, 4.7717e-09,\n         1.0817e-09],\n        [3.9547e-08, 2.4142e-11, 1.7476e-09,  ..., 2.1926e-16, 1.0526e-08,\n         8.0925e-10]], device='cuda:0')"
      }
    },
    "param_groups": [
      {
        "lr": 0.00975530705321762,
        "name": "shared",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          0,
          1
        ]
      },
      {
        "lr": 0.00975530705321762,
        "name": "scale_384",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          2,
          3,
          4
        ]
      },
      {
        "lr": 0.00975530705321762,
        "name": "scale_768",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          5,
          6,
          7
        ]
      },
      {
        "lr": 0.00975530705321762,
        "name": "scale_1024",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          8,
          9,
          10
        ]
      },
      {
        "lr": 0.00975530705321762,
        "name": "scale_1280",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          11,
          12,
          13
        ]
      },
      {
        "lr": 0.004877665762479736,
        "name": "fusion",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.005,
        "params": [
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43
        ]
      }
    ]
  },
  "scheduler_state_dict": {
    "T_0": 10,
    "T_i": 10,
    "T_mult": 2,
    "eta_min": 1e-06,
    "T_cur": 1,
    "base_lrs": [
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.005
    ],
    "last_epoch": 1,
    "_step_count": 0,
    "_is_initial": false,
    "_get_lr_called_within_step": false,
    "_last_lr": [
      0.00975530705321762,
      0.00975530705321762,
      0.00975530705321762,
      0.00975530705321762,
      0.00975530705321762,
      0.004877665762479736
    ]
  },
  "metrics": {
    "best_val_acc": 79.318,
    "best_epoch": 0,
    "scale_accuracies": {
      "384": 79.318
    }
  },
  "train_config": {
    "name": "david_training",
    "run_id": "20251012_041353",
    "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
    "model_variant": "clip_vit_l14",
    "num_classes": 1000,
    "preset": "clip_vit_l14",
    "custom_config_path": null,
    "num_classes_override": null,
    "use_belly_override": null,
    "belly_expand_override": null,
    "progressive_training_override": true,
    "scale_warmup_epochs_override": {
      "384": 0,
      "768": 1,
      "1024": 2,
      "1280": 3
    },
    "num_epochs": 20,
    "batch_size": 1024,
    "learning_rate": 0.01,
    "weight_decay": 1e-05,
    "warmup_epochs": 3,
    "use_rose_loss": true,
    "rose_initial_weight": 0.1,
    "rose_max_weight": 0.5,
    "rose_weight_schedule": "adaptive",
    "use_cayley_loss": false,
    "cayley_weight": 0.001,
    "scale_loss_balance": null,
    "use_mixed_precision": false,
    "gradient_clip": 5.0,
    "scheduler_type": "cosine_restarts",
    "min_lr": 1e-06,
    "freeze_strategy": "never",
    "freeze_threshold": 90.0,
    "unfreeze_on_plateau": true,
    "patience": 10,
    "track_gradients": true,
    "gradient_scale_threshold": 1e-07,
    "gradient_scale_multiplier": 5.0,
    "log_interval": 50,
    "val_interval": 1,
    "save_interval": 5,
    "log_fusion_weights": true,
    "log_loss_components": true,
    "save_format": "safetensors",
    "hf_repo": "AbstractPhil/gated-david",
    "upload_to_hub": true,
    "base_dir": "./david_training",
    "num_workers": 10,
    "pin_memory": true,
    "prefetch_factor": 4,
    "persistent_workers": true
  }
}