{
  "epoch": 0,
  "optimizer_state_dict": {
    "state": {
      "0": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[ 2.1732e-04, -2.4132e-04,  1.1563e-04,  ..., -5.8336e-05,\n          1.2537e-04,  1.3779e-04],\n        [ 1.7667e-04, -3.6072e-04, -1.2474e-05,  ...,  2.0924e-04,\n          2.1620e-04,  4.1487e-05],\n        [-3.0079e-04,  8.5631e-05,  7.0483e-05,  ..., -3.7193e-05,\n         -1.1348e-04, -1.5361e-04],\n        ...,\n        [-8.2470e-04,  1.0081e-03, -1.7612e-04,  ..., -2.3494e-04,\n         -3.9981e-04, -2.6347e-04],\n        [ 1.2842e-04,  3.7175e-05,  7.0022e-05,  ..., -1.4157e-04,\n         -3.5385e-05, -7.2958e-05],\n        [ 2.2581e-04, -1.7839e-04, -5.9781e-05,  ...,  2.7143e-04,\n          1.1676e-04,  8.6820e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[6.1722e-07, 9.9729e-07, 1.2621e-07,  ..., 1.0472e-07, 2.6292e-07,\n         1.6593e-07],\n        [8.7335e-07, 1.2442e-06, 1.2775e-07,  ..., 8.8972e-08, 4.3964e-07,\n         2.0230e-07],\n        [4.5250e-07, 1.2431e-06, 1.3188e-07,  ..., 1.4839e-07, 1.1970e-07,\n         1.2886e-07],\n        ...,\n        [5.4923e-07, 1.1917e-06, 1.1266e-07,  ..., 1.5080e-07, 1.6123e-07,\n         1.1640e-07],\n        [7.2196e-07, 8.9995e-07, 9.6138e-08,  ..., 1.3409e-07, 1.3516e-07,\n         8.6947e-08],\n        [3.4820e-07, 5.3155e-07, 8.0998e-08,  ..., 1.3389e-07, 1.0982e-07,\n         8.2062e-08]], device='cuda:0')"
      },
      "1": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([ 7.0912e-03,  6.0797e-03, -2.2731e-03, -1.1188e-03, -1.7205e-04,\n        -9.0995e-03, -2.1475e-03, -2.6487e-05, -1.2467e-03, -2.5011e-04,\n        -2.9646e-04, -3.2116e-04,  4.7426e-03,  7.0674e-03, -3.0869e-03,\n         3.5444e-03, -2.3375e-03,  2.5352e-03,  2.4416e-03,  1.8826e-03,\n        -4.4655e-03, -2.7397e-03, -1.9701e-03, -2.7799e-04, -5.0051e-03,\n         6.0273e-03,  9.2500e-03,  3.9571e-03,  1.8661e-03, -4.9580e-03,\n        -4.0891e-03, -1.3336e-02, -1.8346e-03, -1.0091e-03, -1.0081e-03,\n         2.2795e-03, -3.0143e-04, -1.0540e-03, -1.3617e-03,  7.0397e-03,\n        -3.7204e-03,  7.1450e-03, -6.5819e-03,  3.1580e-03, -5.3148e-03,\n        -2.6685e-03,  4.9089e-03, -3.5287e-03, -5.4789e-03,  8.7574e-05,\n        -2.4788e-03,  2.4027e-03, -1.2638e-03, -8.4531e-04, -3.2514e-03,\n        -3.0138e-03,  2.1474e-03,  2.5653e-04, -1.6644e-03,  3.1347e-03,\n        -7.8441e-03,  6.6605e-03, -8.5155e-03, -5.3620e-04, -5.2802e-03,\n        -1.6341e-04,  5.6512e-03,  4.7745e-03, -1.0558e-02, -2.7493e-03,\n         1.7077e-03, -5.8065e-03, -4.4423e-03, -5.5764e-03, -4.1995e-03,\n         3.2314e-03, -2.9026e-03, -1.7812e-03, -1.3303e-02,  5.8747e-03,\n         5.3411e-03, -2.8759e-03,  7.2511e-04, -6.4654e-03, -7.9242e-03,\n         1.3007e-02, -3.9319e-03,  6.3910e-03,  9.0574e-04, -7.7051e-04,\n        -3.2434e-04,  2.1786e-04,  3.7087e-03, -2.8836e-03, -3.6037e-03,\n        -7.8597e-03,  1.8171e-04,  8.5504e-04,  3.2590e-04, -2.9010e-04,\n         3.4104e-03,  2.4339e-03, -5.3353e-04,  1.0830e-02, -3.8072e-03,\n        -1.2378e-03,  5.0279e-03,  3.9110e-03, -2.9825e-03,  2.4288e-03,\n         3.0165e-03, -5.5090e-04,  3.5609e-03,  2.6118e-03, -4.1643e-04,\n         4.7672e-03, -3.9978e-04,  6.7410e-03, -7.2266e-03,  3.1271e-03,\n         2.0010e-03, -2.5916e-03, -9.0442e-03, -4.2935e-03, -3.3065e-03,\n        -4.8675e-04, -8.3347e-04, -7.9866e-04, -7.1479e-03,  1.8182e-03,\n         7.2755e-03,  2.2139e-03, -8.0096e-03,  6.8266e-03,  2.1929e-03,\n        -2.6154e-04,  1.7752e-03, -1.2123e-03, -3.3109e-04, -1.5690e-03,\n        -7.0882e-04, -3.8521e-03,  6.8386e-03,  2.3643e-03,  2.3162e-03,\n         2.1250e-03,  3.8087e-04,  2.1138e-03, -3.7927e-03,  5.0401e-03,\n        -2.1434e-03,  2.7185e-03,  8.8836e-03, -2.2127e-03,  7.1977e-03,\n         6.3743e-03,  9.9269e-04, -4.1267e-03,  1.0365e-03, -1.9980e-03,\n         4.8892e-03, -3.2718e-03, -1.1687e-02,  1.4169e-04, -9.5171e-04,\n        -2.7085e-03, -4.0704e-03, -2.1806e-04,  8.2463e-04, -2.4732e-03,\n        -1.7975e-03, -5.0172e-03,  1.1143e-03,  7.1312e-03, -3.8062e-03,\n        -5.0800e-03, -2.5988e-03, -5.6488e-03,  5.5590e-03, -5.1282e-04,\n        -1.3679e-03,  2.4185e-03,  1.4071e-03, -3.8554e-04, -1.4484e-03,\n         1.0441e-03,  1.4888e-03, -6.7347e-04, -9.9482e-04,  2.7858e-03,\n        -1.6250e-03,  9.5575e-03,  1.4734e-03, -6.8645e-03,  5.3222e-03,\n         9.7656e-04,  4.2416e-03, -5.1792e-04,  9.2710e-04,  5.7750e-03,\n        -7.9313e-03,  5.8195e-03, -1.4147e-03, -4.8664e-03,  2.2078e-03,\n         2.5382e-03, -1.5230e-04,  4.3010e-03, -6.6360e-04,  4.3867e-03,\n        -2.6316e-04,  4.4627e-03, -1.8241e-03, -4.5507e-03,  4.7632e-03,\n        -5.0898e-03, -6.4845e-03, -2.9958e-03,  5.4260e-03,  3.3777e-03,\n        -5.0020e-03,  5.4927e-03, -1.5026e-03, -5.9193e-03,  2.2837e-03,\n        -1.2810e-03,  2.6591e-03,  7.4449e-03, -6.0431e-03, -4.8712e-03,\n        -2.3451e-03, -1.0393e-03,  4.2036e-03,  6.1362e-03, -8.8334e-04,\n         3.3778e-03,  3.7556e-04, -1.8910e-03,  3.6084e-03,  2.9121e-03,\n         3.1919e-03, -1.4548e-03,  1.4200e-03,  7.5153e-03,  4.1686e-03,\n        -1.8493e-03,  2.3320e-03, -3.0653e-03,  5.1124e-03, -3.6398e-03,\n        -2.4768e-03,  4.5905e-04, -1.2641e-03, -5.7746e-03,  6.1046e-03,\n        -6.3901e-03, -2.5523e-04, -8.4340e-03,  5.5724e-05, -2.3499e-03,\n        -2.7910e-03, -5.7836e-03,  7.4894e-05, -3.2481e-03,  3.9726e-03,\n         9.2060e-03, -9.2005e-04,  1.4906e-05,  2.1199e-03, -1.8445e-03,\n        -1.7948e-03, -3.5234e-03,  2.0883e-03, -5.0710e-04, -2.5717e-04,\n         4.3037e-03,  4.4966e-03, -7.7924e-03,  8.6677e-03, -1.8408e-03,\n        -6.1555e-03, -7.7540e-03,  7.9262e-03, -2.6324e-03, -4.2165e-03,\n        -5.1994e-03, -3.3296e-04, -1.4055e-03,  3.8356e-03,  7.1755e-04,\n         7.0362e-03,  9.2058e-05,  2.9546e-03,  6.3822e-03, -5.6428e-03,\n         6.6616e-06,  1.9882e-03, -6.6812e-03,  8.3475e-04, -9.3955e-04,\n        -4.1878e-03, -7.2331e-04,  2.2562e-03, -4.1220e-03, -3.4375e-03,\n        -5.5896e-03, -1.4944e-03,  3.9882e-03, -4.3493e-03,  6.8208e-04,\n        -1.3324e-03, -2.2253e-03,  2.3026e-03, -2.7456e-03, -1.8855e-04,\n         3.8285e-04,  9.8479e-04,  8.9096e-04, -4.9162e-04,  2.7539e-03,\n        -3.9258e-03,  3.0915e-03, -2.2254e-03, -1.7285e-04, -3.6156e-04,\n         4.0246e-04, -1.2701e-03,  3.6221e-03, -1.6354e-03,  5.2486e-03,\n         2.8508e-04, -3.7546e-03, -2.0131e-03,  1.8658e-03, -3.4003e-03,\n        -1.1812e-04,  3.5204e-04, -3.0869e-04,  5.7886e-05, -6.8708e-04,\n        -1.8284e-03, -2.0498e-05,  7.5628e-03, -2.7515e-03,  1.6203e-03,\n        -2.8042e-03,  1.7185e-03, -2.4798e-03, -2.6479e-03,  4.7997e-03,\n        -1.9016e-03, -1.7169e-03, -2.3570e-04, -4.6680e-03,  3.2771e-03,\n        -7.5168e-03,  3.7136e-03, -1.9205e-03,  1.9161e-03,  1.9826e-03,\n        -2.6310e-03, -3.8434e-03, -1.6103e-03,  2.9747e-03,  3.0572e-03,\n         1.7101e-04, -6.1491e-03,  4.8488e-03,  9.6064e-04, -8.0626e-04,\n         3.5130e-03, -4.1602e-04, -1.7930e-03,  4.2207e-04,  1.7642e-04,\n        -4.2503e-03,  2.1579e-03,  7.2297e-03, -1.8326e-03, -1.7935e-03,\n        -5.4874e-03,  7.1566e-03,  9.0907e-04, -5.6886e-04, -3.1875e-03,\n        -1.6131e-03, -1.7008e-03, -6.5143e-03,  8.3601e-03, -7.9876e-04,\n         6.4246e-04,  1.5012e-03, -1.2257e-03,  3.5219e-03, -3.7269e-03,\n        -2.1768e-03,  2.5210e-03,  2.6302e-03,  1.0311e-03,  2.0144e-03,\n         3.4861e-03,  1.9706e-04,  1.0585e-03, -4.7431e-03, -2.8216e-03,\n         3.5322e-03, -1.5365e-03,  1.5808e-03,  3.1544e-03, -2.5791e-03,\n        -4.0344e-03, -1.3644e-03,  2.4535e-04,  1.8168e-03, -1.0563e-03,\n         4.3836e-03,  1.2257e-03,  1.5054e-03,  4.7464e-03, -5.1708e-03,\n        -4.4317e-03,  8.9603e-04,  1.6012e-03, -3.1157e-03, -2.4605e-03,\n         1.2364e-03, -2.2034e-03,  1.4938e-05,  1.2810e-03, -7.4490e-03,\n        -1.2025e-03,  2.5513e-03,  1.4516e-03,  4.1075e-03,  1.3590e-03,\n         5.7191e-04, -5.7842e-03,  8.8870e-04,  1.4588e-03,  2.1709e-03,\n        -9.8725e-04,  3.1786e-03, -3.4432e-03, -4.9858e-03,  5.2325e-04,\n        -9.2348e-04,  2.0494e-03,  4.6476e-03, -2.5771e-03, -8.4505e-04,\n         1.8759e-03,  6.2245e-03,  5.6467e-03,  3.8403e-03,  7.7850e-03,\n        -1.3934e-03, -1.0567e-02, -4.6038e-03, -9.9003e-04,  3.1682e-03,\n         5.6918e-03,  1.1739e-04,  4.0534e-03,  5.6161e-03,  5.1312e-03,\n         1.2442e-04,  4.3313e-03,  9.0238e-04, -4.8393e-03, -6.0563e-03,\n         8.0289e-03, -7.2311e-04, -4.6000e-03, -9.5827e-04,  1.3059e-03,\n         1.0847e-03, -1.5818e-03, -2.0929e-04,  4.9184e-03,  5.2542e-04,\n        -1.1754e-03, -3.4989e-03,  3.5434e-03, -1.2947e-03,  9.8844e-04,\n         2.5973e-03, -3.8419e-03,  7.2049e-03,  2.6424e-03,  3.0021e-03,\n         3.5133e-03,  3.4665e-03,  2.6124e-04,  1.3058e-02,  2.2194e-03,\n         8.3854e-03,  2.7876e-03, -4.8036e-04,  1.2251e-04, -3.9441e-03,\n        -2.4125e-03, -1.4467e-02,  1.5160e-03,  7.4386e-03, -6.7232e-04,\n        -2.3544e-04, -1.3008e-04, -3.7047e-03,  1.1277e-03, -1.1985e-02,\n         1.3106e-04,  6.7225e-03], device='cuda:0')",
        "exp_avg_sq": "tensor([2.9447e-04, 3.7317e-04, 3.0769e-04, 2.6471e-04, 2.3692e-04, 3.5314e-04,\n        2.1902e-04, 1.7906e-04, 2.1044e-04, 1.8192e-04, 2.1482e-04, 7.0766e-05,\n        2.9461e-04, 4.9959e-04, 2.0153e-04, 2.6225e-04, 1.9483e-04, 1.9536e-04,\n        2.5183e-04, 2.7501e-04, 2.1652e-04, 2.5576e-04, 3.0764e-04, 1.7653e-04,\n        4.6414e-04, 2.6101e-04, 2.4140e-04, 3.4430e-04, 1.7731e-04, 1.6433e-04,\n        2.6430e-04, 3.7979e-04, 2.1596e-04, 3.1081e-04, 3.7860e-04, 1.8742e-04,\n        3.6558e-04, 2.5975e-04, 1.4181e-04, 3.4011e-04, 2.9547e-04, 2.9818e-04,\n        2.1602e-04, 1.9651e-04, 2.8383e-04, 1.8665e-04, 2.1635e-04, 2.1205e-04,\n        1.6542e-04, 2.4786e-04, 7.4485e-05, 2.4915e-04, 2.5787e-04, 4.0114e-04,\n        1.8783e-04, 1.6829e-04, 1.8226e-04, 1.7397e-04, 1.8009e-04, 2.4305e-04,\n        2.5268e-04, 2.0510e-04, 1.9335e-04, 4.6825e-05, 2.3267e-04, 2.9784e-04,\n        3.1968e-04, 3.3130e-04, 2.9275e-04, 2.5763e-04, 3.5427e-04, 2.9196e-04,\n        2.5550e-04, 2.9405e-04, 2.2513e-04, 3.1311e-04, 2.9520e-04, 2.1316e-04,\n        2.6201e-04, 2.3898e-04, 3.5285e-04, 1.6689e-04, 1.1688e-04, 3.1497e-04,\n        2.7398e-04, 2.7224e-04, 3.9858e-04, 3.3022e-04, 3.1581e-05, 2.1734e-04,\n        2.1575e-04, 3.0422e-04, 2.3974e-04, 2.0378e-04, 1.8246e-04, 2.2434e-04,\n        2.8697e-04, 8.9937e-05, 2.6815e-04, 1.4957e-04, 2.0106e-04, 2.4685e-04,\n        2.0805e-04, 2.6720e-04, 2.6514e-04, 2.5647e-04, 2.1405e-04, 2.5724e-04,\n        2.5247e-04, 2.3343e-04, 1.7841e-04, 2.4671e-04, 2.3688e-04, 4.2637e-04,\n        1.9855e-04, 2.0871e-04, 2.7769e-04, 2.8097e-04, 1.8511e-04, 2.8667e-04,\n        2.1273e-04, 2.6418e-04, 2.1942e-04, 2.1062e-04, 1.4386e-04, 2.2347e-04,\n        3.0800e-04, 2.4416e-04, 2.4733e-04, 1.5518e-04, 2.2140e-04, 2.5184e-04,\n        2.5104e-04, 2.2683e-04, 2.1679e-04, 2.9015e-04, 2.6186e-04, 2.5157e-04,\n        2.6475e-04, 1.8260e-04, 3.2228e-04, 1.7744e-04, 2.9755e-04, 2.0920e-04,\n        2.3092e-04, 3.1963e-04, 2.3765e-04, 1.6940e-04, 3.5729e-04, 2.9008e-04,\n        2.6877e-04, 1.7373e-04, 2.9628e-04, 1.6400e-04, 2.5480e-04, 2.1375e-04,\n        2.7598e-04, 2.4039e-04, 2.9941e-04, 3.1055e-04, 4.2870e-04, 2.1161e-04,\n        2.1921e-04, 3.6805e-04, 1.3545e-04, 2.0458e-04, 3.6451e-04, 2.1763e-04,\n        2.1920e-04, 2.2399e-04, 2.1450e-04, 2.7571e-04, 2.4447e-04, 3.1175e-04,\n        1.9950e-04, 3.2276e-04, 2.2346e-04, 2.0164e-04, 2.0836e-04, 3.0490e-04,\n        5.9661e-04, 2.8911e-04, 1.5643e-04, 2.4104e-04, 3.6681e-04, 2.0591e-04,\n        3.2098e-04, 2.5081e-04, 2.1218e-04, 2.2976e-04, 2.5976e-04, 2.9883e-04,\n        2.4033e-04, 2.8229e-04, 2.7729e-04, 2.4350e-04, 2.2417e-04, 1.7924e-04,\n        2.8155e-04, 1.9334e-04, 2.9187e-04, 1.4910e-04, 1.8579e-04, 2.5475e-04,\n        1.7988e-04, 2.6229e-04, 2.7908e-04, 2.1525e-04, 2.1232e-04, 3.4535e-04,\n        2.0509e-04, 2.0875e-04, 2.6197e-04, 2.0895e-04, 3.1091e-04, 2.2663e-04,\n        3.0048e-04, 1.3750e-04, 1.4466e-04, 2.5741e-04, 2.6456e-04, 4.9971e-04,\n        2.5545e-04, 1.5469e-04, 2.6366e-04, 2.3993e-04, 1.9428e-04, 2.2426e-04,\n        2.2476e-04, 2.1300e-04, 1.8702e-04, 3.4476e-04, 1.6692e-04, 2.0065e-04,\n        1.9138e-04, 1.9048e-04, 1.9580e-04, 3.6866e-04, 2.9075e-04, 2.0295e-04,\n        2.8478e-04, 2.4457e-04, 2.2804e-04, 1.5773e-04, 2.2214e-04, 3.8855e-04,\n        2.2255e-04, 2.2339e-04, 1.1732e-04, 2.4928e-04, 2.5387e-04, 1.7706e-04,\n        2.4940e-04, 2.6220e-04, 2.5909e-04, 2.1448e-04, 2.5817e-04, 2.6417e-04,\n        2.6663e-04, 2.7899e-04, 2.5987e-04, 1.6427e-04, 2.3980e-04, 2.4583e-04,\n        2.1500e-04, 2.5805e-04, 2.3381e-04, 2.8162e-04, 2.3576e-04, 2.4813e-04,\n        2.2833e-04, 1.6486e-04, 2.6447e-04, 2.3271e-04, 2.5335e-04, 1.4849e-04,\n        1.9227e-04, 1.6196e-04, 2.6197e-04, 2.0174e-04, 2.5599e-04, 2.2340e-04,\n        2.6386e-04, 2.1278e-04, 2.4264e-04, 3.0364e-04, 2.7725e-04, 2.1635e-04,\n        2.1511e-04, 1.2012e-04, 2.2719e-04, 4.1056e-04, 3.2885e-04, 2.2142e-04,\n        2.8487e-04, 2.0930e-04, 3.6140e-04, 2.1923e-04, 2.7760e-04, 1.5732e-04,\n        2.2037e-04, 1.1846e-04, 2.0022e-04, 2.4561e-04, 2.4144e-04, 2.9667e-04,\n        2.6137e-04, 1.5714e-04, 2.7525e-04, 2.4622e-04, 1.2938e-04, 2.0910e-04,\n        2.5203e-04, 1.9712e-04, 1.8866e-04, 2.4993e-04, 1.6358e-04, 2.2596e-04,\n        1.7336e-04, 2.8070e-04, 1.8984e-04, 2.1483e-04, 3.4745e-04, 5.0253e-04,\n        2.3227e-04, 2.1675e-04, 2.4302e-04, 2.8549e-04, 1.8574e-04, 2.1854e-04,\n        2.1350e-04, 2.9515e-04, 2.6781e-04, 2.3115e-04, 2.0208e-04, 2.7666e-04,\n        1.8419e-04, 1.8013e-04, 3.1309e-04, 2.5983e-04, 2.3039e-04, 2.4238e-04,\n        2.2123e-04, 2.6165e-04, 3.0882e-04, 2.7453e-04, 2.3319e-04, 2.5678e-04,\n        2.2072e-04, 2.0204e-04, 2.3164e-04, 1.9436e-04, 1.6204e-04, 2.5961e-04,\n        3.0348e-04, 2.1758e-04, 1.6803e-04, 2.9268e-04, 2.3556e-04, 1.6266e-04,\n        2.9910e-04, 1.6006e-04, 2.9268e-04, 1.7687e-04, 2.9815e-04, 3.4582e-04,\n        2.1822e-04, 2.1002e-04, 2.6928e-04, 1.6722e-04, 4.2373e-04, 2.2255e-04,\n        2.5697e-04, 1.8589e-04, 2.8008e-04, 4.1086e-04, 2.6616e-04, 2.3654e-04,\n        2.9476e-04, 1.3487e-04, 2.1656e-04, 2.2312e-04, 4.9007e-04, 1.7495e-04,\n        1.7244e-04, 1.7352e-04, 1.9052e-04, 4.1044e-04, 2.6560e-04, 3.4738e-04,\n        2.9212e-04, 1.4712e-04, 2.8646e-04, 2.7514e-04, 2.3274e-04, 2.4003e-04,\n        3.1161e-04, 1.7938e-04, 2.0162e-04, 2.7117e-04, 2.5148e-04, 3.8533e-04,\n        2.1721e-04, 2.0932e-04, 2.0759e-04, 3.8381e-04, 2.1099e-04, 1.9811e-04,\n        2.0648e-04, 1.9183e-04, 2.6736e-04, 2.6825e-04, 2.4064e-04, 3.0336e-04,\n        2.8484e-04, 1.8200e-04, 2.9309e-04, 3.1992e-04, 1.4031e-04, 3.3981e-04,\n        1.9319e-04, 2.5011e-04, 2.1276e-04, 2.3967e-04, 2.6501e-04, 2.9038e-04,\n        2.3775e-04, 2.3574e-04, 2.3021e-04, 3.6044e-04, 1.9675e-04, 2.4050e-04,\n        1.9979e-04, 1.3166e-04, 2.2467e-04, 2.0088e-04, 2.2093e-04, 2.4106e-04,\n        2.4315e-04, 2.2263e-04, 2.8154e-04, 2.4672e-04, 3.0070e-04, 2.3575e-04,\n        3.2954e-04, 2.0570e-04, 1.9560e-04, 1.7775e-04, 2.6143e-04, 1.7553e-04,\n        2.6129e-04, 2.3124e-04, 2.8916e-04, 1.6066e-04, 3.2773e-04, 1.9670e-04,\n        2.1148e-04, 1.6714e-04, 2.3379e-04, 1.7128e-04, 2.0169e-04, 1.8986e-04,\n        1.4896e-04, 2.1704e-04, 3.1252e-04, 2.1416e-04, 3.4009e-04, 2.8807e-04,\n        3.5600e-04, 2.5140e-04, 4.1171e-04, 1.6049e-04, 1.9790e-04, 2.7214e-04,\n        2.2382e-04, 2.5065e-04, 1.2592e-04, 2.2314e-04, 3.1649e-04, 2.5426e-04,\n        3.4588e-04, 3.3344e-04, 2.5931e-04, 1.9958e-04, 2.1405e-04, 2.5943e-04,\n        3.7225e-04, 3.8034e-04, 2.4375e-04, 2.0339e-04, 2.2120e-04, 1.9557e-04,\n        2.3890e-04, 2.9015e-04, 2.8212e-04, 3.0234e-04, 1.7472e-04, 2.2272e-04,\n        2.5042e-04, 9.4893e-05, 3.0954e-04, 3.1036e-04, 1.9061e-04, 1.9346e-04,\n        2.1624e-04, 2.6488e-04, 1.9919e-04, 2.4612e-04, 2.5312e-04, 2.5250e-04,\n        2.5765e-04, 2.4213e-04], device='cuda:0')"
      },
      "2": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([ 3.4184e-03,  2.7278e-03, -2.8081e-03, -3.7930e-04, -9.7510e-05,\n        -5.8738e-03, -3.5282e-04,  8.3442e-04, -1.2154e-04, -1.4573e-05,\n        -7.2452e-04, -5.6543e-03,  2.9131e-03,  4.6260e-03, -3.5767e-03,\n         3.2509e-03, -1.7123e-03,  1.8318e-03,  2.6752e-03,  1.0468e-03,\n        -1.7199e-03, -9.7654e-04, -4.8754e-04,  3.3165e-04, -4.9488e-03,\n         3.2316e-03,  5.2018e-03,  3.1327e-03,  5.5856e-04, -6.2779e-03,\n        -2.9570e-03, -4.3141e-03, -2.1774e-03, -8.0216e-04, -1.7261e-03,\n         1.9607e-03,  6.1659e-04, -1.7345e-03, -8.7239e-04,  2.6505e-03,\n        -1.9765e-03,  3.8879e-03, -5.6384e-03,  2.5214e-03, -2.8107e-03,\n        -5.8875e-04,  3.6451e-03, -3.3639e-03, -1.0893e-02,  4.5255e-04,\n        -1.0662e-02,  2.5433e-03, -2.1356e-03, -1.4398e-03, -3.8268e-03,\n        -2.1625e-03,  1.3507e-03,  1.1380e-03, -1.7401e-03,  2.9767e-03,\n        -5.9497e-03,  4.2108e-03, -9.8701e-03, -1.4374e-02, -4.0546e-03,\n        -5.2263e-04,  2.0749e-03,  2.9863e-03, -4.8733e-03, -3.3797e-03,\n         1.5042e-04, -4.2552e-03, -2.0624e-03, -5.4418e-03, -2.4114e-03,\n         4.1112e-03, -7.0112e-04, -2.8896e-03, -8.6019e-03,  7.4896e-03,\n         2.8952e-03, -5.7825e-03,  1.1835e-03, -2.2995e-03, -6.0511e-03,\n         6.4554e-03, -2.5340e-03,  3.3304e-03,  9.3410e-24, -4.2280e-04,\n        -4.2691e-03,  1.9652e-03,  3.7844e-03, -3.1400e-03, -3.8535e-03,\n        -6.0053e-03, -1.4311e-03,  3.9327e-03,  1.1910e-03, -1.3046e-03,\n         2.9090e-03,  5.5334e-04, -1.6887e-03,  1.0477e-02, -4.0227e-03,\n        -1.0192e-03,  2.6573e-03,  1.7726e-03, -1.3745e-03,  1.7060e-03,\n         2.8598e-03, -1.4818e-03,  3.3605e-03,  7.5695e-04, -8.2858e-04,\n         3.1565e-03,  1.7958e-03,  4.4471e-03, -6.0539e-03,  1.8275e-03,\n         1.9078e-03, -9.9236e-04, -6.8147e-03, -3.6253e-03, -5.9307e-03,\n        -1.5895e-03, -6.1906e-04, -3.9996e-04, -3.5084e-03,  2.0870e-03,\n         3.9427e-03,  1.8046e-03, -9.7865e-03,  7.4494e-03,  3.2588e-03,\n         1.3857e-04,  1.2492e-03, -2.3269e-03, -5.7836e-04, -2.5859e-03,\n        -1.1721e-03, -4.7796e-03,  4.5991e-03,  4.0910e-03,  3.1960e-03,\n         1.6916e-03, -2.0877e-03,  4.2816e-04, -1.2278e-03,  5.7702e-03,\n        -2.1622e-03,  4.3127e-03,  5.3876e-03, -2.1955e-03,  4.1135e-03,\n         4.2805e-03,  4.7634e-04, -2.3241e-03,  2.9025e-04, -2.4137e-03,\n         2.1759e-03, -1.7161e-03, -1.0870e-02,  3.1825e-04, -8.1508e-04,\n        -2.3249e-03, -2.5426e-03,  8.8758e-04,  9.8898e-04, -3.9451e-03,\n        -1.0564e-03, -2.8019e-03,  1.0875e-03,  3.1644e-03, -5.1027e-03,\n        -1.8086e-03, -3.1807e-03, -6.8280e-03,  5.5181e-03, -2.6287e-03,\n        -1.6519e-03,  1.7492e-03,  8.1751e-04, -2.9324e-03, -1.3657e-03,\n         4.1177e-04,  3.4169e-04, -1.5535e-03,  7.0645e-04,  2.8915e-03,\n        -2.1121e-03,  6.9193e-03,  1.0620e-03, -5.3412e-03,  3.1544e-03,\n         1.6713e-04,  3.7178e-03,  2.4542e-04,  7.8458e-04,  3.8568e-03,\n        -3.0821e-03,  9.5915e-03, -1.5608e-03, -3.7024e-03,  2.0204e-03,\n         1.9235e-03, -1.9786e-04,  2.6449e-03, -5.6376e-04,  1.9634e-03,\n         4.2661e-04,  4.1795e-03, -1.9989e-03, -3.4974e-03,  3.4327e-03,\n        -3.0713e-03, -3.0610e-03, -4.2239e-03,  8.4635e-03,  8.4894e-04,\n        -3.4265e-03,  5.5908e-03, -6.1383e-04, -6.0085e-03, -2.0861e-03,\n        -1.0919e-03,  6.7907e-04,  5.1914e-03, -6.1674e-03, -3.4714e-03,\n        -1.8022e-03,  2.0030e-04,  4.3040e-03,  7.1921e-03, -3.2461e-05,\n         1.2077e-03, -3.2985e-04, -3.6048e-06,  3.1595e-03,  2.3927e-03,\n         2.5950e-03, -1.8790e-04,  1.1631e-03,  6.6310e-03,  4.0112e-03,\n        -2.5934e-04,  2.1013e-03, -2.1607e-03,  7.7924e-03, -1.9381e-03,\n        -1.6783e-03, -2.0762e-04, -1.1051e-03, -1.7680e-03,  3.6529e-03,\n        -8.4169e-03,  1.6156e-04, -6.3086e-03, -2.6614e-04,  7.0708e-04,\n        -3.6475e-03, -8.1781e-03,  7.7157e-04, -2.2509e-03,  3.5293e-03,\n         9.6958e-03, -2.7979e-03, -4.2822e-04,  1.7815e-03, -1.2651e-03,\n        -1.5099e-03, -3.5986e-03,  4.2769e-04, -9.5290e-04, -2.3294e-04,\n         9.9919e-03,  3.6278e-03, -6.3853e-03,  8.4500e-03, -2.1056e-03,\n        -3.9677e-03, -5.5748e-03,  1.0708e-02, -1.2806e-03, -4.4282e-03,\n        -3.9635e-03, -1.0118e-03, -1.6587e-03,  3.4588e-03,  5.7744e-04,\n         3.8342e-03, -7.2854e-04,  1.3008e-03,  3.9695e-03, -1.9321e-03,\n         6.5650e-04,  1.5987e-03, -9.8732e-03,  1.9962e-03, -1.6643e-03,\n        -2.7277e-03, -3.1865e-03,  2.3558e-03, -2.1843e-03, -2.1195e-03,\n        -3.9005e-03, -2.1801e-03,  3.3427e-03, -2.4026e-03,  2.8996e-04,\n        -2.6050e-03, -2.3256e-03,  6.1841e-04, -1.9165e-03,  3.8676e-04,\n         8.1380e-04,  2.2026e-03,  2.2067e-03, -9.1111e-04,  2.0519e-03,\n        -2.7911e-03,  2.1624e-03, -1.9052e-03,  2.6590e-04, -1.7145e-03,\n         3.5091e-04, -5.3331e-04,  1.4208e-03, -1.2433e-03,  7.0732e-03,\n         2.7985e-04, -2.8940e-03, -3.2422e-03,  1.4449e-03, -2.3747e-03,\n        -6.8386e-04,  1.1747e-03, -5.4422e-04, -4.8600e-04, -3.2378e-04,\n        -6.7249e-04, -2.1946e-04,  7.3253e-03, -1.2538e-03,  2.3842e-03,\n        -2.3401e-03, -5.3134e-05, -1.6592e-03, -1.4258e-03,  5.6559e-03,\n        -2.6921e-03, -2.3515e-03,  2.0870e-03, -2.5880e-03,  1.0508e-03,\n        -7.8608e-03,  4.5714e-03, -2.1934e-03,  6.4511e-04,  1.5744e-03,\n        -2.0696e-03, -2.0573e-03, -2.3358e-03,  3.0994e-03,  1.9378e-03,\n         7.4438e-05, -3.9586e-03,  4.2065e-03,  2.8482e-04, -1.8266e-03,\n         9.0880e-04, -9.8569e-04, -2.0896e-03,  1.1372e-03, -7.6679e-05,\n        -3.0742e-03,  6.8614e-04,  8.5280e-03, -1.0057e-03, -3.8477e-03,\n        -7.4526e-03,  4.5727e-03,  1.0802e-03, -4.7644e-05, -2.7261e-03,\n        -4.9294e-03, -4.3644e-04, -5.4936e-03,  6.1642e-03, -1.3486e-03,\n         3.3867e-04,  2.8851e-03, -7.5389e-04,  2.2820e-03, -2.3219e-03,\n         1.9179e-04,  3.7242e-03,  2.1815e-03,  5.2975e-04,  1.3971e-03,\n         2.3667e-03,  3.7373e-04,  4.2356e-04, -2.1948e-03, -3.3116e-03,\n         3.7259e-03, -4.7227e-04,  1.9993e-03,  1.3587e-03, -3.2313e-03,\n        -4.7099e-03, -1.3288e-03, -5.2207e-04, -1.1481e-04, -2.8355e-04,\n         5.9829e-03, -1.2657e-04,  8.9951e-04,  1.1720e-02, -2.6923e-03,\n        -5.7581e-03,  1.1089e-03,  1.7833e-03, -4.4792e-03, -7.0931e-04,\n         3.6109e-04, -1.5927e-03, -1.3835e-04,  8.8109e-04, -3.3068e-03,\n        -2.6530e-04,  2.1425e-03,  4.5240e-04,  1.1654e-02,  1.9390e-03,\n         1.8187e-03, -6.5379e-03,  2.2464e-03,  9.8832e-04,  4.1459e-03,\n        -9.2557e-04,  2.2297e-03, -2.1348e-03, -4.3717e-03, -2.4845e-04,\n        -1.0719e-03,  1.2796e-03,  4.5842e-03, -2.7041e-03, -2.1959e-03,\n         3.0345e-03,  3.1722e-03,  4.3686e-03,  5.7541e-03,  4.7023e-03,\n        -1.8333e-03, -8.9308e-03, -4.5921e-03, -8.9954e-04,  1.8643e-03,\n         4.4605e-03,  1.5317e-04,  5.4704e-03,  4.0192e-03,  3.7174e-03,\n        -7.3009e-05,  3.2739e-03,  1.4126e-03, -2.3728e-03, -4.6553e-03,\n         4.9287e-03, -1.8731e-04, -2.3757e-03, -1.3770e-03,  5.6579e-04,\n         2.0683e-03, -4.7602e-03,  1.3428e-03,  2.3574e-03,  1.2990e-04,\n        -4.4217e-04, -2.0124e-03,  2.9274e-03, -6.2379e-04,  8.5436e-04,\n         1.5156e-03, -1.8395e-03,  3.0566e-03, -4.1310e-04,  2.4849e-03,\n         3.2003e-03,  4.2600e-03, -1.3337e-04,  8.7607e-03,  1.2232e-03,\n         5.4495e-03,  1.4905e-03, -5.0086e-04, -8.5565e-05, -2.1486e-03,\n        -2.1281e-03, -7.0873e-03,  7.3244e-04,  8.8575e-03, -4.5805e-04,\n         6.6243e-04,  1.8176e-04, -2.2098e-03,  5.9491e-04, -7.4454e-03,\n        -3.3656e-05,  4.0627e-03], device='cuda:0')",
        "exp_avg_sq": "tensor([3.9600e-05, 6.4381e-05, 4.8072e-05, 7.3210e-05, 5.8307e-05, 5.3678e-05,\n        3.3623e-05, 4.5860e-05, 6.6022e-05, 6.5272e-05, 5.1552e-05, 1.6738e-03,\n        5.2756e-05, 6.7838e-05, 1.2087e-04, 4.4350e-05, 1.1770e-04, 1.1252e-04,\n        9.6078e-05, 4.4455e-05, 5.1236e-05, 5.0755e-05, 5.2306e-05, 2.5199e-05,\n        7.2600e-05, 3.9548e-05, 7.0392e-05, 6.6353e-05, 8.1098e-05, 1.3693e-04,\n        5.3729e-05, 5.8445e-05, 3.8029e-05, 3.3524e-05, 8.6110e-05, 5.3107e-05,\n        7.8084e-05, 1.0151e-04, 6.6539e-05, 3.3441e-05, 3.6012e-05, 6.6207e-05,\n        5.9212e-05, 6.1578e-05, 4.6035e-05, 4.0579e-05, 6.2152e-05, 1.1118e-04,\n        1.5289e-04, 3.4514e-05, 6.2806e-04, 1.8387e-04, 6.7518e-05, 4.5748e-05,\n        9.9997e-05, 6.7684e-05, 1.0945e-04, 7.7932e-05, 1.1758e-04, 1.1141e-04,\n        1.0643e-04, 4.6503e-05, 1.5542e-04, 5.3542e-03, 7.3961e-05, 8.1964e-05,\n        6.8237e-05, 4.8917e-05, 2.8945e-05, 1.1846e-04, 4.3508e-05, 9.3928e-05,\n        6.4102e-05, 6.5294e-05, 6.3835e-05, 1.5793e-04, 3.6490e-05, 1.2651e-04,\n        8.1779e-05, 1.2541e-04, 7.8363e-05, 3.5845e-04, 4.1844e-04, 6.2583e-05,\n        7.9322e-05, 4.8535e-05, 5.5934e-05, 9.8612e-05, 3.3677e-04, 1.4057e-04,\n        4.2436e-05, 1.0143e-04, 1.3029e-04, 6.4883e-05, 1.6558e-04, 5.1201e-05,\n        4.1590e-05, 6.5359e-04, 4.6331e-05, 9.3253e-05, 1.1327e-04, 3.7158e-05,\n        5.5202e-05, 1.3847e-04, 4.9189e-05, 3.4487e-05, 4.1969e-05, 3.4024e-05,\n        4.2403e-05, 7.0675e-05, 9.9015e-05, 6.3951e-05, 4.4300e-05, 4.6244e-05,\n        5.7491e-05, 6.9334e-05, 4.3085e-05, 4.1878e-05, 4.6723e-05, 4.4373e-05,\n        9.5254e-05, 4.3977e-05, 7.7592e-05, 1.1402e-04, 9.7114e-05, 3.4156e-05,\n        3.1799e-05, 2.5546e-05, 4.0581e-05, 1.3313e-04, 4.6832e-05, 7.4642e-05,\n        1.6675e-04, 1.5696e-04, 1.9662e-04, 1.2953e-04, 7.6995e-05, 7.9739e-05,\n        6.8174e-05, 1.1657e-04, 5.0580e-05, 9.7171e-05, 5.0731e-05, 9.1917e-05,\n        5.1409e-05, 6.6388e-05, 1.3097e-04, 3.2235e-05, 5.1695e-05, 7.5243e-05,\n        7.2492e-05, 1.6994e-04, 5.8420e-05, 9.5364e-05, 4.1267e-05, 5.1751e-05,\n        6.2857e-05, 3.1941e-05, 9.0395e-05, 5.2793e-05, 5.2425e-05, 4.1051e-05,\n        1.4754e-04, 5.2407e-05, 3.2314e-05, 8.6912e-05, 4.6841e-05, 6.0142e-05,\n        6.2479e-05, 1.5286e-04, 5.5130e-05, 6.3721e-05, 8.8627e-05, 6.0204e-05,\n        1.6831e-04, 6.3566e-05, 5.8835e-05, 1.4341e-04, 7.0422e-05, 1.0191e-04,\n        6.5399e-05, 7.5551e-05, 1.0321e-04, 1.9792e-04, 6.3548e-05, 3.7266e-05,\n        5.0458e-05, 1.7176e-04, 5.3855e-05, 6.5435e-05, 4.2406e-05, 7.1112e-05,\n        1.0644e-04, 4.4769e-05, 6.3218e-05, 3.7588e-05, 1.1528e-04, 5.0218e-05,\n        3.4320e-05, 4.0212e-05, 7.1630e-05, 2.3345e-04, 1.0160e-04, 5.7749e-05,\n        6.5414e-05, 7.7594e-05, 6.8500e-05, 8.7619e-05, 2.7256e-05, 3.0500e-05,\n        4.7310e-05, 1.5539e-04, 7.5878e-05, 5.2446e-05, 5.9303e-05, 5.8812e-05,\n        3.8864e-05, 8.8109e-05, 1.5847e-04, 4.4937e-05, 6.9018e-05, 6.3549e-05,\n        7.0232e-05, 1.0525e-04, 1.0612e-04, 2.1898e-05, 5.4272e-05, 6.6958e-05,\n        5.5488e-05, 4.7423e-05, 2.6731e-05, 4.2092e-05, 1.1603e-04, 1.1945e-04,\n        9.9042e-05, 4.9842e-05, 1.1139e-04, 2.1082e-04, 4.2075e-05, 3.6654e-05,\n        5.6222e-05, 2.1926e-05, 4.7523e-05, 6.8757e-05, 4.5510e-05, 6.2872e-05,\n        3.4599e-05, 8.1056e-05, 1.6585e-04, 5.0132e-05, 1.0609e-04, 8.7736e-05,\n        4.5175e-05, 6.0189e-05, 5.0129e-05, 7.2468e-05, 2.9374e-05, 1.1445e-04,\n        4.2141e-05, 6.4726e-05, 8.2706e-05, 1.1134e-04, 8.2270e-05, 3.5101e-05,\n        1.2360e-04, 1.1101e-04, 7.6243e-05, 2.6201e-05, 6.9142e-05, 8.7068e-05,\n        5.0850e-05, 5.8678e-05, 7.8472e-05, 5.6535e-05, 5.1873e-05, 3.5081e-04,\n        9.1862e-05, 8.6318e-05, 9.7763e-05, 8.8791e-05, 7.3812e-05, 6.6584e-05,\n        2.7497e-04, 7.2667e-05, 7.9735e-05, 7.9080e-05, 4.1959e-05, 7.7589e-05,\n        1.4417e-04, 2.1497e-04, 4.3115e-05, 8.6184e-05, 4.9091e-05, 2.4555e-05,\n        6.9723e-05, 4.4433e-05, 8.1440e-05, 1.4370e-04, 3.3483e-05, 1.8678e-04,\n        5.2311e-05, 3.3757e-04, 3.5898e-05, 4.0793e-05, 3.8237e-05, 7.2866e-05,\n        1.0356e-04, 9.9319e-05, 2.8778e-05, 5.0912e-05, 1.2630e-04, 6.1348e-05,\n        2.9934e-05, 7.3993e-05, 3.4409e-05, 3.8406e-05, 1.1915e-04, 1.0946e-04,\n        7.9104e-05, 7.8837e-05, 9.2321e-05, 9.0655e-05, 6.2748e-05, 8.3371e-05,\n        1.8207e-04, 2.5476e-05, 5.4799e-05, 5.5890e-05, 6.8994e-05, 1.0420e-04,\n        5.1943e-05, 1.0004e-04, 1.2829e-04, 5.1255e-05, 5.0546e-05, 4.9478e-05,\n        3.7562e-05, 6.2637e-05, 3.7131e-05, 7.6964e-05, 1.0186e-04, 7.6213e-05,\n        8.2444e-05, 3.8991e-05, 2.9221e-05, 1.1639e-04, 1.1740e-04, 4.2628e-05,\n        4.2042e-05, 1.2221e-04, 5.6198e-05, 1.1700e-04, 1.7684e-04, 3.1066e-05,\n        4.0916e-05, 1.3970e-04, 8.1107e-05, 4.8749e-05, 1.1592e-04, 4.6130e-05,\n        8.5357e-05, 3.1174e-05, 3.6626e-05, 5.8383e-05, 4.2261e-05, 1.7329e-05,\n        3.4454e-05, 1.4393e-04, 7.9968e-05, 5.0011e-05, 7.5636e-05, 8.6797e-05,\n        3.5984e-05, 1.0899e-04, 1.1936e-04, 1.3840e-04, 3.6028e-05, 1.1247e-04,\n        4.4691e-05, 1.2191e-04, 1.6792e-04, 4.7551e-05, 1.2493e-04, 1.0150e-04,\n        1.1795e-04, 1.9566e-04, 6.4463e-05, 6.6510e-05, 9.6799e-05, 6.3704e-05,\n        5.0194e-05, 4.1368e-04, 3.9959e-05, 5.9155e-05, 8.5753e-05, 6.9573e-05,\n        3.8492e-05, 1.5175e-04, 1.2351e-04, 2.3835e-05, 7.4126e-05, 5.0864e-05,\n        1.1487e-04, 3.7643e-05, 5.1128e-05, 1.2279e-04, 3.2085e-05, 9.8932e-05,\n        4.4380e-05, 1.1838e-04, 1.2962e-04, 1.0192e-04, 3.6317e-05, 5.4738e-05,\n        5.0185e-05, 9.4576e-05, 6.7493e-05, 6.2044e-05, 3.4287e-04, 8.8081e-05,\n        9.4533e-05, 1.4711e-04, 7.8249e-05, 9.6513e-05, 3.7715e-05, 1.0235e-04,\n        5.6602e-05, 1.0007e-04, 8.3447e-05, 4.5039e-05, 2.8902e-05, 7.1463e-05,\n        5.6537e-05, 5.7301e-04, 6.0649e-05, 8.6062e-05, 1.5073e-04, 6.3094e-05,\n        9.3308e-05, 1.4244e-04, 1.5843e-04, 5.0681e-05, 5.2643e-05, 6.1280e-05,\n        6.1522e-05, 1.0554e-04, 9.7306e-05, 9.9238e-05, 6.0242e-05, 1.5380e-04,\n        2.1723e-04, 3.8714e-05, 5.0054e-05, 1.3878e-04, 6.4378e-05, 1.4719e-04,\n        6.3152e-05, 1.0181e-04, 8.8848e-05, 4.8031e-05, 6.6484e-05, 1.8200e-04,\n        1.0909e-04, 7.4998e-05, 7.3979e-05, 1.2405e-04, 7.9479e-05, 1.0843e-04,\n        6.2691e-05, 8.6563e-05, 1.2653e-04, 4.7224e-05, 2.3525e-05, 4.2260e-05,\n        8.6819e-05, 5.7005e-05, 2.1489e-04, 1.0874e-04, 4.7308e-05, 5.4752e-05,\n        4.1583e-05, 8.1988e-05, 8.0489e-05, 2.4834e-05, 4.5631e-05, 7.1088e-05,\n        5.5007e-05, 3.7490e-05, 4.9179e-05, 6.5010e-05, 9.8970e-05, 6.6629e-05,\n        4.9302e-05, 8.1281e-05, 3.8743e-05, 8.2108e-05, 4.7178e-05, 5.1445e-05,\n        2.9104e-05, 1.4734e-05, 7.3063e-05, 4.4081e-05, 5.5845e-05, 1.2687e-04,\n        3.0537e-05, 5.5340e-05, 9.5501e-05, 5.7617e-05, 9.7333e-05, 7.5231e-05,\n        9.8795e-05, 7.6599e-05], device='cuda:0')"
      },
      "3": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([ 3.1904e-03,  2.4222e-03, -1.7243e-03, -2.1361e-04, -4.5648e-04,\n        -4.4914e-03, -4.0826e-04,  6.0857e-05, -1.0633e-03, -1.8701e-04,\n        -1.0094e-04, -1.0730e-03,  2.5487e-03,  3.7071e-03, -2.5732e-03,\n         1.7053e-03, -1.6072e-03,  1.2401e-03,  1.9455e-03,  9.8183e-04,\n        -2.3986e-03, -1.3021e-03, -1.2740e-03,  7.8719e-05, -2.2588e-03,\n         2.4844e-03,  4.7591e-03,  2.3537e-03,  8.7274e-04, -2.7813e-03,\n        -2.1584e-03, -5.7121e-03, -1.0514e-03, -4.9135e-04, -5.0863e-04,\n         1.0160e-03,  3.5551e-04, -8.2458e-04, -6.1719e-04,  3.0669e-03,\n        -1.7525e-03,  3.7165e-03, -3.5400e-03,  1.5227e-03, -2.3531e-03,\n        -9.3174e-04,  2.7090e-03, -2.8537e-03, -5.2754e-03,  3.5283e-04,\n        -3.1249e-03,  1.8325e-03, -8.7579e-04, -9.8658e-04, -2.1943e-03,\n        -1.6022e-03,  1.6063e-03, -1.4201e-04, -8.7773e-04,  1.5954e-03,\n        -3.4693e-03,  3.3627e-03, -5.9400e-03, -1.4886e-03, -2.8561e-03,\n         4.9963e-06,  2.6523e-03,  2.1751e-03, -4.5067e-03, -1.6471e-03,\n         6.1238e-04, -3.0726e-03, -2.2776e-03, -3.0961e-03, -2.6491e-03,\n         2.5174e-03, -1.6137e-03, -1.1758e-03, -7.0937e-03,  3.0721e-03,\n         2.4804e-03, -2.9634e-03,  7.5499e-04, -2.5208e-03, -4.3277e-03,\n         6.2486e-03, -1.4583e-03,  2.5355e-03,  6.1928e-25, -4.4319e-04,\n        -9.2588e-05,  6.1980e-04,  2.4074e-03, -1.4264e-03, -2.6702e-03,\n        -5.2027e-03,  3.2811e-04,  7.5429e-04,  4.3014e-04, -2.3400e-04,\n         2.2783e-03,  1.3519e-03, -4.6419e-04,  6.4521e-03, -2.5697e-03,\n        -3.7380e-04,  2.5154e-03,  1.7723e-03, -1.1818e-03,  1.6516e-03,\n         1.4826e-03, -5.6420e-04,  2.0438e-03,  1.1187e-03, -2.9898e-04,\n         2.6698e-03,  7.7897e-05,  3.9188e-03, -4.1652e-03,  1.3913e-03,\n         1.2945e-03, -9.8420e-04, -4.8178e-03, -2.4465e-03, -2.1221e-03,\n        -4.8332e-04, -5.2057e-04, -4.2964e-05, -3.0201e-03,  1.2106e-03,\n         4.0763e-03,  1.4035e-03, -4.7324e-03,  4.2647e-03,  1.7459e-03,\n        -1.6737e-04,  6.8106e-04, -9.5752e-04,  6.2907e-05, -8.4935e-04,\n        -2.0021e-05, -1.6985e-03,  4.1308e-03,  1.9211e-03,  2.1120e-03,\n         1.1449e-03,  7.1857e-05,  1.0149e-03, -1.5457e-03,  3.3767e-03,\n        -1.4126e-03,  2.2736e-03,  4.6065e-03, -1.1290e-03,  3.3845e-03,\n         2.9009e-03,  5.9909e-04, -1.7268e-03,  8.4962e-04, -9.9145e-04,\n         2.3474e-03, -1.6903e-03, -6.8522e-03,  1.5492e-06, -4.3454e-04,\n        -1.6407e-03, -2.2012e-03, -9.6707e-05,  6.2157e-04, -1.7944e-03,\n        -9.5820e-04, -3.2028e-03,  1.0146e-03,  3.0503e-03, -2.6575e-03,\n        -2.3976e-03, -1.2744e-03, -3.7374e-03,  3.2818e-03, -7.9136e-04,\n        -6.7694e-04,  1.3035e-03,  6.8708e-04, -7.5005e-04, -7.9721e-04,\n         3.2199e-04,  8.4443e-04, -1.0358e-03, -5.8625e-04,  2.2493e-03,\n        -1.0940e-03,  4.7334e-03,  7.9526e-04, -3.5127e-03,  2.7730e-03,\n         4.6681e-04,  2.8145e-03,  2.0011e-05,  5.2325e-04,  2.8647e-03,\n        -3.4098e-03,  4.3481e-03, -1.3079e-03, -2.6441e-03,  1.3388e-03,\n         1.3526e-03, -1.6097e-04,  2.0493e-03, -6.5562e-05,  1.9000e-03,\n         1.1330e-04,  3.2703e-03, -1.0130e-03, -2.7398e-03,  2.4459e-03,\n        -2.3422e-03, -3.1243e-03, -2.2579e-03,  3.9420e-03,  1.1702e-03,\n        -2.5664e-03,  2.7142e-03, -5.0034e-04, -3.5694e-03,  1.1598e-03,\n        -5.5907e-04,  1.2753e-03,  4.0660e-03, -2.8114e-03, -2.4260e-03,\n        -1.1035e-03,  8.5328e-05,  2.8403e-03,  4.3773e-03, -5.6567e-04,\n         1.1592e-03, -2.0618e-04, -6.2220e-04,  1.5435e-03,  1.5132e-03,\n         1.4089e-03, -7.0188e-04,  7.6053e-04,  4.6536e-03,  2.1584e-03,\n        -5.8872e-04,  1.2909e-03, -1.6966e-03,  3.5526e-03, -1.4303e-03,\n        -1.1565e-03,  5.5262e-05, -7.4679e-04, -2.2140e-03,  2.8577e-03,\n        -3.3754e-03, -7.0951e-05, -4.3856e-03, -4.9340e-05, -6.7614e-04,\n        -1.5506e-03, -3.6110e-03,  1.7703e-04, -1.5703e-03,  2.7254e-03,\n         5.3880e-03, -8.5066e-04,  8.1779e-05,  1.4173e-03, -9.7033e-04,\n        -9.8494e-04, -1.5660e-03,  9.2168e-04, -2.5521e-04, -2.5823e-04,\n         4.1380e-03,  2.6260e-03, -4.7231e-03,  5.2908e-03, -8.5965e-04,\n        -3.1564e-03, -4.0660e-03,  5.4622e-03, -1.2705e-03, -1.9472e-03,\n        -2.9671e-03, -2.9607e-04, -8.8360e-04,  1.9839e-03,  4.1254e-04,\n         3.4229e-03, -3.6705e-04,  1.1736e-03,  3.2248e-03, -2.5280e-03,\n        -6.7139e-05,  1.2895e-03, -5.3288e-03,  3.5936e-04, -6.0826e-04,\n        -2.0248e-03, -6.7692e-04,  1.4171e-03, -1.7383e-03, -1.5708e-03,\n        -3.3928e-03, -9.3082e-04,  2.1560e-03, -2.1778e-03,  5.4869e-04,\n        -1.0189e-03, -1.1810e-03,  1.3729e-03, -1.1063e-03, -2.6538e-05,\n         2.2007e-04,  1.0640e-03,  1.0277e-03, -3.3726e-04,  1.3428e-03,\n        -2.1437e-03,  1.9025e-03, -1.4138e-03, -2.9612e-04, -6.8967e-04,\n         1.9308e-04, -4.1767e-04,  1.5898e-03, -8.7928e-04,  3.3775e-03,\n         4.6539e-04, -1.9738e-03, -1.6794e-03,  1.2778e-03, -1.5326e-03,\n         1.7231e-04,  3.0655e-04, -1.3434e-04,  3.9599e-05, -3.4258e-05,\n        -7.2063e-04, -3.3310e-04,  4.8141e-03, -1.1483e-03,  6.6917e-04,\n        -1.9422e-03,  1.0457e-03, -1.0133e-03, -9.4841e-04,  2.5923e-03,\n        -1.2265e-03, -1.2267e-03,  3.9142e-04, -2.3726e-03,  1.5940e-03,\n        -5.2374e-03,  2.7246e-03, -1.0910e-03,  1.1526e-03,  1.1178e-03,\n        -1.0941e-03, -1.5307e-03, -1.0303e-03,  1.7788e-03,  1.5859e-03,\n         2.2983e-05, -3.0915e-03,  2.6508e-03,  6.0091e-04, -6.1271e-04,\n         1.8174e-03, -4.5975e-04, -1.0870e-03,  1.7214e-04, -1.6099e-04,\n        -2.1872e-03,  1.1419e-03,  4.6945e-03, -1.2438e-03, -1.4843e-03,\n        -3.4550e-03,  4.5226e-03,  7.1138e-04, -9.7154e-06, -1.8886e-03,\n        -1.9993e-03, -7.4591e-04, -3.9161e-03,  4.0808e-03, -5.0838e-04,\n         6.4355e-04,  8.6135e-04, -7.4710e-04,  2.0459e-03, -2.2105e-03,\n        -8.9224e-04,  1.0878e-03,  1.9240e-03,  4.8070e-04,  1.0789e-03,\n         1.6584e-03,  2.7715e-04,  7.2161e-04, -1.8812e-03, -1.6259e-03,\n         2.0904e-03, -6.4178e-04,  1.3211e-03,  1.5467e-03, -2.5006e-03,\n        -2.2606e-03, -3.9100e-04,  1.4261e-04,  6.3983e-04, -3.4760e-04,\n         2.9336e-03,  7.7692e-04,  8.6507e-04,  4.4178e-03, -2.9295e-03,\n        -2.5126e-03,  9.8705e-04,  9.5787e-04, -1.8633e-03, -1.3545e-03,\n         4.3228e-04, -9.5638e-04,  1.4392e-05,  3.5035e-04, -2.8349e-03,\n        -1.6866e-04,  8.5039e-04,  8.0130e-04,  4.2779e-03,  1.0363e-03,\n         6.0787e-04, -3.8142e-03,  1.2560e-03,  9.5090e-04,  2.0486e-03,\n        -5.7823e-04,  1.6073e-03, -1.0749e-03, -2.6016e-03,  3.6830e-04,\n        -9.2381e-04,  1.2310e-03,  2.7715e-03, -1.7159e-03, -8.0291e-04,\n         2.0933e-03,  3.4137e-03,  2.8626e-03,  2.5903e-03,  3.9098e-03,\n        -9.3130e-04, -6.1518e-03, -3.2061e-03, -4.6206e-04,  1.5748e-03,\n         2.9637e-03, -5.8531e-04,  2.9134e-03,  2.9580e-03,  2.9258e-03,\n        -3.8473e-05,  2.2790e-03,  4.3666e-04, -2.6951e-03, -2.7725e-03,\n         4.4092e-03, -7.0286e-04, -1.7469e-03, -2.5611e-04,  8.0352e-04,\n         6.7673e-04, -1.5469e-03,  9.2937e-06,  2.2508e-03, -9.9972e-05,\n        -2.6990e-04, -1.5148e-03,  2.0812e-03, -8.6283e-04,  3.5961e-04,\n         1.0026e-03, -1.6160e-03,  3.0064e-03,  1.1928e-03,  1.5060e-03,\n         1.7966e-03,  2.4652e-03,  3.1205e-04,  6.5354e-03,  1.1178e-03,\n         3.9190e-03,  1.1978e-03, -3.0784e-04, -3.6838e-05, -1.5580e-03,\n        -1.4839e-03, -6.2756e-03,  5.6554e-04,  4.2337e-03, -4.0199e-04,\n         4.6458e-05, -8.8214e-05, -1.4594e-03,  7.4806e-04, -6.1363e-03,\n         1.9009e-04,  3.3268e-03], device='cuda:0')",
        "exp_avg_sq": "tensor([3.2608e-05, 5.1813e-05, 3.6478e-05, 4.9759e-05, 3.1374e-05, 3.2858e-05,\n        2.3110e-05, 2.6065e-05, 3.7883e-05, 3.0333e-05, 3.0979e-05, 7.9943e-05,\n        3.7628e-05, 5.2990e-05, 5.4106e-05, 2.9069e-05, 4.4855e-05, 3.9952e-05,\n        4.9734e-05, 3.5846e-05, 2.9616e-05, 3.4745e-05, 3.7715e-05, 1.9398e-05,\n        5.5174e-05, 2.8122e-05, 3.4693e-05, 5.4598e-05, 3.5685e-05, 3.6675e-05,\n        3.3971e-05, 4.6584e-05, 2.2161e-05, 2.9817e-05, 5.2569e-05, 2.6219e-05,\n        5.0869e-05, 5.6757e-05, 2.5216e-05, 2.8672e-05, 2.7959e-05, 4.2479e-05,\n        3.1481e-05, 3.1807e-05, 3.0899e-05, 2.5489e-05, 3.3023e-05, 4.9718e-05,\n        4.8909e-05, 2.6523e-05, 5.2396e-05, 6.7479e-05, 3.6356e-05, 4.3392e-05,\n        3.5922e-05, 2.7994e-05, 4.0730e-05, 3.5743e-05, 4.1670e-05, 4.9743e-05,\n        4.1944e-05, 2.7888e-05, 5.0559e-05, 1.1012e-04, 4.3299e-05, 5.2707e-05,\n        4.4338e-05, 3.3324e-05, 2.7188e-05, 5.1808e-05, 3.5586e-05, 4.5448e-05,\n        3.5480e-05, 3.9800e-05, 3.4946e-05, 7.0627e-05, 3.3953e-05, 5.0293e-05,\n        3.8729e-05, 5.1157e-05, 3.6394e-05, 7.3038e-05, 5.8686e-05, 4.0220e-05,\n        4.3622e-05, 3.4566e-05, 4.7022e-05, 5.6091e-05, 5.0546e-06, 5.6641e-05,\n        2.2846e-05, 6.3658e-05, 4.8345e-05, 3.4071e-05, 5.2106e-05, 3.4783e-05,\n        2.9782e-05, 5.9789e-05, 3.6178e-05, 2.9054e-05, 4.7738e-05, 2.9361e-05,\n        2.6725e-05, 5.7036e-05, 3.5507e-05, 2.3917e-05, 2.8500e-05, 2.4112e-05,\n        2.8923e-05, 3.7104e-05, 3.3309e-05, 3.5095e-05, 3.6185e-05, 4.0533e-05,\n        3.2158e-05, 3.5747e-05, 3.5395e-05, 3.7217e-05, 2.5518e-05, 3.6521e-05,\n        4.3698e-05, 3.4422e-05, 3.8187e-05, 4.8232e-05, 3.3016e-05, 2.5286e-05,\n        2.8268e-05, 2.1387e-05, 2.8860e-05, 4.0245e-05, 2.7977e-05, 4.1323e-05,\n        6.1664e-05, 5.6791e-05, 6.3904e-05, 5.7324e-05, 3.8974e-05, 4.2554e-05,\n        3.9765e-05, 3.9540e-05, 3.4547e-05, 2.9966e-05, 4.2294e-05, 4.4849e-05,\n        2.9022e-05, 3.7501e-05, 5.1115e-05, 1.7921e-05, 3.4351e-05, 4.1571e-05,\n        4.1932e-05, 5.1901e-05, 4.3911e-05, 3.8497e-05, 3.1480e-05, 3.0066e-05,\n        3.7794e-05, 2.9852e-05, 4.5148e-05, 3.4660e-05, 3.5515e-05, 2.5521e-05,\n        4.9242e-05, 4.5122e-05, 1.5869e-05, 4.0670e-05, 4.7479e-05, 3.7994e-05,\n        3.4011e-05, 5.2619e-05, 2.7742e-05, 3.1974e-05, 4.4370e-05, 4.3103e-05,\n        5.7954e-05, 3.8967e-05, 2.8050e-05, 4.7580e-05, 3.4764e-05, 5.8400e-05,\n        5.3260e-05, 4.4203e-05, 3.5991e-05, 6.4280e-05, 4.9932e-05, 2.0936e-05,\n        3.9473e-05, 6.2270e-05, 3.2312e-05, 3.9472e-05, 2.5771e-05, 4.7181e-05,\n        4.3664e-05, 3.4987e-05, 3.7633e-05, 2.7576e-05, 4.9282e-05, 2.4964e-05,\n        2.5539e-05, 2.7012e-05, 4.4453e-05, 4.9104e-05, 4.0547e-05, 3.4927e-05,\n        2.9960e-05, 4.3335e-05, 3.9049e-05, 3.9952e-05, 2.0383e-05, 3.2433e-05,\n        2.6951e-05, 4.9201e-05, 3.9310e-05, 3.3337e-05, 4.0379e-05, 3.2107e-05,\n        3.3901e-05, 3.5042e-05, 4.3775e-05, 3.6807e-05, 3.2625e-05, 5.5724e-05,\n        3.5240e-05, 3.2468e-05, 4.3511e-05, 2.1100e-05, 3.0426e-05, 3.5376e-05,\n        3.0155e-05, 2.8411e-05, 1.7705e-05, 3.0617e-05, 4.3398e-05, 4.9885e-05,\n        4.5708e-05, 2.5958e-05, 4.3901e-05, 8.4682e-05, 3.2613e-05, 2.5753e-05,\n        3.1085e-05, 2.1591e-05, 2.7057e-05, 2.9445e-05, 2.7500e-05, 3.5485e-05,\n        2.5766e-05, 4.0222e-05, 3.4392e-05, 2.9452e-05, 5.2441e-05, 3.6463e-05,\n        3.2552e-05, 4.0690e-05, 3.1536e-05, 3.0109e-05, 2.4839e-05, 5.1441e-05,\n        3.7413e-05, 4.1314e-05, 4.2541e-05, 3.6868e-05, 4.4045e-05, 2.9618e-05,\n        4.3116e-05, 4.8569e-05, 3.4432e-05, 2.5326e-05, 4.0064e-05, 4.1969e-05,\n        2.9393e-05, 2.6801e-05, 4.2183e-05, 3.3915e-05, 2.9240e-05, 6.8680e-05,\n        4.3725e-05, 3.7071e-05, 4.7470e-05, 3.4592e-05, 3.5597e-05, 3.5783e-05,\n        7.6772e-05, 4.1478e-05, 4.2449e-05, 4.4364e-05, 3.4347e-05, 3.5186e-05,\n        5.1583e-05, 4.2614e-05, 2.6735e-05, 5.9264e-05, 4.3715e-05, 2.2254e-05,\n        3.5565e-05, 2.5955e-05, 4.8387e-05, 5.9097e-05, 1.7540e-05, 4.1614e-05,\n        3.0456e-05, 5.1736e-05, 2.2741e-05, 2.6639e-05, 2.9400e-05, 5.0960e-05,\n        4.7542e-05, 3.7676e-05, 2.9634e-05, 3.0276e-05, 3.5328e-05, 3.4354e-05,\n        2.6617e-05, 2.8736e-05, 2.1163e-05, 2.8214e-05, 4.9795e-05, 4.2116e-05,\n        3.3335e-05, 4.2205e-05, 3.8198e-05, 4.1428e-05, 5.1545e-05, 6.5996e-05,\n        7.0261e-05, 1.9427e-05, 3.1066e-05, 3.5937e-05, 3.4066e-05, 4.2430e-05,\n        3.0496e-05, 5.9409e-05, 6.0696e-05, 3.3119e-05, 2.2896e-05, 3.4432e-05,\n        2.5491e-05, 2.9343e-05, 3.2953e-05, 4.1691e-05, 4.4710e-05, 4.7396e-05,\n        4.2963e-05, 2.3772e-05, 3.0405e-05, 6.2058e-05, 5.2358e-05, 2.5863e-05,\n        2.7585e-05, 3.7039e-05, 3.6992e-05, 4.5142e-05, 4.5037e-05, 2.9929e-05,\n        3.3757e-05, 4.9805e-05, 2.7740e-05, 4.0557e-05, 5.2582e-05, 2.3903e-05,\n        5.4078e-05, 1.5692e-05, 2.6962e-05, 3.0930e-05, 3.7187e-05, 2.0318e-05,\n        2.5082e-05, 4.2672e-05, 4.6621e-05, 2.8251e-05, 5.3580e-05, 4.2044e-05,\n        2.7176e-05, 3.4785e-05, 5.6138e-05, 6.9396e-05, 3.1785e-05, 4.1883e-05,\n        3.1659e-05, 3.5620e-05, 5.9935e-05, 3.4060e-05, 8.8174e-05, 3.9377e-05,\n        3.6814e-05, 5.3285e-05, 2.9641e-05, 5.4779e-05, 4.1094e-05, 3.8774e-05,\n        3.9281e-05, 6.3158e-05, 2.8784e-05, 3.7206e-05, 4.1169e-05, 3.4387e-05,\n        2.5929e-05, 4.3367e-05, 4.8873e-05, 2.0964e-05, 4.4807e-05, 3.9749e-05,\n        4.5384e-05, 2.0908e-05, 2.5394e-05, 5.6192e-05, 2.5171e-05, 3.7346e-05,\n        3.0626e-05, 4.5672e-05, 4.6783e-05, 4.3141e-05, 2.6629e-05, 4.0067e-05,\n        3.5018e-05, 3.7252e-05, 4.5220e-05, 4.0715e-05, 5.8007e-05, 5.8535e-05,\n        4.1670e-05, 5.8711e-05, 3.9437e-05, 3.8816e-05, 2.8924e-05, 6.0359e-05,\n        3.6461e-05, 4.1827e-05, 3.9043e-05, 3.3750e-05, 2.1728e-05, 3.5902e-05,\n        2.2062e-05, 8.1242e-05, 3.4953e-05, 4.1082e-05, 5.2174e-05, 4.1625e-05,\n        4.5753e-05, 5.9839e-05, 6.8913e-05, 3.0984e-05, 3.7476e-05, 2.9579e-05,\n        3.5499e-05, 4.9814e-05, 4.2719e-05, 4.0008e-05, 4.2896e-05, 5.0278e-05,\n        7.9120e-05, 2.7155e-05, 3.8322e-05, 4.2573e-05, 3.7037e-05, 4.8976e-05,\n        3.0148e-05, 3.6619e-05, 3.9858e-05, 2.2120e-05, 2.9387e-05, 5.5581e-05,\n        3.6904e-05, 3.2654e-05, 5.0724e-05, 4.8904e-05, 4.3524e-05, 4.9652e-05,\n        4.8339e-05, 3.7918e-05, 7.3630e-05, 2.4555e-05, 1.6924e-05, 3.0256e-05,\n        4.0265e-05, 3.2645e-05, 3.8231e-05, 4.3055e-05, 3.7927e-05, 3.3574e-05,\n        3.2205e-05, 4.2003e-05, 4.3956e-05, 1.9243e-05, 2.6357e-05, 3.7517e-05,\n        4.3266e-05, 2.8992e-05, 3.1765e-05, 3.0849e-05, 4.1076e-05, 3.0590e-05,\n        3.0444e-05, 4.5449e-05, 2.6051e-05, 5.0734e-05, 2.5179e-05, 2.8450e-05,\n        2.5363e-05, 8.9012e-06, 4.2007e-05, 3.3325e-05, 2.9209e-05, 3.8279e-05,\n        2.3830e-05, 3.8488e-05, 4.2143e-05, 3.4694e-05, 5.2841e-05, 3.7961e-05,\n        4.8133e-05, 3.6130e-05], device='cuda:0')"
      },
      "4": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[ 4.3313e-06, -8.5848e-05, -2.3694e-05,  ...,  1.9346e-05,\n          1.4683e-05,  8.5242e-05],\n        [-1.2722e-05,  7.6283e-05,  7.3687e-05,  ..., -1.7570e-05,\n          1.7892e-05,  2.9558e-05],\n        [-7.5209e-05, -3.7764e-05,  8.0535e-06,  ...,  2.3776e-05,\n          4.4740e-06, -1.3045e-04],\n        ...,\n        [-3.9314e-06,  1.1009e-04,  2.6299e-05,  ..., -1.3002e-05,\n          1.9424e-05,  7.0476e-05],\n        [-2.0331e-05, -9.2566e-05,  2.2730e-06,  ...,  2.8280e-05,\n         -1.0114e-04,  1.5327e-05],\n        [-3.3568e-05,  1.8985e-05,  1.4551e-05,  ...,  3.5216e-05,\n         -2.1837e-06,  2.8932e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[3.7971e-08, 6.0170e-08, 6.3390e-08,  ..., 2.8713e-08, 3.3194e-08,\n         4.0259e-08],\n        [5.5826e-08, 1.1562e-07, 9.6918e-08,  ..., 4.6189e-08, 4.7870e-08,\n         7.5119e-08],\n        [6.0282e-08, 1.2140e-07, 9.2586e-08,  ..., 6.3903e-08, 3.8224e-08,\n         9.1918e-08],\n        ...,\n        [6.0442e-08, 9.4329e-08, 9.5867e-08,  ..., 5.0398e-08, 5.7388e-08,\n         8.8877e-08],\n        [5.9988e-08, 9.9616e-08, 1.0130e-07,  ..., 4.4964e-08, 7.0245e-08,\n         8.3955e-08],\n        [6.7426e-08, 9.2967e-08, 1.2939e-07,  ..., 6.5939e-08, 5.5437e-08,\n         8.4152e-08]], device='cuda:0')"
      },
      "5": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[ 1.4430e-05, -1.1706e-05, -4.2505e-06,  ...,  9.4333e-06,\n         -4.7393e-05,  4.2187e-05],\n        [-1.1030e-06,  1.7644e-05,  1.4433e-05,  ..., -9.8050e-06,\n         -2.0379e-05,  1.6822e-05],\n        [-3.5394e-05, -3.3732e-05, -2.7051e-05,  ...,  2.9086e-05,\n          2.9219e-05, -5.8495e-05],\n        ...,\n        [ 1.6078e-05, -9.0171e-05,  1.2413e-05,  ..., -2.1114e-05,\n          1.3468e-05,  1.3303e-05],\n        [-3.7386e-06,  2.5794e-05, -1.3835e-05,  ..., -3.7103e-06,\n         -4.3155e-05, -1.8257e-05],\n        [ 2.2013e-05,  3.9908e-05,  3.2269e-05,  ...,  2.0060e-05,\n         -4.2385e-05, -3.9860e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.0855e-08, 1.1875e-08, 1.7739e-08,  ..., 7.9900e-09, 6.0227e-09,\n         1.6394e-08],\n        [1.6176e-08, 2.9787e-08, 2.2523e-08,  ..., 1.2433e-08, 1.4064e-08,\n         1.9748e-08],\n        [2.3372e-08, 3.6078e-08, 2.4778e-08,  ..., 1.9851e-08, 1.1107e-08,\n         4.1721e-08],\n        ...,\n        [1.8227e-08, 2.5559e-08, 3.2876e-08,  ..., 1.1376e-08, 1.5359e-08,\n         2.1241e-08],\n        [1.9527e-08, 4.0396e-08, 2.6205e-08,  ..., 1.6501e-08, 1.6760e-08,\n         2.3153e-08],\n        [1.8697e-08, 2.5146e-08, 2.3862e-08,  ..., 1.5342e-08, 1.0780e-08,\n         2.5310e-08]], device='cuda:0')"
      },
      "6": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([ 0.0004, -0.0004], device='cuda:0')",
        "exp_avg_sq": "tensor([1.2634e-05, 1.2634e-05], device='cuda:0')"
      }
    },
    "param_groups": [
      {
        "lr": 0.00975530705321762,
        "name": "shared",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          0,
          1,
          2,
          3
        ]
      },
      {
        "lr": 0.00975530705321762,
        "name": "scale_256",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          4
        ]
      },
      {
        "lr": 0.00975530705321762,
        "name": "scale_512",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          5
        ]
      },
      {
        "lr": 0.004877665762479736,
        "name": "fusion",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.005,
        "params": [
          6
        ]
      }
    ]
  },
  "scheduler_state_dict": {
    "T_0": 10,
    "T_i": 10,
    "T_mult": 2,
    "eta_min": 1e-06,
    "T_cur": 1,
    "base_lrs": [
      0.01,
      0.01,
      0.01,
      0.005
    ],
    "last_epoch": 1,
    "_step_count": 0,
    "_is_initial": false,
    "_get_lr_called_within_step": false,
    "_last_lr": [
      0.00975530705321762,
      0.00975530705321762,
      0.00975530705321762,
      0.004877665762479736
    ]
  },
  "metrics": {
    "best_val_acc": 72.376,
    "best_epoch": 0,
    "scale_accuracies": {
      "256": 71.828,
      "512": 72.234
    }
  },
  "train_config": {
    "name": "david_training",
    "run_id": "20251012_060013",
    "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
    "model_variant": "clip_vit_b16",
    "num_classes": 1000,
    "preset": "small_fast",
    "custom_config_path": null,
    "num_classes_override": null,
    "use_belly_override": null,
    "belly_expand_override": null,
    "progressive_training_override": false,
    "scale_warmup_epochs_override": null,
    "num_epochs": 10,
    "batch_size": 1024,
    "learning_rate": 0.01,
    "weight_decay": 1e-05,
    "warmup_epochs": 0,
    "use_rose_loss": true,
    "rose_initial_weight": 0.1,
    "rose_max_weight": 0.5,
    "rose_weight_schedule": "adaptive",
    "use_cayley_loss": false,
    "cayley_weight": 0.001,
    "scale_loss_balance": null,
    "use_mixed_precision": false,
    "gradient_clip": 5.0,
    "scheduler_type": "cosine_restarts",
    "min_lr": 1e-06,
    "freeze_strategy": "never",
    "freeze_threshold": 90.0,
    "unfreeze_on_plateau": true,
    "patience": 10,
    "track_gradients": true,
    "gradient_scale_threshold": 1e-07,
    "gradient_scale_multiplier": 5.0,
    "log_interval": 50,
    "val_interval": 1,
    "save_interval": 5,
    "log_fusion_weights": true,
    "log_loss_components": true,
    "save_format": "safetensors",
    "hf_repo": "AbstractPhil/gated-david",
    "upload_to_hub": true,
    "base_dir": "./david_training",
    "num_workers": 10,
    "pin_memory": true,
    "prefetch_factor": 4,
    "persistent_workers": true
  }
}