{
  "epoch": 4,
  "optimizer_state_dict": {
    "state": {
      "0": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([[ 7.7419e-05, -7.2244e-06,  3.0531e-05,  ...,  2.5845e-05,\n          1.8728e-05,  3.9241e-06],\n        [ 3.3904e-06, -1.4326e-05,  4.1395e-06,  ...,  4.7042e-06,\n          1.7799e-05, -3.5841e-06],\n        [-2.3233e-06,  2.1983e-06,  1.2519e-05,  ..., -6.5935e-07,\n         -8.1431e-07, -1.6264e-06],\n        ...,\n        [-3.6582e-06,  4.0445e-05, -2.2387e-05,  ...,  4.7334e-05,\n         -1.9004e-06,  2.3796e-05],\n        [-5.6052e-45, -5.6052e-45,  5.6052e-45,  ..., -5.6052e-45,\n          5.6052e-45, -5.6052e-45],\n        [-3.9960e-05, -3.0237e-05,  2.3135e-05,  ..., -2.7614e-05,\n          1.2621e-05, -4.3781e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.2214e-08, 9.8888e-09, 5.6333e-09,  ..., 6.1682e-09, 6.1113e-09,\n         6.5305e-09],\n        [5.4207e-09, 5.4562e-09, 3.2772e-09,  ..., 3.2730e-09, 3.7076e-09,\n         2.4386e-09],\n        [1.1351e-08, 1.0384e-08, 1.3721e-08,  ..., 8.7800e-09, 7.4439e-09,\n         6.3112e-09],\n        ...,\n        [1.1335e-08, 1.3139e-08, 1.9069e-08,  ..., 8.3830e-09, 7.8853e-09,\n         8.1240e-09],\n        [3.1389e-13, 1.6375e-12, 4.9763e-13,  ..., 2.9173e-15, 1.5702e-12,\n         9.2530e-15],\n        [1.0876e-08, 1.0017e-08, 1.0210e-08,  ..., 8.8808e-09, 5.4898e-09,\n         4.7628e-09]], device='cuda:0')"
      },
      "1": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([ 1.5406e-03, -1.1476e-03, -2.6409e-04,  ...,  2.9145e-04,\n         5.6052e-45, -1.4334e-03], device='cuda:0')",
        "exp_avg_sq": "tensor([1.3140e-05, 7.1135e-06, 1.9650e-05,  ..., 2.1019e-05, 1.0628e-08,\n        1.4784e-05], device='cuda:0')"
      },
      "2": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([[-1.1485e-06, -5.7111e-07,  2.8125e-06,  ..., -1.6099e-06,\n          5.6052e-45, -6.3541e-06],\n        [-5.3498e-06,  5.3070e-06, -2.1827e-06,  ...,  7.7663e-06,\n         -5.6052e-45, -6.0954e-06],\n        [-4.6320e-06,  1.8319e-06, -1.0043e-06,  ...,  6.1784e-06,\n         -5.6052e-45,  5.3614e-06],\n        ...,\n        [-2.2621e-08,  2.2287e-09, -1.9373e-07,  ...,  4.6302e-07,\n          5.6052e-45,  1.4883e-07],\n        [-1.2227e-06, -9.5979e-09,  1.4337e-07,  ..., -1.5761e-08,\n         -5.6052e-45,  1.0289e-05],\n        [-2.3983e-05,  8.2098e-06, -1.4137e-05,  ..., -2.6232e-05,\n          5.6052e-45, -6.9302e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.3307e-09, 3.9215e-10, 2.4352e-09,  ..., 9.7499e-09, 2.5026e-13,\n         1.1423e-09],\n        [3.0932e-09, 1.1406e-09, 8.4448e-10,  ..., 6.5127e-10, 1.3886e-11,\n         4.0199e-09],\n        [8.5765e-10, 8.2466e-10, 8.1234e-10,  ..., 9.9855e-10, 9.5692e-11,\n         2.0900e-09],\n        ...,\n        [4.1790e-11, 2.6058e-11, 3.9500e-10,  ..., 2.8262e-10, 9.6092e-12,\n         2.8260e-10],\n        [1.6500e-09, 2.2122e-11, 4.5411e-10,  ..., 3.4966e-10, 1.9384e-14,\n         1.0797e-09],\n        [4.3279e-09, 9.7757e-10, 1.0829e-08,  ..., 2.3307e-09, 1.7174e-12,\n         3.1455e-09]], device='cuda:0')"
      },
      "3": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([ 1.0477e-04,  1.1376e-04,  6.8580e-05,  1.1062e-04, -1.5102e-04,\n         1.8946e-05,  2.2321e-04,  4.4966e-04, -2.1635e-04,  3.6279e-04,\n        -4.8620e-05, -1.1527e-06,  7.3181e-05,  9.5280e-05,  1.7739e-04,\n         1.6335e-04, -1.1370e-04, -1.1923e-05,  2.4393e-05,  1.0780e-04,\n        -9.4480e-05,  2.5872e-04,  4.0348e-04, -1.9958e-04, -8.5265e-05,\n        -1.9338e-04,  1.9970e-04, -8.4242e-05,  2.3576e-05, -3.3020e-05,\n        -3.9298e-05, -7.7467e-06, -3.9194e-04, -9.2478e-05,  5.7959e-05,\n         5.9237e-05,  4.3912e-04, -1.0686e-04,  1.5071e-04,  8.6443e-06,\n         7.1097e-04,  1.8752e-04,  2.1190e-05,  4.4484e-05,  4.0833e-05,\n         9.2181e-05,  1.0533e-18,  7.5112e-05,  2.1147e-05,  1.4307e-04,\n         4.7949e-05, -1.8811e-05,  3.2247e-05, -2.0407e-06, -4.8476e-05,\n        -2.7448e-04, -2.3936e-04, -9.3065e-05,  4.3918e-05,  8.5998e-05,\n         1.7864e-05, -1.4735e-04, -9.0984e-05, -2.1071e-06,  1.3226e-04,\n         5.5162e-05, -2.7692e-05,  1.8201e-04,  9.0112e-05, -4.0201e-04,\n         1.3602e-04,  5.6052e-45,  6.4871e-06,  2.7337e-04, -1.0230e-04,\n        -1.5436e-04, -4.0674e-05,  5.9901e-05, -1.7824e-04,  1.3852e-04,\n         4.4658e-05, -1.3218e-04, -1.4014e-04,  1.7143e-05, -6.1319e-05,\n        -1.4133e-04, -3.2296e-04,  3.2819e-04, -1.4037e-04, -3.6881e-04,\n         6.7872e-06,  2.7025e-04, -5.5208e-04, -6.8871e-05,  3.1423e-05,\n         1.5790e-04,  1.0097e-05,  1.3257e-04, -2.4127e-04,  3.0916e-05,\n         1.5010e-05, -2.2331e-04, -7.9740e-05, -2.9505e-06, -1.0984e-05,\n        -1.8364e-06, -4.5352e-05, -1.1953e-04,  1.7912e-05, -2.8878e-04,\n         3.9155e-05,  1.4545e-04, -4.1241e-06, -7.3574e-05,  4.0110e-04,\n         5.3793e-05,  1.5396e-04, -2.4731e-04, -3.7000e-05,  1.9019e-04,\n        -1.4076e-04,  1.2596e-04, -3.3846e-04,  2.6942e-04, -2.5929e-04,\n        -2.7995e-04,  7.5528e-06,  2.9249e-05, -3.6848e-04, -1.2839e-04,\n        -7.5530e-05,  1.4641e-04,  1.3239e-05, -7.3279e-05, -1.5624e-04,\n         2.0339e-04,  1.8814e-04,  8.7494e-05, -1.8627e-04,  3.8127e-04,\n         9.6811e-05, -2.5912e-05,  8.9033e-05,  2.4748e-05,  1.6964e-04,\n        -1.7393e-04, -2.3904e-05, -1.5047e-04,  1.1144e-04,  2.8452e-05,\n         1.1765e-06,  4.0289e-05,  1.1107e-04, -6.5209e-05,  2.4792e-05,\n         5.6931e-13,  7.5780e-05,  1.4192e-04,  7.0784e-05, -4.5672e-04,\n         1.4378e-04,  7.7271e-06, -2.6640e-04,  8.8398e-05, -2.3331e-04,\n         1.5183e-05,  3.1811e-04, -1.2715e-05,  1.1393e-05, -2.7617e-04,\n        -2.7581e-05,  8.1515e-05,  1.2959e-07, -4.6871e-04,  4.8529e-04,\n        -3.3767e-04, -2.5262e-04,  1.7203e-04, -1.8694e-04,  1.1845e-04,\n         8.9294e-05,  2.0866e-04,  1.2850e-04,  8.5298e-06,  1.4894e-04,\n        -9.5003e-05,  1.9936e-04, -1.3296e-04, -4.4874e-05,  2.0801e-04,\n         1.2607e-04, -1.7662e-06,  9.1341e-05, -1.8251e-04, -2.0454e-04,\n         4.9065e-05, -2.9016e-05,  2.0897e-04,  5.6052e-45,  9.3667e-05,\n        -2.2721e-06,  2.3989e-04,  2.0541e-04,  4.6781e-06,  1.2744e-04,\n        -4.4005e-05, -4.8953e-06, -2.6030e-04,  1.9106e-04,  9.8366e-05,\n         4.9292e-05, -1.3986e-04,  1.2331e-42, -3.2295e-05, -4.8253e-05,\n        -1.3859e-04,  5.3263e-04,  6.0121e-05, -2.2526e-04, -8.2499e-05,\n         4.7832e-05,  2.7444e-05, -1.1168e-04,  1.5333e-04, -8.4024e-06,\n         3.1697e-04, -2.3630e-04, -2.1602e-04, -1.5837e-04, -5.0515e-04,\n        -1.1856e-04,  4.3696e-05,  5.7240e-05, -1.5466e-04, -2.6130e-04,\n        -2.7558e-05, -5.1042e-05, -5.5390e-05, -1.4133e-04,  1.0975e-05,\n        -4.0222e-04,  1.2004e-04, -8.3381e-05, -1.3901e-04, -4.6265e-05,\n         1.5251e-04,  2.1482e-05, -3.0831e-04,  4.1425e-05, -9.0239e-05,\n         1.5997e-04, -1.0658e-04,  1.9397e-04, -8.7239e-05, -8.7222e-05,\n        -1.9878e-04,  1.4545e-05, -4.9886e-05,  1.6795e-04, -1.6524e-05,\n        -1.7034e-04, -4.8796e-04,  8.6639e-05,  1.0742e-04,  1.3301e-04,\n        -7.4432e-05, -3.7230e-04,  8.4317e-05, -2.9158e-04,  1.5610e-04,\n        -1.6772e-04,  8.4969e-05, -1.0371e-04, -3.1074e-06,  2.4064e-04,\n        -1.6916e-04,  2.6740e-05,  1.1810e-05,  1.1866e-05, -1.3244e-05,\n        -1.0962e-04,  6.9882e-05, -3.3924e-04,  1.7443e-04, -5.3820e-05,\n         6.3928e-05,  6.3257e-05,  9.3041e-05,  1.6266e-04, -3.9549e-04,\n        -1.3156e-05, -2.1692e-04, -1.1161e-04, -6.8772e-05,  2.7330e-10,\n         1.4159e-04,  6.2701e-05,  3.7193e-04,  2.4729e-06,  1.1071e-04,\n         1.1206e-04, -6.4637e-05,  2.0143e-04,  3.8784e-06, -7.4142e-05,\n         6.1758e-05, -4.0520e-04, -8.6071e-05,  1.0703e-04,  2.0677e-05,\n        -2.2336e-05,  2.3289e-05, -8.7161e-05, -1.1777e-04, -2.6742e-04,\n         4.0852e-05, -3.2601e-04, -1.7674e-04, -7.0380e-05,  2.6905e-04,\n        -2.4768e-04,  5.2272e-05,  1.4922e-04,  1.9704e-04, -1.0745e-04,\n         9.4971e-05,  3.7686e-05,  1.2015e-04, -4.0044e-04,  2.5644e-04,\n         4.8441e-05, -1.9149e-04, -3.9381e-05,  5.9652e-05,  1.1766e-04,\n         3.9545e-05,  1.2883e-04,  1.3951e-04,  8.3888e-05,  1.5828e-05,\n         1.7781e-04, -1.2960e-04,  7.2166e-05,  1.9132e-06, -1.0379e-04,\n        -2.5532e-04,  1.9610e-05,  5.7568e-06, -9.5458e-05,  2.9811e-04,\n        -4.2334e-05, -1.7680e-04,  4.2791e-05,  2.5948e-04,  7.9276e-05,\n         2.5633e-05,  3.0468e-05,  2.0320e-04,  8.5785e-05, -1.1651e-04,\n        -5.4273e-05, -7.3785e-04,  1.9157e-04, -3.3559e-04, -2.2174e-05,\n         6.2231e-05,  5.6052e-45, -1.7726e-05, -1.7570e-04,  2.0834e-04,\n        -7.0150e-05,  1.2011e-04,  2.3193e-05,  2.5905e-04,  4.8156e-05,\n         9.4227e-05, -1.5386e-04, -3.2985e-04, -8.4540e-09,  1.3336e-04,\n         3.2558e-11,  7.8638e-05,  1.3845e-04,  2.0236e-04,  2.7482e-05,\n         1.9566e-04,  7.6982e-05,  7.3572e-05,  1.9124e-05, -4.4878e-04,\n        -7.3165e-05,  4.0464e-05,  8.3604e-05,  2.8651e-05, -1.3938e-05,\n        -7.2535e-05, -2.8282e-05, -3.0575e-04,  9.7689e-05,  1.9464e-04,\n         3.7703e-05, -6.0119e-05, -2.4153e-04,  8.2015e-05,  1.4069e-04,\n         8.2936e-05,  5.2972e-05, -3.1064e-05,  1.0269e-04,  2.1319e-05,\n         2.9697e-05,  5.1293e-05, -1.4735e-04, -1.1027e-04, -1.0017e-05,\n         3.7506e-05,  1.9099e-04, -2.7635e-04, -1.7746e-04, -1.0333e-05,\n        -1.1964e-06, -7.3104e-05, -2.7289e-04,  1.2494e-05, -1.1267e-04,\n         9.3180e-05,  8.4389e-05,  1.2577e-04, -3.1900e-05,  6.9829e-05,\n         4.4992e-04, -2.2451e-05, -6.2085e-05,  1.0990e-04,  1.0630e-04,\n         2.3869e-04, -1.3656e-04, -8.4981e-05, -1.8713e-04,  4.7313e-05,\n         3.7676e-04,  1.2203e-04,  1.5593e-04,  7.6599e-05, -4.5713e-04,\n         3.9790e-04,  5.6052e-45, -1.1085e-04,  6.5030e-05,  2.4468e-04,\n        -4.8410e-05, -1.5028e-04,  6.5386e-05,  8.2005e-05,  1.5086e-05,\n         2.5292e-04,  7.3439e-05, -6.3366e-05,  2.4668e-04, -1.3422e-04,\n        -1.8001e-04, -6.6304e-05, -6.5024e-06,  6.7363e-05, -9.5922e-05,\n        -3.2134e-05,  1.9623e-04, -3.5708e-05, -1.3757e-04, -1.3367e-04,\n        -1.0616e-05,  4.4708e-05,  5.9554e-06,  4.1430e-05,  5.7943e-05,\n         3.9001e-05, -1.1295e-04,  5.3587e-05,  1.4498e-04,  1.8632e-04,\n        -3.6238e-05,  1.3988e-04,  1.4748e-04, -2.5542e-04, -1.4115e-04,\n        -7.5265e-05,  1.2344e-04, -1.2543e-04,  1.4823e-04,  3.9968e-04,\n         3.4148e-04, -8.2524e-05,  2.1892e-04,  2.9281e-04,  1.4586e-04,\n         5.6052e-45, -1.4999e-04, -7.8472e-06, -3.9977e-05,  1.2903e-04,\n         8.7892e-06, -5.1465e-05,  1.0537e-04,  5.8208e-05,  1.5826e-04,\n        -1.4232e-04, -1.7625e-05,  8.3911e-05, -2.5242e-05,  3.7977e-05,\n        -1.4690e-04, -4.8904e-05], device='cuda:0')",
        "exp_avg_sq": "tensor([3.4901e-07, 4.9780e-07, 4.5792e-07, 4.8573e-07, 3.5745e-07, 4.6700e-07,\n        2.1051e-07, 5.0464e-07, 3.7209e-07, 2.4541e-07, 7.1930e-07, 3.6780e-07,\n        1.9960e-07, 3.7045e-07, 9.0267e-08, 4.4603e-07, 6.2253e-07, 5.7818e-07,\n        2.6456e-07, 3.0200e-07, 1.0456e-07, 3.3981e-07, 7.9043e-07, 2.8181e-07,\n        4.2366e-07, 4.3592e-07, 5.2335e-07, 5.1445e-07, 1.7712e-07, 1.8183e-07,\n        5.1255e-07, 6.2132e-07, 6.1939e-07, 7.1401e-07, 1.9657e-07, 3.3129e-07,\n        3.2808e-07, 6.1111e-07, 7.0171e-07, 3.2175e-07, 6.3079e-07, 7.1098e-07,\n        5.5448e-07, 2.0753e-07, 5.6254e-07, 5.8143e-07, 4.6728e-07, 2.4432e-07,\n        6.2282e-07, 3.0151e-07, 3.3118e-07, 2.1630e-07, 4.8689e-07, 3.5626e-07,\n        3.2663e-07, 5.7372e-07, 4.4594e-07, 3.6973e-07, 4.8599e-07, 5.2061e-07,\n        3.8244e-07, 5.1999e-07, 4.0964e-07, 5.2100e-07, 3.4740e-07, 1.8908e-07,\n        3.9346e-07, 4.1781e-07, 3.3507e-07, 2.3654e-07, 4.3065e-07, 2.2398e-15,\n        3.7701e-07, 4.1233e-07, 2.1751e-07, 3.8878e-07, 5.4075e-07, 5.4777e-07,\n        5.2886e-07, 4.0562e-07, 2.7422e-07, 4.5003e-07, 2.8292e-07, 4.2921e-07,\n        3.4534e-07, 2.1345e-07, 4.0643e-07, 7.4621e-07, 2.4236e-07, 5.3800e-07,\n        3.2723e-07, 5.9020e-07, 4.1647e-07, 6.1475e-07, 1.6163e-07, 6.0077e-07,\n        6.0399e-07, 6.7833e-07, 8.2945e-07, 3.5063e-07, 5.0820e-07, 4.1673e-07,\n        4.1482e-07, 7.0023e-07, 2.7133e-07, 3.6670e-07, 4.9629e-07, 3.1619e-07,\n        2.9640e-07, 2.3822e-07, 7.6810e-08, 4.8421e-07, 2.2558e-07, 5.3604e-07,\n        4.7887e-07, 5.1480e-07, 3.2878e-07, 6.5371e-07, 2.9507e-07, 2.9290e-07,\n        4.7715e-07, 4.1540e-07, 6.0668e-07, 4.6661e-07, 4.9506e-07, 2.3042e-07,\n        5.2414e-07, 3.8682e-07, 3.0927e-07, 2.1464e-07, 1.5432e-07, 2.3888e-07,\n        3.0761e-07, 1.9994e-07, 2.6545e-07, 3.9507e-07, 6.6907e-07, 4.2116e-07,\n        4.2793e-07, 5.7846e-07, 6.4881e-07, 4.4181e-07, 1.0540e-07, 7.9226e-07,\n        4.4806e-07, 4.6135e-07, 6.6322e-07, 3.3756e-07, 3.0772e-07, 6.5047e-07,\n        5.1120e-07, 4.5501e-07, 2.2361e-07, 7.9076e-07, 3.2080e-07, 1.0430e-09,\n        3.8042e-07, 4.0912e-07, 2.3389e-07, 2.9209e-07, 2.1960e-07, 3.2842e-07,\n        3.3549e-07, 4.0562e-07, 4.4690e-07, 2.8572e-07, 5.8275e-07, 5.0191e-07,\n        5.4766e-07, 2.8384e-07, 4.5149e-07, 2.3664e-07, 1.0455e-08, 6.0392e-07,\n        4.0145e-07, 2.8710e-07, 5.1689e-07, 4.0183e-07, 2.7466e-07, 1.9466e-07,\n        2.4865e-07, 2.5448e-07, 3.2309e-07, 1.9651e-07, 3.2357e-07, 8.9897e-07,\n        3.8397e-07, 4.2723e-07, 5.3729e-07, 4.8969e-07, 6.5440e-07, 5.0850e-07,\n        5.9515e-07, 5.0035e-07, 7.4258e-07, 5.0683e-07, 5.4641e-07, 2.0189e-07,\n        1.6596e-16, 6.6499e-07, 5.5023e-07, 3.1932e-07, 3.6105e-07, 6.1298e-07,\n        3.3548e-07, 3.3084e-07, 3.1938e-07, 6.1478e-07, 1.6463e-07, 1.2872e-07,\n        4.9612e-07, 5.8799e-07, 3.3898e-07, 4.0943e-07, 3.1607e-07, 4.3309e-07,\n        4.1719e-07, 1.7738e-07, 4.9472e-07, 5.5189e-07, 3.2298e-07, 2.5237e-07,\n        3.2492e-07, 4.5085e-07, 5.4742e-07, 5.8879e-07, 6.1076e-07, 8.3486e-07,\n        4.9658e-07, 2.4045e-07, 3.0293e-07, 4.7960e-07, 4.8984e-07, 4.0100e-07,\n        4.7781e-07, 2.6311e-07, 5.0894e-07, 3.5116e-07, 4.7168e-07, 8.5078e-07,\n        2.8361e-07, 5.2249e-07, 2.8813e-07, 3.2718e-07, 1.8626e-07, 2.5006e-07,\n        3.4277e-07, 3.4120e-07, 3.2144e-07, 5.0437e-07, 3.3093e-07, 5.0051e-07,\n        4.5540e-07, 6.1511e-07, 5.1284e-07, 4.3680e-07, 3.3738e-07, 3.0149e-07,\n        4.7970e-07, 2.5696e-07, 4.6504e-07, 3.9066e-07, 4.6030e-07, 3.8832e-07,\n        9.1776e-08, 2.8071e-07, 6.1818e-07, 4.4937e-07, 7.0595e-07, 4.4950e-07,\n        4.6531e-07, 4.7784e-07, 2.8059e-07, 7.4658e-07, 1.8031e-07, 2.5060e-07,\n        3.1726e-07, 6.5142e-07, 3.3772e-07, 1.8310e-07, 4.6568e-07, 4.4181e-07,\n        4.2281e-07, 3.6107e-07, 5.1441e-07, 5.3050e-07, 1.2671e-07, 4.3878e-07,\n        2.9601e-07, 5.5376e-07, 6.0354e-07, 5.0992e-07, 7.5289e-07, 2.2094e-07,\n        6.4931e-08, 5.4879e-07, 3.6594e-07, 3.7682e-07, 5.2444e-07, 3.6397e-07,\n        1.9318e-07, 2.9806e-07, 5.1638e-07, 6.3940e-07, 4.3617e-07, 3.0835e-07,\n        3.4528e-07, 5.7015e-07, 2.6914e-07, 4.9114e-07, 3.9302e-07, 2.3057e-07,\n        2.5393e-07, 3.7498e-07, 6.3723e-07, 4.4607e-07, 2.0030e-07, 5.8750e-07,\n        2.1584e-07, 4.1554e-07, 4.2400e-07, 3.4224e-07, 8.1866e-07, 2.9809e-07,\n        2.0921e-07, 4.7519e-07, 4.5442e-07, 2.4226e-07, 6.2201e-07, 4.0839e-07,\n        5.8200e-07, 6.4224e-07, 4.7172e-07, 2.5576e-07, 5.0983e-07, 5.0638e-07,\n        4.9579e-07, 2.4395e-07, 1.0644e-07, 4.3306e-07, 4.6720e-07, 3.9897e-07,\n        4.4289e-07, 8.2057e-09, 3.1663e-07, 9.4660e-07, 2.1463e-07, 1.7159e-07,\n        2.9124e-07, 5.2223e-07, 3.1503e-07, 5.6536e-07, 3.6594e-07, 4.0489e-07,\n        6.6517e-07, 2.4098e-07, 2.2393e-07, 4.5338e-07, 4.4010e-07, 3.3143e-07,\n        4.5810e-07, 2.1017e-07, 4.6123e-07, 6.2484e-07, 4.4140e-07, 4.5624e-07,\n        1.4758e-11, 5.0992e-07, 2.3912e-07, 5.1579e-07, 3.7319e-07, 5.0737e-07,\n        5.3290e-07, 3.6051e-07, 2.2441e-07, 3.5299e-07, 2.4764e-07, 3.4130e-07,\n        3.6462e-08, 2.5128e-07, 5.4420e-09, 4.0580e-07, 6.0589e-07, 5.6025e-07,\n        4.2791e-07, 5.6832e-07, 4.4755e-07, 1.6124e-07, 3.6226e-07, 6.3319e-07,\n        3.8873e-07, 4.2324e-07, 5.7647e-07, 5.2951e-07, 2.5311e-07, 2.2406e-07,\n        2.7735e-07, 4.2267e-07, 2.0156e-07, 6.0279e-07, 5.0002e-07, 2.6103e-07,\n        6.4814e-07, 3.5559e-07, 7.1142e-07, 4.5199e-07, 4.3067e-07, 7.7637e-07,\n        3.8712e-07, 4.8204e-07, 6.2978e-07, 3.1425e-07, 5.8953e-07, 6.5175e-07,\n        4.2345e-07, 2.8833e-07, 6.9242e-07, 3.0576e-07, 2.9983e-07, 2.4886e-07,\n        2.5190e-07, 3.7736e-07, 2.8962e-07, 2.9376e-07, 7.8339e-07, 3.6087e-07,\n        2.0723e-07, 4.6125e-07, 2.2941e-07, 7.7926e-07, 6.1360e-07, 5.5141e-07,\n        2.8900e-07, 3.6297e-07, 2.6125e-07, 2.0251e-07, 4.6401e-07, 1.6262e-07,\n        1.8248e-07, 3.6406e-07, 5.5654e-07, 4.4869e-07, 2.0911e-07, 2.9894e-07,\n        7.4050e-07, 3.9037e-07, 2.1039e-13, 3.3572e-07, 1.9306e-07, 6.3397e-07,\n        5.2783e-07, 1.8366e-07, 3.5176e-07, 4.9930e-07, 6.5063e-07, 4.7547e-07,\n        1.8812e-07, 3.3188e-07, 7.3594e-08, 4.3111e-07, 4.8595e-07, 2.3549e-07,\n        2.9108e-07, 5.8303e-07, 6.6009e-07, 1.8412e-07, 3.2794e-07, 5.4452e-07,\n        2.2323e-07, 5.0946e-07, 8.1826e-07, 2.7505e-07, 5.3600e-07, 4.5628e-07,\n        3.8038e-07, 2.9003e-07, 3.7763e-07, 4.8779e-07, 2.2477e-07, 3.1693e-07,\n        6.9090e-07, 3.7828e-07, 3.1496e-07, 3.8370e-07, 4.8291e-07, 5.3087e-07,\n        1.3978e-07, 5.1581e-07, 4.9655e-07, 6.9022e-07, 7.8373e-07, 4.8621e-07,\n        3.0570e-07, 6.3252e-07, 3.7424e-07, 5.7693e-18, 4.0560e-07, 7.3005e-07,\n        2.8521e-07, 5.1881e-07, 4.3237e-07, 5.1581e-07, 3.0775e-07, 5.5621e-07,\n        3.6184e-07, 9.2218e-07, 1.2871e-07, 4.3169e-07, 1.9843e-07, 1.0242e-07,\n        2.5739e-07, 4.5693e-07], device='cuda:0')"
      },
      "4": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([[ 1.8051e-06,  7.7717e-06, -1.1762e-05,  ..., -1.7779e-06,\n         -1.6077e-06, -7.2762e-06],\n        [ 3.8044e-06, -8.7437e-06, -2.4373e-05,  ...,  1.3913e-06,\n          9.0766e-08, -6.5949e-06],\n        [ 6.6412e-06, -1.3297e-05,  4.6889e-05,  ...,  2.4836e-07,\n          1.4580e-05,  6.9022e-06],\n        ...,\n        [ 8.5893e-05,  2.2444e-06, -3.4216e-05,  ..., -1.5663e-06,\n          1.0801e-05, -4.8861e-06],\n        [-5.1267e-06, -1.1652e-05, -2.3581e-05,  ..., -1.0031e-07,\n          8.5159e-07,  1.2613e-05],\n        [ 4.8896e-05, -1.6472e-05, -3.4303e-05,  ...,  9.9761e-07,\n         -1.0900e-05, -1.6162e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.1308e-09, 1.6604e-09, 1.8253e-09,  ..., 9.7495e-11, 6.8655e-10,\n         4.2496e-09],\n        [8.3453e-09, 5.4105e-09, 6.6297e-09,  ..., 1.1778e-10, 1.0796e-09,\n         8.1224e-09],\n        [5.3479e-09, 4.2595e-09, 3.1662e-09,  ..., 2.6425e-10, 1.9064e-09,\n         4.0530e-09],\n        ...,\n        [3.1641e-08, 3.7727e-09, 7.7409e-09,  ..., 1.6950e-10, 3.6582e-09,\n         7.5386e-09],\n        [7.7039e-09, 3.5320e-09, 6.5373e-09,  ..., 1.2939e-10, 6.4699e-09,\n         9.9044e-09],\n        [9.4282e-09, 5.2435e-09, 9.9640e-09,  ..., 2.3744e-10, 1.0114e-09,\n         1.0140e-08]], device='cuda:0')"
      },
      "5": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[ 2.9052e-07, -2.2928e-08, -4.3634e-08,  ..., -6.0430e-07,\n          0.0000e+00,  9.5430e-07],\n        [-2.2505e-07,  4.2085e-07,  5.5051e-07,  ...,  1.4380e-06,\n          0.0000e+00, -1.6246e-08],\n        [-1.1071e-06, -1.2538e-07,  1.2875e-07,  ..., -3.5246e-06,\n          0.0000e+00, -2.3430e-06],\n        ...,\n        [ 7.2477e-07,  7.6364e-07,  3.4231e-07,  ..., -9.2964e-08,\n          0.0000e+00, -4.8257e-07],\n        [-8.8239e-08,  1.4731e-07,  1.9609e-06,  ...,  1.0204e-06,\n          0.0000e+00,  1.1838e-06],\n        [-8.0134e-07, -1.5626e-06, -5.1318e-07,  ..., -3.0431e-06,\n          0.0000e+00,  1.0494e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.0832e-11, 7.7793e-12, 1.9464e-11,  ..., 8.1092e-12, 0.0000e+00,\n         1.2444e-11],\n        [4.0658e-11, 8.6062e-12, 2.6192e-11,  ..., 3.9069e-11, 0.0000e+00,\n         1.8913e-11],\n        [2.4880e-11, 1.2028e-11, 2.6414e-11,  ..., 7.8947e-11, 0.0000e+00,\n         1.1963e-10],\n        ...,\n        [4.8356e-11, 1.9637e-11, 1.0225e-11,  ..., 4.4803e-12, 0.0000e+00,\n         8.1128e-12],\n        [1.7608e-12, 4.0500e-12, 1.1602e-10,  ..., 1.0861e-10, 0.0000e+00,\n         2.0399e-11],\n        [1.0155e-11, 1.2418e-11, 2.5896e-11,  ..., 3.7060e-10, 0.0000e+00,\n         4.8629e-11]], device='cuda:0')"
      },
      "6": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-2.9050e-05,  3.7621e-06, -5.0623e-05,  ...,  3.0815e-06,\n        -2.6227e-05,  5.0914e-06], device='cuda:0')",
        "exp_avg_sq": "tensor([4.7790e-09, 3.2526e-09, 4.1045e-09,  ..., 2.5641e-09, 3.0099e-09,\n        2.8913e-09], device='cuda:0')"
      },
      "7": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[ 2.7085e-07, -4.0476e-07,  5.9547e-07,  ...,  5.3408e-07,\n         -1.3915e-06, -1.4484e-06],\n        [ 5.7098e-08,  1.6263e-06, -1.3432e-06,  ..., -8.1884e-07,\n         -9.3186e-07,  4.7864e-07],\n        [-3.6681e-07,  1.1444e-06,  5.7836e-07,  ...,  5.7105e-07,\n         -5.0189e-07,  1.1652e-06],\n        ...,\n        [-9.8853e-07, -3.5932e-06, -2.1081e-06,  ..., -1.0942e-06,\n         -2.0264e-06,  8.8557e-07],\n        [-1.1547e-06,  1.0930e-06, -1.8229e-07,  ..., -6.1655e-07,\n          1.6886e-06,  1.7918e-07],\n        [ 2.8115e-06, -1.4956e-06,  2.8647e-07,  ...,  4.3480e-06,\n          2.1652e-07,  1.1223e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.4066e-11, 6.0948e-12, 7.8347e-12,  ..., 7.1691e-12, 6.5819e-12,\n         7.8594e-12],\n        [1.8706e-11, 1.1371e-11, 1.3297e-11,  ..., 1.5869e-11, 1.2163e-11,\n         1.2811e-11],\n        [1.8208e-11, 1.2180e-11, 1.1863e-11,  ..., 1.2269e-11, 1.0056e-11,\n         1.6568e-11],\n        ...,\n        [1.7239e-11, 2.1737e-11, 2.0287e-11,  ..., 1.3141e-11, 1.5983e-11,\n         1.4509e-11],\n        [1.5653e-11, 1.1913e-11, 1.5217e-11,  ..., 9.6066e-12, 1.0947e-11,\n         1.5791e-11],\n        [2.1784e-11, 1.2656e-11, 1.6639e-11,  ..., 5.4514e-11, 1.2477e-11,\n         1.0659e-11]], device='cuda:0')"
      },
      "32": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([2.3813e-07], device='cuda:0')"
      },
      "33": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45, -5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([2.1677e-09, 1.0537e-08, 3.2088e-09], device='cuda:0')"
      },
      "34": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([1.2936e-05, 1.1728e-07, 1.5841e-07, 1.6438e-07, 1.8847e-07, 2.1650e-07,\n        1.7212e-07, 1.2747e-07, 1.6100e-07, 1.4307e-07], device='cuda:0')"
      },
      "36": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [-5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.9978e-14, 9.0025e-15, 1.3375e-14,  ..., 5.8468e-14, 0.0000e+00,\n         1.9733e-14],\n        [3.5963e-15, 8.6944e-15, 2.6151e-14,  ..., 2.8911e-14, 0.0000e+00,\n         1.1606e-14],\n        [3.2274e-12, 7.6360e-13, 4.2429e-11,  ..., 1.3603e-11, 0.0000e+00,\n         3.7956e-11],\n        ...,\n        [8.9948e-13, 3.7408e-14, 7.0843e-12,  ..., 3.9534e-12, 0.0000e+00,\n         1.0637e-11],\n        [2.4244e-14, 8.7240e-15, 2.5640e-15,  ..., 9.5197e-14, 0.0000e+00,\n         4.9087e-14],\n        [8.1793e-16, 1.5347e-14, 2.0049e-15,  ..., 5.1406e-15, 0.0000e+00,\n         2.0940e-14]], device='cuda:0')"
      },
      "37": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([3.5479e-11, 1.2245e-11, 4.6173e-08, 6.1268e-11, 2.2287e-08, 7.6758e-09,\n        1.4685e-09, 1.6852e-09, 4.2586e-08, 1.5583e-10, 2.1651e-08, 2.6345e-09,\n        2.3354e-11, 4.8711e-10, 3.4389e-12, 2.4702e-10, 3.5240e-09, 3.9909e-09,\n        2.5803e-10, 3.0989e-10, 3.7796e-09, 1.7438e-10, 2.6284e-09, 1.5656e-09,\n        1.3232e-09, 1.3183e-08, 2.5059e-09, 2.1820e-09, 1.6605e-09, 3.3095e-08,\n        1.2159e-08, 7.5240e-09, 8.7648e-08, 1.4254e-10, 1.1743e-09, 2.6267e-10,\n        3.6516e-09, 1.6229e-10, 1.4740e-08, 3.8932e-10, 1.4597e-08, 2.4892e-10,\n        7.7537e-10, 1.3820e-08, 2.2944e-11, 8.3743e-10, 1.2955e-09, 1.2917e-09,\n        2.6382e-10, 9.3042e-09, 3.2149e-09, 3.4474e-09, 2.3094e-08, 1.5104e-09,\n        2.5661e-10, 1.8493e-09, 3.6877e-13, 9.0062e-10, 3.9670e-09, 3.8889e-08,\n        4.7947e-09, 4.2538e-10, 1.3559e-11, 1.4481e-09, 1.6455e-09, 2.1766e-08,\n        1.9861e-10, 1.4583e-09, 4.7860e-10, 3.3452e-09, 2.0477e-09, 1.7861e-09,\n        5.5110e-10, 8.1518e-09, 1.1459e-08, 6.4706e-09, 7.8509e-09, 1.8956e-09,\n        5.6690e-10, 1.5675e-11, 1.9414e-09, 7.5427e-10, 1.9399e-10, 1.0924e-09,\n        1.5380e-09, 4.8332e-12, 1.6051e-08, 2.5757e-09, 4.6993e-10, 2.1004e-11,\n        1.5059e-10, 1.5371e-09, 8.7864e-09, 1.3755e-08, 1.2888e-08, 9.7868e-10,\n        1.1540e-08, 4.8378e-09, 3.3328e-09, 4.8035e-10, 1.4150e-09, 5.9385e-11,\n        2.5289e-10, 1.5162e-08, 6.0929e-09, 1.0643e-08, 3.4683e-09, 3.3259e-11,\n        1.4267e-08, 1.1920e-09, 2.0685e-09, 2.8603e-10, 3.0297e-09, 1.4728e-09,\n        1.3634e-09, 6.4464e-08, 4.7078e-11, 2.1756e-09, 4.1909e-10, 2.0143e-08,\n        2.0516e-08, 2.7997e-09, 6.4070e-09, 2.5240e-10, 3.1640e-10, 5.5276e-09,\n        2.9859e-09, 3.3730e-09, 4.5159e-10, 1.0579e-08, 8.3490e-12, 2.3854e-10,\n        5.7509e-08, 5.2433e-09, 9.8784e-11, 1.1378e-09, 3.5632e-09, 1.4181e-09,\n        2.3336e-08, 4.4948e-08, 1.0506e-09, 3.2503e-09, 2.7599e-10, 4.5542e-11,\n        5.0297e-10, 2.2788e-09, 2.3975e-09, 4.5208e-09, 1.7035e-10, 9.6365e-09,\n        8.6242e-11, 2.3726e-10, 7.9157e-09, 3.5301e-09, 2.9803e-09, 1.7216e-08,\n        2.4856e-10, 1.3209e-08, 1.1434e-10, 6.1727e-10, 1.4812e-11, 1.0650e-08,\n        9.8839e-12, 8.0410e-10, 1.3713e-08, 7.7832e-09, 3.6659e-10, 4.8140e-09,\n        4.0976e-11, 2.3992e-10, 1.0284e-09, 5.7059e-09, 1.3543e-08, 4.7687e-09,\n        2.8803e-09, 9.8009e-10, 1.4175e-08, 1.7174e-08, 4.1154e-11, 3.1486e-12,\n        3.1273e-11, 5.1175e-09, 4.1237e-10, 1.5600e-10, 5.3597e-10, 2.0011e-08,\n        1.2457e-09, 2.4551e-11, 4.4612e-10, 7.3899e-10, 8.7226e-09, 5.1564e-11,\n        8.1586e-09, 1.6295e-08, 4.8456e-09, 1.3196e-09, 3.9527e-10, 4.4637e-09,\n        1.2073e-09, 5.0518e-09, 1.3496e-09, 8.1166e-11, 4.2251e-08, 8.3211e-11,\n        2.8962e-09, 5.4242e-12, 3.2657e-09, 1.0863e-08, 2.8558e-11, 7.4243e-10,\n        1.2487e-08, 7.5474e-10, 5.7565e-09, 3.8747e-08, 3.5719e-08, 3.9376e-09,\n        5.8776e-11, 1.7933e-09, 1.0446e-08, 2.1473e-11, 2.4653e-09, 4.3438e-08,\n        1.3479e-09, 2.3601e-09, 7.8143e-09, 1.0053e-09, 3.8463e-10, 3.5114e-09,\n        2.1156e-09, 1.7640e-09, 9.9896e-10, 4.1275e-09, 1.8535e-09, 9.8731e-10,\n        4.2872e-09, 6.7889e-10, 1.3748e-09, 1.7219e-09, 4.1866e-09, 7.3752e-11,\n        2.0725e-08, 7.1628e-10, 1.9448e-10, 7.0694e-09, 2.4255e-09, 3.6291e-11,\n        3.7791e-09, 2.5139e-08, 1.8705e-08, 2.7413e-08, 9.2042e-10, 3.8629e-09,\n        2.9373e-09, 1.2797e-08, 1.0136e-10, 1.2750e-11], device='cuda:0')"
      },
      "38": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([7.9704e-14, 9.1411e-17, 8.8704e-11, 3.0303e-13, 4.7183e-11, 1.4851e-11,\n        2.5882e-12, 3.7791e-12, 9.8010e-11, 3.8767e-13, 7.7471e-11, 4.5133e-12,\n        1.4122e-13, 6.0863e-13, 1.1052e-14, 8.9470e-13, 7.8248e-12, 1.0980e-11,\n        9.3614e-13, 6.7659e-13, 6.1462e-12, 4.2859e-13, 3.4815e-12, 2.6922e-12,\n        2.4777e-12, 1.6412e-11, 3.8433e-12, 3.9055e-12, 9.5015e-12, 9.6053e-11,\n        1.5007e-11, 1.3207e-11, 1.6564e-10, 2.0921e-13, 2.0429e-12, 9.9529e-13,\n        4.6437e-12, 4.8835e-13, 3.7353e-11, 5.4679e-13, 4.1141e-11, 8.4792e-13,\n        2.2833e-12, 1.8357e-11, 2.3008e-14, 9.2193e-13, 2.0377e-12, 1.8546e-12,\n        7.2091e-13, 2.2153e-11, 4.2467e-12, 9.0624e-12, 6.2485e-11, 2.1398e-12,\n        4.1960e-13, 2.7433e-12, 2.2177e-14, 1.1577e-12, 5.5717e-12, 1.1307e-10,\n        7.8786e-12, 6.6648e-13, 1.3349e-16, 3.2397e-12, 2.9583e-12, 6.0295e-11,\n        5.0269e-13, 2.0251e-12, 9.7442e-13, 4.3214e-12, 3.4468e-12, 3.2460e-12,\n        4.7888e-13, 1.6356e-11, 1.9750e-11, 9.8523e-12, 2.1741e-11, 2.8953e-12,\n        1.4936e-12, 4.0735e-15, 3.4282e-12, 1.0889e-12, 3.7526e-13, 1.3706e-12,\n        2.2512e-12, 1.4358e-14, 2.4454e-11, 6.2712e-12, 1.0598e-12, 3.7810e-15,\n        3.6536e-13, 2.5772e-12, 1.9775e-11, 3.7701e-11, 1.5567e-11, 1.2174e-12,\n        1.6158e-11, 7.1007e-12, 4.7434e-12, 2.1109e-12, 1.9639e-12, 8.5630e-14,\n        4.2130e-13, 3.0330e-11, 1.7723e-11, 2.0543e-11, 5.0156e-12, 1.1215e-13,\n        5.1175e-11, 2.1793e-12, 4.6328e-12, 6.0478e-13, 4.8523e-12, 1.8456e-12,\n        2.5458e-12, 1.6603e-10, 1.1213e-13, 2.2022e-12, 2.1938e-12, 6.3428e-11,\n        3.5417e-11, 3.8519e-12, 8.1148e-12, 2.1524e-12, 9.0250e-13, 1.1855e-11,\n        6.0894e-12, 7.9426e-12, 8.7641e-13, 3.4487e-11, 1.3871e-13, 4.7587e-13,\n        1.5612e-10, 4.7644e-12, 1.4011e-13, 1.5556e-12, 1.2188e-11, 2.0245e-12,\n        6.6321e-11, 1.4456e-10, 2.0891e-12, 6.9279e-12, 1.0518e-12, 1.9212e-13,\n        8.5141e-13, 3.4865e-12, 4.7304e-12, 8.4764e-12, 6.3999e-13, 3.4260e-11,\n        1.3430e-13, 2.7672e-13, 1.6942e-11, 7.0065e-12, 6.5180e-12, 3.0545e-11,\n        3.6096e-13, 1.5320e-11, 1.6713e-13, 9.2611e-13, 5.6083e-14, 1.7613e-11,\n        3.8010e-14, 1.8334e-12, 2.3534e-11, 1.2222e-11, 4.5365e-13, 1.0047e-11,\n        1.6078e-13, 8.0152e-13, 9.5758e-13, 3.3375e-11, 3.3462e-11, 1.4176e-11,\n        8.3792e-12, 2.2743e-12, 2.7943e-11, 4.4034e-11, 1.7528e-13, 2.6871e-15,\n        6.5686e-14, 1.4017e-11, 1.2706e-12, 2.8380e-13, 1.8892e-12, 2.6017e-11,\n        2.0613e-12, 5.1715e-14, 8.7449e-13, 1.3215e-12, 1.9529e-11, 3.2710e-13,\n        1.5227e-11, 5.9264e-11, 7.0989e-12, 3.8427e-12, 7.0962e-13, 7.6742e-12,\n        1.7045e-12, 9.5399e-12, 2.1977e-12, 5.8818e-13, 6.8430e-11, 2.8363e-13,\n        4.4418e-12, 1.0469e-14, 7.4854e-12, 2.4871e-11, 9.7962e-14, 8.2577e-13,\n        2.2010e-11, 1.0252e-12, 9.9002e-12, 7.1685e-11, 5.7302e-11, 7.4050e-12,\n        9.8317e-14, 2.9014e-12, 1.7102e-11, 1.8916e-14, 4.3630e-12, 7.4485e-11,\n        3.2052e-12, 4.3423e-12, 1.1139e-11, 2.3035e-12, 8.9871e-13, 5.7505e-12,\n        2.2569e-12, 4.0470e-12, 1.9029e-12, 1.1356e-11, 2.7720e-12, 1.8108e-12,\n        6.5694e-12, 1.5012e-12, 1.7322e-12, 2.4776e-12, 1.4612e-11, 1.5713e-13,\n        5.7337e-11, 1.1523e-12, 1.8873e-12, 7.5873e-12, 3.3990e-12, 2.7042e-13,\n        8.0982e-12, 7.8071e-11, 3.0536e-11, 9.5873e-11, 1.5020e-12, 7.9711e-12,\n        4.7331e-12, 2.2150e-11, 1.7520e-13, 4.7577e-14], device='cuda:0')"
      },
      "39": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([1.6247e-13, 6.2031e-15, 1.2697e-10, 2.7348e-13, 5.7894e-11, 1.9428e-11,\n        4.6037e-12, 4.9469e-12, 1.1874e-10, 8.2611e-13, 6.2295e-11, 7.5230e-12,\n        2.2212e-13, 1.4894e-12, 4.4692e-14, 1.1311e-12, 1.0582e-11, 1.1909e-11,\n        8.3862e-13, 1.0809e-12, 1.0865e-11, 5.9651e-13, 7.6405e-12, 4.7399e-12,\n        3.9390e-12, 3.5885e-11, 7.5869e-12, 5.8530e-12, 6.3753e-12, 9.1851e-11,\n        3.1557e-11, 2.0942e-11, 2.3561e-10, 3.6821e-13, 3.5264e-12, 1.0358e-12,\n        9.4137e-12, 8.8770e-13, 3.9976e-11, 1.3051e-12, 3.7652e-11, 9.3495e-13,\n        2.5902e-12, 4.0672e-11, 1.6321e-14, 1.8929e-12, 3.8018e-12, 4.0204e-12,\n        7.8766e-13, 2.5730e-11, 9.5770e-12, 7.9764e-12, 6.0478e-11, 4.2259e-12,\n        8.7117e-13, 5.6561e-12, 5.4368e-14, 2.4078e-12, 1.0680e-11, 1.0578e-10,\n        1.2747e-11, 1.2617e-12, 1.2465e-14, 4.7263e-12, 5.1025e-12, 5.5920e-11,\n        6.8172e-13, 4.1648e-12, 1.5258e-12, 9.6215e-12, 6.2051e-12, 5.4793e-12,\n        1.4389e-12, 2.3395e-11, 3.2969e-11, 1.7767e-11, 2.2130e-11, 5.6316e-12,\n        1.9406e-12, 3.7025e-14, 5.7369e-12, 1.9835e-12, 7.6573e-13, 3.4124e-12,\n        3.8937e-12, 6.3891e-14, 4.3504e-11, 7.7361e-12, 2.1372e-12, 4.0352e-15,\n        4.7077e-13, 4.3164e-12, 2.2542e-11, 3.5727e-11, 3.2180e-11, 2.2522e-12,\n        3.0717e-11, 1.4153e-11, 8.6717e-12, 2.3922e-12, 4.2334e-12, 2.0625e-13,\n        8.6434e-13, 3.9743e-11, 1.7889e-11, 2.5054e-11, 1.0350e-11, 1.9718e-13,\n        3.5427e-11, 3.5804e-12, 6.2758e-12, 1.0801e-12, 7.7568e-12, 3.4230e-12,\n        3.7203e-12, 1.7368e-10, 2.5120e-13, 5.8762e-12, 1.9473e-12, 5.7286e-11,\n        5.4959e-11, 7.9737e-12, 1.8000e-11, 2.2323e-12, 1.0703e-12, 1.6426e-11,\n        7.4623e-12, 9.5658e-12, 1.5736e-12, 2.9771e-11, 1.8784e-13, 8.0659e-13,\n        1.5430e-10, 1.3904e-11, 3.0663e-13, 3.5631e-12, 1.0593e-11, 3.4054e-12,\n        6.1111e-11, 1.1934e-10, 3.4170e-12, 9.8687e-12, 8.2126e-13, 2.3470e-13,\n        1.6033e-12, 6.5393e-12, 6.1147e-12, 1.2605e-11, 8.1294e-13, 2.7744e-11,\n        2.8873e-13, 5.8635e-13, 2.0707e-11, 8.7667e-12, 8.5469e-12, 4.7211e-11,\n        7.7112e-13, 3.5071e-11, 3.6136e-13, 1.7112e-12, 2.8352e-13, 3.0442e-11,\n        1.3817e-13, 2.4282e-12, 3.8697e-11, 2.0192e-11, 1.1284e-12, 1.3100e-11,\n        2.1970e-13, 8.2566e-13, 2.5768e-12, 1.7505e-11, 3.4713e-11, 1.4803e-11,\n        7.7263e-12, 3.1823e-12, 4.0023e-11, 4.8944e-11, 4.9264e-13, 9.5902e-15,\n        1.0875e-13, 1.3781e-11, 1.4290e-12, 5.5292e-13, 1.6273e-12, 5.4572e-11,\n        3.4432e-12, 1.0774e-13, 1.5787e-12, 2.2260e-12, 2.3734e-11, 4.3554e-13,\n        2.0820e-11, 4.5715e-11, 1.3124e-11, 4.0293e-12, 1.2340e-12, 1.0899e-11,\n        3.2635e-12, 1.4484e-11, 4.0354e-12, 6.5851e-13, 1.1324e-10, 4.7519e-13,\n        7.8001e-12, 1.3719e-14, 9.3705e-12, 2.8435e-11, 3.3619e-13, 2.4197e-12,\n        3.5289e-11, 1.8121e-12, 1.6451e-11, 1.0148e-10, 9.5363e-11, 1.0583e-11,\n        2.4643e-13, 4.8550e-12, 2.7828e-11, 7.1413e-14, 5.9944e-12, 1.1865e-10,\n        3.9309e-12, 6.6650e-12, 2.2877e-11, 2.4278e-12, 1.3540e-12, 9.9616e-12,\n        6.0236e-12, 5.1833e-12, 3.2359e-12, 1.1182e-11, 4.3313e-12, 2.9331e-12,\n        1.2286e-11, 2.1692e-12, 3.5090e-12, 5.1169e-12, 1.0535e-11, 3.4787e-13,\n        5.5445e-11, 2.1555e-12, 1.4558e-12, 2.0433e-11, 6.6568e-12, 3.6057e-13,\n        1.1403e-11, 6.3919e-11, 5.4355e-11, 7.7766e-11, 2.8420e-12, 9.9083e-12,\n        6.9704e-12, 3.4935e-11, 4.2927e-13, 4.6440e-14], device='cuda:0')"
      },
      "40": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-5.6052e-45,  5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [-5.6052e-45, -5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [-5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[3.5706e-18, 1.4614e-16, 1.0016e-14,  ..., 4.1279e-15, 0.0000e+00,\n         4.1635e-14],\n        [9.5113e-15, 1.1034e-15, 9.9037e-14,  ..., 6.0940e-14, 0.0000e+00,\n         2.6467e-14],\n        [3.0439e-12, 4.6959e-14, 4.0767e-11,  ..., 6.3829e-12, 0.0000e+00,\n         2.9291e-11],\n        ...,\n        [7.0622e-13, 1.9660e-13, 1.2457e-11,  ..., 7.0788e-12, 0.0000e+00,\n         6.3716e-12],\n        [4.0711e-15, 8.9847e-15, 1.5071e-13,  ..., 1.3067e-14, 0.0000e+00,\n         7.2166e-14],\n        [7.5040e-15, 9.8285e-14, 6.6878e-13,  ..., 7.8228e-14, 0.0000e+00,\n         6.5772e-13]], device='cuda:0')"
      },
      "41": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([3.3835e-13, 7.2243e-12, 1.3722e-08, 7.9997e-12, 3.0142e-09, 4.3618e-09,\n        2.2646e-09, 2.9187e-10, 4.6284e-08, 1.3038e-09, 1.0715e-08, 9.6087e-09,\n        2.8775e-11, 8.9983e-10, 1.8683e-11, 3.7350e-11, 4.3476e-09, 2.4361e-09,\n        5.2058e-10, 1.3215e-10, 7.6710e-10, 7.6424e-11, 4.8852e-09, 2.3178e-09,\n        6.1803e-09, 8.3029e-09, 2.6610e-09, 3.8604e-09, 1.5112e-09, 2.1558e-08,\n        2.2770e-08, 7.8243e-10, 9.1130e-08, 8.0198e-10, 9.6481e-10, 8.7110e-11,\n        3.3302e-09, 7.0704e-11, 2.1733e-09, 7.6949e-10, 7.4301e-09, 7.0436e-11,\n        1.7769e-11, 4.5184e-08, 1.1286e-11, 2.0330e-09, 4.7965e-09, 3.8565e-09,\n        4.3637e-11, 2.5268e-09, 3.7897e-09, 4.9238e-10, 1.6053e-08, 1.0532e-09,\n        5.2142e-10, 1.0781e-08, 3.4522e-10, 9.8438e-10, 1.9934e-08, 1.9615e-08,\n        4.1991e-09, 5.5180e-09, 1.3263e-10, 2.6657e-09, 3.2738e-09, 1.8291e-08,\n        1.1067e-10, 4.8104e-09, 1.0136e-09, 2.3180e-09, 8.4550e-10, 9.0208e-10,\n        2.4175e-09, 2.5930e-08, 7.8374e-09, 2.2920e-08, 4.8593e-09, 6.9132e-10,\n        7.2894e-10, 3.3316e-11, 2.2499e-09, 3.2246e-10, 2.0753e-10, 5.8701e-09,\n        1.7047e-09, 1.7269e-10, 3.4212e-08, 3.4423e-09, 9.8817e-10, 8.1816e-12,\n        6.1984e-11, 2.1920e-09, 3.8552e-09, 1.0613e-09, 3.6857e-08, 1.4084e-10,\n        3.4405e-08, 2.3799e-08, 4.1368e-09, 2.7359e-10, 9.2966e-10, 5.5858e-10,\n        6.1777e-11, 1.7443e-08, 2.0307e-10, 2.8901e-09, 2.2910e-09, 3.8629e-10,\n        2.5195e-09, 1.1412e-08, 2.9406e-09, 1.9768e-10, 1.2261e-08, 1.2533e-09,\n        9.6230e-10, 3.6824e-09, 3.3300e-10, 2.8474e-09, 3.5378e-10, 9.6683e-09,\n        2.6582e-08, 3.3735e-09, 3.0178e-09, 3.9328e-11, 6.8490e-10, 3.8755e-08,\n        2.8445e-10, 2.1269e-09, 8.0531e-10, 2.5489e-09, 2.8260e-11, 8.0991e-10,\n        3.7994e-08, 1.7236e-09, 7.6030e-11, 3.2178e-09, 2.0448e-09, 3.1537e-10,\n        1.0529e-08, 1.8702e-08, 3.3728e-10, 9.3636e-10, 1.3631e-08, 2.2796e-11,\n        4.9270e-10, 2.9789e-09, 9.5323e-10, 1.1217e-08, 2.0318e-10, 6.5663e-09,\n        6.0961e-10, 3.7041e-11, 9.9037e-09, 2.6570e-09, 6.8212e-10, 9.6470e-09,\n        3.4870e-09, 1.3253e-08, 1.9312e-11, 1.7053e-09, 7.5054e-11, 9.1497e-09,\n        8.3724e-12, 1.4651e-08, 1.9437e-09, 9.7388e-09, 9.9612e-10, 3.4513e-09,\n        7.0512e-11, 2.4928e-12, 8.3391e-10, 3.1616e-09, 2.8635e-09, 3.5596e-09,\n        1.3242e-09, 1.1196e-09, 1.5780e-08, 3.5556e-09, 2.9469e-11, 5.2370e-12,\n        3.3983e-10, 3.6113e-09, 4.2674e-11, 3.9182e-10, 2.0199e-10, 4.4425e-08,\n        4.2148e-09, 6.0427e-11, 8.3467e-10, 9.9403e-10, 1.3943e-08, 3.9887e-11,\n        5.9162e-09, 1.5078e-08, 9.8644e-09, 1.1682e-10, 1.3915e-10, 4.3525e-10,\n        4.3243e-09, 1.2200e-08, 6.2447e-09, 7.3650e-11, 2.7204e-08, 3.3920e-12,\n        1.8069e-09, 1.2202e-11, 1.0444e-09, 8.8971e-09, 2.4941e-10, 2.8105e-10,\n        1.6745e-08, 5.2622e-10, 6.5089e-09, 3.5855e-08, 1.2062e-07, 8.9336e-10,\n        1.4769e-10, 5.5306e-09, 2.7387e-09, 3.6584e-12, 2.1476e-09, 4.9546e-08,\n        6.9506e-10, 4.5427e-09, 4.0996e-09, 5.1299e-10, 2.8697e-10, 6.8062e-09,\n        8.4046e-09, 4.2396e-10, 8.1681e-09, 1.2358e-09, 3.9323e-09, 5.2638e-09,\n        4.6854e-09, 2.3825e-09, 1.3652e-09, 2.6547e-09, 8.0153e-10, 1.5439e-10,\n        1.3811e-08, 2.8913e-09, 8.9446e-11, 1.1491e-08, 1.1797e-09, 2.3663e-11,\n        1.0791e-08, 1.3129e-08, 3.3048e-08, 1.0195e-08, 3.1693e-10, 7.8598e-09,\n        7.1589e-10, 1.0500e-08, 1.2463e-10, 8.9954e-10], device='cuda:0')"
      },
      "42": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([1.1144e-14, 1.5780e-14, 2.3904e-11, 6.7419e-14, 5.3374e-12, 7.3714e-12,\n        3.8141e-12, 4.7729e-13, 8.3453e-11, 2.8020e-12, 1.7114e-11, 1.5570e-11,\n        8.5791e-14, 1.2713e-12, 5.8064e-14, 9.8766e-14, 8.2010e-12, 1.0174e-11,\n        1.3602e-12, 4.4066e-13, 1.8171e-12, 2.0218e-13, 1.2712e-11, 4.7530e-12,\n        2.2136e-11, 1.2961e-11, 5.3757e-12, 1.1652e-11, 5.8513e-12, 3.5687e-11,\n        4.2077e-11, 1.9233e-12, 1.8372e-10, 8.9440e-13, 1.6030e-12, 1.3001e-13,\n        5.2682e-12, 3.7647e-13, 2.9749e-12, 1.6588e-12, 9.7710e-12, 8.9178e-13,\n        2.3599e-14, 1.5089e-10, 4.6241e-14, 3.5810e-12, 1.3189e-11, 1.5266e-11,\n        1.3295e-13, 4.0147e-12, 7.3643e-12, 6.1669e-13, 2.8043e-11, 1.6976e-12,\n        1.1155e-12, 1.6749e-11, 1.2393e-12, 1.3295e-12, 4.7233e-11, 2.6043e-11,\n        6.1707e-12, 1.5095e-11, 3.5608e-13, 4.2468e-12, 8.3914e-12, 4.0014e-11,\n        3.2634e-13, 7.0845e-12, 1.8767e-12, 2.5925e-12, 1.3799e-12, 1.9873e-12,\n        3.9385e-12, 6.2733e-11, 1.0854e-11, 4.9619e-11, 9.6626e-12, 1.2148e-12,\n        1.4368e-12, 3.5516e-14, 2.7548e-12, 4.8309e-13, 6.3459e-13, 2.3101e-11,\n        2.2156e-12, 9.3900e-13, 9.1007e-11, 1.2032e-11, 2.4887e-12, 4.9114e-14,\n        3.0020e-13, 3.5795e-12, 5.0175e-12, 1.6990e-12, 9.8227e-11, 3.3366e-13,\n        8.7830e-11, 6.0927e-11, 7.7274e-12, 1.0875e-12, 1.7983e-12, 1.1551e-12,\n        1.9002e-13, 3.2457e-11, 7.2612e-13, 4.0040e-12, 2.6859e-12, 9.9456e-13,\n        2.5324e-12, 2.6372e-11, 6.5774e-12, 2.6338e-13, 1.8775e-11, 1.5775e-12,\n        2.4182e-12, 6.1742e-12, 1.2237e-12, 3.1897e-12, 1.3208e-12, 1.3634e-11,\n        5.0544e-11, 1.0006e-11, 3.3348e-12, 9.0311e-14, 2.0913e-12, 7.8762e-11,\n        5.5483e-13, 5.8995e-12, 1.8475e-12, 4.4877e-12, 1.6183e-13, 1.0824e-12,\n        7.1972e-11, 2.7938e-12, 1.9521e-13, 6.5242e-12, 5.5148e-12, 6.9905e-13,\n        1.6890e-11, 2.9662e-11, 6.7764e-13, 1.4140e-12, 2.6057e-11, 1.5221e-13,\n        1.2812e-12, 5.8168e-12, 1.3351e-12, 3.6409e-11, 8.5493e-13, 2.1694e-11,\n        1.1959e-12, 2.2944e-14, 2.0341e-11, 5.0605e-12, 1.2748e-12, 1.0012e-11,\n        5.5102e-12, 1.6991e-11, 2.8451e-14, 3.6085e-12, 8.8608e-13, 1.9916e-11,\n        4.6902e-14, 4.7316e-11, 3.3909e-12, 1.6679e-11, 2.4141e-12, 7.4987e-12,\n        1.6169e-13, 7.2581e-15, 1.1657e-12, 7.8604e-12, 3.6049e-12, 4.9561e-12,\n        2.9625e-12, 2.4753e-12, 3.4780e-11, 6.6265e-12, 1.0419e-13, 1.2524e-14,\n        5.5686e-13, 9.5801e-12, 3.9779e-14, 7.5909e-13, 5.0792e-13, 7.6930e-11,\n        5.3510e-12, 3.2950e-13, 3.7250e-12, 2.6453e-12, 4.2818e-11, 2.9560e-13,\n        8.8627e-12, 3.2578e-11, 1.7285e-11, 3.5031e-13, 1.3469e-13, 8.3069e-13,\n        6.7884e-12, 2.8940e-11, 1.4643e-11, 4.1385e-13, 4.6435e-11, 7.8776e-14,\n        1.9941e-12, 2.3080e-14, 1.3317e-12, 1.8892e-11, 1.0221e-12, 4.3824e-13,\n        5.0826e-11, 5.4774e-13, 8.6526e-12, 7.3214e-11, 3.4870e-10, 1.3132e-12,\n        2.5958e-13, 9.8291e-12, 3.9841e-12, 3.3104e-15, 2.1298e-12, 8.8707e-11,\n        1.8258e-12, 1.4199e-11, 7.4123e-12, 8.3640e-13, 7.4883e-13, 1.4232e-11,\n        1.4536e-11, 6.5740e-13, 1.0378e-11, 2.5494e-12, 8.1695e-12, 1.0883e-11,\n        7.4965e-12, 4.3875e-12, 2.2249e-12, 5.2209e-12, 1.2320e-12, 3.0936e-13,\n        2.8448e-11, 7.2805e-12, 8.7219e-13, 1.5963e-11, 1.2195e-12, 1.5670e-13,\n        2.9385e-11, 3.1586e-11, 7.7952e-11, 1.6647e-11, 5.1266e-13, 1.6946e-11,\n        1.0805e-12, 2.3883e-11, 3.2270e-13, 3.8248e-12], device='cuda:0')"
      },
      "43": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([3.8372e-14, 2.8582e-14, 3.9635e-11, 1.1168e-13, 8.2233e-12, 1.0812e-11,\n        6.4034e-12, 1.0453e-12, 1.2806e-10, 3.7886e-12, 3.0397e-11, 2.3420e-11,\n        8.5709e-14, 2.8103e-12, 8.6232e-14, 2.8357e-13, 1.3065e-11, 7.9260e-12,\n        1.2533e-12, 6.0188e-13, 2.6196e-12, 3.6392e-13, 1.2754e-11, 6.8879e-12,\n        1.8344e-11, 2.1396e-11, 7.8352e-12, 9.9471e-12, 5.6990e-12, 5.9798e-11,\n        5.6411e-11, 2.6150e-12, 2.5393e-10, 1.9486e-12, 3.2871e-12, 4.3734e-13,\n        8.5655e-12, 5.9951e-13, 6.5889e-12, 2.3052e-12, 1.7875e-11, 5.5844e-13,\n        1.0677e-13, 1.2666e-10, 5.0235e-14, 4.7998e-12, 1.2132e-11, 1.1846e-11,\n        2.4029e-13, 7.4572e-12, 1.1787e-11, 1.2492e-12, 4.1549e-11, 2.7304e-12,\n        1.7745e-12, 2.8722e-11, 1.3733e-12, 2.5085e-12, 5.1431e-11, 5.6446e-11,\n        1.0211e-11, 1.2845e-11, 6.1667e-13, 7.7211e-12, 9.4705e-12, 4.5440e-11,\n        4.0474e-13, 1.2844e-11, 3.0411e-12, 6.8206e-12, 2.5546e-12, 2.6655e-12,\n        5.0823e-12, 7.1356e-11, 2.2580e-11, 5.8325e-11, 1.4045e-11, 2.1991e-12,\n        2.4917e-12, 7.6391e-14, 6.2233e-12, 9.9813e-13, 8.9076e-13, 1.6906e-11,\n        4.1738e-12, 8.4012e-13, 9.4449e-11, 1.0843e-11, 3.4772e-12, 6.9717e-14,\n        3.1853e-13, 5.6534e-12, 1.0140e-11, 3.1930e-12, 9.2452e-11, 3.6375e-13,\n        8.7267e-11, 6.0409e-11, 9.3522e-12, 1.7436e-12, 2.8077e-12, 2.0288e-12,\n        1.8045e-13, 4.5200e-11, 6.0468e-13, 7.5727e-12, 6.4819e-12, 1.4859e-12,\n        6.4989e-12, 2.7996e-11, 8.6049e-12, 6.7582e-13, 3.2690e-11, 2.8174e-12,\n        3.1235e-12, 1.0728e-11, 1.3052e-12, 7.1921e-12, 1.7676e-12, 2.7734e-11,\n        7.2600e-11, 1.0576e-11, 8.7110e-12, 6.3178e-13, 2.5184e-12, 9.8882e-11,\n        7.7047e-13, 6.6190e-12, 2.4453e-12, 7.3561e-12, 2.4729e-13, 2.6464e-12,\n        9.7647e-11, 4.6531e-12, 3.3588e-13, 8.4442e-12, 6.5463e-12, 7.7911e-13,\n        2.6410e-11, 4.7160e-11, 1.2673e-12, 2.9850e-12, 3.3607e-11, 2.7859e-13,\n        1.7236e-12, 8.7213e-12, 2.2254e-12, 2.9150e-11, 7.7986e-13, 1.9063e-11,\n        2.0103e-12, 4.5006e-14, 2.5394e-11, 6.2953e-12, 2.3425e-12, 2.5065e-11,\n        8.4711e-12, 3.2081e-11, 1.0592e-13, 5.0280e-12, 1.2092e-12, 2.6745e-11,\n        9.3083e-14, 4.1189e-11, 5.5673e-12, 2.4838e-11, 2.4674e-12, 8.8197e-12,\n        2.6688e-13, 1.9946e-14, 2.1752e-12, 9.6616e-12, 8.0301e-12, 1.0548e-11,\n        3.7527e-12, 3.2831e-12, 4.4374e-11, 9.8977e-12, 4.1077e-13, 2.7107e-14,\n        1.2611e-12, 9.8190e-12, 1.6585e-13, 1.3749e-12, 8.2252e-13, 1.2230e-10,\n        1.0205e-11, 3.0457e-13, 3.0699e-12, 3.4634e-12, 3.4389e-11, 5.3292e-13,\n        1.4902e-11, 4.0170e-11, 2.4017e-11, 3.2676e-13, 5.1703e-13, 1.4943e-12,\n        1.1542e-11, 3.4949e-11, 1.5784e-11, 6.7459e-13, 7.0958e-11, 1.6969e-13,\n        4.6826e-12, 5.0268e-14, 3.3309e-12, 2.1938e-11, 1.2596e-12, 9.0829e-13,\n        4.7524e-11, 1.2111e-12, 1.8752e-11, 9.6213e-11, 3.2716e-10, 2.4016e-12,\n        4.8934e-13, 1.3050e-11, 7.6979e-12, 1.7565e-14, 4.9380e-12, 1.3570e-10,\n        2.3473e-12, 1.3496e-11, 1.2515e-11, 1.1450e-12, 1.0147e-12, 1.8941e-11,\n        2.3434e-11, 1.3669e-12, 2.2276e-11, 3.9148e-12, 9.3192e-12, 1.5611e-11,\n        1.3433e-11, 7.5452e-12, 3.6201e-12, 8.3728e-12, 2.0787e-12, 4.8442e-13,\n        3.5726e-11, 8.7299e-12, 1.1567e-12, 3.1717e-11, 3.1880e-12, 3.7406e-13,\n        2.9842e-11, 3.3346e-11, 9.0927e-11, 2.9975e-11, 9.5348e-13, 1.9026e-11,\n        1.7026e-12, 3.0821e-11, 6.2195e-13, 3.0994e-12], device='cuda:0')"
      },
      "44": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n          0.0000e+00, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [-5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[9.8859e-16, 1.6766e-15, 6.6839e-15,  ..., 1.1997e-15, 0.0000e+00,\n         2.6504e-14],\n        [4.3003e-15, 5.3131e-16, 4.8592e-14,  ..., 3.1650e-14, 0.0000e+00,\n         1.4534e-13],\n        [8.3781e-12, 9.9935e-13, 6.3691e-11,  ..., 3.9717e-11, 0.0000e+00,\n         1.0498e-10],\n        ...,\n        [2.2334e-12, 2.7086e-13, 1.5270e-11,  ..., 7.2101e-12, 0.0000e+00,\n         7.8554e-12],\n        [9.7118e-15, 5.8925e-15, 2.1534e-14,  ..., 1.6115e-13, 0.0000e+00,\n         9.7193e-14],\n        [2.1374e-15, 9.0714e-16, 8.0825e-14,  ..., 6.0784e-14, 0.0000e+00,\n         4.1355e-15]], device='cuda:0')"
      },
      "45": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([4.1587e-12, 1.0258e-11, 5.2222e-08, 1.6983e-12, 1.7321e-08, 1.0815e-08,\n        2.5725e-09, 1.5157e-09, 5.2759e-08, 1.0933e-09, 1.3958e-09, 4.9774e-09,\n        6.0315e-11, 1.1297e-09, 2.4159e-10, 4.3801e-10, 3.9493e-09, 9.7146e-10,\n        1.3592e-10, 4.3434e-11, 3.7198e-09, 1.7985e-10, 5.0085e-09, 5.3230e-10,\n        4.5657e-09, 1.6291e-08, 1.9914e-09, 2.0465e-09, 7.3020e-10, 1.6358e-08,\n        5.6904e-10, 2.2056e-09, 8.5879e-08, 1.0431e-09, 5.6125e-10, 3.2186e-11,\n        5.6316e-09, 3.2565e-11, 3.4699e-09, 1.1054e-09, 2.6281e-09, 9.7948e-11,\n        2.9297e-11, 8.1461e-09, 2.6495e-12, 3.0328e-09, 1.3411e-09, 1.8016e-09,\n        3.4272e-10, 1.3033e-09, 1.0961e-08, 6.9497e-10, 4.2804e-09, 7.9403e-10,\n        4.4144e-10, 2.1614e-08, 1.2758e-10, 1.8357e-09, 2.2877e-09, 3.9338e-08,\n        4.0604e-09, 3.8439e-09, 2.0423e-10, 6.1403e-09, 3.5784e-09, 1.3649e-08,\n        3.9537e-11, 7.4309e-09, 5.3982e-10, 1.8043e-09, 1.8412e-09, 5.4105e-09,\n        2.2138e-10, 1.4547e-08, 1.5592e-08, 1.2709e-08, 8.9123e-10, 9.3765e-10,\n        2.7034e-10, 8.4813e-11, 3.7463e-09, 2.6543e-10, 4.1189e-10, 2.9595e-10,\n        3.7295e-09, 9.4862e-12, 1.1541e-08, 1.6290e-09, 8.8268e-10, 3.4934e-12,\n        2.3644e-11, 2.0222e-09, 6.0008e-09, 4.0728e-09, 1.1907e-08, 6.8144e-10,\n        1.8938e-08, 1.2543e-08, 5.9406e-10, 6.5294e-10, 2.1073e-09, 7.4401e-10,\n        1.3450e-09, 1.9550e-08, 2.8603e-09, 2.6739e-09, 4.2038e-09, 2.9097e-10,\n        2.9240e-09, 4.4050e-10, 1.1259e-10, 7.1108e-10, 7.1173e-10, 1.7799e-09,\n        2.9149e-10, 3.9559e-09, 4.6741e-10, 2.1354e-08, 2.0059e-10, 1.2378e-08,\n        2.2909e-08, 1.8146e-09, 1.6237e-08, 9.5174e-11, 8.0831e-10, 2.9796e-08,\n        1.6219e-09, 8.5196e-10, 8.6713e-10, 2.6594e-09, 7.3665e-11, 1.2729e-09,\n        6.7353e-08, 5.9664e-09, 9.2641e-11, 4.0398e-09, 2.3743e-09, 2.5249e-09,\n        2.1754e-08, 9.2357e-09, 1.8878e-09, 4.1877e-10, 5.8957e-09, 3.4569e-11,\n        2.3900e-11, 4.2855e-09, 3.0735e-09, 3.6681e-09, 5.8919e-11, 9.3767e-10,\n        1.2395e-09, 6.3439e-13, 4.7891e-09, 1.1601e-09, 2.2135e-10, 2.4290e-08,\n        1.4539e-09, 6.0592e-09, 1.7241e-11, 7.2027e-10, 1.6535e-10, 1.4746e-08,\n        2.7548e-11, 7.3235e-09, 2.5575e-08, 4.6651e-09, 7.4057e-10, 3.9279e-09,\n        6.2221e-11, 1.3551e-12, 3.4023e-09, 7.8422e-09, 1.9002e-08, 2.1820e-09,\n        2.6323e-09, 4.9445e-11, 6.8330e-09, 1.6434e-09, 1.3776e-10, 1.6670e-10,\n        4.0919e-10, 4.1001e-09, 1.8277e-11, 1.2644e-10, 5.6685e-10, 7.4711e-08,\n        1.1766e-09, 1.6209e-11, 1.8771e-10, 5.9601e-10, 1.2020e-08, 5.0851e-11,\n        8.8749e-09, 2.8497e-09, 1.5560e-09, 1.8228e-09, 1.5711e-10, 4.5174e-09,\n        3.1762e-09, 5.1915e-09, 4.7196e-09, 2.3353e-11, 2.3779e-08, 7.4926e-12,\n        3.0716e-09, 1.3894e-11, 1.6466e-09, 7.2011e-09, 1.8283e-10, 1.4364e-09,\n        4.9520e-09, 1.7457e-09, 1.6435e-08, 2.3800e-08, 4.2778e-08, 6.0696e-09,\n        7.8245e-10, 1.0180e-09, 3.0060e-08, 1.6216e-10, 2.8963e-09, 4.1981e-08,\n        2.3454e-10, 3.2466e-09, 3.8639e-08, 5.9683e-10, 8.6793e-10, 4.0462e-09,\n        6.4395e-09, 2.2091e-09, 2.4878e-09, 1.8647e-09, 3.7747e-09, 1.3925e-09,\n        4.2200e-09, 5.5317e-09, 6.2899e-10, 4.6773e-10, 2.8839e-09, 8.6494e-11,\n        2.6354e-08, 1.8467e-09, 1.1390e-10, 1.3904e-08, 4.2821e-09, 1.9834e-11,\n        5.1507e-09, 2.0828e-09, 4.6503e-08, 3.2928e-08, 7.1124e-10, 2.0282e-09,\n        8.2025e-10, 1.2842e-08, 2.6166e-10, 7.1461e-11], device='cuda:0')"
      },
      "46": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([1.6910e-14, 1.2480e-13, 1.1601e-10, 2.3355e-15, 3.0010e-11, 2.0215e-11,\n        4.4793e-12, 4.8143e-12, 1.1492e-10, 2.6469e-12, 2.6985e-12, 8.5292e-12,\n        1.1463e-13, 1.8682e-12, 5.5823e-13, 1.6039e-12, 7.8503e-12, 2.0151e-12,\n        3.3576e-13, 2.2860e-13, 5.6979e-12, 9.2634e-13, 1.4247e-11, 8.6733e-13,\n        1.5434e-11, 1.9867e-11, 4.0770e-12, 3.8751e-12, 3.8925e-12, 3.1569e-11,\n        1.8073e-12, 4.2801e-12, 1.7605e-10, 1.4493e-12, 8.9209e-13, 1.5721e-13,\n        9.1879e-12, 5.9724e-13, 5.2179e-12, 2.8333e-12, 3.8141e-12, 1.4796e-13,\n        6.8793e-14, 1.5738e-11, 5.1115e-15, 6.4912e-12, 1.7920e-12, 5.3195e-12,\n        1.1283e-12, 1.7709e-12, 1.9503e-11, 8.7206e-13, 5.0479e-12, 1.2134e-12,\n        2.2808e-12, 6.0230e-11, 4.1495e-13, 2.7983e-12, 3.7216e-12, 7.9669e-11,\n        5.7428e-12, 6.2840e-12, 6.7150e-13, 6.4550e-12, 7.1654e-12, 2.1805e-11,\n        8.8047e-14, 1.2142e-11, 8.0042e-13, 2.6405e-12, 3.3404e-12, 1.3347e-11,\n        2.5740e-13, 2.5090e-11, 3.9393e-11, 1.8914e-11, 1.2205e-12, 1.2580e-12,\n        3.5579e-13, 5.7782e-14, 5.2497e-12, 3.6730e-13, 7.9667e-13, 4.2524e-13,\n        9.5440e-12, 6.8857e-14, 1.6096e-11, 6.9227e-12, 1.9753e-12, 9.1232e-15,\n        3.3390e-15, 3.7197e-12, 1.0894e-11, 5.9461e-12, 1.4687e-11, 6.3752e-13,\n        3.5383e-11, 1.9679e-11, 1.1714e-12, 2.5271e-12, 4.2596e-12, 1.4625e-12,\n        3.1234e-12, 4.6063e-11, 5.2533e-12, 3.5770e-12, 6.1751e-12, 5.9055e-13,\n        2.6355e-12, 1.0549e-12, 3.5702e-13, 2.8473e-12, 2.5490e-12, 2.3952e-12,\n        3.6645e-13, 1.0950e-11, 1.3492e-12, 7.3201e-11, 8.3040e-13, 2.8953e-11,\n        4.1346e-11, 3.9590e-12, 3.8575e-11, 1.0790e-12, 2.3888e-12, 6.4983e-11,\n        1.4648e-12, 9.7196e-13, 1.9230e-12, 5.2255e-12, 2.3006e-13, 2.0217e-12,\n        2.4254e-10, 6.7560e-12, 2.4160e-13, 9.0379e-12, 8.9043e-12, 5.1645e-12,\n        5.2840e-11, 9.6082e-12, 3.8779e-12, 1.0046e-12, 9.4522e-12, 1.1268e-13,\n        8.0429e-14, 1.4449e-11, 4.9085e-12, 5.7726e-12, 1.6560e-13, 1.3987e-12,\n        4.1301e-12, 7.8962e-15, 6.1451e-12, 1.8029e-12, 3.4600e-13, 3.4646e-11,\n        1.9719e-12, 8.0123e-12, 1.7280e-14, 1.3620e-12, 3.6441e-13, 3.0919e-11,\n        2.8876e-13, 1.6381e-11, 6.1469e-11, 6.1147e-12, 1.5772e-12, 8.6822e-12,\n        2.6229e-13, 4.7375e-14, 6.7130e-12, 3.4496e-11, 4.2818e-11, 3.0251e-12,\n        7.2428e-12, 1.3377e-13, 9.2929e-12, 2.5225e-12, 9.0127e-13, 9.4583e-13,\n        1.0173e-12, 1.0329e-11, 6.1456e-14, 3.1722e-13, 2.5713e-12, 2.0816e-10,\n        1.4206e-12, 5.5624e-14, 2.9226e-13, 1.9748e-12, 3.6542e-11, 2.1833e-13,\n        1.3833e-11, 4.0476e-12, 3.4298e-12, 4.5307e-12, 2.3525e-13, 6.3536e-12,\n        4.3504e-12, 7.2294e-12, 9.1392e-12, 1.1665e-13, 2.7484e-11, 4.8954e-14,\n        4.9409e-12, 4.2065e-14, 2.4373e-12, 1.2343e-11, 1.3425e-12, 3.1453e-12,\n        7.6306e-12, 2.2869e-12, 3.6529e-11, 3.6614e-11, 6.5797e-11, 1.6088e-11,\n        1.8145e-12, 1.4024e-12, 5.9630e-11, 5.8338e-13, 4.2019e-12, 1.0144e-10,\n        5.7183e-13, 8.1506e-12, 1.0684e-10, 1.0319e-12, 4.1657e-12, 4.7616e-12,\n        1.0626e-11, 4.4567e-12, 4.5365e-12, 2.5797e-12, 9.7351e-12, 2.4483e-12,\n        5.9293e-12, 1.1525e-11, 9.9460e-13, 7.3762e-13, 5.6430e-12, 2.3315e-13,\n        8.4356e-11, 3.0493e-12, 4.7529e-13, 2.0433e-11, 4.7109e-12, 4.8635e-14,\n        9.8702e-12, 2.4215e-12, 1.1779e-10, 1.3514e-10, 1.0660e-12, 2.3356e-12,\n        1.3917e-12, 3.1707e-11, 6.4994e-13, 1.4066e-13], device='cuda:0')"
      },
      "47": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([4.2961e-14, 8.7322e-14, 1.4351e-10, 2.1152e-14, 4.2460e-11, 2.6469e-11,\n        7.3760e-12, 5.0806e-12, 1.4133e-10, 3.1921e-12, 4.4323e-12, 1.2431e-11,\n        2.5971e-13, 3.7291e-12, 1.0639e-12, 1.4075e-12, 1.1403e-11, 3.1617e-12,\n        5.0781e-13, 3.0541e-13, 1.0853e-11, 9.1541e-13, 1.2907e-11, 1.8306e-12,\n        1.4299e-11, 4.0954e-11, 6.2449e-12, 5.5499e-12, 3.2075e-12, 4.6348e-11,\n        1.6282e-12, 5.7856e-12, 2.3388e-10, 2.5118e-12, 1.7461e-12, 2.2901e-13,\n        1.3107e-11, 6.1610e-13, 1.0645e-11, 3.5860e-12, 6.6771e-12, 4.1129e-13,\n        1.2475e-13, 2.4782e-11, 1.3143e-14, 6.8017e-12, 3.8740e-12, 6.3224e-12,\n        1.2707e-12, 4.0446e-12, 3.3068e-11, 1.4648e-12, 1.0696e-11, 2.1730e-12,\n        1.8403e-12, 5.4709e-11, 6.1511e-13, 4.2284e-12, 5.9706e-12, 1.1046e-10,\n        9.3553e-12, 8.9044e-12, 8.1452e-13, 1.7360e-11, 1.0369e-11, 3.2166e-11,\n        1.9433e-13, 1.8360e-11, 1.6675e-12, 5.5954e-12, 5.9034e-12, 1.6726e-11,\n        4.6873e-13, 4.1145e-11, 4.4548e-11, 3.3011e-11, 2.8812e-12, 3.0741e-12,\n        9.2058e-13, 9.4088e-14, 1.1418e-11, 7.5864e-13, 1.2825e-12, 9.7047e-13,\n        8.4739e-12, 2.0804e-13, 3.3725e-11, 5.9136e-12, 3.1804e-12, 1.2696e-14,\n        3.2986e-14, 5.1512e-12, 1.4728e-11, 1.0767e-11, 3.0582e-11, 1.5479e-12,\n        4.9221e-11, 3.1968e-11, 1.5861e-12, 2.6995e-12, 6.5358e-12, 2.3632e-12,\n        4.4830e-12, 5.0080e-11, 8.4217e-12, 6.6929e-12, 1.2084e-11, 1.0948e-12,\n        6.5900e-12, 1.2920e-12, 3.5401e-13, 2.7275e-12, 1.9555e-12, 4.0471e-12,\n        1.0383e-12, 1.1129e-11, 1.8654e-12, 5.2068e-11, 1.0028e-12, 3.5284e-11,\n        6.4567e-11, 5.9778e-12, 4.1777e-11, 1.6377e-12, 2.9711e-12, 7.9615e-11,\n        3.7869e-12, 2.5918e-12, 2.8660e-12, 7.9046e-12, 3.8584e-13, 4.0753e-12,\n        1.7169e-10, 1.4956e-11, 3.8554e-13, 9.7635e-12, 7.0886e-12, 5.7711e-12,\n        5.3813e-11, 2.5319e-11, 5.5632e-12, 1.2976e-12, 1.4572e-11, 2.4282e-13,\n        1.1378e-13, 1.2989e-11, 6.4210e-12, 1.0382e-11, 3.6319e-13, 3.0793e-12,\n        4.3029e-12, 3.0514e-16, 1.2469e-11, 2.8085e-12, 6.9136e-13, 6.0078e-11,\n        3.8611e-12, 1.5585e-11, 2.3368e-14, 2.5282e-12, 7.0607e-13, 4.1235e-11,\n        3.6250e-13, 2.0890e-11, 7.2194e-11, 1.1930e-11, 1.7617e-12, 9.8458e-12,\n        3.9597e-13, 5.2749e-14, 7.9145e-12, 2.3298e-11, 4.7184e-11, 6.6788e-12,\n        6.5383e-12, 1.9624e-13, 1.9491e-11, 5.0178e-12, 9.2949e-13, 8.2414e-13,\n        1.6430e-12, 1.1008e-11, 8.2518e-14, 5.0388e-13, 2.2672e-12, 2.0388e-10,\n        2.9045e-12, 1.1521e-13, 6.7653e-13, 2.3480e-12, 3.0208e-11, 4.4789e-13,\n        2.2134e-11, 9.0086e-12, 4.0276e-12, 5.9263e-12, 5.2535e-13, 1.1372e-11,\n        8.3141e-12, 1.5294e-11, 1.1722e-11, 3.7822e-13, 6.3699e-11, 1.5565e-13,\n        7.1998e-12, 3.1474e-14, 5.0979e-12, 1.9223e-11, 1.1497e-12, 4.4189e-12,\n        1.3831e-11, 3.9063e-12, 4.5823e-11, 6.2403e-11, 1.2055e-10, 1.4502e-11,\n        2.8466e-12, 2.4143e-12, 7.6664e-11, 7.2881e-13, 6.7959e-12, 1.1203e-10,\n        8.8550e-13, 1.0051e-11, 1.0704e-10, 1.0447e-12, 3.1626e-12, 1.1666e-11,\n        1.8783e-11, 5.9869e-12, 7.4090e-12, 5.3331e-12, 8.5175e-12, 4.2652e-12,\n        1.1864e-11, 1.5898e-11, 1.8333e-12, 1.4560e-12, 6.7515e-12, 3.4409e-13,\n        6.5633e-11, 5.2101e-12, 8.4061e-13, 3.9307e-11, 1.0148e-11, 1.2214e-13,\n        1.4124e-11, 5.1210e-12, 1.2752e-10, 9.2959e-11, 2.3210e-12, 4.9945e-12,\n        1.8612e-12, 3.5700e-11, 9.6382e-13, 3.1836e-13], device='cuda:0')"
      },
      "48": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [-5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45],\n        [ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          0.0000e+00,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[5.9575e-15, 1.0825e-15, 1.6920e-14,  ..., 1.9717e-14, 0.0000e+00,\n         2.6737e-15],\n        [1.2008e-15, 4.9309e-15, 6.2334e-14,  ..., 8.4273e-14, 0.0000e+00,\n         1.4732e-13],\n        [9.9611e-12, 1.9475e-13, 4.1646e-11,  ..., 1.0580e-11, 0.0000e+00,\n         5.9738e-11],\n        ...,\n        [1.3149e-12, 5.4443e-14, 1.5148e-11,  ..., 4.4843e-12, 0.0000e+00,\n         4.8208e-12],\n        [1.6362e-13, 1.9082e-16, 4.9669e-15,  ..., 2.0852e-14, 0.0000e+00,\n         2.1374e-14],\n        [6.1442e-16, 3.6497e-16, 1.3591e-13,  ..., 1.1544e-14, 0.0000e+00,\n         9.1086e-14]], device='cuda:0')"
      },
      "49": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([5.0947e-12, 4.4473e-11, 2.3858e-08, 2.4776e-11, 3.8914e-09, 1.2540e-08,\n        5.8821e-09, 1.3252e-09, 6.2616e-10, 1.2800e-09, 1.6520e-08, 4.8650e-09,\n        9.0830e-10, 3.2470e-09, 1.4097e-11, 9.2087e-10, 2.4485e-09, 1.1582e-09,\n        1.8504e-10, 4.1756e-10, 5.9863e-09, 1.1563e-10, 4.8867e-10, 1.8430e-09,\n        3.0868e-09, 1.6117e-09, 2.5546e-09, 5.1959e-09, 9.2708e-10, 9.3659e-09,\n        1.2694e-08, 1.4827e-08, 8.0513e-08, 2.5380e-10, 3.5108e-09, 6.9901e-11,\n        7.1885e-09, 1.1870e-10, 2.6520e-08, 7.6939e-10, 4.2342e-09, 9.9113e-11,\n        4.3879e-10, 1.7272e-08, 7.8637e-12, 8.6879e-10, 3.7030e-09, 2.5574e-09,\n        5.0259e-10, 4.8721e-09, 8.9902e-09, 3.0031e-09, 9.8094e-09, 7.9978e-10,\n        1.0168e-09, 2.2526e-08, 7.7947e-11, 3.4379e-09, 1.7969e-08, 2.2301e-08,\n        2.8514e-09, 2.5430e-09, 1.0117e-10, 1.7587e-08, 8.3959e-10, 2.3366e-08,\n        2.8388e-11, 5.9323e-09, 3.3395e-09, 3.3541e-09, 6.8937e-09, 1.1108e-08,\n        2.9647e-09, 8.1908e-09, 3.6194e-09, 1.3858e-08, 7.9422e-09, 3.1311e-09,\n        4.8211e-10, 1.8586e-11, 1.6506e-08, 1.1945e-09, 9.5226e-10, 6.4057e-09,\n        1.5079e-09, 2.2129e-10, 3.5769e-08, 1.1017e-09, 1.2612e-09, 5.3666e-11,\n        1.3298e-11, 1.3558e-09, 5.0779e-09, 1.1164e-09, 3.1166e-08, 9.6190e-10,\n        1.8446e-08, 2.5491e-08, 3.6745e-09, 2.2209e-09, 2.2460e-09, 9.5263e-10,\n        4.2489e-10, 7.3610e-09, 4.2538e-09, 3.0078e-09, 4.3458e-09, 3.3514e-10,\n        2.8249e-09, 3.4042e-09, 1.6088e-09, 4.0317e-10, 1.9726e-08, 4.9554e-10,\n        6.1749e-10, 1.9661e-08, 6.3633e-10, 4.0883e-09, 7.4328e-10, 1.2593e-09,\n        4.2560e-08, 3.5303e-09, 1.5861e-08, 2.8327e-10, 1.6853e-10, 4.9753e-08,\n        6.4933e-09, 3.3917e-10, 1.7083e-10, 7.2506e-09, 1.0543e-12, 1.4841e-09,\n        2.6074e-08, 1.6477e-08, 6.6996e-11, 1.6666e-09, 4.0297e-09, 1.3637e-09,\n        1.4747e-08, 1.0786e-08, 1.7243e-09, 1.5718e-09, 7.3272e-09, 1.1334e-11,\n        5.6128e-10, 8.4648e-10, 5.4316e-10, 8.9691e-09, 5.2223e-10, 8.5303e-10,\n        1.1820e-09, 1.5417e-09, 3.4850e-09, 2.6297e-09, 1.0105e-09, 1.2003e-08,\n        2.6695e-10, 3.9351e-08, 1.4173e-11, 7.2817e-10, 2.9532e-10, 1.0118e-09,\n        1.5196e-11, 1.8407e-09, 1.1375e-08, 7.4185e-09, 4.5602e-10, 2.1160e-09,\n        2.1356e-12, 3.6921e-10, 1.7152e-10, 3.1605e-09, 2.5365e-09, 3.1362e-09,\n        1.6517e-09, 4.1372e-10, 1.3535e-08, 7.8884e-09, 7.7845e-11, 1.9900e-11,\n        2.1493e-09, 5.8254e-09, 1.6315e-11, 1.2494e-09, 8.7076e-10, 1.7093e-08,\n        3.4473e-10, 8.8320e-11, 9.2788e-10, 4.0309e-10, 1.7197e-09, 2.0035e-11,\n        4.0691e-09, 1.3312e-09, 9.4103e-09, 1.1462e-09, 5.2530e-10, 3.8064e-09,\n        1.8495e-08, 1.7844e-09, 2.6190e-09, 5.5531e-11, 3.1594e-08, 7.0392e-12,\n        3.9121e-09, 1.5901e-11, 4.3035e-09, 4.2775e-09, 1.8034e-10, 2.6475e-09,\n        2.4955e-08, 1.4796e-09, 1.7597e-09, 2.1040e-08, 8.3409e-08, 1.0594e-09,\n        8.6943e-10, 8.8088e-10, 4.3999e-08, 4.8261e-11, 8.1679e-10, 8.3923e-08,\n        1.1339e-10, 6.3925e-09, 3.0066e-08, 5.2207e-10, 8.9954e-10, 1.3215e-08,\n        3.2176e-09, 9.5854e-10, 1.7540e-08, 3.7799e-09, 2.1637e-10, 1.3968e-09,\n        6.0081e-09, 5.9623e-09, 8.9291e-10, 1.1121e-09, 6.4941e-10, 4.0829e-10,\n        3.6919e-09, 2.7609e-09, 2.2553e-10, 3.8084e-08, 5.0894e-09, 4.6730e-11,\n        4.1755e-09, 1.7187e-08, 2.1480e-08, 1.8723e-08, 5.9150e-10, 2.0446e-09,\n        8.1711e-10, 7.7826e-09, 8.4481e-11, 2.1635e-11], device='cuda:0')"
      },
      "50": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([3.9446e-15, 1.7013e-13, 3.5363e-11, 3.7492e-13, 3.3681e-12, 2.8597e-11,\n        1.6346e-11, 2.9354e-12, 6.1217e-12, 2.2087e-12, 3.5292e-11, 8.6545e-12,\n        3.9509e-12, 5.9049e-12, 5.7731e-14, 3.2940e-12, 4.0845e-12, 2.3014e-12,\n        7.1147e-13, 7.6498e-13, 1.1166e-11, 1.8883e-13, 9.4184e-13, 2.6065e-12,\n        5.8489e-12, 3.3253e-12, 3.2642e-12, 2.2662e-11, 6.0461e-12, 1.1400e-11,\n        1.6907e-11, 3.2851e-11, 1.1364e-10, 2.9929e-13, 1.3659e-11, 9.7027e-14,\n        1.4386e-11, 8.8877e-13, 8.3707e-11, 1.0545e-12, 4.3594e-12, 2.8653e-13,\n        7.8511e-13, 2.5977e-11, 7.0402e-16, 1.0098e-12, 8.8536e-12, 5.5622e-12,\n        2.5512e-12, 8.1338e-12, 1.7832e-11, 5.7583e-12, 1.7500e-11, 1.0162e-12,\n        2.1618e-12, 5.2277e-11, 2.0231e-13, 9.8465e-12, 2.5800e-11, 2.8825e-11,\n        3.0533e-12, 3.6857e-12, 1.6759e-13, 3.2632e-11, 1.3117e-12, 6.8141e-11,\n        7.9564e-14, 7.5297e-12, 8.4702e-12, 3.9836e-12, 1.6289e-11, 2.5989e-11,\n        7.2750e-12, 1.1007e-11, 4.7434e-12, 2.1002e-11, 1.6529e-11, 4.5184e-12,\n        6.7395e-13, 5.6244e-14, 2.7756e-11, 3.4398e-12, 2.5473e-12, 2.3198e-11,\n        1.9107e-12, 5.8781e-13, 7.6221e-11, 1.2223e-12, 2.4039e-12, 8.3100e-14,\n        8.3892e-16, 2.5367e-12, 8.1515e-12, 2.1474e-12, 6.4814e-11, 1.3938e-12,\n        2.4542e-11, 5.5510e-11, 5.1320e-12, 1.2880e-11, 4.2488e-12, 1.6749e-12,\n        7.2756e-13, 1.2870e-11, 7.7191e-12, 4.4262e-12, 7.2851e-12, 1.0160e-12,\n        3.7101e-12, 4.0345e-12, 2.5730e-12, 5.5181e-13, 3.5068e-11, 4.4474e-13,\n        6.8119e-13, 2.8753e-11, 1.5980e-12, 7.0007e-12, 2.6739e-12, 2.0930e-12,\n        1.0492e-10, 5.0731e-12, 3.6473e-11, 2.3785e-12, 5.0099e-13, 1.4207e-10,\n        1.8123e-11, 6.5164e-13, 4.7694e-13, 1.8496e-11, 4.1208e-15, 3.1593e-12,\n        3.6330e-11, 2.9122e-11, 1.7786e-13, 2.5210e-12, 1.3638e-11, 1.5864e-12,\n        2.6609e-11, 1.4127e-11, 3.4335e-12, 2.6931e-12, 9.5511e-12, 8.0765e-14,\n        7.9081e-13, 1.2941e-12, 4.2906e-13, 2.8804e-11, 1.7371e-12, 1.3982e-12,\n        3.5023e-12, 3.5601e-12, 5.6386e-12, 4.6609e-12, 2.2291e-12, 2.4865e-11,\n        4.5483e-13, 9.2786e-11, 1.9414e-14, 1.0120e-12, 6.6343e-13, 2.3220e-12,\n        6.3974e-14, 3.1198e-12, 2.3792e-11, 1.3209e-11, 4.7651e-13, 3.2238e-12,\n        2.8018e-14, 9.0746e-13, 2.7184e-13, 7.0493e-12, 3.6847e-12, 4.9255e-12,\n        3.0911e-12, 6.0458e-13, 2.0842e-11, 1.2759e-11, 8.6891e-13, 7.2465e-14,\n        7.7692e-12, 1.6348e-11, 2.9469e-14, 2.4112e-12, 3.3939e-12, 2.0863e-11,\n        7.9514e-13, 1.9575e-13, 2.9000e-12, 7.0354e-13, 2.6198e-12, 1.0132e-13,\n        4.8448e-12, 1.9972e-12, 1.7503e-11, 2.1070e-12, 7.6691e-13, 5.5358e-12,\n        6.0560e-11, 2.7921e-12, 5.4803e-12, 3.3723e-13, 3.7292e-11, 1.0099e-13,\n        9.6197e-12, 1.3410e-14, 1.4026e-11, 4.8940e-12, 1.2014e-12, 7.7915e-12,\n        7.4482e-11, 2.2428e-12, 4.2812e-12, 2.1836e-11, 1.3815e-10, 1.8123e-12,\n        1.8576e-12, 8.0104e-13, 1.3213e-10, 1.5174e-13, 6.6309e-13, 1.8447e-10,\n        2.9563e-13, 2.1325e-11, 6.4421e-11, 8.4506e-13, 1.7039e-12, 3.4314e-11,\n        3.1699e-12, 1.3728e-12, 3.4108e-11, 7.6055e-12, 3.7477e-13, 1.8525e-12,\n        9.8873e-12, 1.4587e-11, 1.7873e-12, 1.8545e-12, 1.0156e-12, 9.6140e-13,\n        4.7545e-12, 5.6392e-12, 1.0308e-12, 8.6286e-11, 7.9691e-12, 4.2131e-13,\n        6.2941e-12, 3.7936e-11, 2.9666e-11, 3.8387e-11, 9.4171e-13, 3.3925e-12,\n        7.0121e-13, 1.0920e-11, 1.8017e-13, 1.1953e-13], device='cuda:0')"
      },
      "51": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([ 5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([4.0085e-14, 1.9896e-13, 6.3540e-11, 2.1188e-13, 1.0350e-11, 3.1923e-11,\n        1.6062e-11, 4.1986e-12, 2.0162e-12, 3.9005e-12, 4.5847e-11, 1.1898e-11,\n        2.4898e-12, 9.4389e-12, 7.3258e-14, 2.8766e-12, 7.0997e-12, 3.3496e-12,\n        7.1951e-13, 1.3068e-12, 1.6906e-11, 3.5412e-13, 1.9017e-12, 5.5879e-12,\n        8.9230e-12, 4.6037e-12, 6.9851e-12, 1.3850e-11, 4.7247e-12, 2.4627e-11,\n        3.3518e-11, 3.8740e-11, 2.1753e-10, 6.3721e-13, 1.0240e-11, 2.2075e-13,\n        1.8404e-11, 1.0046e-12, 7.2519e-11, 2.3052e-12, 1.1353e-11, 3.5135e-13,\n        1.2879e-12, 4.6827e-11, 2.1746e-14, 2.1334e-12, 1.0096e-11, 7.5828e-12,\n        1.7853e-12, 1.3663e-11, 2.4683e-11, 6.9277e-12, 2.5599e-11, 2.1567e-12,\n        2.7751e-12, 5.6628e-11, 3.5789e-13, 8.7624e-12, 4.6061e-11, 5.8191e-11,\n        7.5362e-12, 6.2666e-12, 3.2223e-13, 4.6420e-11, 2.5206e-12, 6.0031e-11,\n        9.6485e-14, 1.5275e-11, 9.3053e-12, 9.0184e-12, 1.9217e-11, 3.0065e-11,\n        7.1173e-12, 2.2132e-11, 1.0586e-11, 3.5385e-11, 2.1528e-11, 8.7994e-12,\n        1.4174e-12, 5.5902e-14, 4.3463e-11, 3.2915e-12, 2.7927e-12, 1.7783e-11,\n        3.6524e-12, 7.5615e-13, 9.5006e-11, 3.0006e-12, 4.1401e-12, 6.3401e-14,\n        5.5399e-14, 3.7052e-12, 1.3467e-11, 3.2279e-12, 8.0151e-11, 2.3562e-12,\n        4.7693e-11, 6.5444e-11, 8.6732e-12, 7.2659e-12, 6.5077e-12, 2.9720e-12,\n        1.3360e-12, 2.0173e-11, 1.2261e-11, 7.9687e-12, 1.2155e-11, 1.3028e-12,\n        7.0867e-12, 9.0318e-12, 4.7541e-12, 1.1375e-12, 5.1654e-11, 1.1918e-12,\n        1.8711e-12, 5.4707e-11, 2.1565e-12, 1.0726e-11, 2.8754e-12, 3.7087e-12,\n        1.1534e-10, 1.0239e-11, 4.0964e-11, 2.8234e-12, 6.5787e-13, 1.2725e-10,\n        1.5988e-11, 1.0602e-12, 6.1938e-13, 1.9822e-11, 1.3761e-14, 4.5356e-12,\n        6.5283e-11, 4.1253e-11, 2.5504e-13, 4.5234e-12, 1.1883e-11, 3.3010e-12,\n        3.7629e-11, 2.7906e-11, 4.9464e-12, 4.6959e-12, 1.8779e-11, 3.3781e-13,\n        1.7459e-12, 2.3829e-12, 1.3694e-12, 2.3795e-11, 1.5385e-12, 2.4355e-12,\n        3.4555e-12, 3.5687e-12, 9.4784e-12, 6.5100e-12, 3.2782e-12, 3.3078e-11,\n        7.9779e-13, 1.0371e-10, 3.9868e-14, 2.2999e-12, 9.8232e-13, 3.1593e-12,\n        1.4712e-13, 5.7708e-12, 3.0069e-11, 1.9207e-11, 1.2066e-12, 6.6168e-12,\n        1.0036e-13, 1.2355e-12, 4.1535e-13, 9.2143e-12, 7.0252e-12, 9.2337e-12,\n        4.8243e-12, 1.2716e-12, 3.5453e-11, 2.1932e-11, 1.0855e-12, 9.8796e-14,\n        6.0680e-12, 1.6950e-11, 5.7725e-14, 3.5970e-12, 2.9387e-12, 4.7509e-11,\n        9.1291e-13, 3.6445e-13, 2.9615e-12, 1.1935e-12, 5.6111e-12, 3.1009e-13,\n        9.9820e-12, 3.9080e-12, 2.3213e-11, 3.6136e-12, 1.7276e-12, 9.9397e-12,\n        4.6262e-11, 5.5634e-12, 7.4620e-12, 6.2364e-13, 7.8575e-11, 2.4546e-13,\n        9.3465e-12, 1.8893e-14, 1.1978e-11, 1.0655e-11, 1.3060e-12, 8.0197e-12,\n        6.7673e-11, 3.1013e-12, 5.4417e-12, 5.9721e-11, 2.2024e-10, 3.3319e-12,\n        2.3911e-12, 2.0263e-12, 1.1678e-10, 2.3709e-13, 1.9108e-12, 2.2420e-10,\n        4.0373e-13, 1.7079e-11, 8.0098e-11, 1.1628e-12, 2.5577e-12, 3.6156e-11,\n        9.3960e-12, 2.7998e-12, 4.7994e-11, 1.0147e-11, 4.7224e-13, 3.9185e-12,\n        1.7163e-11, 1.6187e-11, 2.7015e-12, 3.5302e-12, 1.8195e-12, 1.1401e-12,\n        1.0332e-11, 8.0481e-12, 1.3956e-12, 1.0014e-10, 1.4000e-11, 5.9840e-13,\n        1.1506e-11, 4.6302e-11, 5.7676e-11, 5.2285e-11, 1.6572e-12, 5.5282e-12,\n        1.9896e-12, 2.1313e-11, 2.5921e-13, 8.9746e-14], device='cuda:0')"
      },
      "52": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [-5.6052e-45, -5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n         -5.6052e-45, -5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.0017e-14, 7.8582e-15, 1.4592e-14,  ..., 9.4973e-16, 4.5026e-16,\n         7.1545e-14],\n        [2.2888e-15, 6.7599e-15, 2.2422e-14,  ..., 8.8568e-14, 5.2899e-15,\n         1.1728e-14],\n        [1.3947e-14, 5.0148e-16, 7.4101e-15,  ..., 4.2796e-14, 4.4201e-16,\n         6.2229e-16],\n        ...,\n        [1.9419e-11, 3.1606e-10, 4.2962e-10,  ..., 7.3530e-10, 3.7532e-10,\n         1.8565e-10],\n        [5.7855e-12, 9.5852e-11, 1.3089e-10,  ..., 2.1153e-10, 1.1913e-10,\n         6.1779e-11],\n        [2.3785e-12, 2.7955e-11, 4.1566e-11,  ..., 6.8377e-11, 2.9529e-11,\n         1.4632e-11]], device='cuda:0')"
      },
      "53": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([7.8881e-14, 6.3152e-14, 1.8977e-13, 2.3054e-14, 9.4219e-14, 1.5038e-13,\n        4.2869e-14, 1.8302e-13, 4.4785e-13, 1.5137e-12, 5.4721e-14, 2.2723e-14,\n        6.3096e-13, 1.7238e-13, 1.1678e-12, 6.8947e-16, 1.1747e-13, 2.6376e-14,\n        3.0992e-13, 8.5476e-13, 7.3031e-14, 1.9072e-16, 3.2698e-15, 1.8174e-14,\n        1.1050e-13, 7.6735e-14, 1.6513e-13, 2.3113e-13, 2.1382e-14, 1.7573e-13,\n        4.8293e-13, 5.5988e-14, 3.3364e-13, 5.2803e-13, 1.1853e-12, 7.2859e-13,\n        1.8720e-13, 1.9919e-13, 3.6595e-13, 8.2147e-14, 1.4604e-14, 2.5379e-13,\n        2.6537e-13, 1.2814e-15, 1.6022e-12, 7.6221e-13, 7.2534e-15, 2.2835e-15,\n        8.6447e-14, 2.7267e-13, 2.9742e-13, 1.8446e-13, 2.1107e-13, 9.5885e-14,\n        8.7379e-13, 3.2639e-13, 4.8503e-14, 9.0398e-13, 6.9266e-15, 1.7224e-13,\n        2.5514e-13, 6.4299e-14, 5.3708e-15, 1.6619e-12, 4.1839e-14, 9.9817e-13,\n        6.1899e-15, 1.0451e-12, 1.9340e-13, 1.9650e-12, 2.3198e-12, 5.6223e-14,\n        5.5957e-13, 1.5452e-12, 1.1014e-12, 1.0241e-13, 2.7244e-12, 1.0432e-12,\n        3.8233e-12, 1.0297e-14, 1.0914e-13, 3.2736e-13, 1.2960e-13, 4.3734e-13,\n        1.3825e-12, 5.0885e-14, 1.0296e-13, 3.3356e-13, 3.9776e-12, 9.5502e-14,\n        7.5428e-13, 5.3684e-15, 3.1300e-12, 3.0092e-12, 5.1618e-13, 4.3518e-13,\n        3.4397e-15, 5.5677e-13, 5.7189e-13, 4.3511e-13, 1.2684e-12, 3.0959e-15,\n        1.6421e-14, 6.4785e-15, 3.3432e-15, 5.2926e-14, 1.3468e-14, 7.7023e-14,\n        3.1301e-13, 9.9282e-13, 1.5244e-13, 7.9610e-14, 4.5644e-14, 2.5114e-13,\n        6.9103e-14, 9.6152e-13, 3.5900e-13, 7.0068e-13, 1.0938e-12, 3.6962e-13,\n        1.4837e-14, 3.5075e-15, 6.9838e-13, 9.4324e-14, 6.4651e-14, 3.7231e-14,\n        1.2123e-12, 1.9558e-13, 3.0082e-14, 1.1104e-12, 4.7382e-14, 1.3405e-13,\n        1.8978e-13, 7.3520e-13, 1.3907e-12, 1.2823e-14, 5.3291e-14, 1.0583e-13,\n        7.5930e-14, 1.9925e-15, 3.1427e-13, 8.7406e-14, 2.5568e-14, 1.1490e-15,\n        2.5335e-13, 4.0270e-13, 6.0054e-13, 2.6327e-13, 4.6681e-14, 1.9170e-13,\n        4.4947e-15, 6.3730e-13, 6.6222e-15, 1.9210e-14, 9.5676e-13, 4.7198e-13,\n        1.5384e-15, 9.4925e-16, 6.3416e-14, 4.2448e-13, 1.8915e-12, 1.1733e-13,\n        1.1364e-13, 6.0120e-14, 1.6646e-13, 1.9633e-13, 1.6038e-13, 3.1349e-13,\n        3.6340e-15, 3.9254e-14, 4.4693e-13, 9.0008e-14, 1.1346e-13, 6.6382e-13,\n        4.1716e-16, 4.7976e-13, 6.2869e-13, 5.7207e-13, 4.8554e-15, 3.1855e-14,\n        3.6443e-14, 2.3160e-13, 5.9311e-16, 1.5005e-14, 1.0812e-13, 1.2628e-12,\n        1.2067e-13, 1.9709e-14, 5.3055e-13, 6.0151e-14, 5.1844e-14, 4.3504e-13,\n        3.0636e-13, 3.6488e-13, 1.9835e-13, 5.6412e-13, 6.8088e-16, 4.1053e-14,\n        3.0615e-13, 3.4820e-13, 5.7890e-15, 8.9699e-13, 4.2852e-14, 5.1847e-13,\n        2.8156e-14, 2.2078e-14, 8.6286e-15, 1.6982e-13, 1.0927e-12, 3.5052e-14,\n        1.7854e-13, 2.8843e-13, 6.9098e-13, 7.3444e-14, 3.8175e-15, 8.2270e-14,\n        4.7991e-14, 4.6594e-14, 2.3304e-14, 5.2066e-14, 1.3280e-14, 3.6116e-12,\n        9.5892e-13, 5.9606e-13, 1.8000e-15, 1.9781e-14, 1.9741e-13, 6.3775e-13,\n        1.8912e-12, 6.0356e-13, 2.1282e-14, 1.1591e-13, 6.7058e-13, 5.8677e-15,\n        3.8415e-13, 2.9286e-13, 7.9222e-13, 1.7309e-13, 1.2496e-13, 2.5783e-14,\n        5.1645e-14, 2.2981e-14, 6.1061e-13, 1.5739e-13, 1.5153e-13, 1.0705e-12,\n        1.8871e-13, 1.8069e-15, 4.1283e-13, 4.1875e-13, 1.7457e-12, 3.5813e-13,\n        5.1397e-13, 2.8108e-13, 4.4070e-16, 2.2262e-13, 1.2059e-27, 2.9533e-29,\n        1.8786e-28, 3.4172e-29, 1.9300e-28, 5.7458e-29, 3.1024e-28, 7.4796e-30,\n        1.6943e-28, 2.6350e-28, 2.4658e-28, 6.3341e-30, 1.6009e-31, 1.3458e-29,\n        7.5691e-29, 5.6567e-30, 1.3143e-28, 1.0067e-29, 1.1986e-28, 2.8843e-29,\n        1.8533e-30, 9.2954e-29, 5.2572e-30, 1.0832e-28, 1.5336e-30, 9.2693e-29,\n        2.0479e-28, 8.8563e-30, 5.3455e-28, 1.8232e-28, 2.9657e-29, 4.2721e-29,\n        5.4718e-29, 3.2744e-28, 3.0566e-29, 2.5194e-29, 1.0241e-29, 5.5503e-29,\n        1.4950e-28, 2.9559e-29, 5.2424e-30, 6.0785e-29, 1.2225e-29, 2.7036e-29,\n        2.5552e-29, 2.3045e-28, 9.4324e-29, 3.1600e-29, 3.3879e-29, 2.1037e-29,\n        4.2117e-29, 1.3955e-28, 1.4076e-28, 1.7641e-28, 7.1848e-29, 1.6523e-29,\n        1.4690e-28, 2.5285e-28, 3.4655e-28, 1.5126e-28, 1.2217e-28, 8.4675e-29,\n        5.8254e-30, 5.2432e-29, 3.9511e-31, 2.0464e-28, 5.4184e-28, 1.6891e-28,\n        4.5634e-30, 2.7119e-29, 1.6489e-28, 5.5138e-29, 8.9425e-29, 5.3785e-30,\n        4.6064e-29, 2.4913e-29, 8.7643e-29, 2.4689e-28, 8.5054e-28, 3.1158e-28,\n        3.3752e-29, 2.5570e-28, 3.0974e-29, 4.1328e-28, 5.0564e-29, 4.6668e-29,\n        4.1546e-28, 1.4445e-28, 7.4850e-29, 9.5776e-30, 1.6729e-29, 9.1067e-29,\n        9.5516e-29, 2.0099e-29, 7.7656e-29, 5.5221e-30, 7.3314e-30, 2.4000e-28,\n        1.2608e-29, 7.2246e-29, 1.0817e-28, 7.5457e-30, 6.5915e-29, 6.2286e-29,\n        2.5000e-29, 3.4303e-29, 2.1408e-29, 3.5735e-28, 1.4515e-28, 2.5846e-29,\n        1.9768e-29, 1.0513e-28, 1.1384e-28, 2.1380e-28, 9.0044e-29, 4.2135e-28,\n        1.1571e-28, 1.5676e-28, 3.6452e-28, 4.4757e-28, 2.7082e-28, 1.4265e-28,\n        1.6829e-28, 8.2413e-29, 1.8870e-29, 1.6904e-29, 9.3875e-28, 1.1028e-30,\n        1.7832e-28, 1.0616e-29, 4.1707e-29, 5.0833e-29, 1.3087e-28, 4.2289e-29,\n        1.1091e-29, 1.2456e-32, 2.8480e-30, 1.6753e-29, 3.7604e-29, 9.7723e-29,\n        9.2936e-30, 2.2078e-28, 8.4587e-28, 9.5487e-29, 2.7320e-28, 3.6403e-29,\n        1.4937e-28, 4.1200e-28, 1.8984e-28, 1.2722e-29, 1.3700e-29, 5.5483e-28,\n        9.5530e-29, 1.0740e-28, 4.9810e-30, 6.8336e-29, 2.3657e-28, 7.7465e-29,\n        4.0592e-31, 1.0684e-28, 3.1446e-29, 3.0608e-29, 1.7375e-28, 1.6061e-29,\n        1.1227e-29, 9.7449e-29, 6.4443e-29, 4.1266e-30, 3.7941e-29, 4.4174e-28,\n        2.6089e-29, 9.0657e-28, 4.6126e-28, 6.4994e-30, 1.5028e-28, 4.7655e-28,\n        4.7156e-28, 1.7825e-29, 2.2368e-29, 3.9071e-29, 2.7428e-29, 1.1799e-28,\n        2.3370e-30, 1.1898e-28, 6.0865e-29, 1.5310e-29, 5.5554e-29, 1.0276e-30,\n        1.9238e-28, 1.1782e-30, 2.3662e-28, 1.3208e-28, 1.4571e-28, 3.3100e-28,\n        3.0172e-28, 1.0937e-29, 6.1652e-28, 5.5999e-29, 1.4270e-28, 3.0816e-29,\n        9.1167e-31, 5.1665e-29, 7.1095e-29, 1.1859e-28, 1.3693e-29, 4.2502e-29,\n        2.7936e-30, 6.4468e-29, 9.6965e-29, 4.2475e-30, 1.9012e-28, 3.6103e-30,\n        1.0901e-28, 4.0638e-29, 2.0797e-28, 9.5189e-29, 5.4217e-30, 4.3073e-29,\n        1.0285e-28, 2.8053e-29, 3.1106e-29, 5.6677e-29, 1.1090e-28, 6.5107e-29,\n        8.5778e-29, 2.4811e-30, 1.1116e-28, 1.1952e-28, 3.1926e-28, 3.2411e-28,\n        8.0598e-30, 5.9329e-29, 8.4628e-30, 5.6572e-29, 1.2108e-30, 1.5605e-29,\n        1.8729e-29, 9.9933e-29, 3.4385e-30, 5.0681e-30, 2.2277e-29, 2.4897e-28,\n        1.7774e-29, 5.1055e-29, 1.3583e-29, 1.7013e-28, 3.5099e-28, 3.6544e-32,\n        3.3407e-28, 2.3512e-28, 3.2173e-28, 1.5142e-30, 5.2955e-29, 3.6289e-28,\n        1.5986e-29, 7.1608e-30, 4.6065e-10, 6.9577e-10, 7.3203e-12, 5.8465e-09,\n        1.5907e-09, 3.9905e-10, 1.0059e-09, 2.8342e-09, 5.6410e-11, 7.0456e-10,\n        4.6601e-09, 1.1058e-11, 3.2767e-10, 1.1874e-09, 5.8934e-09, 1.3348e-10,\n        1.7447e-09, 5.8619e-09, 4.0275e-10, 6.7440e-10, 1.8774e-09, 3.3208e-12,\n        2.4690e-09, 7.5427e-09, 4.7542e-10, 8.2207e-09, 1.7392e-10, 1.3149e-09,\n        1.0810e-09, 4.9524e-09, 1.0573e-09, 1.1469e-11, 1.8648e-10, 1.2138e-11,\n        2.2710e-09, 3.1490e-09, 3.1549e-09, 4.8447e-10, 9.2270e-11, 3.0847e-09,\n        5.8501e-09, 2.9384e-09, 3.4777e-09, 2.7137e-09, 1.1788e-09, 7.7262e-12,\n        2.1123e-09, 6.0154e-10, 2.8835e-09, 2.0549e-09, 1.1501e-08, 2.2368e-09,\n        3.8118e-10, 1.7719e-11, 9.8428e-10, 9.8174e-10, 6.3035e-09, 5.3795e-10,\n        1.5077e-09, 1.5234e-09, 1.5621e-09, 5.9962e-10, 5.7680e-10, 4.2371e-10,\n        1.6382e-10, 1.3198e-11, 3.4268e-09, 1.2779e-09, 1.7378e-09, 2.2539e-09,\n        1.6295e-10, 5.2861e-11, 3.2192e-11, 7.6448e-09, 2.4748e-10, 1.0440e-08,\n        2.0000e-09, 5.7968e-09, 2.6598e-09, 2.6915e-10, 1.6855e-09, 1.9677e-08,\n        6.8846e-10, 4.3500e-09, 1.5803e-10, 5.9127e-09, 2.1364e-09, 1.3540e-10,\n        3.7929e-11, 2.3665e-09, 3.4421e-09, 1.6055e-09, 1.1620e-11, 3.2958e-09,\n        8.4400e-09, 4.1595e-10, 1.7773e-10, 3.4495e-09, 3.2476e-10, 3.2649e-10,\n        4.3707e-10, 2.0542e-09, 5.4998e-11, 2.6963e-10, 7.2852e-10, 7.2741e-09,\n        4.4020e-09, 6.2670e-10, 6.6686e-11, 7.8158e-10, 3.5236e-10, 1.1754e-09,\n        2.1964e-09, 1.2169e-10, 1.6865e-09, 2.3472e-09, 6.2072e-10, 2.9321e-10,\n        4.1434e-09, 8.1221e-11, 2.8912e-10, 1.5894e-08, 2.9077e-09, 1.0040e-08,\n        3.6226e-09, 1.2015e-09, 8.9331e-11, 1.4694e-10, 1.8154e-09, 2.0351e-09,\n        1.1355e-09, 1.6054e-09, 4.0184e-10, 6.7986e-12, 1.7374e-09, 6.4987e-10,\n        9.3318e-10, 4.3033e-09, 4.8560e-10, 9.4932e-11, 7.6813e-13, 3.5586e-09,\n        1.5403e-09, 3.5899e-11, 1.0581e-09, 8.5527e-10, 2.2316e-10, 7.6102e-09,\n        3.2499e-10, 5.8725e-12, 1.0048e-09, 1.1367e-08, 5.9032e-10, 4.8207e-10,\n        4.5078e-09, 1.3521e-10, 5.1234e-11, 8.5184e-10, 7.4108e-13, 1.4445e-09,\n        3.6361e-09, 5.1320e-09, 1.4837e-10, 9.7027e-10, 1.3523e-09, 2.4325e-09,\n        1.1545e-09, 5.6041e-10, 9.3139e-11, 6.0844e-09, 3.7516e-10, 7.8498e-09,\n        4.9659e-09, 3.7467e-10, 1.2649e-09, 3.2852e-10, 2.0277e-11, 4.1618e-10,\n        3.3939e-09, 1.3091e-11, 2.3606e-09, 6.7454e-09, 1.7815e-11, 1.6044e-09,\n        4.4573e-09, 4.1934e-09, 1.1337e-09, 6.5518e-10, 5.1178e-11, 4.3137e-09,\n        7.8639e-11, 1.4605e-09, 6.6196e-09, 6.8517e-09, 1.1926e-09, 2.0088e-09,\n        3.5838e-09, 8.6628e-09, 8.3651e-11, 3.8425e-09, 4.7748e-10, 2.4063e-10,\n        8.9216e-09, 6.1204e-10, 2.8275e-10, 1.0915e-09, 3.5383e-10, 6.4193e-10,\n        9.9752e-10, 3.2701e-10, 1.8837e-09, 1.4131e-09, 4.5348e-09, 3.5161e-10,\n        5.2451e-11, 1.4271e-08, 4.4924e-10, 8.4499e-09, 1.7143e-11, 9.2384e-10,\n        7.2292e-10, 3.2112e-09, 3.6661e-09, 2.1645e-09, 4.6861e-09, 1.6604e-10,\n        2.9713e-09, 3.9374e-10, 1.8311e-08, 1.1854e-10, 4.3243e-10, 1.8001e-09,\n        1.8395e-09, 1.0749e-09, 3.7574e-09, 1.3596e-09, 3.8129e-10, 3.3836e-10,\n        2.2952e-09, 1.3751e-11, 3.3432e-09, 1.2618e-10, 3.1017e-13, 6.2582e-10,\n        3.1831e-09, 4.2028e-11, 8.4115e-10, 8.1981e-11, 1.1073e-10, 7.4330e-10,\n        2.6609e-09, 1.9219e-09, 1.4463e-09, 6.2773e-09, 1.8970e-09, 5.5348e-10],\n       device='cuda:0')"
      },
      "54": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        [-5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45, -5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45, -5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.2702e-10, 4.1625e-12, 2.8990e-12,  ..., 9.6173e-12, 5.1048e-11,\n         6.6579e-11],\n        [6.6957e-11, 3.0669e-12, 2.1936e-12,  ..., 8.1221e-12, 2.9007e-11,\n         3.7984e-11],\n        [1.2248e-09, 2.7770e-11, 2.4284e-11,  ..., 4.0189e-11, 4.8968e-10,\n         6.1500e-10],\n        ...,\n        [3.8525e-10, 9.6462e-12, 7.0010e-12,  ..., 1.7725e-11, 1.5404e-10,\n         2.0452e-10],\n        [9.5558e-12, 7.2371e-13, 2.2394e-13,  ..., 3.6473e-12, 3.7576e-12,\n         3.5078e-12],\n        [2.1599e-11, 1.7497e-12, 1.3623e-12,  ..., 5.8641e-12, 1.0624e-11,\n         1.1174e-11]], device='cuda:0')"
      },
      "55": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n        -5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45, -5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,\n        -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n        -5.6052e-45,  5.6052e-45, -5.6052e-45, -5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.6052e-45,  5.6052e-45,\n        -5.6052e-45], device='cuda:0')",
        "exp_avg_sq": "tensor([2.2734e-09, 1.2179e-09, 2.1638e-08, 1.1654e-08, 8.7356e-09, 1.1925e-11,\n        2.9041e-09, 3.0143e-09, 5.2079e-09, 5.1978e-10, 9.6106e-09, 1.2649e-10,\n        7.6180e-09, 4.4636e-12, 7.4967e-09, 1.8304e-11, 4.0195e-10, 3.5517e-09,\n        3.3235e-09, 5.6938e-09, 8.1539e-09, 1.2462e-09, 9.5351e-09, 4.3001e-09,\n        2.4014e-08, 6.4869e-09, 5.1684e-09, 1.2054e-09, 4.8196e-09, 1.1611e-08,\n        6.0805e-09, 2.1202e-08, 2.3599e-09, 2.7316e-10, 9.7926e-10, 1.0508e-09,\n        2.6955e-09, 8.9280e-09, 5.5316e-09, 1.1384e-09, 3.0832e-09, 8.6444e-09,\n        9.0908e-11, 3.3556e-09, 5.9787e-09, 2.1044e-09, 2.2044e-09, 6.9534e-09,\n        2.5981e-08, 1.2957e-09, 9.8671e-09, 5.0366e-09, 2.4580e-09, 2.0853e-09,\n        5.4186e-11, 1.9056e-08, 3.5981e-10, 8.0027e-09, 1.9482e-09, 1.8788e-08,\n        1.0672e-08, 7.2747e-09, 2.3001e-10, 3.8754e-09, 1.6653e-09, 1.8335e-09,\n        2.0162e-09, 1.9103e-08, 1.9124e-10, 9.0638e-09, 6.8541e-11, 7.3417e-09,\n        1.2151e-10, 7.5243e-09, 4.5795e-11, 2.0102e-09, 1.6758e-08, 9.5867e-10,\n        7.0868e-09, 1.3461e-08, 6.2308e-09, 4.4051e-09, 5.0777e-09, 7.0036e-09,\n        6.3432e-09, 8.9455e-09, 3.0707e-09, 4.3107e-09, 5.2586e-09, 2.8862e-10,\n        1.5060e-09, 1.8918e-09, 3.3457e-08, 9.1239e-10, 4.4927e-09, 4.6961e-12,\n        2.4087e-10, 7.6529e-11, 1.6932e-11, 5.2065e-10, 3.3444e-08, 2.3802e-10,\n        3.1675e-09, 2.7801e-08, 2.1891e-09, 5.8025e-10, 9.1144e-10, 2.3418e-10,\n        6.4023e-09, 1.5225e-08, 8.6314e-11, 4.4757e-09, 1.7830e-09, 1.7441e-09,\n        5.6885e-09, 1.3426e-09, 3.3304e-10, 1.8830e-10, 4.3342e-09, 3.3787e-09,\n        6.6496e-11, 4.7511e-09, 3.3609e-10, 3.3881e-08, 1.3898e-08, 3.4882e-08,\n        1.2996e-08, 8.1725e-10, 7.2263e-10, 2.0222e-09, 1.5701e-08, 2.5724e-10,\n        1.6144e-09, 7.2453e-09, 2.5374e-11, 1.0173e-10, 4.5093e-09, 1.0612e-08,\n        5.1931e-09, 1.2518e-09, 2.3298e-09, 6.1176e-09, 3.1834e-09, 2.9513e-09,\n        3.2016e-09, 7.0853e-09, 2.6427e-09, 4.0349e-09, 2.8563e-09, 1.8513e-08,\n        1.0303e-11, 2.1214e-09, 2.5928e-10, 2.1435e-10, 4.4752e-08, 1.9482e-08,\n        9.7065e-10, 1.8198e-08, 4.1317e-09, 2.5391e-08, 5.1226e-10, 5.2466e-10,\n        3.3700e-09, 1.4313e-11, 2.4735e-08, 4.1492e-09, 5.4605e-09, 8.7364e-10,\n        3.9040e-09, 1.7910e-08, 2.4529e-10, 4.2409e-09, 5.7701e-09, 1.5852e-09,\n        1.7589e-08, 4.3128e-09, 3.4382e-09, 1.2104e-10, 3.0907e-09, 9.3106e-10,\n        8.2117e-09, 3.8059e-09, 5.7461e-10, 1.1728e-09, 1.1351e-10, 2.2711e-09,\n        9.8124e-10, 1.4496e-08, 3.5843e-10, 1.3741e-09, 1.1874e-09, 2.6215e-10,\n        8.5794e-10, 9.3135e-09, 3.3762e-09, 2.9627e-09, 9.0343e-09, 4.9274e-09,\n        5.3025e-09, 2.4659e-09, 2.6375e-09, 1.1096e-08, 6.7776e-09, 1.8295e-10,\n        1.1666e-08, 3.9677e-10, 8.9851e-09, 1.1886e-08, 1.2233e-08, 1.1763e-08,\n        3.3699e-09, 2.5505e-09, 1.0891e-08, 4.3692e-10, 9.9659e-09, 4.7206e-09,\n        6.3555e-12, 6.8700e-09, 1.6387e-11, 5.1280e-09, 2.9559e-09, 7.3815e-08,\n        8.6311e-09, 2.6375e-08, 2.4055e-10, 1.1926e-09, 5.5636e-09, 4.2671e-10,\n        1.5367e-09, 8.5672e-11, 2.6454e-09, 6.7589e-09, 7.0970e-09, 1.0648e-09,\n        2.5231e-08, 3.3046e-09, 1.4597e-09, 1.5932e-10, 1.4346e-09, 1.8586e-11,\n        7.9435e-10, 6.0046e-09, 4.3985e-10, 4.0197e-10, 2.8913e-09, 1.8120e-08,\n        3.0262e-09, 9.4218e-09, 4.6120e-09, 2.2490e-09, 1.9008e-08, 4.4192e-10,\n        1.4920e-08, 6.8747e-09, 1.2970e-10, 3.7848e-10], device='cuda:0')"
      },
      "56": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[5.5761e-08, 1.2097e-08, 3.3469e-08,  ..., 1.4493e-08, 1.2408e-09,\n         1.9037e-10],\n        [5.0245e-10, 1.3094e-10, 2.9727e-10,  ..., 1.1948e-10, 1.1835e-11,\n         2.3523e-12],\n        [6.8433e-10, 1.4440e-10, 4.0833e-10,  ..., 1.8091e-10, 1.5308e-11,\n         2.1946e-12],\n        ...,\n        [5.5947e-10, 1.1797e-10, 3.3771e-10,  ..., 1.4378e-10, 1.3257e-11,\n         2.7554e-12],\n        [6.9308e-10, 1.4966e-10, 4.1660e-10,  ..., 1.8031e-10, 1.2851e-11,\n         1.2891e-12],\n        [6.1503e-10, 1.2976e-10, 3.8465e-10,  ..., 1.6464e-10, 1.4854e-11,\n         3.7643e-12]], device='cuda:0')"
      },
      "57": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([1.4373e-06, 1.3031e-08, 1.7601e-08, 1.8265e-08, 2.0941e-08, 2.4055e-08,\n        1.9125e-08, 1.4163e-08, 1.7889e-08, 1.5897e-08], device='cuda:0')"
      },
      "58": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[5.5755e-08, 1.2094e-08, 3.3459e-08,  ..., 1.4493e-08, 1.2380e-09,\n         1.8845e-10],\n        [5.0237e-10, 1.3091e-10, 2.9713e-10,  ..., 1.1948e-10, 1.1793e-11,\n         2.3242e-12],\n        [6.8427e-10, 1.4437e-10, 4.0823e-10,  ..., 1.8091e-10, 1.5277e-11,\n         2.1731e-12],\n        ...,\n        [5.5938e-10, 1.1793e-10, 3.3755e-10,  ..., 1.4378e-10, 1.3210e-11,\n         2.7234e-12],\n        [6.9305e-10, 1.4966e-10, 4.1656e-10,  ..., 1.8031e-10, 1.2839e-11,\n         1.2815e-12],\n        [6.1486e-10, 1.2970e-10, 3.8437e-10,  ..., 1.6464e-10, 1.4771e-11,\n         3.7080e-12]], device='cuda:0')"
      },
      "59": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([1.4373e-06, 1.3031e-08, 1.7601e-08, 1.8265e-08, 2.0941e-08, 2.4055e-08,\n        1.9125e-08, 1.4163e-08, 1.7889e-08, 1.5897e-08], device='cuda:0')"
      },
      "60": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, -5.6052e-45,  ..., -5.6052e-45,\n         -5.6052e-45, -5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        ...,\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45],\n        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n          5.6052e-45,  5.6052e-45]], device='cuda:0')",
        "exp_avg_sq": "tensor([[5.5761e-08, 1.2097e-08, 3.3469e-08,  ..., 1.4493e-08, 1.2408e-09,\n         1.9037e-10],\n        [5.0245e-10, 1.3094e-10, 2.9727e-10,  ..., 1.1948e-10, 1.1835e-11,\n         2.3523e-12],\n        [6.8433e-10, 1.4440e-10, 4.0833e-10,  ..., 1.8091e-10, 1.5308e-11,\n         2.1946e-12],\n        ...,\n        [5.5947e-10, 1.1797e-10, 3.3771e-10,  ..., 1.4378e-10, 1.3257e-11,\n         2.7554e-12],\n        [6.9308e-10, 1.4966e-10, 4.1660e-10,  ..., 1.8031e-10, 1.2851e-11,\n         1.2891e-12],\n        [6.1503e-10, 1.2976e-10, 3.8465e-10,  ..., 1.6464e-10, 1.4854e-11,\n         3.7643e-12]], device='cuda:0')"
      },
      "61": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([1.4373e-06, 1.3031e-08, 1.7601e-08, 1.8265e-08, 2.0941e-08, 2.4055e-08,\n        1.9125e-08, 1.4163e-08, 1.7889e-08, 1.5897e-08], device='cuda:0')"
      },
      "8": {
        "step": "tensor(3756.)",
        "exp_avg": "tensor([[ 3.3050e-06, -1.4826e-06, -2.4800e-07,  ..., -2.0102e-07,\n          0.0000e+00, -1.5793e-08],\n        [-1.4022e-06,  2.0576e-08, -5.3183e-07,  ..., -7.3770e-07,\n          0.0000e+00,  2.0642e-06],\n        [-1.6411e-07,  9.4075e-09, -6.2795e-07,  ..., -4.2783e-07,\n          0.0000e+00,  1.3053e-07],\n        ...,\n        [-5.7752e-07,  3.2708e-08,  1.2804e-06,  ..., -2.7579e-06,\n          0.0000e+00,  1.0007e-06],\n        [ 8.3237e-08, -8.3440e-09,  4.8899e-07,  ..., -5.5080e-07,\n          0.0000e+00,  1.0731e-06],\n        [-5.8672e-07,  2.2360e-08, -6.9772e-08,  ...,  8.9757e-07,\n          0.0000e+00, -3.7203e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.3802e-11, 2.4263e-12, 3.1701e-11,  ..., 3.9639e-11, 0.0000e+00,\n         4.7108e-12],\n        [2.2565e-11, 6.6864e-13, 6.1874e-11,  ..., 1.6449e-11, 0.0000e+00,\n         3.5699e-11],\n        [5.6193e-12, 4.7289e-12, 5.1355e-11,  ..., 1.0132e-11, 0.0000e+00,\n         9.5389e-12],\n        ...,\n        [1.2220e-11, 2.8527e-12, 2.1157e-11,  ..., 4.9039e-11, 0.0000e+00,\n         2.5976e-11],\n        [1.2748e-11, 2.3249e-12, 1.7470e-10,  ..., 8.4237e-11, 0.0000e+00,\n         8.7184e-11],\n        [4.4334e-11, 1.4753e-11, 5.2298e-11,  ..., 4.7643e-11, 0.0000e+00,\n         1.5019e-11]], device='cuda:0')"
      },
      "9": {
        "step": "tensor(3756.)",
        "exp_avg": "tensor([-7.2511e-06,  1.2043e-05,  9.9536e-07,  ..., -1.9313e-06,\n        -1.7364e-06,  1.1259e-06], device='cuda:0')",
        "exp_avg_sq": "tensor([2.5431e-09, 2.8160e-09, 2.0961e-09,  ..., 2.1836e-09, 2.7892e-09,\n        2.6763e-09], device='cuda:0')"
      },
      "10": {
        "step": "tensor(3756.)",
        "exp_avg": "tensor([[-1.4582e-07, -3.2010e-08,  8.8345e-08,  ..., -8.3714e-07,\n          2.0524e-07, -1.4161e-07],\n        [ 2.5680e-07, -6.5429e-07, -1.0099e-06,  ...,  5.4951e-08,\n          2.9744e-07, -5.2585e-07],\n        [ 2.2452e-08,  3.6399e-07, -1.0430e-06,  ...,  2.9101e-07,\n         -3.9084e-08,  2.8615e-07],\n        ...,\n        [ 1.3794e-07,  4.5533e-07, -2.2350e-07,  ..., -1.5740e-06,\n         -4.9998e-07, -2.8954e-07],\n        [-1.9641e-07,  4.6601e-07,  1.3403e-06,  ...,  8.0811e-08,\n         -3.8842e-07,  1.8245e-07],\n        [-4.6505e-08, -2.4896e-07,  1.7009e-07,  ..., -4.5994e-07,\n          2.9078e-07,  4.7989e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.2996e-12, 2.7972e-12, 3.1599e-12,  ..., 6.2085e-12, 3.3779e-12,\n         3.2575e-12],\n        [6.1283e-12, 7.4758e-12, 4.7177e-12,  ..., 7.6014e-12, 5.9618e-12,\n         5.2781e-12],\n        [5.7199e-12, 7.3769e-12, 6.4712e-12,  ..., 7.2539e-12, 7.4178e-12,\n         5.9844e-12],\n        ...,\n        [7.8544e-12, 5.5159e-12, 6.4635e-12,  ..., 1.2175e-11, 9.2106e-12,\n         5.3115e-12],\n        [5.7846e-12, 5.2673e-12, 7.7382e-12,  ..., 7.8881e-12, 1.0459e-11,\n         5.6582e-12],\n        [8.8593e-12, 7.0071e-12, 5.0220e-12,  ..., 7.6055e-12, 6.9697e-12,\n         8.3465e-12]], device='cuda:0')"
      },
      "11": {
        "step": "tensor(2504.)",
        "exp_avg": "tensor([[-8.8237e-07, -2.4691e-06,  8.1761e-07,  ...,  3.4484e-06,\n          0.0000e+00, -1.1416e-06],\n        [-6.1348e-07, -1.0298e-06, -1.4337e-07,  ...,  3.6510e-07,\n          0.0000e+00, -1.6920e-06],\n        [ 3.2471e-07, -9.0841e-08, -6.9405e-07,  ...,  5.1907e-06,\n          0.0000e+00, -1.2068e-07],\n        ...,\n        [ 1.7956e-07,  2.7996e-07,  1.6133e-06,  ...,  1.5855e-07,\n          0.0000e+00,  2.8758e-07],\n        [ 7.8083e-07, -9.2970e-07, -2.8678e-07,  ...,  5.4130e-08,\n          0.0000e+00, -9.7476e-07],\n        [-3.9340e-07, -1.8520e-10,  1.7226e-07,  ..., -1.1919e-06,\n          0.0000e+00, -7.6001e-08]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.6084e-11, 5.9483e-11, 1.5442e-11,  ..., 1.3043e-11, 0.0000e+00,\n         4.6604e-11],\n        [1.6724e-11, 3.6630e-11, 8.9965e-12,  ..., 2.2404e-11, 0.0000e+00,\n         1.1194e-10],\n        [9.5505e-12, 4.8172e-12, 5.9238e-11,  ..., 8.9772e-11, 0.0000e+00,\n         2.8023e-11],\n        ...,\n        [1.8681e-12, 4.6385e-12, 1.7741e-11,  ..., 8.7402e-12, 0.0000e+00,\n         1.5037e-11],\n        [2.3605e-11, 4.3548e-11, 8.2598e-11,  ..., 9.4976e-11, 0.0000e+00,\n         1.0488e-10],\n        [1.8798e-11, 3.3586e-13, 1.0003e-11,  ..., 4.4677e-11, 0.0000e+00,\n         2.4950e-11]], device='cuda:0')"
      },
      "12": {
        "step": "tensor(2504.)",
        "exp_avg": "tensor([ 2.9945e-05, -5.4609e-06,  9.1374e-06,  ..., -2.6447e-05,\n        -2.7284e-06,  3.6155e-06], device='cuda:0')",
        "exp_avg_sq": "tensor([3.4838e-09, 2.6143e-09, 2.7865e-09,  ..., 2.2366e-09, 4.0454e-09,\n        2.6550e-09], device='cuda:0')"
      },
      "13": {
        "step": "tensor(2504.)",
        "exp_avg": "tensor([[-2.7803e-07, -3.7306e-08,  1.1099e-06,  ...,  4.3446e-07,\n         -2.4451e-07,  5.3858e-07],\n        [ 1.9896e-06, -2.9311e-07, -1.4487e-06,  ..., -7.3788e-08,\n          1.1764e-07,  3.2649e-07],\n        [-2.2956e-07, -4.2627e-08,  5.5378e-07,  ..., -1.2870e-07,\n          1.8180e-07,  5.0446e-07],\n        ...,\n        [-4.1148e-07, -4.8784e-07, -1.2887e-06,  ..., -7.5846e-07,\n         -1.7294e-07,  2.4146e-07],\n        [ 3.0531e-07, -1.0245e-06,  3.2995e-07,  ..., -8.1471e-08,\n         -2.0297e-07, -7.3061e-07],\n        [-1.2335e-06,  4.5136e-07,  5.9361e-08,  ...,  1.0099e-07,\n          3.9649e-07,  2.4582e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.1799e-12, 1.9794e-12, 2.9180e-12,  ..., 3.1936e-12, 4.0457e-12,\n         6.2768e-12],\n        [6.2242e-12, 3.3000e-12, 8.1354e-12,  ..., 7.6155e-12, 8.7068e-12,\n         1.2365e-11],\n        [5.2473e-12, 5.0067e-12, 7.2966e-12,  ..., 8.2098e-12, 8.7280e-12,\n         3.6841e-12],\n        ...,\n        [6.8874e-12, 5.9269e-12, 9.9386e-12,  ..., 1.0752e-11, 9.0643e-12,\n         1.1276e-11],\n        [4.8607e-12, 8.6427e-12, 4.2208e-12,  ..., 3.9808e-12, 8.7024e-12,\n         1.2994e-11],\n        [5.0820e-12, 3.4031e-12, 7.2314e-12,  ..., 8.0791e-12, 9.0745e-12,\n         5.7353e-12]], device='cuda:0')"
      },
      "14": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[-5.5836e-08,  5.9145e-08,  3.1468e-06,  ...,  5.0826e-06,\n          0.0000e+00,  1.1467e-06],\n        [-1.2405e-06,  1.2115e-08,  3.8742e-07,  ...,  8.6341e-07,\n          0.0000e+00,  2.7410e-06],\n        [ 1.2883e-06,  2.8321e-07,  1.7832e-07,  ..., -3.5196e-07,\n          0.0000e+00,  8.1146e-08],\n        ...,\n        [-7.3957e-06,  3.4333e-07,  1.1083e-06,  ..., -1.6878e-07,\n          0.0000e+00,  8.5872e-08],\n        [ 3.9460e-08, -1.4093e-07, -1.3078e-08,  ...,  1.3118e-06,\n          0.0000e+00, -1.2148e-06],\n        [-6.4453e-07,  2.9878e-08, -1.1689e-06,  ...,  6.4297e-06,\n          0.0000e+00,  1.5700e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[9.2732e-12, 2.8539e-12, 2.8385e-10,  ..., 7.6557e-10, 0.0000e+00,\n         1.0338e-10],\n        [3.4943e-11, 2.6239e-12, 1.9723e-10,  ..., 5.9832e-11, 0.0000e+00,\n         5.7539e-11],\n        [5.1052e-11, 9.2555e-12, 1.2074e-11,  ..., 1.0524e-11, 0.0000e+00,\n         1.0740e-11],\n        ...,\n        [3.2242e-11, 5.1022e-12, 3.0993e-11,  ..., 1.1228e-11, 0.0000e+00,\n         1.1824e-11],\n        [8.6551e-12, 7.5511e-12, 9.6267e-12,  ..., 6.9740e-11, 0.0000e+00,\n         3.0929e-10],\n        [6.4763e-11, 2.8280e-12, 3.2130e-10,  ..., 1.6215e-09, 0.0000e+00,\n         3.4952e-11]], device='cuda:0')"
      },
      "15": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([-8.1167e-06,  1.7618e-05, -1.1373e-05,  ...,  1.3225e-05,\n        -2.5681e-06,  9.6634e-06], device='cuda:0')",
        "exp_avg_sq": "tensor([8.3071e-09, 5.5925e-09, 6.2431e-09,  ..., 3.4018e-09, 4.5848e-09,\n        7.2586e-09], device='cuda:0')"
      },
      "16": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[-1.5481e-07,  2.8678e-07,  5.5108e-07,  ..., -2.0205e-07,\n          1.5189e-08,  2.9531e-07],\n        [-3.4451e-08,  6.0454e-08,  6.9110e-07,  ..., -2.5115e-07,\n         -2.3807e-07, -1.4570e-07],\n        [ 5.8860e-09,  3.2474e-07, -5.4785e-07,  ...,  3.3104e-08,\n          1.4735e-08,  3.6657e-07],\n        ...,\n        [ 9.7897e-07, -7.7870e-07,  1.5313e-07,  ..., -5.1950e-07,\n          9.6189e-08,  4.9757e-07],\n        [-1.3136e-06, -5.9382e-07,  3.5333e-07,  ..., -1.2665e-08,\n          2.4294e-07,  3.7891e-07],\n        [-1.3422e-07,  1.2921e-06, -6.8474e-07,  ...,  1.9939e-07,\n          2.1495e-07, -3.0081e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[5.1165e-12, 4.8524e-12, 4.6993e-12,  ..., 2.6035e-12, 3.6581e-12,\n         3.5904e-12],\n        [6.9779e-12, 6.6144e-12, 1.1463e-11,  ..., 6.5572e-12, 1.0694e-11,\n         5.1223e-12],\n        [6.9161e-12, 7.6088e-12, 7.9918e-12,  ..., 8.8075e-12, 3.1320e-11,\n         4.0475e-12],\n        ...,\n        [9.1454e-12, 9.5437e-12, 5.1974e-12,  ..., 6.5395e-12, 5.4946e-12,\n         4.7305e-12],\n        [7.3494e-12, 6.6373e-12, 8.6532e-12,  ..., 5.7239e-12, 2.3864e-11,\n         3.8321e-12],\n        [8.5844e-12, 1.5904e-11, 6.9594e-12,  ..., 7.6194e-12, 8.6619e-12,\n         6.7029e-12]], device='cuda:0')"
      }
    },
    "param_groups": [
      {
        "lr": 0.005000500000000001,
        "name": "shared",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          0,
          1
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_256",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          2,
          3,
          4
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_512",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          5,
          6,
          7
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_768",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          8,
          9,
          10
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_1024",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          11,
          12,
          13
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_1280",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          14,
          15,
          16
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_1536",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          17,
          18,
          19
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_1792",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          20,
          21,
          22
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_2048",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          23,
          24,
          25
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_2304",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          26,
          27,
          28
        ]
      },
      {
        "lr": 0.005000500000000001,
        "name": "scale_2560",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.01,
        "params": [
          29,
          30,
          31
        ]
      },
      {
        "lr": 0.0025005,
        "name": "fusion",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.005,
        "params": [
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61
        ]
      }
    ]
  },
  "scheduler_state_dict": {
    "T_0": 10,
    "T_i": 10,
    "T_mult": 2,
    "eta_min": 1e-06,
    "T_cur": 5,
    "base_lrs": [
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.01,
      0.005
    ],
    "last_epoch": 5,
    "_step_count": 0,
    "_is_initial": false,
    "_get_lr_called_within_step": false,
    "_last_lr": [
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.005000500000000001,
      0.0025005
    ]
  },
  "metrics": {
    "val_acc": 81.822
  },
  "train_config": {
    "name": "david_training",
    "run_id": "20251012_050214",
    "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
    "model_variant": "clip_vit_l14",
    "num_classes": 1000,
    "preset": "clip_vit_l14_deep",
    "custom_config_path": null,
    "num_classes_override": null,
    "use_belly_override": null,
    "belly_expand_override": null,
    "progressive_training_override": true,
    "scale_warmup_epochs_override": {
      "256": 0,
      "512": 1,
      "768": 2,
      "1024": 3,
      "1280": 4,
      "1536": 5,
      "1792": 6,
      "2048": 7,
      "2304": 8,
      "2560": 9
    },
    "num_epochs": 10,
    "batch_size": 1024,
    "learning_rate": 0.01,
    "weight_decay": 1e-05,
    "warmup_epochs": 0,
    "use_rose_loss": true,
    "rose_initial_weight": 0.1,
    "rose_max_weight": 0.5,
    "rose_weight_schedule": "adaptive",
    "use_cayley_loss": false,
    "cayley_weight": 0.001,
    "scale_loss_balance": null,
    "use_mixed_precision": false,
    "gradient_clip": 5.0,
    "scheduler_type": "cosine_restarts",
    "min_lr": 1e-06,
    "freeze_strategy": "never",
    "freeze_threshold": 90.0,
    "unfreeze_on_plateau": true,
    "patience": 10,
    "track_gradients": true,
    "gradient_scale_threshold": 1e-07,
    "gradient_scale_multiplier": 5.0,
    "log_interval": 50,
    "val_interval": 1,
    "save_interval": 5,
    "log_fusion_weights": true,
    "log_loss_components": true,
    "save_format": "safetensors",
    "hf_repo": "AbstractPhil/gated-david",
    "upload_to_hub": true,
    "base_dir": "./david_training",
    "num_workers": 10,
    "pin_memory": true,
    "prefetch_factor": 4,
    "persistent_workers": true
  }
}