diff --git "a/weights/David-decoupled-cantor_scale/20251104_154540/best_model_acc78.90_metadata.json" "b/weights/David-decoupled-cantor_scale/20251104_154540/best_model_acc78.90_metadata.json" new file mode 100644--- /dev/null +++ "b/weights/David-decoupled-cantor_scale/20251104_154540/best_model_acc78.90_metadata.json" @@ -0,0 +1,589 @@ +{ + "epoch": 4, + "optimizer_state_dict": { + "state": { + "0": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 2.6236e-05, 6.2345e-05, 2.5701e-05, ..., -2.8827e-05,\n -2.0923e-05, -5.4955e-05],\n [-1.5383e-04, 4.8279e-05, -2.8512e-05, ..., -4.6996e-05,\n 1.6398e-05, -4.1416e-05],\n [ 1.0205e-04, 1.9383e-05, 2.6759e-05, ..., 4.3240e-05,\n 3.1507e-05, 6.0918e-05],\n ...,\n [-2.3955e-04, 4.0410e-04, 1.6243e-05, ..., -8.3899e-05,\n -1.0010e-04, 6.0447e-05],\n [-5.1438e-05, -5.3339e-05, 4.1330e-05, ..., -6.3216e-05,\n 5.7921e-05, -1.2938e-05],\n [ 3.6672e-04, -2.3131e-04, -6.1925e-05, ..., 8.5121e-05,\n 1.0576e-04, 6.1375e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.9018e-08, 1.1705e-07, 2.0728e-08, ..., 2.9990e-08, 2.2284e-08,\n 3.1879e-08],\n [1.8578e-07, 1.7595e-07, 2.9886e-08, ..., 3.8764e-08, 3.6896e-08,\n 3.4503e-08],\n [1.8004e-07, 1.3975e-07, 3.0209e-08, ..., 3.8941e-08, 3.6331e-08,\n 3.9674e-08],\n ...,\n [1.5577e-07, 1.3351e-07, 3.0342e-08, ..., 5.3238e-08, 3.8715e-08,\n 3.5396e-08],\n [1.6681e-07, 1.1049e-07, 3.4750e-08, ..., 4.1164e-08, 3.5784e-08,\n 2.8691e-08],\n [1.9474e-07, 2.9037e-07, 4.8510e-08, ..., 3.8831e-08, 1.3955e-07,\n 4.5897e-08]], device='cuda:0')" + }, + "1": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-4.6958e-04, -2.8509e-03, 3.7872e-03, -1.3392e-04, 1.3360e-03,\n 1.1212e-03, -1.9277e-03, -2.0797e-03, 8.1463e-04, -6.6961e-04,\n -2.8731e-04, -2.3856e-03, 2.5490e-03, -1.4769e-04, 1.5179e-03,\n 1.2037e-03, -3.8695e-04, 3.6498e-03, 4.3615e-03, -4.9218e-03,\n 1.7850e-03, 1.4139e-03, 1.1184e-04, 1.3795e-03, -8.9785e-04,\n -1.3502e-03, -8.0184e-03, -1.0945e-03, 1.6492e-03, 1.3800e-03,\n -3.1348e-03, 5.0298e-03, -9.6943e-04, 8.9452e-04, -1.5121e-03,\n -1.7712e-03, 3.6206e-04, -3.4468e-03, 2.8491e-03, -1.8378e-03,\n -5.2638e-04, 7.0780e-04, -9.1626e-04, -3.4491e-04, 4.0299e-03,\n -6.3373e-04, 3.4805e-04, 1.9163e-03, 1.5582e-03, -2.2719e-03,\n -6.7960e-04, -1.7876e-03, -9.4374e-04, -3.2924e-03, 1.6594e-03,\n -2.7129e-03, -1.0439e-04, 2.8681e-04, -1.5761e-04, -1.4658e-04,\n -1.8579e-03, -1.7238e-03, 9.4887e-04, 4.5260e-04, -6.7147e-03,\n -9.3330e-04, 9.2071e-04, -3.2067e-04, -8.5045e-04, -1.5515e-04,\n -3.6201e-03, -1.8738e-03, 2.4035e-03, -2.4620e-03, 6.6062e-04,\n 8.2351e-04, -5.1114e-03, 2.2706e-03, 9.4338e-04, -2.1247e-03,\n -1.1905e-04, -2.0811e-03, -1.1117e-03, 1.1899e-03, -1.8625e-04,\n 2.4308e-03, -1.1327e-03, 2.5937e-03, 2.4877e-03, 2.8992e-05,\n 2.1479e-03, -6.6296e-04, 1.5876e-03, -1.0108e-04, -1.7379e-03,\n 1.3652e-03, 1.0314e-03, -7.9480e-04, 6.5997e-04, 3.7703e-03,\n 2.5440e-03, 1.7530e-03, -5.6052e-45, 8.5986e-05, -1.2125e-03,\n -4.4068e-03, 1.9562e-03, 5.5938e-04, 6.3061e-03, -1.7142e-03,\n 1.8454e-03, -1.7381e-03, -1.1815e-03, -4.7339e-05, -3.2327e-03,\n 5.9107e-05, 8.2789e-04, -3.6911e-03, 9.3282e-04, -6.5784e-04,\n 2.9023e-03, -6.5502e-04, 1.1834e-03, 1.1355e-03, -2.0407e-04,\n 3.5745e-03, 1.2451e-03, -1.1711e-03, 3.3985e-03, -2.1445e-03,\n 8.7496e-04, -3.2284e-03, -1.7114e-03, -9.6067e-04, -1.0026e-03,\n -8.0616e-04, -2.0638e-03, -4.7329e-04, 1.9716e-03, -1.8341e-03,\n -2.7636e-04, -1.6879e-03, -1.1868e-03, 2.0205e-03, -1.5179e-03,\n 5.0962e-04, -1.0122e-03, -6.7888e-03, -7.3852e-04, -4.1517e-03,\n -4.7318e-04, 4.3940e-04, -1.8491e-04, -1.9806e-04, 1.2776e-03,\n -8.9457e-04, -1.9398e-03, 3.2357e-03, -8.9029e-04, -4.7193e-03,\n 2.9789e-04, 1.6874e-03, 5.0162e-03, -1.4230e-03, 1.6825e-03,\n -1.3967e-03, -3.9381e-03, 1.5625e-03, -5.5795e-05, 1.5177e-03,\n -2.7719e-03, 2.0563e-03, 3.0882e-03, 3.1224e-03, -7.8552e-04,\n -1.0053e-03, 1.3648e-03, 1.3565e-03, 2.6761e-03, 1.4699e-03,\n -4.3181e-06, 4.8604e-03, -2.1869e-03, -1.2491e-04, 2.1254e-03,\n -7.4630e-04, 3.9108e-03, -4.7495e-03, 3.3035e-04, -3.7640e-03,\n -1.6359e-03, -5.0707e-04, 2.4688e-03, 1.1926e-03, -4.9359e-04,\n -3.0053e-04, 6.2811e-04, 2.8622e-03, -1.3300e-04, 2.0898e-03,\n -1.4205e-03, 2.1457e-03, -1.0887e-04, -2.0601e-04, 2.3235e-03,\n 1.9331e-03, -1.5916e-03, -8.4127e-04, 1.6422e-03, 7.7261e-04,\n 1.4142e-03, -1.1824e-04, 1.8055e-03, 1.3502e-03, 6.8768e-04,\n -5.1213e-04, 2.7740e-03, 2.6561e-03, 6.5403e-04, -6.5524e-05,\n 1.6748e-03, -9.2534e-06, -4.0132e-04, 1.2069e-03, -2.5716e-03,\n 4.9890e-04, 7.3233e-03, 2.1327e-04, 3.6709e-03, -1.3401e-03,\n 1.0011e-03, -1.1240e-03, 2.2180e-03, -3.2212e-03, -3.9527e-03,\n -9.7466e-04, -9.9423e-04, 9.9472e-04, -8.9605e-03, 2.3229e-04,\n 1.6837e-03, 1.0923e-03, 2.4990e-04, -1.0078e-03, -3.5004e-04,\n 3.1826e-03, -8.7937e-04, 6.6399e-04, 4.6696e-03, -1.3107e-03,\n -1.3972e-03, -7.5381e-04, -1.9449e-03, -2.2228e-03, -6.9218e-04,\n 3.2701e-03, 1.0429e-03, 1.0889e-03, -2.4061e-03, 1.2975e-03,\n -2.0470e-03, 8.0845e-04, 1.0524e-03, -2.3067e-03, -2.9543e-03,\n 2.3844e-03, 4.3977e-03, -2.3637e-04, 2.1738e-03, -3.8827e-03,\n 3.1304e-04, 3.5120e-03, 1.3707e-03, -2.7517e-03, 2.2011e-03,\n 1.1413e-03, -4.0549e-04, -2.8065e-03, 3.7792e-04, 1.9544e-03,\n -2.5876e-03, -1.0869e-03, -2.2248e-03, 3.4544e-04, -6.7571e-03,\n -1.7121e-03, -2.6990e-04, 4.4156e-04, 2.6184e-03, 2.9587e-03,\n -1.4398e-03, 5.5322e-04, 1.4126e-03, 1.0534e-04, -6.2703e-04,\n -8.2536e-04, -2.3095e-03, -1.8571e-03, -3.3610e-04, 1.2638e-03,\n 3.3828e-03, 6.5658e-05, -2.4251e-03, 4.6834e-03, 3.7162e-03,\n -1.0450e-03, -6.6407e-03, 3.3043e-04, -5.5323e-03, -2.0566e-03,\n -2.7929e-05, 8.0954e-04, -2.1130e-04, 1.8525e-03, -8.2468e-05,\n 1.7273e-03, -2.3040e-03, -5.6189e-04, 4.5474e-04, -1.8820e-03,\n 1.8147e-03, 3.7096e-04, 2.6072e-03, 1.5498e-04, -2.8563e-03,\n -6.0929e-03, -3.8504e-03, 3.7941e-03, -8.3326e-04, -1.3043e-03,\n 2.4317e-03, 1.8671e-03, -4.3496e-07, -1.2456e-03, 2.8679e-03,\n 2.3688e-03, 7.9251e-04, -1.8509e-03, -1.4463e-03, -8.5672e-04,\n 6.4504e-04, 3.9685e-06, 1.0830e-03, -3.9481e-04, 7.3469e-04,\n 1.0193e-03, 3.2131e-04, -2.2276e-03, -5.6733e-04, -6.7398e-04,\n -1.3128e-03, 1.5830e-03, -5.6951e-04, 8.7773e-04, -7.6339e-04,\n 3.1981e-04, -9.7994e-04, -6.1512e-04, -3.3912e-03, -1.1100e-03,\n -1.5996e-03, 5.5892e-04, -6.9186e-05, 1.0263e-03, 1.3723e-03,\n -2.3313e-03, 5.3676e-04, -4.2060e-03, 5.4055e-04, -2.3191e-03,\n 2.1026e-03, 2.8293e-03, 4.1653e-04, -2.5466e-05, 2.2552e-03,\n 5.2356e-04, 1.2551e-03, -3.2862e-03, 1.1517e-03, 3.1768e-03,\n -2.9799e-03, 7.2636e-04, 1.1449e-04, -6.9111e-05, 5.3015e-03,\n 3.1738e-03, 1.9885e-03, -2.0636e-03, 2.0207e-03, 1.8853e-03,\n 6.5450e-04, 3.2117e-03, 3.9545e-04, 2.7314e-03, 1.8122e-04,\n 1.5487e-03, -3.7915e-03, 3.1955e-04, 1.9321e-03, 4.5496e-03,\n 1.1692e-03, 2.1848e-03, 2.0591e-03, -1.1185e-03, 5.4931e-03,\n 1.3779e-03, 4.1976e-03, -5.5308e-04, -1.5159e-03, -3.3551e-04,\n -1.7586e-03, 4.1974e-03, 7.4445e-04, -1.4186e-03, 7.3137e-04,\n 1.9609e-03, -1.4985e-03, -2.4543e-04, -2.4011e-03, -1.0783e-03,\n -1.0270e-03, 1.0891e-03, -2.6078e-03, -4.1536e-03, -2.1719e-03,\n -7.0278e-04, -1.6267e-03, 1.5192e-03, 4.7992e-03, 2.0564e-03,\n -1.9909e-03, -1.3424e-04, 2.2547e-03, 1.7628e-03, 1.0990e-03,\n -2.9331e-04, -3.4695e-04, 1.1148e-03, -1.9802e-03, 1.1166e-03,\n -9.7615e-04, -1.0988e-03, -1.6561e-03, 1.9045e-03, -2.4185e-03,\n 2.4594e-03, -6.9926e-03, 2.8428e-03, 1.2575e-03, 2.2710e-03,\n -7.2790e-04, -1.1250e-03, -1.9120e-03, -1.6437e-03, -2.3881e-03,\n 9.0485e-03, 7.9422e-04, -7.3068e-06, -1.3394e-04, 1.0817e-03,\n 1.3225e-03, -3.3196e-03, 4.5689e-04, -5.1550e-04, -9.7485e-04,\n 3.1560e-04, -3.1916e-03, -7.8208e-04, -7.8478e-04, 1.5089e-03,\n 1.6504e-03, -4.3780e-04, 1.6333e-03, 3.4250e-03, 1.9985e-04,\n 2.6474e-03, 1.2279e-03, 2.5853e-03, -3.3961e-03, 1.1008e-03,\n -2.0965e-03, -1.2371e-03, 9.9103e-04, -2.5548e-04, 8.2381e-04,\n 2.6736e-03, -3.0234e-03, -2.4377e-03, -7.8171e-04, -4.1897e-04,\n 8.8875e-04, 3.1349e-03, 1.5657e-03, -2.3189e-03, 2.9581e-04,\n 4.5555e-04, -3.2056e-03, -6.8754e-04, -9.3667e-06, -1.5923e-03,\n 5.2077e-04, -7.0209e-04, -6.4724e-04, -6.0410e-04, 2.3214e-04,\n -7.4845e-04, 1.3646e-03, 4.8812e-04, 3.0800e-03, -2.6510e-03,\n -2.7581e-03, 4.8308e-03], device='cuda:0')", + "exp_avg_sq": "tensor([3.9541e-05, 6.0157e-05, 6.5035e-05, 4.5914e-05, 7.0415e-05, 5.6864e-05,\n 5.6505e-05, 5.5233e-05, 4.2714e-05, 6.6897e-05, 6.7580e-05, 4.5772e-05,\n 6.4500e-05, 5.0026e-05, 5.0496e-05, 7.6882e-05, 5.4875e-05, 5.8478e-05,\n 9.0888e-05, 8.9536e-05, 6.6368e-05, 4.5908e-05, 4.7714e-05, 4.3342e-05,\n 4.6506e-05, 7.5061e-05, 6.0938e-05, 7.1696e-05, 5.6298e-05, 7.5688e-05,\n 1.0094e-04, 5.5889e-05, 5.0933e-05, 4.5485e-05, 5.6224e-05, 8.6968e-05,\n 6.0383e-05, 5.2227e-05, 7.2985e-05, 8.2274e-05, 5.4437e-05, 9.0206e-05,\n 5.4434e-05, 4.8363e-05, 6.4602e-05, 7.8647e-05, 4.2743e-05, 5.8857e-05,\n 5.1250e-05, 4.1409e-05, 4.8037e-05, 5.3334e-05, 5.4890e-05, 1.1414e-04,\n 6.3911e-05, 4.8077e-05, 1.8286e-05, 5.3344e-05, 5.1581e-05, 5.4572e-05,\n 3.9106e-05, 4.5417e-05, 5.3652e-05, 7.8698e-05, 7.9982e-05, 9.9051e-05,\n 4.9138e-05, 5.7570e-05, 5.5114e-05, 7.8085e-05, 5.6934e-05, 6.5139e-05,\n 5.6467e-05, 4.8656e-05, 8.3686e-05, 6.6108e-05, 5.8366e-05, 5.9761e-05,\n 4.9499e-05, 8.3341e-05, 7.0416e-05, 6.7552e-05, 4.9542e-05, 6.4936e-05,\n 5.5035e-05, 4.7151e-05, 5.8076e-05, 4.7532e-05, 5.1319e-05, 5.5406e-05,\n 6.0067e-05, 4.3851e-05, 7.3921e-05, 6.4402e-05, 5.4718e-05, 7.3416e-05,\n 5.0385e-05, 7.2929e-05, 5.8634e-05, 7.4782e-05, 5.6234e-05, 5.7720e-05,\n 1.0822e-09, 4.3036e-05, 7.0153e-05, 6.4226e-05, 4.5829e-05, 4.0599e-05,\n 7.5300e-05, 3.4100e-05, 6.2978e-05, 7.0087e-05, 7.8048e-05, 7.3279e-05,\n 5.3484e-05, 6.1833e-05, 4.6713e-05, 7.7523e-05, 4.0646e-05, 4.4991e-05,\n 6.5249e-05, 4.1916e-05, 5.6980e-05, 4.4619e-05, 4.3665e-05, 8.5941e-05,\n 5.0713e-05, 4.6596e-05, 5.2990e-05, 5.7952e-05, 4.2936e-05, 5.0796e-05,\n 7.2740e-05, 5.8932e-05, 5.5376e-05, 5.5872e-05, 5.4670e-05, 4.9991e-05,\n 5.5600e-05, 7.8250e-05, 5.7969e-05, 4.1370e-05, 7.2577e-05, 7.5846e-05,\n 8.4318e-05, 6.2249e-05, 7.8870e-05, 7.2742e-05, 4.6082e-05, 6.1937e-05,\n 5.6875e-05, 6.5583e-05, 5.6055e-05, 5.9460e-05, 9.3298e-05, 7.2707e-05,\n 4.3600e-05, 5.5419e-05, 6.2712e-05, 1.2615e-04, 5.9854e-05, 8.5986e-05,\n 6.1593e-05, 5.3187e-05, 4.4901e-05, 5.9429e-05, 6.0618e-05, 4.7709e-05,\n 9.4876e-05, 7.2951e-05, 5.6866e-05, 7.1599e-05, 5.7220e-05, 6.3185e-05,\n 4.8011e-05, 6.0472e-05, 5.2267e-05, 5.3282e-05, 9.5294e-05, 5.6227e-05,\n 6.1468e-05, 8.9963e-05, 5.5281e-05, 8.1716e-05, 6.0121e-05, 8.4221e-05,\n 6.8248e-05, 4.7413e-05, 5.1257e-05, 4.8652e-05, 4.9847e-05, 1.1355e-04,\n 5.2545e-05, 4.0754e-05, 6.2320e-05, 4.7618e-05, 4.8968e-05, 1.1148e-04,\n 6.5384e-05, 3.0445e-05, 6.5401e-05, 5.7762e-05, 5.5864e-05, 6.1487e-05,\n 6.6586e-05, 6.6473e-05, 6.2239e-05, 4.3562e-05, 1.7204e-05, 7.3715e-05,\n 6.3070e-05, 6.7847e-05, 4.9958e-05, 5.3616e-05, 5.8606e-05, 3.3256e-05,\n 5.8982e-05, 4.8708e-05, 7.5287e-05, 5.5009e-05, 1.0835e-04, 3.8467e-05,\n 5.2169e-05, 5.1693e-05, 4.0418e-05, 4.7480e-05, 8.8489e-05, 5.9983e-05,\n 4.5984e-05, 6.3337e-05, 7.2939e-05, 5.0371e-05, 7.3824e-05, 3.8082e-05,\n 4.3153e-05, 5.0910e-05, 6.7361e-05, 8.1115e-05, 1.0119e-04, 4.9039e-05,\n 6.6047e-05, 4.7813e-05, 5.9540e-05, 2.6235e-05, 4.9737e-05, 5.5474e-05,\n 5.1474e-05, 7.2641e-05, 5.8445e-05, 9.1003e-05, 5.9819e-05, 5.8065e-05,\n 4.8605e-05, 5.0905e-05, 5.7332e-05, 4.8730e-05, 4.7442e-05, 5.6691e-05,\n 7.0589e-05, 5.9463e-05, 7.6239e-05, 6.9445e-05, 5.8918e-05, 9.0936e-05,\n 6.7952e-05, 5.2210e-05, 5.1513e-05, 4.9897e-05, 6.0089e-05, 6.3440e-05,\n 3.8022e-05, 7.4572e-05, 4.6923e-05, 4.7408e-05, 9.3873e-05, 5.6461e-05,\n 7.2209e-05, 7.9817e-05, 5.3297e-05, 5.2919e-05, 6.7124e-05, 6.0025e-05,\n 6.5178e-05, 3.8070e-05, 8.6547e-05, 4.9355e-05, 7.8485e-05, 5.4542e-05,\n 7.6952e-05, 5.2673e-05, 5.1024e-05, 5.0042e-05, 4.6868e-05, 3.2014e-05,\n 4.5795e-05, 5.2998e-05, 5.5500e-05, 5.0589e-05, 6.5842e-05, 6.1737e-05,\n 7.0516e-05, 5.7542e-05, 5.5184e-05, 6.3476e-05, 6.2495e-05, 4.5424e-05,\n 5.1936e-05, 4.6233e-05, 7.8375e-05, 4.6972e-05, 5.4701e-05, 4.4948e-05,\n 6.3548e-05, 4.3473e-05, 5.2680e-05, 5.4288e-05, 6.4957e-05, 5.1053e-05,\n 7.7307e-05, 2.6801e-05, 8.3308e-05, 9.0652e-05, 5.6727e-05, 5.9400e-05,\n 3.8085e-05, 4.4590e-05, 7.9894e-05, 3.9323e-05, 6.5200e-05, 6.9192e-05,\n 6.7066e-05, 6.6821e-05, 4.6018e-05, 6.8029e-05, 8.3762e-05, 5.3943e-05,\n 6.3597e-05, 6.4463e-05, 5.4775e-05, 6.8215e-05, 5.0443e-05, 5.9005e-05,\n 5.2982e-05, 5.6406e-05, 5.7802e-05, 6.1516e-05, 6.2055e-05, 7.0833e-05,\n 3.9396e-05, 6.6291e-05, 6.6788e-05, 3.8195e-05, 5.2236e-05, 6.3607e-05,\n 7.9670e-05, 7.4498e-05, 6.7701e-05, 3.7463e-05, 4.5860e-05, 5.0158e-05,\n 5.6064e-05, 4.2904e-05, 7.0671e-05, 4.1224e-05, 6.3664e-05, 6.4786e-05,\n 3.1004e-05, 5.9782e-05, 6.1363e-05, 4.6511e-05, 4.7440e-05, 4.6809e-05,\n 6.5737e-05, 5.9011e-05, 6.8897e-05, 6.0403e-05, 8.1022e-05, 6.6795e-05,\n 5.5059e-05, 5.9239e-05, 5.2561e-05, 9.4740e-05, 7.3362e-05, 7.3179e-05,\n 7.2299e-05, 8.0523e-05, 5.5136e-05, 7.2055e-05, 4.6499e-05, 4.2164e-05,\n 8.3443e-05, 4.1522e-05, 4.9932e-05, 3.3685e-05, 5.0827e-05, 4.0686e-05,\n 6.2128e-05, 7.2561e-05, 7.6401e-05, 6.5771e-05, 5.3436e-05, 6.2127e-05,\n 6.2917e-05, 6.9648e-05, 5.6534e-05, 8.1586e-05, 7.1132e-05, 5.0030e-05,\n 5.1511e-05, 5.1618e-05, 5.4773e-05, 6.8774e-05, 6.6581e-05, 5.3789e-05,\n 5.8868e-05, 5.3953e-05, 2.9705e-05, 4.0181e-05, 6.7566e-05, 5.6832e-05,\n 4.3164e-05, 3.6991e-05, 6.0079e-05, 5.8172e-05, 4.7733e-05, 8.3194e-05,\n 4.4127e-05, 5.4413e-05, 6.9344e-05, 7.3839e-05, 7.9112e-05, 5.6574e-05,\n 4.5783e-05, 4.1758e-05, 5.5050e-05, 3.5437e-05, 5.2198e-05, 5.8889e-05,\n 5.3176e-05, 5.2228e-05, 5.7209e-05, 4.6493e-05, 4.4962e-05, 6.4392e-05,\n 8.2795e-05, 4.9058e-05, 5.0965e-05, 5.8258e-05, 6.0316e-05, 7.2463e-05,\n 4.8247e-05, 4.7983e-05, 5.8867e-05, 5.4292e-05, 4.8633e-05, 1.0601e-04,\n 4.7141e-05, 4.9505e-05, 4.8711e-05, 5.3719e-05, 5.0464e-05, 9.0048e-05,\n 4.1457e-05, 5.1789e-05, 4.9528e-05, 6.0149e-05, 6.3449e-05, 7.5129e-05,\n 5.1274e-05, 5.8388e-05, 5.2290e-05, 4.8510e-05, 5.1716e-05, 6.3444e-05,\n 5.8907e-05, 4.8354e-05, 6.2645e-05, 4.1057e-05, 7.0356e-05, 5.5858e-05,\n 5.8580e-05, 5.3566e-05, 5.3299e-05, 7.0404e-05, 7.5602e-05, 8.4523e-05,\n 6.0258e-05, 4.7723e-05, 5.7825e-05, 9.0492e-05, 7.8411e-05, 3.9131e-05,\n 3.9297e-05, 5.4922e-05, 4.7673e-05, 7.6807e-05, 7.0658e-05, 6.6829e-05,\n 5.4621e-05, 7.6818e-05, 6.4141e-05, 5.9126e-05, 6.8847e-05, 4.7876e-05,\n 6.2278e-05, 5.4746e-05, 6.8685e-05, 7.5026e-05, 5.1421e-05, 6.2557e-05,\n 5.6539e-05, 9.1148e-05], device='cuda:0')" + }, + "2": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 4.7691e-05, 6.0629e-05, 2.9088e-06, ..., 6.1724e-06,\n 6.8852e-06, 1.4174e-05],\n [ 6.9664e-06, -2.3161e-05, -3.0773e-06, ..., -2.9374e-05,\n 7.4491e-05, 2.4848e-05],\n [-1.5844e-05, -2.0607e-05, -6.7906e-05, ..., 1.1564e-04,\n 9.8007e-06, -3.1191e-05],\n ...,\n [-4.2718e-05, 5.0768e-05, 1.3528e-05, ..., 2.9366e-05,\n -3.1631e-05, -5.0623e-05],\n [ 3.0493e-05, -1.2292e-05, 8.0793e-06, ..., -8.5852e-05,\n 1.2692e-04, 8.6685e-06],\n [-1.3834e-05, -3.6061e-06, 1.7246e-05, ..., -5.7238e-05,\n -5.6513e-05, 5.2166e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[7.4772e-09, 8.4885e-09, 1.0471e-08, ..., 1.2605e-08, 1.8407e-08,\n 1.6508e-08],\n [1.2361e-08, 1.8285e-08, 2.1034e-08, ..., 1.4959e-08, 3.3661e-08,\n 3.2319e-08],\n [1.1556e-08, 1.8545e-08, 2.0142e-08, ..., 1.6700e-08, 3.1456e-08,\n 3.5753e-08],\n ...,\n [1.3194e-08, 1.4320e-08, 2.7425e-08, ..., 1.9308e-08, 4.8713e-08,\n 3.4276e-08],\n [1.3553e-08, 1.2955e-08, 1.3859e-08, ..., 1.7855e-08, 3.7478e-08,\n 1.7564e-08],\n [1.2671e-08, 2.0589e-08, 2.0915e-08, ..., 1.8336e-08, 4.7979e-08,\n 2.3332e-08]], device='cuda:0')" + }, + "3": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 3.7683e-05, 3.6953e-05, 4.3404e-06, ..., -2.0047e-05,\n -2.9326e-05, -1.0694e-05],\n [ 5.5647e-05, 1.7720e-05, 4.7133e-05, ..., 5.0546e-06,\n -1.2374e-06, -1.0628e-06],\n [-3.5499e-06, -6.0231e-05, -8.9577e-06, ..., 9.6451e-06,\n 3.3818e-06, -1.2340e-05],\n ...,\n [-2.2072e-06, 1.6710e-05, -1.2588e-05, ..., -2.1016e-05,\n -2.4045e-05, 1.0436e-06],\n [ 1.0325e-04, -6.2328e-05, -8.3159e-06, ..., 3.4036e-05,\n -1.0327e-05, -3.4474e-05],\n [-3.0303e-05, -1.4669e-04, 3.5641e-05, ..., 8.2398e-05,\n 5.9819e-05, 7.4675e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.8225e-08, 3.0433e-08, 1.4023e-08, ..., 9.2876e-09, 8.9665e-09,\n 1.8445e-08],\n [8.8797e-08, 5.3529e-08, 1.9224e-08, ..., 1.6739e-08, 1.6111e-08,\n 1.7900e-08],\n [3.6656e-08, 4.6641e-08, 1.9333e-08, ..., 2.5839e-08, 1.5312e-08,\n 1.7178e-08],\n ...,\n [2.3648e-08, 3.8743e-08, 1.3841e-08, ..., 1.2917e-08, 1.5406e-08,\n 9.2949e-09],\n [1.1033e-07, 5.8743e-08, 1.2641e-08, ..., 1.1843e-08, 1.3074e-08,\n 1.2869e-08],\n [1.9238e-08, 8.3251e-08, 2.0988e-08, ..., 1.6410e-08, 1.2361e-08,\n 3.3280e-08]], device='cuda:0')" + }, + "4": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 1.8137e-04, 2.6490e-04, -3.8390e-05, ..., -9.4848e-04,\n 1.4926e-03, 2.8457e-03], device='cuda:0')", + "exp_avg_sq": "tensor([1.7007e-05, 2.5048e-05, 3.3590e-05, ..., 1.6285e-05, 1.7629e-05,\n 2.1459e-05], device='cuda:0')" + }, + "5": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 6.4501e-06, -4.5897e-07, -6.9317e-08, ..., 1.3155e-06,\n -8.2166e-06, 1.4215e-05],\n [ 2.6129e-06, -7.4149e-07, 2.1134e-06, ..., -1.3550e-06,\n 6.4733e-06, 1.6059e-05],\n [ 1.0375e-05, -7.4240e-07, 4.6742e-07, ..., -3.6816e-06,\n -1.4833e-05, -1.7899e-05],\n ...,\n [ 1.5687e-06, -3.5652e-06, -4.2141e-06, ..., -6.5842e-06,\n 9.8251e-06, -4.7222e-06],\n [ 1.0387e-06, 2.1494e-06, -4.5678e-06, ..., -1.3256e-06,\n 4.4465e-06, 2.2512e-05],\n [-1.2923e-06, 2.9924e-06, 1.3996e-06, ..., -4.1484e-06,\n -4.9859e-06, -2.1455e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.8687e-10, 4.3967e-10, 1.2944e-09, ..., 2.6319e-10, 3.0522e-10,\n 1.6981e-09],\n [6.9384e-10, 1.9197e-09, 4.0029e-09, ..., 3.8830e-10, 8.6036e-10,\n 2.3830e-09],\n [7.9555e-10, 9.6629e-10, 2.4880e-09, ..., 8.1120e-10, 2.0233e-09,\n 2.5418e-09],\n ...,\n [8.1419e-10, 8.7529e-10, 2.4082e-09, ..., 6.2948e-10, 8.2833e-10,\n 4.4176e-09],\n [5.8327e-10, 7.2575e-10, 3.2258e-09, ..., 4.2415e-10, 6.0208e-10,\n 2.3120e-09],\n [1.0503e-09, 2.5829e-09, 2.7248e-09, ..., 5.8513e-10, 1.1493e-09,\n 3.6728e-09]], device='cuda:0')" + }, + "6": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.7232e-06, -1.7260e-05, -4.3929e-06, ..., 2.1024e-05,\n 4.7011e-06, 1.0608e-05],\n [-1.8364e-05, 8.8223e-05, -2.9135e-05, ..., 3.0133e-06,\n -2.5822e-05, -2.0140e-05],\n [ 1.8320e-05, -2.5761e-05, -1.4089e-05, ..., -3.8198e-07,\n 1.1501e-05, 3.5789e-06],\n ...,\n [-4.6842e-07, -1.0607e-05, -4.0497e-06, ..., -1.1158e-06,\n 8.0192e-07, 1.8159e-06],\n [ 2.3621e-06, -2.5845e-05, -1.5537e-05, ..., 7.4166e-06,\n 1.7239e-05, 2.6995e-05],\n [-4.3962e-12, 4.0877e-12, -1.9600e-12, ..., -3.9033e-12,\n 1.7806e-12, 2.1327e-12]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.3555e-08, 2.2740e-08, 3.6067e-09, ..., 3.9163e-09, 5.1483e-09,\n 5.6799e-09],\n [3.2127e-08, 6.6781e-08, 1.8711e-08, ..., 1.3870e-08, 1.5659e-08,\n 1.0208e-08],\n [4.5361e-08, 3.5126e-08, 3.6050e-09, ..., 5.0081e-09, 5.7692e-09,\n 3.5375e-09],\n ...,\n [1.5222e-08, 3.2747e-08, 3.7966e-09, ..., 3.7195e-09, 7.4843e-09,\n 2.9216e-09],\n [3.7550e-08, 5.3304e-08, 8.5398e-09, ..., 1.1164e-08, 8.3774e-09,\n 1.0667e-08],\n [1.4889e-14, 5.1123e-14, 3.8076e-15, ..., 6.2751e-15, 6.7898e-15,\n 1.8162e-15]], device='cuda:0')" + }, + "7": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 5.5431e-04, -3.2257e-04, 1.9091e-04, ..., 1.5275e-04,\n 3.0025e-04, -5.4940e-11], device='cuda:0')", + "exp_avg_sq": "tensor([7.3281e-06, 2.4019e-05, 6.6348e-06, ..., 6.3687e-06, 1.5377e-05,\n 6.6956e-12], device='cuda:0')" + }, + "8": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.8173e-06, -6.4850e-06, 1.9640e-07, ..., 2.6395e-07,\n 2.4293e-07, 5.5127e-13],\n [-2.7685e-07, -8.6282e-07, -1.8781e-06, ..., -8.7293e-07,\n -1.6414e-06, 6.7698e-14],\n [ 1.7745e-06, -6.9675e-08, -2.0016e-06, ..., 6.7856e-07,\n -1.4857e-06, 1.2252e-13],\n ...,\n [-5.3859e-06, 4.9779e-06, -2.9636e-06, ..., -2.3031e-07,\n 1.7022e-06, -2.3513e-13],\n [-4.1937e-06, -2.0052e-06, -9.1262e-07, ..., 3.0709e-08,\n -4.1841e-07, -1.6776e-13],\n [ 1.5927e-06, -3.1442e-06, -1.3026e-06, ..., 3.2509e-07,\n -2.1780e-07, 3.6810e-13]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.5015e-10, 4.5746e-10, 2.0170e-10, ..., 3.5332e-11, 1.6302e-10,\n 8.2207e-16],\n [2.9914e-10, 4.2145e-10, 3.4914e-10, ..., 9.3640e-11, 3.6105e-10,\n 1.8110e-15],\n [3.3403e-10, 5.1903e-10, 4.5673e-10, ..., 8.4055e-11, 7.8882e-10,\n 8.1966e-16],\n ...,\n [4.1714e-10, 8.2624e-10, 4.1119e-10, ..., 2.1524e-10, 3.9891e-10,\n 1.5145e-15],\n [4.0956e-10, 5.5704e-10, 4.2109e-10, ..., 1.1048e-10, 5.7027e-10,\n 1.1376e-15],\n [2.3875e-10, 5.4538e-10, 3.3899e-10, ..., 1.0676e-10, 3.2751e-10,\n 5.7419e-16]], device='cuda:0')" + }, + "9": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 4.0126e-05, -2.6925e-06, 7.1722e-06, ..., 1.6131e-05,\n 6.9838e-07, 2.3798e-05],\n [-2.1072e-05, 1.7635e-05, -1.0482e-06, ..., 1.0088e-05,\n 3.7079e-06, 1.4078e-06],\n [ 6.9073e-05, -6.0749e-05, -3.5022e-06, ..., 9.6767e-07,\n 1.7209e-05, -8.1306e-06],\n ...,\n [ 2.5279e-05, -1.0390e-04, -6.4204e-06, ..., -3.7922e-05,\n -2.9836e-05, 1.6683e-05],\n [-6.0481e-06, 9.5223e-05, 1.3056e-05, ..., 3.0215e-05,\n 2.8358e-05, 2.5365e-05],\n [-2.0147e-05, -5.6817e-05, 3.4647e-05, ..., 2.7737e-05,\n 4.4752e-05, 1.9533e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.0801e-07, 1.0789e-07, 1.9064e-08, ..., 3.7080e-08, 3.9838e-08,\n 2.8051e-08],\n [2.8772e-08, 2.5750e-08, 5.6389e-09, ..., 7.3821e-09, 6.5834e-09,\n 9.2242e-09],\n [4.2301e-08, 3.7461e-08, 5.1138e-09, ..., 4.4914e-09, 8.4969e-09,\n 4.6522e-09],\n ...,\n [1.0932e-08, 4.3531e-08, 7.8270e-09, ..., 5.8993e-09, 8.2319e-09,\n 1.4414e-08],\n [9.9944e-09, 2.7052e-08, 5.2372e-09, ..., 5.8189e-09, 5.0067e-09,\n 7.8663e-09],\n [1.7018e-08, 1.4290e-07, 1.3454e-08, ..., 1.2409e-08, 2.7023e-08,\n 1.6973e-08]], device='cuda:0')" + }, + "10": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 0.0010, 0.0002, 0.0007, ..., 0.0005, -0.0005, 0.0014],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.9803e-05, 1.0861e-05, 8.5639e-06, ..., 9.0871e-06, 7.4858e-06,\n 2.4909e-05], device='cuda:0')" + }, + "11": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 9.2679e-07, 3.8269e-06, -1.5380e-06, ..., -4.8819e-06,\n -2.0129e-06, -2.1236e-06],\n [ 2.5069e-07, 1.0141e-06, 5.8003e-08, ..., -8.1919e-06,\n 3.0242e-07, 2.4617e-06],\n [-1.6484e-06, -1.7063e-06, -2.8557e-07, ..., 6.5747e-06,\n 3.4891e-06, -1.3548e-07],\n ...,\n [ 2.4584e-06, 3.4417e-06, -2.1103e-06, ..., -9.3282e-06,\n 1.3768e-06, -5.2184e-06],\n [ 5.4128e-07, 1.5489e-06, 5.4678e-06, ..., -4.5956e-06,\n -3.3878e-06, 4.3037e-06],\n [-9.2988e-07, 1.9734e-06, -3.5516e-06, ..., -5.6395e-06,\n -2.6942e-06, -5.8501e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[7.7445e-11, 4.4584e-11, 9.9969e-11, ..., 9.6912e-11, 1.1512e-10,\n 5.1591e-11],\n [1.3560e-10, 1.0284e-10, 1.2138e-10, ..., 2.0161e-10, 1.4854e-10,\n 1.3594e-10],\n [2.5075e-10, 7.8247e-11, 1.9390e-10, ..., 2.6698e-10, 2.3481e-10,\n 8.6892e-11],\n ...,\n [2.5641e-10, 9.6647e-11, 1.2845e-10, ..., 1.4801e-10, 1.8781e-10,\n 3.7449e-10],\n [8.6462e-11, 7.2057e-11, 1.7492e-10, ..., 1.8693e-10, 3.3000e-10,\n 2.4306e-10],\n [3.5435e-10, 8.5082e-11, 1.8564e-10, ..., 1.7403e-10, 2.0411e-10,\n 2.7286e-10]], device='cuda:0')" + }, + "12": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-3.1391e-05, 2.5249e-06, 2.2338e-05, ..., -1.3032e-05,\n -3.2736e-05, -6.3401e-06],\n [-8.1715e-06, -8.3547e-06, -1.2609e-05, ..., 1.2368e-06,\n -1.0378e-05, 4.9242e-06],\n [ 1.1975e-05, 1.9630e-05, 1.5762e-05, ..., 2.0149e-05,\n 5.4732e-06, -2.4316e-06],\n ...,\n [ 1.1099e-06, -1.1161e-06, -2.1792e-07, ..., 3.5976e-07,\n -5.2919e-07, 2.4669e-08],\n [ 7.4895e-09, 2.3958e-09, 7.0821e-09, ..., 5.2725e-09,\n 1.1663e-08, -1.6495e-08],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.8032e-07, 1.5099e-07, 1.4304e-08, ..., 1.1272e-08, 6.3357e-08,\n 1.5299e-08],\n [4.8344e-08, 2.1106e-08, 3.6855e-09, ..., 4.0086e-09, 5.0650e-09,\n 5.0215e-09],\n [3.9203e-08, 3.1327e-08, 9.3108e-09, ..., 5.7558e-09, 6.6764e-09,\n 1.5483e-08],\n ...,\n [8.9383e-11, 8.2455e-11, 8.0672e-12, ..., 7.0032e-12, 9.7530e-12,\n 4.5086e-12],\n [1.9818e-12, 4.1789e-12, 9.1872e-13, ..., 1.1453e-12, 1.1950e-12,\n 2.0994e-12],\n [1.0606e-14, 2.4930e-14, 4.2084e-15, ..., 3.1218e-15, 1.6271e-14,\n 3.3886e-15]], device='cuda:0')" + }, + "13": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-3.5548e-04, 1.8567e-04, 9.2850e-04, ..., 1.4980e-05,\n -3.1037e-07, 5.6052e-45], device='cuda:0')", + "exp_avg_sq": "tensor([2.8382e-05, 1.0690e-05, 1.3022e-05, ..., 1.5454e-08, 2.3652e-09,\n 1.0489e-11], device='cuda:0')" + }, + "14": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 6.3776e-07, -8.7343e-07, 1.5187e-06, ..., -1.8470e-09,\n -1.9681e-10, -5.6052e-45],\n [ 7.2902e-07, -5.6012e-07, 1.0399e-06, ..., -6.3860e-10,\n 4.4208e-10, -5.6052e-45],\n [-1.8728e-06, 1.5767e-06, -2.1574e-06, ..., 1.3161e-09,\n 2.7770e-10, 5.6052e-45],\n ...,\n [-1.2146e-06, -1.0625e-06, 1.3900e-06, ..., 2.2537e-09,\n -2.8595e-11, 5.6052e-45],\n [ 1.3816e-06, -9.2287e-08, 3.8688e-08, ..., 1.9306e-09,\n -1.2382e-11, -5.6052e-45],\n [ 8.2633e-09, -1.5112e-06, 1.4549e-06, ..., 2.3343e-11,\n -5.1994e-11, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.9447e-11, 6.1868e-12, 2.0718e-11, ..., 8.3044e-14, 7.9441e-14,\n 2.4092e-16],\n [9.8206e-11, 1.2323e-11, 3.6289e-11, ..., 1.3991e-13, 1.2220e-13,\n 4.3695e-16],\n [1.8171e-10, 1.2605e-11, 2.8123e-11, ..., 4.4059e-13, 1.1533e-13,\n 7.5071e-16],\n ...,\n [1.1920e-10, 1.0563e-11, 2.7098e-11, ..., 5.1650e-13, 1.3372e-13,\n 6.0836e-16],\n [1.2153e-10, 8.6381e-12, 2.8122e-11, ..., 1.8433e-13, 1.5781e-13,\n 7.0418e-16],\n [2.4059e-10, 1.1663e-11, 2.2391e-11, ..., 1.0737e-13, 9.4470e-14,\n 5.6509e-16]], device='cuda:0')" + }, + "15": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 9.4385e-08, -7.9499e-05, -7.6891e-05, ..., 1.5104e-05,\n -7.2462e-05, 8.4229e-05],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 7.7019e-08, -1.6429e-07, -6.0791e-08, ..., -2.0108e-08,\n -1.1931e-07, 3.1762e-08],\n ...,\n [ 9.6973e-05, -2.5829e-05, 1.3184e-05, ..., -2.7458e-05,\n -1.2814e-05, 1.3922e-05],\n [-1.8272e-05, -1.0914e-05, -1.9463e-05, ..., -3.4216e-06,\n -3.6324e-05, -1.7131e-05],\n [-2.4912e-04, 1.9035e-04, -2.5781e-05, ..., -4.0476e-05,\n -5.7424e-05, 7.2013e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.7550e-08, 6.7778e-08, 1.7225e-08, ..., 3.3528e-08, 2.7529e-08,\n 4.5216e-08],\n [4.7677e-14, 8.1295e-14, 1.6686e-14, ..., 8.5662e-15, 1.0530e-14,\n 5.3782e-15],\n [3.9335e-10, 7.1781e-10, 5.1046e-10, ..., 7.0942e-10, 4.3379e-10,\n 5.6322e-10],\n ...,\n [6.9710e-08, 6.7564e-08, 1.6644e-08, ..., 1.5676e-08, 1.6099e-08,\n 2.1436e-08],\n [4.1212e-08, 1.0227e-07, 4.8623e-08, ..., 1.6396e-08, 1.9828e-08,\n 2.8403e-08],\n [3.1151e-07, 2.5951e-07, 1.6635e-08, ..., 2.0998e-08, 3.7333e-08,\n 2.1295e-08]], device='cuda:0')" + }, + "16": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 7.8214e-04, -5.6052e-45, 3.9169e-06, ..., 7.3931e-04,\n -1.0981e-03, -1.6453e-03], device='cuda:0')", + "exp_avg_sq": "tensor([3.3423e-05, 1.4090e-11, 6.6363e-07, ..., 3.5368e-05, 3.4848e-05,\n 4.0135e-05], device='cuda:0')" + }, + "17": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.8756e-06, -5.6052e-45, -2.4463e-10, ..., -7.9339e-07,\n 1.9921e-07, -3.6811e-07],\n [-1.4184e-07, -5.6052e-45, 4.9426e-09, ..., -1.0211e-06,\n -1.4363e-07, 1.0384e-06],\n [-1.7965e-06, 5.6052e-45, -1.6861e-09, ..., -1.5844e-06,\n 1.3121e-06, -2.1566e-06],\n ...,\n [-1.9044e-06, 5.6052e-45, 3.9954e-09, ..., -1.2907e-06,\n -1.1382e-06, 1.7969e-06],\n [-1.0125e-06, -5.6052e-45, 8.6547e-10, ..., 5.5349e-08,\n -1.4131e-06, -1.5545e-06],\n [ 1.2480e-06, 5.6052e-45, 2.8686e-09, ..., 4.9369e-07,\n -9.2380e-07, 2.4504e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.4639e-11, 6.3777e-17, 1.2459e-12, ..., 1.2459e-11, 9.5652e-12,\n 1.3716e-11],\n [3.3157e-11, 8.9090e-17, 1.8042e-12, ..., 4.0981e-11, 1.8109e-11,\n 4.1533e-11],\n [2.8122e-11, 1.0128e-16, 1.2198e-12, ..., 3.5453e-11, 1.7677e-11,\n 3.3135e-11],\n ...,\n [3.3548e-11, 3.7923e-17, 2.2724e-12, ..., 1.7991e-11, 2.6909e-11,\n 5.4391e-11],\n [4.5538e-11, 1.1164e-16, 1.1047e-12, ..., 2.1516e-11, 3.2696e-11,\n 3.8425e-11],\n [2.7122e-11, 9.1564e-17, 1.9720e-12, ..., 2.4542e-11, 2.5501e-11,\n 3.1828e-11]], device='cuda:0')" + }, + "18": { + "step": "tensor(12515.)", + "exp_avg": "tensor(-2.4637e-05, device='cuda:0')", + "exp_avg_sq": "tensor(5.5971e-07, device='cuda:0')" + }, + "19": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-1.2042e-06, 8.8363e-06, 1.4672e-06, ..., -3.2586e-07,\n 3.9932e-06, -3.8428e-06],\n [ 5.5100e-06, 2.1013e-05, 9.0435e-07, ..., -1.3686e-06,\n -1.5825e-05, -4.6869e-06],\n [ 9.7775e-07, 1.2331e-06, -2.6618e-06, ..., 1.2466e-06,\n 9.7760e-07, 1.7729e-06],\n [ 2.2864e-06, -3.8709e-07, -1.3963e-05, ..., 5.6196e-06,\n 1.2484e-05, -1.6543e-05],\n [-4.6403e-06, -6.7790e-06, 4.0789e-06, ..., 2.5253e-06,\n 1.3110e-05, -3.9909e-06],\n [-1.6546e-05, -9.6438e-06, -3.1217e-06, ..., 1.9981e-06,\n 2.1163e-05, -1.1902e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.2944e-10, 1.5081e-09, 2.3059e-10, ..., 1.9816e-10, 6.4875e-10,\n 3.1585e-10],\n [3.3168e-09, 1.0786e-08, 1.7343e-09, ..., 1.8987e-09, 6.4947e-09,\n 2.8885e-09],\n [1.4248e-09, 4.2922e-09, 6.9166e-10, ..., 6.8454e-10, 2.3263e-09,\n 9.3358e-10],\n [3.8872e-09, 1.1257e-08, 2.2458e-09, ..., 2.4112e-09, 5.0010e-09,\n 3.9425e-09],\n [2.6925e-09, 9.0007e-09, 1.6700e-09, ..., 1.9867e-09, 4.3173e-09,\n 2.8622e-09],\n [1.6327e-09, 4.6652e-09, 9.2050e-10, ..., 9.2046e-10, 3.6992e-09,\n 1.4523e-09]], device='cuda:0')" + }, + "20": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.3009e-07, -6.6685e-08, 1.0070e-07, ..., 1.3217e-07,\n -2.1140e-07, 6.2797e-08],\n [ 1.1122e-07, -3.1276e-07, 3.1282e-07, ..., -2.3002e-07,\n 2.3268e-07, 1.7292e-10],\n [-1.6275e-07, 8.5500e-08, -1.5355e-07, ..., 1.4128e-07,\n -6.2811e-08, 1.1310e-08],\n ...,\n [ 1.7353e-07, -1.0904e-08, 2.3410e-07, ..., -7.0360e-08,\n -1.6431e-07, -1.5530e-07],\n [ 2.7769e-07, 3.9212e-07, 7.2639e-08, ..., -1.3492e-07,\n -4.7366e-07, -4.8372e-07],\n [-1.3703e-07, -2.4333e-07, 3.4662e-07, ..., -9.8859e-08,\n -2.4810e-08, -9.1002e-08]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.5871e-13, 8.9009e-13, 5.9441e-13, ..., 8.0516e-13, 9.0971e-13,\n 7.7908e-13],\n [4.6131e-12, 5.6060e-12, 4.5324e-12, ..., 5.5655e-12, 5.6828e-12,\n 4.8132e-12],\n [5.0055e-13, 1.0406e-12, 5.6867e-13, ..., 8.4489e-13, 9.2034e-13,\n 7.1343e-13],\n ...,\n [4.7853e-13, 8.4137e-13, 4.9442e-13, ..., 7.1524e-13, 6.9772e-13,\n 6.6889e-13],\n [1.6517e-12, 2.6617e-12, 1.9091e-12, ..., 2.2600e-12, 2.1950e-12,\n 2.3735e-12],\n [7.0795e-13, 1.3075e-12, 8.0781e-13, ..., 1.1326e-12, 1.0867e-12,\n 1.1509e-12]], device='cuda:0')" + }, + "21": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-1.2042e-06, 8.8363e-06, 1.4672e-06, 3.3788e-06, 5.5388e-06,\n -2.8798e-06, 9.1521e-06, -6.6564e-07, -2.0665e-06, -3.4548e-06,\n 1.8231e-06, 4.1312e-06, 8.9411e-06, -1.1482e-06, -7.2571e-07,\n -2.3811e-06, 1.0760e-06, 3.7043e-06, -6.4195e-07, -9.0443e-07,\n 6.4955e-07, -4.0716e-06, -2.0068e-06, -5.0657e-06, -5.1874e-06,\n -1.5485e-07, 5.1118e-07, 3.0652e-06, -3.5615e-06, -2.9529e-07,\n -1.9651e-06, -1.2553e-07, -2.1692e-06, -4.1059e-06, -4.7186e-07,\n 2.6526e-06, -2.7809e-06, -8.9990e-07, 1.5551e-07, -5.0544e-06,\n -5.2839e-06, 2.6836e-07, -1.1829e-05, -3.7852e-07, -5.1404e-06,\n -3.8751e-06, 1.3240e-06, -4.9074e-07, 1.9073e-06, 8.8001e-07,\n -2.7892e-06, -2.8528e-06, 1.6728e-06, -5.3522e-06, 6.7619e-07,\n 8.1280e-07, 3.1728e-06, 2.1965e-06, -4.3701e-06, -1.6988e-06,\n -4.7744e-06, 1.3019e-07, -1.8511e-06, 1.1866e-06, -1.2808e-06,\n 2.6273e-06, -2.0048e-06, 6.2800e-06, 7.1965e-08, -1.0297e-05,\n 8.7407e-07, -3.5704e-06, -2.2211e-06, -6.2157e-06, 3.2167e-06,\n -4.6849e-06, 3.4966e-06, -4.8241e-06, 1.8750e-07, -1.0645e-07,\n -1.9865e-07, 7.9019e-07, -4.7221e-06, -6.8146e-07, -4.2897e-06,\n -3.3561e-06, 3.9000e-07, 5.7097e-06, -6.0355e-07, -2.3048e-06,\n 1.3484e-06, -1.3436e-06, 1.2342e-06, -2.3285e-06, -2.1092e-06,\n 7.4754e-06, -3.2679e-06, -1.3022e-06, 4.2184e-06, 2.9717e-06,\n -4.5996e-06, -1.4160e-06, -4.7873e-06, 8.9624e-08, 4.0981e-06,\n -2.7543e-06, -1.6264e-06, -1.6246e-06, -8.9466e-06, 1.4263e-06,\n -7.4578e-07, -1.0142e-05, -5.8584e-08, -2.9797e-06, -2.0145e-06,\n 2.8796e-06, 6.7742e-06, -1.1539e-06, -1.1452e-05, -9.5391e-07,\n 2.3004e-06, -6.3788e-06, 1.5067e-06, 4.0441e-06, -1.1818e-06,\n 2.7343e-06, -3.9005e-06, -1.8266e-08, -2.6877e-06, -2.6806e-06,\n -6.2856e-07, 1.4850e-06, -4.9774e-06, 2.5710e-06, -1.1934e-06,\n -2.6881e-06, -2.2205e-06, -8.1107e-07, -1.8771e-07, -1.2462e-05,\n 5.1503e-06, 2.9728e-06, -1.5281e-06, -8.0246e-07, 2.5584e-06,\n -7.2815e-06, 1.0396e-06, 3.5323e-06, -4.8939e-07, 2.9718e-06,\n -5.1025e-06, -3.3874e-06, 8.2295e-06, 8.6723e-07, -1.3014e-07,\n 2.4682e-06, 3.5137e-06, 5.7704e-07, -5.4872e-06, 1.5786e-06,\n 1.0758e-05, -3.5503e-06, -1.1967e-06, -5.5565e-07, -2.6538e-06,\n -4.1240e-06, 2.3669e-06, -4.3096e-07, 2.3855e-07, 2.2944e-07,\n -5.4188e-07, -1.7459e-06, -3.7143e-07, -1.0179e-06, -3.2814e-06,\n 6.8497e-06, -2.8893e-06, 9.0586e-06, -3.3774e-06, -3.1137e-06,\n -4.0516e-06, -1.1323e-06, 6.4535e-07, -9.0944e-07, 6.6206e-06,\n -3.1080e-06, 2.9605e-06, 7.1356e-07, -1.2819e-06, 6.0876e-06,\n -2.3439e-06, -5.1166e-06, -3.9820e-06, 4.4966e-07, -4.2018e-06,\n -9.7044e-06, 2.9239e-06, -5.1030e-06, 6.6692e-08, -1.6478e-06,\n -5.0986e-06, -4.4228e-06, 1.5033e-06, 7.3227e-06, -2.2086e-06,\n -1.1604e-06, 1.1657e-06, 1.1971e-06, -3.0062e-06, 5.2970e-06,\n -1.8089e-06, 2.6705e-06, 3.1129e-07, -3.4280e-06, 3.1320e-06,\n -2.4017e-06, 7.7081e-07, -1.1166e-05, 6.7466e-06, 8.6179e-06,\n -1.9300e-06, -6.6755e-07, -1.7680e-07, -1.4308e-06, 8.5073e-07,\n -4.7441e-06, -3.8734e-06, 7.0087e-07, 3.5938e-06, 5.6883e-07,\n -1.7719e-06, -1.1445e-06, 4.9141e-06, -2.3742e-06, -2.4866e-06,\n -5.3734e-06, -1.9246e-07, -1.8312e-06, 3.8486e-06, 1.2724e-07,\n 4.9097e-07, 4.5627e-06, -1.0401e-06, 5.0416e-07, -4.2258e-07,\n 8.0117e-08, -1.8858e-09, 2.4741e-06, -4.7726e-07, 6.9916e-06,\n 4.2795e-06, -4.1993e-06, -3.4995e-06, 1.3678e-06, 2.1522e-06,\n -4.2537e-06, 3.9671e-07, 2.6604e-06, -4.6995e-06, -8.2167e-06,\n -2.3112e-06, 4.2228e-06, -6.5671e-06, 9.2918e-06, -5.7117e-06,\n 1.1743e-06, 4.4958e-06, -2.4914e-06, 4.8162e-06, 8.1389e-07,\n -1.1878e-06, 9.7517e-07, 4.4324e-06, 1.4324e-06, -7.5361e-06,\n -2.5458e-06, 4.8520e-06, 2.8056e-06, 3.5587e-06, 2.2347e-06,\n -9.1074e-06, 2.1431e-06, 1.2004e-06, -1.2632e-06, -3.6521e-06,\n 7.5907e-06, -5.6155e-07, -4.6731e-07, 2.7923e-06, 1.9627e-06,\n -2.1501e-06, 8.4688e-06, -2.1693e-06, 4.3171e-06, -4.6481e-06,\n 1.4762e-06, 3.6154e-07, 5.2238e-06, -2.1793e-08, 5.6133e-06,\n -4.2444e-07, -2.5681e-07, -4.8288e-06, 4.4037e-07, -2.0070e-07,\n -3.2816e-06, 5.0667e-07, 6.2903e-06, 6.2171e-07, 2.2749e-06,\n -3.0475e-06, -8.6061e-06, 2.3573e-06, -5.0424e-07, -1.4465e-07,\n -2.0087e-06, 2.2205e-06, 6.4259e-06, -1.5632e-06, 4.7544e-06,\n 2.6614e-06, -1.1264e-07, -7.2951e-07, -1.5347e-06, 2.9669e-06,\n -2.5112e-06, -4.4477e-06, 3.0955e-07, -1.6361e-06, 4.9688e-07,\n 3.2871e-07, -1.5052e-07, 2.6910e-06, -2.2929e-07, 2.2357e-06,\n -2.8350e-06, 5.3562e-06, 2.2635e-06, -3.5634e-06, 1.2104e-06,\n 4.0088e-07, 2.5112e-07, 3.0535e-06, -3.2800e-06, -4.3994e-07,\n -8.6007e-07, -3.4953e-06, 4.8769e-07, -2.6661e-07, 3.1428e-06,\n -7.2264e-07, -1.1884e-06, 1.7263e-06, 4.0939e-06, 1.9052e-06,\n 8.1283e-07, 2.0080e-06, 8.2394e-06, -2.8449e-06, -1.4265e-06,\n -1.8857e-06, 1.6636e-06, -3.6691e-06, -1.4707e-06, 3.6715e-06,\n 4.3647e-06, 8.0968e-07, -2.0999e-06, -3.6481e-06, -3.3878e-06,\n 7.9403e-06, 1.0148e-06, -1.7691e-06, -1.2333e-06, -1.4160e-06,\n -6.6413e-06, -3.5660e-06, -1.9045e-07, -1.8647e-06, 4.1893e-06,\n 4.6165e-06, -5.7341e-07, -1.0761e-06, -3.8094e-06, -5.1536e-06,\n 4.4942e-06, 3.9308e-06, 3.1062e-06, 3.4416e-06, -1.7938e-06,\n 3.1237e-06, 1.0839e-06, -1.2981e-07, -4.2606e-06, -1.5561e-06,\n 1.0453e-06, 1.7608e-06, 2.9455e-06, -1.1954e-06, 2.5747e-06,\n 4.6660e-07, 5.3246e-06, -1.1411e-06, 2.0836e-06, -5.1035e-06,\n -3.3748e-07, -1.8551e-06, 5.5033e-06, -1.7341e-06, -1.7080e-07,\n -1.1571e-05, 1.8913e-05, -2.4310e-06, 2.0827e-06, -8.0021e-06,\n 1.5052e-06, -1.8308e-06, -1.7234e-07, 5.1487e-06, 1.5651e-06,\n -1.8639e-06, -4.0421e-06, -1.3293e-06, 2.6710e-06, -1.9974e-06,\n 8.6039e-06, 3.1629e-06, -4.0663e-06, 2.9466e-07, -8.7285e-06,\n -1.7325e-06, -6.4326e-07, 3.8306e-06, -2.3758e-06, -6.4061e-07,\n -3.7064e-07, -4.0938e-06, 2.9147e-06, -1.4528e-06, -8.8986e-06,\n 9.7994e-06, -1.8247e-06, 2.6451e-06, -1.6850e-06, -4.4458e-06,\n 9.8587e-07, 2.5511e-06, 5.2044e-06, 2.4764e-06, 2.9501e-06,\n -1.1277e-06, -1.1256e-06, -5.9659e-06, 4.6859e-06, -5.5080e-07,\n 2.1308e-06, 2.3892e-06, -1.8424e-06, -3.3642e-06, 5.3492e-06,\n 7.7364e-06, 3.7736e-07, -4.9435e-07, -1.5688e-06, -1.3170e-06,\n -1.2504e-07, 3.9000e-06, 2.8257e-06, -8.1336e-08, 1.2944e-06,\n -5.4063e-07, 3.6845e-06, -1.6944e-06, -2.8732e-06, -4.1867e-06,\n -7.0411e-06, 2.2356e-06, -2.9940e-06, 6.2947e-06, -4.2174e-06,\n -7.8725e-08, -4.3605e-06, 7.7019e-06, 1.9780e-07, 1.2078e-06,\n -3.9357e-06, 2.6189e-09, -3.8150e-06, 4.0178e-07, 1.1916e-06,\n 1.1852e-06, -7.5635e-07, -3.9006e-06, -3.4068e-06, -1.5612e-06,\n -2.9337e-06, -4.4170e-06, 2.3404e-06, -8.5796e-07, 1.8429e-06,\n 6.5602e-07, 6.9809e-07, -2.1126e-06, -5.1163e-06, 5.3695e-06,\n -5.9563e-06, 5.4309e-06, -1.9199e-06, 4.3555e-06, -3.2586e-07,\n 3.9932e-06, -3.8428e-06], device='cuda:0')", + "exp_avg_sq": "tensor([2.2944e-10, 1.5081e-09, 2.3059e-10, 2.4576e-10, 8.0178e-10, 3.0943e-10,\n 7.0602e-10, 1.8771e-10, 1.9302e-10, 1.0287e-09, 1.0628e-09, 3.1426e-10,\n 1.6963e-09, 1.7472e-10, 3.3142e-10, 2.8628e-10, 1.7697e-10, 5.8261e-10,\n 1.1474e-10, 1.8905e-10, 2.3580e-10, 9.7576e-10, 2.4818e-10, 6.1335e-10,\n 1.8476e-10, 2.3021e-10, 5.6652e-10, 5.7835e-10, 1.9609e-10, 2.2240e-10,\n 2.4770e-10, 2.0571e-10, 3.0688e-10, 4.5851e-10, 1.2856e-10, 2.7672e-10,\n 4.7086e-10, 2.0252e-10, 2.7641e-10, 2.1755e-10, 1.0263e-09, 3.2570e-10,\n 9.5353e-10, 4.9034e-10, 2.9157e-10, 2.9184e-10, 2.4162e-10, 4.5922e-10,\n 3.0760e-10, 3.6671e-10, 1.6928e-10, 2.4433e-10, 8.6792e-10, 3.3377e-10,\n 4.3494e-10, 4.5316e-10, 4.8981e-10, 2.1407e-10, 9.3120e-10, 1.7413e-10,\n 1.8775e-10, 2.5480e-10, 3.6933e-10, 2.6025e-10, 2.7952e-10, 1.5158e-10,\n 2.9914e-10, 9.4919e-10, 4.9176e-10, 6.8736e-10, 1.8547e-10, 3.5574e-10,\n 1.9016e-10, 3.1080e-10, 3.9679e-10, 1.6436e-10, 2.0932e-10, 1.3495e-09,\n 4.2572e-10, 8.6126e-10, 5.6019e-10, 2.9045e-10, 4.0753e-10, 4.3884e-10,\n 1.1813e-10, 3.4150e-10, 3.1827e-10, 9.9427e-10, 3.3066e-10, 2.3442e-10,\n 1.8081e-10, 7.2552e-10, 2.5409e-10, 6.8315e-10, 3.9777e-10, 3.0502e-10,\n 2.5013e-10, 6.3168e-10, 2.3339e-10, 5.6106e-10, 6.7873e-10, 1.5203e-10,\n 1.4582e-09, 3.7781e-10, 1.6548e-10, 3.2092e-10, 2.6325e-10, 5.0332e-10,\n 7.2326e-10, 1.4415e-10, 3.9140e-10, 1.1180e-09, 2.1713e-10, 3.8854e-10,\n 1.3461e-09, 3.3352e-10, 4.4183e-10, 1.7497e-10, 3.7224e-09, 4.1132e-10,\n 4.1572e-10, 1.2630e-09, 3.6226e-10, 3.0733e-10, 2.0701e-10, 3.1289e-10,\n 3.8147e-10, 6.3742e-10, 3.3211e-10, 2.1690e-10, 1.3664e-09, 3.1656e-10,\n 1.4158e-09, 8.3625e-10, 1.5168e-10, 1.7424e-10, 9.7931e-10, 3.4701e-10,\n 3.0459e-10, 1.0903e-09, 2.8827e-10, 3.4609e-10, 1.0531e-09, 3.0332e-10,\n 2.5095e-10, 3.0887e-10, 4.5912e-10, 1.6719e-10, 1.2948e-10, 3.3805e-10,\n 3.9573e-10, 1.6952e-10, 4.0880e-10, 3.6814e-10, 3.3885e-10, 2.3239e-10,\n 1.3428e-10, 1.5305e-10, 3.0952e-10, 5.2807e-10, 2.1422e-09, 2.1723e-10,\n 2.8692e-10, 3.1721e-10, 1.6219e-10, 3.0409e-10, 1.2932e-10, 4.5128e-10,\n 4.4804e-10, 3.5083e-10, 9.2299e-10, 3.0164e-10, 4.3919e-10, 2.9645e-10,\n 3.9817e-10, 7.4835e-10, 1.5221e-10, 1.2325e-09, 3.0020e-10, 2.9356e-10,\n 2.1426e-10, 1.1542e-10, 2.1338e-10, 4.5374e-10, 5.1595e-10, 1.2157e-10,\n 2.8216e-10, 1.8161e-10, 1.6138e-10, 1.7835e-09, 6.5136e-10, 9.2726e-10,\n 5.0277e-10, 1.2403e-10, 3.8451e-10, 4.7244e-10, 3.3592e-10, 5.1906e-10,\n 4.6123e-10, 1.0287e-10, 2.3148e-10, 4.0457e-10, 1.5918e-10, 1.0106e-09,\n 1.3644e-10, 3.0444e-10, 2.0725e-10, 2.8577e-10, 1.1775e-09, 3.6811e-10,\n 2.4178e-10, 2.8208e-10, 3.2226e-10, 3.4473e-10, 2.3397e-10, 9.0941e-10,\n 1.0310e-09, 1.1668e-09, 8.2208e-10, 4.9907e-10, 8.7553e-10, 2.6492e-10,\n 2.1301e-09, 3.7346e-10, 2.4187e-10, 1.7517e-10, 3.4026e-10, 4.1188e-10,\n 4.4467e-10, 3.9059e-10, 3.7136e-10, 1.3183e-10, 2.9787e-10, 4.2162e-10,\n 6.6703e-10, 1.7172e-09, 1.3339e-10, 2.1629e-10, 3.6734e-10, 3.9419e-10,\n 1.9015e-10, 2.0256e-10, 4.2343e-10, 3.8787e-10, 1.9546e-10, 2.0421e-10,\n 1.8039e-10, 9.7920e-10, 6.3035e-10, 5.6837e-10, 3.8997e-10, 5.1553e-10,\n 6.6722e-10, 2.4137e-10, 4.4462e-10, 1.2813e-10, 7.3772e-10, 4.6477e-10,\n 2.3763e-10, 1.6604e-09, 1.5434e-10, 1.2191e-09, 4.1071e-10, 1.6863e-09,\n 1.1098e-09, 1.4935e-10, 7.7185e-10, 6.4842e-10, 4.6167e-10, 2.4112e-10,\n 1.4701e-10, 3.8998e-10, 6.6842e-10, 2.7500e-10, 2.9344e-10, 5.5196e-10,\n 5.8305e-10, 3.2554e-10, 1.9624e-10, 2.1345e-10, 4.5610e-10, 2.1062e-10,\n 4.7551e-10, 1.5795e-10, 3.5360e-10, 3.0124e-10, 2.5711e-10, 2.5507e-10,\n 6.0636e-10, 1.6743e-10, 5.8793e-10, 1.0792e-09, 5.2629e-10, 6.2593e-10,\n 2.7842e-10, 4.9456e-10, 1.4378e-10, 2.8095e-10, 1.8613e-10, 1.2370e-09,\n 5.7362e-10, 3.2937e-10, 3.8358e-10, 6.0833e-10, 3.1451e-10, 6.2360e-10,\n 1.1638e-10, 7.4472e-10, 6.7752e-10, 4.6018e-10, 8.5885e-10, 7.5479e-10,\n 1.2948e-10, 3.3734e-10, 2.1225e-10, 2.8755e-10, 2.9351e-10, 3.5697e-10,\n 3.6215e-10, 6.6899e-10, 2.1931e-10, 2.8353e-10, 3.7538e-10, 2.4482e-10,\n 3.5651e-10, 1.4624e-10, 3.5287e-10, 2.6524e-09, 3.7972e-10, 3.5238e-10,\n 1.6718e-10, 2.7186e-10, 5.9350e-10, 1.0256e-09, 2.6340e-10, 3.4930e-10,\n 3.1757e-10, 6.3915e-10, 2.8024e-10, 2.8509e-10, 1.0493e-09, 3.5616e-10,\n 3.2092e-10, 3.1615e-10, 1.6092e-10, 2.2597e-10, 1.7679e-10, 2.3767e-10,\n 2.8103e-10, 4.1328e-10, 1.8914e-10, 2.5164e-10, 3.0982e-10, 7.4774e-10,\n 2.0601e-10, 2.1715e-10, 2.1896e-10, 4.2614e-10, 4.8934e-10, 4.0566e-10,\n 2.0679e-10, 2.4923e-10, 2.2767e-10, 3.3630e-10, 2.6909e-10, 2.8902e-10,\n 3.3068e-10, 1.1462e-10, 6.7477e-10, 3.5318e-10, 7.6201e-10, 1.4014e-10,\n 1.8974e-10, 1.2137e-09, 2.6953e-10, 3.4632e-09, 5.4007e-10, 4.3153e-10,\n 3.1834e-10, 2.6805e-10, 1.1195e-09, 3.7290e-10, 1.9001e-10, 2.2419e-10,\n 3.0198e-10, 6.9121e-10, 3.1198e-10, 4.3392e-10, 8.1828e-10, 2.0769e-10,\n 2.3923e-10, 2.3102e-10, 4.7555e-10, 3.2525e-10, 3.0944e-10, 2.2786e-10,\n 1.9127e-10, 3.0247e-10, 2.1822e-10, 2.2428e-10, 2.8080e-10, 2.3854e-10,\n 1.2544e-10, 7.4978e-10, 2.3943e-10, 1.3510e-09, 2.4764e-10, 3.8983e-10,\n 3.4904e-10, 4.2369e-10, 1.3037e-09, 2.1723e-09, 4.5519e-10, 3.2535e-10,\n 6.0892e-10, 5.1309e-10, 8.5464e-11, 2.3669e-10, 2.5653e-10, 4.8598e-10,\n 3.4706e-10, 1.2692e-10, 1.4486e-10, 9.7317e-10, 3.3915e-10, 3.1908e-09,\n 4.8079e-10, 5.2912e-10, 3.5568e-10, 1.9838e-09, 3.1851e-10, 1.2343e-10,\n 6.5564e-10, 1.4002e-10, 1.1924e-10, 2.5505e-10, 5.2487e-10, 2.8452e-10,\n 4.9253e-10, 1.4541e-09, 1.2516e-09, 2.1021e-10, 2.7498e-10, 3.2893e-10,\n 3.9946e-10, 5.9761e-10, 6.9417e-10, 7.2444e-10, 1.5911e-10, 7.9340e-10,\n 3.5720e-10, 7.5432e-10, 3.8611e-10, 1.5558e-10, 3.1373e-10, 1.0372e-09,\n 6.5170e-10, 1.7310e-10, 4.3543e-10, 9.3059e-10, 1.2396e-09, 2.0810e-10,\n 3.6587e-10, 5.9531e-10, 7.2200e-10, 3.8558e-10, 1.3177e-10, 2.1361e-10,\n 3.7704e-10, 3.4285e-10, 1.0880e-10, 5.9377e-10, 2.8637e-10, 3.1031e-10,\n 1.9924e-10, 5.8744e-10, 2.6033e-10, 1.0379e-10, 1.8277e-10, 2.6347e-10,\n 2.7361e-10, 1.8679e-10, 1.2011e-09, 2.8134e-10, 2.0653e-10, 1.0396e-09,\n 3.4727e-10, 7.9382e-10, 1.7770e-10, 9.9505e-10, 2.5788e-10, 6.2059e-10,\n 3.9862e-10, 2.4404e-10, 4.6657e-10, 2.0251e-10, 1.6062e-10, 3.0252e-10,\n 6.9170e-10, 3.1281e-10, 2.9064e-10, 2.9988e-10, 3.5341e-10, 2.1505e-09,\n 4.1756e-10, 1.7264e-10, 3.0750e-10, 6.6954e-10, 3.5634e-10, 1.9816e-10,\n 6.4875e-10, 3.1585e-10], device='cuda:0')" + }, + "22": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-8.0584e-07, -2.8871e-07, 1.5006e-06, ..., -3.1148e-07,\n 2.6384e-08, 9.0573e-07],\n [-7.9481e-08, 1.1765e-06, -3.2110e-09, ..., 4.5217e-08,\n -1.5797e-08, -3.8347e-07],\n [ 2.4331e-07, -7.9877e-07, -6.5744e-07, ..., -4.8364e-07,\n -1.7780e-07, 5.2916e-07],\n ...,\n [-2.8170e-07, 1.9121e-07, 8.2213e-08, ..., -1.8574e-08,\n 3.0128e-07, 9.0368e-07],\n [-6.0699e-07, -1.5089e-06, 4.1106e-07, ..., -9.2504e-07,\n 9.4848e-07, 1.7523e-06],\n [-8.1317e-07, 8.7940e-07, 2.6136e-07, ..., 3.4329e-07,\n -7.3223e-07, -6.0246e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[4.2862e-12, 6.2013e-12, 6.8349e-12, ..., 6.4823e-12, 6.9055e-12,\n 6.8357e-12],\n [1.2045e-11, 1.9640e-11, 2.0179e-11, ..., 1.9175e-11, 2.0379e-11,\n 2.0244e-11],\n [1.9579e-12, 3.1139e-12, 3.1516e-12, ..., 3.1406e-12, 3.4154e-12,\n 3.3847e-12],\n ...,\n [2.3023e-12, 3.8010e-12, 3.6827e-12, ..., 3.6725e-12, 3.8749e-12,\n 4.0634e-12],\n [7.9799e-12, 1.1611e-11, 1.2877e-11, ..., 1.3373e-11, 1.2779e-11,\n 1.2481e-11],\n [3.3841e-12, 5.1044e-12, 5.7837e-12, ..., 5.8930e-12, 5.6829e-12,\n 6.2051e-12]], device='cuda:0')" + }, + "23": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 5.5100e-06, 2.1013e-05, 9.0435e-07, -1.1986e-05, -1.4026e-05,\n -5.6809e-06, 2.1157e-05, 1.8707e-05, 4.4125e-06, 3.4224e-05,\n 3.1897e-05, 1.2879e-05, 8.4504e-06, -3.2906e-06, -1.2346e-05,\n -1.2867e-06, -8.9094e-07, 2.0792e-05, 7.6552e-06, -1.2597e-05,\n 1.5593e-05, -2.8042e-06, -6.3596e-06, 1.3125e-05, -5.6715e-06,\n 2.1684e-05, -4.0542e-05, -8.7839e-06, -1.2524e-05, 5.0247e-06,\n 3.1513e-06, 9.4396e-06, -4.8962e-06, -1.4273e-05, -5.5897e-07,\n -1.8343e-05, 1.3504e-05, 1.5740e-05, -1.2555e-05, 8.5191e-06,\n -2.9128e-05, -1.4460e-05, -1.6760e-05, -1.8089e-05, -1.9697e-05,\n -7.7207e-06, -1.0714e-05, -4.9815e-06, -1.1795e-05, -3.1259e-05,\n 7.4076e-07, 6.1044e-06, -1.9765e-05, 1.1133e-05, 3.7288e-05,\n -9.0934e-06, -1.6977e-05, -3.1900e-05, 2.7106e-05, 4.3904e-06,\n 9.9379e-06, 1.2953e-05, 1.6624e-05, 6.5794e-06, 3.0599e-06,\n 7.4255e-06, 1.6732e-05, 2.8772e-05, -2.1814e-05, -1.2056e-05,\n -1.8483e-05, -1.3695e-06, -1.0751e-05, 5.6892e-06, -8.4630e-06,\n -6.8419e-06, -1.5228e-07, 8.1182e-06, 2.9134e-05, 1.3765e-05,\n 2.2198e-07, 5.1393e-08, -2.4358e-06, 1.5084e-05, -4.3775e-07,\n 6.7452e-06, -3.6443e-06, 1.1237e-05, -9.7119e-06, -1.2198e-05,\n -1.3711e-06, 1.8163e-05, -6.6563e-07, -2.3990e-05, -9.6826e-07,\n 2.3553e-06, -1.1692e-05, 2.2151e-05, 7.1339e-06, 2.5507e-05,\n 1.7995e-05, 5.8339e-06, 3.9829e-05, -5.3518e-06, -7.7940e-07,\n 3.0166e-05, -1.6171e-05, -8.2798e-06, -1.5300e-05, 3.2256e-06,\n 2.4952e-05, -1.3794e-05, 3.1710e-05, -8.6336e-06, 3.3596e-05,\n -1.5862e-05, 1.4615e-05, 1.6285e-05, -4.7396e-06, -6.5671e-06,\n -1.3549e-05, 3.9746e-05, -1.3414e-05, 9.4477e-06, -5.5261e-06,\n 7.5860e-06, 2.9466e-06, -2.7409e-05, 1.8726e-05, -1.1710e-05,\n 4.0098e-05, 6.9057e-06, 2.5482e-05, -9.1583e-06, 7.5027e-07,\n -6.1267e-06, 1.5079e-05, 1.0522e-05, -2.9411e-05, -6.6591e-06,\n -1.2281e-05, 1.5299e-06, 2.3833e-05, -2.6113e-05, -1.9990e-05,\n -2.2189e-07, -2.1357e-05, 5.7370e-06, -6.1421e-06, 1.8417e-06,\n -1.0714e-05, -3.3256e-06, -4.9316e-06, 1.0410e-05, -1.2662e-05,\n 2.8341e-05, -1.0902e-06, 9.1936e-06, 1.6962e-05, -1.4467e-05,\n 2.7080e-05, 9.7442e-07, 3.5009e-06, 1.0010e-05, -1.1461e-05,\n 7.9072e-06, 2.2739e-06, -7.3958e-07, -1.4143e-06, 2.1639e-05,\n -9.6419e-06, 1.9382e-05, 1.9404e-07, 1.8523e-05, -1.3540e-05,\n 3.6582e-05, 4.2720e-06, 1.0653e-05, -7.5114e-06, 3.4583e-06,\n 1.5149e-05, 2.1303e-05, 1.2756e-05, 1.2682e-05, -1.6357e-06,\n -9.3289e-06, -3.3424e-06, -1.5578e-05, 7.1849e-06, 3.5885e-06,\n 3.5125e-05, -8.8680e-07, 1.7615e-05, 4.6952e-06, 1.4494e-05,\n -6.4179e-06, -1.0189e-05, 3.7479e-05, 2.6061e-07, 4.2372e-06,\n -7.2578e-07, 1.5462e-05, -1.1646e-05, -3.2482e-05, -2.0421e-05,\n -9.1434e-06, 5.2789e-06, 9.2567e-06, -8.3283e-07, 1.4838e-05,\n 2.3938e-06, -8.4841e-06, 3.5430e-06, -7.8662e-06, 1.9516e-05,\n -1.4002e-05, -2.2011e-05, -3.0309e-05, 1.1114e-05, 1.3123e-06,\n -8.3692e-06, 2.6277e-05, -7.3310e-06, -1.6184e-06, -6.6766e-06,\n 5.3768e-06, -1.3148e-05, -1.3768e-05, -8.7069e-06, 2.5758e-06,\n -4.2472e-06, 5.8674e-06, -1.5083e-05, -1.5480e-06, -1.5418e-05,\n -6.5097e-06, -7.0255e-06, -8.2752e-07, -2.4516e-05, 1.9163e-05,\n -1.8991e-06, 1.4545e-05, 3.5234e-05, 1.4143e-05, 1.4509e-07,\n -1.9986e-06, 1.9048e-05, -3.0335e-05, -1.7560e-05, 6.8990e-06,\n 7.8708e-06, -3.5557e-05, 8.0418e-06, 5.9436e-06, -2.4416e-05,\n -2.6198e-06, 1.4102e-05, -2.3014e-05, 1.5829e-05, -3.4757e-05,\n 1.5098e-05, -3.1583e-05, 9.9725e-06, 2.1093e-05, 2.4697e-05,\n -1.3612e-05, 2.0188e-05, 3.2203e-05, 8.4816e-07, -1.5657e-05,\n -5.1585e-06, -7.4951e-06, -2.4079e-06, -5.3353e-06, 7.6903e-06,\n 1.3704e-05, -1.6669e-05, 5.4309e-06, -4.6025e-06, -1.4553e-05,\n 6.1491e-06, -2.5946e-05, -4.3220e-06, 4.9240e-06, -2.5046e-05,\n -2.9561e-05, -1.8876e-05, -2.6042e-05, -1.3232e-05, -1.6048e-05,\n -8.3112e-06, 7.1987e-06, 2.5647e-05, 9.3209e-06, 4.7370e-06,\n -3.9709e-06, 1.3815e-06, -2.0226e-05, 5.5147e-06, -3.9811e-07,\n 3.7055e-05, -8.9672e-06, -1.1486e-05, -1.6330e-05, -6.0081e-06,\n -1.8869e-05, -7.2728e-06, -1.9118e-06, 3.4414e-05, -1.1981e-05,\n 2.1710e-05, -7.0189e-06, 8.2509e-06, -9.6941e-07, -4.9352e-07,\n 1.9613e-05, 1.2260e-05, 2.9492e-05, -7.3355e-06, -2.6428e-05,\n -1.8659e-05, -8.0595e-06, -1.6381e-05, -2.4903e-05, 8.7132e-07,\n -1.3092e-05, -1.3346e-05, 5.2249e-06, -6.0174e-07, 2.6775e-05,\n -1.9972e-05, -2.6260e-05, -2.1495e-05, -1.5607e-05, -8.3165e-06,\n -7.5504e-06, 2.1862e-05, -4.1896e-05, 5.5689e-06, 2.2803e-05,\n 3.2073e-05, -2.6867e-07, -1.7802e-07, 2.8671e-05, 5.2326e-06,\n 2.0084e-05, 1.9808e-06, 9.5837e-06, -6.3101e-07, 1.5488e-05,\n 1.4512e-05, 3.3626e-06, 1.7884e-05, 2.4188e-05, 1.3677e-05,\n -1.6815e-05, -1.1455e-05, 9.7987e-06, 1.7857e-05, 7.5139e-07,\n 1.0513e-05, -1.9947e-06, -1.9501e-06, 9.6202e-07, -3.6853e-06,\n -4.0272e-06, -2.2479e-06, -3.8615e-06, 1.4187e-05, 8.5544e-06,\n 9.2347e-06, 2.5107e-06, 4.1783e-06, -2.5198e-05, -8.3858e-06,\n 2.1363e-07, 1.9427e-05, 1.2683e-05, -3.0369e-06, 1.3481e-05,\n 1.9387e-05, -4.0807e-06, -1.3192e-05, -1.5937e-05, 1.5795e-05,\n 1.0241e-05, -5.1454e-06, -1.4663e-05, -6.6645e-06, 1.2845e-05,\n 1.5701e-05, 5.2624e-06, -9.0677e-06, -9.9361e-06, 4.6963e-06,\n 1.6182e-05, -1.4626e-08, -3.3171e-06, -3.5155e-06, 1.1042e-06,\n 5.1067e-06, 1.4736e-05, -1.2210e-05, 3.3104e-05, 2.6651e-05,\n 4.0698e-06, 8.0840e-06, -1.5601e-05, 1.6649e-05, 1.5808e-05,\n -1.8469e-05, 4.9196e-05, 1.0578e-05, 1.0855e-05, -1.3498e-05,\n 3.3911e-05, 6.3933e-06, -1.6888e-05, -3.9941e-06, -8.1765e-06,\n -2.0148e-05, -1.3833e-05, -2.8331e-07, 7.4838e-06, 2.5542e-05,\n -2.0522e-05, 2.4442e-05, -1.1897e-06, -2.4586e-05, 4.9309e-06,\n -3.0353e-06, 5.0734e-06, -3.1385e-05, 1.2639e-06, -8.7947e-07,\n -1.3746e-08, 3.9285e-06, -5.4851e-06, 2.2500e-05, -4.0261e-06,\n 1.9103e-05, -8.2596e-06, -8.1129e-06, 5.2356e-06, -1.1123e-05,\n -9.5744e-06, -8.9056e-06, -2.4038e-05, -1.2402e-05, -2.9502e-05,\n -4.5715e-06, -2.0561e-05, 1.9399e-06, -1.6741e-05, -1.8959e-05,\n -7.6699e-06, -2.9415e-06, 1.5254e-05, -8.0267e-07, 3.5813e-06,\n 9.8081e-06, 4.1717e-06, -1.5909e-05, -2.2507e-05, 2.0222e-05,\n -2.8835e-05, -1.1185e-05, 1.7828e-06, -6.8799e-06, -2.5345e-07,\n -6.6996e-06, 5.1365e-06, 2.8745e-05, 4.3310e-06, 1.7053e-05,\n 1.4040e-06, -1.3954e-06, 1.5234e-06, -1.1512e-05, 1.8344e-06,\n 1.3388e-05, 3.1363e-06, -1.7458e-05, -1.4263e-06, 4.3866e-06,\n 2.6843e-05, 7.1009e-08, 2.9806e-05, -6.8849e-06, -1.0301e-05,\n -7.9216e-06, -1.6593e-05, -1.5750e-05, 6.4854e-06, 3.1893e-05,\n 2.0668e-05, 4.2614e-06, -1.2438e-05, 1.6731e-05, 4.1656e-06,\n -2.1799e-05, -1.3561e-05, 7.1276e-06, 3.7166e-05, -7.8912e-06,\n -8.9788e-06, 7.8531e-06, -1.3691e-05, 2.3507e-05, -1.3686e-06,\n -1.5825e-05, -4.6869e-06], device='cuda:0')", + "exp_avg_sq": "tensor([3.3168e-09, 1.0786e-08, 1.7343e-09, 2.4520e-09, 7.8630e-09, 3.1419e-09,\n 5.4908e-09, 2.0258e-09, 1.4897e-09, 9.7944e-09, 5.9678e-09, 3.1294e-09,\n 9.8137e-09, 1.4841e-09, 2.7889e-09, 2.5494e-09, 1.7481e-09, 5.5707e-09,\n 1.2571e-09, 1.3960e-09, 3.1723e-09, 6.5397e-09, 2.1539e-09, 7.1065e-09,\n 1.2347e-09, 2.2366e-09, 5.1465e-09, 5.1418e-09, 2.3058e-09, 1.6222e-09,\n 2.3162e-09, 1.8191e-09, 2.8663e-09, 5.4455e-09, 1.3366e-09, 2.9715e-09,\n 4.0840e-09, 2.9250e-09, 3.0955e-09, 2.3813e-09, 1.1353e-08, 2.5070e-09,\n 9.0187e-09, 4.6193e-09, 2.5384e-09, 1.9991e-09, 2.0226e-09, 4.3968e-09,\n 2.2272e-09, 3.6405e-09, 1.7533e-09, 2.0577e-09, 6.6678e-09, 2.8635e-09,\n 4.8326e-09, 4.1501e-09, 4.8000e-09, 2.4792e-09, 9.5796e-09, 1.5870e-09,\n 1.4006e-09, 2.1197e-09, 3.4743e-09, 3.2050e-09, 1.9091e-09, 1.3023e-09,\n 1.9520e-09, 7.3489e-09, 5.6824e-09, 5.9795e-09, 2.0364e-09, 3.4325e-09,\n 2.1284e-09, 2.6492e-09, 3.2968e-09, 2.2000e-09, 2.2603e-09, 7.5570e-09,\n 4.0669e-09, 9.9691e-09, 4.0316e-09, 4.2496e-09, 4.0517e-09, 4.0284e-09,\n 1.2999e-09, 4.3207e-09, 2.8005e-09, 7.3945e-09, 2.8897e-09, 2.4947e-09,\n 1.6255e-09, 9.1673e-09, 1.8335e-09, 4.8530e-09, 2.2791e-09, 2.5729e-09,\n 2.4723e-09, 4.6647e-09, 2.8871e-09, 6.4919e-09, 6.5185e-09, 1.5364e-09,\n 1.7050e-08, 3.3403e-09, 1.7706e-09, 3.7750e-09, 1.9550e-09, 4.6063e-09,\n 6.6700e-09, 1.2413e-09, 4.6752e-09, 8.5485e-09, 3.6574e-09, 4.0589e-09,\n 6.7389e-09, 2.5760e-09, 3.9461e-09, 1.6477e-09, 2.3695e-08, 3.7659e-09,\n 4.0178e-09, 1.4619e-08, 4.5432e-09, 3.5923e-09, 1.5676e-09, 3.4599e-09,\n 2.7103e-09, 4.9673e-09, 2.3597e-09, 1.8282e-09, 1.5291e-08, 3.4879e-09,\n 1.3853e-08, 6.4491e-09, 1.4008e-09, 1.6105e-09, 5.0808e-09, 3.2791e-09,\n 4.1871e-09, 1.2139e-08, 2.7435e-09, 4.5082e-09, 5.7689e-09, 2.6316e-09,\n 3.1747e-09, 4.1961e-09, 4.8311e-09, 1.6181e-09, 1.2560e-09, 3.2660e-09,\n 4.2148e-09, 1.9201e-09, 4.1564e-09, 3.0308e-09, 2.6214e-09, 2.5321e-09,\n 1.7554e-09, 1.6318e-09, 2.7910e-09, 3.2585e-09, 1.3978e-08, 2.3407e-09,\n 2.0481e-09, 4.4477e-09, 2.1534e-09, 4.8122e-09, 1.3252e-09, 5.1197e-09,\n 4.2428e-09, 3.1337e-09, 6.1151e-09, 3.1268e-09, 4.4321e-09, 2.9389e-09,\n 3.7684e-09, 5.2814e-09, 1.8292e-09, 7.5540e-09, 2.3684e-09, 3.0288e-09,\n 2.6974e-09, 1.2788e-09, 2.1007e-09, 4.5937e-09, 5.3805e-09, 9.9736e-10,\n 2.6560e-09, 1.8931e-09, 1.7992e-09, 1.0701e-08, 7.6404e-09, 5.3163e-09,\n 4.1946e-09, 1.6883e-09, 2.7847e-09, 3.3490e-09, 2.4411e-09, 5.7938e-09,\n 4.4878e-09, 1.1457e-09, 1.7422e-09, 5.1925e-09, 1.7402e-09, 1.1685e-08,\n 1.0977e-09, 1.7561e-09, 1.8368e-09, 2.3275e-09, 6.8105e-09, 4.1155e-09,\n 2.5942e-09, 2.3055e-09, 2.4900e-09, 3.9158e-09, 2.2465e-09, 7.0662e-09,\n 9.1118e-09, 1.5732e-08, 6.8397e-09, 5.0173e-09, 8.5750e-09, 2.8546e-09,\n 1.1178e-08, 2.8202e-09, 2.4983e-09, 1.5849e-09, 2.7707e-09, 4.3851e-09,\n 3.1767e-09, 3.1009e-09, 2.7371e-09, 1.5340e-09, 3.1933e-09, 3.1091e-09,\n 5.9751e-09, 1.0278e-08, 1.3988e-09, 1.8151e-09, 4.0100e-09, 4.5857e-09,\n 1.6217e-09, 1.7149e-09, 4.6082e-09, 3.8895e-09, 2.0162e-09, 1.8012e-09,\n 2.2132e-09, 9.6339e-09, 5.8826e-09, 5.9217e-09, 2.7109e-09, 3.9765e-09,\n 5.2650e-09, 2.6276e-09, 4.7432e-09, 1.2786e-09, 4.6641e-09, 3.0729e-09,\n 2.5658e-09, 1.3372e-08, 2.1295e-09, 1.2281e-08, 4.0857e-09, 1.0812e-08,\n 1.0654e-08, 1.6298e-09, 4.4702e-09, 6.2938e-09, 3.8389e-09, 2.6893e-09,\n 1.4531e-09, 3.8847e-09, 6.3708e-09, 2.6398e-09, 3.1942e-09, 6.6751e-09,\n 7.1748e-09, 2.4098e-09, 2.8171e-09, 2.7887e-09, 3.3765e-09, 2.8225e-09,\n 5.3467e-09, 1.8267e-09, 5.0118e-09, 3.2140e-09, 2.4567e-09, 3.5480e-09,\n 7.3733e-09, 2.1654e-09, 4.9700e-09, 7.6590e-09, 3.2681e-09, 7.2753e-09,\n 2.4442e-09, 3.7540e-09, 1.2383e-09, 2.8716e-09, 2.1619e-09, 1.6500e-08,\n 6.7086e-09, 3.5041e-09, 3.0587e-09, 7.3838e-09, 2.9644e-09, 4.2655e-09,\n 1.2259e-09, 9.1690e-09, 5.8155e-09, 4.1151e-09, 9.9615e-09, 6.1851e-09,\n 1.4396e-09, 2.9401e-09, 1.8889e-09, 2.9198e-09, 3.3784e-09, 4.2039e-09,\n 2.8354e-09, 5.6244e-09, 3.0792e-09, 1.8770e-09, 4.1315e-09, 2.0234e-09,\n 3.6860e-09, 1.3627e-09, 3.6976e-09, 2.5470e-08, 2.7320e-09, 4.6545e-09,\n 1.5932e-09, 2.5914e-09, 3.9976e-09, 1.0734e-08, 2.7445e-09, 3.1750e-09,\n 2.8871e-09, 5.9424e-09, 3.3275e-09, 2.1579e-09, 1.2321e-08, 3.0549e-09,\n 2.4288e-09, 3.2101e-09, 1.7122e-09, 2.3565e-09, 2.2118e-09, 2.5319e-09,\n 2.3477e-09, 2.7116e-09, 1.8512e-09, 2.0361e-09, 3.9079e-09, 5.0847e-09,\n 2.1431e-09, 2.1326e-09, 1.8978e-09, 4.2013e-09, 5.0388e-09, 3.6525e-09,\n 2.1469e-09, 1.8492e-09, 2.1586e-09, 2.9785e-09, 2.1283e-09, 3.0488e-09,\n 2.6432e-09, 1.5571e-09, 7.5347e-09, 3.3143e-09, 5.3251e-09, 1.0093e-09,\n 1.5637e-09, 1.4324e-08, 1.7662e-09, 2.7258e-08, 8.0807e-09, 2.9019e-09,\n 3.1304e-09, 1.9654e-09, 7.9171e-09, 4.6710e-09, 1.5637e-09, 2.3938e-09,\n 2.8012e-09, 6.3354e-09, 2.3908e-09, 4.7541e-09, 6.1461e-09, 2.5055e-09,\n 1.9763e-09, 1.8377e-09, 4.3049e-09, 4.3773e-09, 3.0242e-09, 2.5168e-09,\n 1.9188e-09, 2.8260e-09, 1.9685e-09, 2.3082e-09, 2.0391e-09, 1.9982e-09,\n 8.5673e-10, 6.5196e-09, 2.7898e-09, 1.3108e-08, 2.9730e-09, 3.6810e-09,\n 2.3187e-09, 3.6079e-09, 1.0507e-08, 2.7644e-08, 4.4401e-09, 2.8950e-09,\n 5.6352e-09, 4.7453e-09, 1.1511e-09, 3.0881e-09, 2.2045e-09, 4.5463e-09,\n 3.8225e-09, 1.3219e-09, 1.2151e-09, 6.6083e-09, 3.6627e-09, 1.6143e-08,\n 3.9488e-09, 5.3330e-09, 3.1211e-09, 1.0814e-08, 1.9166e-09, 1.7376e-09,\n 5.3158e-09, 1.6553e-09, 1.3635e-09, 1.6159e-09, 5.2853e-09, 1.5401e-09,\n 5.0705e-09, 1.2612e-08, 1.3086e-08, 2.3762e-09, 2.6466e-09, 3.4490e-09,\n 3.6131e-09, 5.7904e-09, 4.3552e-09, 4.5226e-09, 1.4960e-09, 1.1732e-08,\n 3.3827e-09, 9.3746e-09, 3.5743e-09, 1.7494e-09, 3.2296e-09, 6.4670e-09,\n 4.4299e-09, 1.5665e-09, 4.4435e-09, 9.1862e-09, 1.3988e-08, 2.6404e-09,\n 3.3970e-09, 6.2106e-09, 6.5269e-09, 5.7827e-09, 1.6800e-09, 1.4776e-09,\n 4.9474e-09, 3.5477e-09, 1.0108e-09, 7.8343e-09, 3.6720e-09, 2.5553e-09,\n 1.8591e-09, 5.3826e-09, 2.0888e-09, 1.1293e-09, 1.7569e-09, 2.1861e-09,\n 2.8290e-09, 1.8928e-09, 1.1043e-08, 4.2501e-09, 2.1330e-09, 1.3594e-08,\n 2.5255e-09, 9.0695e-09, 2.3810e-09, 9.5957e-09, 2.0784e-09, 4.5802e-09,\n 4.6089e-09, 1.9197e-09, 3.9214e-09, 1.7474e-09, 1.5730e-09, 2.0577e-09,\n 6.7888e-09, 3.2604e-09, 2.7377e-09, 3.2910e-09, 3.1647e-09, 2.1755e-08,\n 3.7800e-09, 1.5393e-09, 2.3762e-09, 6.4953e-09, 3.8213e-09, 1.8987e-09,\n 6.4947e-09, 2.8885e-09], device='cuda:0')" + }, + "24": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-9.6262e-09, 1.8758e-07, -3.5083e-07, ..., -2.5668e-07,\n -2.0587e-07, 2.0654e-07],\n [ 5.3500e-07, 3.6718e-07, 5.3651e-07, ..., -1.2186e-07,\n 5.5820e-08, 8.2350e-07],\n [-1.2711e-07, 4.8281e-08, -2.3066e-07, ..., 1.4943e-07,\n -6.8295e-08, -1.3274e-07],\n ...,\n [ 2.0590e-07, 1.8924e-07, -1.6145e-07, ..., -1.4247e-07,\n -1.3245e-07, 3.1698e-07],\n [-4.7339e-08, 3.6966e-07, -4.6252e-07, ..., -2.9455e-07,\n -1.7691e-07, 2.5807e-07],\n [ 6.6890e-08, -1.0580e-07, 1.4642e-07, ..., 2.6921e-07,\n -8.9393e-08, 1.6884e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.6697e-13, 2.2432e-12, 1.7244e-12, ..., 2.6479e-12, 1.5691e-12,\n 1.6906e-12],\n [2.4353e-12, 7.0818e-12, 5.2681e-12, ..., 6.8327e-12, 4.4678e-12,\n 5.2908e-12],\n [3.4017e-13, 1.0168e-12, 7.5490e-13, ..., 9.5357e-13, 7.4442e-13,\n 8.0052e-13],\n ...,\n [3.7302e-13, 1.0167e-12, 9.2551e-13, ..., 1.2125e-12, 7.9599e-13,\n 8.6847e-13],\n [1.2323e-12, 3.5714e-12, 2.7105e-12, ..., 3.8801e-12, 2.5986e-12,\n 2.7587e-12],\n [5.1565e-13, 1.1443e-12, 1.1765e-12, ..., 1.3985e-12, 9.4763e-13,\n 1.1291e-12]], device='cuda:0')" + }, + "25": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 9.7775e-07, 1.2331e-06, -2.6618e-06, -2.9965e-06, 2.1609e-06,\n 1.2453e-06, 8.9719e-06, 2.9043e-06, 3.4176e-06, 1.5784e-06,\n 7.3915e-06, -4.4688e-07, -3.1847e-06, -2.8159e-07, -2.2409e-06,\n -4.5551e-06, -1.8558e-07, 8.7752e-06, 3.1380e-06, -2.7740e-06,\n 4.5019e-06, 1.6531e-06, -2.5296e-06, -1.6148e-06, -5.1568e-06,\n 2.8711e-06, -2.0937e-06, -6.5395e-06, -9.8472e-06, 4.6756e-06,\n 1.4259e-06, 2.3206e-06, -1.8324e-06, -2.0090e-06, 1.4216e-06,\n -1.0717e-06, -3.6442e-07, 4.3178e-06, -8.9066e-07, -4.0340e-06,\n -2.9319e-07, -5.3741e-06, -8.2480e-06, -8.0515e-06, 7.8663e-07,\n -8.5152e-06, -2.1760e-06, -1.0066e-05, -1.3384e-06, -9.3405e-06,\n -3.0422e-06, -1.3009e-06, -9.1285e-06, 1.7076e-06, 4.2923e-06,\n -2.1490e-07, 6.0573e-07, -6.9612e-06, 2.1495e-06, -2.5897e-06,\n -2.5032e-06, 4.4614e-06, 4.2500e-06, 2.0962e-06, -1.4166e-06,\n 4.3976e-06, -1.9848e-06, 9.5742e-06, -5.5721e-06, -1.6570e-06,\n -1.3685e-06, -8.9756e-07, -3.6940e-06, 2.4267e-07, -3.5337e-06,\n -4.2943e-06, -1.3593e-06, 4.3632e-06, 8.3496e-06, 1.5941e-06,\n 1.5779e-06, 6.5960e-06, 1.3677e-06, 6.5805e-06, -4.5575e-07,\n -7.5691e-07, -5.1023e-06, 8.7741e-08, -5.6026e-06, -3.3094e-06,\n -2.6227e-06, -3.7967e-06, 2.7196e-06, -6.7689e-06, 1.6143e-06,\n 7.9669e-06, -5.2052e-06, 5.2452e-06, 8.3323e-06, 4.7847e-06,\n 6.1900e-07, -2.6360e-06, 3.3330e-06, 1.0599e-05, 2.5053e-06,\n 3.8997e-06, -2.9506e-06, -2.3680e-06, -5.1739e-06, 3.1576e-06,\n 1.0405e-05, -1.0558e-06, 5.8676e-07, -4.0757e-06, 6.6903e-06,\n -5.9786e-06, 3.9292e-06, 5.3963e-06, 2.5252e-06, -1.2252e-06,\n 2.3869e-06, 3.9594e-06, -5.1980e-06, 3.5128e-06, -5.7023e-06,\n 1.9061e-06, -1.0719e-07, -1.8322e-06, 4.4938e-07, -3.2475e-06,\n 4.9214e-06, 2.2926e-06, -9.1968e-07, 8.0402e-06, 6.1012e-07,\n -2.3984e-06, 6.7653e-06, 3.0034e-06, -9.0066e-07, -2.4401e-06,\n -1.4351e-07, 3.2434e-06, 1.0854e-05, -3.0006e-06, -1.6846e-06,\n -4.5373e-06, -5.3725e-06, 1.0673e-06, -6.9823e-06, -7.2300e-07,\n -2.7886e-06, -2.2145e-06, -2.6655e-06, -7.6653e-07, -1.0352e-06,\n 6.2980e-06, -1.7610e-06, 3.7712e-06, 2.8881e-06, -7.1959e-08,\n -1.7865e-06, -2.6462e-06, 2.8229e-06, -7.0045e-07, -4.9590e-06,\n -3.3981e-07, -9.1849e-09, 2.7599e-06, -3.4673e-06, 4.1023e-06,\n 3.2304e-06, 3.0165e-07, 4.2003e-06, 7.7934e-06, -6.7257e-06,\n 8.0984e-06, 2.4145e-06, 1.5071e-06, -4.7381e-06, -7.4675e-07,\n -1.7372e-06, 5.5852e-06, 4.5813e-06, -2.6944e-06, 1.3794e-06,\n 1.6312e-06, 3.2748e-06, -7.0811e-06, 4.1789e-07, 9.5499e-07,\n 3.6962e-06, 2.0271e-06, -7.5686e-07, 3.1308e-08, 2.2192e-06,\n -8.1704e-06, -2.5741e-06, -3.0290e-06, -7.9773e-07, 6.7302e-07,\n -1.8794e-06, -5.6282e-07, 4.9501e-06, -3.5387e-06, -5.5232e-06,\n 1.1502e-06, 5.3595e-07, 3.4513e-06, 6.6266e-06, 7.6140e-06,\n 2.8203e-09, 3.0414e-06, 8.9031e-07, -2.9217e-06, 5.7039e-06,\n -1.0546e-05, -1.2386e-06, -5.0343e-06, -4.4878e-06, 6.7191e-06,\n 1.4139e-06, 5.2587e-06, -9.9728e-06, 2.1319e-06, 7.7488e-07,\n -1.1207e-06, -5.6230e-06, -5.2663e-06, 2.9063e-06, 6.1732e-06,\n 1.0264e-05, 1.4862e-06, 2.2670e-06, 2.4498e-06, -5.2198e-06,\n 2.2108e-06, -5.5485e-06, 9.5829e-07, -6.1006e-06, 5.8824e-07,\n -2.5874e-06, 3.4391e-06, 4.0920e-06, 3.3283e-06, -2.0898e-06,\n -1.5488e-06, -4.1427e-07, -6.3336e-06, 4.5200e-06, 3.2831e-06,\n 3.9252e-06, -1.1286e-05, 8.1787e-07, -3.8082e-06, 8.3730e-07,\n 3.7391e-08, -5.0585e-07, -1.3910e-06, 1.7163e-06, 1.2441e-06,\n -2.1435e-06, -2.1305e-06, -1.0077e-06, -1.4590e-06, -1.4554e-06,\n 2.3113e-06, 7.3214e-06, 1.9059e-07, 3.9135e-06, -5.4010e-06,\n -2.9887e-06, -4.3453e-06, 2.4183e-06, -3.5781e-06, -6.1204e-06,\n -4.5293e-06, -2.9746e-07, 4.9461e-06, -2.4837e-06, -4.3542e-06,\n -5.2813e-06, -1.9508e-06, 2.3140e-06, -3.7709e-06, -3.2034e-06,\n 4.1544e-06, -2.5971e-06, -5.7884e-06, -6.6277e-07, -5.6760e-06,\n -2.9432e-06, -9.3027e-07, -3.7812e-06, 1.4097e-06, -1.9039e-06,\n -7.6546e-07, 1.4946e-06, -2.3434e-06, 1.4032e-07, 1.7972e-06,\n 7.9288e-06, -8.1322e-07, -5.0556e-06, -1.8703e-06, -7.8482e-07,\n -5.9981e-06, -3.0815e-06, 2.4476e-06, 6.4661e-06, -4.8578e-06,\n 1.3968e-06, 1.6730e-06, 7.1556e-06, -1.9721e-06, 6.0215e-06,\n 2.6972e-06, 3.7496e-06, 7.8897e-06, -4.3690e-06, -4.0345e-06,\n -4.4402e-06, -8.8098e-07, -4.2832e-06, -6.6785e-07, 1.3273e-06,\n -5.4614e-06, -4.5764e-06, 1.1950e-06, 1.1031e-06, 5.8206e-06,\n -5.3283e-06, -3.7114e-06, -3.4937e-06, -9.7406e-08, 3.4880e-06,\n 1.6213e-06, 3.9731e-06, -6.5380e-06, 6.5330e-07, 6.8276e-06,\n 8.2447e-06, -4.0133e-06, 4.1905e-06, 4.8900e-06, 7.8746e-07,\n 7.4669e-06, -4.1937e-06, 1.0174e-06, 4.4658e-07, 1.0096e-05,\n 1.9643e-06, -3.4214e-06, 7.2663e-06, 4.0703e-06, 5.0648e-06,\n 9.9603e-07, 4.4619e-08, 1.3144e-05, -1.5662e-08, -1.0469e-06,\n -1.4360e-06, 9.1974e-06, -2.8825e-06, 1.8728e-06, -1.2154e-06,\n 8.3849e-07, -5.0796e-06, 2.4652e-06, -1.9512e-07, -4.2833e-07,\n -7.4020e-07, 4.2882e-07, -3.9910e-06, 1.2853e-06, -5.5939e-06,\n -6.8576e-06, -6.6907e-07, 4.3186e-06, 2.1732e-06, 8.1724e-06,\n 3.1239e-06, -1.6134e-06, -7.0117e-06, -2.3290e-06, 8.6680e-06,\n 1.4757e-06, -5.6538e-07, -3.4657e-06, -2.9750e-06, 4.1946e-06,\n 4.3361e-06, -2.3542e-06, -5.4104e-06, 2.2950e-06, 5.1039e-07,\n 5.7793e-06, -2.0328e-06, 2.9882e-06, 1.6936e-06, -1.1772e-06,\n 1.1245e-06, 4.9188e-06, 6.6686e-07, 8.4668e-06, 2.4457e-07,\n 7.5002e-06, 1.3924e-06, -7.1932e-07, -2.8364e-06, 3.7902e-06,\n -1.6437e-06, -3.3779e-06, 6.3400e-06, 1.9084e-06, -7.2784e-06,\n 6.4480e-06, -1.4679e-06, -3.7181e-06, 3.7291e-06, -1.4079e-06,\n -1.3738e-06, -3.0858e-06, -5.5382e-06, 1.6919e-06, 3.6963e-06,\n -7.7519e-06, 9.3899e-06, -6.9929e-07, -6.2484e-06, 9.0931e-06,\n -5.9403e-07, 4.2634e-06, 1.2046e-06, 1.6240e-06, -2.7480e-06,\n 6.7742e-06, -7.6564e-06, 7.3876e-06, 3.3661e-06, 5.4045e-07,\n 7.7872e-06, -4.1280e-06, -1.4119e-07, 6.0463e-06, -8.1102e-06,\n 4.1522e-06, -3.9131e-06, -1.1617e-07, -8.3954e-06, -3.5793e-07,\n -8.8381e-06, -1.8833e-06, -4.6257e-06, -2.3180e-06, 5.8120e-07,\n 1.4151e-06, -7.3924e-06, -1.4048e-06, 3.6920e-06, -2.1556e-06,\n 3.0438e-06, -1.6481e-06, -2.0798e-06, -5.8491e-06, 4.8774e-06,\n -5.0860e-06, -7.7472e-07, -3.3447e-06, -9.3370e-07, 1.1258e-06,\n -4.2088e-06, 4.2354e-06, 3.1646e-06, 1.0536e-07, -3.6525e-06,\n -6.0384e-06, 5.5685e-06, -7.0602e-06, 5.0579e-06, -4.2228e-06,\n -1.5425e-06, 1.2510e-06, -5.3561e-06, -1.5740e-06, 1.8767e-06,\n 2.9984e-07, -2.3364e-06, -8.0968e-07, -1.1553e-06, 1.9849e-06,\n 2.8738e-06, 7.5615e-07, -5.3931e-06, 5.9468e-07, 4.8785e-06,\n 2.3801e-06, -2.6563e-08, -1.0305e-06, 4.9469e-07, 3.9639e-06,\n -1.2995e-06, 9.4916e-10, 1.9416e-06, -3.8152e-07, -4.3828e-06,\n 9.2160e-07, 1.5441e-06, -9.4491e-07, 8.0539e-06, 1.2466e-06,\n 9.7760e-07, 1.7729e-06], device='cuda:0')", + "exp_avg_sq": "tensor([1.4248e-09, 4.2922e-09, 6.9166e-10, 1.0912e-09, 2.5021e-09, 1.3167e-09,\n 2.4122e-09, 6.5325e-10, 6.3520e-10, 2.8288e-09, 2.1897e-09, 1.0909e-09,\n 3.6506e-09, 6.0467e-10, 9.6349e-10, 8.5941e-10, 7.3123e-10, 2.2830e-09,\n 4.0625e-10, 5.6279e-10, 1.0731e-09, 2.7044e-09, 8.9494e-10, 2.2047e-09,\n 5.2394e-10, 7.7208e-10, 1.3940e-09, 2.2619e-09, 8.2587e-10, 6.7594e-10,\n 9.3545e-10, 7.2027e-10, 1.1970e-09, 1.8839e-09, 5.0255e-10, 9.5194e-10,\n 1.4823e-09, 1.1454e-09, 1.2197e-09, 9.0095e-10, 4.1626e-09, 8.2339e-10,\n 3.3397e-09, 1.9613e-09, 1.0432e-09, 7.2431e-10, 7.6578e-10, 1.8491e-09,\n 9.1041e-10, 1.0971e-09, 7.1138e-10, 8.3313e-10, 2.2861e-09, 1.1670e-09,\n 1.3484e-09, 1.6325e-09, 1.3928e-09, 8.8081e-10, 3.4184e-09, 4.9778e-10,\n 5.3326e-10, 8.3140e-10, 1.4213e-09, 1.1011e-09, 8.2330e-10, 4.6703e-10,\n 7.7846e-10, 2.8157e-09, 2.3072e-09, 2.2765e-09, 7.0404e-10, 1.5265e-09,\n 6.8217e-10, 9.7734e-10, 1.2788e-09, 8.4005e-10, 7.6306e-10, 3.1517e-09,\n 1.4730e-09, 3.7130e-09, 1.5103e-09, 1.9553e-09, 1.5823e-09, 1.3811e-09,\n 4.7105e-10, 1.3750e-09, 1.2289e-09, 2.3837e-09, 1.1206e-09, 9.4816e-10,\n 7.3659e-10, 3.1721e-09, 6.3743e-10, 1.9065e-09, 8.4401e-10, 1.1136e-09,\n 9.2619e-10, 1.9521e-09, 1.3387e-09, 2.2096e-09, 2.7897e-09, 6.2321e-10,\n 5.8649e-09, 1.2016e-09, 6.9913e-10, 1.1491e-09, 7.0216e-10, 1.6994e-09,\n 2.3637e-09, 4.5322e-10, 1.5928e-09, 3.3510e-09, 1.1285e-09, 1.6239e-09,\n 2.8158e-09, 1.0977e-09, 1.4954e-09, 5.6383e-10, 8.0255e-09, 1.4087e-09,\n 1.2618e-09, 5.2481e-09, 1.8983e-09, 1.4011e-09, 6.1654e-10, 1.1092e-09,\n 1.2984e-09, 1.5966e-09, 1.0011e-09, 7.4198e-10, 4.8642e-09, 1.3553e-09,\n 4.1299e-09, 2.8260e-09, 4.9980e-10, 5.5564e-10, 2.3410e-09, 1.1286e-09,\n 1.2150e-09, 5.0320e-09, 1.1790e-09, 1.8727e-09, 2.2940e-09, 1.0119e-09,\n 9.4880e-10, 1.7581e-09, 2.0255e-09, 5.8008e-10, 4.6538e-10, 1.1025e-09,\n 1.9567e-09, 7.7201e-10, 1.6412e-09, 1.0363e-09, 8.1075e-10, 7.8882e-10,\n 6.2744e-10, 5.6776e-10, 1.0139e-09, 1.2148e-09, 5.5184e-09, 9.0897e-10,\n 7.0805e-10, 1.4454e-09, 8.0234e-10, 2.3002e-09, 4.9090e-10, 2.1858e-09,\n 1.4483e-09, 1.3779e-09, 2.4223e-09, 8.2772e-10, 1.5752e-09, 9.9792e-10,\n 1.3042e-09, 2.0225e-09, 6.8881e-10, 2.9131e-09, 8.6683e-10, 1.1167e-09,\n 8.9653e-10, 3.9650e-10, 6.9458e-10, 1.6553e-09, 1.8223e-09, 3.7209e-10,\n 9.9002e-10, 5.8853e-10, 6.2868e-10, 3.6206e-09, 2.8957e-09, 2.0683e-09,\n 1.4187e-09, 5.5789e-10, 9.6841e-10, 1.3326e-09, 9.5848e-10, 1.4777e-09,\n 1.6123e-09, 4.0041e-10, 6.9495e-10, 1.7083e-09, 5.6745e-10, 3.7388e-09,\n 4.2626e-10, 6.0999e-10, 7.8345e-10, 9.9541e-10, 2.7227e-09, 1.5055e-09,\n 8.9367e-10, 7.9828e-10, 1.0143e-09, 1.4598e-09, 6.6242e-10, 2.7793e-09,\n 4.0251e-09, 5.6463e-09, 2.7256e-09, 2.1761e-09, 2.5694e-09, 1.0891e-09,\n 4.0665e-09, 1.1390e-09, 9.2619e-10, 5.3115e-10, 1.0694e-09, 1.8308e-09,\n 1.0773e-09, 1.0949e-09, 1.1296e-09, 5.5841e-10, 8.9644e-10, 1.4069e-09,\n 2.1240e-09, 3.5515e-09, 4.9999e-10, 6.8350e-10, 1.2619e-09, 1.3076e-09,\n 6.1683e-10, 7.6207e-10, 1.5965e-09, 1.2090e-09, 6.1657e-10, 6.1539e-10,\n 7.6574e-10, 3.6896e-09, 1.8449e-09, 2.1865e-09, 1.0220e-09, 1.3881e-09,\n 2.1631e-09, 8.5326e-10, 1.5226e-09, 4.4328e-10, 1.5877e-09, 1.2335e-09,\n 8.6801e-10, 4.4838e-09, 6.5310e-10, 4.6984e-09, 1.9019e-09, 4.0253e-09,\n 3.4977e-09, 6.6496e-10, 2.0993e-09, 1.7153e-09, 1.5514e-09, 1.0478e-09,\n 4.6273e-10, 1.6079e-09, 2.5019e-09, 1.1643e-09, 1.4471e-09, 2.3116e-09,\n 2.5664e-09, 9.0266e-10, 8.9397e-10, 1.1083e-09, 1.1526e-09, 9.2315e-10,\n 2.3243e-09, 6.0401e-10, 1.5206e-09, 1.0260e-09, 9.1523e-10, 9.3724e-10,\n 2.4417e-09, 6.5490e-10, 1.9362e-09, 3.1586e-09, 1.2692e-09, 2.7327e-09,\n 8.5963e-10, 1.5386e-09, 4.5930e-10, 9.4108e-10, 9.0898e-10, 6.0246e-09,\n 2.0469e-09, 1.5686e-09, 1.2522e-09, 1.8678e-09, 1.0701e-09, 1.6912e-09,\n 4.3278e-10, 3.6682e-09, 1.8045e-09, 1.4870e-09, 3.4262e-09, 2.1751e-09,\n 5.1540e-10, 1.1933e-09, 6.6762e-10, 9.7048e-10, 1.2010e-09, 1.4026e-09,\n 1.3260e-09, 1.7200e-09, 1.1359e-09, 6.0632e-10, 1.7247e-09, 8.3286e-10,\n 1.4468e-09, 5.8912e-10, 1.5473e-09, 8.3092e-09, 1.2166e-09, 1.5764e-09,\n 5.8331e-10, 8.0782e-10, 1.5323e-09, 4.0559e-09, 8.2839e-10, 1.4238e-09,\n 1.1160e-09, 2.2039e-09, 1.2552e-09, 8.1318e-10, 4.5539e-09, 1.0883e-09,\n 9.2038e-10, 1.1771e-09, 5.8387e-10, 8.2578e-10, 6.1028e-10, 1.0549e-09,\n 8.8352e-10, 1.1027e-09, 6.4987e-10, 7.0783e-10, 1.4635e-09, 2.1838e-09,\n 7.7401e-10, 9.9487e-10, 5.9430e-10, 1.4662e-09, 1.8899e-09, 1.1972e-09,\n 6.6974e-10, 8.8518e-10, 7.2829e-10, 1.1922e-09, 8.3967e-10, 1.3551e-09,\n 1.0347e-09, 4.4669e-10, 2.6927e-09, 1.0939e-09, 2.2088e-09, 3.4217e-10,\n 5.2782e-10, 4.8956e-09, 8.1424e-10, 7.3901e-09, 2.4959e-09, 1.1315e-09,\n 1.1375e-09, 7.8946e-10, 3.0352e-09, 1.8300e-09, 5.8990e-10, 9.5864e-10,\n 1.0651e-09, 2.2546e-09, 9.7251e-10, 1.8825e-09, 2.7680e-09, 7.6063e-10,\n 7.6382e-10, 7.2409e-10, 1.9153e-09, 1.7580e-09, 1.3118e-09, 1.0320e-09,\n 6.7898e-10, 1.2074e-09, 7.7683e-10, 6.9866e-10, 8.9819e-10, 8.3817e-10,\n 3.0134e-10, 2.1097e-09, 9.2045e-10, 3.8006e-09, 1.0949e-09, 1.3072e-09,\n 9.8834e-10, 1.3918e-09, 4.4598e-09, 1.0771e-08, 1.8647e-09, 1.2887e-09,\n 2.3227e-09, 1.9227e-09, 3.8223e-10, 1.0134e-09, 8.5167e-10, 1.8553e-09,\n 1.3200e-09, 4.0646e-10, 5.0693e-10, 2.1239e-09, 1.0998e-09, 6.3878e-09,\n 1.2726e-09, 1.9211e-09, 1.1656e-09, 4.1127e-09, 7.6735e-10, 6.0137e-10,\n 1.6688e-09, 5.2082e-10, 5.6900e-10, 6.9124e-10, 1.6591e-09, 6.6176e-10,\n 1.3804e-09, 5.0442e-09, 5.1235e-09, 8.7812e-10, 9.1015e-10, 1.3778e-09,\n 9.8949e-10, 1.7712e-09, 1.9758e-09, 1.6499e-09, 5.8733e-10, 3.8586e-09,\n 1.5160e-09, 2.9907e-09, 1.1064e-09, 5.5166e-10, 1.1936e-09, 2.4159e-09,\n 1.8594e-09, 5.9148e-10, 1.6115e-09, 3.7146e-09, 5.0826e-09, 9.3320e-10,\n 1.1108e-09, 1.8609e-09, 2.7328e-09, 2.2170e-09, 5.1032e-10, 6.2528e-10,\n 1.6564e-09, 1.2575e-09, 3.4141e-10, 2.4357e-09, 1.1251e-09, 1.1562e-09,\n 5.8613e-10, 2.1544e-09, 8.6669e-10, 4.9422e-10, 5.8242e-10, 8.1974e-10,\n 1.0082e-09, 6.1440e-10, 5.5426e-09, 1.6361e-09, 8.3136e-10, 4.4962e-09,\n 1.1281e-09, 2.9027e-09, 8.7105e-10, 2.8371e-09, 8.2346e-10, 2.0016e-09,\n 1.5367e-09, 7.9553e-10, 1.3664e-09, 6.5353e-10, 5.8270e-10, 7.3163e-10,\n 1.9909e-09, 1.4109e-09, 8.6430e-10, 9.1510e-10, 1.2972e-09, 5.9281e-09,\n 1.5077e-09, 5.6252e-10, 9.9682e-10, 2.1813e-09, 1.2851e-09, 6.8454e-10,\n 2.3263e-09, 9.3358e-10], device='cuda:0')" + }, + "26": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-3.5437e-07, 3.6547e-07, 2.4394e-07, ..., 2.5023e-07,\n 9.6407e-07, 3.3310e-07],\n [-5.3378e-07, 1.0291e-06, 4.4875e-07, ..., -9.2255e-07,\n 4.5860e-08, -1.7421e-07],\n [ 1.8160e-07, -7.2916e-07, -5.5797e-07, ..., 2.5960e-08,\n -7.4174e-08, -4.1563e-07],\n ...,\n [ 1.7075e-07, 8.1681e-07, 1.1952e-07, ..., 2.8317e-07,\n 7.9683e-07, -4.3700e-07],\n [ 3.4226e-07, 5.7860e-07, -1.7330e-07, ..., 2.9356e-07,\n 1.3430e-06, -2.9741e-07],\n [-1.3678e-07, -2.9939e-07, -3.8236e-07, ..., 9.1096e-07,\n -5.4526e-07, 1.9082e-08]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.1771e-12, 3.7391e-12, 3.8309e-12, ..., 4.1802e-12, 3.6971e-12,\n 4.3944e-12],\n [6.2560e-12, 1.0642e-11, 9.6585e-12, ..., 1.1235e-11, 1.0126e-11,\n 1.2310e-11],\n [1.2130e-12, 2.0699e-12, 2.0429e-12, ..., 2.0806e-12, 2.0256e-12,\n 2.3911e-12],\n ...,\n [1.2835e-12, 2.3141e-12, 2.3638e-12, ..., 2.3484e-12, 2.2531e-12,\n 2.7911e-12],\n [2.4976e-12, 4.2637e-12, 4.2526e-12, ..., 4.6482e-12, 4.5659e-12,\n 5.2016e-12],\n [1.9314e-12, 3.3269e-12, 3.7008e-12, ..., 3.5998e-12, 3.7234e-12,\n 4.3404e-12]], device='cuda:0')" + }, + "27": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 2.2864e-06, -3.8709e-07, -1.3963e-05, 1.2801e-05, -1.7110e-06,\n 8.7704e-06, -4.8982e-06, -1.1293e-05, -5.8874e-06, -1.9290e-05,\n -2.2410e-05, -5.0416e-06, 1.3816e-05, -6.5128e-06, 1.8069e-05,\n 7.1169e-06, -1.9214e-06, -1.0416e-05, -4.8020e-06, -7.1071e-06,\n -1.0240e-05, -7.3037e-06, 4.0449e-06, -1.8363e-05, 6.3941e-07,\n 1.1097e-06, 1.8166e-05, 7.7087e-06, 1.0077e-05, 5.9076e-07,\n -1.0578e-05, -1.0600e-05, -1.6523e-07, 1.3084e-06, -2.1213e-06,\n 2.3684e-05, -4.4243e-06, -1.2700e-05, 3.0012e-06, 2.4079e-06,\n 1.0575e-05, 7.3908e-06, 2.6329e-05, 3.8497e-06, 9.8297e-06,\n 2.9084e-06, 4.1852e-06, 6.2243e-06, -4.3656e-06, 1.1102e-05,\n -1.0291e-05, -8.9543e-06, 1.6868e-05, 1.0739e-05, -1.6955e-05,\n 9.5117e-06, 1.4849e-05, 1.1665e-05, -8.0491e-06, 1.1767e-05,\n 7.2546e-06, 6.1622e-06, -9.0783e-06, -1.1377e-05, -9.7185e-06,\n -1.2778e-05, 7.0666e-06, -1.5522e-05, 4.1657e-06, 3.0193e-06,\n -2.6953e-06, -3.2324e-07, -5.1328e-07, -8.5713e-07, 1.3472e-05,\n 6.2598e-06, 3.5747e-06, -1.5959e-05, -2.4720e-05, -1.1499e-05,\n 4.7352e-06, -2.9322e-06, 7.4937e-06, -7.3510e-06, 4.1904e-07,\n -8.8340e-06, 2.9349e-06, -1.9494e-05, -2.1736e-06, -1.7204e-06,\n -8.6550e-07, -5.1654e-07, -1.0594e-05, 8.8907e-06, 1.6472e-06,\n -8.8322e-06, 7.5889e-06, -1.7995e-05, -1.7469e-06, -1.5728e-06,\n -1.0238e-05, -2.6056e-06, -1.4703e-05, 5.0207e-07, -1.9610e-06,\n -2.3221e-06, 1.3830e-05, 5.0966e-06, 6.2579e-06, -9.2207e-06,\n -1.3568e-05, 1.9411e-06, -6.2910e-06, 3.8482e-06, -2.7280e-05,\n 4.7860e-06, 3.9236e-06, -2.3874e-06, -1.3798e-05, 4.3197e-06,\n 1.1449e-05, -1.7932e-05, 3.9465e-06, -1.2462e-05, 6.8063e-07,\n -1.6138e-05, -5.8829e-06, 1.6930e-05, -5.4683e-06, 3.5010e-06,\n -1.3335e-05, -1.7749e-06, -2.1054e-05, -2.5167e-06, -2.8783e-06,\n 9.3205e-06, -1.9235e-05, -6.1489e-06, 1.4258e-05, 2.2523e-05,\n 7.0676e-06, -1.4549e-05, -1.9195e-05, 6.2541e-06, 9.1606e-06,\n 1.1969e-05, 1.4052e-06, -1.1994e-06, 8.1897e-06, 4.1001e-06,\n -3.0032e-06, 1.1279e-05, -1.2848e-06, -4.2564e-06, 1.3071e-05,\n -3.5530e-06, 4.5453e-07, 1.5789e-06, -1.2933e-05, 5.3170e-06,\n 1.6406e-06, 3.4803e-06, -6.3621e-07, 6.0181e-06, -4.0476e-06,\n 2.8779e-06, 6.6582e-06, -1.6270e-05, 1.8077e-05, -2.5757e-06,\n 2.3627e-06, -1.2690e-05, 1.5493e-06, -1.3377e-05, 1.4132e-05,\n -5.6015e-06, -1.3544e-05, 1.2545e-06, 1.7123e-05, 1.9871e-06,\n -8.5779e-06, -1.3064e-05, -1.3151e-06, -7.0038e-06, -6.1033e-07,\n -6.7116e-06, -4.4391e-06, -6.6886e-06, 4.9514e-06, 1.0150e-05,\n -3.0148e-06, -3.0660e-06, -3.1889e-07, -1.4115e-05, -2.3580e-06,\n 5.4263e-06, 1.7069e-05, -1.3418e-05, 5.2886e-06, -5.6324e-06,\n -1.0790e-07, -1.1303e-05, 6.3476e-06, 2.1658e-05, 3.6183e-07,\n 1.3782e-05, 1.7831e-06, -2.1833e-06, -1.3676e-05, -2.1211e-06,\n -1.0984e-05, -4.0492e-06, 1.6726e-05, 1.4120e-05, 4.4185e-06,\n 1.5830e-05, 8.1902e-06, 2.7023e-05, 8.2407e-06, -8.7707e-06,\n 2.3270e-05, -9.4594e-06, 1.1101e-05, -2.8417e-06, 1.2232e-05,\n -3.6456e-06, 6.9677e-06, 4.1779e-06, 4.3266e-06, -6.4975e-06,\n -6.5947e-06, -5.7663e-06, 7.2607e-06, 3.2197e-06, 1.1392e-05,\n -5.9137e-06, 5.5325e-06, 1.3927e-06, 1.2458e-05, 9.6288e-08,\n -6.4503e-06, -7.1177e-06, -1.6403e-05, 1.1291e-05, -9.8229e-06,\n -9.9804e-06, -5.3127e-06, 7.1957e-06, -5.9560e-06, -2.2983e-06,\n -7.3847e-06, 1.7766e-05, 1.0882e-06, 1.2381e-06, 1.3798e-05,\n 7.5991e-06, 1.8582e-05, 1.5293e-05, -8.8192e-06, -1.0979e-06,\n -1.8399e-06, 1.4037e-05, -4.9580e-06, -1.6859e-06, -2.9705e-05,\n -3.3403e-06, -1.3952e-05, -1.1196e-05, -9.3027e-06, 5.3176e-06,\n 5.3375e-07, 7.2469e-06, -2.0131e-05, -5.0677e-06, -1.0422e-06,\n 6.2862e-06, 1.0203e-05, 3.3239e-06, 9.8049e-06, 7.5108e-06,\n 1.0039e-05, 1.6860e-05, 6.1260e-06, -5.6838e-06, 1.4532e-05,\n 1.0684e-05, 1.0340e-05, 7.2641e-06, 1.8445e-05, 3.2533e-06,\n 1.3798e-05, 4.3194e-06, -1.7316e-05, -2.8767e-05, -4.7352e-06,\n -4.2020e-07, -1.0177e-05, 6.2443e-06, -1.0343e-05, -2.9480e-05,\n -2.3017e-05, 1.4778e-06, -2.8706e-07, 2.8232e-05, 8.0441e-06,\n -2.0913e-06, 2.6636e-06, -1.5739e-05, -1.6622e-05, 1.3403e-05,\n -7.3892e-06, 2.1153e-05, 4.5700e-06, -7.6041e-06, -1.6284e-05,\n -3.9060e-06, 5.7495e-06, -1.8312e-05, 1.4170e-05, 2.1402e-05,\n 8.5872e-06, 1.2603e-05, 1.3053e-05, 4.8984e-06, 7.3283e-06,\n -3.8928e-06, -4.6539e-07, -4.2864e-05, -1.6091e-06, -5.1566e-06,\n 7.8402e-06, 9.2121e-06, 7.7040e-06, 2.1627e-05, 3.7735e-06,\n -4.2616e-06, -1.7167e-05, 1.5827e-05, -1.5786e-07, -1.3903e-05,\n -1.2936e-05, 1.8678e-05, -5.5139e-06, -4.6640e-06, -2.8233e-06,\n -7.1109e-06, -6.8307e-06, -1.6614e-06, -1.4218e-05, -5.6452e-06,\n 4.3709e-07, 6.2961e-07, -7.3212e-06, -1.4115e-05, 8.0707e-07,\n 9.6546e-06, 1.1050e-05, -1.7833e-05, -1.3401e-06, -2.7065e-06,\n -9.3608e-07, 1.8844e-06, 1.0991e-06, -6.7446e-06, 3.2770e-06,\n -2.4079e-08, 1.2618e-06, -6.0802e-06, -6.4425e-06, -1.8177e-06,\n 7.5557e-06, 8.8461e-07, 2.5161e-06, 1.7248e-05, -1.3245e-06,\n -1.6599e-05, -8.0622e-06, -5.4908e-06, -5.4387e-06, 2.7862e-06,\n -2.6950e-05, 1.2060e-05, 1.0513e-05, 1.2334e-05, -9.9234e-06,\n 3.4661e-06, -7.5266e-06, 4.0932e-06, -7.0612e-06, 3.3494e-06,\n -1.1653e-05, -6.1165e-06, 1.0429e-05, 3.5526e-06, -1.0396e-06,\n -3.4876e-06, -7.9323e-06, 2.6464e-06, 2.9029e-07, -7.1322e-06,\n -3.7913e-06, -1.5014e-05, 7.4101e-06, -1.8500e-05, -1.4320e-05,\n 2.6211e-08, 1.9728e-06, 2.0996e-06, -8.7198e-06, -2.0042e-05,\n -6.4984e-06, -2.4434e-05, -1.0541e-05, -2.2266e-06, 3.3826e-06,\n -1.5093e-05, -5.3623e-06, 1.0869e-05, -5.8989e-06, 5.5749e-06,\n -1.8247e-06, 4.0521e-06, 1.0900e-05, 1.9186e-05, -1.8068e-05,\n 2.8620e-05, 2.1764e-06, 1.3636e-06, 6.6324e-06, -6.5618e-06,\n -9.3010e-06, -3.3998e-06, 6.0785e-06, 5.7226e-06, 6.8640e-06,\n 5.3766e-06, 4.2224e-07, 2.3391e-06, -1.7428e-05, 7.2604e-06,\n -2.8181e-05, 1.3965e-05, 4.7934e-06, 2.5349e-06, 1.5910e-06,\n -1.1968e-05, 9.2956e-06, 1.8095e-05, 1.6729e-05, 8.9386e-06,\n -6.8491e-06, 1.6328e-05, -3.2862e-06, 8.2395e-06, 2.6592e-06,\n 4.4267e-06, 1.3146e-05, 4.4106e-06, 6.0963e-07, -6.0490e-06,\n -4.8363e-05, -1.9927e-06, 1.9418e-06, 6.0050e-06, -4.8750e-06,\n 6.3246e-06, 7.8401e-06, 2.6926e-06, 9.9961e-06, 2.4954e-06,\n 8.6258e-06, 8.3913e-06, -1.9400e-05, -2.6508e-06, 8.1279e-06,\n 5.8713e-06, -1.2839e-06, -1.3860e-06, 8.3162e-06, 1.5760e-05,\n -3.6461e-06, -1.3420e-05, 1.3534e-05, -1.3050e-05, -3.9495e-06,\n -2.1965e-05, 7.4262e-06, -1.9726e-05, -1.2001e-06, 2.6685e-05,\n -4.1095e-06, -1.0868e-05, -5.4619e-06, -3.7156e-06, -1.1957e-05,\n -5.1651e-06, -2.6077e-06, 7.6430e-06, -1.6927e-05, -9.1036e-06,\n 1.1292e-05, 6.7462e-06, -3.9359e-06, -2.3263e-05, 9.7792e-06,\n -5.5512e-06, 2.8352e-07, 4.9336e-06, 4.5506e-07, 5.6196e-06,\n 1.2484e-05, -1.6543e-05], device='cuda:0')", + "exp_avg_sq": "tensor([3.8872e-09, 1.1257e-08, 2.2458e-09, 2.7172e-09, 6.0620e-09, 3.8811e-09,\n 6.4928e-09, 2.0105e-09, 1.9500e-09, 7.4429e-09, 9.6222e-09, 3.9501e-09,\n 1.2330e-08, 1.8812e-09, 2.9419e-09, 3.0287e-09, 2.0941e-09, 6.8186e-09,\n 1.9938e-09, 2.0979e-09, 2.5357e-09, 7.3270e-09, 2.8800e-09, 5.0509e-09,\n 1.5279e-09, 3.0618e-09, 5.6392e-09, 7.4087e-09, 2.5515e-09, 2.1483e-09,\n 2.6873e-09, 2.2249e-09, 3.0438e-09, 4.4977e-09, 1.7067e-09, 3.5588e-09,\n 3.3272e-09, 2.6472e-09, 2.7994e-09, 3.3164e-09, 1.3448e-08, 3.1899e-09,\n 1.2752e-08, 5.2085e-09, 4.0158e-09, 2.6740e-09, 2.4726e-09, 5.0646e-09,\n 2.5532e-09, 3.7894e-09, 2.6263e-09, 2.4011e-09, 8.2538e-09, 3.0507e-09,\n 5.0435e-09, 3.5727e-09, 4.7249e-09, 2.5209e-09, 7.1343e-09, 2.0435e-09,\n 1.6236e-09, 2.6475e-09, 3.5473e-09, 3.0649e-09, 2.4881e-09, 1.7309e-09,\n 2.4069e-09, 1.0294e-08, 5.7807e-09, 6.3062e-09, 1.9340e-09, 3.4256e-09,\n 2.1204e-09, 3.1550e-09, 3.7367e-09, 2.2531e-09, 3.3814e-09, 8.0329e-09,\n 4.1272e-09, 8.2329e-09, 6.2000e-09, 5.0875e-09, 4.5716e-09, 5.7948e-09,\n 1.6545e-09, 2.9754e-09, 2.4711e-09, 1.0312e-08, 4.8267e-09, 2.9220e-09,\n 1.6593e-09, 6.4857e-09, 2.1582e-09, 5.9289e-09, 2.8640e-09, 3.9675e-09,\n 3.1369e-09, 6.2158e-09, 3.7260e-09, 4.7010e-09, 7.0423e-09, 1.8250e-09,\n 1.2118e-08, 3.4404e-09, 2.3212e-09, 3.1821e-09, 2.5203e-09, 5.6023e-09,\n 7.2129e-09, 1.9561e-09, 4.3355e-09, 8.8830e-09, 2.6045e-09, 4.7055e-09,\n 8.8786e-09, 3.0043e-09, 4.6961e-09, 2.2198e-09, 2.5811e-08, 4.4091e-09,\n 3.6596e-09, 1.1080e-08, 3.9126e-09, 4.9029e-09, 2.6170e-09, 3.8700e-09,\n 3.2061e-09, 4.2972e-09, 2.9034e-09, 2.4260e-09, 1.0583e-08, 4.0237e-09,\n 1.1200e-08, 7.9591e-09, 1.9721e-09, 1.9817e-09, 6.8280e-09, 4.0821e-09,\n 3.3372e-09, 1.4454e-08, 2.8978e-09, 5.3293e-09, 7.1997e-09, 3.2710e-09,\n 3.1478e-09, 5.1988e-09, 5.3927e-09, 2.1515e-09, 1.5788e-09, 4.3696e-09,\n 5.3987e-09, 2.3086e-09, 5.0742e-09, 3.8572e-09, 3.9478e-09, 2.8919e-09,\n 1.7398e-09, 2.1138e-09, 3.0965e-09, 4.7323e-09, 1.5070e-08, 2.4411e-09,\n 3.4195e-09, 3.6478e-09, 2.0883e-09, 4.7835e-09, 1.5669e-09, 4.9240e-09,\n 4.7007e-09, 3.2936e-09, 8.3412e-09, 2.5257e-09, 4.4154e-09, 3.7743e-09,\n 4.0081e-09, 6.9351e-09, 2.3149e-09, 8.0286e-09, 2.9990e-09, 3.4281e-09,\n 3.2114e-09, 1.9056e-09, 2.4359e-09, 4.4289e-09, 5.1156e-09, 1.3858e-09,\n 3.3693e-09, 2.2260e-09, 1.5600e-09, 1.2438e-08, 8.1431e-09, 7.5742e-09,\n 5.1180e-09, 2.2773e-09, 3.2399e-09, 3.6761e-09, 2.8662e-09, 5.6568e-09,\n 5.5540e-09, 1.3705e-09, 2.3031e-09, 3.6984e-09, 2.0599e-09, 8.9886e-09,\n 1.3090e-09, 2.2796e-09, 2.2920e-09, 2.9662e-09, 7.4610e-09, 4.1650e-09,\n 2.8453e-09, 3.4197e-09, 2.9920e-09, 4.2326e-09, 2.5675e-09, 7.6890e-09,\n 1.0329e-08, 1.6978e-08, 7.3819e-09, 5.6246e-09, 9.7961e-09, 2.9626e-09,\n 1.3116e-08, 3.8435e-09, 2.3815e-09, 2.0298e-09, 3.3285e-09, 4.1784e-09,\n 4.5933e-09, 3.7200e-09, 3.4118e-09, 1.5286e-09, 3.6479e-09, 4.0811e-09,\n 7.4731e-09, 1.1694e-08, 1.9141e-09, 2.4865e-09, 3.4619e-09, 4.2055e-09,\n 2.4258e-09, 2.5751e-09, 5.3962e-09, 4.2201e-09, 2.0155e-09, 2.2239e-09,\n 2.7845e-09, 8.5874e-09, 6.7313e-09, 6.9701e-09, 3.2338e-09, 4.3375e-09,\n 6.7064e-09, 3.3249e-09, 4.2315e-09, 1.7009e-09, 6.6860e-09, 4.2550e-09,\n 2.4123e-09, 1.3896e-08, 2.0152e-09, 9.9095e-09, 5.3788e-09, 1.2810e-08,\n 1.1676e-08, 2.3420e-09, 5.7998e-09, 5.6880e-09, 4.4391e-09, 2.6373e-09,\n 1.8775e-09, 4.6837e-09, 9.3273e-09, 2.7496e-09, 3.7115e-09, 5.4574e-09,\n 5.3317e-09, 3.3009e-09, 2.3719e-09, 2.9865e-09, 3.4969e-09, 3.4273e-09,\n 5.2696e-09, 1.9577e-09, 5.0920e-09, 3.5529e-09, 3.0447e-09, 3.7598e-09,\n 5.8849e-09, 2.0866e-09, 6.0162e-09, 7.8236e-09, 4.6487e-09, 1.1216e-08,\n 2.9021e-09, 4.3883e-09, 1.7912e-09, 2.8789e-09, 2.3195e-09, 1.9225e-08,\n 6.0035e-09, 4.1653e-09, 3.8145e-09, 7.9010e-09, 3.0575e-09, 5.9080e-09,\n 1.3436e-09, 1.1030e-08, 5.5409e-09, 3.6641e-09, 6.9936e-09, 8.4661e-09,\n 1.8307e-09, 3.7972e-09, 2.5990e-09, 2.4195e-09, 3.9187e-09, 5.9359e-09,\n 4.1124e-09, 5.1572e-09, 3.3699e-09, 2.5309e-09, 3.7730e-09, 2.7868e-09,\n 3.4333e-09, 1.5850e-09, 4.6465e-09, 3.3139e-08, 3.2451e-09, 4.1958e-09,\n 1.5706e-09, 3.2120e-09, 4.4818e-09, 9.9502e-09, 2.8131e-09, 3.2688e-09,\n 3.2755e-09, 6.1568e-09, 3.3689e-09, 2.8662e-09, 1.0709e-08, 4.6554e-09,\n 2.8861e-09, 4.0237e-09, 2.1607e-09, 3.3377e-09, 2.0616e-09, 3.0883e-09,\n 2.6584e-09, 3.6402e-09, 2.2410e-09, 2.6305e-09, 4.6733e-09, 8.4014e-09,\n 2.8001e-09, 3.0402e-09, 2.0405e-09, 4.9427e-09, 4.9742e-09, 3.9557e-09,\n 2.0118e-09, 2.8068e-09, 2.5687e-09, 3.4697e-09, 2.5391e-09, 3.3397e-09,\n 3.1657e-09, 1.5925e-09, 5.6107e-09, 3.2375e-09, 6.1624e-09, 1.4529e-09,\n 2.5327e-09, 1.1829e-08, 2.2706e-09, 2.9909e-08, 5.3219e-09, 3.3414e-09,\n 3.8429e-09, 2.7209e-09, 1.2810e-08, 4.3580e-09, 1.9210e-09, 2.5785e-09,\n 3.8362e-09, 6.6397e-09, 3.0608e-09, 5.6910e-09, 7.9318e-09, 2.3632e-09,\n 2.6833e-09, 2.3921e-09, 4.8984e-09, 3.4667e-09, 3.3061e-09, 2.9473e-09,\n 2.5362e-09, 3.5911e-09, 2.6473e-09, 3.4355e-09, 2.7641e-09, 2.8384e-09,\n 1.2422e-09, 8.2760e-09, 3.3356e-09, 1.6973e-08, 2.5912e-09, 3.9173e-09,\n 3.3657e-09, 4.7994e-09, 1.0874e-08, 2.8935e-08, 5.1678e-09, 3.4108e-09,\n 6.3785e-09, 5.3315e-09, 1.2516e-09, 2.9594e-09, 2.6489e-09, 4.1575e-09,\n 4.5939e-09, 1.3968e-09, 1.6949e-09, 9.9397e-09, 3.7287e-09, 1.9442e-08,\n 5.9257e-09, 5.9213e-09, 3.0185e-09, 1.4024e-08, 2.1700e-09, 1.6069e-09,\n 4.7753e-09, 1.7333e-09, 1.6305e-09, 2.2470e-09, 4.7877e-09, 2.1539e-09,\n 4.7931e-09, 1.3034e-08, 1.5900e-08, 2.5018e-09, 2.4016e-09, 4.4068e-09,\n 4.5021e-09, 5.6235e-09, 5.3030e-09, 4.4595e-09, 1.8687e-09, 8.0072e-09,\n 5.6205e-09, 7.6899e-09, 4.2807e-09, 1.9342e-09, 2.8811e-09, 8.5128e-09,\n 5.0883e-09, 2.2584e-09, 5.5064e-09, 9.5672e-09, 2.1341e-08, 3.1024e-09,\n 3.6484e-09, 6.4720e-09, 6.4646e-09, 4.8984e-09, 1.8025e-09, 1.7051e-09,\n 5.0132e-09, 4.4098e-09, 1.2573e-09, 7.2485e-09, 4.5832e-09, 3.5383e-09,\n 2.8002e-09, 5.9642e-09, 2.7164e-09, 1.3749e-09, 2.4609e-09, 2.9139e-09,\n 3.2048e-09, 2.2782e-09, 1.1890e-08, 4.4737e-09, 2.7371e-09, 9.3410e-09,\n 3.8733e-09, 6.6568e-09, 2.3865e-09, 1.0215e-08, 2.1734e-09, 5.6719e-09,\n 5.1944e-09, 2.4324e-09, 3.6401e-09, 2.1679e-09, 1.6428e-09, 2.3612e-09,\n 6.0358e-09, 3.0856e-09, 3.1455e-09, 4.2134e-09, 4.1500e-09, 1.5815e-08,\n 3.9349e-09, 2.2617e-09, 2.8702e-09, 8.1783e-09, 4.1398e-09, 2.4112e-09,\n 5.0010e-09, 3.9425e-09], device='cuda:0')" + }, + "28": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-2.0788e-07, -2.8917e-07, 5.0366e-07, ..., -8.9726e-08,\n -5.6738e-07, 2.6383e-07],\n [ 1.7978e-07, -1.3870e-07, 3.3729e-07, ..., -1.2083e-06,\n -6.5913e-07, 7.6241e-08],\n [-3.7165e-08, 4.4065e-08, -1.2224e-07, ..., 2.9945e-07,\n 4.3992e-07, -4.1263e-08],\n ...,\n [-5.8395e-08, 3.5408e-08, 1.7215e-07, ..., -5.0305e-07,\n -5.9888e-07, 3.4784e-08],\n [ 1.3114e-07, -3.1020e-09, 4.2106e-07, ..., -5.2571e-07,\n -8.0221e-07, -3.8425e-08],\n [-3.5026e-07, -1.8154e-07, -2.4880e-07, ..., 4.7817e-07,\n -4.6713e-07, 7.9253e-08]], device='cuda:0')", + "exp_avg_sq": "tensor([[7.4115e-13, 1.3000e-12, 1.5203e-12, ..., 1.2977e-12, 1.3327e-12,\n 1.1782e-12],\n [2.1120e-12, 4.1777e-12, 4.9116e-12, ..., 3.9395e-12, 4.4514e-12,\n 3.8421e-12],\n [4.4608e-13, 7.6099e-13, 9.4036e-13, ..., 7.8550e-13, 8.5973e-13,\n 7.3917e-13],\n ...,\n [5.2256e-13, 9.2090e-13, 1.1178e-12, ..., 9.2125e-13, 1.0125e-12,\n 8.4677e-13],\n [1.0307e-12, 1.7598e-12, 2.3010e-12, ..., 1.8349e-12, 2.0501e-12,\n 1.7816e-12],\n [8.0929e-13, 1.3094e-12, 1.6812e-12, ..., 1.4124e-12, 1.4212e-12,\n 1.2970e-12]], device='cuda:0')" + }, + "29": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-4.6403e-06, -6.7790e-06, 4.0789e-06, 2.1507e-05, 7.3379e-06,\n 1.5183e-05, -1.5666e-05, -1.0957e-05, -1.5072e-06, -1.4331e-05,\n -1.8487e-05, -6.1235e-06, 1.5543e-05, 1.1652e-05, 1.8324e-05,\n 3.1673e-06, -8.3873e-06, -1.2303e-05, -7.9510e-06, -2.0810e-06,\n -1.2761e-05, -2.1306e-06, -5.4558e-07, -1.2915e-05, 1.1639e-05,\n -1.2415e-05, 1.4212e-05, 5.6429e-06, 2.8746e-05, -4.1060e-06,\n -7.8416e-06, -2.6252e-06, 4.4959e-06, -8.8147e-07, -5.2754e-06,\n 1.3973e-05, -7.4114e-06, -1.9650e-05, 4.0077e-06, -8.4457e-06,\n 4.5403e-06, 2.0473e-05, 3.6917e-06, 7.3266e-06, -4.3975e-06,\n 1.6038e-05, 1.6085e-06, 2.1116e-05, -5.8365e-06, 1.6431e-05,\n 1.7821e-06, -3.6281e-06, 9.1027e-06, 1.2325e-06, -2.4431e-05,\n 1.7544e-06, 1.2845e-05, 1.9309e-05, -1.2544e-05, -1.6214e-06,\n 3.9004e-06, -5.3963e-06, -1.4810e-05, 1.5836e-06, -5.6773e-06,\n -3.8090e-06, 4.1724e-06, -2.0064e-05, 1.2123e-05, 8.0892e-07,\n 2.7041e-06, 1.0040e-05, 7.1902e-06, 5.9099e-07, 4.4638e-06,\n 4.9387e-06, -2.5476e-06, -1.5583e-05, -2.7244e-05, -4.0119e-06,\n -2.5930e-06, -5.0170e-06, 1.3077e-05, -1.2784e-05, -2.3402e-06,\n -9.9525e-06, 5.8841e-06, 1.0148e-05, -9.9664e-07, 5.9534e-06,\n 3.9059e-07, 4.3204e-06, -7.4841e-06, 1.5437e-05, 6.2662e-06,\n -1.8066e-06, 8.1124e-06, -1.6513e-05, -2.8529e-06, -9.2939e-06,\n -1.8907e-05, 3.4301e-07, -2.4505e-05, -4.5139e-06, 6.4992e-06,\n -2.0232e-05, 1.7098e-05, 1.2594e-05, 1.2725e-05, -6.3917e-06,\n -2.9861e-05, 1.1959e-06, -1.7951e-05, 1.5081e-05, -1.9095e-05,\n 6.9441e-06, 4.8450e-06, -1.3016e-05, 7.5836e-06, 1.4839e-05,\n 6.5779e-06, -2.8546e-05, 1.0553e-05, -2.7305e-06, 6.5033e-06,\n -9.3849e-06, 2.8198e-06, 1.1782e-05, -2.2436e-06, 7.3388e-06,\n -2.0615e-05, -1.7791e-05, -1.6004e-05, 4.3942e-06, -1.0171e-05,\n 9.3715e-06, -1.3847e-05, -1.9673e-05, 2.0084e-05, -5.7881e-06,\n 7.3990e-06, 6.4936e-06, -2.7557e-05, 7.7687e-06, 9.1260e-06,\n 2.7126e-07, 7.9637e-06, -2.7288e-06, 7.9873e-06, 2.3434e-06,\n 3.6007e-06, 7.7593e-06, 6.0083e-06, -6.2868e-06, 5.8157e-06,\n -1.6175e-05, 6.3663e-06, -2.4870e-06, -1.2042e-05, 9.2143e-07,\n 2.8930e-06, 4.8723e-06, 1.1542e-06, -1.0086e-05, 3.0720e-06,\n -6.6039e-06, 8.8678e-06, -1.7824e-06, 1.8460e-05, -1.3057e-05,\n -2.2001e-05, -1.2138e-05, -2.5337e-05, -1.5994e-05, 1.4608e-05,\n -2.1866e-05, -2.3611e-05, -9.5336e-07, 7.5760e-06, -2.0198e-06,\n -1.2353e-05, -1.3166e-05, -3.3374e-06, 6.8165e-07, 4.9154e-06,\n -4.8654e-06, 5.1561e-06, 8.6515e-06, -2.8863e-06, -1.1739e-07,\n -4.4414e-06, -4.1533e-06, -8.2059e-07, -2.8315e-06, -1.4601e-05,\n 1.1355e-05, 8.4205e-06, -2.4549e-05, 1.6247e-05, -1.7502e-06,\n 1.1261e-06, -1.3168e-05, -7.4673e-06, 2.0358e-05, 1.5225e-05,\n 7.0704e-06, 1.0858e-05, -4.8733e-06, -1.1913e-05, -1.8665e-05,\n -9.0214e-06, 3.9486e-06, 7.4377e-06, 5.2274e-06, -9.1277e-06,\n 2.3204e-05, -5.2011e-06, 7.5447e-06, 1.6455e-06, -2.2191e-06,\n 1.1219e-05, -1.3513e-05, 1.1582e-05, -3.5539e-06, 1.3891e-05,\n 3.2302e-07, 6.8162e-06, 7.9873e-06, 1.0508e-05, -9.8152e-06,\n -6.1182e-06, -6.3722e-06, 8.9734e-06, 2.1067e-06, 1.9321e-05,\n -3.8263e-06, 8.6817e-06, -5.2566e-06, 2.1761e-05, -1.1438e-05,\n 5.9249e-06, -1.3421e-05, -2.0093e-05, -1.4752e-05, -3.0325e-06,\n 5.2271e-06, -8.7721e-07, 1.5063e-05, 5.1985e-06, -6.0688e-06,\n -2.6660e-06, 2.7872e-05, 7.3891e-06, 4.1438e-06, 9.9457e-06,\n -4.6166e-07, 6.1268e-06, 2.0955e-05, -1.1298e-05, 3.4359e-06,\n 5.5649e-06, 2.2396e-05, -1.0680e-05, 9.7058e-06, -1.2823e-05,\n 4.8222e-06, -2.6000e-05, -7.7935e-06, -1.6667e-05, 1.1046e-05,\n 1.0063e-07, 1.1730e-05, -5.4610e-08, 4.9439e-06, 3.7505e-06,\n 7.1707e-06, 1.5163e-05, 6.2864e-06, 6.0752e-06, 1.0997e-06,\n 3.9190e-06, 1.4376e-05, 4.2602e-06, -2.6629e-06, 1.6836e-05,\n 1.4240e-05, 1.2556e-05, 1.9451e-05, 6.4870e-06, 9.1706e-06,\n 8.8310e-06, -2.8477e-06, -1.7532e-05, 7.5394e-06, -5.4058e-06,\n 8.6125e-07, 3.7757e-06, 9.2145e-06, -2.4940e-06, 3.6502e-06,\n -3.1681e-05, 1.3182e-05, -7.2574e-06, 2.0259e-05, 7.2334e-06,\n 8.5176e-06, -3.0336e-06, 4.7591e-06, -2.5977e-05, 1.8473e-05,\n -1.2657e-05, -1.1362e-05, -2.8120e-06, -4.7989e-06, -4.4328e-06,\n -1.9992e-05, -9.2541e-06, -1.4545e-05, 9.2005e-06, 2.7826e-05,\n 2.1153e-05, 3.4239e-06, 1.4425e-05, 5.1430e-06, 8.3105e-06,\n 8.5442e-06, 9.6107e-06, 4.4951e-06, 5.7472e-06, -7.8583e-06,\n 1.5264e-05, 2.0271e-05, 1.0631e-05, 3.4761e-06, -5.1909e-07,\n -6.3378e-06, -2.0750e-05, 2.9181e-05, -7.5538e-06, -1.8939e-05,\n -2.5962e-05, 5.2884e-06, -8.6756e-07, -1.3840e-05, 2.6375e-06,\n -6.7424e-06, -2.8447e-06, -1.0656e-05, -7.6816e-06, -1.0356e-05,\n -5.3978e-06, 1.0232e-06, -3.6814e-06, -8.1972e-06, -3.4056e-06,\n 5.8032e-06, 6.0824e-06, -1.3485e-05, -6.6063e-06, 8.2806e-06,\n -7.5027e-07, -5.3383e-06, -3.9290e-06, 2.2445e-06, 4.5442e-06,\n -1.3444e-06, 8.5599e-07, -8.5171e-06, -9.9233e-07, -4.3276e-06,\n 9.8066e-06, 3.2894e-06, 7.7608e-07, 1.5569e-05, 4.3207e-06,\n 2.6397e-05, -7.8704e-06, -1.0940e-05, -3.3853e-06, -1.4279e-05,\n -1.1318e-05, 6.9907e-06, 1.1719e-05, 9.5960e-06, -1.9149e-05,\n 3.4298e-06, 2.4334e-06, 1.5032e-05, -7.9562e-07, -1.0807e-05,\n -5.2657e-06, -7.0590e-06, 1.5281e-05, 9.7877e-07, -4.2919e-06,\n -8.5433e-06, -5.6921e-06, -2.5787e-06, -3.4902e-06, 5.8997e-07,\n -5.5061e-06, -7.6516e-06, 8.5336e-06, -3.7597e-05, -1.1018e-05,\n -5.0203e-05, -3.0422e-06, 8.6752e-06, -8.0539e-07, -1.8584e-05,\n -8.4954e-06, 8.3561e-06, -1.9721e-05, 9.9476e-06, 1.4159e-05,\n -1.8379e-05, -8.9456e-06, 2.7754e-05, -7.3396e-06, 2.0734e-06,\n -1.0341e-06, 8.5343e-06, 2.8378e-06, -8.4851e-07, -2.8090e-05,\n 2.2003e-05, -2.3694e-05, 1.3716e-05, 1.4479e-05, -1.5207e-05,\n -7.0645e-06, -4.4425e-06, 5.4109e-06, 7.7754e-07, 9.5286e-06,\n 8.6306e-06, -1.0025e-06, -1.5223e-06, -1.6008e-05, -1.1346e-05,\n -1.5893e-05, 1.6457e-05, -2.1817e-06, 2.3266e-06, 7.8568e-06,\n 4.7725e-06, 9.5365e-06, 1.1633e-05, 2.3175e-05, 1.4846e-05,\n 1.6649e-06, 1.0651e-05, 1.0279e-05, 1.2781e-05, -2.4685e-07,\n -1.4428e-05, 5.8246e-06, -5.6873e-06, -1.3476e-05, 1.4935e-06,\n 1.9674e-06, -4.0215e-06, 1.1327e-05, 1.7413e-05, -1.2037e-05,\n 1.7176e-05, 6.2370e-06, 7.9658e-06, 1.1621e-05, 4.7989e-06,\n 9.6617e-06, 2.1605e-06, -1.9323e-05, 1.0093e-05, -2.3164e-06,\n 1.4255e-05, -5.4343e-06, 1.7205e-06, 8.5085e-06, 3.9212e-06,\n -1.4378e-06, -4.0416e-06, 3.0854e-05, -1.8669e-06, -4.7548e-06,\n -1.2945e-05, -4.2409e-06, -2.0103e-05, -6.5609e-06, 1.2361e-05,\n 5.2249e-07, -4.0261e-06, 5.0181e-06, -1.0837e-06, -1.1716e-05,\n -8.3499e-07, -3.2092e-07, 9.9126e-06, -5.2309e-06, -1.1934e-05,\n 8.5642e-06, 1.5702e-05, -8.7670e-06, -1.7693e-05, 1.6262e-05,\n -3.8596e-06, -1.3398e-05, 2.1068e-05, -1.0187e-05, 2.5253e-06,\n 1.3110e-05, -3.9909e-06], device='cuda:0')", + "exp_avg_sq": "tensor([2.6925e-09, 9.0007e-09, 1.6700e-09, 2.4734e-09, 5.2607e-09, 2.9427e-09,\n 5.2234e-09, 1.6463e-09, 1.4595e-09, 6.7843e-09, 6.3038e-09, 2.8596e-09,\n 8.2682e-09, 1.5015e-09, 2.4542e-09, 2.4182e-09, 1.6707e-09, 5.3237e-09,\n 1.3169e-09, 1.4627e-09, 2.2894e-09, 5.5774e-09, 2.1674e-09, 4.8721e-09,\n 1.1672e-09, 2.2584e-09, 4.4318e-09, 4.5090e-09, 2.4653e-09, 1.7205e-09,\n 2.0216e-09, 1.4972e-09, 2.2746e-09, 4.0738e-09, 1.4337e-09, 2.7602e-09,\n 3.0966e-09, 2.5097e-09, 2.5453e-09, 2.7609e-09, 1.0262e-08, 2.2898e-09,\n 1.1299e-08, 3.5020e-09, 3.6138e-09, 1.7389e-09, 1.8571e-09, 4.5160e-09,\n 2.0208e-09, 3.1102e-09, 2.3337e-09, 1.9860e-09, 6.0611e-09, 2.2156e-09,\n 4.0686e-09, 3.0625e-09, 3.8659e-09, 2.4559e-09, 6.5722e-09, 1.4493e-09,\n 1.2129e-09, 2.2482e-09, 2.8839e-09, 2.8711e-09, 1.8598e-09, 1.3745e-09,\n 1.6761e-09, 7.8701e-09, 5.0001e-09, 4.8036e-09, 1.6613e-09, 3.0128e-09,\n 1.8986e-09, 2.5991e-09, 3.0439e-09, 1.9260e-09, 2.0316e-09, 6.8414e-09,\n 3.5095e-09, 7.8794e-09, 3.7957e-09, 3.6135e-09, 3.5080e-09, 3.5637e-09,\n 1.3791e-09, 2.8130e-09, 2.5695e-09, 7.8911e-09, 2.9111e-09, 2.6843e-09,\n 1.3355e-09, 6.3242e-09, 1.7296e-09, 4.4622e-09, 1.9181e-09, 2.6734e-09,\n 2.4092e-09, 4.6912e-09, 2.7010e-09, 4.5019e-09, 6.4847e-09, 1.4338e-09,\n 1.1296e-08, 2.7811e-09, 2.0844e-09, 2.8881e-09, 1.8934e-09, 4.3575e-09,\n 5.7046e-09, 1.2662e-09, 4.5072e-09, 6.6028e-09, 2.6071e-09, 4.1658e-09,\n 6.3296e-09, 2.4905e-09, 4.3304e-09, 2.0562e-09, 2.0650e-08, 3.8476e-09,\n 2.8136e-09, 9.9950e-09, 3.2464e-09, 3.8401e-09, 1.8203e-09, 3.2612e-09,\n 2.6452e-09, 3.4203e-09, 1.9742e-09, 1.7937e-09, 1.0186e-08, 3.5186e-09,\n 1.0572e-08, 5.8876e-09, 1.4561e-09, 1.5131e-09, 4.5481e-09, 3.2754e-09,\n 3.0841e-09, 1.1579e-08, 2.4576e-09, 4.7155e-09, 5.7958e-09, 2.4900e-09,\n 2.6939e-09, 4.1731e-09, 4.1372e-09, 1.3505e-09, 1.2069e-09, 3.0659e-09,\n 3.4494e-09, 1.6664e-09, 3.6149e-09, 2.9715e-09, 2.8075e-09, 2.6055e-09,\n 1.4524e-09, 1.4913e-09, 2.4444e-09, 2.9954e-09, 1.1397e-08, 2.0136e-09,\n 1.9667e-09, 3.3760e-09, 1.8668e-09, 4.0380e-09, 1.3892e-09, 4.4201e-09,\n 3.9426e-09, 2.4552e-09, 6.6188e-09, 2.1306e-09, 4.2275e-09, 2.9212e-09,\n 3.3316e-09, 5.3065e-09, 1.8385e-09, 5.9847e-09, 2.7133e-09, 2.7544e-09,\n 2.6834e-09, 1.4101e-09, 1.9811e-09, 3.8805e-09, 4.3961e-09, 1.0592e-09,\n 2.9386e-09, 1.9051e-09, 1.4154e-09, 8.5127e-09, 6.2257e-09, 4.9902e-09,\n 3.9320e-09, 1.4180e-09, 2.5778e-09, 2.7724e-09, 2.1226e-09, 5.0919e-09,\n 4.1708e-09, 1.1154e-09, 1.7068e-09, 3.4378e-09, 1.5278e-09, 8.2711e-09,\n 1.0761e-09, 1.7155e-09, 1.7981e-09, 2.2090e-09, 5.7413e-09, 3.5576e-09,\n 2.2224e-09, 2.4215e-09, 2.5432e-09, 3.9325e-09, 2.1078e-09, 5.5534e-09,\n 9.3756e-09, 1.4281e-08, 5.6057e-09, 4.8906e-09, 7.6392e-09, 2.3547e-09,\n 1.0120e-08, 3.0405e-09, 1.9401e-09, 1.7236e-09, 2.7230e-09, 3.5188e-09,\n 2.9457e-09, 2.8277e-09, 2.7372e-09, 1.2896e-09, 2.6537e-09, 2.8710e-09,\n 6.0228e-09, 8.8047e-09, 1.3429e-09, 2.0435e-09, 3.1905e-09, 3.6897e-09,\n 1.9620e-09, 1.8625e-09, 4.6486e-09, 3.7596e-09, 1.7742e-09, 1.5321e-09,\n 2.2273e-09, 7.6800e-09, 4.8483e-09, 5.6010e-09, 2.7836e-09, 3.7841e-09,\n 5.8368e-09, 2.3074e-09, 3.3347e-09, 1.3379e-09, 4.8633e-09, 3.1054e-09,\n 1.9640e-09, 1.0945e-08, 1.6917e-09, 8.8132e-09, 4.2877e-09, 9.4595e-09,\n 9.0907e-09, 1.8464e-09, 4.4852e-09, 4.7064e-09, 4.1192e-09, 2.1443e-09,\n 1.5584e-09, 3.8893e-09, 8.0120e-09, 2.2956e-09, 3.0087e-09, 4.8398e-09,\n 4.6739e-09, 2.6728e-09, 2.1171e-09, 2.3911e-09, 3.1599e-09, 2.7257e-09,\n 4.3346e-09, 1.5519e-09, 4.2242e-09, 2.8227e-09, 2.5153e-09, 3.2783e-09,\n 5.3530e-09, 1.9210e-09, 4.9137e-09, 6.0981e-09, 2.9963e-09, 9.6194e-09,\n 2.2892e-09, 3.1567e-09, 1.2538e-09, 2.4824e-09, 1.8653e-09, 1.6142e-08,\n 5.3852e-09, 3.2203e-09, 2.8681e-09, 5.9085e-09, 2.4048e-09, 4.2214e-09,\n 1.2147e-09, 9.9174e-09, 4.6285e-09, 3.2710e-09, 6.6254e-09, 7.1630e-09,\n 1.5445e-09, 3.0873e-09, 1.8254e-09, 2.1781e-09, 3.4799e-09, 4.7086e-09,\n 2.7316e-09, 4.4152e-09, 3.4641e-09, 1.9194e-09, 3.1683e-09, 2.1559e-09,\n 3.0949e-09, 1.3576e-09, 3.4711e-09, 2.6910e-08, 2.6281e-09, 3.6481e-09,\n 1.3830e-09, 2.4658e-09, 3.5586e-09, 8.7089e-09, 2.2600e-09, 2.6714e-09,\n 2.5300e-09, 5.2970e-09, 2.9246e-09, 2.0527e-09, 9.6096e-09, 3.3827e-09,\n 2.1733e-09, 3.2314e-09, 1.5527e-09, 2.2540e-09, 1.7137e-09, 2.4238e-09,\n 2.1911e-09, 2.4216e-09, 2.0927e-09, 2.0276e-09, 3.9403e-09, 5.7831e-09,\n 2.1657e-09, 2.3165e-09, 1.5182e-09, 4.1947e-09, 4.1494e-09, 3.0798e-09,\n 1.8845e-09, 2.0035e-09, 2.1037e-09, 2.7212e-09, 2.2285e-09, 2.7832e-09,\n 2.5494e-09, 1.3951e-09, 5.0505e-09, 2.8500e-09, 5.2048e-09, 1.0462e-09,\n 1.6385e-09, 1.1054e-08, 1.6543e-09, 2.2344e-08, 5.2207e-09, 2.6030e-09,\n 2.8455e-09, 2.0251e-09, 8.3192e-09, 3.9503e-09, 1.5668e-09, 2.1578e-09,\n 3.0225e-09, 5.5972e-09, 2.4620e-09, 4.9467e-09, 5.5356e-09, 2.0684e-09,\n 1.9687e-09, 1.7449e-09, 3.8712e-09, 3.0373e-09, 3.1791e-09, 2.3779e-09,\n 1.8423e-09, 2.4291e-09, 1.9505e-09, 2.3823e-09, 2.1015e-09, 2.3537e-09,\n 9.5231e-10, 6.4318e-09, 3.0309e-09, 1.3014e-08, 2.3408e-09, 3.1018e-09,\n 2.1813e-09, 3.9718e-09, 8.7911e-09, 2.2391e-08, 4.4662e-09, 2.5700e-09,\n 4.8346e-09, 4.2629e-09, 1.0325e-09, 3.3537e-09, 2.2504e-09, 3.2109e-09,\n 3.8391e-09, 1.1712e-09, 1.2729e-09, 6.0650e-09, 3.2593e-09, 1.4320e-08,\n 3.5890e-09, 4.7873e-09, 2.6712e-09, 1.0451e-08, 1.6314e-09, 1.3502e-09,\n 4.0886e-09, 1.5060e-09, 1.3490e-09, 1.4820e-09, 3.9159e-09, 1.4925e-09,\n 4.0375e-09, 1.0572e-08, 1.4080e-08, 2.2199e-09, 1.9751e-09, 3.4188e-09,\n 2.5591e-09, 5.2117e-09, 4.5208e-09, 3.4306e-09, 1.3860e-09, 8.0231e-09,\n 3.2908e-09, 7.0884e-09, 3.1257e-09, 1.4872e-09, 2.4068e-09, 6.1737e-09,\n 4.0055e-09, 1.6655e-09, 5.0324e-09, 8.0472e-09, 1.6507e-08, 2.4937e-09,\n 2.8155e-09, 5.9126e-09, 5.1502e-09, 4.0775e-09, 1.4455e-09, 1.5284e-09,\n 4.0955e-09, 3.4783e-09, 1.0269e-09, 5.8003e-09, 3.3957e-09, 2.7621e-09,\n 1.9860e-09, 4.7051e-09, 2.0886e-09, 1.1433e-09, 1.8923e-09, 2.0662e-09,\n 2.6034e-09, 1.8823e-09, 9.9256e-09, 3.9709e-09, 2.1172e-09, 8.9403e-09,\n 2.8089e-09, 6.2668e-09, 1.9915e-09, 7.4986e-09, 1.8139e-09, 4.4716e-09,\n 4.8154e-09, 1.9733e-09, 3.0465e-09, 1.6210e-09, 1.3612e-09, 1.7340e-09,\n 5.0847e-09, 2.6776e-09, 2.5672e-09, 2.8730e-09, 3.3172e-09, 1.4528e-08,\n 3.0013e-09, 1.6353e-09, 2.2919e-09, 6.2713e-09, 3.4402e-09, 1.9867e-09,\n 4.3173e-09, 2.8622e-09], device='cuda:0')" + }, + "30": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.6145e-07, 3.1764e-08, -2.5192e-07, ..., -9.3284e-08,\n 1.4599e-08, 5.3231e-08],\n [-2.6228e-07, 1.1640e-07, -2.3886e-07, ..., 2.8433e-07,\n -4.4154e-08, -2.2271e-07],\n [ 4.0968e-08, 3.6996e-08, 1.3183e-07, ..., -1.9575e-08,\n 7.7835e-08, 1.6541e-07],\n ...,\n [ 1.0034e-07, 8.2843e-08, -1.0309e-07, ..., -1.3175e-07,\n -6.1527e-08, 2.2923e-08],\n [ 4.1647e-07, 4.0058e-07, -2.6787e-07, ..., -4.4377e-07,\n 1.6590e-07, 3.7439e-07],\n [-1.2588e-07, -9.9156e-08, 2.1106e-07, ..., 1.3751e-07,\n -1.5136e-07, -2.9788e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.0286e-13, 3.6786e-13, 3.9585e-13, ..., 3.4353e-13, 3.5674e-13,\n 3.3830e-13],\n [5.7770e-13, 1.0649e-12, 1.1227e-12, ..., 9.2193e-13, 1.0280e-12,\n 8.8769e-13],\n [1.2599e-13, 2.1875e-13, 2.3436e-13, ..., 2.1139e-13, 2.1747e-13,\n 1.8936e-13],\n ...,\n [1.2538e-13, 2.2620e-13, 2.3621e-13, ..., 2.0490e-13, 2.0947e-13,\n 1.9117e-13],\n [5.3473e-13, 9.0330e-13, 8.8480e-13, ..., 8.9351e-13, 8.7277e-13,\n 7.4672e-13],\n [1.8731e-13, 3.3559e-13, 3.8169e-13, ..., 3.4399e-13, 3.4936e-13,\n 3.0997e-13]], device='cuda:0')" + }, + "31": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-1.6546e-05, -9.6438e-06, -3.1217e-06, 2.1828e-05, 1.4169e-05,\n 1.3414e-05, -1.6340e-05, -2.3349e-05, -3.4120e-06, -4.1383e-05,\n -5.6243e-05, -6.5484e-06, 1.8501e-05, -3.2583e-06, 2.7802e-05,\n 1.4473e-05, 2.0279e-06, -1.8603e-05, -2.9754e-06, 3.6468e-06,\n -1.3668e-05, -1.3437e-05, 2.2052e-06, -1.5965e-05, 8.6101e-06,\n -1.2735e-05, 5.1426e-05, 7.1523e-06, 3.5094e-06, -2.8776e-06,\n -7.3540e-06, -9.0615e-06, 1.0253e-05, 1.5872e-05, 6.3487e-06,\n 3.4868e-05, -1.2758e-05, -2.5020e-05, 1.2649e-05, -7.6362e-06,\n 3.9585e-05, 1.5652e-05, 3.7098e-05, 1.2868e-05, 3.2989e-05,\n -1.9649e-07, 1.2487e-05, -2.8868e-06, 1.1444e-05, 3.3632e-05,\n -1.8914e-05, -1.4845e-05, 3.3657e-05, -4.2804e-06, -4.1083e-05,\n 1.2689e-05, 2.6515e-05, 3.2418e-05, -2.3816e-05, -2.4851e-06,\n -1.4558e-05, -1.2627e-05, -1.5670e-05, -1.3466e-05, -2.1504e-05,\n -1.9341e-05, -1.3940e-05, -4.0726e-05, 1.8484e-05, 1.0200e-06,\n 1.7353e-05, 1.8582e-06, 7.9166e-06, -6.4595e-06, 6.7293e-06,\n 3.9414e-07, -5.6743e-06, -1.8613e-05, -3.8656e-05, -1.8347e-05,\n 4.8726e-06, 6.6507e-06, 2.5305e-06, -1.5867e-05, 4.7690e-06,\n -9.3069e-06, 1.0870e-05, -3.3627e-05, 6.2624e-06, 1.0739e-06,\n -4.0380e-06, -1.5238e-05, -9.7566e-06, 2.2705e-05, 1.2096e-05,\n 1.7704e-06, 1.7240e-05, -3.4038e-05, -4.8226e-07, -2.6221e-05,\n -3.0368e-05, -1.4289e-05, -3.7053e-05, 3.2469e-06, 1.2492e-06,\n -2.8623e-05, 2.4423e-05, 1.0775e-05, 4.3795e-06, -6.2844e-06,\n -2.7094e-05, 3.0904e-06, -2.3190e-05, -5.0515e-06, -5.1555e-05,\n 1.2729e-05, -1.6582e-05, -1.0224e-05, -2.7286e-05, 7.4494e-06,\n 2.1447e-05, -4.1676e-05, 1.3849e-05, -1.7060e-05, 8.5162e-06,\n -2.2488e-05, -1.1088e-05, 3.2932e-05, -1.6289e-05, 1.3090e-05,\n -3.7896e-05, -4.8872e-06, -3.6297e-05, 9.3999e-06, 2.5170e-06,\n 7.7154e-06, -3.3793e-05, -1.5028e-05, 3.1776e-05, 1.6082e-05,\n 1.3170e-05, -1.1248e-05, -3.5378e-05, 2.3932e-05, 2.5916e-05,\n 5.4705e-06, 1.5509e-05, -8.6009e-06, 5.4316e-06, -5.1750e-06,\n -8.7809e-08, 5.3212e-07, 1.5863e-06, -7.9245e-06, 2.8309e-05,\n -1.7620e-05, 7.1024e-07, -1.7800e-05, -2.3068e-05, 2.7730e-05,\n -8.0925e-06, 5.2200e-06, -1.1563e-06, -8.1680e-07, 7.9809e-06,\n -9.8027e-06, -2.2548e-06, -7.5953e-06, 1.2897e-05, -1.5133e-05,\n 2.3361e-05, -2.2235e-05, 5.6554e-06, -2.2573e-05, 1.4030e-05,\n -2.6376e-05, -9.6551e-06, 3.2287e-06, 2.0549e-05, -9.4872e-06,\n -1.3586e-05, -1.7476e-05, -3.2388e-06, -1.8885e-05, -9.7900e-07,\n 8.2284e-06, 8.0670e-06, 9.3396e-06, -4.2990e-06, 1.2593e-05,\n -2.7774e-05, -1.4309e-05, -1.1101e-05, -6.3930e-06, -1.8042e-05,\n 3.2893e-06, 2.0423e-05, -4.4538e-05, 7.2549e-06, -7.5053e-06,\n -6.2598e-06, -1.8533e-05, 1.7078e-05, 3.9120e-05, 2.2799e-05,\n 2.1759e-05, -3.9986e-06, -2.0938e-06, -7.7831e-06, -4.5991e-06,\n -1.8696e-06, 6.3657e-06, 2.0165e-06, 9.6107e-06, -6.4625e-06,\n 1.6219e-05, 3.3956e-05, 3.9900e-05, -3.0529e-06, -5.7644e-06,\n 2.7199e-05, -2.6871e-05, 2.7448e-05, -1.8280e-06, 7.8726e-06,\n -1.4354e-05, 1.2439e-05, 7.8582e-06, 1.8575e-05, -1.8921e-06,\n 2.7573e-06, -1.1160e-05, 3.2419e-05, 2.2977e-06, 2.6934e-05,\n -1.0506e-05, 1.0203e-05, 4.5765e-06, 2.5406e-05, -1.7331e-05,\n -6.6159e-06, -1.6452e-05, -3.4027e-05, -9.1958e-06, -4.4474e-06,\n 2.8456e-06, -1.9509e-05, 3.0474e-05, 2.3192e-05, -8.6782e-06,\n -1.2106e-05, 3.8851e-05, -9.8916e-06, -5.8676e-06, 3.0182e-05,\n 7.9401e-06, 7.2540e-06, 3.4008e-05, -1.5587e-05, 2.1198e-05,\n -1.2917e-05, 3.0315e-05, -1.7737e-05, -1.3519e-05, -4.8397e-05,\n 1.2038e-05, -1.6115e-05, -3.6046e-05, -1.4521e-06, 1.5609e-05,\n 4.6392e-06, 1.5147e-05, -1.7634e-05, -4.2743e-06, -2.2600e-05,\n -4.8950e-06, 1.4277e-05, -1.1852e-05, 4.3817e-06, 1.6538e-05,\n -6.4169e-07, 2.9599e-05, 1.3682e-05, -9.2339e-06, 3.2283e-05,\n 3.0842e-05, 2.3997e-05, 2.7762e-05, 2.4124e-05, 1.2227e-05,\n 2.0581e-05, 7.2069e-06, -3.3987e-05, -3.5613e-05, -9.4721e-06,\n 6.1152e-06, 9.8783e-07, 2.7079e-05, -7.8725e-06, -2.2694e-05,\n -4.3137e-05, 9.2546e-06, 2.7773e-06, 3.7183e-05, 6.7431e-06,\n 1.4229e-05, 4.2589e-06, -1.3831e-05, -3.9563e-05, 1.7070e-05,\n -2.0727e-05, 2.1536e-05, 3.5875e-06, -1.0508e-05, 1.3761e-07,\n -2.3471e-05, 4.0674e-06, -4.2390e-05, 1.0400e-05, 4.0911e-05,\n 1.9001e-05, 1.9357e-05, 1.6833e-05, 2.8017e-05, 2.4735e-06,\n 1.3021e-05, 5.4789e-06, -4.6092e-05, 5.8251e-06, -2.4227e-05,\n 2.3868e-05, 3.6903e-05, 2.8706e-05, 2.8736e-05, 7.5355e-06,\n 7.3641e-06, -2.9975e-05, 4.6091e-05, -1.2432e-05, -2.3691e-05,\n -2.5172e-05, 2.8837e-06, -1.1200e-05, -2.5343e-05, -4.1990e-06,\n -1.6949e-05, -1.1987e-05, -1.2708e-05, -2.9639e-06, -1.4595e-05,\n -1.2573e-05, -4.3944e-06, -1.5982e-05, -3.6734e-05, -1.4171e-05,\n 3.2718e-05, 1.9342e-05, -1.7884e-05, -1.0302e-05, -1.6011e-06,\n -9.4760e-06, 8.7471e-06, -3.8532e-06, -1.0833e-05, 5.5890e-06,\n 7.9546e-06, 4.9462e-06, -2.1813e-06, -1.2457e-05, -6.1419e-06,\n 3.9605e-06, -5.7797e-06, -1.1554e-05, 2.9735e-05, 1.3451e-06,\n -2.8772e-05, -1.7989e-05, -9.4180e-06, 1.5344e-05, -3.8715e-07,\n -4.1322e-05, 7.7878e-06, 7.1868e-06, 2.0039e-05, -2.3010e-05,\n -1.0220e-05, -8.8314e-08, 1.4767e-05, 3.4413e-07, -9.9834e-06,\n -1.6773e-05, -6.2051e-06, 1.1722e-05, 1.0138e-05, -6.7370e-06,\n -1.5620e-05, 1.0183e-06, 2.1934e-06, 1.2568e-05, -8.4873e-06,\n -1.6562e-05, -1.8863e-05, 1.6540e-05, -4.4639e-05, -2.8954e-05,\n 1.2844e-05, -3.0375e-06, 1.7609e-05, -2.6111e-05, -3.3069e-05,\n 7.4040e-07, -5.3387e-05, -1.9369e-05, -3.9956e-06, 7.4255e-06,\n -4.1832e-05, -1.3079e-05, 1.4167e-05, -6.9483e-07, 6.7485e-06,\n 2.1494e-05, 1.2234e-05, 2.4415e-06, 1.7335e-05, -3.6529e-05,\n 5.3959e-05, -4.4185e-06, -6.6357e-06, 2.4624e-05, -3.0009e-05,\n 2.7960e-06, -8.9115e-06, 3.8797e-05, 4.7175e-07, 1.3914e-06,\n 7.9200e-06, 1.9017e-08, 1.8792e-05, -3.0369e-05, 4.5747e-06,\n -3.4562e-05, 1.1276e-05, 1.2643e-05, -1.8229e-05, 5.5887e-07,\n -4.8764e-06, 1.6835e-05, 3.4686e-05, 1.4572e-05, 3.1427e-05,\n -8.6623e-06, 2.8542e-05, -1.3465e-08, 2.1202e-05, 1.6952e-05,\n 2.8248e-05, 1.3497e-05, -1.5202e-05, 1.6049e-05, -7.8165e-07,\n -4.0841e-05, -7.9280e-06, 1.9354e-05, 2.3432e-05, -1.6796e-05,\n 2.4687e-05, 1.3738e-05, 1.5708e-06, 8.2267e-06, -3.6145e-06,\n 2.6045e-06, 3.3480e-07, -4.2599e-05, 2.2162e-06, -2.0269e-05,\n -6.8186e-06, 8.4788e-06, -1.7869e-06, 1.3135e-05, -7.7653e-06,\n -1.5770e-05, -1.1754e-05, 3.7361e-05, -2.1249e-06, -5.3796e-06,\n -3.3650e-05, -6.3262e-06, -3.2875e-05, 4.0950e-06, 3.3945e-05,\n 1.0531e-05, 6.1925e-06, -1.1433e-05, 6.5689e-07, -3.1676e-05,\n -2.4181e-05, -3.9947e-06, 1.5458e-05, -2.4657e-05, -2.8186e-06,\n 3.5523e-05, 2.7170e-05, -1.2112e-05, -4.9032e-05, 4.6091e-06,\n 1.0629e-05, -7.9909e-06, 1.8534e-05, -2.1026e-05, 1.9981e-06,\n 2.1163e-05, -1.1902e-05], device='cuda:0')", + "exp_avg_sq": "tensor([1.6327e-09, 4.6652e-09, 9.2050e-10, 1.3785e-09, 3.6638e-09, 1.5154e-09,\n 3.5584e-09, 1.2533e-09, 7.4717e-10, 4.9806e-09, 3.2315e-09, 1.3959e-09,\n 4.0432e-09, 5.9268e-10, 1.3817e-09, 1.1606e-09, 1.2473e-09, 2.5214e-09,\n 1.3884e-09, 7.2517e-10, 1.7306e-09, 2.8908e-09, 1.0015e-09, 3.5088e-09,\n 5.7731e-10, 1.1505e-09, 2.4593e-09, 3.3628e-09, 1.5716e-09, 8.8950e-10,\n 1.4806e-09, 1.2254e-09, 1.0669e-09, 2.7101e-09, 6.5632e-10, 1.6067e-09,\n 2.3514e-09, 1.2769e-09, 1.9339e-09, 1.2432e-09, 5.2043e-09, 1.8431e-09,\n 4.4347e-09, 2.5695e-09, 1.7530e-09, 1.2268e-09, 1.1409e-09, 2.6430e-09,\n 9.8016e-10, 2.0591e-09, 9.5649e-10, 1.0547e-09, 4.1528e-09, 1.4657e-09,\n 2.3275e-09, 1.6838e-09, 2.0348e-09, 1.7589e-09, 5.0170e-09, 8.5387e-10,\n 6.4981e-10, 1.0325e-09, 2.6755e-09, 1.3980e-09, 9.8681e-10, 7.2047e-10,\n 7.8012e-10, 4.2451e-09, 2.4394e-09, 2.5840e-09, 1.0192e-09, 1.3497e-09,\n 1.2207e-09, 1.1208e-09, 1.6039e-09, 1.3252e-09, 1.1456e-09, 3.2869e-09,\n 2.7690e-09, 4.5670e-09, 2.5373e-09, 2.3875e-09, 1.8639e-09, 3.4077e-09,\n 6.6583e-10, 2.3578e-09, 1.1745e-09, 4.2527e-09, 2.3002e-09, 1.3167e-09,\n 9.9046e-10, 3.7319e-09, 8.7785e-10, 3.3317e-09, 1.2104e-09, 1.3782e-09,\n 1.5425e-09, 2.2301e-09, 1.7128e-09, 3.3426e-09, 2.7423e-09, 8.6877e-10,\n 1.0794e-08, 1.4407e-09, 8.4443e-10, 3.0207e-09, 1.0034e-09, 2.1985e-09,\n 3.0562e-09, 8.9877e-10, 3.4223e-09, 3.4514e-09, 2.1760e-09, 3.1890e-09,\n 2.9219e-09, 1.1837e-09, 1.9852e-09, 1.0071e-09, 1.0385e-08, 1.9449e-09,\n 1.6939e-09, 9.0278e-09, 2.5430e-09, 1.8219e-09, 8.9932e-10, 1.5725e-09,\n 1.2814e-09, 2.4470e-09, 1.1367e-09, 1.0014e-09, 8.5372e-09, 2.2118e-09,\n 7.3996e-09, 3.0055e-09, 1.1338e-09, 1.0254e-09, 2.2983e-09, 1.7062e-09,\n 2.0915e-09, 5.8967e-09, 1.2875e-09, 2.1206e-09, 3.0781e-09, 1.2908e-09,\n 1.8587e-09, 1.8275e-09, 3.4595e-09, 1.0696e-09, 7.4232e-10, 2.5010e-09,\n 2.2737e-09, 1.0084e-09, 2.2826e-09, 1.4956e-09, 1.7329e-09, 1.5225e-09,\n 8.5940e-10, 9.9507e-10, 1.7350e-09, 1.6467e-09, 6.3375e-09, 1.2228e-09,\n 1.1669e-09, 2.2148e-09, 9.0841e-10, 1.8446e-09, 9.0587e-10, 2.0696e-09,\n 2.5688e-09, 2.3665e-09, 3.9269e-09, 1.4504e-09, 2.4448e-09, 2.8592e-09,\n 2.0836e-09, 2.9668e-09, 9.9218e-10, 3.2557e-09, 1.1361e-09, 1.4059e-09,\n 1.1442e-09, 9.1996e-10, 1.2877e-09, 2.5278e-09, 2.0875e-09, 6.6540e-10,\n 1.4393e-09, 1.0258e-09, 1.1813e-09, 4.9082e-09, 4.4964e-09, 2.6459e-09,\n 1.6941e-09, 1.1374e-09, 1.2598e-09, 1.5431e-09, 1.0466e-09, 2.7764e-09,\n 2.2057e-09, 5.8778e-10, 9.0615e-10, 2.5395e-09, 8.6596e-10, 6.7824e-09,\n 6.8447e-10, 9.2236e-10, 7.2013e-10, 1.1278e-09, 2.9205e-09, 2.0819e-09,\n 1.3845e-09, 1.3146e-09, 1.1485e-09, 1.9894e-09, 1.0862e-09, 3.3087e-09,\n 4.3786e-09, 7.0496e-09, 2.7736e-09, 2.0225e-09, 4.0233e-09, 1.8958e-09,\n 4.9872e-09, 1.3020e-09, 1.2447e-09, 7.6076e-10, 1.3185e-09, 2.6371e-09,\n 1.6603e-09, 1.5047e-09, 1.4092e-09, 7.4346e-10, 1.4647e-09, 1.5180e-09,\n 3.6652e-09, 4.3072e-09, 1.2009e-09, 8.8474e-10, 2.8818e-09, 2.4627e-09,\n 8.4007e-10, 8.2217e-10, 2.5026e-09, 1.9733e-09, 1.0560e-09, 1.0181e-09,\n 1.0505e-09, 5.7945e-09, 2.7086e-09, 2.9427e-09, 1.4124e-09, 2.1213e-09,\n 3.1187e-09, 1.1607e-09, 1.8767e-09, 6.4621e-10, 2.1285e-09, 1.4415e-09,\n 1.6693e-09, 6.0009e-09, 8.2176e-10, 8.2103e-09, 1.7800e-09, 5.7369e-09,\n 4.6184e-09, 8.5046e-10, 2.6969e-09, 2.9107e-09, 1.8325e-09, 1.4452e-09,\n 7.6961e-10, 1.8603e-09, 3.3550e-09, 1.5015e-09, 1.4530e-09, 3.2078e-09,\n 3.4033e-09, 1.3353e-09, 1.5219e-09, 1.5560e-09, 1.4554e-09, 1.8009e-09,\n 2.7697e-09, 6.7239e-10, 2.7805e-09, 1.2608e-09, 1.4597e-09, 2.1211e-09,\n 3.5766e-09, 1.6483e-09, 2.9956e-09, 3.1136e-09, 1.5123e-09, 4.1113e-09,\n 1.6330e-09, 1.8327e-09, 7.3841e-10, 1.7620e-09, 1.0787e-09, 7.4079e-09,\n 4.5930e-09, 1.5977e-09, 1.3968e-09, 3.3389e-09, 1.3348e-09, 2.6037e-09,\n 6.8598e-10, 4.2234e-09, 2.7483e-09, 3.2337e-09, 5.6464e-09, 3.2096e-09,\n 7.5354e-10, 1.3275e-09, 1.1636e-09, 1.6350e-09, 1.7941e-09, 2.2230e-09,\n 1.8153e-09, 3.1783e-09, 1.5751e-09, 8.6765e-10, 2.0745e-09, 1.2280e-09,\n 1.7538e-09, 6.5740e-10, 1.7529e-09, 1.2018e-08, 1.2907e-09, 2.4030e-09,\n 1.0525e-09, 1.8085e-09, 1.8794e-09, 4.2242e-09, 1.0304e-09, 1.5914e-09,\n 1.4015e-09, 3.5750e-09, 1.6728e-09, 1.5052e-09, 8.3133e-09, 1.6295e-09,\n 1.0714e-09, 2.0588e-09, 1.1211e-09, 2.0458e-09, 9.4421e-10, 1.5610e-09,\n 1.0346e-09, 1.9411e-09, 9.7172e-10, 1.0800e-09, 1.9192e-09, 3.0139e-09,\n 1.0722e-09, 1.1846e-09, 8.4111e-10, 2.0640e-09, 2.9752e-09, 1.7100e-09,\n 1.2385e-09, 1.1572e-09, 1.1655e-09, 1.6330e-09, 1.1335e-09, 1.3029e-09,\n 1.3735e-09, 1.1017e-09, 3.0409e-09, 2.2501e-09, 2.5620e-09, 7.9276e-10,\n 9.0168e-10, 7.4548e-09, 9.4501e-10, 1.2874e-08, 3.2125e-09, 1.6407e-09,\n 1.4709e-09, 1.2124e-09, 4.4197e-09, 2.3472e-09, 8.6997e-10, 1.0328e-09,\n 1.9202e-09, 2.9774e-09, 1.3686e-09, 2.4429e-09, 3.2427e-09, 1.5331e-09,\n 1.0277e-09, 7.9364e-10, 3.7516e-09, 1.6932e-09, 1.3035e-09, 1.5402e-09,\n 9.0373e-10, 1.3790e-09, 1.2489e-09, 1.2536e-09, 1.0517e-09, 1.0054e-09,\n 4.9459e-10, 3.8266e-09, 1.6380e-09, 8.4727e-09, 1.3034e-09, 1.3966e-09,\n 1.1602e-09, 1.8512e-09, 4.5979e-09, 1.2454e-08, 1.9719e-09, 1.1989e-09,\n 2.6009e-09, 2.5131e-09, 5.4153e-10, 2.1727e-09, 1.1527e-09, 2.3867e-09,\n 2.0719e-09, 7.1074e-10, 6.6606e-10, 3.4179e-09, 2.0637e-09, 6.7932e-09,\n 2.8846e-09, 2.3999e-09, 1.6495e-09, 5.2448e-09, 1.0390e-09, 9.5947e-10,\n 2.6950e-09, 8.1306e-10, 1.1687e-09, 9.0234e-10, 2.0149e-09, 8.8272e-10,\n 2.9473e-09, 5.5384e-09, 6.3375e-09, 1.2758e-09, 1.2724e-09, 1.6683e-09,\n 2.1970e-09, 2.8055e-09, 1.9904e-09, 2.3439e-09, 1.1754e-09, 4.9920e-09,\n 2.4995e-09, 4.3525e-09, 1.9041e-09, 1.1365e-09, 1.2305e-09, 3.3913e-09,\n 2.4004e-09, 9.0410e-10, 2.7177e-09, 4.0161e-09, 7.6037e-09, 1.1573e-09,\n 1.5925e-09, 3.4455e-09, 4.4273e-09, 3.3073e-09, 7.3124e-10, 9.9694e-10,\n 2.0228e-09, 2.1645e-09, 5.9231e-10, 3.2267e-09, 2.1140e-09, 1.2927e-09,\n 1.0043e-09, 2.3056e-09, 1.2054e-09, 7.2109e-10, 1.1393e-09, 1.1459e-09,\n 1.3194e-09, 9.0073e-10, 4.8433e-09, 2.4407e-09, 1.0175e-09, 6.9884e-09,\n 1.2770e-09, 5.5312e-09, 1.1728e-09, 3.9643e-09, 1.0334e-09, 2.2413e-09,\n 2.5118e-09, 9.4035e-10, 2.0213e-09, 1.2308e-09, 8.9605e-10, 7.9322e-10,\n 2.7286e-09, 1.9500e-09, 1.3433e-09, 1.6249e-09, 1.6056e-09, 1.0346e-08,\n 2.9821e-09, 1.0196e-09, 1.0752e-09, 3.4015e-09, 1.8471e-09, 9.2046e-10,\n 3.6992e-09, 1.4523e-09], device='cuda:0')" + }, + "32": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-3.1850e-07, 5.7439e-07, -3.4244e-08, ..., -5.8710e-08,\n -9.6470e-08, -5.4412e-08],\n [ 5.7016e-07, -9.3643e-07, -1.7977e-08, ..., 1.7288e-09,\n 2.0540e-07, 1.4000e-07],\n [ 4.0108e-07, -6.7456e-07, -1.0061e-07, ..., -1.9561e-07,\n -8.4610e-08, -8.9876e-09],\n ...,\n [-3.6093e-07, 2.9989e-07, -4.0628e-08, ..., 1.9005e-08,\n -9.1029e-08, -1.6310e-07],\n [-1.3158e-08, 1.8036e-08, -5.5018e-08, ..., -9.5296e-08,\n -5.2678e-08, -6.4878e-08],\n [ 5.2528e-07, -9.0726e-07, 3.3198e-07, ..., 1.0530e-07,\n 5.9511e-07, 3.7157e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.7208e-12, 2.7342e-12, 4.0522e-13, ..., 5.2740e-13, 6.9732e-13,\n 4.5474e-13],\n [5.7857e-12, 6.7782e-12, 9.6534e-13, ..., 1.3517e-12, 1.4267e-12,\n 1.0924e-12],\n [4.1183e-12, 4.3392e-12, 6.5460e-13, ..., 9.5021e-13, 1.1519e-12,\n 7.3301e-13],\n ...,\n [2.6442e-12, 2.4571e-12, 3.8341e-13, ..., 4.9163e-13, 5.9603e-13,\n 4.0467e-13],\n [8.3343e-13, 7.4019e-13, 1.3357e-13, ..., 1.8726e-13, 2.0374e-13,\n 1.3735e-13],\n [1.0752e-11, 9.5541e-12, 1.5623e-12, ..., 2.1538e-12, 2.5306e-12,\n 1.6974e-12]], device='cuda:0')" + }, + "33": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-7.4875e-07, 1.1912e-05, 2.8351e-06, -1.8455e-06, -9.6374e-06,\n -1.9488e-05, 6.1232e-06, -1.4380e-05, 1.9367e-05, -9.9119e-07,\n -6.7563e-06, -2.3417e-06, 8.7615e-06, 1.6985e-05, 1.1402e-05,\n 5.9434e-06, 2.0781e-05, -1.4785e-05, 3.7910e-06, -1.4085e-05,\n -1.0101e-05, 9.0798e-06, 1.6292e-05, -7.7492e-06, 8.6701e-06,\n -2.1881e-05, -4.0908e-06, -6.6181e-06, 4.0211e-06, 1.4182e-05,\n 5.0277e-06, 5.4812e-06, 1.9753e-06, 4.8155e-06, -2.3113e-05,\n 2.5443e-05, 1.0481e-05, -1.2827e-05, -7.3225e-06, -4.0169e-06,\n 1.3080e-05, -1.3243e-05, 4.2880e-06, -1.0446e-05, 1.4423e-05,\n -1.4886e-05, 6.0831e-06, 1.6316e-05, -1.3917e-05, -1.0229e-05,\n -1.5777e-05, -1.9684e-05, -9.6406e-06, -1.5491e-05, -8.1743e-06,\n -4.0065e-06, -1.0946e-06, 2.1820e-05, -2.4907e-05, -8.5432e-06,\n -1.0402e-05, 2.0798e-05, -2.2770e-06, -7.3006e-06, 2.6547e-05,\n 4.3745e-06, -1.4983e-06, -2.0519e-06, -1.6578e-05, 1.0796e-05,\n 2.2641e-05, -4.4969e-06, 3.5674e-07, 1.8561e-06, 2.1743e-05,\n -1.7417e-05, -6.5148e-06, 9.4538e-06, -2.3757e-05, -1.1818e-05,\n -1.2913e-05, 7.1401e-06, 2.6853e-07, 1.9109e-05, 8.6154e-06,\n -2.1561e-05, -5.9289e-06, 6.1494e-06, -9.9933e-06, 9.9289e-06,\n 6.2179e-06, -4.9378e-06, 5.3709e-06, 1.5456e-05, 2.5439e-05,\n 4.0707e-06, 1.3062e-05, 4.8401e-07, -1.3627e-05, 1.2081e-05,\n 2.5691e-05, 1.4987e-05, -4.8326e-06, -8.6370e-06, 8.9147e-07,\n 2.0502e-05, -6.1904e-06, 6.3173e-06, -9.3734e-06, -1.0971e-05,\n -1.8584e-05, 3.9502e-06, 6.3690e-06, 8.7739e-06, 8.3271e-06,\n -1.1541e-05, -7.8058e-06, -2.5022e-05, 2.0330e-06, 7.6578e-06,\n 1.1369e-05, 8.1149e-06, -1.7006e-06, 2.3748e-05, -2.4791e-05,\n -1.0916e-05, 5.0736e-06, 2.9765e-06, 3.1119e-07, 4.5806e-06,\n -3.8102e-06, 4.7589e-06, -4.6228e-07, 2.8289e-06, -3.4005e-06,\n -6.1501e-06, 4.6970e-06, 4.8042e-06, 2.7663e-06, -9.8840e-07,\n -1.0645e-06, 1.7228e-06, -8.5992e-06, 4.3670e-06, 1.3297e-06,\n -7.4485e-07, 4.2980e-07, 6.6033e-06, 1.0404e-06, -4.7945e-06,\n 5.0312e-06, 9.9536e-06, 2.6900e-06, -4.2903e-06, -2.2253e-06,\n -7.1429e-07, 4.3237e-06, 2.6483e-06, -1.9731e-06, 4.0958e-06,\n -3.4235e-07, 4.0388e-07, -7.4683e-07, 2.3865e-06, 6.4664e-06,\n 3.8959e-07, 2.2775e-06, 2.5515e-07, -4.5503e-06, 8.2875e-07,\n 1.3024e-06, 7.7934e-08, -2.1446e-06, 2.2646e-06, 1.6565e-07,\n -1.1029e-06, 5.9728e-06, 3.0732e-06, -3.7101e-06, -1.4240e-06,\n -2.8455e-07, 4.5054e-06, 6.9737e-06, -4.7046e-06, -4.7631e-06,\n -1.5244e-06, -1.0140e-06, -4.5517e-06, 2.6387e-06, -3.5316e-06,\n 5.5845e-06, -6.5611e-06, 1.3609e-06, -2.8525e-06, -4.9597e-06,\n -1.6893e-06, 3.5423e-06, -5.1575e-06, -4.1545e-06, 2.7063e-06,\n -2.3536e-06, -2.7909e-07, 5.3598e-06, 5.3027e-06, -1.3710e-06,\n -1.6423e-06, -4.7041e-06, -3.2661e-06, 1.2368e-06, 2.9156e-06,\n 3.5378e-07, 3.3205e-06, 5.9047e-06, -4.0394e-06, 1.6701e-06,\n 2.2165e-06, 5.0008e-06, -2.0839e-07, 3.3467e-06, -5.8035e-06,\n -6.1446e-06, 5.0467e-06, -4.2441e-06, 2.2414e-06, -4.4621e-06,\n -2.2442e-06, -2.4044e-07, -2.7006e-06, -8.9252e-06, 1.6833e-06,\n -2.5492e-06, 4.5683e-07, -2.7769e-06, 3.0301e-06, 2.5590e-06,\n -2.5153e-06, 2.6558e-06, 7.4942e-06, 3.3201e-06, 2.1428e-06,\n 3.6820e-06, -4.4602e-06, -2.3747e-06, 4.3429e-06, 3.2498e-06,\n 4.0553e-06, 2.9205e-06, -1.0656e-06, -3.9640e-06, 1.4039e-06,\n -3.8484e-06, 3.2542e-06, -9.1155e-07, -4.7854e-06, -5.0983e-08,\n -4.0932e-06, 2.4922e-05, 3.4967e-06, -1.0964e-06, -1.4957e-05,\n 2.5313e-06, -3.0754e-06, -2.8348e-06, 3.7648e-06, -5.9979e-06,\n 9.4722e-06, 1.3880e-07, -1.2522e-06, -1.2141e-06, 1.0533e-05,\n 1.0129e-05, -7.2396e-06, -7.4327e-06, -3.9799e-07, 2.5241e-06,\n 1.1949e-05, -4.2867e-06, 1.1494e-05, -8.3063e-06, -5.1797e-06,\n -5.8159e-06, 1.0240e-05, -1.7093e-07, 8.5322e-07, -4.1882e-06,\n -5.9362e-06, 1.9055e-05, -4.7255e-06, 1.0291e-05, -8.3621e-06,\n -8.3654e-06, -1.4653e-05, 4.3683e-06, -5.8478e-06, 1.9222e-05,\n 4.1939e-06, 2.1757e-06, 1.0959e-06, 6.4287e-06, 2.0001e-06,\n -3.5733e-06, 5.3492e-06, -5.2246e-06, 1.0494e-06, 6.0855e-06,\n 1.8341e-05, -1.1320e-05, -1.9919e-06, 7.4037e-06, -1.1304e-06,\n -1.0457e-05, 1.6973e-05, -8.6492e-06, -7.5762e-06, -1.9728e-05,\n -9.1421e-06, -3.2016e-06, 2.4986e-06, -7.2078e-06, 1.7322e-05,\n 1.0550e-05, -3.2002e-06, -1.5797e-06, -5.5582e-07, 1.6504e-06,\n 2.4713e-07, -4.5150e-07, 6.1631e-06, 9.2274e-06, -6.5139e-06,\n 4.1000e-06, -1.4551e-05, 9.4909e-07, 1.1968e-05, -7.9807e-06,\n -6.1060e-06, -1.1165e-05, -5.1033e-06, -4.2404e-06, 8.7057e-06,\n 6.6043e-06, -2.8110e-06, -8.7276e-06, -1.6393e-05, 2.3556e-06,\n 4.8250e-06, -4.6277e-06, -3.6161e-07, -5.9663e-06, 1.7513e-06,\n -1.1688e-06, -1.2842e-05, -5.1788e-06, -4.7008e-06, -5.0370e-06,\n -5.5817e-06, -3.8133e-06, 8.5663e-06, 8.6997e-06, -1.3186e-05,\n 2.5148e-06, 8.8664e-06, -5.8977e-06, -1.0065e-05, 1.9931e-05,\n 6.2885e-06, 1.2515e-05, -1.9014e-05, 1.4241e-06, 2.0906e-05,\n 3.9618e-06, 1.2816e-06, 7.8119e-06, -3.2635e-06, 8.3883e-07,\n 4.3577e-06, 2.0677e-05, -5.7744e-06, 2.3475e-05, -2.8232e-06,\n 9.7028e-07, -1.4845e-05, -7.1082e-06, 3.0342e-06, 1.0783e-07,\n -1.2409e-05, -7.9197e-06, -9.9508e-06, 9.9209e-06, 9.7384e-06,\n 4.1280e-06, 1.3715e-05, 1.3854e-05, 8.3578e-06, 6.0449e-06,\n 9.4910e-06, 4.8820e-06, -7.2316e-06, -3.0258e-06, 4.9532e-06,\n -2.0759e-07, -8.6899e-09, 5.3074e-06, -6.0939e-06, 4.1752e-06,\n 3.7959e-06, 4.1925e-06, 3.1601e-06, -7.0302e-06, 3.4783e-06,\n 7.6286e-06, 5.8349e-08, -8.3885e-06, -1.6338e-06, -1.6865e-06,\n -4.0213e-06, 7.9430e-06, -8.7435e-06, 1.2045e-06, 9.5734e-06,\n 8.2737e-06, 1.7568e-06, 7.7757e-06, 3.9589e-06, 9.1490e-06,\n -7.6374e-06, -5.0074e-06, 1.0040e-05, 1.2871e-05, -3.0615e-06,\n 7.0069e-06, -1.7207e-06, 5.2783e-06, -5.2864e-06, 1.0428e-05,\n -1.1138e-05, -7.1667e-06, 7.5761e-06, 1.1915e-05, -9.0277e-06,\n 1.0280e-05, 6.2234e-06, -1.0538e-05, 1.4836e-05, -3.4914e-06,\n 8.9567e-06, 1.0074e-05, 7.1457e-06, -7.8308e-06, -9.8337e-06,\n 9.8663e-07, -1.3899e-05, 1.7026e-06, 1.7043e-06, -6.8871e-06,\n -8.1367e-07, -7.4765e-06, -1.1392e-05, 4.4950e-06, 3.6636e-07,\n 1.5273e-06, 8.5281e-06, -1.4737e-05, -1.0699e-05, 5.7910e-06,\n -2.3048e-06, -1.1506e-05, 1.0788e-05, -1.6825e-06, 1.0962e-05,\n -8.0038e-06, 2.1387e-06, -6.9735e-06, -1.4039e-06, -3.4149e-06,\n 8.4505e-07, -1.2566e-05, -3.3922e-06, 9.8366e-06, 1.1010e-05,\n -1.0388e-05, 4.5540e-06, 7.1107e-06, -3.8566e-06, -1.2213e-05,\n -6.0700e-06, -5.2967e-06, 7.6847e-07, -4.5652e-06, 4.5131e-06,\n -6.7989e-07, -5.6431e-06, -1.2151e-05, -5.7601e-06, -6.3313e-06,\n -1.0951e-05, 8.4508e-06, -3.8312e-06, -8.3581e-06, 5.4108e-07,\n 4.6663e-06, 1.0787e-06, -6.9419e-06, 3.6581e-06, -6.9338e-06,\n 8.0257e-06, 2.4990e-06, 3.4614e-06, 2.4446e-06, -7.1655e-07,\n 7.4931e-07, -7.6934e-06], device='cuda:0')", + "exp_avg_sq": "tensor([7.7452e-10, 1.7621e-09, 1.2717e-09, 6.9625e-10, 1.8252e-09, 3.1341e-09,\n 3.4774e-09, 2.3946e-09, 3.7967e-09, 9.5857e-10, 1.3652e-09, 8.1130e-10,\n 1.0780e-09, 2.7297e-09, 1.5043e-09, 4.2401e-10, 3.0904e-09, 2.4348e-09,\n 5.3717e-10, 2.0401e-09, 1.0099e-09, 7.6441e-10, 2.2378e-09, 9.7843e-10,\n 1.1803e-09, 2.6279e-09, 2.3799e-09, 6.5697e-10, 3.6591e-10, 1.9847e-09,\n 2.0506e-09, 3.1660e-10, 5.5163e-10, 9.1641e-10, 4.9100e-09, 3.5772e-09,\n 9.4296e-10, 1.3907e-09, 1.1773e-09, 1.6628e-10, 2.0734e-09, 3.0359e-09,\n 9.0551e-10, 1.5825e-09, 1.4167e-09, 2.5477e-09, 4.4997e-10, 2.5762e-09,\n 2.1196e-09, 1.0100e-09, 1.7547e-09, 3.0636e-09, 1.8904e-09, 2.3416e-09,\n 1.0024e-09, 7.2838e-10, 1.7779e-10, 2.8630e-09, 4.7561e-09, 6.4999e-10,\n 1.3608e-09, 3.8426e-09, 5.3489e-10, 6.8959e-10, 5.3129e-09, 9.2928e-10,\n 2.5058e-10, 7.3380e-10, 2.6894e-09, 1.3310e-09, 5.5355e-09, 3.1225e-10,\n 8.4642e-10, 8.0165e-10, 3.1536e-09, 3.2601e-09, 1.1269e-09, 1.2852e-09,\n 3.9593e-09, 1.2963e-09, 2.1836e-09, 4.1491e-10, 9.5580e-10, 2.9871e-09,\n 4.3014e-10, 2.6967e-09, 2.6374e-10, 1.6675e-09, 6.0185e-10, 1.2027e-09,\n 6.9478e-10, 5.2425e-10, 2.4074e-10, 3.4142e-09, 4.9318e-09, 2.8030e-10,\n 1.9072e-09, 9.9661e-10, 2.4486e-09, 1.1697e-09, 3.8043e-09, 2.2772e-09,\n 3.6250e-10, 1.5493e-09, 4.1323e-10, 3.7898e-09, 3.8561e-10, 5.9920e-10,\n 1.5588e-09, 1.3151e-09, 4.2336e-09, 6.7799e-10, 9.4581e-10, 7.2431e-10,\n 5.6615e-10, 1.8562e-09, 1.2798e-09, 3.3968e-09, 9.2393e-10, 1.3373e-09,\n 1.0630e-09, 7.2904e-10, 3.3944e-10, 4.2869e-09, 4.2642e-09, 1.7733e-09,\n 3.3515e-10, 6.7219e-10, 8.5214e-10, 1.3318e-09, 1.0514e-09, 2.5781e-09,\n 1.0133e-09, 1.6367e-09, 7.9712e-10, 2.6514e-09, 1.3021e-09, 9.5426e-10,\n 1.1151e-09, 7.5122e-10, 1.4871e-09, 1.1626e-09, 2.1269e-09, 3.0828e-09,\n 2.8060e-10, 1.0318e-09, 7.2922e-10, 2.8612e-09, 6.9638e-10, 7.1723e-10,\n 1.2426e-09, 2.1724e-09, 1.4873e-09, 1.6170e-09, 1.5032e-09, 6.5390e-10,\n 2.7487e-09, 1.4996e-09, 1.1762e-09, 2.9015e-09, 7.8995e-10, 1.2456e-09,\n 6.8579e-10, 1.1309e-09, 2.0189e-09, 8.3729e-10, 6.6922e-10, 8.7154e-10,\n 2.4803e-09, 1.3085e-09, 1.4837e-09, 1.4462e-09, 4.4285e-10, 1.5046e-09,\n 7.3925e-10, 6.2126e-10, 3.5785e-09, 4.1768e-10, 8.2554e-10, 5.4872e-10,\n 5.5864e-10, 2.4742e-09, 2.5804e-09, 8.5077e-10, 6.5997e-10, 2.2055e-10,\n 6.4286e-10, 1.6143e-09, 8.1098e-10, 1.2789e-09, 1.8416e-09, 2.5034e-09,\n 6.5790e-10, 1.8702e-09, 2.2231e-09, 5.3593e-10, 2.9574e-09, 2.7203e-09,\n 1.2685e-09, 1.4478e-09, 1.7347e-09, 4.1364e-10, 2.1602e-09, 1.4611e-09,\n 5.9100e-10, 5.5668e-10, 1.0073e-09, 9.3604e-10, 4.9166e-10, 1.4771e-09,\n 2.2707e-10, 2.7063e-09, 3.0909e-09, 2.4833e-09, 5.6974e-10, 7.0940e-10,\n 5.9236e-10, 1.3437e-10, 2.9048e-09, 4.3273e-09, 1.5937e-09, 2.6358e-09,\n 2.9112e-09, 6.5972e-10, 4.4322e-10, 9.0680e-10, 3.9022e-10, 1.1409e-09,\n 2.8291e-09, 4.2386e-10, 2.8385e-09, 4.8814e-10, 2.0409e-09, 7.4913e-10,\n 1.4814e-09, 9.0583e-10, 4.1277e-10, 2.6267e-09, 2.9840e-09, 8.8485e-10,\n 1.2848e-09, 2.0086e-09, 1.7599e-09, 1.9837e-09, 8.8009e-10, 1.8015e-09,\n 1.8604e-09, 1.1435e-09, 1.6938e-09, 3.8497e-10, 1.7481e-09, 6.4795e-10,\n 2.0112e-09, 5.9007e-10, 6.7899e-10, 7.0162e-10, 3.0458e-09, 6.3902e-10,\n 8.4952e-10, 1.8527e-09, 2.3301e-10, 3.8413e-10, 8.9582e-10, 8.2162e-10,\n 1.8221e-09, 1.6325e-09, 3.8800e-10, 9.1018e-10, 3.5317e-10, 1.5806e-09,\n 1.5462e-09, 1.2530e-09, 1.1489e-09, 5.4843e-10, 2.2626e-10, 1.4088e-09,\n 9.3262e-10, 1.1487e-09, 1.0916e-09, 1.8713e-09, 1.5999e-09, 1.0448e-09,\n 1.7065e-10, 4.1266e-10, 4.1715e-10, 1.3786e-09, 2.0955e-09, 5.2381e-10,\n 7.7588e-10, 1.0312e-09, 1.6829e-09, 1.2320e-09, 2.8138e-10, 9.4101e-10,\n 1.6490e-09, 3.6851e-10, 6.1561e-10, 2.3287e-10, 2.7643e-10, 6.0009e-10,\n 8.7508e-10, 7.5912e-10, 9.5041e-10, 6.4122e-10, 6.1280e-10, 2.0398e-09,\n 1.4077e-09, 1.1224e-09, 1.0487e-09, 2.8286e-10, 5.1267e-10, 1.8016e-09,\n 5.3212e-10, 4.7974e-10, 2.1783e-09, 1.5673e-09, 3.8907e-10, 3.9991e-10,\n 1.6089e-09, 2.0884e-09, 1.1822e-09, 1.5915e-10, 1.1462e-09, 3.2276e-10,\n 6.3186e-10, 5.4849e-10, 2.6534e-10, 7.0905e-10, 1.3012e-09, 5.4847e-10,\n 8.4472e-10, 2.1208e-09, 7.6613e-10, 1.0734e-09, 1.4661e-09, 7.1500e-10,\n 1.9418e-09, 1.2299e-09, 3.7921e-10, 1.3610e-09, 1.3190e-09, 1.2416e-09,\n 8.6894e-10, 1.8828e-09, 1.3203e-09, 4.7457e-10, 1.8762e-09, 9.2571e-10,\n 1.1037e-09, 6.7097e-10, 2.3347e-10, 1.1176e-09, 5.7070e-10, 5.3905e-10,\n 1.6646e-09, 8.7204e-10, 7.1837e-10, 1.5399e-09, 1.3185e-09, 8.7598e-10,\n 7.2551e-10, 1.6074e-09, 3.8918e-10, 2.3280e-09, 2.2786e-09, 7.6509e-10,\n 2.2292e-09, 2.3388e-09, 7.7933e-10, 2.7483e-09, 5.3851e-10, 1.5835e-09,\n 5.1987e-10, 4.2765e-10, 1.0041e-09, 1.1708e-09, 2.6938e-09, 2.0748e-09,\n 2.7053e-09, 7.0136e-10, 2.2777e-09, 1.2057e-09, 5.6866e-10, 3.4909e-10,\n 2.8799e-10, 1.4008e-09, 1.2954e-09, 6.5024e-10, 8.7392e-10, 1.8473e-09,\n 2.2874e-10, 2.8690e-09, 1.3005e-09, 9.2749e-10, 3.2240e-10, 2.3962e-09,\n 4.4251e-10, 9.0268e-10, 9.9684e-10, 1.1789e-09, 9.9633e-10, 1.7755e-10,\n 2.8446e-10, 5.5609e-10, 1.3293e-09, 9.3096e-10, 4.4293e-10, 3.3989e-10,\n 1.1367e-09, 4.3192e-10, 9.3111e-10, 1.1372e-10, 8.6386e-10, 6.1010e-10,\n 8.4518e-10, 3.2008e-10, 6.2047e-10, 5.5755e-10, 5.3552e-10, 9.8686e-10,\n 1.3487e-09, 3.3793e-10, 6.7405e-10, 4.5088e-10, 6.2696e-10, 4.7665e-09,\n 6.2941e-10, 7.6737e-10, 1.7330e-09, 7.3693e-10, 1.0946e-09, 3.2721e-10,\n 8.2542e-10, 9.3242e-10, 1.0982e-09, 9.9253e-10, 5.1073e-09, 1.9093e-09,\n 2.0766e-09, 2.8052e-09, 1.7412e-09, 4.7154e-10, 1.4898e-09, 1.5619e-09,\n 3.7881e-10, 6.5243e-09, 4.9055e-09, 1.4880e-09, 1.3923e-09, 8.3282e-10,\n 2.2904e-10, 1.7452e-09, 4.5467e-10, 5.6121e-10, 9.9872e-10, 7.9548e-10,\n 1.0223e-09, 8.5965e-10, 3.3588e-10, 9.5224e-10, 6.8336e-10, 1.5201e-09,\n 2.2066e-09, 2.9336e-09, 3.2727e-10, 2.1522e-09, 2.9033e-09, 1.1810e-09,\n 6.1529e-10, 1.5838e-09, 2.2506e-09, 5.9502e-10, 9.9337e-10, 1.2767e-09,\n 5.4538e-10, 7.5818e-10, 5.0323e-09, 2.7094e-10, 2.6985e-09, 1.2347e-09,\n 1.4424e-09, 1.4396e-09, 9.4164e-10, 1.3845e-09, 4.0105e-09, 7.1931e-10,\n 6.2306e-10, 3.9652e-10, 1.2665e-09, 6.2311e-10, 4.0212e-10, 6.9481e-10,\n 3.1552e-09, 5.5699e-10, 9.2404e-10, 1.3140e-09, 8.2872e-10, 4.9022e-10,\n 2.5103e-09, 1.8159e-09, 7.9546e-10, 3.5991e-10, 7.3365e-10, 1.0113e-09,\n 8.3755e-10, 7.7503e-10, 3.1192e-10, 2.1142e-09, 2.1886e-10, 6.9459e-10,\n 2.3033e-10, 2.8264e-09], device='cuda:0')" + }, + "34": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[-2.0222e-07, -8.5402e-08, 1.8041e-07, ..., 1.9843e-08,\n -3.6483e-07, 1.1091e-07],\n [ 6.9998e-09, 1.5792e-07, 6.8100e-08, ..., 8.7619e-09,\n 1.9892e-07, 6.2535e-08],\n [ 2.3905e-08, 6.2830e-08, -3.0449e-08, ..., -7.1235e-08,\n 2.3807e-07, -9.4597e-08],\n ...,\n [ 1.2734e-08, -3.6170e-08, 1.3500e-08, ..., -1.5624e-07,\n 1.2328e-07, -9.0109e-08],\n [ 3.6320e-08, -2.0564e-08, -7.4204e-08, ..., -6.3295e-08,\n -1.1969e-08, -7.8466e-08],\n [-9.0531e-08, -5.3652e-08, -6.6287e-08, ..., -1.1148e-07,\n 4.6953e-08, 1.4598e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.0237e-13, 1.8694e-13, 1.1290e-13, ..., 1.1954e-13, 2.6410e-13,\n 1.2350e-13],\n [2.6028e-13, 2.2526e-13, 1.3545e-13, ..., 1.3606e-13, 2.8518e-13,\n 1.4497e-13],\n [2.3403e-13, 2.0729e-13, 1.2373e-13, ..., 1.2940e-13, 2.9693e-13,\n 1.4579e-13],\n ...,\n [2.2443e-13, 2.3549e-13, 1.1865e-13, ..., 1.3818e-13, 2.7901e-13,\n 1.4519e-13],\n [1.5483e-13, 1.5640e-13, 9.1470e-14, ..., 1.0345e-13, 2.1997e-13,\n 1.1575e-13],\n [8.5757e-13, 8.3737e-13, 4.2881e-13, ..., 5.5611e-13, 1.1436e-12,\n 5.8743e-13]], device='cuda:0')" + }, + "35": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 1.2831e-12, -1.3012e-12, 7.0086e-13, 2.3103e-12, -2.7922e-12,\n 7.8804e-13, 1.9391e-12, -2.2045e-12, 1.3643e-12, -1.8217e-12,\n -7.1867e-13, 2.3787e-12, -1.5472e-13, -3.6211e-12, 9.6485e-13,\n -8.3877e-13, -2.0553e-12, -1.6428e-12, -1.1673e-12, 3.3301e-12,\n 2.3888e-13, -7.5802e-13, -4.2871e-12, -1.3736e-12, 4.9755e-14,\n 4.5120e-12, -1.0257e-12, 8.6149e-13, -1.4623e-12, 6.0197e-13,\n 1.5744e-12, 8.6723e-13, 3.9669e-14, 1.9024e-12, 1.2436e-12,\n -1.4268e-12, -8.8870e-13, 1.5033e-12, -3.5204e-14, -4.3893e-13,\n 7.5391e-13, 3.1712e-12, 7.3902e-13, 1.0614e-12, -2.1380e-12,\n -1.6463e-12, -2.4505e-12, 1.6749e-13, 3.2247e-12, -3.1365e-13,\n -2.1692e-12, -4.5167e-13, 1.4167e-13, -3.8244e-13, -5.9135e-13,\n -3.4359e-13, 2.2647e-12, 4.5204e-13, -1.1644e-12, 2.3247e-12,\n 8.8945e-13, -4.7488e-12, 1.6825e-12, -1.4206e-13, 9.0253e-13,\n -6.3063e-13, 1.2668e-13, -2.3661e-12, 8.9752e-13, -1.8184e-13,\n 3.0794e-12, 2.9486e-12, 2.9038e-12, -2.3945e-13, -1.5011e-12,\n 3.2439e-12, 1.8891e-13, -1.7127e-12, 1.8965e-12, 2.7883e-13,\n 1.8707e-12, -2.8903e-13, 2.1900e-12, -9.5030e-12, -3.6088e-12,\n 1.5335e-12, 1.7708e-12, -4.0349e-12, 5.4778e-13, 2.5656e-12,\n -2.1042e-12, 1.2756e-12, -1.2208e-12, -8.5971e-13, -2.2351e-12,\n -1.2830e-12, 4.5746e-13, -3.7898e-12, -1.6684e-13, -3.9545e-12,\n -1.4241e-12, -2.3791e-12, 8.4515e-14, 6.0237e-13, 8.2807e-13,\n -2.1995e-12, 3.4184e-12, -2.5124e-12, -4.8267e-13, -1.9554e-12,\n -2.8119e-12, -1.8062e-12, 1.2593e-12, 1.1661e-12, -2.6507e-12,\n -1.6778e-12, 3.4223e-13, 2.2848e-12, -9.3844e-13, 4.5369e-13,\n -4.7624e-12, 9.0569e-14, 3.9872e-13, -5.2902e-12, 1.9520e-12,\n -9.1319e-13, -6.8890e-13, 4.1075e-13, -4.3225e-12, 3.0498e-12,\n -3.4805e-12, 5.3436e-12, -1.7698e-12, 2.5799e-12, -3.2720e-12,\n -1.3895e-11, 4.9195e-12, 4.6835e-12, 4.0981e-12, -5.0206e-13,\n -2.0404e-12, 3.5734e-12, -4.9667e-12, 5.6639e-12, 3.5088e-12,\n -1.7188e-12, -4.0489e-12, 8.3491e-12, -6.0526e-12, -7.6860e-13,\n -3.6625e-13, 4.1748e-12, 4.9670e-12, -3.8744e-12, -6.2701e-12,\n -1.8294e-12, 4.2329e-12, 4.3187e-12, -4.0204e-12, 7.6436e-12,\n -3.3536e-12, 3.8143e-12, 5.0785e-12, 4.8527e-12, 3.1445e-12,\n 3.0655e-12, 4.7627e-12, 3.8398e-12, -4.0990e-12, 7.5606e-12,\n 3.2232e-12, -3.8375e-12, -2.3501e-12, 3.8382e-12, 4.8012e-12,\n 7.0682e-13, 5.7622e-12, 4.0140e-13, -2.6460e-12, 1.3778e-12,\n -3.5104e-12, -2.3579e-13, 5.4343e-12, -2.3887e-12, 1.6617e-12,\n -1.5598e-12, 4.6530e-13, -3.8609e-12, 1.9960e-12, -4.6140e-12,\n 3.7262e-12, -3.6877e-12, -1.0013e-13, -4.6032e-12, -4.5258e-12,\n -1.5675e-12, 3.0215e-12, -5.8139e-12, -2.4895e-12, 4.2469e-12,\n -4.8164e-12, 3.9378e-12, 4.2748e-12, 1.6050e-12, -1.3393e-12,\n -1.9436e-12, -2.5046e-12, -1.3262e-12, 3.0731e-12, 4.4885e-12,\n -3.1004e-12, 7.3737e-12, 5.3924e-12, -6.9160e-12, 3.2708e-12,\n 1.5859e-12, 1.0080e-12, 1.1199e-12, 4.9961e-12, -1.2345e-11,\n -5.1427e-12, 6.6655e-12, -5.6274e-12, 1.7145e-12, -2.0958e-12,\n -1.8863e-12, -1.6009e-12, -2.8464e-12, -3.9477e-12, -2.1975e-13,\n -5.1530e-12, -4.2506e-12, -5.4812e-12, 4.2205e-12, 2.8341e-12,\n 3.7380e-13, -1.4556e-12, 5.1952e-12, 4.0312e-12, 5.5292e-12,\n 4.0211e-12, -5.9489e-12, -6.2137e-12, 5.2796e-12, 4.0538e-12,\n 6.6584e-12, 4.5058e-12, -2.6112e-12, -6.4330e-12, 3.3950e-12,\n -6.0120e-12, 2.1421e-12, 4.3333e-12, 9.3889e-13, -7.3316e-13,\n 1.9408e-12, -6.1220e-12, -2.4905e-13, -1.2366e-12, 1.0202e-11,\n 3.6348e-13, -5.8463e-15, 3.2795e-12, -2.0215e-12, 9.6320e-12,\n -1.0077e-11, -4.4561e-12, -5.7278e-13, 2.1955e-12, -5.7367e-12,\n -2.6027e-12, 6.3607e-13, -2.6257e-13, -2.3682e-12, 1.5161e-13,\n -6.1612e-12, 3.1837e-12, -2.7180e-12, 6.0755e-12, 2.6267e-12,\n 7.3177e-12, -6.5599e-13, 4.2603e-13, -2.0045e-12, 2.0041e-12,\n 4.9571e-12, -8.2728e-12, 2.2517e-12, -3.7539e-12, 4.0010e-12,\n 4.8185e-12, 9.6574e-12, -1.2362e-12, 5.7507e-12, -7.2628e-12,\n -4.0705e-12, -2.3170e-13, -9.0195e-13, 6.3878e-13, -1.4448e-12,\n 2.4659e-12, -3.1995e-12, 2.4206e-12, 1.4607e-12, -3.9336e-12,\n -5.7615e-12, 3.5727e-12, 2.6216e-12, -4.6218e-12, -4.8764e-12,\n 1.3673e-12, -1.2565e-11, 8.5108e-13, 4.9814e-13, 1.0001e-11,\n 8.9195e-12, 1.5076e-12, 3.4226e-12, 2.8263e-12, -1.6208e-12,\n -4.5267e-12, -1.4952e-12, 2.9329e-12, -4.3386e-13, 8.3013e-14,\n -5.9292e-13, 1.1921e-12, -3.5284e-12, -1.0639e-12, 3.1676e-12,\n -4.1094e-12, 3.3175e-12, 8.0563e-13, -2.5833e-12, 3.6827e-12,\n -4.5603e-12, 6.9969e-12, -4.2953e-13, -1.4323e-12, -5.5104e-12,\n -8.4272e-12, 5.7419e-12, 2.2989e-12, 4.5941e-12, -2.2622e-12,\n -4.0741e-12, 2.8537e-12, -2.7053e-12, 5.7880e-12, 2.0607e-13,\n 1.4429e-13, 2.5522e-12, 7.5086e-13, 1.9076e-12, 6.6545e-12,\n 2.5481e-12, 1.7794e-12, -2.8134e-12, -8.7089e-12, 1.9132e-12,\n 2.2507e-12, -3.5973e-12, -2.5590e-13, 8.5290e-12, -9.8876e-12,\n -1.1459e-12, -1.1642e-11, 1.4321e-11, -1.4617e-12, -1.0233e-11,\n -4.5448e-13, -1.8578e-12, 2.6855e-13, 3.9769e-13, -8.7971e-13,\n -5.0345e-12, -8.6702e-12, 6.0694e-12, -2.4441e-12, 1.3748e-12,\n -3.5153e-12, 1.8558e-12, 1.5420e-12, -3.4086e-12, -2.8828e-12,\n 2.8496e-12, -2.0148e-12, -1.5675e-13, -1.6678e-12, -3.8843e-12,\n -2.2714e-12, 2.0277e-12, -2.2845e-13, -2.4749e-12, -9.1608e-13,\n -7.6561e-13, 2.4283e-12, -4.5628e-13, 1.7753e-12, -3.5747e-12,\n -1.4312e-12, -1.2695e-12, 4.2956e-13, -4.8182e-13, 2.2117e-13,\n -3.2961e-12, 1.6367e-13, -1.2900e-12, -1.4446e-12, 1.4437e-13,\n -3.1665e-12, 6.9042e-13, -9.4483e-13, -2.0647e-12, -2.6882e-13,\n 3.3284e-14, -6.2038e-13, -8.2338e-13, 1.7269e-12, -2.7611e-13,\n 1.0716e-12, -1.9612e-12, -2.0915e-12, 3.2115e-12, 8.1463e-13,\n 4.6331e-12, -8.0462e-13, -5.7485e-12, -1.8988e-12, 3.1485e-12,\n -1.6795e-12, -4.1286e-13, -1.6934e-12, 1.6713e-12, -9.4288e-13,\n -2.2440e-12, 3.9204e-12, -3.0365e-12, -6.3733e-13, 2.4314e-12,\n -4.7650e-12, -1.1368e-12, 2.0267e-12, -2.6154e-12, 1.1548e-12,\n -8.9274e-12, -4.1351e-12, -2.2653e-12, -1.2815e-13, 1.9132e-12,\n -1.2150e-12, 5.1663e-12, -6.0073e-13, 1.5275e-12, 1.8552e-12,\n -2.7583e-13, 2.2774e-12, 1.0568e-12, -6.3564e-13, -4.3348e-13,\n -2.0680e-12, -3.7673e-12, 1.7327e-12, 5.4731e-12, 2.5321e-12,\n 1.4576e-12, -6.9290e-13, -7.2006e-13, -3.8948e-13, 1.9558e-12,\n 5.4357e-12, 7.1051e-13, 9.5857e-13, -1.6952e-14, 1.0107e-12,\n 2.1221e-12, 7.7738e-12, 9.4039e-13, -2.2395e-12, -1.2216e-12,\n -2.2788e-13, -1.1315e-12, -1.1984e-12, 2.3749e-12, -2.9478e-12,\n 1.8224e-12, 4.4359e-13, 8.5118e-13, -2.8726e-12, 1.9014e-12,\n 2.5071e-12, 1.8756e-12, 7.6015e-12, 4.1663e-13, 2.4085e-12,\n 3.8884e-12, -6.0662e-12, -9.5933e-13, -1.6347e-12, 2.5418e-13,\n -3.3159e-13, 3.4625e-12, 1.9130e-12, 2.7363e-12, 7.0337e-14,\n -1.6600e-12, -1.4410e-12, -2.1433e-12, 1.4707e-12, 3.8346e-13,\n -4.1132e-13, 3.1391e-12], device='cuda:0')", + "exp_avg_sq": "tensor([5.0218e-23, 6.3690e-23, 5.7836e-23, 5.9482e-23, 1.1182e-22, 2.7799e-22,\n 5.5631e-23, 2.7996e-22, 4.5540e-22, 7.8820e-23, 6.4022e-23, 8.3739e-23,\n 5.3723e-23, 2.2670e-22, 1.0816e-22, 5.2177e-23, 2.0010e-22, 3.7467e-22,\n 4.2005e-23, 1.2202e-22, 7.5160e-23, 7.9544e-23, 1.4454e-22, 4.6655e-23,\n 5.7798e-23, 1.3970e-22, 6.4650e-23, 6.8196e-23, 5.7275e-23, 8.2607e-23,\n 5.6530e-23, 4.4920e-23, 3.7455e-23, 4.7707e-23, 2.4958e-22, 1.9936e-22,\n 7.1651e-23, 9.4284e-23, 4.1445e-23, 4.3689e-23, 2.2402e-22, 9.7081e-23,\n 5.5713e-23, 1.5257e-22, 1.0586e-22, 1.3244e-22, 9.8834e-23, 3.8096e-22,\n 1.5940e-22, 8.1778e-23, 1.2213e-22, 3.9843e-22, 7.2692e-23, 1.2288e-22,\n 8.4288e-23, 7.0907e-23, 4.4041e-23, 1.7790e-22, 4.3189e-22, 8.2116e-23,\n 7.0030e-23, 2.6668e-22, 3.6715e-23, 6.4780e-23, 4.9432e-22, 4.5450e-23,\n 4.3884e-23, 3.6864e-23, 1.7645e-22, 1.0998e-22, 3.3396e-22, 4.4026e-23,\n 6.8652e-23, 8.0676e-23, 2.1877e-22, 2.4058e-22, 5.1354e-23, 8.8482e-23,\n 3.0814e-22, 1.2452e-22, 7.3091e-23, 7.0962e-23, 5.3098e-23, 2.9062e-22,\n 7.4332e-23, 1.4892e-22, 5.1925e-23, 1.3299e-22, 1.0450e-22, 7.4087e-23,\n 8.5931e-23, 8.6622e-23, 4.9750e-23, 9.9312e-23, 4.1265e-22, 4.4692e-23,\n 9.4645e-23, 6.2481e-23, 1.1788e-22, 1.1005e-22, 3.1958e-22, 8.1630e-23,\n 5.0798e-23, 4.8082e-23, 5.0713e-23, 1.2384e-22, 6.2334e-23, 8.6154e-23,\n 4.3980e-23, 1.2496e-22, 2.8119e-22, 4.3142e-23, 9.8614e-23, 9.0779e-23,\n 7.1787e-23, 1.3371e-22, 5.6384e-23, 1.6113e-22, 4.2077e-23, 4.5206e-23,\n 9.5499e-23, 7.5690e-23, 2.9533e-23, 2.2648e-22, 4.0852e-22, 4.9736e-23,\n 5.0924e-23, 1.0339e-22, 4.7967e-23, 9.5898e-23, 6.5722e-23, 2.4624e-22,\n 1.1988e-22, 7.3164e-23, 6.0032e-23, 3.1563e-22, 9.3768e-23, 6.9871e-23,\n 7.3111e-23, 3.6216e-23, 7.3920e-23, 8.5134e-23, 1.4721e-22, 3.6242e-22,\n 5.6470e-23, 8.8653e-23, 4.1098e-23, 3.2876e-22, 7.5300e-23, 3.0773e-23,\n 5.4965e-23, 1.3773e-22, 1.0449e-22, 8.5395e-23, 1.2276e-22, 4.7392e-23,\n 1.4045e-22, 1.2573e-22, 8.3252e-23, 3.9988e-22, 4.8519e-23, 1.1819e-22,\n 7.6308e-23, 9.7777e-23, 1.0342e-22, 8.8589e-23, 5.8437e-23, 6.0024e-23,\n 1.0725e-22, 1.7219e-22, 7.8789e-23, 1.1184e-22, 5.5035e-23, 6.6455e-23,\n 6.2397e-23, 3.6734e-23, 5.4186e-22, 5.2827e-23, 5.4735e-23, 3.7680e-23,\n 6.3269e-23, 2.9050e-22, 2.5327e-22, 8.1920e-23, 6.8965e-23, 3.2492e-23,\n 4.1668e-23, 1.4339e-22, 3.9169e-23, 7.3224e-23, 9.7369e-23, 1.4428e-22,\n 3.0639e-23, 1.4402e-22, 8.9438e-23, 5.2420e-23, 1.0444e-22, 1.4535e-22,\n 7.4273e-23, 7.0611e-23, 8.0835e-23, 4.5372e-23, 2.2103e-22, 9.5843e-23,\n 4.2622e-23, 2.8552e-23, 6.2589e-23, 5.0839e-23, 5.1136e-23, 8.6333e-23,\n 2.9972e-23, 1.9819e-22, 1.4551e-22, 1.7418e-22, 5.0719e-23, 5.2608e-23,\n 5.2130e-23, 3.1590e-23, 1.5866e-22, 4.4618e-22, 1.4161e-22, 1.8217e-22,\n 2.1041e-22, 8.7955e-23, 4.5399e-23, 5.3288e-23, 3.2925e-23, 4.8386e-23,\n 1.8508e-22, 4.2655e-23, 2.4976e-22, 9.1932e-23, 1.7358e-22, 5.4175e-23,\n 1.1209e-22, 5.4395e-23, 3.0716e-23, 2.1538e-22, 1.2112e-22, 6.6095e-23,\n 8.3558e-23, 1.8439e-22, 1.5313e-22, 1.7651e-22, 8.5689e-23, 1.4822e-22,\n 1.1412e-22, 7.5700e-23, 1.2331e-22, 3.2019e-23, 9.8044e-23, 3.6110e-23,\n 2.3677e-22, 3.5967e-23, 3.4157e-23, 4.1973e-23, 3.0175e-22, 6.2193e-23,\n 5.3586e-23, 4.2715e-22, 3.2263e-23, 3.8712e-23, 3.4147e-23, 8.2210e-23,\n 2.3734e-22, 4.1435e-22, 8.2681e-23, 3.6969e-23, 3.4418e-23, 2.8769e-22,\n 1.2281e-22, 1.0212e-22, 1.0894e-22, 4.6849e-23, 4.1201e-23, 1.8273e-22,\n 1.7468e-22, 1.1216e-22, 1.5917e-22, 9.4322e-23, 2.9959e-22, 1.8572e-22,\n 2.5177e-23, 4.1651e-23, 6.7870e-23, 1.3910e-22, 3.7562e-22, 6.4854e-23,\n 1.3367e-22, 9.8175e-23, 3.8803e-22, 1.6559e-22, 2.8551e-23, 6.2180e-23,\n 2.1181e-22, 6.1892e-23, 3.2154e-23, 2.2164e-23, 4.2727e-23, 7.9880e-23,\n 6.5598e-23, 8.2523e-23, 1.1020e-22, 4.2150e-23, 1.0308e-22, 2.1150e-22,\n 1.8174e-22, 5.2696e-23, 1.8859e-22, 4.4731e-23, 5.9336e-23, 3.3238e-22,\n 7.0581e-23, 5.9508e-23, 3.3225e-22, 2.2695e-22, 5.1805e-23, 3.4086e-23,\n 1.3211e-22, 3.2420e-22, 1.5127e-22, 2.6682e-23, 8.9428e-23, 3.6615e-23,\n 3.5104e-23, 3.0989e-23, 3.5234e-23, 8.9232e-23, 1.1618e-22, 8.7833e-23,\n 1.4418e-22, 4.1460e-22, 6.0465e-23, 1.2009e-22, 7.4941e-23, 4.3589e-23,\n 2.6581e-22, 1.7023e-22, 3.3056e-23, 2.1126e-22, 2.4698e-22, 1.6014e-22,\n 7.6762e-23, 3.2210e-22, 7.3633e-23, 9.0847e-23, 1.1055e-22, 9.1755e-23,\n 1.2308e-22, 4.8592e-23, 4.8540e-23, 1.5233e-22, 5.3842e-23, 8.5058e-23,\n 3.8719e-22, 1.4119e-22, 1.0581e-22, 2.1296e-22, 2.3232e-22, 1.1349e-22,\n 1.3834e-22, 1.2213e-22, 4.3851e-23, 2.8943e-22, 2.9083e-22, 1.0367e-22,\n 4.5162e-22, 4.8180e-22, 3.9820e-23, 5.6129e-22, 3.7962e-23, 9.3298e-23,\n 5.1195e-23, 4.8682e-23, 5.0731e-23, 9.6327e-23, 2.5579e-22, 2.2956e-22,\n 3.5465e-22, 4.3795e-23, 1.2562e-22, 1.5601e-22, 5.3327e-23, 3.9078e-23,\n 6.6012e-23, 1.1899e-22, 1.2597e-22, 5.4244e-23, 7.3430e-23, 1.9097e-22,\n 5.8548e-23, 2.6696e-22, 5.6541e-23, 8.3926e-23, 9.1578e-23, 1.8520e-22,\n 1.1677e-22, 9.1588e-23, 9.6504e-23, 1.3221e-22, 1.4543e-22, 3.7337e-23,\n 4.9375e-23, 3.9101e-23, 5.2793e-23, 9.9201e-23, 7.9436e-23, 4.6461e-23,\n 1.2009e-22, 1.0545e-22, 9.9901e-23, 3.2930e-23, 9.0694e-23, 5.4655e-23,\n 1.1399e-22, 3.6934e-23, 7.1654e-23, 9.2058e-23, 5.1358e-23, 8.3873e-23,\n 9.6318e-23, 6.0411e-23, 9.1615e-23, 8.0058e-23, 5.6428e-23, 4.4216e-22,\n 4.5572e-23, 8.1321e-23, 1.1826e-22, 1.1152e-22, 1.1776e-22, 4.2399e-23,\n 4.9888e-23, 8.3860e-23, 9.0946e-23, 1.3085e-22, 4.6295e-22, 2.1257e-22,\n 2.6721e-22, 3.1999e-22, 1.2888e-22, 5.7733e-23, 1.1953e-22, 1.4354e-22,\n 4.1648e-23, 7.0530e-22, 8.1477e-22, 1.1335e-22, 1.1426e-22, 7.4351e-23,\n 3.7351e-23, 2.1190e-22, 6.7607e-23, 5.3108e-23, 8.4532e-23, 6.1866e-23,\n 1.0226e-22, 8.8384e-23, 4.1992e-23, 1.3220e-22, 4.4995e-23, 1.4930e-22,\n 2.0082e-22, 2.2611e-22, 6.6987e-23, 1.2794e-22, 1.6697e-22, 8.6805e-23,\n 7.3301e-23, 1.1660e-22, 1.2881e-22, 7.7572e-23, 1.3068e-22, 5.0013e-23,\n 6.3541e-23, 1.2121e-22, 1.2329e-21, 4.9502e-23, 3.0295e-22, 1.0752e-22,\n 1.2395e-22, 1.1061e-22, 4.8879e-23, 9.1891e-23, 3.9545e-22, 6.0537e-23,\n 5.0388e-23, 6.8356e-23, 6.3669e-23, 6.2024e-23, 7.5876e-23, 7.7651e-23,\n 3.7575e-22, 6.8343e-23, 9.6393e-23, 2.0627e-22, 1.6172e-22, 6.7223e-23,\n 3.0748e-22, 8.8000e-23, 8.0581e-23, 6.0488e-23, 7.7912e-23, 1.2666e-22,\n 4.8240e-23, 1.0610e-22, 5.3153e-23, 1.6088e-22, 5.2319e-23, 6.7008e-23,\n 4.5008e-23, 2.3523e-22], device='cuda:0')" + }, + "36": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.0746e-06, 1.8055e-06, 2.0165e-07, ..., 2.0066e-06,\n 1.6146e-06, 5.0777e-09],\n [-2.5553e-06, -4.4900e-07, 6.5473e-07, ..., -1.5542e-06,\n -1.9551e-06, 6.1188e-08],\n [-1.3772e-06, 4.5698e-07, 7.8571e-07, ..., -6.7072e-07,\n -2.1458e-07, 1.3904e-07],\n ...,\n [-6.9211e-07, -1.7209e-07, 5.8065e-07, ..., -1.2349e-06,\n -1.9925e-06, 1.5810e-06],\n [-1.2275e-06, -7.3286e-08, 7.4251e-08, ..., -5.6018e-07,\n -7.6085e-07, 8.8167e-08],\n [ 8.6026e-07, -1.4248e-06, -1.2289e-06, ..., 3.1158e-07,\n 9.7797e-08, -4.2684e-08]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.8050e-11, 4.8108e-11, 2.3748e-11, ..., 3.0424e-11, 2.8517e-11,\n 2.9617e-11],\n [4.5573e-11, 8.1503e-11, 3.8951e-11, ..., 4.1215e-11, 4.7843e-11,\n 4.6514e-11],\n [2.3303e-11, 3.0214e-11, 1.6668e-11, ..., 1.8866e-11, 2.1327e-11,\n 2.0962e-11],\n ...,\n [2.9940e-11, 3.4612e-11, 2.4473e-11, ..., 2.3343e-11, 2.8383e-11,\n 3.0932e-11],\n [8.5886e-12, 9.8220e-12, 6.5923e-12, ..., 6.5944e-12, 9.9357e-12,\n 9.4370e-12],\n [1.6323e-11, 2.2462e-11, 1.4309e-11, ..., 1.4908e-11, 1.7198e-11,\n 1.7931e-11]], device='cuda:0')" + }, + "37": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-1.1788e-05, 1.9121e-05, 1.8624e-05, 1.6449e-05, 1.6810e-05,\n -9.7557e-06, 1.3770e-05, -5.3443e-06, -6.8513e-06, 1.3991e-05,\n 4.0041e-06, 3.2169e-06, -5.1365e-05, -5.1612e-06, -2.2675e-05,\n -2.4433e-06, -8.1040e-06, 2.1075e-05, 1.4718e-05, 9.4843e-06,\n -2.6063e-05, 3.0032e-07, 7.2355e-06, 3.0219e-05, 8.0530e-06,\n -3.7507e-06, -1.4014e-05, 3.8856e-06, -1.9312e-06, -1.2945e-05,\n 4.2336e-06, -1.5581e-05, 1.3428e-05, -2.5647e-05, 2.0097e-05,\n 3.5994e-06, 1.4740e-05, 5.6997e-06, -1.3673e-06, 8.5160e-06,\n -3.5896e-05, -5.4136e-06, 4.9877e-06, -8.4901e-06, -3.2764e-06,\n -2.6866e-05, -5.5955e-06, 2.1695e-05, 1.7214e-05, -3.8286e-06,\n -2.5948e-09, 3.4508e-06, 4.6575e-06, 5.9216e-08, -4.3663e-06,\n 1.0827e-05, -3.2395e-06, -2.2505e-05, 5.8753e-06, 2.6745e-05,\n 1.4631e-05, 1.2712e-05, 9.8963e-06, 1.6828e-05, -3.7627e-06,\n 2.9913e-06, -5.8197e-06, -1.0469e-05, 4.8614e-06, -3.6139e-05,\n 2.1071e-05, -2.0630e-05, -8.1724e-06, -1.1649e-05, 1.4573e-05,\n -1.2995e-05, 1.8624e-05, 1.6922e-05, -9.0585e-06, -1.2161e-05,\n -3.3835e-06, -4.1399e-06, 2.4632e-06, -2.3518e-05, 1.0008e-05,\n 1.5426e-05, -1.4391e-05, -2.0997e-05, 2.9295e-06, -2.1037e-06,\n -1.1423e-05, 3.7337e-06, 1.4032e-05, -1.0225e-05, -6.7366e-06,\n 3.0619e-05, -4.3874e-06, -8.5646e-06, 2.1859e-05, 2.7595e-06,\n -2.1983e-05, -2.0380e-05, 4.6053e-05, 1.2582e-06, 2.8096e-06,\n -6.1967e-06, -1.8394e-05, 4.3664e-06, 6.8700e-06, -8.0141e-06,\n -1.9257e-05, 5.7958e-06, -2.0283e-05, -1.4842e-05, 1.3324e-06,\n 2.5831e-07, -1.4311e-06, 1.2236e-05, 1.1014e-05, 1.5959e-05,\n 1.1700e-05, 2.4198e-06, -1.9519e-05, -1.9938e-05, -1.7382e-05,\n 9.5213e-06, -1.9600e-05, -2.8929e-06, -2.8743e-05, 7.0541e-06,\n 1.9445e-06, -1.4851e-05, 4.3597e-06, -1.6132e-05, 1.2032e-05,\n -2.6286e-06, 2.9493e-06, -1.8626e-05, 3.6845e-06, -1.5965e-05,\n -5.3079e-06, -3.6693e-07, 5.5692e-06, -1.7154e-05, 1.2100e-05,\n -8.1168e-06, 1.4275e-05, 3.3634e-07, -1.0404e-05, -1.6628e-06,\n -5.1486e-06, -1.7806e-05, 2.9606e-06, -6.6252e-06, 1.3721e-05,\n -3.1456e-06, 5.5729e-06, 2.5671e-06, -2.8715e-07, -1.6473e-05,\n -1.0080e-05, -1.2060e-05, -8.8861e-06, 1.6479e-06, 1.9446e-06,\n -3.1436e-07, 1.9586e-05, -4.0570e-07, 6.3377e-06, 1.7658e-05,\n -5.1626e-07, 1.9339e-05, -4.9011e-06, -1.6791e-05, -4.8787e-06,\n -1.6083e-05, 1.3214e-05, 3.4965e-06, -9.9396e-06, 3.2722e-06,\n -6.8194e-06, -2.8666e-05, -3.8188e-06, -9.1112e-06, -5.1916e-06,\n 1.1170e-05, 1.0482e-06, -5.4466e-06, 1.1460e-05, -1.4710e-06,\n 2.0394e-06, 9.7346e-06, 8.0866e-06, -1.6200e-05, 1.0037e-05,\n 9.6145e-06, -1.4028e-05, -2.6769e-05, 1.0053e-05, 2.2315e-06,\n 1.8628e-05, 3.8652e-06, -1.6133e-05, -1.9185e-05, 1.0966e-05,\n 1.0960e-05, 1.7664e-05, 1.6837e-05, -2.4984e-06, 1.8562e-05,\n 5.0888e-06, 1.1626e-05, -5.6721e-05, -9.5403e-06, 4.2289e-07,\n -1.0833e-05, -1.7613e-05, 4.4242e-06, -6.7699e-06, -5.8827e-06,\n 1.5933e-05, -1.8723e-05, 2.5510e-05, 3.2419e-07, -4.6554e-06,\n -6.3611e-06, -5.4152e-06, -3.4916e-06, -3.6784e-06, 1.2830e-05,\n -1.6168e-05, 1.8129e-06, 1.2579e-04, 1.2957e-06, -3.9468e-06,\n 4.5927e-06, 8.8680e-07, 2.7636e-06, 1.5204e-05, 1.6025e-05,\n 1.0441e-05, -2.0177e-06, -1.3554e-05, 6.4417e-06, -1.0009e-05,\n 1.7998e-05, 2.1906e-06, -2.7081e-06, 4.6787e-06, 1.3588e-05,\n -4.0044e-06, 3.6537e-06, -2.3509e-05, -7.9052e-06, -6.5109e-06,\n -1.3843e-05, 5.3357e-06, -8.9568e-06, -3.0966e-06, -1.2100e-05,\n -1.4321e-06, -7.2227e-06, 5.4556e-06, 6.9534e-06, 5.5406e-06,\n 4.6700e-06, 1.0932e-05, -5.3493e-06, -1.1268e-05, -1.1609e-05,\n 7.8621e-06, -8.2888e-06, -3.7706e-06, -6.0352e-06, -3.3355e-06,\n -9.9240e-06, -8.0113e-06, 5.9456e-06, -3.4961e-06, -1.7544e-05,\n -7.9692e-06, -2.0436e-06, -5.2839e-07, -1.9523e-05, 1.7612e-05,\n -1.3226e-05, -1.0912e-05, -1.1051e-05, -2.2795e-06, 5.1057e-07,\n 6.1763e-06, -4.3476e-06, 7.2224e-06, 1.9367e-06, 1.3852e-05,\n -7.1315e-06, 1.3356e-05, -5.1284e-06, -1.7552e-05, 9.3837e-08,\n 7.8142e-06, -3.7243e-06, -4.1050e-06, 3.5052e-06, -2.7923e-06,\n 4.0979e-06, -2.3784e-07, -1.7670e-06, 1.1108e-05, -2.2295e-06,\n 4.7958e-06, 1.6362e-06, -2.1043e-07, 6.2266e-06, 3.9364e-06,\n -4.5253e-05, -1.0096e-05, 3.4340e-06, -2.7272e-07, 4.4057e-06,\n 1.5234e-05, 1.2262e-05, 9.6454e-06, 2.5662e-07, 1.1215e-06,\n 1.0788e-06, 2.4957e-05, -1.8050e-05, 2.6282e-06, -8.4773e-06,\n -1.0363e-05, 2.5319e-06, -7.1240e-06, -9.8874e-06, 5.2446e-06,\n 9.1343e-06, -2.2630e-07, -5.2561e-06, -2.4066e-06, 3.3555e-06,\n -3.1069e-06, -2.3925e-06, -3.3269e-06, 9.1084e-07, -6.3445e-06,\n 5.8069e-06, 1.7548e-06, 2.9977e-06, 4.4016e-06, 7.2898e-06,\n 7.3555e-06, 1.6110e-06, 9.2650e-06, -2.0124e-05, -1.8142e-06,\n -4.8872e-06, -8.7517e-06, 1.3673e-05, -1.3949e-05, -1.9027e-06,\n -8.2688e-07, -1.5631e-05, 1.6733e-05, 3.0849e-06, 1.3761e-05,\n -9.3092e-07, 1.2085e-05, -8.1081e-06, -4.9074e-06, 8.7873e-06,\n 1.3269e-05, -1.0415e-05, 1.4018e-06, -3.6992e-07, -5.3951e-06,\n 3.9156e-06, 6.7560e-08, 9.1611e-06, -2.7995e-06, -5.2827e-06,\n -2.0850e-05, 1.4619e-05, -3.3439e-06, 5.1894e-06, -1.4374e-05,\n 1.8127e-05, -4.5135e-06, 8.5135e-06, -6.7608e-06, 1.3401e-06,\n -3.7061e-06, -2.0097e-05, 1.5747e-05, -1.2871e-05, 1.0024e-05,\n 2.1368e-05, 7.7202e-07, 1.4773e-06, -1.1221e-05, 2.2803e-06,\n -3.1057e-05, -8.0921e-06, 2.6216e-05, 3.5445e-06, -6.2990e-06,\n -3.3521e-06, 1.2048e-05, -1.4570e-05, 5.8615e-06, -7.8355e-06,\n -1.3922e-05, -3.6713e-06, -9.2055e-05, -2.0168e-05, -1.5404e-05,\n -1.0310e-05, -7.9178e-06, -3.4181e-06, 2.5747e-07, 1.3177e-05,\n 1.0784e-05, 1.9091e-06, -1.8405e-06, 7.7072e-06, -7.5613e-06,\n -9.8459e-06, -2.0762e-05, -1.6583e-05, -2.8919e-06, -8.5794e-06,\n -1.7574e-05, 2.4016e-07, 1.1052e-05, 2.7498e-06, 2.7033e-06,\n -9.8246e-06, 1.1376e-05, 2.5365e-05, -7.8051e-06, -1.8160e-05,\n 9.5941e-06, -2.1814e-05, -8.4735e-06, 1.0089e-05, -1.8752e-05,\n -3.1097e-05, -6.3095e-06, -1.6411e-05, 1.7941e-07, -3.9207e-06,\n 1.1878e-05, -2.6191e-05, 1.1016e-05, -3.8274e-06, -4.6090e-06,\n -1.0696e-06, -6.3139e-06, -1.8524e-05, 1.0306e-05, -8.8572e-06,\n 3.1937e-06, -1.5617e-05, -6.5274e-06, -8.2805e-06, 9.2046e-06,\n 1.0428e-05, -1.1423e-05, 8.4831e-06, 1.2858e-06, -3.0197e-06,\n -2.0197e-05, 9.9484e-06, -1.4916e-05, 1.1784e-05, -2.5026e-05,\n -5.7624e-06, 1.7993e-05, 8.4375e-06, 7.6305e-06, 1.8845e-06,\n 1.0392e-05, 4.9690e-06, 7.6517e-06, -5.4566e-06, -4.7284e-06,\n 1.0708e-05, -1.1516e-06, 9.0121e-06, 4.2026e-06, 9.9089e-06,\n 2.3196e-06, -4.7690e-07, 1.6044e-05, -1.7533e-05, -3.1113e-06,\n 4.8278e-07, -1.2604e-05, -5.0211e-06, 3.4413e-05, 1.7530e-05,\n -2.4869e-05, 1.0608e-05, 1.2041e-05, 1.4050e-05, -2.3438e-05,\n 3.6131e-06, -1.7167e-05, -1.0068e-05, -4.5131e-07, -3.1852e-05,\n -2.5414e-06, 1.0447e-05], device='cuda:0')", + "exp_avg_sq": "tensor([6.0547e-09, 9.4206e-09, 4.3976e-09, 6.9443e-09, 7.9782e-09, 4.8151e-09,\n 2.6572e-09, 5.1451e-09, 4.3477e-09, 5.3336e-09, 3.1314e-09, 7.6955e-09,\n 2.7897e-08, 4.7615e-09, 6.1403e-09, 3.6997e-09, 9.7774e-09, 3.1952e-08,\n 5.1267e-09, 4.0438e-09, 3.1990e-09, 6.4905e-09, 4.6134e-09, 1.1962e-08,\n 3.3090e-09, 3.5576e-09, 6.7270e-09, 6.3661e-09, 5.2055e-09, 5.4094e-09,\n 5.1301e-09, 6.5993e-09, 6.1652e-09, 6.0239e-09, 1.0554e-08, 4.1172e-09,\n 6.3448e-09, 6.4934e-09, 3.8129e-09, 6.3841e-09, 1.0534e-08, 5.6026e-09,\n 7.6862e-09, 4.6462e-09, 8.5958e-09, 1.1940e-08, 5.1725e-09, 5.2723e-09,\n 3.5646e-09, 5.0024e-09, 4.7937e-09, 3.4179e-09, 5.2575e-09, 2.0132e-09,\n 2.8796e-09, 5.3501e-09, 2.3960e-09, 4.2429e-09, 6.8119e-09, 5.6693e-09,\n 5.5144e-09, 4.2345e-09, 1.8175e-09, 5.2659e-09, 5.7587e-09, 3.8438e-09,\n 4.2038e-09, 3.2455e-09, 4.7447e-09, 1.0077e-08, 3.2094e-09, 3.7989e-09,\n 4.7588e-09, 2.2258e-09, 4.2014e-09, 3.2579e-09, 3.4077e-09, 5.8262e-09,\n 4.6877e-09, 3.1162e-09, 3.7590e-09, 4.7684e-09, 2.7445e-09, 4.0001e-09,\n 3.8310e-09, 5.0592e-09, 7.1243e-09, 3.5686e-09, 3.5289e-09, 4.8954e-09,\n 6.8242e-09, 3.7110e-09, 3.4698e-09, 2.7340e-09, 3.1533e-09, 1.3605e-08,\n 6.6514e-09, 2.7796e-09, 5.8434e-09, 4.6143e-09, 4.5812e-09, 5.7208e-09,\n 4.6788e-08, 2.1841e-09, 3.2683e-09, 6.0573e-09, 4.3802e-09, 3.6996e-09,\n 5.5405e-09, 3.2534e-09, 3.2090e-09, 3.6248e-09, 4.3128e-09, 2.8419e-09,\n 2.9380e-09, 6.1618e-09, 4.8196e-09, 5.0659e-09, 3.5821e-09, 1.0824e-08,\n 2.8365e-09, 4.1412e-09, 5.9057e-09, 5.7958e-09, 5.1369e-09, 5.2337e-09,\n 5.0973e-09, 6.6399e-09, 7.7728e-09, 4.0936e-09, 2.4739e-09, 6.1145e-09,\n 3.4595e-09, 1.9875e-08, 3.2008e-09, 2.4136e-09, 1.6627e-09, 1.0425e-08,\n 6.4718e-09, 1.2344e-08, 2.7059e-09, 8.1329e-09, 3.2004e-09, 8.3506e-09,\n 1.9648e-09, 7.9284e-09, 5.9683e-09, 2.4603e-09, 5.7916e-09, 2.8719e-09,\n 4.6395e-09, 3.9522e-09, 2.5128e-09, 4.0643e-09, 1.6610e-08, 2.5561e-09,\n 2.2500e-09, 4.0024e-09, 4.3425e-09, 7.5543e-09, 5.2568e-09, 9.2443e-09,\n 2.0948e-09, 2.5296e-09, 1.0746e-08, 1.0948e-08, 3.2729e-09, 2.5914e-09,\n 6.1479e-09, 6.9559e-09, 4.0589e-09, 5.6906e-09, 2.2575e-09, 7.0805e-09,\n 4.1132e-09, 6.4281e-09, 6.4534e-09, 2.2377e-09, 5.6474e-09, 2.4774e-09,\n 4.0764e-09, 7.7699e-09, 3.0252e-09, 5.4834e-09, 3.8640e-09, 5.0809e-09,\n 4.6503e-09, 1.9617e-09, 5.0904e-09, 2.6068e-09, 6.6529e-09, 7.2040e-09,\n 2.1149e-09, 8.6429e-09, 6.5739e-09, 4.4870e-09, 3.8605e-09, 1.2226e-08,\n 3.0695e-09, 3.8654e-09, 6.2801e-09, 5.4067e-09, 4.3979e-09, 7.0405e-09,\n 2.9684e-09, 3.2682e-09, 3.1035e-09, 4.8622e-09, 3.3269e-09, 5.3631e-09,\n 4.5791e-09, 4.1593e-09, 4.0377e-08, 6.5039e-09, 3.8387e-09, 2.9426e-09,\n 2.5050e-09, 5.8646e-09, 3.1668e-09, 3.2913e-09, 6.2681e-09, 8.8226e-09,\n 3.3575e-09, 2.3507e-09, 2.5333e-09, 4.4155e-09, 2.9317e-09, 4.6030e-09,\n 6.2324e-09, 1.7359e-09, 6.8718e-09, 7.6831e-09, 1.0477e-07, 2.9532e-09,\n 8.0394e-09, 4.6535e-09, 5.6210e-09, 4.5220e-09, 3.4998e-09, 2.3633e-09,\n 2.2668e-09, 3.3582e-09, 2.0561e-09, 2.7775e-09, 6.4290e-09, 7.2699e-09,\n 3.7232e-09, 3.7290e-09, 5.6146e-09, 3.1642e-09, 3.5065e-09, 5.3839e-09,\n 9.1219e-09, 5.8240e-09, 3.6848e-09, 1.9987e-09, 8.2054e-09, 4.1331e-09,\n 4.3427e-09, 2.4343e-09, 2.8843e-09, 2.0373e-09, 5.2891e-09, 5.2965e-09,\n 4.8341e-09, 1.5611e-09, 2.1814e-09, 2.1651e-09, 3.2266e-09, 2.8615e-09,\n 1.4477e-08, 3.3573e-09, 4.0071e-09, 3.4922e-09, 7.3125e-09, 1.9569e-09,\n 3.1648e-09, 3.4600e-09, 2.5563e-09, 3.3607e-09, 1.0801e-08, 3.0952e-09,\n 5.3643e-09, 3.2272e-08, 2.8357e-09, 4.8517e-09, 3.5655e-09, 3.3085e-09,\n 6.3396e-09, 3.4506e-09, 5.1824e-09, 5.2107e-09, 7.1766e-09, 2.1325e-09,\n 3.2421e-09, 5.8097e-09, 3.8960e-09, 2.9204e-09, 2.8240e-09, 4.9437e-09,\n 3.8697e-09, 6.5641e-09, 7.5016e-09, 4.1814e-09, 1.8512e-09, 3.0320e-09,\n 4.3631e-09, 3.8236e-09, 4.9264e-09, 4.1220e-09, 3.6708e-09, 4.9632e-09,\n 1.9110e-09, 2.5911e-09, 4.5714e-09, 1.0837e-07, 6.6455e-09, 4.1119e-09,\n 4.4498e-09, 6.2115e-09, 4.1217e-09, 3.2054e-09, 3.4605e-09, 5.1309e-09,\n 4.7442e-09, 2.1094e-09, 1.3914e-08, 4.7275e-09, 4.0494e-09, 5.1061e-09,\n 4.2785e-08, 5.9862e-09, 3.5406e-09, 3.1725e-09, 4.5544e-09, 2.9467e-09,\n 7.3636e-09, 3.9071e-09, 5.8104e-09, 3.1129e-09, 5.3688e-09, 5.6617e-09,\n 1.0693e-08, 5.4161e-09, 7.3153e-09, 3.4973e-09, 2.5315e-09, 6.1002e-09,\n 6.5597e-09, 4.4357e-09, 5.1950e-09, 2.7219e-09, 2.8817e-09, 3.3005e-09,\n 7.5604e-09, 1.7239e-09, 7.8771e-09, 2.0673e-09, 3.3892e-09, 3.2516e-09,\n 5.6975e-09, 3.1389e-09, 2.4269e-09, 1.7897e-08, 3.8819e-09, 3.4777e-09,\n 6.3327e-09, 3.0673e-09, 3.0844e-09, 3.4743e-09, 4.0379e-09, 6.0170e-09,\n 5.2259e-09, 6.0312e-09, 4.7836e-09, 3.4536e-09, 2.7939e-09, 2.2661e-09,\n 1.4078e-08, 2.6306e-09, 3.1376e-09, 6.9770e-09, 9.2924e-09, 3.6188e-09,\n 1.2039e-08, 5.3781e-09, 4.0782e-09, 2.7185e-09, 4.9568e-09, 2.5258e-09,\n 2.9329e-09, 4.7760e-09, 4.3690e-09, 5.2139e-09, 3.6237e-09, 7.5735e-09,\n 4.6332e-09, 4.3717e-09, 7.8102e-09, 2.7462e-09, 6.7144e-09, 2.5802e-09,\n 3.1880e-08, 2.4133e-09, 3.7172e-09, 2.8705e-09, 4.0967e-09, 7.7115e-09,\n 3.9178e-09, 3.7106e-09, 3.8728e-09, 3.3258e-09, 1.4521e-07, 6.6530e-09,\n 5.1624e-09, 4.7212e-09, 1.8916e-09, 3.6272e-09, 2.4441e-09, 3.7771e-09,\n 8.0394e-09, 3.0890e-09, 4.9707e-09, 2.6276e-09, 4.3911e-09, 2.8170e-09,\n 6.8605e-09, 4.2248e-09, 3.4674e-09, 4.1150e-09, 6.0904e-09, 4.0220e-09,\n 1.1714e-08, 2.2103e-09, 1.0485e-08, 2.5915e-09, 2.0151e-09, 4.6583e-09,\n 5.5837e-09, 3.2813e-09, 4.7944e-09, 9.3517e-09, 1.2804e-08, 3.7895e-09,\n 4.1031e-09, 3.7995e-09, 1.7662e-09, 3.2145e-09, 3.0878e-09, 4.3569e-09,\n 6.0839e-09, 4.5266e-09, 3.8431e-09, 2.7836e-09, 4.0030e-09, 5.9335e-09,\n 6.4217e-09, 1.2009e-08, 5.5585e-09, 4.5135e-09, 3.2265e-09, 4.4799e-09,\n 5.2592e-09, 3.2850e-09, 5.1467e-09, 3.9539e-09, 6.4306e-09, 3.5341e-09,\n 5.6510e-09, 3.0433e-09, 6.3955e-09, 3.7336e-09, 3.7620e-09, 4.1701e-09,\n 9.8535e-09, 4.5163e-09, 3.0862e-09, 6.0104e-09, 3.9560e-09, 4.7243e-09,\n 1.0043e-08, 3.2241e-09, 1.4942e-09, 9.1077e-09, 2.9204e-09, 5.3082e-09,\n 2.2235e-09, 2.1786e-08, 1.8390e-08, 2.9299e-09, 1.6865e-09, 3.3148e-09,\n 6.1895e-09, 5.8234e-09, 4.0348e-09, 8.6605e-09, 3.4833e-09, 3.2780e-09,\n 7.5580e-09, 2.7608e-09, 4.4290e-09, 4.3665e-09, 6.0810e-09, 3.9655e-09,\n 6.4321e-09, 2.4085e-09, 3.2793e-09, 4.3496e-09, 2.8354e-09, 5.4939e-09,\n 1.9038e-09, 3.8305e-09], device='cuda:0')" + }, + "40": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 7.8223e-06, 2.4348e-05, -6.4955e-06, ..., 1.0136e-07,\n 4.3710e-06, -4.1836e-06],\n [ 3.6400e-05, -3.3081e-05, -9.6707e-07, ..., -2.6714e-05,\n -1.7610e-05, 1.3256e-05],\n [-3.1024e-07, -5.1122e-06, 1.5728e-05, ..., 8.2056e-06,\n -2.9461e-05, 1.6519e-05],\n ...,\n [-1.1842e-05, -1.7701e-06, -2.4971e-06, ..., 2.6799e-07,\n -6.0356e-06, 5.3318e-07],\n [-8.6667e-06, 5.7816e-07, -6.1048e-06, ..., 4.9390e-06,\n 2.3413e-06, 4.3739e-06],\n [-6.3315e-05, -1.1106e-05, -1.7734e-05, ..., -1.3707e-05,\n -4.0498e-06, 2.9074e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.4419e-09, 7.3591e-09, 1.0276e-09, ..., 1.2561e-09, 1.4808e-09,\n 1.0506e-09],\n [4.4758e-08, 6.2065e-08, 9.7279e-09, ..., 1.0228e-08, 1.1142e-08,\n 8.7729e-09],\n [5.5426e-08, 4.5831e-08, 8.1118e-09, ..., 7.7008e-09, 1.4689e-08,\n 7.0917e-09],\n ...,\n [4.9289e-10, 5.5567e-10, 9.1196e-11, ..., 1.1561e-10, 1.2765e-10,\n 1.0268e-10],\n [2.1906e-09, 2.6359e-09, 4.5809e-10, ..., 5.4615e-10, 6.6719e-10,\n 5.8990e-10],\n [2.7521e-08, 3.1147e-08, 5.4398e-09, ..., 6.1794e-09, 6.3418e-09,\n 5.6017e-09]], device='cuda:0')" + }, + "41": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 4.7218e-06, 2.6022e-04, -3.3463e-06, -1.8186e-04, -7.6840e-05,\n 8.5956e-04, -2.4307e-04, 7.9241e-04, -7.3168e-05, -8.5633e-05,\n -1.2291e-04, -1.9080e-04, -5.1632e-04, -3.3361e-04, -8.7449e-05,\n -5.1208e-04, -5.4509e-05, -1.0940e-04, 6.4798e-04, 1.1604e-04,\n 6.4253e-04, -2.7899e-05, 8.8936e-05, 3.0324e-04, -8.0635e-04,\n -9.8201e-05, -2.2828e-04, -8.4167e-05, -6.0834e-04, -2.8517e-05,\n -1.1221e-04, -4.8781e-05, -4.3580e-04, 3.6055e-04, -6.7866e-05,\n 3.6371e-05, -2.8745e-05, -9.4410e-05, -6.4600e-05, 2.5711e-03,\n -2.9758e-04, 4.2007e-04, -8.7313e-05, -3.5346e-05, -1.5931e-05,\n -9.0542e-05, -2.9645e-05, -1.8941e-05, -8.1135e-04, -8.3080e-05,\n -8.0618e-04, -3.8752e-05, -6.8348e-05, -2.1360e-04, 5.9789e-05,\n -3.7353e-05, -2.8223e-04, 1.9917e-04, 3.8752e-04, -1.7851e-04,\n -2.4017e-04, -6.2451e-05, -4.3980e-05, 9.6172e-05, -3.4955e-04,\n 3.2066e-06, -6.7093e-05, -4.6850e-04, 8.1583e-04, -1.2991e-03,\n 9.4471e-05, -6.0920e-05, 5.3501e-04, 1.0474e-04, -7.3071e-05,\n 2.5301e-03, -1.0773e-04, -4.2494e-06, -1.0523e-04, -3.1950e-04,\n -1.1168e-04, -4.4070e-04, -4.2756e-04, -1.5543e-05, -1.0572e-04,\n -5.5995e-05, -7.5696e-05, -6.6861e-05, -3.1437e-04, 4.0650e-05,\n 9.7402e-05, -7.7097e-05, -7.6339e-05, -5.1009e-05, -1.1727e-04,\n -9.6658e-05, 9.0826e-04, -4.6459e-05, -1.8674e-04, -9.1979e-05,\n -1.5416e-04, -7.4879e-05, -1.2475e-04, 1.6099e-03, -4.9956e-05,\n -6.2507e-06, 2.8132e-04, -1.0071e-04, -8.9312e-05, -1.4153e-04,\n -7.6204e-05, -2.7410e-04, -6.5370e-05, 4.3601e-04, -1.2433e-03,\n -1.7904e-04, -1.5368e-04, -2.8238e-04, -1.0696e-04, -2.5590e-04,\n 5.1551e-04, -1.2807e-04, -8.0144e-05, -2.8405e-04, 3.2471e-04,\n 1.3214e-03, -2.1991e-04, -1.8858e-04, -5.9072e-05, -2.9351e-04,\n -8.0028e-05, -1.2086e-04, 2.9483e-05, 4.2375e-04, -1.6577e-04,\n 2.6889e-04, -4.4108e-04, -5.3306e-04, 3.2826e-03, -1.3416e-04,\n -2.7644e-04, -3.5749e-04, -4.9430e-04, -2.3418e-04, -7.4881e-05,\n 4.7119e-05, -8.0307e-05, -1.1507e-03, -8.6873e-04, -4.9457e-05,\n -2.2220e-05, -3.9887e-05, -9.7753e-05, 4.6692e-05, -9.6889e-05,\n 1.8779e-04, -8.6948e-05, -1.5434e-04, -3.5676e-04, 1.9246e-05,\n -2.8165e-04, -9.4003e-04, -1.3281e-04, 4.8090e-04, 2.8396e-03,\n -1.1399e-04, -2.1471e-04, -3.6391e-04, 1.6463e-03, 1.0626e-04,\n -1.4194e-04, -1.7038e-04, -3.7130e-04, -1.8988e-04, -2.0495e-04,\n -7.4659e-05, 1.8914e-04, -2.1255e-03, 1.5160e-05, -2.1681e-04,\n -3.2734e-04, -1.6375e-05, -1.3296e-04, -1.0783e-04, -1.3553e-04,\n -3.5420e-05, -1.7022e-04, -1.0483e-04, -5.1656e-05, -2.5540e-04,\n -1.1157e-04, -1.9229e-04, -2.8643e-04, 4.4749e-04, -6.6107e-05,\n -5.5673e-05, -5.1814e-04, 1.0078e-04, -2.1195e-04, 1.4946e-03,\n -8.0366e-05, -1.6978e-04, -1.4214e-04, 1.0115e-05, 4.2920e-05,\n -1.1484e-04, -1.1281e-04, -3.0415e-05, -8.9334e-05, -1.3615e-04,\n -3.1735e-06, -5.3059e-05, 3.3226e-04, 1.5649e-03, -8.2515e-05,\n -4.2109e-05, -2.3123e-04, -1.8299e-04, 4.2875e-04, -2.3409e-04,\n -1.9896e-05, -1.7216e-04, -2.3422e-04, 1.6832e-04, -8.0923e-05,\n -7.3710e-05, 3.7945e-03, -4.4058e-05, -3.9415e-05, -1.1788e-04,\n -5.8231e-06, -5.2986e-05, 4.7312e-04, -8.7741e-05, -1.7716e-04,\n -5.3452e-05, -1.4785e-04, -1.2199e-04, 6.6157e-05, -9.9269e-05,\n -2.0666e-04, 3.6210e-04, -2.5394e-04, -7.4391e-05, -6.6800e-06,\n -1.9432e-04, -1.0014e-04, -3.2122e-05, -1.0064e-04, -1.5800e-04,\n 6.7917e-05, -9.8061e-06, 1.3870e-03, 3.1526e-05, 1.2717e-04,\n -7.2772e-04], device='cuda:0')", + "exp_avg_sq": "tensor([1.8628e-06, 1.4729e-05, 1.3446e-05, 3.4904e-06, 2.7715e-05, 2.7464e-06,\n 8.1853e-07, 2.5812e-06, 2.0318e-07, 1.7165e-07, 5.5833e-07, 2.8988e-06,\n 4.8082e-06, 1.6591e-05, 6.1204e-07, 8.3746e-06, 1.3916e-06, 2.4354e-05,\n 9.5600e-06, 1.1798e-05, 2.1634e-06, 1.1378e-07, 5.9620e-06, 4.7592e-07,\n 1.0362e-05, 3.0966e-07, 4.0072e-07, 3.4959e-07, 3.5512e-06, 3.0865e-07,\n 6.7415e-06, 3.5590e-06, 2.3951e-06, 3.8925e-05, 2.7251e-07, 1.6678e-07,\n 2.4604e-06, 6.3172e-08, 1.7799e-07, 4.0638e-05, 3.4680e-07, 3.7819e-06,\n 2.3751e-07, 8.1292e-08, 1.7036e-07, 1.6564e-07, 3.6104e-07, 4.9591e-07,\n 3.1683e-05, 1.2816e-06, 2.8002e-05, 1.1780e-07, 2.8328e-07, 1.4638e-06,\n 9.4855e-08, 1.8172e-07, 1.9222e-06, 2.6144e-05, 3.2014e-06, 7.7150e-06,\n 1.4426e-06, 2.9171e-07, 1.7754e-07, 6.4713e-06, 4.0532e-07, 4.1491e-07,\n 3.4597e-07, 5.6976e-07, 5.5079e-06, 2.2419e-05, 2.8308e-06, 8.3689e-07,\n 1.3673e-05, 3.0718e-07, 1.3256e-06, 1.1921e-05, 3.3514e-07, 2.8991e-07,\n 9.2335e-07, 3.9161e-06, 2.1897e-07, 6.5880e-07, 2.6087e-06, 2.9170e-07,\n 2.1105e-07, 1.6838e-07, 2.1769e-07, 7.6730e-07, 3.0481e-06, 1.5289e-07,\n 3.4710e-07, 1.2393e-07, 1.1414e-06, 3.8344e-07, 1.3446e-07, 2.3704e-07,\n 3.7000e-06, 4.0988e-06, 2.5226e-07, 2.4774e-07, 4.4771e-06, 1.3162e-07,\n 2.6806e-07, 3.0695e-05, 3.7611e-07, 8.4853e-07, 1.1334e-05, 1.8387e-07,\n 1.3516e-07, 6.5095e-07, 3.4986e-07, 5.1724e-07, 2.6956e-07, 1.6522e-06,\n 1.4447e-05, 2.5656e-06, 2.1821e-07, 5.1632e-07, 2.4102e-07, 1.1117e-06,\n 3.5058e-05, 3.1530e-07, 3.0892e-07, 2.9424e-06, 9.1284e-06, 3.2464e-05,\n 1.7795e-06, 5.4014e-07, 2.0249e-07, 2.0359e-06, 2.7847e-07, 3.3896e-07,\n 4.2461e-06, 1.0730e-05, 1.2962e-07, 2.6934e-06, 7.9750e-07, 5.7014e-06,\n 3.7114e-05, 1.3601e-05, 2.8786e-07, 2.3396e-06, 1.1741e-05, 1.0123e-06,\n 3.5387e-07, 5.2395e-07, 1.9907e-07, 3.8306e-06, 2.5894e-05, 2.9346e-05,\n 1.5151e-06, 1.7602e-06, 3.4232e-07, 2.5264e-06, 3.1109e-07, 3.1482e-06,\n 8.4973e-08, 1.6361e-06, 2.2276e-06, 2.8102e-06, 1.9264e-06, 6.7213e-06,\n 6.0095e-07, 4.4379e-06, 7.4076e-05, 2.6216e-07, 3.1425e-07, 2.0018e-06,\n 2.6312e-05, 5.1553e-06, 3.9774e-06, 2.0957e-07, 1.5014e-06, 5.2017e-07,\n 7.6077e-06, 2.7505e-07, 2.6698e-06, 2.0028e-05, 2.8186e-07, 2.8078e-07,\n 2.0671e-06, 4.0674e-07, 9.5549e-07, 5.1362e-07, 4.1712e-07, 2.0251e-07,\n 1.8838e-07, 1.6396e-07, 2.3972e-06, 1.7714e-06, 2.0675e-07, 1.6337e-06,\n 2.6528e-06, 1.1710e-06, 4.2382e-07, 2.6262e-07, 3.0730e-06, 2.5872e-06,\n 2.8888e-07, 6.0496e-06, 1.7880e-07, 1.2309e-06, 2.1888e-07, 3.3903e-06,\n 1.1948e-06, 2.0364e-07, 1.1713e-07, 1.2286e-07, 1.4488e-07, 6.7568e-07,\n 2.1705e-07, 5.5911e-07, 5.2506e-06, 1.9677e-05, 1.8724e-06, 4.3036e-07,\n 1.1241e-06, 1.0437e-05, 2.9158e-05, 9.3076e-06, 2.5489e-07, 1.1144e-05,\n 1.0503e-06, 2.1084e-06, 3.5590e-07, 1.2896e-06, 9.9658e-05, 1.9996e-07,\n 9.2183e-07, 1.9087e-07, 4.2749e-07, 2.2356e-07, 6.7962e-06, 1.3409e-07,\n 8.4374e-07, 3.3912e-07, 7.1996e-07, 6.0445e-07, 1.7664e-06, 1.7663e-06,\n 1.7113e-06, 9.2772e-07, 1.3859e-06, 2.1101e-07, 6.7713e-07, 7.7884e-07,\n 1.6865e-07, 1.5168e-07, 2.1025e-07, 8.5721e-07, 1.2862e-06, 1.6875e-07,\n 9.8857e-06, 1.5681e-07, 8.1302e-07, 8.7478e-06], device='cuda:0')" + }, + "42": { + "step": "tensor(12515.)", + "exp_avg": "tensor([-1.1752e-07, -4.2701e-05, 1.2966e-05, -1.4209e-05, -3.4288e-05,\n 2.3603e-05, -1.4478e-05, -9.8808e-06, 1.0122e-06, 6.0022e-07,\n -7.1208e-06, 2.0518e-06, 2.2705e-05, -2.8786e-05, 1.3269e-06,\n -2.3574e-05, 1.2326e-05, 7.3779e-05, 4.3673e-05, 1.3385e-05,\n 2.3083e-05, -1.9937e-06, 1.7770e-05, 1.5705e-05, -3.5482e-05,\n 2.7718e-07, 1.9931e-07, 2.0053e-07, -1.9454e-05, -2.8272e-06,\n -2.3510e-05, 1.1643e-05, 2.9515e-06, 6.2774e-05, -8.5098e-07,\n -1.0816e-07, -7.1194e-07, 2.8895e-06, 3.3859e-06, 1.3180e-04,\n -9.8612e-06, 2.6658e-05, -1.6369e-06, 3.2214e-06, -1.2383e-06,\n 7.5493e-06, 3.0897e-06, -4.8177e-06, -1.7507e-05, 5.2756e-06,\n -3.9834e-05, 3.0344e-06, 2.4509e-06, 1.9332e-05, -3.7449e-06,\n 2.3084e-06, -7.3982e-06, 9.8729e-06, -2.2476e-05, -5.6636e-05,\n -1.0769e-05, 1.9955e-06, 1.6410e-06, -8.8247e-06, -1.4742e-06,\n 4.2733e-07, 5.5528e-06, -1.3918e-06, -2.5403e-06, -1.4860e-05,\n 1.9022e-06, 3.4292e-06, -4.7507e-06, -1.2218e-08, 2.1540e-05,\n 4.2735e-05, -1.1586e-06, -2.1118e-06, -8.8741e-06, -4.0631e-05,\n 3.9375e-07, -1.5912e-05, -1.3428e-05, 5.5711e-06, -4.4780e-06,\n 4.4613e-06, 6.4705e-06, 1.6512e-06, 2.3811e-05, 6.8674e-06,\n 1.6562e-07, 1.5675e-06, 6.8895e-06, -3.9567e-06, -1.3591e-07,\n -1.8265e-06, 1.8119e-05, 3.0389e-05, -7.5862e-06, -2.8510e-07,\n -6.6031e-07, 2.6588e-06, 1.6919e-06, 1.4842e-04, -6.7953e-07,\n 1.7317e-05, -1.7965e-06, 1.9527e-06, 7.2504e-07, -8.6569e-06,\n 1.0449e-05, 3.8007e-06, -4.4253e-06, 5.1888e-06, -4.8201e-05,\n 1.3701e-05, 8.5264e-07, -3.2793e-06, -1.2940e-06, 4.1097e-05,\n 6.3977e-05, -9.2598e-07, 3.2105e-07, 1.0957e-05, -3.1379e-05,\n 3.3490e-05, -1.3749e-05, -2.0372e-06, 2.1960e-06, -1.1519e-06,\n 1.5561e-06, -1.2647e-06, 8.1949e-06, 1.9895e-05, -6.3126e-06,\n 3.7519e-06, -1.4246e-05, -3.6922e-05, 2.9312e-05, -4.2815e-05,\n -3.3639e-06, 1.5467e-05, 9.3037e-06, -1.7858e-06, -4.1093e-06,\n 8.6567e-06, 2.3540e-06, -5.0243e-05, -3.4907e-05, -3.9575e-05,\n -2.2878e-06, 1.7706e-05, -8.4318e-07, 3.6869e-06, 1.7466e-05,\n 4.4946e-05, 1.7399e-06, -1.1022e-05, 5.8014e-06, -7.8702e-08,\n 9.3225e-06, -2.0072e-05, -1.0891e-06, 2.2266e-05, -4.7673e-05,\n 3.9901e-06, -1.6195e-06, -1.7083e-05, 2.9906e-05, -2.2843e-05,\n -1.9376e-05, -5.9225e-06, 5.9601e-06, 8.2120e-07, -1.8305e-06,\n 4.0055e-07, 3.2857e-06, -1.0334e-05, 1.1520e-07, -8.0697e-06,\n -9.7771e-06, 2.3862e-06, 2.0071e-06, 1.2848e-07, -3.7526e-06,\n 4.3553e-06, -6.7962e-06, -3.6845e-07, -1.1810e-05, -1.3457e-05,\n -3.6432e-06, 3.0062e-05, 7.5719e-06, 2.2236e-05, -3.4399e-06,\n -3.2125e-06, 2.8099e-05, 2.9003e-05, -2.8789e-06, -1.4741e-05,\n -1.1785e-06, -1.1335e-06, -2.9783e-06, 3.4357e-07, 2.2863e-05,\n -1.6328e-06, -3.1821e-08, -2.2710e-07, 6.4052e-06, 2.5503e-06,\n 8.7624e-06, 1.2111e-06, 3.1802e-05, 2.9214e-05, 1.0321e-05,\n 7.9589e-07, -9.4915e-06, 3.5762e-05, 2.5671e-05, 2.9466e-05,\n 1.1953e-05, 5.7756e-06, -2.0211e-06, -2.4381e-06, 1.3319e-06,\n 9.1136e-06, 2.1437e-04, 7.7108e-07, 1.6779e-06, 5.6500e-07,\n 5.8388e-06, 3.5221e-06, 2.2721e-05, 1.1944e-06, -7.2566e-06,\n 7.6120e-06, -2.0176e-06, 2.1384e-06, 1.2547e-05, -1.0575e-05,\n 4.3456e-05, 2.6751e-05, -5.8833e-06, 5.1660e-06, -6.2298e-06,\n -1.7388e-07, 3.7014e-06, 4.5818e-06, 1.0931e-05, -8.9612e-06,\n 1.6290e-05, 1.2905e-07, 4.5980e-05, 1.3068e-05, 2.3798e-06,\n -2.3352e-05], device='cuda:0')", + "exp_avg_sq": "tensor([6.7688e-09, 3.9749e-08, 4.3258e-08, 1.2357e-08, 4.7559e-08, 1.0009e-08,\n 5.3207e-09, 8.8288e-09, 2.2994e-10, 1.2710e-10, 7.0219e-10, 1.2012e-08,\n 3.8478e-08, 5.4008e-08, 9.9694e-10, 3.1404e-08, 7.0616e-09, 9.4481e-08,\n 3.0369e-08, 1.5596e-08, 7.0407e-09, 7.0609e-11, 2.8946e-08, 6.2940e-10,\n 3.6622e-08, 4.6875e-10, 9.4324e-10, 4.4287e-10, 1.0462e-08, 1.3921e-10,\n 4.0138e-08, 1.5130e-08, 5.7713e-09, 1.2159e-07, 6.6444e-10, 1.4041e-10,\n 8.0100e-09, 2.1808e-10, 3.0067e-10, 8.1141e-08, 1.4323e-09, 1.3200e-08,\n 3.6494e-10, 1.9766e-10, 9.2258e-11, 3.0787e-10, 4.0204e-10, 6.1451e-10,\n 1.0370e-07, 4.9569e-09, 7.7677e-08, 1.0034e-10, 3.9248e-10, 2.0641e-08,\n 2.0729e-10, 1.1852e-10, 9.7027e-09, 4.1821e-08, 8.4823e-09, 2.3473e-08,\n 4.9782e-09, 2.3727e-10, 9.0796e-11, 1.6994e-08, 7.9262e-10, 3.1198e-10,\n 9.8548e-10, 1.4989e-09, 1.3415e-08, 3.8187e-08, 2.2727e-08, 1.5548e-09,\n 2.7874e-08, 3.0395e-10, 1.9585e-08, 1.4977e-08, 5.3505e-10, 7.0650e-10,\n 1.6883e-09, 2.5467e-08, 3.8521e-10, 1.6173e-08, 5.2878e-09, 1.3615e-09,\n 8.3724e-11, 4.8336e-10, 4.2105e-10, 1.8237e-09, 1.0044e-08, 1.3939e-10,\n 7.9624e-10, 5.2816e-11, 7.6682e-09, 7.7628e-10, 2.9838e-11, 1.7382e-10,\n 9.4755e-09, 1.4270e-08, 2.3238e-10, 4.4834e-10, 1.2219e-08, 4.2653e-11,\n 1.1053e-09, 9.8804e-08, 5.2095e-10, 3.7831e-09, 2.3982e-08, 2.6124e-10,\n 5.5605e-11, 1.8961e-09, 1.9778e-09, 7.3223e-09, 2.2575e-10, 5.6617e-09,\n 4.9228e-08, 1.5524e-08, 5.0407e-10, 1.3606e-09, 1.6658e-10, 1.2413e-08,\n 3.9474e-08, 4.8147e-10, 4.9272e-10, 8.0369e-09, 3.0401e-08, 8.5182e-08,\n 2.7385e-09, 1.6260e-09, 1.6716e-10, 6.0099e-09, 1.2341e-10, 3.2068e-10,\n 1.0969e-08, 2.1939e-08, 1.1598e-10, 5.0631e-09, 2.3560e-08, 6.2026e-08,\n 4.9232e-08, 3.0099e-08, 3.2133e-09, 2.4625e-08, 2.6113e-08, 2.1254e-09,\n 4.8627e-10, 1.1179e-09, 3.6435e-10, 1.5219e-08, 6.2949e-08, 4.2144e-08,\n 3.1518e-09, 4.4237e-09, 4.3207e-10, 1.6380e-08, 1.0971e-09, 3.9521e-08,\n 7.8148e-11, 2.8882e-09, 3.2051e-08, 8.5782e-09, 6.6333e-09, 2.2036e-08,\n 1.6717e-09, 1.2675e-08, 9.5657e-08, 5.0616e-10, 9.4264e-10, 9.1258e-09,\n 5.0369e-08, 1.5892e-08, 2.6908e-08, 1.3171e-10, 1.4904e-08, 3.3073e-09,\n 2.1909e-08, 4.0757e-10, 4.9435e-09, 3.7127e-08, 1.7517e-10, 3.3554e-10,\n 1.2220e-08, 6.8085e-10, 4.1650e-09, 3.2160e-09, 5.3541e-10, 5.7372e-10,\n 8.1272e-11, 2.9514e-11, 5.1426e-09, 1.4674e-08, 2.1516e-10, 9.7396e-09,\n 7.4729e-09, 2.3065e-09, 5.5463e-10, 2.4221e-10, 2.8635e-08, 1.7036e-08,\n 4.1101e-10, 1.9131e-08, 7.7648e-11, 3.2970e-09, 2.3938e-10, 2.2079e-08,\n 5.3056e-09, 7.0935e-10, 4.9834e-11, 4.5564e-11, 3.1965e-10, 2.0289e-09,\n 3.9057e-10, 1.2536e-09, 1.1008e-08, 5.3062e-08, 3.6489e-09, 5.8902e-10,\n 3.2148e-09, 3.0194e-08, 6.8288e-08, 5.9119e-08, 1.1823e-09, 2.5414e-08,\n 7.8699e-09, 4.9261e-09, 2.1520e-10, 4.5606e-09, 1.5949e-07, 3.6967e-10,\n 1.3587e-09, 3.6389e-11, 1.1295e-09, 3.7487e-10, 1.9187e-08, 8.2194e-11,\n 3.1668e-09, 1.5073e-09, 2.4156e-09, 1.2078e-09, 5.2915e-09, 4.1603e-09,\n 1.2096e-08, 2.5302e-09, 7.7855e-09, 5.4584e-10, 1.0611e-09, 2.3322e-09,\n 1.2829e-10, 9.6232e-11, 8.8612e-10, 3.5061e-09, 3.3466e-09, 5.3423e-11,\n 3.8174e-08, 2.8501e-10, 9.9412e-10, 1.8598e-08], device='cuda:0')" + }, + "43": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 1.0297e-05, 2.8139e-06, 3.2508e-05, 4.2666e-06, 1.9496e-05,\n 5.8615e-05, -8.2283e-06, 3.9067e-05, 1.3949e-06, 1.4854e-06,\n 2.9987e-06, 3.8806e-06, -1.7428e-05, 4.2363e-06, -1.7691e-06,\n -1.8308e-05, -4.9033e-06, -7.1393e-06, 2.4853e-05, 2.2368e-06,\n 3.9102e-05, 4.2042e-06, 1.2314e-05, 2.0756e-05, -3.9098e-05,\n 2.1147e-07, -6.2709e-06, 2.8666e-06, -4.4164e-05, 3.5962e-06,\n 1.6411e-05, 1.0264e-05, -3.5287e-05, 5.2702e-05, 3.9971e-06,\n 3.4386e-06, -3.6851e-06, -3.2417e-06, 1.7921e-06, 1.2468e-04,\n -1.5224e-05, 1.9754e-05, 3.7894e-06, -1.4372e-06, 2.9657e-06,\n 5.0227e-06, 3.9873e-06, 4.2813e-06, -3.9866e-05, 5.1516e-06,\n -4.3775e-05, 3.0619e-06, -8.6834e-07, 5.7249e-06, 5.7211e-06,\n 3.9483e-06, -1.0150e-05, 1.9782e-05, 3.1269e-05, -3.2799e-05,\n -6.8821e-06, -4.7697e-06, 2.3096e-06, -1.0438e-05, -1.6632e-05,\n 4.1599e-06, 7.1898e-06, -2.3853e-05, 3.9078e-05, -7.7801e-05,\n 1.5398e-05, -1.5764e-06, 2.4363e-05, 1.3485e-05, 1.7304e-05,\n 1.2669e-04, -2.1077e-06, -2.1285e-07, -1.1555e-05, -2.8264e-05,\n -3.7410e-06, -1.5270e-05, -3.3775e-05, 9.2462e-06, -1.9616e-06,\n 5.0417e-06, 6.2152e-06, 2.2464e-06, -3.0408e-05, 1.0035e-05,\n 1.7896e-05, 8.1604e-07, -7.5436e-06, 2.6975e-06, -2.4475e-06,\n -1.6415e-06, 5.1103e-05, 1.2906e-05, -1.1098e-05, 2.4806e-08,\n 7.2739e-06, 2.5081e-06, 2.1092e-06, 8.2002e-05, 5.2367e-06,\n 1.4963e-05, 6.9480e-06, 1.7274e-06, 1.2077e-06, -7.2340e-06,\n 6.1805e-06, -1.0415e-05, 2.2759e-06, 3.4704e-05, -7.5323e-05,\n -2.3204e-06, -2.9307e-06, -1.4184e-05, -1.3982e-06, 1.4677e-06,\n 2.7078e-05, -9.6160e-06, 1.5465e-06, -1.7416e-05, 6.7397e-06,\n 8.1905e-05, -6.7858e-06, -1.1806e-05, 3.6767e-06, -1.9308e-05,\n -2.2923e-06, -3.9869e-06, -2.3415e-07, 3.9534e-05, -5.1612e-06,\n 1.3195e-05, -1.5567e-05, -2.5840e-05, 1.4769e-04, -2.5332e-05,\n -9.2284e-06, -1.6192e-05, 5.4598e-06, -4.5665e-06, -4.8817e-06,\n 8.8758e-06, 8.3999e-07, -8.4340e-05, -5.1759e-05, 8.5767e-06,\n 5.8214e-06, 1.3797e-06, 6.4477e-07, 7.2082e-06, -3.6185e-06,\n 2.6508e-05, -2.6336e-06, -1.0323e-05, -1.1358e-05, 4.6012e-06,\n -7.3087e-06, -5.9630e-05, -1.1001e-05, 1.2705e-05, 1.4608e-04,\n 8.5175e-07, -7.1995e-06, -1.8188e-05, 9.1179e-05, -5.3642e-06,\n -1.3705e-05, -2.7388e-06, -1.6462e-05, -4.1625e-06, -2.4818e-05,\n 2.7759e-06, 1.5541e-05, -1.3237e-04, 4.7528e-06, -1.0274e-05,\n -9.4036e-06, 4.1150e-06, -3.4469e-06, -2.8746e-06, -5.0351e-06,\n 8.0996e-06, -5.9022e-06, -3.7478e-08, 1.0207e-05, -8.2507e-06,\n -6.9786e-07, 5.0778e-06, -1.2592e-05, 3.2575e-05, 1.6918e-06,\n 3.3671e-06, -1.7565e-05, 1.8794e-05, -1.4932e-05, 8.1168e-05,\n 2.1821e-06, -1.5067e-05, -7.5774e-06, 6.3988e-06, 1.1251e-05,\n -1.0644e-06, -5.3713e-07, 1.3247e-06, 2.1255e-06, -8.2657e-06,\n 1.5158e-06, -2.4429e-06, 3.3299e-05, 8.8073e-05, 1.9105e-05,\n 2.2748e-06, -2.1299e-05, -1.2646e-05, 3.2378e-05, 1.2019e-05,\n 1.1063e-05, 2.0837e-05, -1.1447e-05, 6.9656e-06, -5.8053e-06,\n -1.5901e-06, 1.8955e-04, 3.1601e-06, 3.5682e-06, 9.6882e-07,\n 8.4596e-06, 4.6264e-06, 3.3619e-05, 1.5966e-06, -3.5447e-06,\n 6.9901e-06, -1.0025e-05, 2.4344e-06, 6.6646e-06, 4.0645e-06,\n 2.3808e-06, 2.3050e-05, -1.7501e-05, 5.9423e-06, -3.2081e-06,\n -1.1453e-05, 2.1011e-06, 6.4403e-06, 7.1755e-06, -4.7380e-06,\n 5.8491e-06, 3.9282e-06, 8.6339e-05, 1.1440e-05, 1.2074e-05,\n -5.3347e-05], device='cuda:0')", + "exp_avg_sq": "tensor([1.0751e-08, 4.6361e-08, 6.3487e-08, 1.5321e-08, 7.6609e-08, 1.3410e-08,\n 3.3939e-09, 1.1098e-08, 4.4221e-10, 6.3738e-10, 2.4697e-09, 1.6207e-08,\n 3.2326e-08, 7.8957e-08, 2.7792e-09, 3.9600e-08, 9.0115e-09, 7.5715e-08,\n 3.5297e-08, 5.0307e-08, 9.2863e-09, 3.5162e-10, 3.0054e-08, 2.1648e-09,\n 5.4346e-08, 1.1346e-09, 2.4363e-09, 1.6354e-09, 1.4120e-08, 1.4293e-09,\n 3.1944e-08, 1.8300e-08, 1.2403e-08, 1.5064e-07, 1.3444e-09, 5.5766e-10,\n 1.6043e-08, 1.1947e-10, 1.0499e-09, 1.1995e-07, 2.8055e-09, 2.2490e-08,\n 1.4374e-09, 1.9485e-10, 4.8331e-10, 4.9802e-10, 1.6077e-09, 2.6282e-09,\n 9.1204e-08, 5.9645e-09, 8.8399e-08, 4.6712e-10, 1.0024e-09, 6.9256e-09,\n 2.0605e-10, 4.2744e-10, 1.3332e-08, 1.0287e-07, 1.5223e-08, 2.7039e-08,\n 9.0698e-09, 1.1427e-09, 3.9797e-10, 2.5300e-08, 2.5619e-09, 1.8148e-09,\n 1.0857e-09, 2.0898e-09, 2.2764e-08, 6.1922e-08, 1.4926e-08, 5.3690e-09,\n 3.8829e-08, 1.7053e-09, 8.6217e-09, 3.6185e-08, 1.3719e-09, 1.2914e-09,\n 3.9385e-09, 1.9996e-08, 1.0281e-09, 3.7260e-09, 1.3076e-08, 1.7525e-09,\n 5.4684e-10, 1.0211e-09, 6.9534e-10, 4.7580e-09, 1.2250e-08, 8.5162e-10,\n 1.8917e-09, 1.9386e-10, 6.8467e-09, 1.8938e-09, 2.2081e-10, 4.3958e-10,\n 1.4493e-08, 2.1550e-08, 8.7470e-10, 1.0990e-09, 2.0836e-08, 2.3567e-10,\n 1.3496e-09, 9.4375e-08, 1.8911e-09, 5.4949e-09, 3.9236e-08, 7.2456e-10,\n 2.4939e-10, 2.9402e-09, 2.3631e-09, 4.0437e-09, 9.1981e-10, 1.0731e-08,\n 7.1388e-08, 1.0528e-08, 9.7758e-10, 2.4936e-09, 6.7732e-10, 1.0608e-08,\n 9.9614e-08, 1.7667e-09, 5.8012e-10, 1.2234e-08, 2.7662e-08, 1.1296e-07,\n 8.3561e-09, 3.2964e-09, 2.9758e-10, 1.0409e-08, 1.0658e-09, 1.7674e-09,\n 1.8068e-08, 4.2858e-08, 4.2972e-10, 1.2720e-08, 4.8671e-09, 3.4897e-08,\n 9.1929e-08, 3.4361e-08, 1.6007e-09, 1.3781e-08, 4.7482e-08, 5.7036e-09,\n 1.6249e-09, 2.4652e-09, 7.8611e-10, 2.0998e-08, 7.1586e-08, 8.9801e-08,\n 7.9867e-09, 7.2382e-09, 1.5327e-09, 1.5169e-08, 1.6013e-09, 2.1712e-08,\n 1.4521e-10, 5.6843e-09, 1.2299e-08, 1.3385e-08, 1.4722e-08, 3.7597e-08,\n 3.0184e-09, 1.9895e-08, 1.9166e-07, 1.7841e-09, 1.4137e-09, 1.1658e-08,\n 7.0969e-08, 2.0739e-08, 2.2051e-08, 8.1724e-10, 1.1313e-08, 3.7922e-09,\n 3.3296e-08, 8.2694e-10, 1.0909e-08, 8.0375e-08, 1.4048e-09, 1.0386e-09,\n 1.4812e-08, 1.8501e-09, 5.1715e-09, 3.3997e-09, 1.4823e-09, 1.1425e-09,\n 2.6571e-10, 6.9109e-11, 1.1609e-08, 1.1125e-08, 5.6519e-10, 1.1517e-08,\n 1.3869e-08, 5.3163e-09, 2.3596e-09, 1.4225e-09, 2.1701e-08, 1.7160e-08,\n 1.7867e-09, 2.3876e-08, 4.2061e-10, 7.0264e-09, 6.1342e-10, 1.7766e-08,\n 6.1131e-09, 1.0761e-09, 2.0854e-10, 4.4101e-10, 5.0213e-10, 3.0875e-09,\n 1.0888e-09, 2.5625e-09, 2.0467e-08, 7.2322e-08, 1.1855e-08, 1.7255e-09,\n 6.1704e-09, 4.6369e-08, 1.0008e-07, 5.2793e-08, 1.8829e-09, 4.7103e-08,\n 5.7740e-09, 1.0282e-08, 1.5043e-09, 6.8524e-09, 2.4528e-07, 1.5816e-09,\n 4.1308e-09, 5.5875e-11, 2.4890e-09, 7.9849e-10, 3.2793e-08, 2.6145e-10,\n 5.2002e-09, 1.8113e-09, 4.3008e-09, 2.6255e-09, 9.5200e-09, 8.8700e-09,\n 1.2280e-08, 5.0931e-09, 8.6391e-09, 8.5707e-10, 3.3027e-09, 4.2823e-09,\n 5.1036e-10, 2.7450e-10, 1.3115e-09, 4.0779e-09, 5.2820e-09, 4.2116e-10,\n 4.0821e-08, 5.6595e-10, 4.5196e-09, 4.4847e-08], device='cuda:0')" + }, + "44": { + "step": "tensor(12515.)", + "exp_avg": "tensor([[ 1.4630e-08, 2.8455e-08, 3.6804e-08, ..., 2.0015e-08,\n -5.3790e-09, 4.4662e-09],\n [ 4.2723e-08, 7.4143e-08, 1.1372e-07, ..., 3.4877e-08,\n -1.9451e-08, 3.3144e-09],\n [ 9.7812e-06, 2.9438e-04, 1.8072e-06, ..., 3.9215e-05,\n -1.6177e-05, -5.0492e-06],\n [-1.5202e-05, 1.3661e-04, -4.8936e-05, ..., -4.7981e-05,\n -1.8269e-05, 9.2453e-05],\n [-3.6810e-05, -4.0603e-04, 3.2490e-04, ..., -2.1266e-04,\n -5.2393e-05, 1.5387e-04],\n [ 4.2163e-05, -2.5083e-05, -2.7847e-04, ..., 2.2139e-04,\n 8.6861e-05, -2.4127e-04]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.3340e-13, 2.4755e-13, 2.9711e-13, ..., 1.0344e-13, 3.5053e-13,\n 2.2174e-11],\n [3.6852e-13, 1.4759e-12, 5.6955e-13, ..., 1.5682e-13, 3.4075e-13,\n 3.2934e-11],\n [2.0667e-08, 8.0183e-07, 1.0773e-08, ..., 9.9342e-09, 7.8126e-09,\n 5.9028e-10],\n [1.5339e-07, 2.6672e-06, 6.1716e-07, ..., 5.9231e-08, 2.9827e-08,\n 8.7971e-08],\n [5.3683e-07, 3.3441e-06, 2.9934e-05, ..., 1.6742e-07, 1.0878e-07,\n 3.3388e-07],\n [4.5912e-07, 9.2057e-08, 3.0116e-05, ..., 1.6039e-07, 9.2303e-08,\n 4.4649e-07]], device='cuda:0')" + }, + "45": { + "step": "tensor(12515.)", + "exp_avg": "tensor([ 7.2885e-08, 1.7900e-07, 1.1427e-04, -3.0988e-05, -2.5808e-05,\n -5.7722e-05], device='cuda:0')", + "exp_avg_sq": "tensor([4.7878e-12, 7.4995e-12, 2.2945e-07, 1.0374e-06, 3.7013e-06, 3.1743e-06],\n device='cuda:0')" + } + }, + "param_groups": [ + { + "lr": 0.0005005000000000001, + "name": "scale_256", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 0, + 1, + 2 + ] + }, + { + "lr": 0.0005005000000000001, + "name": "scale_512", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 3, + 4, + 5 + ] + }, + { + "lr": 0.0005005000000000001, + "name": "scale_768", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 6, + 7, + 8 + ] + }, + { + "lr": 0.0005005000000000001, + "name": "scale_1024", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 9, + 10, + 11 + ] + }, + { + "lr": 0.0005005000000000001, + "name": "scale_2048", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 12, + 13, + 14 + ] + }, + { + "lr": 0.0005005000000000001, + "name": "scale_4096", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 15, + 16, + 17 + ] + }, + { + "lr": 0.0002505, + "name": "fusion", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0005, + "params": [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45 + ] + } + ] + }, + "scheduler_state_dict": { + "T_0": 10, + "T_i": 10, + "T_mult": 2, + "eta_min": 1e-06, + "T_cur": 5, + "base_lrs": [ + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.0005 + ], + "last_epoch": 5, + "_step_count": 0, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0005005000000000001, + 0.0005005000000000001, + 0.0005005000000000001, + 0.0005005000000000001, + 0.0005005000000000001, + 0.0005005000000000001, + 0.0002505 + ] + }, + "metrics": { + "best_val_acc": 78.904, + "best_epoch": 4, + "scale_accuracies": { + "256": 74.616, + "512": 77.184, + "768": 77.982, + "1024": 77.986, + "2048": 77.906, + "4096": 77.97 + }, + "training_history": { + "epochs": [ + 1, + 2, + 3, + 4, + 5 + ], + "train_loss": [ + 2.66668762367247, + 1.9412667955630216, + 1.7480393170453146, + 1.593776029584125, + 1.4550095543688029 + ], + "train_acc": [ + 75.19659810157458, + 80.85815510390137, + 82.94141200951944, + 84.7887121663296, + 86.63156325443911 + ], + "val_acc": [ + 76.602, + 77.83, + 78.244, + 78.386, + 78.904 + ], + "scale_accs": { + "256": [ + 71.976, + 73.134, + 73.802, + 74.2, + 74.616 + ], + "512": [ + 74.694, + 76.192, + 76.514, + 76.866, + 77.184 + ], + "768": [ + 75.594, + 76.828, + 77.4, + 77.528, + 77.982 + ], + "1024": [ + 75.882, + 77.01, + 77.448, + 77.684, + 77.986 + ], + "2048": [ + 76.116, + 77.174, + 77.336, + 77.404, + 77.906 + ], + "4096": [ + 75.64, + 77.066, + 77.386, + 77.624, + 77.97 + ] + }, + "lr": [ + 0.0009755527298894294, + 0.0009046039886902864, + 0.0007940987335200904, + 0.0006548539886902864, + 0.0005005000000000001 + ] + } + }, + "train_config": { + "name": "david_training", + "run_id": "20251104_154540", + "seed": 42, + "deterministic": false, + "dataset_name": "AbstractPhil/imagenet-clip-features-orderly", + "model_variant": "clip_vit_b16", + "num_classes": 1000, + "preset": "clip_vit_b16_cantor_big_window", + "custom_config_path": null, + "num_classes_override": null, + "use_belly_override": null, + "belly_expand_override": null, + "progressive_training_override": false, + "scale_warmup_epochs_override": null, + "num_epochs": 5, + "batch_size": 512, + "learning_rate": 0.001, + "weight_decay": 1e-05, + "warmup_epochs": 1, + "use_rose_loss": true, + "rose_initial_weight": 0.1, + "rose_max_weight": 0.5, + "rose_weight_schedule": "adaptive", + "use_cayley_loss": false, + "cayley_weight": 0.001, + "scale_loss_balance": null, + "use_mixed_precision": false, + "gradient_clip": 10.0, + "scheduler_type": "cosine_restarts", + "min_lr": 1e-06, + "freeze_strategy": "none", + "freeze_threshold": 90.0, + "unfreeze_on_plateau": true, + "patience": 10, + "track_gradients": true, + "gradient_scale_threshold": 1e-05, + "gradient_scale_multiplier": 10.0, + "log_interval": 50, + "val_interval": 1, + "save_interval": 5, + "log_fusion_weights": true, + "log_loss_components": true, + "save_format": "safetensors", + "hf_repo": "AbstractPhil/gated-david", + "upload_to_hub": true, + "base_dir": "./david_training", + "num_workers": 8, + "pin_memory": true, + "prefetch_factor": 4, + "persistent_workers": true + } +} \ No newline at end of file