Training in progress, step 450
Browse files
adapter_model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 70430032
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fd57c1420c3542a77e83ba3c94e14009e93dc495cee37644f59f7f5877d00f4e
|
| 3 |
size 70430032
|
kernel_metrics/kernel_behavior_analysis.txt
CHANGED
|
@@ -3,8 +3,8 @@ KERNEL BEHAVIOR ANALYSIS
|
|
| 3 |
Based on Malladi et al. (2023) Section 6.2
|
| 4 |
================================================================================
|
| 5 |
|
| 6 |
-
Training steps analyzed: 0 to
|
| 7 |
-
Number of checkpoints:
|
| 8 |
Number of eval prompts: 45
|
| 9 |
Kernel types computed: sgd, signsgd, asymmetric_signsgd
|
| 10 |
|
|
@@ -14,7 +14,7 @@ Kernel types computed: sgd, signsgd, asymmetric_signsgd
|
|
| 14 |
Tests if f(x; θ_FT) ≈ f(x; θ_0) + <∇f(x; θ_0), θ_FT − θ_0>
|
| 15 |
Note: gradient is at θ_0 (initial), NOT at θ_FT.
|
| 16 |
|
| 17 |
-
Final linearization error:
|
| 18 |
|
| 19 |
❌ FAIL: Linearization property does not hold
|
| 20 |
Behavior has departed significantly from linear regime.
|
|
@@ -26,9 +26,9 @@ Reference: Malladi Figure 2: >50% of improvement recovered by linearization.
|
|
| 26 |
--------------------------------------------------------------------------------
|
| 27 |
Tests if K(θ_t) ≈ K(θ_0) throughout training
|
| 28 |
|
| 29 |
-
[sgd] Final kernel distance:
|
| 30 |
-
[signsgd] Final kernel distance:
|
| 31 |
-
[asymmetric_signsgd] Final kernel distance:
|
| 32 |
|
| 33 |
Thresholds: <10% strong lazy learning | <30% moderate | >30% feature learning
|
| 34 |
Reference: Malladi Table 7: ~2-6% distance for tasks with kernel behavior.
|
|
|
|
| 3 |
Based on Malladi et al. (2023) Section 6.2
|
| 4 |
================================================================================
|
| 5 |
|
| 6 |
+
Training steps analyzed: 0 to 400
|
| 7 |
+
Number of checkpoints: 9
|
| 8 |
Number of eval prompts: 45
|
| 9 |
Kernel types computed: sgd, signsgd, asymmetric_signsgd
|
| 10 |
|
|
|
|
| 14 |
Tests if f(x; θ_FT) ≈ f(x; θ_0) + <∇f(x; θ_0), θ_FT − θ_0>
|
| 15 |
Note: gradient is at θ_0 (initial), NOT at θ_FT.
|
| 16 |
|
| 17 |
+
Final linearization error: 423.85%
|
| 18 |
|
| 19 |
❌ FAIL: Linearization property does not hold
|
| 20 |
Behavior has departed significantly from linear regime.
|
|
|
|
| 26 |
--------------------------------------------------------------------------------
|
| 27 |
Tests if K(θ_t) ≈ K(θ_0) throughout training
|
| 28 |
|
| 29 |
+
[sgd] Final kernel distance: 85.04%
|
| 30 |
+
[signsgd] Final kernel distance: 70.42% ← correct kernel for AdamW
|
| 31 |
+
[asymmetric_signsgd] Final kernel distance: 72.91%
|
| 32 |
|
| 33 |
Thresholds: <10% strong lazy learning | <30% moderate | >30% feature learning
|
| 34 |
Reference: Malladi Table 7: ~2-6% distance for tasks with kernel behavior.
|
kernel_metrics/kernel_metrics.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
kernel_metrics/kernel_summary.json
CHANGED
|
@@ -134,5 +134,22 @@
|
|
| 134 |
"kernel_distance_asymmetric_signsgd": 0.7073289752006531,
|
| 135 |
"linearization_error": 4.403439813055527,
|
| 136 |
"kernel_distance": 0.8333620429039001
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
}
|
| 138 |
]
|
|
|
|
| 134 |
"kernel_distance_asymmetric_signsgd": 0.7073289752006531,
|
| 135 |
"linearization_error": 4.403439813055527,
|
| 136 |
"kernel_distance": 0.8333620429039001
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"step": 400,
|
| 140 |
+
"kernel_trace": 411456.625,
|
| 141 |
+
"kernel_frobenius_norm": 151568.5625,
|
| 142 |
+
"mean_gradient_norm": 88.02743045243014,
|
| 143 |
+
"kernel_trace_sgd": 411456.625,
|
| 144 |
+
"kernel_frobenius_norm_sgd": 151568.5625,
|
| 145 |
+
"kernel_distance_sgd": 0.8504035472869873,
|
| 146 |
+
"kernel_trace_signsgd": 791838720.0,
|
| 147 |
+
"kernel_frobenius_norm_signsgd": 136973952.0,
|
| 148 |
+
"kernel_distance_signsgd": 0.704200804233551,
|
| 149 |
+
"kernel_trace_asymmetric_signsgd": 6100248.0,
|
| 150 |
+
"kernel_frobenius_norm_asymmetric_signsgd": 1615226.625,
|
| 151 |
+
"kernel_distance_asymmetric_signsgd": 0.7290645241737366,
|
| 152 |
+
"linearization_error": 4.238533284174912,
|
| 153 |
+
"kernel_distance": 0.8504035472869873
|
| 154 |
}
|
| 155 |
]
|