| { |
| "base_model": "krea/Krea-2-Turbo", |
| "format": "krea2-svdquant-transformer-v1", |
| "target_component": "transformer", |
| "transformer_class": "Krea2Transformer2DModel", |
| "runtime_target": "w4a16_plus_bf16_lowrank", |
| "weight_bits": 4, |
| "activation_bits": 16, |
| "group_size": 128, |
| "calibrated": true, |
| "num_blocks": 28, |
| "options": { |
| "rank": 32, |
| "attn_rank": 64, |
| "mlp_rank": 128, |
| "skip_first_blocks": 0, |
| "skip_last_blocks": 0, |
| "skip_qk": false, |
| "max_layers": 0, |
| "svd_oversample": 16, |
| "svd_niter": 2, |
| "svd_exact": false |
| }, |
| "layers": { |
| "transformer_blocks.0.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.0.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.1.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.2.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.3.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.4.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.5.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.6.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.7.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.8.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.9.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.10.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.11.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.12.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.13.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.14.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.15.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.16.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.17.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.18.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.19.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.20.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.21.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.22.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.23.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.24.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.25.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.26.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.attn.to_q": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.attn.to_k": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.attn.to_v": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 1536, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.attn.to_gate": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.attn.to_out.0": { |
| "rank": 64, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.ff.gate": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.ff.up": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 16384, |
| 6144 |
| ], |
| "calibrated": true |
| }, |
| "transformer_blocks.27.ff.down": { |
| "rank": 128, |
| "group_size": 128, |
| "shape": [ |
| 6144, |
| 16384 |
| ], |
| "calibrated": true |
| } |
| } |
| } |
