{ "model": "Zyphra/ZAYA1-8B-legacy", "quantization": { "method": "compressed-tensors", "format": "float-quantized", "scheme": "NVFP4", "scheme_variant": "w4a4", "num_bits_weight": 4, "num_bits_activation": 4, "type": "float", "strategy": "tensor_group", "group_size": 16, "target_modules": [ "Linear" ], "ignore": [ "lm_head", "re:.*router.*", "re:.*norm.*", "re:.*qkv.*", "re:.*cca.*" ], "weight_packed_dtype": "uint8", "weight_scale_dtype": "float8_e4m3fn", "weight_global_scale_dtype": "float32", "input_global_scale_dtype": "float32", "zero_point": "none (symmetric)", "compressor": "NVFP4PackedCompressor" }, "calibration": { "data_path": "data/calibration/arcmix/calibration_data.pt", "num_samples_used": 977, "batch_size": 1, "linears_calibrated": 1320, "soar": true, "mr_gptq": false }, "mixed_precision": { "threshold": 500.0, "outlier_layers": [ 1, 19, 31, 37, 39, 65, 69, 71, 73, 75, 77, 79 ], "bf16_exempted_modules": 0, "w4a4_compressed_modules": 1320, "arcquant_target": false, "description": "12 MoE layers with max_abs > 500 were detected and COMPRESSED to W4A4 anyway (--no-bf16-exempt), giving 0 BF16-exempted modules. Evaluated in this state, with no residual corrections applied: -0.71 pp vs the 384-exemption checkpoint on HellaSwag (n=10,042, 95% CI [-1.26, -0.15], paired McNemar). The per-layer BF16 exemption cost 3.44 GB because FusedMoE requires uniform quantization per layer, turning 24 offending linear_fc2 modules into 384 exempted Linears.", "arcquant_note": "Originally flagged true on the assumption that removing exemptions would require residual correction. Phase A measured the uncorrected checkpoint at -0.71 pp on HellaSwag, so no correction was applied or is required." }, "activation_max_per_module": { "model.layers.0.self_attn.o_proj": 9.5625, "model.layers.1.zaya_block.experts.local_experts.0.linear_fc1": 7.25, "model.layers.1.zaya_block.experts.local_experts.0.linear_fc2": 540.0, "model.layers.1.zaya_block.experts.local_experts.1.linear_fc1": 7.9062, "model.layers.1.zaya_block.experts.local_experts.1.linear_fc2": 502.0, "model.layers.1.zaya_block.experts.local_experts.2.linear_fc1": 7.4375, "model.layers.1.zaya_block.experts.local_experts.2.linear_fc2": 504.0, "model.layers.1.zaya_block.experts.local_experts.3.linear_fc1": 7.7188, "model.layers.1.zaya_block.experts.local_experts.3.linear_fc2": 632.0, "model.layers.1.zaya_block.experts.local_experts.4.linear_fc1": 6.9688, "model.layers.1.zaya_block.experts.local_experts.4.linear_fc2": 368.0, "model.layers.1.zaya_block.experts.local_experts.5.linear_fc1": 7.5625, "model.layers.1.zaya_block.experts.local_experts.5.linear_fc2": 300.0, "model.layers.1.zaya_block.experts.local_experts.6.linear_fc1": 8.75, "model.layers.1.zaya_block.experts.local_experts.6.linear_fc2": 588.0, "model.layers.1.zaya_block.experts.local_experts.7.linear_fc1": 8.1875, "model.layers.1.zaya_block.experts.local_experts.7.linear_fc2": 326.0, "model.layers.1.zaya_block.experts.local_experts.8.linear_fc1": 7.4688, "model.layers.1.zaya_block.experts.local_experts.8.linear_fc2": 255.0, "model.layers.1.zaya_block.experts.local_experts.9.linear_fc1": 7.5, "model.layers.1.zaya_block.experts.local_experts.9.linear_fc2": 332.0, "model.layers.1.zaya_block.experts.local_experts.10.linear_fc1": 7.0312, "model.layers.1.zaya_block.experts.local_experts.10.linear_fc2": 314.0, "model.layers.1.zaya_block.experts.local_experts.11.linear_fc1": 6.0938, "model.layers.1.zaya_block.experts.local_experts.11.linear_fc2": 580.0, "model.layers.1.zaya_block.experts.local_experts.12.linear_fc1": 8.3125, "model.layers.1.zaya_block.experts.local_experts.12.linear_fc2": 280.0, "model.layers.1.zaya_block.experts.local_experts.13.linear_fc1": 6.0938, "model.layers.1.zaya_block.experts.local_experts.13.linear_fc2": 354.0, "model.layers.1.zaya_block.experts.local_experts.14.linear_fc1": 8.375, "model.layers.1.zaya_block.experts.local_experts.14.linear_fc2": 308.0, "model.layers.1.zaya_block.experts.local_experts.15.linear_fc1": 8.0, "model.layers.1.zaya_block.experts.local_experts.15.linear_fc2": 344.0, "model.layers.2.self_attn.o_proj": 9.4375, "model.layers.3.zaya_block.experts.local_experts.0.linear_fc1": 9.6875, "model.layers.3.zaya_block.experts.local_experts.0.linear_fc2": 44.75, "model.layers.3.zaya_block.experts.local_experts.1.linear_fc1": 9.6875, "model.layers.3.zaya_block.experts.local_experts.1.linear_fc2": 33.75, "model.layers.3.zaya_block.experts.local_experts.2.linear_fc1": 10.1875, "model.layers.3.zaya_block.experts.local_experts.2.linear_fc2": 38.5, "model.layers.3.zaya_block.experts.local_experts.3.linear_fc1": 10.1875, "model.layers.3.zaya_block.experts.local_experts.3.linear_fc2": 42.75, "model.layers.3.zaya_block.experts.local_experts.4.linear_fc1": 10.25, "model.layers.3.zaya_block.experts.local_experts.4.linear_fc2": 44.0, "model.layers.3.zaya_block.experts.local_experts.5.linear_fc1": 11.5, "model.layers.3.zaya_block.experts.local_experts.5.linear_fc2": 48.75, "model.layers.3.zaya_block.experts.local_experts.6.linear_fc1": 9.875, "model.layers.3.zaya_block.experts.local_experts.6.linear_fc2": 37.75, "model.layers.3.zaya_block.experts.local_experts.7.linear_fc1": 10.1875, "model.layers.3.zaya_block.experts.local_experts.7.linear_fc2": 35.5, "model.layers.3.zaya_block.experts.local_experts.8.linear_fc1": 9.75, "model.layers.3.zaya_block.experts.local_experts.8.linear_fc2": 29.875, "model.layers.3.zaya_block.experts.local_experts.9.linear_fc1": 10.3125, "model.layers.3.zaya_block.experts.local_experts.9.linear_fc2": 40.25, "model.layers.3.zaya_block.experts.local_experts.10.linear_fc1": 9.875, "model.layers.3.zaya_block.experts.local_experts.10.linear_fc2": 37.25, "model.layers.3.zaya_block.experts.local_experts.11.linear_fc1": 10.125, "model.layers.3.zaya_block.experts.local_experts.11.linear_fc2": 37.0, "model.layers.3.zaya_block.experts.local_experts.12.linear_fc1": 10.0625, "model.layers.3.zaya_block.experts.local_experts.12.linear_fc2": 33.5, "model.layers.3.zaya_block.experts.local_experts.13.linear_fc1": 9.9375, "model.layers.3.zaya_block.experts.local_experts.13.linear_fc2": 32.5, "model.layers.3.zaya_block.experts.local_experts.14.linear_fc1": 10.1875, "model.layers.3.zaya_block.experts.local_experts.14.linear_fc2": 47.0, "model.layers.3.zaya_block.experts.local_experts.15.linear_fc1": 10.0625, "model.layers.3.zaya_block.experts.local_experts.15.linear_fc2": 34.75, "model.layers.4.self_attn.o_proj": 33.75, "model.layers.5.zaya_block.experts.local_experts.0.linear_fc1": 8.0625, "model.layers.5.zaya_block.experts.local_experts.0.linear_fc2": 41.25, "model.layers.5.zaya_block.experts.local_experts.1.linear_fc1": 8.0, "model.layers.5.zaya_block.experts.local_experts.1.linear_fc2": 29.375, "model.layers.5.zaya_block.experts.local_experts.2.linear_fc1": 7.7188, "model.layers.5.zaya_block.experts.local_experts.2.linear_fc2": 33.5, "model.layers.5.zaya_block.experts.local_experts.3.linear_fc1": 7.8438, "model.layers.5.zaya_block.experts.local_experts.3.linear_fc2": 30.875, "model.layers.5.zaya_block.experts.local_experts.4.linear_fc1": 7.7812, "model.layers.5.zaya_block.experts.local_experts.4.linear_fc2": 29.625, "model.layers.5.zaya_block.experts.local_experts.5.linear_fc1": 7.8438, "model.layers.5.zaya_block.experts.local_experts.5.linear_fc2": 23.125, "model.layers.5.zaya_block.experts.local_experts.6.linear_fc1": 7.5938, "model.layers.5.zaya_block.experts.local_experts.6.linear_fc2": 27.0, "model.layers.5.zaya_block.experts.local_experts.7.linear_fc1": 8.0625, "model.layers.5.zaya_block.experts.local_experts.7.linear_fc2": 36.75, "model.layers.5.zaya_block.experts.local_experts.8.linear_fc1": 8.375, "model.layers.5.zaya_block.experts.local_experts.8.linear_fc2": 29.25, "model.layers.5.zaya_block.experts.local_experts.9.linear_fc1": 5.9062, "model.layers.5.zaya_block.experts.local_experts.9.linear_fc2": 28.0, "model.layers.5.zaya_block.experts.local_experts.10.linear_fc1": 8.0625, "model.layers.5.zaya_block.experts.local_experts.10.linear_fc2": 26.75, "model.layers.5.zaya_block.experts.local_experts.11.linear_fc1": 8.0, "model.layers.5.zaya_block.experts.local_experts.11.linear_fc2": 29.375, "model.layers.5.zaya_block.experts.local_experts.12.linear_fc1": 7.9375, "model.layers.5.zaya_block.experts.local_experts.12.linear_fc2": 41.0, "model.layers.5.zaya_block.experts.local_experts.13.linear_fc1": 7.9375, "model.layers.5.zaya_block.experts.local_experts.13.linear_fc2": 27.75, "model.layers.5.zaya_block.experts.local_experts.14.linear_fc1": 8.1875, "model.layers.5.zaya_block.experts.local_experts.14.linear_fc2": 26.375, "model.layers.5.zaya_block.experts.local_experts.15.linear_fc1": 7.9062, "model.layers.5.zaya_block.experts.local_experts.15.linear_fc2": 36.25, "model.layers.6.self_attn.o_proj": 10.125, "model.layers.7.zaya_block.experts.local_experts.0.linear_fc1": 6.2812, "model.layers.7.zaya_block.experts.local_experts.0.linear_fc2": 26.625, "model.layers.7.zaya_block.experts.local_experts.1.linear_fc1": 6.4062, "model.layers.7.zaya_block.experts.local_experts.1.linear_fc2": 29.375, "model.layers.7.zaya_block.experts.local_experts.2.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.2.linear_fc2": 30.125, "model.layers.7.zaya_block.experts.local_experts.3.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.3.linear_fc2": 29.125, "model.layers.7.zaya_block.experts.local_experts.4.linear_fc1": 6.2812, "model.layers.7.zaya_block.experts.local_experts.4.linear_fc2": 33.5, "model.layers.7.zaya_block.experts.local_experts.5.linear_fc1": 6.375, "model.layers.7.zaya_block.experts.local_experts.5.linear_fc2": 32.25, "model.layers.7.zaya_block.experts.local_experts.6.linear_fc1": 6.375, "model.layers.7.zaya_block.experts.local_experts.6.linear_fc2": 36.25, "model.layers.7.zaya_block.experts.local_experts.7.linear_fc1": 6.375, "model.layers.7.zaya_block.experts.local_experts.7.linear_fc2": 23.625, "model.layers.7.zaya_block.experts.local_experts.8.linear_fc1": 6.375, "model.layers.7.zaya_block.experts.local_experts.8.linear_fc2": 26.375, "model.layers.7.zaya_block.experts.local_experts.9.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.9.linear_fc2": 24.875, "model.layers.7.zaya_block.experts.local_experts.10.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.10.linear_fc2": 24.75, "model.layers.7.zaya_block.experts.local_experts.11.linear_fc1": 6.375, "model.layers.7.zaya_block.experts.local_experts.11.linear_fc2": 26.5, "model.layers.7.zaya_block.experts.local_experts.12.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.12.linear_fc2": 31.125, "model.layers.7.zaya_block.experts.local_experts.13.linear_fc1": 6.2188, "model.layers.7.zaya_block.experts.local_experts.13.linear_fc2": 31.875, "model.layers.7.zaya_block.experts.local_experts.14.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.14.linear_fc2": 20.25, "model.layers.7.zaya_block.experts.local_experts.15.linear_fc1": 6.3125, "model.layers.7.zaya_block.experts.local_experts.15.linear_fc2": 33.0, "model.layers.8.self_attn.o_proj": 7.75, "model.layers.9.zaya_block.experts.local_experts.0.linear_fc1": 6.3438, "model.layers.9.zaya_block.experts.local_experts.0.linear_fc2": 29.25, "model.layers.9.zaya_block.experts.local_experts.1.linear_fc1": 6.3438, "model.layers.9.zaya_block.experts.local_experts.1.linear_fc2": 34.0, "model.layers.9.zaya_block.experts.local_experts.2.linear_fc1": 6.3125, "model.layers.9.zaya_block.experts.local_experts.2.linear_fc2": 44.75, "model.layers.9.zaya_block.experts.local_experts.3.linear_fc1": 6.3438, "model.layers.9.zaya_block.experts.local_experts.3.linear_fc2": 25.875, "model.layers.9.zaya_block.experts.local_experts.4.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.4.linear_fc2": 25.75, "model.layers.9.zaya_block.experts.local_experts.5.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.5.linear_fc2": 35.5, "model.layers.9.zaya_block.experts.local_experts.6.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.6.linear_fc2": 31.625, "model.layers.9.zaya_block.experts.local_experts.7.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.7.linear_fc2": 32.75, "model.layers.9.zaya_block.experts.local_experts.8.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.8.linear_fc2": 26.125, "model.layers.9.zaya_block.experts.local_experts.9.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.9.linear_fc2": 35.5, "model.layers.9.zaya_block.experts.local_experts.10.linear_fc1": 6.3438, "model.layers.9.zaya_block.experts.local_experts.10.linear_fc2": 26.625, "model.layers.9.zaya_block.experts.local_experts.11.linear_fc1": 6.4375, "model.layers.9.zaya_block.experts.local_experts.11.linear_fc2": 36.0, "model.layers.9.zaya_block.experts.local_experts.12.linear_fc1": 6.3438, "model.layers.9.zaya_block.experts.local_experts.12.linear_fc2": 28.125, "model.layers.9.zaya_block.experts.local_experts.13.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.13.linear_fc2": 24.125, "model.layers.9.zaya_block.experts.local_experts.14.linear_fc1": 6.3438, "model.layers.9.zaya_block.experts.local_experts.14.linear_fc2": 27.25, "model.layers.9.zaya_block.experts.local_experts.15.linear_fc1": 6.375, "model.layers.9.zaya_block.experts.local_experts.15.linear_fc2": 29.625, "model.layers.10.self_attn.o_proj": 7.8125, "model.layers.11.zaya_block.experts.local_experts.0.linear_fc1": 14.8125, "model.layers.11.zaya_block.experts.local_experts.0.linear_fc2": 26.0, "model.layers.11.zaya_block.experts.local_experts.1.linear_fc1": 14.8125, "model.layers.11.zaya_block.experts.local_experts.1.linear_fc2": 26.75, "model.layers.11.zaya_block.experts.local_experts.2.linear_fc1": 13.1875, "model.layers.11.zaya_block.experts.local_experts.2.linear_fc2": 38.25, "model.layers.11.zaya_block.experts.local_experts.3.linear_fc1": 14.3125, "model.layers.11.zaya_block.experts.local_experts.3.linear_fc2": 26.0, "model.layers.11.zaya_block.experts.local_experts.4.linear_fc1": 13.75, "model.layers.11.zaya_block.experts.local_experts.4.linear_fc2": 25.0, "model.layers.11.zaya_block.experts.local_experts.5.linear_fc1": 13.75, "model.layers.11.zaya_block.experts.local_experts.5.linear_fc2": 29.25, "model.layers.11.zaya_block.experts.local_experts.6.linear_fc1": 15.6875, "model.layers.11.zaya_block.experts.local_experts.6.linear_fc2": 21.25, "model.layers.11.zaya_block.experts.local_experts.7.linear_fc1": 5.6875, "model.layers.11.zaya_block.experts.local_experts.7.linear_fc2": 32.75, "model.layers.11.zaya_block.experts.local_experts.8.linear_fc1": 12.375, "model.layers.11.zaya_block.experts.local_experts.8.linear_fc2": 28.375, "model.layers.11.zaya_block.experts.local_experts.9.linear_fc1": 6.9375, "model.layers.11.zaya_block.experts.local_experts.9.linear_fc2": 29.125, "model.layers.11.zaya_block.experts.local_experts.10.linear_fc1": 13.5, "model.layers.11.zaya_block.experts.local_experts.10.linear_fc2": 38.25, "model.layers.11.zaya_block.experts.local_experts.11.linear_fc1": 14.1875, "model.layers.11.zaya_block.experts.local_experts.11.linear_fc2": 27.25, "model.layers.11.zaya_block.experts.local_experts.12.linear_fc1": 14.875, "model.layers.11.zaya_block.experts.local_experts.12.linear_fc2": 28.5, "model.layers.11.zaya_block.experts.local_experts.13.linear_fc1": 14.875, "model.layers.11.zaya_block.experts.local_experts.13.linear_fc2": 22.25, "model.layers.11.zaya_block.experts.local_experts.14.linear_fc1": 15.0, "model.layers.11.zaya_block.experts.local_experts.14.linear_fc2": 32.25, "model.layers.11.zaya_block.experts.local_experts.15.linear_fc1": 14.0625, "model.layers.11.zaya_block.experts.local_experts.15.linear_fc2": 38.0, "model.layers.12.self_attn.o_proj": 8.5625, "model.layers.13.zaya_block.experts.local_experts.0.linear_fc1": 9.25, "model.layers.13.zaya_block.experts.local_experts.0.linear_fc2": 22.375, "model.layers.13.zaya_block.experts.local_experts.1.linear_fc1": 10.625, "model.layers.13.zaya_block.experts.local_experts.1.linear_fc2": 24.875, "model.layers.13.zaya_block.experts.local_experts.2.linear_fc1": 10.75, "model.layers.13.zaya_block.experts.local_experts.2.linear_fc2": 24.0, "model.layers.13.zaya_block.experts.local_experts.3.linear_fc1": 10.625, "model.layers.13.zaya_block.experts.local_experts.3.linear_fc2": 28.375, "model.layers.13.zaya_block.experts.local_experts.4.linear_fc1": 4.8125, "model.layers.13.zaya_block.experts.local_experts.4.linear_fc2": 22.75, "model.layers.13.zaya_block.experts.local_experts.5.linear_fc1": 11.0625, "model.layers.13.zaya_block.experts.local_experts.5.linear_fc2": 24.625, "model.layers.13.zaya_block.experts.local_experts.6.linear_fc1": 10.3125, "model.layers.13.zaya_block.experts.local_experts.6.linear_fc2": 24.0, "model.layers.13.zaya_block.experts.local_experts.7.linear_fc1": 10.9375, "model.layers.13.zaya_block.experts.local_experts.7.linear_fc2": 22.375, "model.layers.13.zaya_block.experts.local_experts.8.linear_fc1": 11.375, "model.layers.13.zaya_block.experts.local_experts.8.linear_fc2": 22.5, "model.layers.13.zaya_block.experts.local_experts.9.linear_fc1": 10.6875, "model.layers.13.zaya_block.experts.local_experts.9.linear_fc2": 23.125, "model.layers.13.zaya_block.experts.local_experts.10.linear_fc1": 5.875, "model.layers.13.zaya_block.experts.local_experts.10.linear_fc2": 29.5, "model.layers.13.zaya_block.experts.local_experts.11.linear_fc1": 4.9375, "model.layers.13.zaya_block.experts.local_experts.11.linear_fc2": 28.25, "model.layers.13.zaya_block.experts.local_experts.12.linear_fc1": 10.875, "model.layers.13.zaya_block.experts.local_experts.12.linear_fc2": 22.625, "model.layers.13.zaya_block.experts.local_experts.13.linear_fc1": 4.5312, "model.layers.13.zaya_block.experts.local_experts.13.linear_fc2": 26.625, "model.layers.13.zaya_block.experts.local_experts.14.linear_fc1": 9.875, "model.layers.13.zaya_block.experts.local_experts.14.linear_fc2": 36.5, "model.layers.13.zaya_block.experts.local_experts.15.linear_fc1": 10.75, "model.layers.13.zaya_block.experts.local_experts.15.linear_fc2": 21.875, "model.layers.14.self_attn.o_proj": 7.2812, "model.layers.15.zaya_block.experts.local_experts.0.linear_fc1": 9.1875, "model.layers.15.zaya_block.experts.local_experts.0.linear_fc2": 25.375, "model.layers.15.zaya_block.experts.local_experts.1.linear_fc1": 8.5, "model.layers.15.zaya_block.experts.local_experts.1.linear_fc2": 24.625, "model.layers.15.zaya_block.experts.local_experts.2.linear_fc1": 9.3125, "model.layers.15.zaya_block.experts.local_experts.2.linear_fc2": 27.625, "model.layers.15.zaya_block.experts.local_experts.3.linear_fc1": 8.625, "model.layers.15.zaya_block.experts.local_experts.3.linear_fc2": 38.25, "model.layers.15.zaya_block.experts.local_experts.4.linear_fc1": 9.0, "model.layers.15.zaya_block.experts.local_experts.4.linear_fc2": 32.75, "model.layers.15.zaya_block.experts.local_experts.5.linear_fc1": 8.9375, "model.layers.15.zaya_block.experts.local_experts.5.linear_fc2": 34.5, "model.layers.15.zaya_block.experts.local_experts.6.linear_fc1": 9.25, "model.layers.15.zaya_block.experts.local_experts.6.linear_fc2": 38.25, "model.layers.15.zaya_block.experts.local_experts.7.linear_fc1": 9.3125, "model.layers.15.zaya_block.experts.local_experts.7.linear_fc2": 42.5, "model.layers.15.zaya_block.experts.local_experts.8.linear_fc1": 9.1875, "model.layers.15.zaya_block.experts.local_experts.8.linear_fc2": 29.625, "model.layers.15.zaya_block.experts.local_experts.9.linear_fc1": 9.1875, "model.layers.15.zaya_block.experts.local_experts.9.linear_fc2": 24.625, "model.layers.15.zaya_block.experts.local_experts.10.linear_fc1": 9.375, "model.layers.15.zaya_block.experts.local_experts.10.linear_fc2": 29.5, "model.layers.15.zaya_block.experts.local_experts.11.linear_fc1": 6.0938, "model.layers.15.zaya_block.experts.local_experts.11.linear_fc2": 19.875, "model.layers.15.zaya_block.experts.local_experts.12.linear_fc1": 9.125, "model.layers.15.zaya_block.experts.local_experts.12.linear_fc2": 23.5, "model.layers.15.zaya_block.experts.local_experts.13.linear_fc1": 9.1875, "model.layers.15.zaya_block.experts.local_experts.13.linear_fc2": 18.875, "model.layers.15.zaya_block.experts.local_experts.14.linear_fc1": 5.0625, "model.layers.15.zaya_block.experts.local_experts.14.linear_fc2": 30.75, "model.layers.15.zaya_block.experts.local_experts.15.linear_fc1": 9.1875, "model.layers.15.zaya_block.experts.local_experts.15.linear_fc2": 27.75, "model.layers.16.self_attn.o_proj": 7.4375, "model.layers.17.zaya_block.experts.local_experts.0.linear_fc1": 13.9375, "model.layers.17.zaya_block.experts.local_experts.0.linear_fc2": 27.5, "model.layers.17.zaya_block.experts.local_experts.1.linear_fc1": 15.25, "model.layers.17.zaya_block.experts.local_experts.1.linear_fc2": 21.5, "model.layers.17.zaya_block.experts.local_experts.2.linear_fc1": 15.25, "model.layers.17.zaya_block.experts.local_experts.2.linear_fc2": 20.75, "model.layers.17.zaya_block.experts.local_experts.3.linear_fc1": 5.75, "model.layers.17.zaya_block.experts.local_experts.3.linear_fc2": 54.0, "model.layers.17.zaya_block.experts.local_experts.4.linear_fc1": 15.75, "model.layers.17.zaya_block.experts.local_experts.4.linear_fc2": 24.125, "model.layers.17.zaya_block.experts.local_experts.5.linear_fc1": 4.3125, "model.layers.17.zaya_block.experts.local_experts.5.linear_fc2": 32.0, "model.layers.17.zaya_block.experts.local_experts.6.linear_fc1": 4.4688, "model.layers.17.zaya_block.experts.local_experts.6.linear_fc2": 20.0, "model.layers.17.zaya_block.experts.local_experts.7.linear_fc1": 16.0, "model.layers.17.zaya_block.experts.local_experts.7.linear_fc2": 21.25, "model.layers.17.zaya_block.experts.local_experts.8.linear_fc1": 4.7812, "model.layers.17.zaya_block.experts.local_experts.8.linear_fc2": 22.5, "model.layers.17.zaya_block.experts.local_experts.9.linear_fc1": 15.9375, "model.layers.17.zaya_block.experts.local_experts.9.linear_fc2": 18.375, "model.layers.17.zaya_block.experts.local_experts.10.linear_fc1": 5.1875, "model.layers.17.zaya_block.experts.local_experts.10.linear_fc2": 25.875, "model.layers.17.zaya_block.experts.local_experts.11.linear_fc1": 4.8438, "model.layers.17.zaya_block.experts.local_experts.11.linear_fc2": 22.375, "model.layers.17.zaya_block.experts.local_experts.12.linear_fc1": 16.25, "model.layers.17.zaya_block.experts.local_experts.12.linear_fc2": 22.25, "model.layers.17.zaya_block.experts.local_experts.13.linear_fc1": 4.6875, "model.layers.17.zaya_block.experts.local_experts.13.linear_fc2": 23.0, "model.layers.17.zaya_block.experts.local_experts.14.linear_fc1": 15.5625, "model.layers.17.zaya_block.experts.local_experts.14.linear_fc2": 21.875, "model.layers.17.zaya_block.experts.local_experts.15.linear_fc1": 15.75, "model.layers.17.zaya_block.experts.local_experts.15.linear_fc2": 21.25, "model.layers.18.self_attn.o_proj": 9.25, "model.layers.19.zaya_block.experts.local_experts.0.linear_fc1": 5.5625, "model.layers.19.zaya_block.experts.local_experts.0.linear_fc2": 26.25, "model.layers.19.zaya_block.experts.local_experts.1.linear_fc1": 5.3438, "model.layers.19.zaya_block.experts.local_experts.1.linear_fc2": 46.75, "model.layers.19.zaya_block.experts.local_experts.2.linear_fc1": 6.3125, "model.layers.19.zaya_block.experts.local_experts.2.linear_fc2": 52.25, "model.layers.19.zaya_block.experts.local_experts.3.linear_fc1": 5.5, "model.layers.19.zaya_block.experts.local_experts.3.linear_fc2": 28.625, "model.layers.19.zaya_block.experts.local_experts.4.linear_fc1": 6.0625, "model.layers.19.zaya_block.experts.local_experts.4.linear_fc2": 38.0, "model.layers.19.zaya_block.experts.local_experts.5.linear_fc1": 5.125, "model.layers.19.zaya_block.experts.local_experts.5.linear_fc2": 30.875, "model.layers.19.zaya_block.experts.local_experts.6.linear_fc1": 4.875, "model.layers.19.zaya_block.experts.local_experts.6.linear_fc2": 30.75, "model.layers.19.zaya_block.experts.local_experts.7.linear_fc1": 7.3125, "model.layers.19.zaya_block.experts.local_experts.7.linear_fc2": 97.5, "model.layers.19.zaya_block.experts.local_experts.8.linear_fc1": 38.0, "model.layers.19.zaya_block.experts.local_experts.8.linear_fc2": 592.0, "model.layers.19.zaya_block.experts.local_experts.9.linear_fc1": 5.4375, "model.layers.19.zaya_block.experts.local_experts.9.linear_fc2": 32.75, "model.layers.19.zaya_block.experts.local_experts.10.linear_fc1": 5.8438, "model.layers.19.zaya_block.experts.local_experts.10.linear_fc2": 26.875, "model.layers.19.zaya_block.experts.local_experts.11.linear_fc1": 5.0312, "model.layers.19.zaya_block.experts.local_experts.11.linear_fc2": 29.375, "model.layers.19.zaya_block.experts.local_experts.12.linear_fc1": 6.5625, "model.layers.19.zaya_block.experts.local_experts.12.linear_fc2": 33.25, "model.layers.19.zaya_block.experts.local_experts.13.linear_fc1": 6.0625, "model.layers.19.zaya_block.experts.local_experts.13.linear_fc2": 34.0, "model.layers.19.zaya_block.experts.local_experts.14.linear_fc1": 6.2812, "model.layers.19.zaya_block.experts.local_experts.14.linear_fc2": 26.625, "model.layers.19.zaya_block.experts.local_experts.15.linear_fc1": 5.3438, "model.layers.19.zaya_block.experts.local_experts.15.linear_fc2": 55.5, "model.layers.20.self_attn.o_proj": 11.8125, "model.layers.21.zaya_block.experts.local_experts.0.linear_fc1": 5.0312, "model.layers.21.zaya_block.experts.local_experts.0.linear_fc2": 24.375, "model.layers.21.zaya_block.experts.local_experts.1.linear_fc1": 4.5, "model.layers.21.zaya_block.experts.local_experts.1.linear_fc2": 21.875, "model.layers.21.zaya_block.experts.local_experts.2.linear_fc1": 5.125, "model.layers.21.zaya_block.experts.local_experts.2.linear_fc2": 28.0, "model.layers.21.zaya_block.experts.local_experts.3.linear_fc1": 64.0, "model.layers.21.zaya_block.experts.local_experts.3.linear_fc2": 38.75, "model.layers.21.zaya_block.experts.local_experts.4.linear_fc1": 5.75, "model.layers.21.zaya_block.experts.local_experts.4.linear_fc2": 24.75, "model.layers.21.zaya_block.experts.local_experts.5.linear_fc1": 5.8125, "model.layers.21.zaya_block.experts.local_experts.5.linear_fc2": 27.75, "model.layers.21.zaya_block.experts.local_experts.6.linear_fc1": 4.625, "model.layers.21.zaya_block.experts.local_experts.6.linear_fc2": 30.25, "model.layers.21.zaya_block.experts.local_experts.7.linear_fc1": 64.0, "model.layers.21.zaya_block.experts.local_experts.7.linear_fc2": 46.25, "model.layers.21.zaya_block.experts.local_experts.8.linear_fc1": 4.9688, "model.layers.21.zaya_block.experts.local_experts.8.linear_fc2": 22.875, "model.layers.21.zaya_block.experts.local_experts.9.linear_fc1": 4.875, "model.layers.21.zaya_block.experts.local_experts.9.linear_fc2": 35.5, "model.layers.21.zaya_block.experts.local_experts.10.linear_fc1": 5.5938, "model.layers.21.zaya_block.experts.local_experts.10.linear_fc2": 17.0, "model.layers.21.zaya_block.experts.local_experts.11.linear_fc1": 4.7188, "model.layers.21.zaya_block.experts.local_experts.11.linear_fc2": 20.375, "model.layers.21.zaya_block.experts.local_experts.12.linear_fc1": 5.25, "model.layers.21.zaya_block.experts.local_experts.12.linear_fc2": 32.5, "model.layers.21.zaya_block.experts.local_experts.13.linear_fc1": 4.375, "model.layers.21.zaya_block.experts.local_experts.13.linear_fc2": 37.25, "model.layers.21.zaya_block.experts.local_experts.14.linear_fc1": 5.0938, "model.layers.21.zaya_block.experts.local_experts.14.linear_fc2": 30.25, "model.layers.21.zaya_block.experts.local_experts.15.linear_fc1": 4.6562, "model.layers.21.zaya_block.experts.local_experts.15.linear_fc2": 22.375, "model.layers.22.self_attn.o_proj": 8.5, "model.layers.23.zaya_block.experts.local_experts.0.linear_fc1": 5.7188, "model.layers.23.zaya_block.experts.local_experts.0.linear_fc2": 482.0, "model.layers.23.zaya_block.experts.local_experts.1.linear_fc1": 4.1562, "model.layers.23.zaya_block.experts.local_experts.1.linear_fc2": 22.375, "model.layers.23.zaya_block.experts.local_experts.2.linear_fc1": 4.4688, "model.layers.23.zaya_block.experts.local_experts.2.linear_fc2": 30.625, "model.layers.23.zaya_block.experts.local_experts.4.linear_fc1": 4.2188, "model.layers.23.zaya_block.experts.local_experts.4.linear_fc2": 16.375, "model.layers.23.zaya_block.experts.local_experts.5.linear_fc1": 4.5, "model.layers.23.zaya_block.experts.local_experts.5.linear_fc2": 22.875, "model.layers.23.zaya_block.experts.local_experts.6.linear_fc1": 4.5312, "model.layers.23.zaya_block.experts.local_experts.6.linear_fc2": 17.5, "model.layers.23.zaya_block.experts.local_experts.7.linear_fc1": 4.5, "model.layers.23.zaya_block.experts.local_experts.7.linear_fc2": 45.25, "model.layers.23.zaya_block.experts.local_experts.8.linear_fc1": 6.0312, "model.layers.23.zaya_block.experts.local_experts.8.linear_fc2": 120.0, "model.layers.23.zaya_block.experts.local_experts.9.linear_fc1": 4.2188, "model.layers.23.zaya_block.experts.local_experts.9.linear_fc2": 28.625, "model.layers.23.zaya_block.experts.local_experts.10.linear_fc1": 3.9844, "model.layers.23.zaya_block.experts.local_experts.10.linear_fc2": 19.25, "model.layers.23.zaya_block.experts.local_experts.11.linear_fc1": 5.1562, "model.layers.23.zaya_block.experts.local_experts.11.linear_fc2": 21.875, "model.layers.23.zaya_block.experts.local_experts.12.linear_fc1": 4.7188, "model.layers.23.zaya_block.experts.local_experts.12.linear_fc2": 19.125, "model.layers.23.zaya_block.experts.local_experts.13.linear_fc1": 4.3438, "model.layers.23.zaya_block.experts.local_experts.13.linear_fc2": 18.75, "model.layers.23.zaya_block.experts.local_experts.14.linear_fc1": 64.5, "model.layers.23.zaya_block.experts.local_experts.14.linear_fc2": 39.5, "model.layers.23.zaya_block.experts.local_experts.15.linear_fc1": 4.9688, "model.layers.23.zaya_block.experts.local_experts.15.linear_fc2": 22.75, "model.layers.23.zaya_block.experts.local_experts.3.linear_fc1": 4.75, "model.layers.23.zaya_block.experts.local_experts.3.linear_fc2": 20.25, "model.layers.24.self_attn.o_proj": 17.125, "model.layers.25.zaya_block.experts.local_experts.0.linear_fc1": 8.8125, "model.layers.25.zaya_block.experts.local_experts.0.linear_fc2": 11.0625, "model.layers.25.zaya_block.experts.local_experts.1.linear_fc1": 19.625, "model.layers.25.zaya_block.experts.local_experts.1.linear_fc2": 14.0625, "model.layers.25.zaya_block.experts.local_experts.2.linear_fc1": 3.9844, "model.layers.25.zaya_block.experts.local_experts.2.linear_fc2": 18.5, "model.layers.25.zaya_block.experts.local_experts.3.linear_fc1": 14.625, "model.layers.25.zaya_block.experts.local_experts.3.linear_fc2": 21.5, "model.layers.25.zaya_block.experts.local_experts.4.linear_fc1": 15.125, "model.layers.25.zaya_block.experts.local_experts.4.linear_fc2": 21.75, "model.layers.25.zaya_block.experts.local_experts.5.linear_fc1": 3.7812, "model.layers.25.zaya_block.experts.local_experts.5.linear_fc2": 14.25, "model.layers.25.zaya_block.experts.local_experts.6.linear_fc1": 22.75, "model.layers.25.zaya_block.experts.local_experts.6.linear_fc2": 28.375, "model.layers.25.zaya_block.experts.local_experts.7.linear_fc1": 30.5, "model.layers.25.zaya_block.experts.local_experts.7.linear_fc2": 14.1875, "model.layers.25.zaya_block.experts.local_experts.8.linear_fc1": 81.0, "model.layers.25.zaya_block.experts.local_experts.8.linear_fc2": 72.5, "model.layers.25.zaya_block.experts.local_experts.9.linear_fc1": 4.2812, "model.layers.25.zaya_block.experts.local_experts.9.linear_fc2": 15.0, "model.layers.25.zaya_block.experts.local_experts.11.linear_fc1": 7.6875, "model.layers.25.zaya_block.experts.local_experts.11.linear_fc2": 19.5, "model.layers.25.zaya_block.experts.local_experts.12.linear_fc1": 4.0, "model.layers.25.zaya_block.experts.local_experts.12.linear_fc2": 16.5, "model.layers.25.zaya_block.experts.local_experts.13.linear_fc1": 80.5, "model.layers.25.zaya_block.experts.local_experts.13.linear_fc2": 69.0, "model.layers.25.zaya_block.experts.local_experts.14.linear_fc1": 6.2188, "model.layers.25.zaya_block.experts.local_experts.14.linear_fc2": 17.0, "model.layers.25.zaya_block.experts.local_experts.15.linear_fc1": 4.4688, "model.layers.25.zaya_block.experts.local_experts.15.linear_fc2": 13.8125, "model.layers.25.zaya_block.experts.local_experts.10.linear_fc1": 8.25, "model.layers.25.zaya_block.experts.local_experts.10.linear_fc2": 21.125, "model.layers.26.self_attn.o_proj": 19.625, "model.layers.27.zaya_block.experts.local_experts.0.linear_fc1": 4.1875, "model.layers.27.zaya_block.experts.local_experts.0.linear_fc2": 12.5625, "model.layers.27.zaya_block.experts.local_experts.1.linear_fc1": 11.625, "model.layers.27.zaya_block.experts.local_experts.1.linear_fc2": 14.125, "model.layers.27.zaya_block.experts.local_experts.2.linear_fc1": 4.6875, "model.layers.27.zaya_block.experts.local_experts.2.linear_fc2": 29.5, "model.layers.27.zaya_block.experts.local_experts.3.linear_fc1": 4.1875, "model.layers.27.zaya_block.experts.local_experts.3.linear_fc2": 10.5, "model.layers.27.zaya_block.experts.local_experts.4.linear_fc1": 3.625, "model.layers.27.zaya_block.experts.local_experts.4.linear_fc2": 15.5, "model.layers.27.zaya_block.experts.local_experts.5.linear_fc1": 4.7188, "model.layers.27.zaya_block.experts.local_experts.5.linear_fc2": 12.375, "model.layers.27.zaya_block.experts.local_experts.6.linear_fc1": 6.6875, "model.layers.27.zaya_block.experts.local_experts.6.linear_fc2": 12.5, "model.layers.27.zaya_block.experts.local_experts.7.linear_fc1": 3.6875, "model.layers.27.zaya_block.experts.local_experts.7.linear_fc2": 19.125, "model.layers.27.zaya_block.experts.local_experts.8.linear_fc1": 104.5, "model.layers.27.zaya_block.experts.local_experts.8.linear_fc2": 71.5, "model.layers.27.zaya_block.experts.local_experts.9.linear_fc1": 6.5, "model.layers.27.zaya_block.experts.local_experts.9.linear_fc2": 16.75, "model.layers.27.zaya_block.experts.local_experts.10.linear_fc1": 11.75, "model.layers.27.zaya_block.experts.local_experts.10.linear_fc2": 13.375, "model.layers.27.zaya_block.experts.local_experts.11.linear_fc1": 11.375, "model.layers.27.zaya_block.experts.local_experts.11.linear_fc2": 14.25, "model.layers.27.zaya_block.experts.local_experts.12.linear_fc1": 104.5, "model.layers.27.zaya_block.experts.local_experts.12.linear_fc2": 98.5, "model.layers.27.zaya_block.experts.local_experts.13.linear_fc1": 4.8438, "model.layers.27.zaya_block.experts.local_experts.13.linear_fc2": 13.5, "model.layers.27.zaya_block.experts.local_experts.14.linear_fc1": 4.5, "model.layers.27.zaya_block.experts.local_experts.14.linear_fc2": 13.625, "model.layers.27.zaya_block.experts.local_experts.15.linear_fc1": 4.375, "model.layers.27.zaya_block.experts.local_experts.15.linear_fc2": 13.25, "model.layers.28.self_attn.o_proj": 19.75, "model.layers.29.zaya_block.experts.local_experts.0.linear_fc1": 5.3125, "model.layers.29.zaya_block.experts.local_experts.0.linear_fc2": 10.9375, "model.layers.29.zaya_block.experts.local_experts.1.linear_fc1": 5.5625, "model.layers.29.zaya_block.experts.local_experts.1.linear_fc2": 14.125, "model.layers.29.zaya_block.experts.local_experts.2.linear_fc1": 4.0, "model.layers.29.zaya_block.experts.local_experts.2.linear_fc2": 15.0, "model.layers.29.zaya_block.experts.local_experts.3.linear_fc1": 5.0, "model.layers.29.zaya_block.experts.local_experts.3.linear_fc2": 12.75, "model.layers.29.zaya_block.experts.local_experts.4.linear_fc1": 4.4688, "model.layers.29.zaya_block.experts.local_experts.4.linear_fc2": 11.1875, "model.layers.29.zaya_block.experts.local_experts.5.linear_fc1": 4.0312, "model.layers.29.zaya_block.experts.local_experts.5.linear_fc2": 13.9375, "model.layers.29.zaya_block.experts.local_experts.6.linear_fc1": 46.5, "model.layers.29.zaya_block.experts.local_experts.6.linear_fc2": 118.0, "model.layers.29.zaya_block.experts.local_experts.7.linear_fc1": 5.1875, "model.layers.29.zaya_block.experts.local_experts.7.linear_fc2": 13.5625, "model.layers.29.zaya_block.experts.local_experts.8.linear_fc1": 4.9688, "model.layers.29.zaya_block.experts.local_experts.8.linear_fc2": 13.5625, "model.layers.29.zaya_block.experts.local_experts.9.linear_fc1": 4.5312, "model.layers.29.zaya_block.experts.local_experts.9.linear_fc2": 13.625, "model.layers.29.zaya_block.experts.local_experts.10.linear_fc1": 5.375, "model.layers.29.zaya_block.experts.local_experts.10.linear_fc2": 21.125, "model.layers.29.zaya_block.experts.local_experts.11.linear_fc1": 49.75, "model.layers.29.zaya_block.experts.local_experts.11.linear_fc2": 12.0, "model.layers.29.zaya_block.experts.local_experts.12.linear_fc1": 5.0938, "model.layers.29.zaya_block.experts.local_experts.12.linear_fc2": 14.0625, "model.layers.29.zaya_block.experts.local_experts.13.linear_fc1": 4.4688, "model.layers.29.zaya_block.experts.local_experts.13.linear_fc2": 17.25, "model.layers.29.zaya_block.experts.local_experts.14.linear_fc1": 4.3125, "model.layers.29.zaya_block.experts.local_experts.14.linear_fc2": 14.1875, "model.layers.29.zaya_block.experts.local_experts.15.linear_fc1": 4.6562, "model.layers.29.zaya_block.experts.local_experts.15.linear_fc2": 11.25, "model.layers.30.self_attn.o_proj": 8.5625, "model.layers.31.zaya_block.experts.local_experts.0.linear_fc1": 20.5, "model.layers.31.zaya_block.experts.local_experts.0.linear_fc2": 10.5625, "model.layers.31.zaya_block.experts.local_experts.1.linear_fc1": 15.5625, "model.layers.31.zaya_block.experts.local_experts.1.linear_fc2": 10.3125, "model.layers.31.zaya_block.experts.local_experts.2.linear_fc1": 46.0, "model.layers.31.zaya_block.experts.local_experts.2.linear_fc2": 36.5, "model.layers.31.zaya_block.experts.local_experts.3.linear_fc1": 140.0, "model.layers.31.zaya_block.experts.local_experts.3.linear_fc2": 944.0, "model.layers.31.zaya_block.experts.local_experts.4.linear_fc1": 27.875, "model.layers.31.zaya_block.experts.local_experts.4.linear_fc2": 10.6875, "model.layers.31.zaya_block.experts.local_experts.5.linear_fc1": 8.625, "model.layers.31.zaya_block.experts.local_experts.5.linear_fc2": 16.25, "model.layers.31.zaya_block.experts.local_experts.6.linear_fc1": 14.6875, "model.layers.31.zaya_block.experts.local_experts.6.linear_fc2": 14.6875, "model.layers.31.zaya_block.experts.local_experts.7.linear_fc1": 12.6875, "model.layers.31.zaya_block.experts.local_experts.7.linear_fc2": 14.4375, "model.layers.31.zaya_block.experts.local_experts.8.linear_fc1": 4.375, "model.layers.31.zaya_block.experts.local_experts.8.linear_fc2": 9.3125, "model.layers.31.zaya_block.experts.local_experts.9.linear_fc1": 54.25, "model.layers.31.zaya_block.experts.local_experts.9.linear_fc2": 30.625, "model.layers.31.zaya_block.experts.local_experts.10.linear_fc1": 28.625, "model.layers.31.zaya_block.experts.local_experts.10.linear_fc2": 55.25, "model.layers.31.zaya_block.experts.local_experts.11.linear_fc1": 3.9375, "model.layers.31.zaya_block.experts.local_experts.11.linear_fc2": 11.25, "model.layers.31.zaya_block.experts.local_experts.12.linear_fc1": 4.1562, "model.layers.31.zaya_block.experts.local_experts.12.linear_fc2": 10.1875, "model.layers.31.zaya_block.experts.local_experts.13.linear_fc1": 55.5, "model.layers.31.zaya_block.experts.local_experts.13.linear_fc2": 62.0, "model.layers.31.zaya_block.experts.local_experts.14.linear_fc1": 9.375, "model.layers.31.zaya_block.experts.local_experts.14.linear_fc2": 9.375, "model.layers.31.zaya_block.experts.local_experts.15.linear_fc1": 14.9375, "model.layers.31.zaya_block.experts.local_experts.15.linear_fc2": 10.5625, "model.layers.32.self_attn.o_proj": 10.8125, "model.layers.33.zaya_block.experts.local_experts.0.linear_fc1": 5.9062, "model.layers.33.zaya_block.experts.local_experts.0.linear_fc2": 9.25, "model.layers.33.zaya_block.experts.local_experts.1.linear_fc1": 5.2812, "model.layers.33.zaya_block.experts.local_experts.1.linear_fc2": 10.5625, "model.layers.33.zaya_block.experts.local_experts.2.linear_fc1": 10.5, "model.layers.33.zaya_block.experts.local_experts.2.linear_fc2": 11.5, "model.layers.33.zaya_block.experts.local_experts.3.linear_fc1": 14.75, "model.layers.33.zaya_block.experts.local_experts.3.linear_fc2": 10.6875, "model.layers.33.zaya_block.experts.local_experts.4.linear_fc1": 9.5, "model.layers.33.zaya_block.experts.local_experts.4.linear_fc2": 9.375, "model.layers.33.zaya_block.experts.local_experts.5.linear_fc1": 10.1875, "model.layers.33.zaya_block.experts.local_experts.5.linear_fc2": 13.0, "model.layers.33.zaya_block.experts.local_experts.6.linear_fc1": 3.5625, "model.layers.33.zaya_block.experts.local_experts.6.linear_fc2": 8.375, "model.layers.33.zaya_block.experts.local_experts.7.linear_fc1": 8.375, "model.layers.33.zaya_block.experts.local_experts.7.linear_fc2": 14.5, "model.layers.33.zaya_block.experts.local_experts.8.linear_fc1": 17.125, "model.layers.33.zaya_block.experts.local_experts.8.linear_fc2": 14.125, "model.layers.33.zaya_block.experts.local_experts.9.linear_fc1": 4.0, "model.layers.33.zaya_block.experts.local_experts.9.linear_fc2": 9.125, "model.layers.33.zaya_block.experts.local_experts.10.linear_fc1": 8.25, "model.layers.33.zaya_block.experts.local_experts.10.linear_fc2": 10.625, "model.layers.33.zaya_block.experts.local_experts.11.linear_fc1": 4.25, "model.layers.33.zaya_block.experts.local_experts.11.linear_fc2": 7.5625, "model.layers.33.zaya_block.experts.local_experts.12.linear_fc1": 29.875, "model.layers.33.zaya_block.experts.local_experts.12.linear_fc2": 14.25, "model.layers.33.zaya_block.experts.local_experts.13.linear_fc1": 115.5, "model.layers.33.zaya_block.experts.local_experts.13.linear_fc2": 170.0, "model.layers.33.zaya_block.experts.local_experts.14.linear_fc1": 19.0, "model.layers.33.zaya_block.experts.local_experts.14.linear_fc2": 9.9375, "model.layers.33.zaya_block.experts.local_experts.15.linear_fc1": 10.875, "model.layers.33.zaya_block.experts.local_experts.15.linear_fc2": 10.6875, "model.layers.34.self_attn.o_proj": 6.7188, "model.layers.35.zaya_block.experts.local_experts.0.linear_fc1": 18.75, "model.layers.35.zaya_block.experts.local_experts.0.linear_fc2": 12.5, "model.layers.35.zaya_block.experts.local_experts.1.linear_fc1": 4.9375, "model.layers.35.zaya_block.experts.local_experts.1.linear_fc2": 7.6562, "model.layers.35.zaya_block.experts.local_experts.2.linear_fc1": 5.625, "model.layers.35.zaya_block.experts.local_experts.2.linear_fc2": 9.125, "model.layers.35.zaya_block.experts.local_experts.3.linear_fc1": 9.8125, "model.layers.35.zaya_block.experts.local_experts.3.linear_fc2": 9.0625, "model.layers.35.zaya_block.experts.local_experts.4.linear_fc1": 13.5, "model.layers.35.zaya_block.experts.local_experts.4.linear_fc2": 11.3125, "model.layers.35.zaya_block.experts.local_experts.5.linear_fc1": 5.125, "model.layers.35.zaya_block.experts.local_experts.5.linear_fc2": 11.25, "model.layers.35.zaya_block.experts.local_experts.6.linear_fc1": 102.0, "model.layers.35.zaya_block.experts.local_experts.6.linear_fc2": 181.0, "model.layers.35.zaya_block.experts.local_experts.7.linear_fc1": 10.625, "model.layers.35.zaya_block.experts.local_experts.7.linear_fc2": 14.375, "model.layers.35.zaya_block.experts.local_experts.8.linear_fc1": 4.5, "model.layers.35.zaya_block.experts.local_experts.8.linear_fc2": 10.0625, "model.layers.35.zaya_block.experts.local_experts.9.linear_fc1": 4.5938, "model.layers.35.zaya_block.experts.local_experts.9.linear_fc2": 10.125, "model.layers.35.zaya_block.experts.local_experts.10.linear_fc1": 79.5, "model.layers.35.zaya_block.experts.local_experts.10.linear_fc2": 37.0, "model.layers.35.zaya_block.experts.local_experts.11.linear_fc1": 12.5625, "model.layers.35.zaya_block.experts.local_experts.11.linear_fc2": 10.3125, "model.layers.35.zaya_block.experts.local_experts.12.linear_fc1": 15.3125, "model.layers.35.zaya_block.experts.local_experts.12.linear_fc2": 8.25, "model.layers.35.zaya_block.experts.local_experts.13.linear_fc1": 17.625, "model.layers.35.zaya_block.experts.local_experts.13.linear_fc2": 11.3125, "model.layers.35.zaya_block.experts.local_experts.14.linear_fc1": 4.5625, "model.layers.35.zaya_block.experts.local_experts.14.linear_fc2": 11.625, "model.layers.35.zaya_block.experts.local_experts.15.linear_fc1": 9.4375, "model.layers.35.zaya_block.experts.local_experts.15.linear_fc2": 9.9375, "model.layers.36.self_attn.o_proj": 7.1562, "model.layers.37.zaya_block.experts.local_experts.0.linear_fc1": 81.5, "model.layers.37.zaya_block.experts.local_experts.0.linear_fc2": 6944.0, "model.layers.37.zaya_block.experts.local_experts.1.linear_fc1": 5.3125, "model.layers.37.zaya_block.experts.local_experts.1.linear_fc2": 8.625, "model.layers.37.zaya_block.experts.local_experts.2.linear_fc1": 4.5625, "model.layers.37.zaya_block.experts.local_experts.2.linear_fc2": 12.8125, "model.layers.37.zaya_block.experts.local_experts.3.linear_fc1": 57.5, "model.layers.37.zaya_block.experts.local_experts.3.linear_fc2": 9.9375, "model.layers.37.zaya_block.experts.local_experts.4.linear_fc1": 4.4375, "model.layers.37.zaya_block.experts.local_experts.4.linear_fc2": 10.3125, "model.layers.37.zaya_block.experts.local_experts.5.linear_fc1": 57.0, "model.layers.37.zaya_block.experts.local_experts.5.linear_fc2": 89.5, "model.layers.37.zaya_block.experts.local_experts.6.linear_fc1": 6.3125, "model.layers.37.zaya_block.experts.local_experts.6.linear_fc2": 15.0625, "model.layers.37.zaya_block.experts.local_experts.7.linear_fc1": 5.5, "model.layers.37.zaya_block.experts.local_experts.7.linear_fc2": 17.125, "model.layers.37.zaya_block.experts.local_experts.8.linear_fc1": 4.9688, "model.layers.37.zaya_block.experts.local_experts.8.linear_fc2": 9.4375, "model.layers.37.zaya_block.experts.local_experts.9.linear_fc1": 4.5, "model.layers.37.zaya_block.experts.local_experts.9.linear_fc2": 10.25, "model.layers.37.zaya_block.experts.local_experts.10.linear_fc1": 4.3125, "model.layers.37.zaya_block.experts.local_experts.10.linear_fc2": 9.1875, "model.layers.37.zaya_block.experts.local_experts.11.linear_fc1": 5.2188, "model.layers.37.zaya_block.experts.local_experts.11.linear_fc2": 11.0625, "model.layers.37.zaya_block.experts.local_experts.12.linear_fc1": 4.4688, "model.layers.37.zaya_block.experts.local_experts.12.linear_fc2": 7.5625, "model.layers.37.zaya_block.experts.local_experts.13.linear_fc1": 4.7188, "model.layers.37.zaya_block.experts.local_experts.13.linear_fc2": 9.8125, "model.layers.37.zaya_block.experts.local_experts.14.linear_fc1": 4.7812, "model.layers.37.zaya_block.experts.local_experts.14.linear_fc2": 10.0, "model.layers.37.zaya_block.experts.local_experts.15.linear_fc1": 5.3125, "model.layers.37.zaya_block.experts.local_experts.15.linear_fc2": 10.625, "model.layers.38.self_attn.o_proj": 5.75, "model.layers.39.zaya_block.experts.local_experts.0.linear_fc1": 5.5312, "model.layers.39.zaya_block.experts.local_experts.0.linear_fc2": 9.6875, "model.layers.39.zaya_block.experts.local_experts.1.linear_fc1": 18.875, "model.layers.39.zaya_block.experts.local_experts.1.linear_fc2": 42.0, "model.layers.39.zaya_block.experts.local_experts.2.linear_fc1": 5.2188, "model.layers.39.zaya_block.experts.local_experts.2.linear_fc2": 10.875, "model.layers.39.zaya_block.experts.local_experts.3.linear_fc1": 4.7188, "model.layers.39.zaya_block.experts.local_experts.3.linear_fc2": 10.3125, "model.layers.39.zaya_block.experts.local_experts.4.linear_fc1": 5.2188, "model.layers.39.zaya_block.experts.local_experts.4.linear_fc2": 10.4375, "model.layers.39.zaya_block.experts.local_experts.5.linear_fc1": 224.0, "model.layers.39.zaya_block.experts.local_experts.5.linear_fc2": 2528.0, "model.layers.39.zaya_block.experts.local_experts.6.linear_fc1": 7.5, "model.layers.39.zaya_block.experts.local_experts.6.linear_fc2": 16.75, "model.layers.39.zaya_block.experts.local_experts.7.linear_fc1": 57.25, "model.layers.39.zaya_block.experts.local_experts.7.linear_fc2": 23.875, "model.layers.39.zaya_block.experts.local_experts.8.linear_fc1": 5.375, "model.layers.39.zaya_block.experts.local_experts.8.linear_fc2": 9.8125, "model.layers.39.zaya_block.experts.local_experts.9.linear_fc1": 5.5938, "model.layers.39.zaya_block.experts.local_experts.9.linear_fc2": 8.3125, "model.layers.39.zaya_block.experts.local_experts.10.linear_fc1": 5.0, "model.layers.39.zaya_block.experts.local_experts.10.linear_fc2": 11.125, "model.layers.39.zaya_block.experts.local_experts.11.linear_fc1": 4.875, "model.layers.39.zaya_block.experts.local_experts.11.linear_fc2": 9.0625, "model.layers.39.zaya_block.experts.local_experts.12.linear_fc1": 4.8438, "model.layers.39.zaya_block.experts.local_experts.12.linear_fc2": 8.625, "model.layers.39.zaya_block.experts.local_experts.13.linear_fc1": 6.7188, "model.layers.39.zaya_block.experts.local_experts.13.linear_fc2": 12.6875, "model.layers.39.zaya_block.experts.local_experts.14.linear_fc1": 5.5312, "model.layers.39.zaya_block.experts.local_experts.14.linear_fc2": 11.0625, "model.layers.39.zaya_block.experts.local_experts.15.linear_fc1": 5.1562, "model.layers.39.zaya_block.experts.local_experts.15.linear_fc2": 10.5625, "model.layers.40.self_attn.o_proj": 17.75, "model.layers.41.zaya_block.experts.local_experts.0.linear_fc1": 7.375, "model.layers.41.zaya_block.experts.local_experts.0.linear_fc2": 13.1875, "model.layers.41.zaya_block.experts.local_experts.1.linear_fc1": 4.8125, "model.layers.41.zaya_block.experts.local_experts.1.linear_fc2": 8.5, "model.layers.41.zaya_block.experts.local_experts.2.linear_fc1": 5.4688, "model.layers.41.zaya_block.experts.local_experts.2.linear_fc2": 8.25, "model.layers.41.zaya_block.experts.local_experts.3.linear_fc1": 5.4062, "model.layers.41.zaya_block.experts.local_experts.3.linear_fc2": 9.1875, "model.layers.41.zaya_block.experts.local_experts.4.linear_fc1": 20.375, "model.layers.41.zaya_block.experts.local_experts.4.linear_fc2": 10.9375, "model.layers.41.zaya_block.experts.local_experts.5.linear_fc1": 5.1562, "model.layers.41.zaya_block.experts.local_experts.5.linear_fc2": 10.1875, "model.layers.41.zaya_block.experts.local_experts.6.linear_fc1": 5.1562, "model.layers.41.zaya_block.experts.local_experts.6.linear_fc2": 10.6875, "model.layers.41.zaya_block.experts.local_experts.7.linear_fc1": 6.25, "model.layers.41.zaya_block.experts.local_experts.7.linear_fc2": 12.5625, "model.layers.41.zaya_block.experts.local_experts.8.linear_fc1": 6.4688, "model.layers.41.zaya_block.experts.local_experts.8.linear_fc2": 11.6875, "model.layers.41.zaya_block.experts.local_experts.9.linear_fc1": 5.4375, "model.layers.41.zaya_block.experts.local_experts.9.linear_fc2": 9.8125, "model.layers.41.zaya_block.experts.local_experts.10.linear_fc1": 95.5, "model.layers.41.zaya_block.experts.local_experts.10.linear_fc2": 170.0, "model.layers.41.zaya_block.experts.local_experts.11.linear_fc1": 5.875, "model.layers.41.zaya_block.experts.local_experts.11.linear_fc2": 16.375, "model.layers.41.zaya_block.experts.local_experts.12.linear_fc1": 6.5938, "model.layers.41.zaya_block.experts.local_experts.12.linear_fc2": 11.8125, "model.layers.41.zaya_block.experts.local_experts.13.linear_fc1": 16.625, "model.layers.41.zaya_block.experts.local_experts.13.linear_fc2": 10.75, "model.layers.41.zaya_block.experts.local_experts.14.linear_fc1": 28.125, "model.layers.41.zaya_block.experts.local_experts.14.linear_fc2": 21.0, "model.layers.41.zaya_block.experts.local_experts.15.linear_fc1": 5.3125, "model.layers.41.zaya_block.experts.local_experts.15.linear_fc2": 10.1875, "model.layers.42.self_attn.o_proj": 8.0625, "model.layers.43.zaya_block.experts.local_experts.0.linear_fc1": 5.8125, "model.layers.43.zaya_block.experts.local_experts.0.linear_fc2": 12.75, "model.layers.43.zaya_block.experts.local_experts.1.linear_fc1": 8.1875, "model.layers.43.zaya_block.experts.local_experts.1.linear_fc2": 41.75, "model.layers.43.zaya_block.experts.local_experts.2.linear_fc1": 5.75, "model.layers.43.zaya_block.experts.local_experts.2.linear_fc2": 12.0, "model.layers.43.zaya_block.experts.local_experts.3.linear_fc1": 6.0938, "model.layers.43.zaya_block.experts.local_experts.3.linear_fc2": 11.375, "model.layers.43.zaya_block.experts.local_experts.4.linear_fc1": 5.375, "model.layers.43.zaya_block.experts.local_experts.4.linear_fc2": 15.6875, "model.layers.43.zaya_block.experts.local_experts.5.linear_fc1": 6.2812, "model.layers.43.zaya_block.experts.local_experts.5.linear_fc2": 21.375, "model.layers.43.zaya_block.experts.local_experts.6.linear_fc1": 5.875, "model.layers.43.zaya_block.experts.local_experts.6.linear_fc2": 20.625, "model.layers.43.zaya_block.experts.local_experts.7.linear_fc1": 6.2812, "model.layers.43.zaya_block.experts.local_experts.7.linear_fc2": 11.375, "model.layers.43.zaya_block.experts.local_experts.8.linear_fc1": 5.2812, "model.layers.43.zaya_block.experts.local_experts.8.linear_fc2": 12.0625, "model.layers.43.zaya_block.experts.local_experts.9.linear_fc1": 6.25, "model.layers.43.zaya_block.experts.local_experts.9.linear_fc2": 10.75, "model.layers.43.zaya_block.experts.local_experts.10.linear_fc1": 26.125, "model.layers.43.zaya_block.experts.local_experts.10.linear_fc2": 19.75, "model.layers.43.zaya_block.experts.local_experts.11.linear_fc1": 67.5, "model.layers.43.zaya_block.experts.local_experts.11.linear_fc2": 106.0, "model.layers.43.zaya_block.experts.local_experts.12.linear_fc1": 5.9375, "model.layers.43.zaya_block.experts.local_experts.12.linear_fc2": 10.875, "model.layers.43.zaya_block.experts.local_experts.13.linear_fc1": 5.2812, "model.layers.43.zaya_block.experts.local_experts.13.linear_fc2": 12.25, "model.layers.43.zaya_block.experts.local_experts.14.linear_fc1": 8.4375, "model.layers.43.zaya_block.experts.local_experts.14.linear_fc2": 12.1875, "model.layers.43.zaya_block.experts.local_experts.15.linear_fc1": 67.5, "model.layers.43.zaya_block.experts.local_experts.15.linear_fc2": 70.5, "model.layers.44.self_attn.o_proj": 9.9375, "model.layers.45.zaya_block.experts.local_experts.0.linear_fc1": 25.5, "model.layers.45.zaya_block.experts.local_experts.0.linear_fc2": 43.5, "model.layers.45.zaya_block.experts.local_experts.1.linear_fc1": 7.2188, "model.layers.45.zaya_block.experts.local_experts.1.linear_fc2": 16.0, "model.layers.45.zaya_block.experts.local_experts.2.linear_fc1": 9.0625, "model.layers.45.zaya_block.experts.local_experts.2.linear_fc2": 14.8125, "model.layers.45.zaya_block.experts.local_experts.3.linear_fc1": 6.6875, "model.layers.45.zaya_block.experts.local_experts.3.linear_fc2": 21.0, "model.layers.45.zaya_block.experts.local_experts.4.linear_fc1": 15.25, "model.layers.45.zaya_block.experts.local_experts.4.linear_fc2": 20.5, "model.layers.45.zaya_block.experts.local_experts.5.linear_fc1": 91.5, "model.layers.45.zaya_block.experts.local_experts.5.linear_fc2": 141.0, "model.layers.45.zaya_block.experts.local_experts.6.linear_fc1": 8.3125, "model.layers.45.zaya_block.experts.local_experts.6.linear_fc2": 21.375, "model.layers.45.zaya_block.experts.local_experts.7.linear_fc1": 5.9688, "model.layers.45.zaya_block.experts.local_experts.7.linear_fc2": 27.25, "model.layers.45.zaya_block.experts.local_experts.8.linear_fc1": 6.9688, "model.layers.45.zaya_block.experts.local_experts.8.linear_fc2": 15.4375, "model.layers.45.zaya_block.experts.local_experts.9.linear_fc1": 8.25, "model.layers.45.zaya_block.experts.local_experts.9.linear_fc2": 24.75, "model.layers.45.zaya_block.experts.local_experts.10.linear_fc1": 7.9062, "model.layers.45.zaya_block.experts.local_experts.10.linear_fc2": 17.5, "model.layers.45.zaya_block.experts.local_experts.11.linear_fc1": 8.5625, "model.layers.45.zaya_block.experts.local_experts.11.linear_fc2": 18.75, "model.layers.45.zaya_block.experts.local_experts.12.linear_fc1": 8.0, "model.layers.45.zaya_block.experts.local_experts.12.linear_fc2": 20.625, "model.layers.45.zaya_block.experts.local_experts.13.linear_fc1": 7.0938, "model.layers.45.zaya_block.experts.local_experts.13.linear_fc2": 20.125, "model.layers.45.zaya_block.experts.local_experts.14.linear_fc1": 7.3438, "model.layers.45.zaya_block.experts.local_experts.14.linear_fc2": 16.0, "model.layers.45.zaya_block.experts.local_experts.15.linear_fc1": 6.8125, "model.layers.45.zaya_block.experts.local_experts.15.linear_fc2": 13.5625, "model.layers.46.self_attn.o_proj": 15.1875, "model.layers.47.zaya_block.experts.local_experts.0.linear_fc1": 9.6875, "model.layers.47.zaya_block.experts.local_experts.0.linear_fc2": 17.875, "model.layers.47.zaya_block.experts.local_experts.1.linear_fc1": 9.6875, "model.layers.47.zaya_block.experts.local_experts.1.linear_fc2": 17.0, "model.layers.47.zaya_block.experts.local_experts.2.linear_fc1": 8.625, "model.layers.47.zaya_block.experts.local_experts.2.linear_fc2": 15.3125, "model.layers.47.zaya_block.experts.local_experts.3.linear_fc1": 8.125, "model.layers.47.zaya_block.experts.local_experts.3.linear_fc2": 17.875, "model.layers.47.zaya_block.experts.local_experts.4.linear_fc1": 9.9375, "model.layers.47.zaya_block.experts.local_experts.4.linear_fc2": 16.625, "model.layers.47.zaya_block.experts.local_experts.5.linear_fc1": 9.9375, "model.layers.47.zaya_block.experts.local_experts.5.linear_fc2": 18.25, "model.layers.47.zaya_block.experts.local_experts.6.linear_fc1": 7.9062, "model.layers.47.zaya_block.experts.local_experts.6.linear_fc2": 11.6875, "model.layers.47.zaya_block.experts.local_experts.7.linear_fc1": 13.1875, "model.layers.47.zaya_block.experts.local_experts.7.linear_fc2": 87.0, "model.layers.47.zaya_block.experts.local_experts.8.linear_fc1": 7.1875, "model.layers.47.zaya_block.experts.local_experts.8.linear_fc2": 14.3125, "model.layers.47.zaya_block.experts.local_experts.9.linear_fc1": 9.25, "model.layers.47.zaya_block.experts.local_experts.9.linear_fc2": 20.5, "model.layers.47.zaya_block.experts.local_experts.10.linear_fc1": 7.9688, "model.layers.47.zaya_block.experts.local_experts.10.linear_fc2": 19.0, "model.layers.47.zaya_block.experts.local_experts.11.linear_fc1": 6.6562, "model.layers.47.zaya_block.experts.local_experts.11.linear_fc2": 20.625, "model.layers.47.zaya_block.experts.local_experts.12.linear_fc1": 6.1562, "model.layers.47.zaya_block.experts.local_experts.12.linear_fc2": 21.0, "model.layers.47.zaya_block.experts.local_experts.13.linear_fc1": 7.0625, "model.layers.47.zaya_block.experts.local_experts.13.linear_fc2": 17.875, "model.layers.47.zaya_block.experts.local_experts.14.linear_fc1": 6.0938, "model.layers.47.zaya_block.experts.local_experts.14.linear_fc2": 18.75, "model.layers.47.zaya_block.experts.local_experts.15.linear_fc1": 97.0, "model.layers.47.zaya_block.experts.local_experts.15.linear_fc2": 111.5, "model.layers.48.self_attn.o_proj": 7.5938, "model.layers.49.zaya_block.experts.local_experts.0.linear_fc1": 7.6562, "model.layers.49.zaya_block.experts.local_experts.0.linear_fc2": 30.875, "model.layers.49.zaya_block.experts.local_experts.1.linear_fc1": 10.4375, "model.layers.49.zaya_block.experts.local_experts.1.linear_fc2": 31.25, "model.layers.49.zaya_block.experts.local_experts.2.linear_fc1": 15.0, "model.layers.49.zaya_block.experts.local_experts.2.linear_fc2": 21.625, "model.layers.49.zaya_block.experts.local_experts.3.linear_fc1": 11.6875, "model.layers.49.zaya_block.experts.local_experts.3.linear_fc2": 21.125, "model.layers.49.zaya_block.experts.local_experts.4.linear_fc1": 7.0625, "model.layers.49.zaya_block.experts.local_experts.4.linear_fc2": 22.0, "model.layers.49.zaya_block.experts.local_experts.5.linear_fc1": 88.0, "model.layers.49.zaya_block.experts.local_experts.5.linear_fc2": 65.0, "model.layers.49.zaya_block.experts.local_experts.6.linear_fc1": 8.9375, "model.layers.49.zaya_block.experts.local_experts.6.linear_fc2": 24.625, "model.layers.49.zaya_block.experts.local_experts.7.linear_fc1": 12.125, "model.layers.49.zaya_block.experts.local_experts.7.linear_fc2": 31.25, "model.layers.49.zaya_block.experts.local_experts.8.linear_fc1": 12.5, "model.layers.49.zaya_block.experts.local_experts.8.linear_fc2": 29.375, "model.layers.49.zaya_block.experts.local_experts.9.linear_fc1": 10.4375, "model.layers.49.zaya_block.experts.local_experts.9.linear_fc2": 22.25, "model.layers.49.zaya_block.experts.local_experts.10.linear_fc1": 7.5938, "model.layers.49.zaya_block.experts.local_experts.10.linear_fc2": 15.9375, "model.layers.49.zaya_block.experts.local_experts.11.linear_fc1": 10.25, "model.layers.49.zaya_block.experts.local_experts.11.linear_fc2": 19.875, "model.layers.49.zaya_block.experts.local_experts.12.linear_fc1": 13.0, "model.layers.49.zaya_block.experts.local_experts.12.linear_fc2": 15.375, "model.layers.49.zaya_block.experts.local_experts.13.linear_fc1": 11.0625, "model.layers.49.zaya_block.experts.local_experts.13.linear_fc2": 18.625, "model.layers.49.zaya_block.experts.local_experts.14.linear_fc1": 11.125, "model.layers.49.zaya_block.experts.local_experts.14.linear_fc2": 27.875, "model.layers.49.zaya_block.experts.local_experts.15.linear_fc1": 7.5938, "model.layers.49.zaya_block.experts.local_experts.15.linear_fc2": 24.875, "model.layers.50.self_attn.o_proj": 7.4688, "model.layers.51.zaya_block.experts.local_experts.0.linear_fc1": 86.0, "model.layers.51.zaya_block.experts.local_experts.0.linear_fc2": 49.0, "model.layers.51.zaya_block.experts.local_experts.1.linear_fc1": 6.0312, "model.layers.51.zaya_block.experts.local_experts.1.linear_fc2": 22.5, "model.layers.51.zaya_block.experts.local_experts.2.linear_fc1": 8.1875, "model.layers.51.zaya_block.experts.local_experts.2.linear_fc2": 41.5, "model.layers.51.zaya_block.experts.local_experts.3.linear_fc1": 8.6875, "model.layers.51.zaya_block.experts.local_experts.3.linear_fc2": 31.125, "model.layers.51.zaya_block.experts.local_experts.4.linear_fc1": 7.4688, "model.layers.51.zaya_block.experts.local_experts.4.linear_fc2": 20.5, "model.layers.51.zaya_block.experts.local_experts.5.linear_fc1": 6.4062, "model.layers.51.zaya_block.experts.local_experts.5.linear_fc2": 31.0, "model.layers.51.zaya_block.experts.local_experts.6.linear_fc1": 10.9375, "model.layers.51.zaya_block.experts.local_experts.6.linear_fc2": 26.375, "model.layers.51.zaya_block.experts.local_experts.7.linear_fc1": 9.4375, "model.layers.51.zaya_block.experts.local_experts.7.linear_fc2": 26.25, "model.layers.51.zaya_block.experts.local_experts.8.linear_fc1": 10.375, "model.layers.51.zaya_block.experts.local_experts.8.linear_fc2": 32.75, "model.layers.51.zaya_block.experts.local_experts.9.linear_fc1": 15.25, "model.layers.51.zaya_block.experts.local_experts.9.linear_fc2": 23.5, "model.layers.51.zaya_block.experts.local_experts.10.linear_fc1": 14.125, "model.layers.51.zaya_block.experts.local_experts.10.linear_fc2": 25.25, "model.layers.51.zaya_block.experts.local_experts.11.linear_fc1": 5.9062, "model.layers.51.zaya_block.experts.local_experts.11.linear_fc2": 23.25, "model.layers.51.zaya_block.experts.local_experts.12.linear_fc1": 8.375, "model.layers.51.zaya_block.experts.local_experts.12.linear_fc2": 24.125, "model.layers.51.zaya_block.experts.local_experts.13.linear_fc1": 6.375, "model.layers.51.zaya_block.experts.local_experts.13.linear_fc2": 22.875, "model.layers.51.zaya_block.experts.local_experts.14.linear_fc1": 6.6562, "model.layers.51.zaya_block.experts.local_experts.14.linear_fc2": 21.625, "model.layers.51.zaya_block.experts.local_experts.15.linear_fc1": 9.25, "model.layers.51.zaya_block.experts.local_experts.15.linear_fc2": 28.25, "model.layers.52.self_attn.o_proj": 10.5, "model.layers.53.zaya_block.experts.local_experts.0.linear_fc1": 67.0, "model.layers.53.zaya_block.experts.local_experts.0.linear_fc2": 37.75, "model.layers.53.zaya_block.experts.local_experts.1.linear_fc1": 67.0, "model.layers.53.zaya_block.experts.local_experts.1.linear_fc2": 346.0, "model.layers.53.zaya_block.experts.local_experts.2.linear_fc1": 8.375, "model.layers.53.zaya_block.experts.local_experts.2.linear_fc2": 20.125, "model.layers.53.zaya_block.experts.local_experts.3.linear_fc1": 67.0, "model.layers.53.zaya_block.experts.local_experts.3.linear_fc2": 36.0, "model.layers.53.zaya_block.experts.local_experts.4.linear_fc1": 11.5625, "model.layers.53.zaya_block.experts.local_experts.4.linear_fc2": 24.25, "model.layers.53.zaya_block.experts.local_experts.5.linear_fc1": 6.5938, "model.layers.53.zaya_block.experts.local_experts.5.linear_fc2": 39.0, "model.layers.53.zaya_block.experts.local_experts.6.linear_fc1": 5.9688, "model.layers.53.zaya_block.experts.local_experts.6.linear_fc2": 12.6875, "model.layers.53.zaya_block.experts.local_experts.7.linear_fc1": 9.0, "model.layers.53.zaya_block.experts.local_experts.7.linear_fc2": 23.5, "model.layers.53.zaya_block.experts.local_experts.8.linear_fc1": 67.0, "model.layers.53.zaya_block.experts.local_experts.8.linear_fc2": 29.0, "model.layers.53.zaya_block.experts.local_experts.9.linear_fc1": 9.1875, "model.layers.53.zaya_block.experts.local_experts.9.linear_fc2": 19.375, "model.layers.53.zaya_block.experts.local_experts.10.linear_fc1": 8.5625, "model.layers.53.zaya_block.experts.local_experts.10.linear_fc2": 30.0, "model.layers.53.zaya_block.experts.local_experts.11.linear_fc1": 5.8438, "model.layers.53.zaya_block.experts.local_experts.11.linear_fc2": 28.75, "model.layers.53.zaya_block.experts.local_experts.12.linear_fc1": 67.0, "model.layers.53.zaya_block.experts.local_experts.12.linear_fc2": 41.25, "model.layers.53.zaya_block.experts.local_experts.13.linear_fc1": 8.3125, "model.layers.53.zaya_block.experts.local_experts.13.linear_fc2": 21.375, "model.layers.53.zaya_block.experts.local_experts.14.linear_fc1": 8.5, "model.layers.53.zaya_block.experts.local_experts.14.linear_fc2": 23.5, "model.layers.53.zaya_block.experts.local_experts.15.linear_fc1": 8.3125, "model.layers.53.zaya_block.experts.local_experts.15.linear_fc2": 26.25, "model.layers.54.self_attn.o_proj": 12.5, "model.layers.55.zaya_block.experts.local_experts.0.linear_fc1": 56.25, "model.layers.55.zaya_block.experts.local_experts.0.linear_fc2": 23.125, "model.layers.55.zaya_block.experts.local_experts.1.linear_fc1": 7.875, "model.layers.55.zaya_block.experts.local_experts.1.linear_fc2": 25.75, "model.layers.55.zaya_block.experts.local_experts.2.linear_fc1": 7.7188, "model.layers.55.zaya_block.experts.local_experts.2.linear_fc2": 24.5, "model.layers.55.zaya_block.experts.local_experts.3.linear_fc1": 56.25, "model.layers.55.zaya_block.experts.local_experts.3.linear_fc2": 38.75, "model.layers.55.zaya_block.experts.local_experts.4.linear_fc1": 4.3438, "model.layers.55.zaya_block.experts.local_experts.4.linear_fc2": 14.75, "model.layers.55.zaya_block.experts.local_experts.5.linear_fc1": 7.9062, "model.layers.55.zaya_block.experts.local_experts.5.linear_fc2": 26.375, "model.layers.55.zaya_block.experts.local_experts.6.linear_fc1": 56.25, "model.layers.55.zaya_block.experts.local_experts.6.linear_fc2": 31.375, "model.layers.55.zaya_block.experts.local_experts.7.linear_fc1": 56.25, "model.layers.55.zaya_block.experts.local_experts.7.linear_fc2": 26.375, "model.layers.55.zaya_block.experts.local_experts.8.linear_fc1": 6.5, "model.layers.55.zaya_block.experts.local_experts.8.linear_fc2": 20.875, "model.layers.55.zaya_block.experts.local_experts.9.linear_fc1": 8.0625, "model.layers.55.zaya_block.experts.local_experts.9.linear_fc2": 25.75, "model.layers.55.zaya_block.experts.local_experts.10.linear_fc1": 8.125, "model.layers.55.zaya_block.experts.local_experts.10.linear_fc2": 30.125, "model.layers.55.zaya_block.experts.local_experts.11.linear_fc1": 8.125, "model.layers.55.zaya_block.experts.local_experts.11.linear_fc2": 24.75, "model.layers.55.zaya_block.experts.local_experts.12.linear_fc1": 7.875, "model.layers.55.zaya_block.experts.local_experts.12.linear_fc2": 25.375, "model.layers.55.zaya_block.experts.local_experts.13.linear_fc1": 7.8438, "model.layers.55.zaya_block.experts.local_experts.13.linear_fc2": 24.625, "model.layers.55.zaya_block.experts.local_experts.14.linear_fc1": 7.6562, "model.layers.55.zaya_block.experts.local_experts.14.linear_fc2": 24.375, "model.layers.55.zaya_block.experts.local_experts.15.linear_fc1": 8.0, "model.layers.55.zaya_block.experts.local_experts.15.linear_fc2": 23.5, "model.layers.56.self_attn.o_proj": 11.5625, "model.layers.57.zaya_block.experts.local_experts.0.linear_fc1": 7.8125, "model.layers.57.zaya_block.experts.local_experts.0.linear_fc2": 20.875, "model.layers.57.zaya_block.experts.local_experts.1.linear_fc1": 7.4062, "model.layers.57.zaya_block.experts.local_experts.1.linear_fc2": 22.5, "model.layers.57.zaya_block.experts.local_experts.2.linear_fc1": 51.0, "model.layers.57.zaya_block.experts.local_experts.2.linear_fc2": 27.375, "model.layers.57.zaya_block.experts.local_experts.3.linear_fc1": 7.8125, "model.layers.57.zaya_block.experts.local_experts.3.linear_fc2": 14.25, "model.layers.57.zaya_block.experts.local_experts.4.linear_fc1": 7.9375, "model.layers.57.zaya_block.experts.local_experts.4.linear_fc2": 14.75, "model.layers.57.zaya_block.experts.local_experts.5.linear_fc1": 8.125, "model.layers.57.zaya_block.experts.local_experts.5.linear_fc2": 20.625, "model.layers.57.zaya_block.experts.local_experts.6.linear_fc1": 7.875, "model.layers.57.zaya_block.experts.local_experts.6.linear_fc2": 21.625, "model.layers.57.zaya_block.experts.local_experts.7.linear_fc1": 8.0, "model.layers.57.zaya_block.experts.local_experts.7.linear_fc2": 38.75, "model.layers.57.zaya_block.experts.local_experts.8.linear_fc1": 51.0, "model.layers.57.zaya_block.experts.local_experts.8.linear_fc2": 19.25, "model.layers.57.zaya_block.experts.local_experts.9.linear_fc1": 5.4062, "model.layers.57.zaya_block.experts.local_experts.9.linear_fc2": 21.125, "model.layers.57.zaya_block.experts.local_experts.10.linear_fc1": 7.875, "model.layers.57.zaya_block.experts.local_experts.10.linear_fc2": 43.5, "model.layers.57.zaya_block.experts.local_experts.11.linear_fc1": 8.0, "model.layers.57.zaya_block.experts.local_experts.11.linear_fc2": 27.375, "model.layers.57.zaya_block.experts.local_experts.12.linear_fc1": 7.9062, "model.layers.57.zaya_block.experts.local_experts.12.linear_fc2": 45.0, "model.layers.57.zaya_block.experts.local_experts.13.linear_fc1": 5.3125, "model.layers.57.zaya_block.experts.local_experts.13.linear_fc2": 20.125, "model.layers.57.zaya_block.experts.local_experts.14.linear_fc1": 51.0, "model.layers.57.zaya_block.experts.local_experts.14.linear_fc2": 15.1875, "model.layers.57.zaya_block.experts.local_experts.15.linear_fc1": 7.9688, "model.layers.57.zaya_block.experts.local_experts.15.linear_fc2": 26.5, "model.layers.58.self_attn.o_proj": 8.3125, "model.layers.59.zaya_block.experts.local_experts.0.linear_fc1": 43.25, "model.layers.59.zaya_block.experts.local_experts.0.linear_fc2": 25.875, "model.layers.59.zaya_block.experts.local_experts.1.linear_fc1": 4.25, "model.layers.59.zaya_block.experts.local_experts.1.linear_fc2": 19.375, "model.layers.59.zaya_block.experts.local_experts.2.linear_fc1": 4.5625, "model.layers.59.zaya_block.experts.local_experts.2.linear_fc2": 21.75, "model.layers.59.zaya_block.experts.local_experts.3.linear_fc1": 43.25, "model.layers.59.zaya_block.experts.local_experts.3.linear_fc2": 17.75, "model.layers.59.zaya_block.experts.local_experts.4.linear_fc1": 6.4375, "model.layers.59.zaya_block.experts.local_experts.4.linear_fc2": 19.375, "model.layers.59.zaya_block.experts.local_experts.5.linear_fc1": 6.3438, "model.layers.59.zaya_block.experts.local_experts.5.linear_fc2": 20.5, "model.layers.59.zaya_block.experts.local_experts.6.linear_fc1": 5.6875, "model.layers.59.zaya_block.experts.local_experts.6.linear_fc2": 13.3125, "model.layers.59.zaya_block.experts.local_experts.7.linear_fc1": 43.25, "model.layers.59.zaya_block.experts.local_experts.7.linear_fc2": 17.875, "model.layers.59.zaya_block.experts.local_experts.8.linear_fc1": 6.5, "model.layers.59.zaya_block.experts.local_experts.8.linear_fc2": 20.25, "model.layers.59.zaya_block.experts.local_experts.9.linear_fc1": 5.875, "model.layers.59.zaya_block.experts.local_experts.9.linear_fc2": 26.25, "model.layers.59.zaya_block.experts.local_experts.10.linear_fc1": 6.6875, "model.layers.59.zaya_block.experts.local_experts.10.linear_fc2": 17.875, "model.layers.59.zaya_block.experts.local_experts.11.linear_fc1": 4.9375, "model.layers.59.zaya_block.experts.local_experts.11.linear_fc2": 28.0, "model.layers.59.zaya_block.experts.local_experts.12.linear_fc1": 5.9375, "model.layers.59.zaya_block.experts.local_experts.12.linear_fc2": 25.625, "model.layers.59.zaya_block.experts.local_experts.13.linear_fc1": 5.3125, "model.layers.59.zaya_block.experts.local_experts.13.linear_fc2": 19.25, "model.layers.59.zaya_block.experts.local_experts.14.linear_fc1": 6.3125, "model.layers.59.zaya_block.experts.local_experts.14.linear_fc2": 24.625, "model.layers.59.zaya_block.experts.local_experts.15.linear_fc1": 7.0625, "model.layers.59.zaya_block.experts.local_experts.15.linear_fc2": 21.0, "model.layers.60.self_attn.o_proj": 8.6875, "model.layers.61.zaya_block.experts.local_experts.0.linear_fc1": 6.5312, "model.layers.61.zaya_block.experts.local_experts.0.linear_fc2": 25.25, "model.layers.61.zaya_block.experts.local_experts.1.linear_fc1": 60.0, "model.layers.61.zaya_block.experts.local_experts.1.linear_fc2": 24.75, "model.layers.61.zaya_block.experts.local_experts.2.linear_fc1": 6.2188, "model.layers.61.zaya_block.experts.local_experts.2.linear_fc2": 21.125, "model.layers.61.zaya_block.experts.local_experts.3.linear_fc1": 6.5312, "model.layers.61.zaya_block.experts.local_experts.3.linear_fc2": 16.625, "model.layers.61.zaya_block.experts.local_experts.4.linear_fc1": 6.9375, "model.layers.61.zaya_block.experts.local_experts.4.linear_fc2": 23.25, "model.layers.61.zaya_block.experts.local_experts.5.linear_fc1": 6.5938, "model.layers.61.zaya_block.experts.local_experts.5.linear_fc2": 30.875, "model.layers.61.zaya_block.experts.local_experts.6.linear_fc1": 7.125, "model.layers.61.zaya_block.experts.local_experts.6.linear_fc2": 27.5, "model.layers.61.zaya_block.experts.local_experts.7.linear_fc1": 6.4375, "model.layers.61.zaya_block.experts.local_experts.7.linear_fc2": 30.75, "model.layers.61.zaya_block.experts.local_experts.8.linear_fc1": 4.625, "model.layers.61.zaya_block.experts.local_experts.8.linear_fc2": 22.625, "model.layers.61.zaya_block.experts.local_experts.9.linear_fc1": 5.4375, "model.layers.61.zaya_block.experts.local_experts.9.linear_fc2": 26.625, "model.layers.61.zaya_block.experts.local_experts.10.linear_fc1": 6.0625, "model.layers.61.zaya_block.experts.local_experts.10.linear_fc2": 22.75, "model.layers.61.zaya_block.experts.local_experts.11.linear_fc1": 6.3125, "model.layers.61.zaya_block.experts.local_experts.11.linear_fc2": 26.125, "model.layers.61.zaya_block.experts.local_experts.12.linear_fc1": 5.9688, "model.layers.61.zaya_block.experts.local_experts.12.linear_fc2": 18.75, "model.layers.61.zaya_block.experts.local_experts.13.linear_fc1": 6.4375, "model.layers.61.zaya_block.experts.local_experts.13.linear_fc2": 21.375, "model.layers.61.zaya_block.experts.local_experts.14.linear_fc1": 6.8125, "model.layers.61.zaya_block.experts.local_experts.14.linear_fc2": 22.125, "model.layers.61.zaya_block.experts.local_experts.15.linear_fc1": 6.5938, "model.layers.61.zaya_block.experts.local_experts.15.linear_fc2": 26.25, "model.layers.62.self_attn.o_proj": 6.5625, "model.layers.63.zaya_block.experts.local_experts.0.linear_fc1": 5.5938, "model.layers.63.zaya_block.experts.local_experts.0.linear_fc2": 16.25, "model.layers.63.zaya_block.experts.local_experts.1.linear_fc1": 4.3125, "model.layers.63.zaya_block.experts.local_experts.1.linear_fc2": 36.75, "model.layers.63.zaya_block.experts.local_experts.2.linear_fc1": 5.1875, "model.layers.63.zaya_block.experts.local_experts.2.linear_fc2": 19.25, "model.layers.63.zaya_block.experts.local_experts.3.linear_fc1": 5.4375, "model.layers.63.zaya_block.experts.local_experts.3.linear_fc2": 29.375, "model.layers.63.zaya_block.experts.local_experts.4.linear_fc1": 5.0625, "model.layers.63.zaya_block.experts.local_experts.4.linear_fc2": 19.625, "model.layers.63.zaya_block.experts.local_experts.5.linear_fc1": 4.8125, "model.layers.63.zaya_block.experts.local_experts.5.linear_fc2": 14.5, "model.layers.63.zaya_block.experts.local_experts.6.linear_fc1": 4.2812, "model.layers.63.zaya_block.experts.local_experts.6.linear_fc2": 18.0, "model.layers.63.zaya_block.experts.local_experts.7.linear_fc1": 5.8125, "model.layers.63.zaya_block.experts.local_experts.7.linear_fc2": 24.625, "model.layers.63.zaya_block.experts.local_experts.8.linear_fc1": 4.4375, "model.layers.63.zaya_block.experts.local_experts.8.linear_fc2": 18.125, "model.layers.63.zaya_block.experts.local_experts.9.linear_fc1": 4.6562, "model.layers.63.zaya_block.experts.local_experts.9.linear_fc2": 18.5, "model.layers.63.zaya_block.experts.local_experts.10.linear_fc1": 5.6562, "model.layers.63.zaya_block.experts.local_experts.10.linear_fc2": 67.5, "model.layers.63.zaya_block.experts.local_experts.11.linear_fc1": 4.7188, "model.layers.63.zaya_block.experts.local_experts.11.linear_fc2": 17.625, "model.layers.63.zaya_block.experts.local_experts.12.linear_fc1": 5.3125, "model.layers.63.zaya_block.experts.local_experts.12.linear_fc2": 14.8125, "model.layers.63.zaya_block.experts.local_experts.13.linear_fc1": 6.2188, "model.layers.63.zaya_block.experts.local_experts.13.linear_fc2": 24.25, "model.layers.63.zaya_block.experts.local_experts.14.linear_fc1": 41.75, "model.layers.63.zaya_block.experts.local_experts.14.linear_fc2": 124.5, "model.layers.63.zaya_block.experts.local_experts.15.linear_fc1": 5.3125, "model.layers.63.zaya_block.experts.local_experts.15.linear_fc2": 41.25, "model.layers.64.self_attn.o_proj": 11.5625, "model.layers.65.zaya_block.experts.local_experts.0.linear_fc1": 7.1562, "model.layers.65.zaya_block.experts.local_experts.0.linear_fc2": 19.875, "model.layers.65.zaya_block.experts.local_experts.1.linear_fc1": 6.9375, "model.layers.65.zaya_block.experts.local_experts.1.linear_fc2": 18.875, "model.layers.65.zaya_block.experts.local_experts.2.linear_fc1": 4.7188, "model.layers.65.zaya_block.experts.local_experts.2.linear_fc2": 37.0, "model.layers.65.zaya_block.experts.local_experts.3.linear_fc1": 4.9062, "model.layers.65.zaya_block.experts.local_experts.3.linear_fc2": 18.125, "model.layers.65.zaya_block.experts.local_experts.4.linear_fc1": 5.3125, "model.layers.65.zaya_block.experts.local_experts.4.linear_fc2": 22.375, "model.layers.65.zaya_block.experts.local_experts.5.linear_fc1": 5.5625, "model.layers.65.zaya_block.experts.local_experts.5.linear_fc2": 24.125, "model.layers.65.zaya_block.experts.local_experts.6.linear_fc1": 7.0938, "model.layers.65.zaya_block.experts.local_experts.6.linear_fc2": 61.75, "model.layers.65.zaya_block.experts.local_experts.7.linear_fc1": 5.625, "model.layers.65.zaya_block.experts.local_experts.7.linear_fc2": 13.8125, "model.layers.65.zaya_block.experts.local_experts.8.linear_fc1": 5.3125, "model.layers.65.zaya_block.experts.local_experts.8.linear_fc2": 24.125, "model.layers.65.zaya_block.experts.local_experts.9.linear_fc1": 4.4062, "model.layers.65.zaya_block.experts.local_experts.9.linear_fc2": 22.5, "model.layers.65.zaya_block.experts.local_experts.10.linear_fc1": 5.0625, "model.layers.65.zaya_block.experts.local_experts.10.linear_fc2": 27.625, "model.layers.65.zaya_block.experts.local_experts.11.linear_fc1": 72.5, "model.layers.65.zaya_block.experts.local_experts.11.linear_fc2": 1344.0, "model.layers.65.zaya_block.experts.local_experts.12.linear_fc1": 4.0625, "model.layers.65.zaya_block.experts.local_experts.12.linear_fc2": 27.5, "model.layers.65.zaya_block.experts.local_experts.13.linear_fc1": 5.1562, "model.layers.65.zaya_block.experts.local_experts.13.linear_fc2": 29.0, "model.layers.65.zaya_block.experts.local_experts.14.linear_fc1": 5.9062, "model.layers.65.zaya_block.experts.local_experts.14.linear_fc2": 22.25, "model.layers.65.zaya_block.experts.local_experts.15.linear_fc1": 5.2188, "model.layers.65.zaya_block.experts.local_experts.15.linear_fc2": 17.0, "model.layers.66.self_attn.o_proj": 9.1875, "model.layers.67.zaya_block.experts.local_experts.0.linear_fc1": 6.3438, "model.layers.67.zaya_block.experts.local_experts.0.linear_fc2": 92.5, "model.layers.67.zaya_block.experts.local_experts.1.linear_fc1": 4.4062, "model.layers.67.zaya_block.experts.local_experts.1.linear_fc2": 22.875, "model.layers.67.zaya_block.experts.local_experts.2.linear_fc1": 4.4688, "model.layers.67.zaya_block.experts.local_experts.2.linear_fc2": 26.0, "model.layers.67.zaya_block.experts.local_experts.3.linear_fc1": 4.3125, "model.layers.67.zaya_block.experts.local_experts.3.linear_fc2": 22.125, "model.layers.67.zaya_block.experts.local_experts.4.linear_fc1": 5.6562, "model.layers.67.zaya_block.experts.local_experts.4.linear_fc2": 27.75, "model.layers.67.zaya_block.experts.local_experts.5.linear_fc1": 6.9375, "model.layers.67.zaya_block.experts.local_experts.5.linear_fc2": 34.25, "model.layers.67.zaya_block.experts.local_experts.6.linear_fc1": 6.0, "model.layers.67.zaya_block.experts.local_experts.6.linear_fc2": 26.25, "model.layers.67.zaya_block.experts.local_experts.7.linear_fc1": 102.0, "model.layers.67.zaya_block.experts.local_experts.7.linear_fc2": 227.0, "model.layers.67.zaya_block.experts.local_experts.8.linear_fc1": 4.0625, "model.layers.67.zaya_block.experts.local_experts.8.linear_fc2": 49.0, "model.layers.67.zaya_block.experts.local_experts.9.linear_fc1": 8.5625, "model.layers.67.zaya_block.experts.local_experts.9.linear_fc2": 21.25, "model.layers.67.zaya_block.experts.local_experts.10.linear_fc1": 5.5938, "model.layers.67.zaya_block.experts.local_experts.10.linear_fc2": 23.875, "model.layers.67.zaya_block.experts.local_experts.11.linear_fc1": 7.5312, "model.layers.67.zaya_block.experts.local_experts.11.linear_fc2": 22.375, "model.layers.67.zaya_block.experts.local_experts.12.linear_fc1": 7.3125, "model.layers.67.zaya_block.experts.local_experts.12.linear_fc2": 52.75, "model.layers.67.zaya_block.experts.local_experts.14.linear_fc1": 5.3438, "model.layers.67.zaya_block.experts.local_experts.14.linear_fc2": 24.25, "model.layers.67.zaya_block.experts.local_experts.15.linear_fc1": 4.6562, "model.layers.67.zaya_block.experts.local_experts.15.linear_fc2": 22.625, "model.layers.67.zaya_block.experts.local_experts.13.linear_fc1": 5.5625, "model.layers.67.zaya_block.experts.local_experts.13.linear_fc2": 22.75, "model.layers.68.self_attn.o_proj": 15.0625, "model.layers.69.zaya_block.experts.local_experts.0.linear_fc1": 4.125, "model.layers.69.zaya_block.experts.local_experts.0.linear_fc2": 23.5, "model.layers.69.zaya_block.experts.local_experts.1.linear_fc1": 5.4062, "model.layers.69.zaya_block.experts.local_experts.1.linear_fc2": 28.625, "model.layers.69.zaya_block.experts.local_experts.2.linear_fc1": 5.5, "model.layers.69.zaya_block.experts.local_experts.2.linear_fc2": 87.5, "model.layers.69.zaya_block.experts.local_experts.3.linear_fc1": 4.6875, "model.layers.69.zaya_block.experts.local_experts.3.linear_fc2": 45.75, "model.layers.69.zaya_block.experts.local_experts.4.linear_fc1": 5.0, "model.layers.69.zaya_block.experts.local_experts.4.linear_fc2": 80.5, "model.layers.69.zaya_block.experts.local_experts.5.linear_fc1": 146.0, "model.layers.69.zaya_block.experts.local_experts.5.linear_fc2": 1784.0, "model.layers.69.zaya_block.experts.local_experts.6.linear_fc1": 5.625, "model.layers.69.zaya_block.experts.local_experts.6.linear_fc2": 30.625, "model.layers.69.zaya_block.experts.local_experts.7.linear_fc1": 4.875, "model.layers.69.zaya_block.experts.local_experts.7.linear_fc2": 62.25, "model.layers.69.zaya_block.experts.local_experts.8.linear_fc1": 4.9688, "model.layers.69.zaya_block.experts.local_experts.8.linear_fc2": 27.25, "model.layers.69.zaya_block.experts.local_experts.9.linear_fc1": 5.2812, "model.layers.69.zaya_block.experts.local_experts.9.linear_fc2": 60.75, "model.layers.69.zaya_block.experts.local_experts.10.linear_fc1": 5.0312, "model.layers.69.zaya_block.experts.local_experts.10.linear_fc2": 44.0, "model.layers.69.zaya_block.experts.local_experts.11.linear_fc1": 4.4062, "model.layers.69.zaya_block.experts.local_experts.11.linear_fc2": 22.25, "model.layers.69.zaya_block.experts.local_experts.12.linear_fc1": 4.3438, "model.layers.69.zaya_block.experts.local_experts.12.linear_fc2": 78.0, "model.layers.69.zaya_block.experts.local_experts.13.linear_fc1": 9.75, "model.layers.69.zaya_block.experts.local_experts.13.linear_fc2": 59.0, "model.layers.69.zaya_block.experts.local_experts.14.linear_fc1": 13.9375, "model.layers.69.zaya_block.experts.local_experts.14.linear_fc2": 26.75, "model.layers.69.zaya_block.experts.local_experts.15.linear_fc1": 4.375, "model.layers.69.zaya_block.experts.local_experts.15.linear_fc2": 34.25, "model.layers.70.self_attn.o_proj": 13.5, "model.layers.71.zaya_block.experts.local_experts.0.linear_fc1": 5.6875, "model.layers.71.zaya_block.experts.local_experts.0.linear_fc2": 107.5, "model.layers.71.zaya_block.experts.local_experts.1.linear_fc1": 5.5, "model.layers.71.zaya_block.experts.local_experts.1.linear_fc2": 26.875, "model.layers.71.zaya_block.experts.local_experts.2.linear_fc1": 4.7188, "model.layers.71.zaya_block.experts.local_experts.2.linear_fc2": 27.125, "model.layers.71.zaya_block.experts.local_experts.3.linear_fc1": 4.8125, "model.layers.71.zaya_block.experts.local_experts.3.linear_fc2": 26.25, "model.layers.71.zaya_block.experts.local_experts.4.linear_fc1": 5.7812, "model.layers.71.zaya_block.experts.local_experts.4.linear_fc2": 134.0, "model.layers.71.zaya_block.experts.local_experts.5.linear_fc1": 5.9375, "model.layers.71.zaya_block.experts.local_experts.5.linear_fc2": 26.125, "model.layers.71.zaya_block.experts.local_experts.6.linear_fc1": 5.6875, "model.layers.71.zaya_block.experts.local_experts.6.linear_fc2": 107.5, "model.layers.71.zaya_block.experts.local_experts.7.linear_fc1": 5.5312, "model.layers.71.zaya_block.experts.local_experts.7.linear_fc2": 40.75, "model.layers.71.zaya_block.experts.local_experts.9.linear_fc1": 6.2812, "model.layers.71.zaya_block.experts.local_experts.9.linear_fc2": 46.75, "model.layers.71.zaya_block.experts.local_experts.10.linear_fc1": 135.0, "model.layers.71.zaya_block.experts.local_experts.10.linear_fc2": 2480.0, "model.layers.71.zaya_block.experts.local_experts.11.linear_fc1": 4.6875, "model.layers.71.zaya_block.experts.local_experts.11.linear_fc2": 23.25, "model.layers.71.zaya_block.experts.local_experts.12.linear_fc1": 7.1875, "model.layers.71.zaya_block.experts.local_experts.12.linear_fc2": 58.5, "model.layers.71.zaya_block.experts.local_experts.13.linear_fc1": 4.8125, "model.layers.71.zaya_block.experts.local_experts.13.linear_fc2": 37.75, "model.layers.71.zaya_block.experts.local_experts.14.linear_fc1": 4.25, "model.layers.71.zaya_block.experts.local_experts.14.linear_fc2": 2352.0, "model.layers.71.zaya_block.experts.local_experts.8.linear_fc1": 5.0938, "model.layers.71.zaya_block.experts.local_experts.8.linear_fc2": 26.5, "model.layers.71.zaya_block.experts.local_experts.15.linear_fc1": 4.3438, "model.layers.71.zaya_block.experts.local_experts.15.linear_fc2": 25.125, "model.layers.72.self_attn.o_proj": 17.25, "model.layers.73.zaya_block.experts.local_experts.0.linear_fc1": 5.6562, "model.layers.73.zaya_block.experts.local_experts.0.linear_fc2": 34.0, "model.layers.73.zaya_block.experts.local_experts.1.linear_fc1": 5.9062, "model.layers.73.zaya_block.experts.local_experts.1.linear_fc2": 30.75, "model.layers.73.zaya_block.experts.local_experts.2.linear_fc1": 5.6875, "model.layers.73.zaya_block.experts.local_experts.2.linear_fc2": 70.5, "model.layers.73.zaya_block.experts.local_experts.3.linear_fc1": 6.7812, "model.layers.73.zaya_block.experts.local_experts.3.linear_fc2": 72.0, "model.layers.73.zaya_block.experts.local_experts.4.linear_fc1": 7.4062, "model.layers.73.zaya_block.experts.local_experts.4.linear_fc2": 92.5, "model.layers.73.zaya_block.experts.local_experts.5.linear_fc1": 96.5, "model.layers.73.zaya_block.experts.local_experts.5.linear_fc2": 6240.0, "model.layers.73.zaya_block.experts.local_experts.6.linear_fc1": 5.8125, "model.layers.73.zaya_block.experts.local_experts.6.linear_fc2": 75.5, "model.layers.73.zaya_block.experts.local_experts.7.linear_fc1": 5.6875, "model.layers.73.zaya_block.experts.local_experts.7.linear_fc2": 246.0, "model.layers.73.zaya_block.experts.local_experts.8.linear_fc1": 6.7188, "model.layers.73.zaya_block.experts.local_experts.8.linear_fc2": 59.5, "model.layers.73.zaya_block.experts.local_experts.9.linear_fc1": 5.125, "model.layers.73.zaya_block.experts.local_experts.9.linear_fc2": 37.0, "model.layers.73.zaya_block.experts.local_experts.10.linear_fc1": 7.625, "model.layers.73.zaya_block.experts.local_experts.10.linear_fc2": 51.75, "model.layers.73.zaya_block.experts.local_experts.11.linear_fc1": 6.4062, "model.layers.73.zaya_block.experts.local_experts.11.linear_fc2": 150.0, "model.layers.73.zaya_block.experts.local_experts.12.linear_fc1": 5.7812, "model.layers.73.zaya_block.experts.local_experts.12.linear_fc2": 51.25, "model.layers.73.zaya_block.experts.local_experts.14.linear_fc1": 6.0312, "model.layers.73.zaya_block.experts.local_experts.14.linear_fc2": 177.0, "model.layers.73.zaya_block.experts.local_experts.15.linear_fc1": 5.7188, "model.layers.73.zaya_block.experts.local_experts.15.linear_fc2": 34.75, "model.layers.73.zaya_block.experts.local_experts.13.linear_fc1": 6.0938, "model.layers.73.zaya_block.experts.local_experts.13.linear_fc2": 28.75, "model.layers.74.self_attn.o_proj": 26.5, "model.layers.75.zaya_block.experts.local_experts.0.linear_fc1": 11.125, "model.layers.75.zaya_block.experts.local_experts.0.linear_fc2": 230.0, "model.layers.75.zaya_block.experts.local_experts.1.linear_fc1": 10.4375, "model.layers.75.zaya_block.experts.local_experts.1.linear_fc2": 8896.0, "model.layers.75.zaya_block.experts.local_experts.2.linear_fc1": 12.6875, "model.layers.75.zaya_block.experts.local_experts.2.linear_fc2": 188.0, "model.layers.75.zaya_block.experts.local_experts.3.linear_fc1": 11.25, "model.layers.75.zaya_block.experts.local_experts.3.linear_fc2": 136.0, "model.layers.75.zaya_block.experts.local_experts.4.linear_fc1": 12.5, "model.layers.75.zaya_block.experts.local_experts.4.linear_fc2": 142.0, "model.layers.75.zaya_block.experts.local_experts.5.linear_fc1": 11.4375, "model.layers.75.zaya_block.experts.local_experts.5.linear_fc2": 784.0, "model.layers.75.zaya_block.experts.local_experts.6.linear_fc1": 11.125, "model.layers.75.zaya_block.experts.local_experts.6.linear_fc2": 117.0, "model.layers.75.zaya_block.experts.local_experts.7.linear_fc1": 11.0, "model.layers.75.zaya_block.experts.local_experts.7.linear_fc2": 202.0, "model.layers.75.zaya_block.experts.local_experts.8.linear_fc1": 11.5, "model.layers.75.zaya_block.experts.local_experts.8.linear_fc2": 406.0, "model.layers.75.zaya_block.experts.local_experts.9.linear_fc1": 11.1875, "model.layers.75.zaya_block.experts.local_experts.9.linear_fc2": 116.0, "model.layers.75.zaya_block.experts.local_experts.10.linear_fc1": 10.5, "model.layers.75.zaya_block.experts.local_experts.10.linear_fc2": 104.0, "model.layers.75.zaya_block.experts.local_experts.11.linear_fc1": 11.6875, "model.layers.75.zaya_block.experts.local_experts.11.linear_fc2": 187.0, "model.layers.75.zaya_block.experts.local_experts.12.linear_fc1": 10.3125, "model.layers.75.zaya_block.experts.local_experts.12.linear_fc2": 127.0, "model.layers.75.zaya_block.experts.local_experts.13.linear_fc1": 11.1875, "model.layers.75.zaya_block.experts.local_experts.13.linear_fc2": 155.0, "model.layers.75.zaya_block.experts.local_experts.14.linear_fc1": 10.5, "model.layers.75.zaya_block.experts.local_experts.14.linear_fc2": 63.75, "model.layers.75.zaya_block.experts.local_experts.15.linear_fc1": 10.125, "model.layers.75.zaya_block.experts.local_experts.15.linear_fc2": 70.5, "model.layers.76.self_attn.o_proj": 39.75, "model.layers.77.zaya_block.experts.local_experts.0.linear_fc1": 17.5, "model.layers.77.zaya_block.experts.local_experts.0.linear_fc2": 318.0, "model.layers.77.zaya_block.experts.local_experts.1.linear_fc1": 17.875, "model.layers.77.zaya_block.experts.local_experts.1.linear_fc2": 189.0, "model.layers.77.zaya_block.experts.local_experts.2.linear_fc1": 16.625, "model.layers.77.zaya_block.experts.local_experts.2.linear_fc2": 131.0, "model.layers.77.zaya_block.experts.local_experts.3.linear_fc1": 16.875, "model.layers.77.zaya_block.experts.local_experts.3.linear_fc2": 2496.0, "model.layers.77.zaya_block.experts.local_experts.4.linear_fc1": 17.125, "model.layers.77.zaya_block.experts.local_experts.4.linear_fc2": 6656.0, "model.layers.77.zaya_block.experts.local_experts.5.linear_fc1": 13.5625, "model.layers.77.zaya_block.experts.local_experts.5.linear_fc2": 314.0, "model.layers.77.zaya_block.experts.local_experts.6.linear_fc1": 19.875, "model.layers.77.zaya_block.experts.local_experts.6.linear_fc2": 506.0, "model.layers.77.zaya_block.experts.local_experts.7.linear_fc1": 21.25, "model.layers.77.zaya_block.experts.local_experts.7.linear_fc2": 856.0, "model.layers.77.zaya_block.experts.local_experts.8.linear_fc1": 15.25, "model.layers.77.zaya_block.experts.local_experts.8.linear_fc2": 170.0, "model.layers.77.zaya_block.experts.local_experts.9.linear_fc1": 14.75, "model.layers.77.zaya_block.experts.local_experts.9.linear_fc2": 736.0, "model.layers.77.zaya_block.experts.local_experts.10.linear_fc1": 15.25, "model.layers.77.zaya_block.experts.local_experts.10.linear_fc2": 125.5, "model.layers.77.zaya_block.experts.local_experts.11.linear_fc1": 19.25, "model.layers.77.zaya_block.experts.local_experts.11.linear_fc2": 2592.0, "model.layers.77.zaya_block.experts.local_experts.12.linear_fc1": 16.375, "model.layers.77.zaya_block.experts.local_experts.12.linear_fc2": 115.0, "model.layers.77.zaya_block.experts.local_experts.13.linear_fc1": 16.625, "model.layers.77.zaya_block.experts.local_experts.13.linear_fc2": 161.0, "model.layers.77.zaya_block.experts.local_experts.14.linear_fc1": 23.25, "model.layers.77.zaya_block.experts.local_experts.14.linear_fc2": 552.0, "model.layers.77.zaya_block.experts.local_experts.15.linear_fc1": 20.75, "model.layers.77.zaya_block.experts.local_experts.15.linear_fc2": 218.0, "model.layers.78.self_attn.o_proj": 30.125, "model.layers.79.zaya_block.experts.local_experts.0.linear_fc1": 25.25, "model.layers.79.zaya_block.experts.local_experts.0.linear_fc2": 652.0, "model.layers.79.zaya_block.experts.local_experts.1.linear_fc1": 21.25, "model.layers.79.zaya_block.experts.local_experts.1.linear_fc2": 219.0, "model.layers.79.zaya_block.experts.local_experts.2.linear_fc1": 19.625, "model.layers.79.zaya_block.experts.local_experts.2.linear_fc2": 346.0, "model.layers.79.zaya_block.experts.local_experts.3.linear_fc1": 24.25, "model.layers.79.zaya_block.experts.local_experts.3.linear_fc2": 460.0, "model.layers.79.zaya_block.experts.local_experts.4.linear_fc1": 24.125, "model.layers.79.zaya_block.experts.local_experts.4.linear_fc2": 346.0, "model.layers.79.zaya_block.experts.local_experts.5.linear_fc1": 22.5, "model.layers.79.zaya_block.experts.local_experts.5.linear_fc2": 314.0, "model.layers.79.zaya_block.experts.local_experts.6.linear_fc1": 27.5, "model.layers.79.zaya_block.experts.local_experts.6.linear_fc2": 460.0, "model.layers.79.zaya_block.experts.local_experts.7.linear_fc1": 26.0, "model.layers.79.zaya_block.experts.local_experts.7.linear_fc2": 324.0, "model.layers.79.zaya_block.experts.local_experts.8.linear_fc1": 19.625, "model.layers.79.zaya_block.experts.local_experts.8.linear_fc2": 112.0, "model.layers.79.zaya_block.experts.local_experts.9.linear_fc1": 26.125, "model.layers.79.zaya_block.experts.local_experts.9.linear_fc2": 161.0, "model.layers.79.zaya_block.experts.local_experts.10.linear_fc1": 21.0, "model.layers.79.zaya_block.experts.local_experts.10.linear_fc2": 116.0, "model.layers.79.zaya_block.experts.local_experts.11.linear_fc1": 20.75, "model.layers.79.zaya_block.experts.local_experts.11.linear_fc2": 330.0, "model.layers.79.zaya_block.experts.local_experts.12.linear_fc1": 25.375, "model.layers.79.zaya_block.experts.local_experts.12.linear_fc2": 392.0, "model.layers.79.zaya_block.experts.local_experts.13.linear_fc1": 26.0, "model.layers.79.zaya_block.experts.local_experts.13.linear_fc2": 448.0, "model.layers.79.zaya_block.experts.local_experts.14.linear_fc1": 23.5, "model.layers.79.zaya_block.experts.local_experts.14.linear_fc2": 286.0, "model.layers.79.zaya_block.experts.local_experts.15.linear_fc1": 25.875, "model.layers.79.zaya_block.experts.local_experts.15.linear_fc2": 372.0 }, "hardware": { "gpu": "NVIDIA GeForce RTX 5070 Ti", "vram": "16 GB", "cuda": "13.0" }, "modules_compressed": 1320, "output_size_bytes": 6020194188, "dry_run": false, "evaluation": { "method": "paired exact-binomial McNemar on discordant items, joined per doc_id", "control": "Ttimms/zaya1-8b-nvfp4-w4a4 (9.46 GB, 384 BF16-exempted Linears)", "items_per_checkpoint": 14319, "chat_template_applied": false, "num_fewshot": 0, "results": { "hellaswag_acc": { "this": 0.4579, "control": 0.4649, "delta_pp": -0.71, "ci95_pp": [ -1.26, -0.15 ], "p": 0.014, "n": 10042 }, "hellaswag_acc_norm": { "this": 0.6065, "control": 0.6134, "delta_pp": -0.7, "ci95_pp": [ -1.39, -0.01 ], "p": 0.0504, "n": 10042 }, "arc_challenge_acc": { "this": 0.3797, "control": 0.3695, "delta_pp": 1.02, "ci95_pp": [ -1.42, 3.47 ], "p": 0.4522, "n": 1172 }, "arc_challenge_acc_norm": { "this": 0.3797, "control": 0.4036, "delta_pp": -2.39, "ci95_pp": [ -4.97, 0.19 ], "p": 0.0799, "n": 1172 }, "winogrande_acc": { "this": 0.562, "control": 0.5904, "delta_pp": -2.84, "ci95_pp": [ -6.04, 0.36 ], "p": 0.0906, "n": 1267 }, "piqa_acc": { "this": 0.6942, "control": 0.7002, "delta_pp": -0.6, "ci95_pp": [ -2.41, 1.21 ], "p": 0.5564, "n": 1838 }, "piqa_acc_norm": { "this": 0.7089, "control": 0.7008, "delta_pp": 0.82, "ci95_pp": [ -1.02, 2.65 ], "p": 0.4166, "n": 1838 } }, "interpretation": "No comparison survives Bonferroni (alpha=0.05/7=0.0071). That is absence of resolution, not evidence of absence: HellaSwag is the only adequately powered task and its CI excludes zero, while the smaller benchmarks still admit -5 to -6 pp. Defensible claim: -0.71 pp on HellaSwag for -36% checkpoint size.", "reproduce": "https://github.com/t-timms/zaya1-godspeed - scripts/run_phase_a.py, scripts/analyze_phase_a.py" } }