[
  {
    "mixlab": "w0_embed",
    "hf": "embed_tokens.weight",
    "shape": [
      16384,
      384
    ]
  },
  {
    "mixlab": "w1_shared_relative_embeddings",
    "hf": "relative_embeddings",
    "shape": [
      63,
      384
    ]
  },
  {
    "mixlab": "w2_shared_relative_norm_scale",
    "hf": "relative_layer_norm.weight",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w3_shared_relative_norm_bias",
    "hf": "relative_layer_norm.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w4_wq",
    "hf": "blocks.0.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w5_wq_bias",
    "hf": "blocks.0.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w6_wk",
    "hf": "blocks.0.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w7_wk_bias",
    "hf": "blocks.0.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w8_wv",
    "hf": "blocks.0.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w9_wv_bias",
    "hf": "blocks.0.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w10_wo",
    "hf": "blocks.0.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w11_wo_bias",
    "hf": "blocks.0.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w12_ff_gate",
    "hf": "blocks.0.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w13_ff1",
    "hf": "blocks.0.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w14_ff2",
    "hf": "blocks.0.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w15_wq",
    "hf": "blocks.1.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w16_wq_bias",
    "hf": "blocks.1.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w17_wk",
    "hf": "blocks.1.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w18_wk_bias",
    "hf": "blocks.1.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w19_wv",
    "hf": "blocks.1.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w20_wv_bias",
    "hf": "blocks.1.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w21_wo",
    "hf": "blocks.1.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w22_wo_bias",
    "hf": "blocks.1.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w23_ff_gate",
    "hf": "blocks.1.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w24_ff1",
    "hf": "blocks.1.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w25_ff2",
    "hf": "blocks.1.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w26_wq",
    "hf": "blocks.2.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w27_wq_bias",
    "hf": "blocks.2.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w28_wk",
    "hf": "blocks.2.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w29_wk_bias",
    "hf": "blocks.2.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w30_wv",
    "hf": "blocks.2.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w31_wv_bias",
    "hf": "blocks.2.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w32_wo",
    "hf": "blocks.2.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w33_wo_bias",
    "hf": "blocks.2.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w34_ff_gate",
    "hf": "blocks.2.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w35_ff1",
    "hf": "blocks.2.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w36_ff2",
    "hf": "blocks.2.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w37_wq",
    "hf": "blocks.3.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w38_wq_bias",
    "hf": "blocks.3.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w39_wk",
    "hf": "blocks.3.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w40_wk_bias",
    "hf": "blocks.3.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w41_wv",
    "hf": "blocks.3.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w42_wv_bias",
    "hf": "blocks.3.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w43_wo",
    "hf": "blocks.3.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w44_wo_bias",
    "hf": "blocks.3.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w45_ff_gate",
    "hf": "blocks.3.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w46_ff1",
    "hf": "blocks.3.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w47_ff2",
    "hf": "blocks.3.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w48_wq",
    "hf": "blocks.4.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w49_wq_bias",
    "hf": "blocks.4.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w50_wk",
    "hf": "blocks.4.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w51_wk_bias",
    "hf": "blocks.4.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w52_wv",
    "hf": "blocks.4.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w53_wv_bias",
    "hf": "blocks.4.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w54_wo",
    "hf": "blocks.4.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w55_wo_bias",
    "hf": "blocks.4.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w56_ff_gate",
    "hf": "blocks.4.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w57_ff1",
    "hf": "blocks.4.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w58_ff2",
    "hf": "blocks.4.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w59_wq",
    "hf": "blocks.5.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w60_wq_bias",
    "hf": "blocks.5.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w61_wk",
    "hf": "blocks.5.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w62_wk_bias",
    "hf": "blocks.5.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w63_wv",
    "hf": "blocks.5.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w64_wv_bias",
    "hf": "blocks.5.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w65_wo",
    "hf": "blocks.5.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w66_wo_bias",
    "hf": "blocks.5.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w67_ff_gate",
    "hf": "blocks.5.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w68_ff1",
    "hf": "blocks.5.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w69_ff2",
    "hf": "blocks.5.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w70_wq",
    "hf": "blocks.6.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w71_wq_bias",
    "hf": "blocks.6.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w72_wk",
    "hf": "blocks.6.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w73_wk_bias",
    "hf": "blocks.6.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w74_wv",
    "hf": "blocks.6.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w75_wv_bias",
    "hf": "blocks.6.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w76_wo",
    "hf": "blocks.6.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w77_wo_bias",
    "hf": "blocks.6.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w78_ff_gate",
    "hf": "blocks.6.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w79_ff1",
    "hf": "blocks.6.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w80_ff2",
    "hf": "blocks.6.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w81_wq",
    "hf": "blocks.7.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w82_wq_bias",
    "hf": "blocks.7.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w83_wk",
    "hf": "blocks.7.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w84_wk_bias",
    "hf": "blocks.7.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w85_wv",
    "hf": "blocks.7.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w86_wv_bias",
    "hf": "blocks.7.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w87_wo",
    "hf": "blocks.7.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w88_wo_bias",
    "hf": "blocks.7.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w89_ff_gate",
    "hf": "blocks.7.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w90_ff1",
    "hf": "blocks.7.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w91_ff2",
    "hf": "blocks.7.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w92_wq",
    "hf": "blocks.8.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w93_wq_bias",
    "hf": "blocks.8.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w94_wk",
    "hf": "blocks.8.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w95_wk_bias",
    "hf": "blocks.8.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w96_wv",
    "hf": "blocks.8.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w97_wv_bias",
    "hf": "blocks.8.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w98_wo",
    "hf": "blocks.8.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w99_wo_bias",
    "hf": "blocks.8.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w100_ff_gate",
    "hf": "blocks.8.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w101_ff1",
    "hf": "blocks.8.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w102_ff2",
    "hf": "blocks.8.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w103_wq",
    "hf": "blocks.9.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w104_wq_bias",
    "hf": "blocks.9.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w105_wk",
    "hf": "blocks.9.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w106_wk_bias",
    "hf": "blocks.9.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w107_wv",
    "hf": "blocks.9.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w108_wv_bias",
    "hf": "blocks.9.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w109_wo",
    "hf": "blocks.9.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w110_wo_bias",
    "hf": "blocks.9.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w111_ff_gate",
    "hf": "blocks.9.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w112_ff1",
    "hf": "blocks.9.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w113_ff2",
    "hf": "blocks.9.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w114_wq",
    "hf": "blocks.10.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w115_wq_bias",
    "hf": "blocks.10.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w116_wk",
    "hf": "blocks.10.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w117_wk_bias",
    "hf": "blocks.10.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w118_wv",
    "hf": "blocks.10.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w119_wv_bias",
    "hf": "blocks.10.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w120_wo",
    "hf": "blocks.10.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w121_wo_bias",
    "hf": "blocks.10.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w122_ff_gate",
    "hf": "blocks.10.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w123_ff1",
    "hf": "blocks.10.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w124_ff2",
    "hf": "blocks.10.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w125_wq",
    "hf": "blocks.11.wq.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w126_wq_bias",
    "hf": "blocks.11.wq.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w127_wk",
    "hf": "blocks.11.wk.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w128_wk_bias",
    "hf": "blocks.11.wk.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w129_wv",
    "hf": "blocks.11.wv.weight",
    "shape": [
      384,
      768
    ]
  },
  {
    "mixlab": "w130_wv_bias",
    "hf": "blocks.11.wv.bias",
    "shape": [
      768
    ]
  },
  {
    "mixlab": "w131_wo",
    "hf": "blocks.11.wo.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w132_wo_bias",
    "hf": "blocks.11.wo.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w133_ff_gate",
    "hf": "blocks.11.ff_gate.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w134_ff1",
    "hf": "blocks.11.ff1.weight",
    "shape": [
      384,
      1280
    ]
  },
  {
    "mixlab": "w135_ff2",
    "hf": "blocks.11.ff2.weight",
    "shape": [
      1280,
      384
    ]
  },
  {
    "mixlab": "w136_mlm_head_dense",
    "hf": "mlm_head_dense.weight",
    "shape": [
      384,
      384
    ]
  },
  {
    "mixlab": "w137_mlm_head_dense_bias",
    "hf": "mlm_head_dense.bias",
    "shape": [
      384
    ]
  },
  {
    "mixlab": "w138_mlm_head_output_bias",
    "hf": "mlm_head_output_bias",
    "shape": [
      16384
    ]
  },
  {
    "mixlab": "w139_dwa_alpha_0",
    "hf": "dwa_alphas.0",
    "shape": [
      2
    ]
  },
  {
    "mixlab": "w140_dwa_alpha_1",
    "hf": "dwa_alphas.1",
    "shape": [
      3
    ]
  },
  {
    "mixlab": "w141_dwa_alpha_2",
    "hf": "dwa_alphas.2",
    "shape": [
      4
    ]
  },
  {
    "mixlab": "w142_dwa_alpha_3",
    "hf": "dwa_alphas.3",
    "shape": [
      5
    ]
  },
  {
    "mixlab": "w143_dwa_alpha_4",
    "hf": "dwa_alphas.4",
    "shape": [
      6
    ]
  },
  {
    "mixlab": "w144_dwa_alpha_5",
    "hf": "dwa_alphas.5",
    "shape": [
      7
    ]
  },
  {
    "mixlab": "w145_dwa_alpha_6",
    "hf": "dwa_alphas.6",
    "shape": [
      8
    ]
  },
  {
    "mixlab": "w146_dwa_alpha_7",
    "hf": "dwa_alphas.7",
    "shape": [
      9
    ]
  },
  {
    "mixlab": "w147_dwa_alpha_8",
    "hf": "dwa_alphas.8",
    "shape": [
      10
    ]
  },
  {
    "mixlab": "w148_dwa_alpha_9",
    "hf": "dwa_alphas.9",
    "shape": [
      11
    ]
  },
  {
    "mixlab": "w149_dwa_alpha_10",
    "hf": "dwa_alphas.10",
    "shape": [
      12
    ]
  },
  {
    "mixlab": "w150_dwa_alpha_11",
    "hf": "dwa_alphas.11",
    "shape": [
      13
    ]
  },
  {
    "mixlab": "w151_dwa_alpha_12",
    "hf": "dwa_alphas.12",
    "shape": [
      14
    ]
  },
  {
    "mixlab": "w152_dwa_alpha_13",
    "hf": "dwa_alphas.13",
    "shape": [
      15
    ]
  },
  {
    "mixlab": "w153_dwa_alpha_14",
    "hf": "dwa_alphas.14",
    "shape": [
      16
    ]
  },
  {
    "mixlab": "w154_dwa_alpha_15",
    "hf": "dwa_alphas.15",
    "shape": [
      17
    ]
  },
  {
    "mixlab": "w155_dwa_alpha_16",
    "hf": "dwa_alphas.16",
    "shape": [
      18
    ]
  },
  {
    "mixlab": "w156_dwa_alpha_17",
    "hf": "dwa_alphas.17",
    "shape": [
      19
    ]
  },
  {
    "mixlab": "w157_dwa_alpha_18",
    "hf": "dwa_alphas.18",
    "shape": [
      20
    ]
  },
  {
    "mixlab": "w158_dwa_alpha_19",
    "hf": "dwa_alphas.19",
    "shape": [
      21
    ]
  },
  {
    "mixlab": "w159_dwa_alpha_20",
    "hf": "dwa_alphas.20",
    "shape": [
      22
    ]
  },
  {
    "mixlab": "w160_dwa_alpha_21",
    "hf": "dwa_alphas.21",
    "shape": [
      23
    ]
  },
  {
    "mixlab": "w161_dwa_alpha_22",
    "hf": "dwa_alphas.22",
    "shape": [
      24
    ]
  },
  {
    "mixlab": "w162_dwa_alpha_23",
    "hf": "dwa_alphas.23",
    "shape": [
      25
    ]
  },
  {
    "mixlab": "w163_head",
    "hf": "lm_head_weight",
    "shape": [
      384,
      16384
    ]
  }
]