{
  "model_type": "mixlab",
  "architectures": [
    "MixlabForCausalLM",
    "MixlabForMaskedLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_mixlab.MixlabConfig",
    "AutoModel": "modeling_mixlab.MixlabModel",
    "AutoModelForCausalLM": "modeling_mixlab.MixlabForCausalLM",
    "AutoModelForMaskedLM": "modeling_mixlab.MixlabForMaskedLM"
  },
  "name": "gptbert_masked_focus_reftok_full",
  "model_dim": 384,
  "hidden_size": 384,
  "vocab_size": 16384,
  "seq_len": 512,
  "max_position_embeddings": 512,
  "mlp_mult": 3.3333333,
  "norm_type": "layernorm",
  "norm_eps": 1e-7,
  "norm_affine": false,
  "norm_placement": "pre",
  "ffn_internal_norm": true,
  "mlm_head": "bert",
  "layer_aggregation": "dwa",
  "hidden_dropout": 0.1,
  "pad_token_id": 3,
  "eos_token_id": 2,
  "bos_token_id": 1,
  "unk_token_id": 0,
  "blocks": [
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "causal",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    }
  ],
  "masked_blocks": [
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    },
    {
      "attention_mask": "bidirectional",
      "attn_bias": true,
      "attn_post_norm": "before_outproj",
      "attn_value_gate": true,
      "ffn_activation": "geglu",
      "heads": 6,
      "relative_attention": "deberta_p2c_c2p",
      "relative_attention_embedding_norm": "layernorm",
      "relative_attention_parameterization": "shared_qk_reuse",
      "relative_attention_window": 32,
      "type": "plain"
    }
  ],
  "mixlab": {
    "format": "mixlab_hf_export_v1",
    "requires_trust": "trust_remote_code=True loads repository-provided Python modeling code",
    "source": "mixlab",
    "supported_blocks": [
      "plain",
      "plain.attn_bias",
      "plain.attn_value_gate",
      "plain.attn_post_norm",
      "plain.ffn_activation=geglu",
      "plain.ffn_activation=swiglu",
      "plain.qk_norm",
      "plain.xsa",
      "plain.sparse_attn_gate",
      "plain.relative_attention=deberta_p2c_c2p",
      "plain.relative_attention_parameterization=shared_qk_reuse",
      "plain.relative_attention_embedding_norm=layernorm",
      "layer_aggregation=dwa",
      "mlm_head=bert",
      "swiglu",
      "geglu",
      "mlp",
      "moe"
    ],
    "unsupported_fails": true,
    "weight_map": "weight_map.json"
  }
}