{
  "architectures": [
    "CognicaPoEForCausalLM"
  ],
  "model_type": "cognica_poe",
  "auto_map": {
    "AutoConfig": "configuration_cognica_poe.CognicaPoEConfig",
    "AutoModelForCausalLM": "modeling_cognica_poe.CognicaPoEForCausalLM",
    "AutoTokenizer": [
      "tokenization_cognica_poe.CognicaPoETokenizer",
      null
    ]
  },
  "hidden_size": 1536,
  "intermediate_size": 6144,
  "num_hidden_layers": 24,
  "num_attention_heads": 12,
  "num_key_value_heads": 12,
  "head_dim": 128,
  "max_position_embeddings": 2048,
  "vocab_size": 32768,
  "padded_vocab_size": 32768,
  "hidden_act": "relu_squared",
  "rms_norm_eps": 1e-06,
  "rope_theta": 100000,
  "tie_word_embeddings": false,
  "window_pattern": "SSSL",
  "use_cache": true,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.45.0",
  "bos_token_id": 32759,
  "eos_token_id": 32759,
  "pad_token_id": null,
  "poe_mode": "flat",
  "poe_every": 6,
  "poe_alpha": 0.0,
  "poe_head_count": 4,
  "poe_note": "Trained with poe_mode=flat, poe_alpha=0.0 (uniform stage average), poe_every=6 -> 4 stages of 6 layers. Each stage's output is a valid predictor through the shared lm_head, enabling stage prefix pruning, WAND-style adaptive depth, speculative decoding, and post-hoc specialist attach at inference. The final-layer path is a standard causal LM; stage paths are additional (not replacements). See README and the companion paper.",
  "training": {
    "total_tokens": 27713863680,
    "num_iterations": 26430,
    "total_batch_size": 1048576,
    "sequence_len": 2048,
    "optimizer": "MuonAdamW",
    "embedding_lr": 0.3,
    "unembedding_lr": 0.008,
    "matrix_lr": 0.02,
    "weight_decay": 0.28,
    "warmdown_ratio": 0.65,
    "final_lr_frac": 0.05,
    "chinchilla_ratio": 20,
    "dataset": "ClimbMix (karpathy/climbmix-400b-shuffle mirror of NVIDIA ClimbMix-400B, 700 shards, ~35GB)"
  },
  "training_step": 26430,
  "training_val_bpb": 0.7209354364389962
}