{
  "architectures": [
    "CognicaPoEForCausalLM"
  ],
  "model_type": "cognica_poe",
  "auto_map": {
    "AutoConfig": "configuration_cognica_poe.CognicaPoEConfig",
    "AutoModelForCausalLM": "modeling_cognica_poe.CognicaPoEForCausalLM",
    "AutoTokenizer": [
      "tokenization_cognica_poe.CognicaPoETokenizer",
      null
    ]
  },
  "base_model_name_or_path": "cognica/Cognica-PoE-v1.0-1.3B-base",
  "stage_depth": 1,
  "new_layers": 4,
  "frozen_layers": 24,
  "dual_head": true,
  "hidden_size": 1536,
  "intermediate_size": 6144,
  "num_hidden_layers": 28,
  "num_attention_heads": 12,
  "num_key_value_heads": 12,
  "head_dim": 128,
  "max_position_embeddings": 2048,
  "vocab_size": 32768,
  "padded_vocab_size": 32768,
  "hidden_act": "relu_squared",
  "rms_norm_eps": 1e-06,
  "rope_theta": 100000,
  "tie_word_embeddings": false,
  "window_pattern": "SSSL",
  "use_cache": true,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.45.0",
  "bos_token_id": 32759,
  "eos_token_id": 32759,
  "pad_token_id": null,
  "poe_mode": "flat",
  "poe_every": 6,
  "poe_alpha": 0.0,
  "poe_head_count": 4,
  "stage_training": {
    "stage_type": "dual_head_sft_math",
    "parent_model": "cognica/Cognica-PoE-v1.0-1.3B-base",
    "parent_step": 26430,
    "datasets": [
      "openai/gsm8k (main/train x20 epochs)",
      "TIGER-Lab/MathInstruct (x4 epochs)"
    ],
    "total_convs_available": 1197616,
    "train_convs": 1197104,
    "val_convs": 512,
    "case_augment": true,
    "train_convs_after_case_aug": 1198173,
    "num_iterations": 2267,
    "shipped_step": 2200,
    "total_batch_size": 131072,
    "sequence_len": 2048,
    "init_lr_frac": 0.2,
    "warmup_ratio": 0.05,
    "warmdown_ratio": 0.9,
    "adamw_scaling": 0.707107,
    "matrix_lr": 0.002,
    "lm_head_stage_lr": 0.0001,
    "lm_head_stage_weight_decay": 0.1,
    "trainable_params": 163577912,
    "delta_tensors": 28,
    "delta_params": 213909560,
    "delta_size_mb": 428.0,
    "best_val_bpb": 2.4118,
    "best_val_bpb_step": 2200,
    "note": "Math-domain specialist (Paper Section 6.5 extended to 4-layer specialist). Base lm_head frozen; additive lm_head_stage (zero-init) learns math-token biases while 4 new transformer layers (layers 24-27) specialize on step-by-step math reasoning. Trained on GSM8K main/train (x20) + MathInstruct (x4). Total ~297M trainable tokens (tok/param 1.82). Capacity-constrained at 1.3B base: arithmetic answers often wrong but reasoning format (chain-of-thought) is learned."
  },
  "poe_note": "This stage is a sibling specialist to Cognica-PoE-v1.0-1.3B-stage-chat, trained directly on the base (not on top of chat). Multi-stage composition is supported by the cascade loader: ancestor stages' lm_head_stage folds into lm_head_base so logits = lm_head_base(x) + Sigma_k lm_head_stage_k(x)."
}