model_parameters.json

{
  "experiment_mode": "moe_head",
  "architecture": "dense_backbone_language_moe_head",
  "source_model": "swiss-ai/Apertus-v1.1-0.5B",
  "upstream_model": "swiss-ai/Apertus-8B-2509",
  "apertus_version": "1.1",
  "base_model": "andreasmartin/apertus-v1.1-swiss-embed-0.4b-bidir",
  "base_revision": "875c1f99d7b20e261ea43b3586fdde5f8d5648e5",
  "hidden_size": 1024,
  "embedding_dimension": 1024,
  "matryoshka_dimensions": [
    1024,
    768,
    512,
    256
  ],
  "expert_names": [
    "de",
    "en",
    "fr",
    "it",
    "rm",
    "gsw",
    "shared"
  ],
  "num_experts": 7,
  "experts_active_per_sentence": 2,
  "router_parameters": 7175,
  "expert_parameters_each": 1048576,
  "language_moe_head_parameters": 7347207,
  "active_head_parameters_per_sentence": 2104327,
  "base_parameters_including_old_dense": 439397928,
  "old_dense_projection_parameters": 1048576,
  "final_parameters": 445696559,
  "final_parameters_billions": 0.445696559,
  "active_parameters_per_sentence": 440453679,
  "repository_name": "apertus-v1.1-swiss-embed-0.4b-bidir-langmoe",
  "training_profile": "quality",
  "moe_training_max_seq_length": 512,
  "inference_max_seq_length": 1024,
  "sparse_projection_equivalence_max_abs_diff": 1.430511474609375e-06
}