training_metadata.json

{
  "experiment_mode": "moe_head",
  "source_model": "swiss-ai/Apertus-v1.1-0.5B",
  "dense_parent": "andreasmartin/apertus-v1.1-swiss-embed-0.4b-bidir",
  "dense_parent_revision": "875c1f99d7b20e261ea43b3586fdde5f8d5648e5",
  "backbone_frozen_during_moe_training": true,
  "training_profile": "quality",
  "data_recipe_signature": "fd8f7d87d50416d4d732c68dfffd2ce3de5a58aac879ccb677185d06186c400e",
  "train_moe_sha256": "287bee32718f74d333f48e94c38e5150ed70921f47e15871940c83ce7e13d22a",
  "dataset_revisions": {
    "wikimedia/wikipedia": "b04c8d1ceb2f5cd4588862100d08de323dccfbaa",
    "ZurichNLP/SwissGov-RSD": "f16aa0536811b8c0cf975fede21eec5996e4b3c1",
    "eljuanina/VotingBooklets-v1": "8db11c674d6993aee084fe65ed3ab6d9ee102fd7",
    "eljuanina/VotingBooklets-Diamond-v1": "476c96e74a72b65122cf6fc01943be125f7f3324"
  },
  "training_triplets": 31483,
  "unique_prompted_texts": 56324,
  "moe_training_max_seq_length": 512,
  "inference_max_seq_length": 1024,
  "expert_names": [
    "de",
    "en",
    "fr",
    "it",
    "rm",
    "gsw",
    "shared"
  ],
  "top_k": 2,
  "router_temperature": 1.0,
  "language_loss_weight": 0.05,
  "balance_loss_weight": 0.01,
  "head_learning_rate": 0.0002,
  "head_batch_size": 1024,
  "head_epochs": 5,
  "matryoshka_dims": [
    1024,
    768,
    512,
    256
  ],
  "matryoshka_weights": [
    1.0,
    0.75,
    0.5,
    0.25
  ],
  "training_history": [
    {
      "epoch": 1,
      "loss": 2.023314436276754,
      "retrieval_loss": 1.9348004142443338,
      "language_loss": 1.7696655670801797,
      "balance_loss": 0.0030751408770205066,
      "lr": 0.00018090169943749468
    },
    {
      "epoch": 2,
      "loss": 1.9000611066818238,
      "retrieval_loss": 1.8191587686538697,
      "language_loss": 1.616682501633962,
      "balance_loss": 0.006820600215966503,
      "lr": 0.00013090169943749468
    },
    {
      "epoch": 3,
      "loss": 1.8358161449432373,
      "retrieval_loss": 1.7584825873374939,
      "language_loss": 1.5452160716056824,
      "balance_loss": 0.007276078267022967,
      "lr": 6.909830056250522e-05
    },
    {
      "epoch": 4,
      "loss": 1.7946220676104228,
      "retrieval_loss": 1.7193971276283264,
      "language_loss": 1.5032105565071106,
      "balance_loss": 0.00644152524570624,
      "lr": 1.909830056250526e-05
    },
    {
      "epoch": 5,
      "loss": 1.7804000655810037,
      "retrieval_loss": 1.7059372186660766,
      "language_loss": 1.4880693872769675,
      "balance_loss": 0.0059369114848474664,
      "lr": 0.0
    }
  ]
}