{
  "model_family": "nllb",
  "recipe_id": "nllb200-spm8k-directional-v5",
  "mt_standardization": {
    "id": "formosan-mt-standard-v3",
    "sha256": "3c93c1abecfc68c72e3bb1d547fbe0bdebb14bb74e7a5d3c5c519bd18d34bf6f",
    "namespace": "formosan-mt"
  },
  "step": 180000,
  "best_metric": "chrF2",
  "best_value": 10.495374236654708,
  "direction": "f2zh",
  "run_contract_sha256": "111b9dc57ffdca3e935c42c1238e0e736d280e5757f93f33c52fcbf12aa6bd80",
  "validation": {
    "mean_token_loss": 3.3944601443496327,
    "ppl": 29.798562209666933,
    "by_language": {
      "ami": 3.3714552120973833,
      "bnn": 3.14808146180145,
      "ckv": 2.934123443312717,
      "dru": 3.4917125453661906,
      "pwn": 3.53915243110911,
      "pyu": 3.4347382228197794,
      "ssf": 2.8487141390241217,
      "sxr": 3.814022432376603,
      "szy": 3.8078580233271935,
      "tao": 3.8635977302701963,
      "tay": 3.6466925498905796,
      "trv": 3.299698793430247,
      "tsu": 3.372065121409949,
      "xnb": 2.672724130618039,
      "xsy": 3.536146654313843
    },
    "generation": {
      "samples": 1920,
      "bleu_tokenize": "zh",
      "global": {
        "BLEU": 9.553552590441944,
        "chrF2": 10.495374236654708,
        "TER": 86.49963979828705,
        "exact_match_rate": 0.003125,
        "empty_output_rate": 0.0,
        "character_length_ratio": 0.902794523893103,
        "signatures": {
          "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
          "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
          "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
        },
        "source_copy_rate": 0.0
      },
      "by_language": {
        "ami": {
          "samples": 128,
          "BLEU": 6.739395879666625,
          "chrF2": 8.550576819970608,
          "TER": 85.77801102072947,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8417373939091363,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "bnn": {
          "samples": 128,
          "BLEU": 9.03100783225077,
          "chrF2": 9.673533144677636,
          "TER": 92.53121452894439,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9612031386224935,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "ckv": {
          "samples": 128,
          "BLEU": 14.910290211179818,
          "chrF2": 14.488005580655953,
          "TER": 80.7921866521975,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9030239833159541,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "dru": {
          "samples": 128,
          "BLEU": 7.58083053823673,
          "chrF2": 8.445640978783661,
          "TER": 89.8639455782313,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.829225352112676,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "pwn": {
          "samples": 128,
          "BLEU": 8.917907809504971,
          "chrF2": 9.594233572699789,
          "TER": 94.77611940298507,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9406930253220791,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "pyu": {
          "samples": 128,
          "BLEU": 9.99478744219959,
          "chrF2": 11.050469004899806,
          "TER": 86.71958807122935,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8844772545889864,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "ssf": {
          "samples": 128,
          "BLEU": 11.407580609585827,
          "chrF2": 12.468157911213968,
          "TER": 78.45888724766125,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9317748091603053,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "sxr": {
          "samples": 128,
          "BLEU": 8.60679264907891,
          "chrF2": 9.442801071016975,
          "TER": 87.63796909492274,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8389339513325609,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "szy": {
          "samples": 128,
          "BLEU": 9.552183839137873,
          "chrF2": 11.692885022789454,
          "TER": 81.0523886452804,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.7304252998909487,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tao": {
          "samples": 128,
          "BLEU": 7.863049398718416,
          "chrF2": 8.95577309108151,
          "TER": 90.89008363201911,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9032258064516129,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tay": {
          "samples": 128,
          "BLEU": 6.656066655635352,
          "chrF2": 8.015456306038828,
          "TER": 91.69574319609211,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9085221508285425,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "trv": {
          "samples": 128,
          "BLEU": 8.627011305713472,
          "chrF2": 10.684555566264951,
          "TER": 88.67114093959731,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9550190597204574,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tsu": {
          "samples": 128,
          "BLEU": 8.825403224046577,
          "chrF2": 9.846751617905833,
          "TER": 89.0005431830527,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.1370466321243524,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "xnb": {
          "samples": 128,
          "BLEU": 17.2367080996832,
          "chrF2": 16.780842104118364,
          "TER": 72.3022491786707,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9598853868194842,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "xsy": {
          "samples": 128,
          "BLEU": 7.071599613458036,
          "chrF2": 8.936112405321474,
          "TER": 84.60820895522389,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8568597225280775,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          },
          "source_copy_rate": 0.0
        }
      }
    },
    "step": 180000,
    "selection": {
      "metric": "chrF2",
      "value": 10.495374236654708,
      "improved": true,
      "best_value_before_eval": 10.18533043198548
    }
  }
}