{
  "model_family": "nllb",
  "recipe_id": "nllb200-spm8k-directional-v5",
  "mt_standardization": {
    "id": "formosan-mt-standard-v3",
    "sha256": "3c93c1abecfc68c72e3bb1d547fbe0bdebb14bb74e7a5d3c5c519bd18d34bf6f",
    "namespace": "formosan-mt"
  },
  "step": 250000,
  "best_metric": "chrF2",
  "best_value": 30.439946182229566,
  "direction": "f2en",
  "run_contract_sha256": "5fca5dfcba02f51a5cf9ac8aac90ab005cda4bcaf1b33624dcf9c6c627619574",
  "validation": {
    "mean_token_loss": 3.0167718605405716,
    "ppl": 20.425249590827736,
    "by_language": {
      "ami": 3.057929772634218,
      "bnn": 3.193844542010077,
      "ckv": 3.2313190032380796,
      "dru": 2.8312536303828697,
      "pwn": 2.525507771895442,
      "pyu": 3.2068601836596184,
      "ssf": 2.791594610182075,
      "sxr": 3.3397939340795,
      "szy": 2.7686075285329657,
      "tao": 3.341498091969177,
      "tay": 2.5584949801861248,
      "trv": 2.689101498087646,
      "tsu": 3.1778880092475204,
      "xnb": 3.2671826911801234,
      "xsy": 3.147372350368602
    },
    "generation": {
      "samples": 1920,
      "bleu_tokenize": "13a",
      "global": {
        "BLEU": 10.853407823073317,
        "chrF2": 30.439946182229566,
        "TER": 83.50190729395446,
        "exact_match_rate": 0.019270833333333334,
        "empty_output_rate": 0.0,
        "character_length_ratio": 0.9867910329531052,
        "signatures": {
          "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
          "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
          "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
        },
        "source_copy_rate": 0.0
      },
      "by_language": {
        "ami": {
          "samples": 128,
          "BLEU": 9.818103571217206,
          "chrF2": 29.599159485374614,
          "TER": 85.06769825918762,
          "exact_match_rate": 0.0234375,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9231089984154782,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "bnn": {
          "samples": 128,
          "BLEU": 10.75882451585392,
          "chrF2": 28.4326372468245,
          "TER": 85.13513513513513,
          "exact_match_rate": 0.0390625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.958946143265263,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "ckv": {
          "samples": 128,
          "BLEU": 11.4388963059305,
          "chrF2": 30.53880748361492,
          "TER": 81.19619392840961,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8653027313324333,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "dru": {
          "samples": 128,
          "BLEU": 8.457910317896003,
          "chrF2": 28.22388302816341,
          "TER": 88.71709151692436,
          "exact_match_rate": 0.0234375,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0327089474184654,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "pwn": {
          "samples": 128,
          "BLEU": 11.343137381300266,
          "chrF2": 32.98219396549702,
          "TER": 77.23235156615242,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9546626557799742,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "pyu": {
          "samples": 128,
          "BLEU": 10.730301630122836,
          "chrF2": 28.4299358531323,
          "TER": 79.34990439770554,
          "exact_match_rate": 0.0390625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8756619593998235,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "ssf": {
          "samples": 128,
          "BLEU": 17.499567472616505,
          "chrF2": 37.76181379980078,
          "TER": 78.52380952380953,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0950808847804556,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "sxr": {
          "samples": 128,
          "BLEU": 8.649569123397955,
          "chrF2": 28.49503543851517,
          "TER": 87.22067039106145,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9902259754476503,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "szy": {
          "samples": 128,
          "BLEU": 12.773098535761278,
          "chrF2": 32.35043674601012,
          "TER": 84.52088452088452,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0803105934553523,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tao": {
          "samples": 128,
          "BLEU": 8.875475316305913,
          "chrF2": 29.259433494033964,
          "TER": 88.72146118721462,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0934985175772978,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tay": {
          "samples": 128,
          "BLEU": 9.210064081804864,
          "chrF2": 29.55174819033911,
          "TER": 84.72786352558896,
          "exact_match_rate": 0.0390625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0347908745247147,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "trv": {
          "samples": 128,
          "BLEU": 12.641091402250876,
          "chrF2": 31.671537539919974,
          "TER": 78.76344086021506,
          "exact_match_rate": 0.03125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9652755654667091,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tsu": {
          "samples": 128,
          "BLEU": 11.482446363406487,
          "chrF2": 30.763271436325688,
          "TER": 84.05797101449275,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0576454967913256,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "xnb": {
          "samples": 128,
          "BLEU": 7.863973781490952,
          "chrF2": 30.316148114374013,
          "TER": 81.76737687740133,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9130294992363958,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "xsy": {
          "samples": 128,
          "BLEU": 9.531107665990689,
          "chrF2": 29.792325545563834,
          "TER": 85.66936208445642,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9986703487777436,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        }
      }
    },
    "step": 250000,
    "selection": {
      "metric": "chrF2",
      "value": 30.439946182229566,
      "improved": true,
      "best_value_before_eval": 30.311676923623388
    }
  }
}