{
  "model_family": "nllb",
  "recipe_id": "nllb200-spm8k-directional-v5",
  "mt_standardization": {
    "id": "formosan-mt-standard-v3",
    "sha256": "3c93c1abecfc68c72e3bb1d547fbe0bdebb14bb74e7a5d3c5c519bd18d34bf6f",
    "namespace": "formosan-mt"
  },
  "step": 300000,
  "best_metric": "chrF2",
  "best_value": 35.82202484658809,
  "direction": "zh2f",
  "run_contract_sha256": "cc251cf00ec88f29e0d8407267f5f6d00ec19fe2fc5adfd7ab8f61c20c4ec897",
  "validation": {
    "mean_token_loss": 3.3005945952051094,
    "ppl": 27.12876475945609,
    "by_language": {
      "ami": 2.9977749379532206,
      "bnn": 3.690821646233282,
      "ckv": 2.94499888262739,
      "dru": 4.181627018271752,
      "pwn": 3.4254762849185942,
      "pyu": 3.304404493729452,
      "ssf": 2.497812805786773,
      "sxr": 3.7283250016987934,
      "szy": 3.4226339202524496,
      "tao": 4.027004377501526,
      "tay": 3.8187424449188474,
      "trv": 3.3320100487357895,
      "tsu": 3.1443220605594697,
      "xnb": 2.505188786058678,
      "xsy": 2.4550011498098887
    },
    "generation": {
      "samples": 1920,
      "bleu_tokenize": "13a",
      "global": {
        "BLEU": 9.455191646228812,
        "chrF2": 35.82202484658809,
        "TER": 83.08488862131246,
        "exact_match_rate": 0.0020833333333333333,
        "empty_output_rate": 0.0,
        "character_length_ratio": 0.9482003611684212,
        "signatures": {
          "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
          "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
          "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
        },
        "source_copy_rate": 0.0
      },
      "by_language": {
        "ami": {
          "samples": 128,
          "BLEU": 6.471396361006349,
          "chrF2": 30.301236626781282,
          "TER": 90.9090909090909,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8739165329052969,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "bnn": {
          "samples": 128,
          "BLEU": 5.466768069013215,
          "chrF2": 33.750602012686926,
          "TER": 88.137321549966,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9296519493297257,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "ckv": {
          "samples": 128,
          "BLEU": 15.527114881978997,
          "chrF2": 41.66076326407682,
          "TER": 74.10375369042598,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9610020876826723,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "dru": {
          "samples": 128,
          "BLEU": 2.209520710659227,
          "chrF2": 27.983762152508092,
          "TER": 101.71972191730698,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0066437007874016,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "pwn": {
          "samples": 128,
          "BLEU": 4.6023247601103465,
          "chrF2": 33.737940135980466,
          "TER": 89.06685236768801,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9638833451361617,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "pyu": {
          "samples": 128,
          "BLEU": 9.007496571772867,
          "chrF2": 35.476914949292,
          "TER": 84.61979913916787,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9999442431000837,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "ssf": {
          "samples": 128,
          "BLEU": 18.004249116567237,
          "chrF2": 47.24722590802964,
          "TER": 59.93303571428571,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8978708791208792,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "sxr": {
          "samples": 128,
          "BLEU": 3.610289433997478,
          "chrF2": 38.23203639485925,
          "TER": 91.54639175257732,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.957137966449846,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "szy": {
          "samples": 128,
          "BLEU": 12.018448268563786,
          "chrF2": 38.38498932105344,
          "TER": 79.93158494868872,
          "exact_match_rate": 0.0234375,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0317768627146298,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tao": {
          "samples": 128,
          "BLEU": 9.388220471677311,
          "chrF2": 32.52694722566377,
          "TER": 73.00040600893219,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8613728129205922,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tay": {
          "samples": 128,
          "BLEU": 2.7297561079820287,
          "chrF2": 21.796275475634115,
          "TER": 112.19386439804073,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0032571521632918,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "trv": {
          "samples": 128,
          "BLEU": 8.33659966162617,
          "chrF2": 32.0446575794997,
          "TER": 74.22053231939162,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8481531004512787,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "tsu": {
          "samples": 128,
          "BLEU": 10.036666281229088,
          "chrF2": 35.833768781078454,
          "TER": 81.94852941176471,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9632515066882258,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "xnb": {
          "samples": 128,
          "BLEU": 15.073988584777457,
          "chrF2": 48.23658323417333,
          "TER": 63.860369609856264,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9253807476499307,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        },
        "xsy": {
          "samples": 128,
          "BLEU": 16.437505767524137,
          "chrF2": 43.27827590109571,
          "TER": 68.47792528558196,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9321233674658164,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          },
          "source_copy_rate": 0.0
        }
      }
    },
    "step": 300000,
    "selection": {
      "metric": "chrF2",
      "value": 35.82202484658809,
      "improved": true,
      "best_value_before_eval": 35.57269991348219
    }
  }
}