{
  "tokenizer": "MorpheL",
  "config": {
    "vocab_size": 8097,
    "top_k": 4,
    "temperature": 1.0,
    "mi_threshold": 0.0,
    "min_frequency": 2,
    "special_tokens": [
      "<s>",
      "<pad>",
      "</s>",
      "<unk>",
      "<mask>"
    ],
    "dataset": "facebook/xnli/tr",
    "corpus_lines": 800404,
    "mi_index_words": 97975
  },
  "metrics": {
    "fertility": 1.8496005424179818,
    "tokens_per_char": 0.24694732958592314,
    "avg_seq_len": 21.2776,
    "vocab_coverage": 1.0,
    "oov_rate": 0.0,
    "fallback_event_rate": 0.2401350927803399,
    "char_shatter_rate": 0.0,
    "n_sentences": 15000,
    "n_word_types": 26764
  },
  "baselines": {
    "BPE": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "WordPiece": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "Unigram": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    }
  }
}