{
  "tokenizer": "MorpheL-ablation-random-baseline",
  "config": {
    "vocab_size": 32054,
    "algorithm": "random_uniform",
    "random_max_cuts": 2,
    "seed": 42,
    "min_frequency": 2,
    "special_tokens": [
      "<s>",
      "<pad>",
      "</s>",
      "<unk>",
      "<mask>"
    ],
    "dataset": "facebook/xnli/tr",
    "corpus_lines": 800404,
    "mi_index_words": 0
  },
  "metrics": {
    "fertility": 3.053953958435943,
    "tokens_per_char": 0.407745217098712,
    "avg_seq_len": 33.833866666666665,
    "vocab_coverage": 1.0,
    "oov_rate": 0.0,
    "fallback_event_rate": 0.0,
    "char_shatter_rate": 0.07757600568003735,
    "n_sentences": 15000,
    "n_word_types": 26764
  },
  "baselines": {
    "BPE": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "WordPiece": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "Unigram": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    }
  }
}