{
  "tokenizer": "MorpheL-ablation-random-baseline",
  "config": {
    "vocab_size": 32054,
    "algorithm": "random_uniform",
    "random_max_cuts": 2,
    "seed": 42,
    "min_frequency": 2,
    "special_tokens": [
      "<s>",
      "<pad>",
      "</s>",
      "<unk>",
      "<mask>"
    ],
    "dataset": "facebook/xnli/tr",
    "corpus_lines": 800404,
    "mi_index_words": 0
  },
  "metrics": {
    "fertility": 1.7109577387310746,
    "tokens_per_char": 0.22843659207714814,
    "avg_seq_len": 19.8416,
    "vocab_coverage": 1.0,
    "oov_rate": 0.0,
    "fallback_event_rate": 0.10898891497214351,
    "char_shatter_rate": 0.0,
    "n_sentences": 15000,
    "n_word_types": 26764
  },
  "baselines": {
    "BPE": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "WordPiece": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "Unigram": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    }
  }
}