{
  "tokenizer": "MorpheL-ablation-random-baseline",
  "config": {
    "vocab_size": 32055,
    "algorithm": "random_uniform",
    "random_max_cuts": 2,
    "seed": 42,
    "min_frequency": 2,
    "special_tokens": [
      "<s>",
      "<pad>",
      "</s>",
      "<unk>",
      "<mask>"
    ],
    "dataset": "facebook/xnli/tr",
    "corpus_lines": 800404,
    "mi_index_words": 0
  },
  "metrics": {
    "fertility": 1.4316636496798583,
    "tokens_per_char": 0.19114695689453448,
    "avg_seq_len": 16.921666666666667,
    "vocab_coverage": 1.0,
    "oov_rate": 0.0,
    "fallback_event_rate": 0.130001215323308,
    "char_shatter_rate": 0.0,
    "n_sentences": 15000,
    "n_word_types": 26764
  },
  "baselines": {
    "BPE": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "WordPiece": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    },
    "Unigram": {
      "fertility": null,
      "tokens_per_char": null,
      "avg_seq_len": null,
      "vocab_coverage": null,
      "oov_rate": null
    }
  }
}