| { | |
| "tokenizer": { | |
| "vocab_size": 8000, | |
| "min_frequency": 2, | |
| "special_tokens": [ | |
| "<s>", | |
| "<pad>", | |
| "</s>", | |
| "<unk>", | |
| "<mask>" | |
| ], | |
| "dataset": "facebook/xnli/ru", | |
| "corpus_lines": 800404 | |
| }, | |
| "metrics": { | |
| "tokens_per_char": 0.33150864759813903, | |
| "fertility": 2.2806765738512156, | |
| "avg_seq_len": 27.542666666666666, | |
| "vocab_coverage": 1.0, | |
| "total_sentences": 15000, | |
| "unique_word_types": 32586 | |
| } | |
| } |