Create core/tokenizer.py

core/tokenizer.py

@@ -0,0 +1,6 @@
+from transformers import AutoTokenizer
+
+def load_tokenizer(model_name: str):
+    print(f"🔤 Chargement du tokenizer pour {model_name}...")
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    return tokenizer