Update .env.example to include TOKEN for gated Hugging Face repos and modify app.py to support TOKEN or HF_TOKEN for loading tokenizers.

.env.example

@@ -1,2 +1,3 @@
-HF_TOKEN=your_huggingface_token_here
-
+# For gated Hugging Face repos (e.g. some LLaMA, Bielik). Use one of:
+TOKEN=your_huggingface_token_here
+# HF_TOKEN=your_huggingface_token_here

app.py

@@ -3,10 +3,14 @@ Tokenizer Playground — inspect how text is tokenized, view config and chat tem
 Runs locally and on Hugging Face Spaces.
 """
 import html
+import os
 import random
 import gradio as gr
 from transformers import AutoTokenizer
 
+# For gated repos: set TOKEN or HF_TOKEN (e.g. in .env or Hugging Face Space secrets)
+HF_TOKEN = os.getenv("TOKEN") or os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN") or None
+
 
 def _token_color(index: int, total: int) -> str:
     """Distinct hue per token, pastel background (HSL)."""
@@ -99,11 +103,16 @@ def get_tokenizer(model_id: str, use_fast: bool = True):
     key = (model_id or DEFAULT_MODEL, use_fast)
     if key not in _tokenizer_cache:
         try:
-            _tokenizer_cache[key] = AutoTokenizer.from_pretrained(
-                model_id or DEFAULT_MODEL,
+            kwargs = dict(
                 trust_remote_code=True,
                 use_fast=use_fast,
             )
+            if HF_TOKEN:
+                kwargs["token"] = HF_TOKEN
+            _tokenizer_cache[key] = AutoTokenizer.from_pretrained(
+                model_id or DEFAULT_MODEL,
+                **kwargs,
+            )
         except Exception as e:
             raise RuntimeError(f"Failed to load tokenizer for '{model_id}': {e}") from e
     return _tokenizer_cache[key]