| import json |
| from pathlib import Path |
| from types import SimpleNamespace |
|
|
| from transformers import PreTrainedTokenizerFast |
|
|
|
|
| def load_text_runtime_config(model_dir: str): |
| model_path = Path(model_dir) |
| with open(model_path / "config.json", encoding="utf-8") as f: |
| raw_config = json.load(f) |
|
|
| text_config = dict(raw_config.get("text_config") or raw_config) |
| text_config["model_type"] = text_config.get("model_type", "gemma4_text") |
| text_config["eos_token_id"] = raw_config.get("eos_token_id", text_config.get("eos_token_id")) |
| text_config["image_token_id"] = raw_config.get("image_token_id") |
| text_config["vision_config"] = raw_config.get("vision_config") |
| text_config["vision_soft_tokens_per_image"] = raw_config.get("vision_soft_tokens_per_image") |
| return SimpleNamespace(**text_config) |
|
|
|
|
| def load_tokenizer(model_dir: str): |
| model_path = Path(model_dir) |
| with open(model_path / "tokenizer_config.json", encoding="utf-8") as f: |
| tokenizer_config = json.load(f) |
|
|
| init_kwargs = { |
| "tokenizer_file": str(model_path / "tokenizer.json"), |
| "bos_token": tokenizer_config.get("bos_token"), |
| "eos_token": tokenizer_config.get("eos_token"), |
| "pad_token": tokenizer_config.get("pad_token"), |
| "unk_token": tokenizer_config.get("unk_token"), |
| "mask_token": tokenizer_config.get("mask_token"), |
| "padding_side": tokenizer_config.get("padding_side", "left"), |
| } |
| tokenizer = PreTrainedTokenizerFast(**{k: v for k, v in init_kwargs.items() if v is not None}) |
|
|
| chat_template_path = model_path / "chat_template.jinja" |
| if chat_template_path.exists(): |
| tokenizer.chat_template = chat_template_path.read_text(encoding="utf-8") |
|
|
| return tokenizer |
|
|
