diff --git a/conversion/__init__.py b/conversion/__init__.py index 5aad203..feec11c 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -262,6 +262,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Gemma4UnifiedForConditionalGeneration": "gemma", "Glm4vForConditionalGeneration": "qwen3vl", "Glm4vMoeForConditionalGeneration": "qwen3vl", + "Glm5vForConditionalGeneration": "kimivl", "GlmOcrForConditionalGeneration": "qwen3vl", "GlmasrModel": "ultravox", "Granite4VisionForConditionalGeneration": "granite", diff --git a/conversion/kimivl.py b/conversion/kimivl.py index 63b8a07..d9aa212 100644 --- a/conversion/kimivl.py +++ b/conversion/kimivl.py @@ -152,3 +152,30 @@ class KimiK25Model(MmprojModel): name = name.replace(".proj.2.", ".proj.linear_2.") yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Glm5vForConditionalGeneration") +class Glm5vModel(KimiK25Model): + """GLM-5.2-Vision: MoonViT3d encoder reused verbatim, projector maps to 6144""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # in_patch_limit sits under media_proc_cfg here, not at the top level + media_proc_cfg = self.preprocessor_config.get("media_proc_cfg") + if media_proc_cfg is not None: + self.preprocessor_config = {**media_proc_cfg, **self.preprocessor_config} + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + # encoder graph is shared with kimik25; the type differs only so that + # mtmd wraps the embeddings in GLM's image delimiters + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLM5V) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # projector is flat here (mm_projector.linear_N) rather than nested under proj + if name.startswith("mm_projector.linear_"): + name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_") + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 1bda945..d9b58ed 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4539,6 +4539,7 @@ class VisionProjectorType: KIMIVL = "kimivl" PADDLEOCR = "paddleocr" KIMIK25 = "kimik25" + GLM5V = "glm5v" LIGHTONOCR = "lightonocr" COGVLM = "cogvlm" JANUS_PRO = "janus_pro" diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 5b41368..8897cd8 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -364,6 +364,7 @@ enum projector_type { PROJECTOR_TYPE_YOUTUVL, PROJECTOR_TYPE_YASA2, PROJECTOR_TYPE_KIMIK25, + PROJECTOR_TYPE_GLM5V, PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_HUNYUANVL, PROJECTOR_TYPE_EXAONE4_5, @@ -418,6 +419,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, { PROJECTOR_TYPE_YASA2, "yasa2"}, { PROJECTOR_TYPE_KIMIK25, "kimik25"}, + { PROJECTOR_TYPE_GLM5V, "glm5v"}, { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index d2226b3..ff3a73f 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -957,6 +957,7 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const builder = std::make_unique(ctx, img); } break; case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_GLM5V: { builder = std::make_unique(ctx, img); } break; @@ -1399,6 +1400,7 @@ struct clip_model_loader { hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_GLM5V: { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.rope_theta = 10000.0f; @@ -2307,6 +2309,7 @@ struct clip_model_loader { case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_GLM5V: { model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); @@ -3381,6 +3384,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_GLM5V: { // dynamic size int out_patch_size = params.patch_size * ctx->model.hparams.n_merge; @@ -4040,6 +4044,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_GLM5V: case PROJECTOR_TYPE_LIGHTONOCR: { // set the 2D positions @@ -4588,6 +4593,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_GLM5V: case PROJECTOR_TYPE_YASA2: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 724538b..bd44fd4 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -560,6 +560,13 @@ struct mtmd_context { img_end = "<|media_end|>"; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_GLM5V: + { + // same encoder as kimik25, but GLM's vocab has no <|media_*|> + img_beg = "<|begin_of_image|>"; + img_end = "<|end_of_image|>"; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_LIGHTONOCR: { // <|im_start|> ... (image embeddings) ... <|im_end|>