diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..b025259edec1d6ced28917e49a7e9f25b1c130de 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,35 +1,13 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text +*.axmodel filter=lfs diff=lfs merge=lfs -text *.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.JPG filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +bin/axllm filter=lfs diff=lfs merge=lfs -text +*.gguf filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text *.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text +minicpm_v46_tokenizer.txt filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..fc3a738b860cf22e553825428e3b82a83fbf24cf --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +vision_cache/ +*.tmp diff --git a/README.md b/README.md index 555b4aed7fba6658868d69c9844d1dcb224a0d31..41d64e8aaacc101386da41d0bb2f5b3120090332 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,403 @@ --- -license: bsd-3-clause +library_name: transformers +license: apache-2.0 +base_model: +- openbmb/MiniCPM-V-4.6 +pipeline_tag: image-text-to-text +tags: +- minicpm-v +- vlm +- axera +- AX650 +language: +- zh +- en --- + +# MiniCPM-V-4.6 on AXERA NPU + +Ready-to-run AX650 deployment package for `openbmb/MiniCPM-V-4.6`. + +- Runtime: `axllm serve` and `axllm run` +- Target: `AX650 / AX650N aarch64` +- Text backend: Qwen3.5 dense LLM, `bf16`, non-GPTQ weights +- Vision backend: fixed-shape `448x448` MiniCPM-V-4.6 vision encoder +- Context layout: `prefill_len=128`, `kv_cache_len=2047`, `prefill_max_token_num=1280` +- Validated capabilities in this release: + - text-only chat + - long prompt multi-chunk prefill + - single-image understanding + - video understanding +- Included board binary: + - `bin/axllm` for `AX650/AX650N aarch64` + +This package has been validated for text chat, single-image understanding, and video understanding through the OpenAI-compatible `axllm serve` API. + +## Supported Platform + +- [x] AX650 / AX650N + +## Performance and Footprint + +Measurements below are from the preserved AX650 runtime log in [Inference Log](#inference-log), plus a repeated text-only TTFT recheck on `2026-06-05`. `TTFT` means time to first token. +The text-only smoke prompt is intentionally kept within one `128`-token prefill group. Its TTFT is the 5-run average for the prompt `1+1等于几？只输出数字。`; the response is one token, so decode throughput is not reported for that row. + +| Scenario | Input tokens | Prefill chunks | TTFT | Decode | +|---|---:|---:|---:|---:| +| Text-only smoke prompt | `22` | `1 x 128` | `240.45 ms avg` (`239.20-242.75 ms`) | `n/a` | +| Video red-panda prompt | `1278` | `10 x 128` | `1795.28 ms` | `18.79 token/s` | +| Image prompt | `358` | `3 x 128` | `518.23 ms` | `18.64 token/s` | + +### Flash Footprint + +| Item | Value | +|---|---:| +| Text + post + vision axmodels | `1460.19 MiB` | +| Package flash footprint, excluding `vision_cache/` | `1999.34 MiB` | + +### Startup Runtime Footprint + +Measured on AX650 on `2026-06-05` by reading `/proc/ax_proc/mem_cmm_info` before startup and after the `/health` endpoint became ready. The measurement kept the existing GPTQ service on port `8000` running and records only the incremental CMM used by this extra service. + +| Item | Value | +|---|---:| +| Baseline CMM before startup | `1823 MB` | +| CMM after `/health` ready | `3388 MB` | +| Incremental runtime CMM | `1565 MB` | +| Startup log `remain_cmm` after post axmodel init | `5388 MB` | + +The preserved standalone inference log below records `remain_cmm(6933 MB)` for an earlier single-service startup; use the incremental CMM row above for the board-side footprint measured during this README refresh. + +The runtime log also records the video sampling decision: + +```text +MiniCPM-V-4.6 video frames selected: 19/59 (configured_cap=0, tail_tokens=1278, max_tail=1280, precompute_len=0) +``` + +## Repository Layout + +```text +. +├── README.md +├── bin/ +│ ├── axllm +│ └── axllm.version.json +├── assets/ +│ └── smoke_image.png +├── minicpm_v46_tokenizer/ +├── python/ +│ ├── infer_axmodel.py +│ ├── infer_torch.py +│ └── minicpm_v46_tokenizer/ +├── minicpmv4_6_vision_448.axmodel +├── qwen3_5_text_p128_l0_together.axmodel +├── ... +├── qwen3_5_text_p128_l23_together.axmodel +├── qwen3_5_text_post.axmodel +├── model.embed_tokens.weight.bfloat16.bin +├── config.json +├── post_config.json +└── minicpm_v46_tokenizer.txt +``` + +## Install `axllm` + +Option 1: use the validated binary included in this repository: + +```bash +chmod +x ./bin/axllm +``` + +Option 2: build or install the latest `ax-llm` yourself: + +- Repo: `https://github.com/AXERA-TECH/ax-llm` +- The included binary metadata is recorded in `bin/axllm.version.json`. + +## Run on AX650 + +From the repository root on the board: + +```bash +chmod +x ./bin/axllm +./bin/axllm serve . --port 18080 +``` + +Expected model id: + +```text +AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047 +``` + +### Text Request + +```bash +curl http://127.0.0.1:18080/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "1+1等于几？只输出数字。"} + ] + } + ], + "max_tokens": 32 + }' +``` + +### Image Request + +```bash +python - <<'PY' +import base64 +import json +from pathlib import Path +from urllib.request import Request, urlopen + +img = Path("assets/smoke_image.png").read_bytes() +payload = { + "model": "AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Please briefly describe this image."}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + base64.b64encode(img).decode() + }, + }, + ], + } + ], + "max_tokens": 64, +} +req = Request( + "http://127.0.0.1:18080/v1/chat/completions", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, +) +with urlopen(req, timeout=60) as resp: + print(resp.read().decode()) +PY +``` + +### Video Request + +`axllm serve` supports a frames directory or a raw video file: + +```bash +curl http://127.0.0.1:18080/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047", + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "video:/path/to/frames_dir"}}, + {"type": "text", "text": "Describe this video."} + ] + } + ], + "max_tokens": 256 + }' +``` + +For a raw video file, use `video:/path/to/video.mp4` or `video:/path/to/video.mp4:2` to request a sampling FPS of `2`. + +### Interactive CLI + +```bash +./bin/axllm run . +``` + +After each prompt: + +- press `Enter` directly for text-only chat +- input an image path for single-image chat +- input `video:` for video chat + +## Python Reference Scripts + +### Official PyTorch Reference + +`python/infer_torch.py` is a developer-side reference script. It requires the full upstream HuggingFace model with weights and is intended for x86/GPU validation. + +```bash +cd python +python infer_torch.py \ + --model-path /data/tmp/yongqiang/nfs/auto_model_deployment/Minicpm-V-4.6-hf-original/MiniCPM-V-4.6 \ + --prompt "1+1等于几？请直接回答。" +``` + +### Python `.axmodel` Debug Runner + +`python/infer_axmodel.py` is a text-only per-layer debug runner for the compiled LLM package: + +```bash +cd python +python3 infer_axmodel.py \ + --hf-model ./minicpm_v46_tokenizer \ + --axmodel-dir .. \ + --mode generate \ + --prompt "1+1等于几？请直接回答。" \ + --prompt-mode prefill \ + --max-new-tokens 16 \ + --kv-cache-len 2047 +``` + +## Notes + +1. `.axmodel` files are board-only. Do not run them on x86. +2. This package was compiled with `--prefill_len 128 --kv_cache_len 2047`. +3. The AX650 build keeps `FLOAT_MATMUL_USE_CONV_EU=1` enabled to reduce TTFT. +4. `bin/axllm.version.json` records the exact runtime binary commit and verification metadata. +5. The packaged vision encoder is fixed at `448x448`; arbitrary image/video sizes are resized by the runtime preprocessor. +6. Video requests are isolated to the current user turn and drop KV cache after the response to avoid stale multimodal history reuse. + + +## Inference Log + +```sh +root@ax650 ~/yongqiang/auto_model_deployment/MiniCPM-V-4.6 # ./bin/axllm serve . +14:59:39.575 INF sidecar_string_list_value:251 | loaded layer_types[24] from ./minicpm_v46_tokenizer/config.json +14:59:39.577 INF Init:2239 | LLM init start +14:59:39.577 INF Init:2248 | mixed attention enabled: full_attention_interval=4 ref_full_layer_idx=3 +14:59:39.577 INF Init:2270 | attention config: layers=24 sliding=0 full=6 linear=18 sliding_window=0 ref_full_layer_idx=3 +tokenizer_type = 3 +huggingface tokenizer mode = gpt2_byte_bpe + 44% | ############## | 12 / 27 [4.73s<10.63s, 2.54 count/s] init 10 axmodel ok,remain_cmm(7587 MB 96% | ############################## | 26 / 27 [6.15s<6.39s, 4.23 count/s] init post axmodel ok,remain_cmm(6933 MB) +14:59:45.729 INF Init:2420 | max_token_len : 2047 +14:59:45.729 INF Init:2423 | kv_cache_size : 512, kv_cache_num: 2047 +14:59:45.729 INF init_groups_from_model:1622 | prefill_token_num : 128 +14:59:45.729 INF init_groups_from_model:1870 | decode grp: 0, gid: 0, max_token_len : 2047 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 0, gid: 1, history_cap: 0, total_cap: 128, symbolic_cap: 1 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 1, gid: 2, history_cap: 128, total_cap: 256, symbolic_cap: 128 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 2, gid: 3, history_cap: 256, total_cap: 384, symbolic_cap: 256 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 3, gid: 4, history_cap: 384, total_cap: 512, symbolic_cap: 384 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 4, gid: 5, history_cap: 512, total_cap: 640, symbolic_cap: 512 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 5, gid: 6, history_cap: 640, total_cap: 768, symbolic_cap: 640 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 6, gid: 7, history_cap: 768, total_cap: 896, symbolic_cap: 768 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 7, gid: 8, history_cap: 896, total_cap: 1024, symbolic_cap: 896 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 8, gid: 9, history_cap: 1024, total_cap: 1152, symbolic_cap: 1024 +14:59:45.729 INF init_groups_from_model:1874 | prefill grp: 9, gid: 10, history_cap: 1152, total_cap: 1280, symbolic_cap: 1152 +14:59:45.729 INF init_groups_from_model:1881 | prefill_max_token_num : 1280 +14:59:45.729 INF init_layer_groups:1026 | layer 0 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 1 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 2 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 4 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 5 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 6 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 8 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 9 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 10 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 12 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 13 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.729 INF init_layer_groups:1026 | layer 14 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF init_layer_groups:1026 | layer 16 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF init_layer_groups:1026 | layer 17 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF init_layer_groups:1026 | layer 18 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF init_layer_groups:1026 | layer 20 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF init_layer_groups:1026 | layer 21 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF init_layer_groups:1026 | layer 22 prefill groups=2 ref=10, reuse gid 2 for later prefill chunks +14:59:45.730 INF Init:27 | LLaMaEmbedSelector use mmap +100% | ################################ | 27 / 27 [6.16s<6.16s, 4.39 count/s] embed_selector init ok +14:59:47.002 INF Init:1507 | MiniCPM-V-4.6 token ids: image_pad=248056 video_pad=248057 +14:59:47.002 INF Init:1514 | VisionModule init ok: type=MiniCPMV46VL, tokens_per_block=64, embed_size=1024, out_dtype=fp32 +14:59:47.002 WRN Init:1523 | Vision preprocess backend: SimpleCV (OpenCV not found at build time; minor differences vs OpenCV are possible) +14:59:47.004 INF load_config:444 | load config: +14:59:47.004 INF load_config:444 | { +14:59:47.004 INF load_config:444 | "enable_repetition_penalty": false, +14:59:47.004 INF load_config:444 | "enable_temperature": false, +14:59:47.004 INF load_config:444 | "enable_top_k_sampling": false, +14:59:47.004 INF load_config:444 | "enable_top_p_sampling": false, +14:59:47.004 INF load_config:444 | "penalty_window": 20, +14:59:47.004 INF load_config:444 | "repetition_penalty": 1.2, +14:59:47.004 INF load_config:444 | "temperature": 0.9, +14:59:47.004 INF load_config:444 | "top_k": 10, +14:59:47.004 INF load_config:444 | "top_p": 0.8 +14:59:47.004 INF load_config:444 | } +14:59:47.004 INF Init:2532 | LLM init ok +Starting server on port 8000 with model 'AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047'... +API URLs: + GET http://127.0.0.1:8000/health + GET http://127.0.0.1:8000/v1/models + POST http://127.0.0.1:8000/v1/chat/completions + GET http://10.168.232.217:8000/health + GET http://10.168.232.217:8000/v1/models + POST http://10.168.232.217:8000/v1/chat/completions + GET http://172.17.0.1:8000/health + GET http://172.17.0.1:8000/v1/models + POST http://172.17.0.1:8000/v1/chat/completions +Aliases: + GET http://127.0.0.1:8000/models + POST http://127.0.0.1:8000/chat/completions + GET http://10.168.232.217:8000/models + POST http://10.168.232.217:8000/chat/completions + GET http://172.17.0.1:8000/models + POST http://172.17.0.1:8000/chat/completions +OpenAI API Server starting on http://0.0.0.0:8000 +Max concurrency: 1 +Models: AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047 +15:00:35.571 INF operator():2136 | OpenAI chat request: model=AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047 stream=1 max_tokens=2048 has_temperature=0 temperature=1.0000 has_top_p=0 top_p=1.0000 messages=1 stop=0 +15:00:35.572 INF SetKVCache:3134 | decode_grpid:0 prefill_grpid:1 history_cap:0 total_cap:128 symbolic_cap:1 precompute_len:0 input_num_token:16 prefer_symbolic_group:0 +15:00:35.572 INF SetKVCache:3156 | current prefill_max_token_num:1280 +15:00:35.588 INF SetKVCache:3172 | first run +15:00:35.602 INF Run:3376 | input token num : 16, prefill_split_num : 1 +15:00:35.602 INF Run:3387 | MiniCPM-V-4.6 short text prefill uses decode replay: input_tokens=16 prefill_tokens=128 precompute_len=0 +15:00:35.602 INF Run:3471 | prefill chunk p=0 history_len=0 grpid=1 kv_cache_num=0 input_tokens=16 +15:00:36.091 INF Run:3886 | ttft: 488.83 ms +I'm a model from the MiniCPM series, developed by Modelbest and OpenBMB. For more details, you can visit https://github.com/OpenBMB/ + +15:00:37.997 NTC Run:4298 | hit eos,decode avg 17.84 token/s +15:00:37.997 INF GetKVCache:3087 | precompute_len:51, remaining:1229 (tracked) +15:01:11.728 INF operator():2136 | OpenAI chat request: model=AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047 stream=1 max_tokens=2048 has_temperature=0 temperature=1.0000 has_top_p=0 top_p=1.0000 messages=3 stop=0 +15:01:11.941 WRN Run:4330 | video history is isolated to current user turn: old_history=3 new_history=1 old_media_inputs=1 new_media_inputs=1 +15:01:12.096 INF extract_video_frames_ffmpeg:386 | Extracting raw video container to frames: /tmp/axllm_media/media_863804170273309_0.mp4 -> /tmp/axllm_video_frames/video_863804326631725_0 fps=1.9907760708716282 +15:01:13.679 INF collect_video_frame_paths:466 | Video fps sampling: path=/tmp/axllm_media/media_863804170273309_0.mp4 fps=2 duration=30.139s target_frames=60 selected=59 +15:01:13.687 INF Prepare:2594 | MiniCPM-V-4.6 video frames selected: 19/59 (configured_cap=0, tail_tokens=1278, max_tail=1280, precompute_len=0) +15:01:24.499 INF SetKVCache:3134 | decode_grpid:0 prefill_grpid:1 history_cap:0 total_cap:128 symbolic_cap:1 precompute_len:0 input_num_token:1278 prefer_symbolic_group:0 +15:01:24.499 INF SetKVCache:3156 | current prefill_max_token_num:1280 +15:01:24.516 INF SetKVCache:3172 | first run +15:01:24.531 INF Run:3376 | input token num : 1278, prefill_split_num : 10 +15:01:24.531 INF Run:3471 | prefill chunk p=0 history_len=0 grpid=1 kv_cache_num=0 input_tokens=128 +15:01:24.689 INF Run:3471 | prefill chunk p=1 history_len=128 grpid=3 kv_cache_num=256 input_tokens=128 +15:01:24.865 INF Run:3471 | prefill chunk p=2 history_len=256 grpid=4 kv_cache_num=384 input_tokens=128 +15:01:25.041 INF Run:3471 | prefill chunk p=3 history_len=384 grpid=5 kv_cache_num=512 input_tokens=128 +15:01:25.219 INF Run:3471 | prefill chunk p=4 history_len=512 grpid=6 kv_cache_num=640 input_tokens=128 +15:01:25.398 INF Run:3471 | prefill chunk p=5 history_len=640 grpid=7 kv_cache_num=768 input_tokens=128 +15:01:25.578 INF Run:3471 | prefill chunk p=6 history_len=768 grpid=8 kv_cache_num=896 input_tokens=128 +15:01:25.759 INF Run:3471 | prefill chunk p=7 history_len=896 grpid=9 kv_cache_num=1024 input_tokens=128 +15:01:25.943 INF Run:3471 | prefill chunk p=8 history_len=1024 grpid=10 kv_cache_num=1152 input_tokens=128 +15:01:26.128 INF Run:3471 | prefill chunk p=9 history_len=1152 grpid=10 kv_cache_num=1152 input_tokens=126 +15:01:26.327 INF Run:3886 | ttft: 1795.28 ms +好的，我很乐意为您描述这个视频的内容。 + +这张图片展示了一个活泼可爱的红熊猫，它正积极地参与着某种游戏或探索活动。从画面来看，这只红熊猫有着标志性的红棕色和黑色相间的毛色，面部和胸部有白色的标记，非常醒目。它正用前爪紧紧抓住一根由竹子或类似材料搭建的木制梯子或支架，身体微微前倾，似乎正在用力或试图攀爬。它的姿态充满了动感和专注，显示出它对这个环境非常感兴趣。 + +在画面的下方，还有一只较小的红熊猫，它正抬头向上看，似乎是在观察上方那只正在攀爬的同伴，或者是在等待自己的机会。这两只红熊猫被安置在一个绿色的草地上，背景中可以看到一些绿色的围栏或墙壁，暗示这可能是在一个动物园或野生动物保护区的围栏内。 + +整个场景充满了自然和活力，红熊猫们通过互动和探索，展现了它们的天性和对环境的适应。这个视频捕捉到了它们 playful 和 curious 的瞬间，让人感受到它们活泼可爱的性格。 + +15:01:37.610 NTC Run:4298 | hit eos,decode avg 18.79 token/s +15:01:37.610 WRN Run:4631 | drop KV cache after isolated video-history request +^@15:02:51.406 INF operator():2136 | OpenAI chat request: model=AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047 stream=1 max_tokens=2048 has_temperature=0 temperature=1.0000 has_top_p=0 top_p=1.0000 messages=4 stop=0 +15:02:51.622 INF EncodeForContent:2080 | MiniCPM-V-4.6 pixel_values bytes=602112 min=2 max=250 (w=448 h=448 ps=14) +15:02:52.151 INF EncodeForContent:2124 | vision cache store: /tmp/axllm_media/media_863903737595189_1.png +15:02:52.225 INF SetKVCache:3134 | decode_grpid:0 prefill_grpid:1 history_cap:0 total_cap:128 symbolic_cap:1 precompute_len:0 input_num_token:358 prefer_symbolic_group:0 +15:02:52.225 INF SetKVCache:3156 | current prefill_max_token_num:1280 +15:02:52.242 INF SetKVCache:3172 | first run +15:02:52.258 INF Run:3376 | input token num : 358, prefill_split_num : 3 +15:02:52.258 INF Run:3471 | prefill chunk p=0 history_len=0 grpid=1 kv_cache_num=0 input_tokens=128 +15:02:52.414 INF Run:3471 | prefill chunk p=1 history_len=128 grpid=3 kv_cache_num=256 input_tokens=128 +15:02:52.587 INF Run:3471 | prefill chunk p=2 history_len=256 grpid=4 kv_cache_num=384 input_tokens=102 +15:02:52.776 INF Run:3886 | ttft: 518.23 ms +这张图片展示了一个年轻的女性，她有着长长的银灰色头发，头发上装饰着一些花朵。她穿着一件浅灰色的比基尼，比基尼上有褶皱和蕾丝细节，显得非常优雅。她的面部表情平静，眼神直视前方，给人一种温柔和自信的感觉。背景是海滩，可以看到海浪轻轻拍打着沙滩，远处有绿色的树木，整个场景显得非常宁静和自然。女性的姿态轻松，似乎在享受海滩上的美好时光。 + +15:02:57.605 NTC Run:4298 | hit eos,decode avg 18.64 token/s +15:02:57.605 INF GetKVCache:3087 | precompute_len:449, remaining:831 (tracked) +``` diff --git a/assets/smoke_image.png b/assets/smoke_image.png new file mode 100644 index 0000000000000000000000000000000000000000..97e4ed3533707c045a4dd76a50fa66125acd8a2c --- /dev/null +++ b/assets/smoke_image.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:622ae2d01ff4467fa69a7888728d776650117a0f4887e96ba0fb9a8a6d77b3c3 +size 355241 diff --git a/bin/axllm b/bin/axllm new file mode 100644 index 0000000000000000000000000000000000000000..5d8c5598030c5c347ec7fac237204093d0e49fce --- /dev/null +++ b/bin/axllm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16a71dad6e173db36de93f4e57c6587461005c6e9e20f4c491c6538d4ba63268 +size 2327944 diff --git a/bin/axllm.version.json b/bin/axllm.version.json new file mode 100644 index 0000000000000000000000000000000000000000..32f04bd0ac5203751b23e1a1d62be6257807f071 --- /dev/null +++ b/bin/axllm.version.json @@ -0,0 +1,26 @@ +{ + "binary": "bin/axllm", + "target": "AX650/AX650N aarch64", + "ax_llm_branch": "ax-minicpm-v-4-6", + "ax_llm_commit": "0152d36423c888f28922722951d1d9ca08c1b840", + "simplecv_commit": "7729953c7c75ffaea441f70e0a9eb62790f3eb2b", + "openai_api_cpp_commit": "f56cf8c296d1002f6602226db392325ba42f6775", + "tokenizer_axera_commit": "731018211a62dcb0d606d474af9e760a0c4c44a5", + "build_command": "./build.sh", + "sha256": "16a71dad6e173db36de93f4e57c6587461005c6e9e20f4c491c6538d4ba63268", + "verified": { + "date": "2026-06-05", + "board": "AX650", + "command": "./bin/axllm serve . --port 18080", + "api_url": "http://10.168.232.217:18080/v1/chat/completions", + "model": "AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047", + "smoke_tests": [ + "text-only chat returned correct answer for 1+1", + "long prompt multi-chunk prefill completed successfully", + "single-image request via OpenAI-compatible API returned a valid image description", + "append-only text history reused KV cache correctly", + "image added after long text history reused prior text KV incrementally", + "MiniCPM text, image, and video requests completed with native prefill after removing decode replay" + ] + } +} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7fccd1371f0b796b1772817c25956c6167104ce4 --- /dev/null +++ b/config.json @@ -0,0 +1,25 @@ +{ + "system_prompt": "", + "model_name": "AXERA-TECH/MiniCPM-V-4.6-AX650-C128-P1152-CTX2047", + "url_tokenizer_model": "minicpm_v46_tokenizer.txt", + "tokenizer_type": "MiniCPMV46VL", + "post_config_path": "post_config.json", + "template_filename_axmodel": "qwen3_5_text_p128_l%d_together.axmodel", + "axmodel_num": 24, + "full_attention_interval": 4, + "filename_post_axmodel": "qwen3_5_text_post.axmodel", + "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin", + "tokens_embed_num": 248094, + "tokens_embed_size": 1024, + "vlm_type": "MiniCPMV46VL", + "filename_image_encoder_axmodel": "minicpmv4_6_vision_448.axmodel", + "vision_cache_dir": "vision_cache", + "vision_width": 448, + "vision_height": 448, + "vision_patch_size": 14, + "b_use_mmap_load_embed": true, + "b_use_mmap_load_layer": true, + "devices": [ + 0 + ] +} diff --git a/minicpm_v46_tokenizer.txt b/minicpm_v46_tokenizer.txt new file mode 100644 index 0000000000000000000000000000000000000000..023e9af8ff23bc4c970624828e031ccdb5d06a07 --- /dev/null +++ b/minicpm_v46_tokenizer.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef38624ba70ed2757dd6c8c98c125bf620b99714fd2318f4fa1d383029eb2a05 +size 12102524 diff --git a/minicpm_v46_tokenizer/.gitattributes b/minicpm_v46_tokenizer/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..52373fe24473b1aa44333d318f578ae6bf04b49b --- /dev/null +++ b/minicpm_v46_tokenizer/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/minicpm_v46_tokenizer/README.md b/minicpm_v46_tokenizer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..80a14d84d0ac0a0d00cd27f02adb8888d14d15eb --- /dev/null +++ b/minicpm_v46_tokenizer/README.md @@ -0,0 +1,498 @@ +--- +license: apache-2.0 +pipeline_tag: image-text-to-text +tags: +- minicpm-v +- multimodal +- On-Device Model +- lightweight +--- + +A Pocket-Sized MLLM for Ultra-Efficient Image and Video Understanding on Your Phone + +[GitHub](https://github.com/OpenBMB/MiniCPM-o) | [CookBook](https://github.com/OpenSQZ/MiniCPM-V-CookBook) | [Demo](https://huggingface.co/spaces/openbmb/MiniCPM-V-4.6-Demo) | +[Feishu (Lark)](https://raw.githubusercontent.com/openbmb/MiniCPM-V/main/assets/feishu_qrcode.png) + +## MiniCPM-V 4.6 + +**MiniCPM-V 4.6** is our most edge-deployment-friendly model to date. The model is built based on SigLIP2-400M and the Qwen3.5-0.8B LLM. It inherits the strong single-image, multi-image, and video understanding capabilities of MiniCPM-V family, while significantly improving computation efficiency. It also introduces mixed 4x/16x visual token compression. Notable features of MiniCPM-V 4.6 include: + +- 🔥 **Leading Foundation Capability.** + MiniCPM-V 4.6 scores 13 on the Artificial Analysis Intelligence Index benchmark, outperforming Qwen3.5-0.8B's score of 10 with 19x fewer token cost, and Qwen3.5-0.8B-Thinking's score of 11 with 43x fewer token cost. It also surpasses the larger Ministral 3 3B (score of 11). + +- 💪 **Strong Multimodal Capability.** + MiniCPM-V 4.6 outperforms Qwen3.5-0.8B on most vision-language understanding tasks, and reaches Qwen3.5 2B-level capability on many benchmarks including OpenCompass, RefCOCO, HallusionBench, MUIRBench, and OCRBench. +- 🚀 **Ultra-Efficient Architecture.** + Based on the latest technique in [LLaVA-UHD v4](https://github.com/THUMAI-Lab/LLaVA-UHD-v4), MiniCPM-V 4.6 reduces the visual encoding computation FLOPs by more than 50%. It enables MiniCPM-V 4.6 to achieve better efficiency to even smaller models, achieving ~1.5x token throughput compared to Qwen3.5-0.8B. + It also supports mixed 4x/16x visual token compression rate, allowing flexible switching between accuracy and speed. +- 📱 **Broad Mobile Platform Coverage.** + MiniCPM-V 4.6 can be deployed across all three mainstream mobile platforms — iOS, Android, and HarmonyOS. With every edge adaptation code open-sourced, developers can reproduce the on-device experience in [just a few steps](#deploy-minicpm-v-46-on-ios-android-and-harmonyos-platforms). +- 🛠️ **Developer Friendly.** + MiniCPM-V 4.6 is adapted to [inference frameworks](#inference-and-training) such as vLLM, SGLang, llama.cpp, Ollama, and supports [fine-tuning ecosystems](#inference-and-training) such as SWIFT and LLaMA-Factory. Developers can quickly customize models for new domains and tasks on consumer-grade GPUs. We provide multiple quantized variants across GGUF, BNB, AWQ, and GPTQ formats. + + +### Evaluation + +**Overall Performance (Instruct)** + +

+ +

+ + +
+Click to view MiniCPM-V 4.6-Thinking performance. + + +

+ +

+ + +
+ + +
+Click to view MiniCPM-V 4.6 inference efficiency results. + + +**High-Concurrency Throughput** + +

+ +

+ +**Single Request TTFT (ms)** + +

+ +

+ + +
+ + +### Examples + +#### Overall + +
+ +
+ +MiniCPM-V 4.6 can be deployed across three mainstream end-side platforms — **iOS, Android and HarmonyOS**. The clips below are raw screen recordings on phone devices without edition. + + + + + + + + + + + + +
iPhone
iPhone 17 Pro Max		Android
Redmi K70		HarmonyOS
HUAWEI nova 14
+ + +### Usages + +#### Inference with Transformers +##### Installation + +```bash +pip install "transformers[torch]>=5.7.0" torchvision torchcodec +``` + +> **Note on CUDA compatibility:** `torchcodec` (used for video decoding) may have compatibility issues with certain CUDA versions. For example, `torch>=2.11` bundles CUDA 13.1 by default, while environments with CUDA 12.x may encounter errors such as `RuntimeError: Could not load libtorchcodec`. Two workarounds: +> +> 1. **Replace `torchcodec` with `PyAV`** — supports both image and video inference without CUDA version constraints: +> ```bash +> pip install "transformers[torch]>=5.7.0" torchvision av +> ``` +> 2. **Pin the CUDA version** when installing torch to match your environment (e.g. CUDA 12.8): +> ```bash +> pip install "transformers>=5.7.0" torchvision torchcodec --index-url https://download.pytorch.org/whl/cu128 +> ``` + +##### Load Model + +```python +from transformers import AutoModelForImageTextToText, AutoProcessor + +model_id = "openbmb/MiniCPM-V-4.6" + +processor = AutoProcessor.from_pretrained(model_id) +model = AutoModelForImageTextToText.from_pretrained( + model_id, torch_dtype="auto", device_map="auto" +) + +# Flash Attention 2 is recommended for better acceleration and memory saving, +# especially in multi-image and video scenarios. +# model = AutoModelForImageTextToText.from_pretrained( +# model_id, +# torch_dtype=torch.bfloat16, +# attn_implementation="flash_attention_2", +# device_map="auto", +# ) +``` + +##### Image Inference + +```python +messages = [ + { + "role": "user", + "content": [ + {"type": "image", "url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}, + {"type": "text", "text": "What causes this phenomenon?"}, + ], + } +] + +downsample_mode = "16x" # Using `downsample_mode="4x"` for Finer Detail + +inputs = processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, + return_dict=True, return_tensors="pt", + downsample_mode=downsample_mode, + max_slice_nums=36, +).to(model.device) + +generated_ids = model.generate(**inputs, downsample_mode=downsample_mode, max_new_tokens=512) +generated_ids_trimmed = [ + out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) +] +output_text = processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False +) +print(output_text[0]) +``` + +##### Video Inference + +```python +messages = [ + { + "role": "user", + "content": [ + {"type": "video", "url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/football.mp4"}, + {"type": "text", "text": "Describe this video in detail. Follow the timeline and focus on on-screen text, interface changes, main actions, and scene changes."}, + ], + } +] + +downsample_mode = "16x" # Using `downsample_mode="4x"` for Finer Detail + +inputs = processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, + return_dict=True, return_tensors="pt", + downsample_mode=downsample_mode, + max_num_frames=128, + stack_frames=1, + max_slice_nums=1, + use_image_id=False, +).to(model.device) + +generated_ids = model.generate(**inputs, downsample_mode=downsample_mode, max_new_tokens=2048) +generated_ids_trimmed = [ + out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) +] +output_text = processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False +) +print(output_text[0]) +``` + +##### Advanced Parameters + +You can customize image/video processing by passing additional parameters to `apply_chat_template`: + +| Parameter | Default | Applies to | Description | +|-----------|---------|------------|-------------| +| `downsample_mode` | `"16x"` | Image & Video | Visual token downsampling. `"16x"` merges tokens for efficiency; `"4x"` keeps 4× more tokens for finer detail. Must also be passed to `generate()`. | +| `max_slice_nums` | `9` | Image & Video | Maximum number of slices when splitting a high-resolution image. Higher values preserve more detail for large images. Recommended: `36` for image, `1` for video. | +| `max_num_frames` | `128` | Video only | Maximum number of main frames sampled from the video. | +| `stack_frames` | `1` | Video only | Total sample points per second. `1` = main frame only (no stacking). `N` (N>1) = 1 main frame + N−1 sub-frames per second; the sub-frames are composited into a grid image and interleaved with main frames. Recommended setting is `1` for short videos, and `3` or `5` for long videos. | +| `use_image_id` | `True` | Image & Video | Whether to prepend `N` tags before each image/frame placeholder. Set `True` for image, `False` for video. | + +> **Note:** `downsample_mode` must be passed to **both** `apply_chat_template` (for correct placeholder count) and `generate` (for the vision encoder). All other parameters only need to be passed to `apply_chat_template`. + +##### Serving with `transformers serve` + +Hugging Face Transformers includes a lightweight OpenAI-compatible server for quick testing and moderate-load deployment. + +```bash +pip install "transformers[serving]>=5.7.0" +``` + +Start the server: + +```bash +transformers serve openbmb/MiniCPM-V-4.6 --port 8000 --host 0.0.0.0 --continuous-batching +``` + +Send a request: + +```bash +curl -s http://localhost:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ] + }] + }' +``` + +Tool calling example: + +```bash +curl -s http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "the weather of Beijing"} + ]}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"} + }, + "required": ["location"] + } + } + }] +}' +``` + +The model returns a natural-language explanation followed by a structured block embedded in the content field. Note that a dedicated tool call parser for this format has not yet been added to the transformers library, so the tool calls need to be extracted manually via regex for now. + +``` +{ + "id": "f4f09c7d-8045-4cb1-ade9-07aa5dee637d", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "I need to check the current weather for Beijing, so I will call the get_weather function.\n\n\n\n\nBeijing\n\n\n", + "role": "assistant" + } + } + ], + "created": 1778748859, + "model": "openbmb/MiniCPM-V-4.6@main", + "object": "chat.completion", + "usage": { + "completion_tokens": 47, + "prompt_tokens": 283, + "total_tokens": 330 + } +} +``` + +#### Handling Escaped Newlines in Model Outputs + +In some cases, the model might output escaped newline characters `\n` as string literals instead of actual newlines. To render the text correctly, especially in UI layers, you can use the following utility function. This function carefully replaces literal `\n` with real newlines while protecting scenarios where `\n` has specific semantic meaning. + +**Utility Function:** + +```python +import re + +_PATTERN = re.compile( + r'(```[\s\S]*?```' # fenced code blocks + r'|`[^`]+`' # inline code + r'|\$\$[\s\S]*?\$\$' # display math + r'|\$[^$]+\$' # inline math + r'|\\\([\s\S]*?\\\)' # \(...\) + r'|\\\[[\s\S]*?\\\]' # \[...\] + r')' + r'|(? str: + """ + Lightweight post-processing: Converts literal '\\n' to actual newlines, + while protecting code blocks, inline code, and LaTeX commands. + """ + if not isinstance(text, str) or "\\" not in text: + return text + return _PATTERN.sub(lambda m: m.group(1) or '\n', text) +``` + +#### Deploy MiniCPM-V 4.6 on iOS, Android, and HarmonyOS Platforms + +We have adapted MiniCPM-V 4.6 for deployment on **iOS, Android, and HarmonyOS** platforms, with **all edge adaptation code fully open-sourced**. Developers can reproduce the on-device experience in just a few steps. Visit our [edge deployment repository](https://github.com/OpenBMB/MiniCPM-V-edge-demo) for platform-specific build guides, or go to the [download page](https://github.com/OpenBMB/MiniCPM-V-edge-demo/blob/main/DOWNLOAD.md) to try pre-built apps directly. + + +#### Use MiniCPM-V 4.6 in Other Inference and Training Frameworks + +MiniCPM-V 4.6 supports multiple inference and training frameworks. Below are quick-start commands for each. For full details, see our [Cookbook](https://github.com/OpenSQZ/MiniCPM-V-CookBook). + +
+vLLM — Full Guide + +```bash +vllm serve openbmb/MiniCPM-V-4.6 \ + --port 8000 \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --default-chat-template-kwargs '{"enable_thinking": false}' +``` + +> **Note:** `--enable-auto-tool-choice` and `--tool-call-parser qwen3_coder` enable tool/function calling support. If you don't need tool use, you can omit these flags and simply run `vllm serve openbmb/MiniCPM-V-4.6`. + +```bash +curl -s http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ]}] +}' +``` + + +Tool calling example: + +```bash +curl -s http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "北京的天气"} + ]}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"} + }, + "required": ["location"] + } + } + }] +}' +``` + +
+ +
+SGLang — Full Guide + +```bash +python -m sglang.launch_server --model openbmb/MiniCPM-V-4.6 --port 30000 +``` + +```bash +curl -s http://localhost:30000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ]}] +}' +``` + +
+ +
+llama.cpp — Full Guide + +```bash +llama-server -m MiniCPM-V-4.6-Q4_K_M.gguf --port 8080 +``` + +```bash +curl -s http://localhost:8080/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ]}] +}' +``` + +
+ +
+Ollama — Full Guide + +```bash +ollama run minicpm-v-4.6 +``` + +In the interactive session, paste an image path or URL directly to chat with the model. + +
+ +
+LLaMA-Factory (Fine-tuning) — Full Guide + +```bash +llamafactory-cli train examples/train_lora/minicpmv4_6_lora_sft.yaml +``` + +
+ +
+ms-swift (Fine-tuning) — Full Guide + +```bash +swift sft --model_type minicpm-v-4_6 --dataset +``` + +
+ +## License + +#### Model License +* The MiniCPM-o/V model weights and code are open-sourced under the [Apache-2.0](https://github.com/OpenBMB/MiniCPM-V/blob/main/LICENSE) license. + +#### Statement +* As MLLMs, MiniCPM-o/V models generate content by learning a large number of multimodal corpora, but they cannot comprehend, express personal opinions, or make value judgements. Anything generated by MiniCPM-o/V models does not represent the views and positions of the model developers +* We will not be liable for any problems arising from the use of MiniCPM-o/V models, including but not limited to data security issues, risk of public opinion, or any risks and problems arising from the misdirection, misuse, dissemination, or misuse of the model. + + +## Technical Reports and Key Techniques Papers + +👏 Welcome to explore key techniques of MiniCPM-o/V and other multimodal projects of our team: + +**Technical Reports:** [MiniCPM-o 4.5](https://huggingface.co/papers/2604.27393) | [MiniCPM-V 4.5](https://arxiv.org/abs/2509.18154) | [MiniCPM-o 2.6](https://openbmb.notion.site/MiniCPM-o-2-6-A-GPT-4o-Level-MLLM-for-Vision-Speech-and-Multimodal-Live-Streaming-on-Your-Phone-185ede1b7a558042b5d5e45e6b237da9) | [MiniCPM-Llama3-V 2.5](https://arxiv.org/abs/2408.01800) | [MiniCPM-V 2.0](https://openbmb.vercel.app/minicpm-v-2) + +**Other Multimodal Projects:** [VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLPR](https://github.com/OpenBMB/RLPR) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) | [LLaVA-UHD-v4](https://arxiv.org/abs/2605.08985 ) + + +## Citation + +If you find our model/code/paper helpful, please consider citing our papers 📝 and staring us ⭐️！ + +```bib +@proceedings{yu2025minicpmv45cookingefficient, + title={MiniCPM-V 4.5: Cooking Efficient MLLMs via Architecture, Data, and Training Recipe}, + author={Tianyu Yu and Zefan Wang and Chongyi Wang and Fuwei Huang and Wenshuo Ma and Zhihui He and Tianchi Cai and Weize Chen and Yuxiang Huang and Yuanqian Zhao and others}, + year={2025}, + url={https://arxiv.org/abs/2509.18154}, +} + +@article{yao2024minicpm, + title={MiniCPM-V: A GPT-4V Level MLLM on Your Phone}, + author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others}, + journal={arXiv preprint arXiv:2408.01800}, + year={2024} +} +``` \ No newline at end of file diff --git a/minicpm_v46_tokenizer/chat_template.jinja b/minicpm_v46_tokenizer/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..f25b6ac32579e33b2d7041442e76d9137a0311fa --- /dev/null +++ b/minicpm_v46_tokenizer/chat_template.jinja @@ -0,0 +1,145 @@ +{%- if enable_thinking is not defined -%} + {%- set enable_thinking = false -%} +{%- endif -%} +{%- macro render_content(content, is_system_content=false) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is iterable and content is not mapping -%} + {%- set ns = namespace(parts=[]) -%} + {%- for item in content -%} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%} + {%- if is_system_content -%} + {{- raise_exception('System message cannot contain images.') -}} + {%- endif -%} + {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%} + {%- elif 'video' in item or item.type == 'video' -%} + {%- if is_system_content -%} + {{- raise_exception('System message cannot contain videos.') -}} + {%- endif -%} + {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%} + {%- elif 'text' in item -%} + {%- set ns.parts = ns.parts + [item.text] -%} + {%- else -%} + {{- raise_exception('Unexpected item type in content.') -}} + {%- endif -%} + {%- endfor -%} + {{- ns.parts | join('\n') -}} + {%- elif content is none or content is undefined -%} + {{- '' -}} + {%- else -%} + {{- raise_exception('Unexpected content type.') -}} + {%- endif -%} +{%- endmacro -%} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/minicpm_v46_tokenizer/config.json b/minicpm_v46_tokenizer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dbe7f74b10746936eb2637ba14f5bf5515b5721c --- /dev/null +++ b/minicpm_v46_tokenizer/config.json @@ -0,0 +1,90 @@ +{ + "architectures": [ + "MiniCPMV4_6ForConditionalGeneration" + ], + "bos_token_id": null, + "drop_vision_last_layer": false, + "eos_token_id": 248044, + "image_size": 1120, + "model_type": "minicpmv4_6", + "pad_token_id": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": true, + "vision_config": { + "attention_dropout": 0.0, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 980, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "minicpmv4_6_vision", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14 + }, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 3584, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 16, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "vocab_size": 248094, + "model_type": "qwen3_5_text", + "tie_word_embeddings": true + }, + "insert_layer_id": 6, + "image_token_id": 248056, + "video_token_id": 248057 +} diff --git a/minicpm_v46_tokenizer/generation_config.json b/minicpm_v46_tokenizer/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b80dd7cba268869847631d044ba360aeff64ba7c --- /dev/null +++ b/minicpm_v46_tokenizer/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 248045, + "do_sample": true, + "eos_token_id": [ + 248044, + 248046 + ], + "temperature": 0.7, + "top_k": 0, + "top_p": 1.0, + "repetition_penalty": 1.0, + "transformers_version": "5.7.0" +} diff --git a/minicpm_v46_tokenizer/preprocessor_config.json b/minicpm_v46_tokenizer/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8619291054b6d574c16d9903c0070aa61c73bd0 --- /dev/null +++ b/minicpm_v46_tokenizer/preprocessor_config.json @@ -0,0 +1,19 @@ +{ + "image_processor_type": "MiniCPMV4_6ImageProcessor", + "processor_class": "MiniCPMV4_6Processor", + "max_slice_nums": 9, + "scale_resolution": 448, + "patch_size": 14, + "use_image_id": true, + "slice_mode": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ] +} \ No newline at end of file diff --git a/minicpm_v46_tokenizer/tokenizer.json b/minicpm_v46_tokenizer/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..20e07d534707e3e879933e9f39dfb631ff1cbd9a --- /dev/null +++ b/minicpm_v46_tokenizer/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33861e37bb955af1e3f3061182b820f347eba2b9c2c1011c82794bf0d6e77b54 +size 19992481 diff --git a/minicpm_v46_tokenizer/tokenizer_config.json b/minicpm_v46_tokenizer/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9b6fab078fcc380bdd4ab6fa8d9ac3310d6709d9 --- /dev/null +++ b/minicpm_v46_tokenizer/tokenizer_config.json @@ -0,0 +1,42 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": "<|im_start|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "image_start_token": "", + "image_end_token": "", + "slice_start_token": "", + "slice_end_token": "", + "image_id_start_token": "", + "image_id_end_token": "" + }, + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "unk_token": "", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>", + "chat_template": "{%- if enable_thinking is not defined -%}\n {%- set enable_thinking = false -%}\n{%- endif -%}\n{%- macro render_content(content, is_system_content=false) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is iterable and content is not mapping -%}\n {%- set ns = namespace(parts=[]) -%}\n {%- for item in content -%}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n {%- if is_system_content -%}\n {{- raise_exception('System message cannot contain images.') -}}\n {%- endif -%}\n {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}\n {%- elif 'video' in item or item.type == 'video' -%}\n {%- if is_system_content -%}\n {{- raise_exception('System message cannot contain videos.') -}}\n {%- endif -%}\n {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}\n {%- elif 'text' in item -%}\n {%- set ns.parts = ns.parts + [item.text] -%}\n {%- else -%}\n {{- raise_exception('Unexpected item type in content.') -}}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.parts | join('\\n') -}}\n {%- elif content is none or content is undefined -%}\n {{- '' -}}\n {%- else -%}\n {{- raise_exception('Unexpected content type.') -}}\n {%- endif -%}\n{%- endmacro -%}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content)|trim %}\n {%- if not(content.startswith('') and content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n\\n\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n' }}\n {%- endif %}\n{%- endif %}\n", + "tokenizer_class": "Qwen2Tokenizer" +} \ No newline at end of file diff --git a/minicpmv4_6_vision_448.axmodel b/minicpmv4_6_vision_448.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..479fd6de54097bb0b0002fe32c6cd09ae587e1dd --- /dev/null +++ b/minicpmv4_6_vision_448.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1cb971418dfe32ad4af06634c994e3f94720b5c24cb901393b9242e7e908e20 +size 609776421 diff --git a/model.embed_tokens.weight.bfloat16.bin b/model.embed_tokens.weight.bfloat16.bin new file mode 100644 index 0000000000000000000000000000000000000000..d88496b9c1f1ee0efa6f5c1d200bdc12b93da9d6 --- /dev/null +++ b/model.embed_tokens.weight.bfloat16.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0f4077f6d1946e32888542927e12b8772a49f3a8b93622f7670320149911b69 +size 508096512 diff --git a/post_config.json b/post_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1d6b0aa669ce6b2fb506209abc447289fd8331 --- /dev/null +++ b/post_config.json @@ -0,0 +1,14 @@ +{ + "enable_temperature" : false, + "temperature" : 0.9, + + "enable_repetition_penalty" : false, + "repetition_penalty" : 1.2, + "penalty_window" : 20, + + "enable_top_p_sampling" : false, + "top_p" : 0.8, + + "enable_top_k_sampling" : false, + "top_k" : 10 +} \ No newline at end of file diff --git a/python/infer_axmodel.py b/python/infer_axmodel.py new file mode 100644 index 0000000000000000000000000000000000000000..f9788a326a63e29aefd2f6b58e7617851ca4e760 --- /dev/null +++ b/python/infer_axmodel.py @@ -0,0 +1,701 @@ +import argparse +import atexit +import hashlib +import json +import os +import re +from dataclasses import dataclass +from typing import List, Optional, Sequence, Tuple + +import numpy as np +from axengine import InferenceSession +from ml_dtypes import bfloat16 +from transformers import AutoProcessor + + +def resolve_default_hf_model() -> str: + base_dir = os.path.dirname(__file__) + candidates = [ + os.path.join(base_dir, "minicpm_v46_tokenizer"), + os.path.join(base_dir, "MiniCPM-V-4.6"), + os.path.join(base_dir, "MiniCPM-V-4.6-GPTQ"), + ] + for path in candidates: + if os.path.exists(os.path.join(path, "config.json")): + return path + return candidates[0] + + +DEFAULT_HF_MODEL = resolve_default_hf_model() + + +def release_ax_inference_session(session): + inner = getattr(session, "_sess", None) + unload = getattr(inner, "_unload", None) + if not callable(unload): + return + + try: + unload() + except Exception as exc: + print(f"[WARN] Failed to unload axengine session cleanly: {exc}") + finally: + try: + inner._unload = lambda: None + except Exception: + pass + + +def bf16_zeros(shape: Sequence[int]) -> np.ndarray: + return np.zeros(tuple(shape), dtype=bfloat16) + + +def dtype_from_axengine(dtype) -> np.dtype: + name = str(dtype).lower() + if "bfloat16" in name or "bf16" in name: + return bfloat16 + if "float32" in name or "fp32" in name: + return np.float32 + if "float16" in name or "fp16" in name: + return np.float16 + if "uint32" in name or "u32" in name: + return np.uint32 + raise ValueError(f"Unsupported axengine dtype: {dtype}") + + +def tensor_digest(arr: np.ndarray) -> str: + arr = np.asarray(arr) + if arr.dtype == bfloat16: + raw = arr.view(np.uint16).tobytes() + else: + raw = arr.tobytes() + return hashlib.sha256(raw).hexdigest()[:16] + + +def tensor_stats(arr: np.ndarray) -> str: + arr32 = np.asarray(arr, dtype=np.float32) + finite = bool(np.isfinite(arr32).all()) + if finite: + return ( + f"finite=True hash={tensor_digest(np.asarray(arr))} " + f"sum={float(arr32.sum()):.6f} max={float(arr32.max()):.6f} min={float(arr32.min()):.6f}" + ) + return ( + f"finite=False hash={tensor_digest(np.asarray(arr))} " + f"nans={int(np.isnan(arr32).sum())} infs={int(np.isinf(arr32).sum())}" + ) + + +def ensure_finite(name: str, arr: np.ndarray): + arr32 = np.asarray(arr, dtype=np.float32) + if not np.isfinite(arr32).all(): + raise RuntimeError(f"{name} is non-finite: {tensor_stats(np.asarray(arr))}") + + +def load_text_config(hf_model: str) -> Tuple[dict, int]: + with open(os.path.join(hf_model, "config.json"), "r", encoding="utf-8") as f: + cfg = json.load(f) + text_cfg = cfg["text_config"] + eos_token_id = cfg.get("eos_token_id") + if eos_token_id is None: + eos_token_id = text_cfg.get("eos_token_id") + if isinstance(eos_token_id, list): + eos_token_id = eos_token_id[0] + return text_cfg, int(eos_token_id if eos_token_id is not None else 248044) + + +@dataclass +class LayerFiles: + layer_paths: List[str] + post_path: str + + +def detect_layer_files(model_dir: str, max_layers: Optional[int] = None) -> LayerFiles: + names = os.listdir(model_dir) + layer_pattern = re.compile(r"^(?P.*)_p(?P\d+)_l(?P\d+)_together\.axmodel$") + decode_layer_pattern = re.compile(r"^(?P.*)_l(?P\d+)\.axmodel$") + post_pattern = re.compile(r"^(?P.*)_post\.axmodel$") + + prefix_map = {} + for name in names: + m = layer_pattern.match(name) + if not m: + m = decode_layer_pattern.match(name) + if not m: + continue + prefix = m.group("prefix") + idx = int(m.group("idx")) + prefix_map.setdefault(prefix, []).append((idx, name)) + + if not prefix_map: + raise FileNotFoundError(f"No layer axmodel found under {model_dir}") + + prefix = max(prefix_map.items(), key=lambda kv: len(kv[1]))[0] + layer_items = sorted(prefix_map[prefix], key=lambda it: it[0]) + if max_layers is not None: + layer_items = layer_items[: max_layers] + layer_paths = [os.path.join(model_dir, name) for _, name in layer_items] + + post_name = None + for name in names: + m = post_pattern.match(name) + if m and m.group("prefix") == prefix: + post_name = name + break + if post_name is None: + raise FileNotFoundError(f"No post axmodel found for prefix {prefix} under {model_dir}") + + return LayerFiles(layer_paths=layer_paths, post_path=os.path.join(model_dir, post_name)) + + +class MiniCPMTextAxModelRunner: + def __init__( + self, + hf_model: str, + axmodel_dir: str, + embed_bin: Optional[str], + max_layers: Optional[int], + kv_cache_len: int = 255, + ): + self.hf_model = hf_model + self.axmodel_dir = axmodel_dir + self.text_cfg, self.eos_token_id = load_text_config(hf_model) + self.hidden_size = int(self.text_cfg["hidden_size"]) + self.vocab_size = int(self.text_cfg["vocab_size"]) + self.kv_cache_len = int(kv_cache_len) + self.layer_types = list(self.text_cfg.get("layer_types") or []) + self.num_attention_heads = int(self.text_cfg["num_attention_heads"]) + self.num_key_value_heads = int(self.text_cfg["num_key_value_heads"]) + self.head_dim = int(self.text_cfg.get("head_dim") or (self.hidden_size // self.num_attention_heads)) + self.full_attn_kv_dim = self.num_key_value_heads * self.head_dim + + self.processor = AutoProcessor.from_pretrained(hf_model, trust_remote_code=True) + self.layer_files = detect_layer_files(axmodel_dir, max_layers=max_layers) + + if embed_bin is None: + candidate = os.path.join(axmodel_dir, "model.embed_tokens.weight.bfloat16.bin") + if not os.path.exists(candidate): + raise FileNotFoundError( + "Embedding bin not found under axmodel_dir, please pass --embed-bin explicitly" + ) + embed_bin = candidate + self.embed_bin = embed_bin + self.embed_matrix = np.memmap(embed_bin, mode="r", dtype=np.uint16).view(bfloat16).reshape( + self.vocab_size, self.hidden_size + ) + + self.decoder_sessions = [InferenceSession(path) for path in self.layer_files.layer_paths] + self.post_session = None + self._closed = False + atexit.register(self.close) + + self.layer_decode_input_shapes = [] + self.layer_decode_input_dtypes = [] + self.layer_prefill_input_shapes = [] + self.layer_prefill_input_dtypes = [] + self.layer_decode_output_names = [] + self.layer_prefill_output_names = [] + for layer_idx, session in enumerate(self.decoder_sessions): + decode_input_shapes = {x.name: tuple(x.shape) for x in session.get_inputs(shape_group=0)} + decode_input_dtypes = {x.name: dtype_from_axengine(x.dtype) for x in session.get_inputs(shape_group=0)} + # Some decode-only AX650 exports hide `indices` / `mask` from get_inputs(), + # but axengine still validates them as required runtime inputs. + decode_input_shapes.setdefault("indices", (1, 1)) + decode_input_dtypes.setdefault("indices", np.uint32) + decode_input_shapes.setdefault("mask", (1, 1)) + decode_input_dtypes.setdefault("mask", bfloat16) + if layer_idx < len(self.layer_types) and self.layer_types[layer_idx] == "full_attention": + decode_input_shapes["K_cache"] = (1, self.kv_cache_len, self.full_attn_kv_dim) + decode_input_shapes["V_cache"] = (1, self.kv_cache_len, self.full_attn_kv_dim) + self.layer_decode_input_shapes.append(decode_input_shapes) + self.layer_decode_input_dtypes.append(decode_input_dtypes) + self.layer_decode_output_names.append([x.name for x in session.get_outputs(shape_group=0)]) + prefill_shape_groups = [] + prefill_dtype_groups = [] + prefill_output_groups = [] + for shape_group in range(1, 64): + try: + prefill_inputs = session.get_inputs(shape_group=shape_group) + prefill_shape_groups.append({x.name: tuple(x.shape) for x in prefill_inputs}) + prefill_dtype_groups.append({x.name: dtype_from_axengine(x.dtype) for x in prefill_inputs}) + prefill_output_groups.append([x.name for x in session.get_outputs(shape_group=shape_group)]) + except Exception: + break + self.layer_prefill_input_shapes.append(prefill_shape_groups) + self.layer_prefill_input_dtypes.append(prefill_dtype_groups) + self.layer_prefill_output_names.append(prefill_output_groups) + self.decode_input_shapes = self.layer_decode_input_shapes[0] + self.decode_input_dtypes = self.layer_decode_input_dtypes[0] + self.prefill_input_shapes = ( + self.layer_prefill_input_shapes[0][0] + if self.layer_prefill_input_shapes and self.layer_prefill_input_shapes[0] + else {} + ) + self.prefill_input_dtypes = ( + self.layer_prefill_input_dtypes[0][0] + if self.layer_prefill_input_dtypes and self.layer_prefill_input_dtypes[0] + else {} + ) + self.prefill_len = int(self.prefill_input_shapes["input"][1]) if "input" in self.prefill_input_shapes else 0 + self.decode_output_names = self.layer_decode_output_names[0] + self.prefill_output_names = ( + self.layer_prefill_output_names[0][0] + if self.layer_prefill_output_names and self.layer_prefill_output_names[0] + else [] + ) + self.hidden_dtype = self.decode_input_dtypes["input"] + + def close(self): + if self._closed: + return + for session in getattr(self, "decoder_sessions", []): + release_ax_inference_session(session) + if getattr(self, "post_session", None) is not None: + release_ax_inference_session(self.post_session) + self.decoder_sessions = [] + self.post_session = None + self._closed = True + + def tokenize_prompt(self, prompt: str) -> List[int]: + messages = [{"role": "user", "content": [{"type": "text", "text": prompt}]}] + inputs = self.processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="np", + ) + return inputs["input_ids"][0].astype(np.int64).tolist() + + def decode_tokens(self, token_ids: Sequence[int]) -> str: + return self.processor.decode( + list(token_ids), + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + ) + + def embed_token(self, token_id: int) -> np.ndarray: + return np.asarray(self.embed_matrix[int(token_id)], dtype=self.hidden_dtype).reshape(1, 1, self.hidden_size) + + def alloc_layer_states(self) -> Tuple[List[np.ndarray], List[np.ndarray]]: + k_states = [ + np.zeros(shapes["K_cache"], dtype=dtypes["K_cache"]) + for shapes, dtypes in zip(self.layer_decode_input_shapes, self.layer_decode_input_dtypes) + ] + v_states = [ + np.zeros(shapes["V_cache"], dtype=dtypes["V_cache"]) + for shapes, dtypes in zip(self.layer_decode_input_shapes, self.layer_decode_input_dtypes) + ] + return k_states, v_states + + @staticmethod + def make_feed(shapes: dict, values: dict) -> dict: + return {name: value for name, value in values.items() if name in shapes} + + def is_linear_layer(self, layer_idx: int) -> bool: + return layer_idx >= len(self.layer_types) or self.layer_types[layer_idx] != "full_attention" + + def prefill_history_capacity(self, shapes: dict) -> int: + if not shapes: + return -1 + input_len = int(shapes.get("input", (1, self.prefill_len))[1]) + mask_shape = shapes.get("mask") + if mask_shape is not None and len(mask_shape) == 3: + return max(0, int(mask_shape[-1]) - input_len) + k_shape = shapes.get("K_cache") + if k_shape is not None and len(k_shape) >= 2: + return int(k_shape[1]) + return 0 + + def select_prefill_shape_group(self, layer_idx: int, history_len: int) -> int: + groups = self.layer_prefill_input_shapes[layer_idx] + if not groups: + raise RuntimeError(f"layer {layer_idx} has no prefill shape_group") + if history_len <= 0: + return 1 + + candidates = [] + for offset, shapes in enumerate(groups): + cap = self.prefill_history_capacity(shapes) + if cap >= history_len: + candidates.append((cap, offset + 1)) + if candidates: + return min(candidates)[1] + + # Linear-attention layers may prune reusable warm groups. Reuse the + # largest available group when its state shape is independent of history. + return max(range(1, len(groups) + 1), key=lambda gid: self.prefill_history_capacity(groups[gid - 1])) + + def inspect(self): + print("hf_model:", self.hf_model) + print("axmodel_dir:", self.axmodel_dir) + print("embed_bin:", self.embed_bin) + print("num_layers:", len(self.decoder_sessions)) + print("hidden_size:", self.hidden_size) + print("vocab_size:", self.vocab_size) + print("eos_token_id:", self.eos_token_id) + print("kv_cache_len:", self.kv_cache_len) + print("prefill_len:", self.prefill_len) + print("decode_inputs:", sorted((k, v, str(self.decode_input_dtypes[k])) for k, v in self.decode_input_shapes.items())) + print("decode_outputs:", [(x.name, tuple(x.shape), str(x.dtype)) for x in self.decoder_sessions[0].get_outputs(0)]) + unique_k_shapes = sorted({tuple(spec["K_cache"]) for spec in self.layer_decode_input_shapes}) + unique_v_shapes = sorted({tuple(spec["V_cache"]) for spec in self.layer_decode_input_shapes}) + print("decode_k_cache_shapes:", unique_k_shapes) + print("decode_v_cache_shapes:", unique_v_shapes) + if self.prefill_input_shapes: + print("prefill_group_count_layer0:", len(self.layer_prefill_input_shapes[0])) + unique_prefill_k_shapes = sorted( + { + tuple(spec["K_cache"]) + for groups in self.layer_prefill_input_shapes + for spec in groups + if "K_cache" in spec + } + ) + unique_prefill_v_shapes = sorted( + { + tuple(spec["V_cache"]) + for groups in self.layer_prefill_input_shapes + for spec in groups + if "V_cache" in spec + } + ) + unique_prefill_mask_shapes = sorted( + { + tuple(spec["mask"]) + for groups in self.layer_prefill_input_shapes + for spec in groups + if "mask" in spec + } + ) + print("prefill_inputs_layer0:", sorted((k, v, str(self.prefill_input_dtypes[k])) for k, v in self.prefill_input_shapes.items())) + print("prefill_k_cache_shapes:", unique_prefill_k_shapes) + print("prefill_v_cache_shapes:", unique_prefill_v_shapes) + print("prefill_mask_shapes:", unique_prefill_mask_shapes) + print("prefill_outputs:", [(x.name, tuple(x.shape), str(x.dtype)) for x in self.decoder_sessions[0].get_outputs(1)]) + else: + print("prefill_inputs: []") + print("prefill_outputs: []") + post_session = self.get_post_session() + print("post_inputs:", [(x.name, tuple(x.shape), str(x.dtype)) for x in post_session.get_inputs()]) + print("post_outputs:", [(x.name, tuple(x.shape), str(x.dtype)) for x in post_session.get_outputs()]) + + def get_post_session(self): + if self.post_session is None: + self.post_session = InferenceSession(self.layer_files.post_path) + return self.post_session + + def run_prefill(self, token_ids: Sequence[int], verbose: bool = False, return_states: bool = False): + k_states, v_states = self.alloc_layer_states() + last_hidden = None + for start in range(0, len(token_ids), self.prefill_len): + chunk_ids = token_ids[start : start + self.prefill_len] + chunk_len = len(chunk_ids) + data = np.zeros((1, self.prefill_len, self.hidden_size), dtype=self.hidden_dtype) + data[0, :chunk_len, :] = np.asarray( + self.embed_matrix[np.asarray(chunk_ids, dtype=np.int64)], dtype=self.hidden_dtype + ) + + for layer_idx, session in enumerate(self.decoder_sessions): + if self.is_linear_layer(layer_idx): + out = np.zeros_like(data) + for j in range(chunk_len): + hidden = data[:, j : j + 1, :] + hidden = self.run_single_layer_decode_step( + layer_idx, + hidden, + start + j, + k_states, + v_states, + verbose=False, + ) + out[:, j : j + 1, :] = hidden + data = out + if verbose: + print( + f"prefill chunk={start // self.prefill_len} layer={layer_idx} " + f"linear_decode_replay tokens={chunk_len} {tensor_stats(data)}" + ) + continue + + shape_group = self.select_prefill_shape_group(layer_idx, start) + layer_shapes = self.layer_prefill_input_shapes[layer_idx][shape_group - 1] + layer_dtypes = self.layer_prefill_input_dtypes[layer_idx][shape_group - 1] + if not layer_shapes: + raise RuntimeError(f"layer {layer_idx} has no prefill shape_group={shape_group}") + history_cap = self.prefill_history_capacity(layer_shapes) + history_len = min(start, history_cap) + indices = None + if "indices" in layer_shapes: + indices = np.zeros(layer_shapes["indices"], dtype=layer_dtypes["indices"]) + indices.reshape(-1)[:chunk_len] = np.arange(start, start + chunk_len, dtype=np.uint32) + mask = None + if "mask" in layer_shapes: + if self.layer_types[layer_idx] == "full_attention" and len(layer_shapes["mask"]) == 3: + mask = np.full(layer_shapes["mask"], -65536.0, dtype=np.float32) + for q in range(chunk_len): + mask[:, q, : history_len + q + 1] = 0.0 + else: + mask = np.zeros(layer_shapes["mask"], dtype=np.float32) + mask.reshape(-1)[:chunk_len] = 1.0 + mask = mask.astype(layer_dtypes["mask"]) + + k_feed = k_states[layer_idx] + v_feed = v_states[layer_idx] + if self.layer_types[layer_idx] == "full_attention": + k_feed = np.zeros(layer_shapes["K_cache"], dtype=layer_dtypes["K_cache"]) + v_feed = np.zeros(layer_shapes["V_cache"], dtype=layer_dtypes["V_cache"]) + if history_len > 0: + k_feed[:, :history_len, :] = k_states[layer_idx][:, :history_len, :] + v_feed[:, :history_len, :] = v_states[layer_idx][:, :history_len, :] + outputs = session.run( + None, + self.make_feed( + layer_shapes, + { + "K_cache": k_feed, + "V_cache": v_feed, + **({"indices": indices} if indices is not None else {}), + "input": data.astype(layer_dtypes["input"], copy=False), + **({"mask": mask} if mask is not None else {}), + }, + ), + shape_group=shape_group, + ) + output_map = dict(zip(self.layer_prefill_output_names[layer_idx][shape_group - 1], outputs)) + k_out = output_map.get("K_cache_out") + if k_out is not None: + if self.layer_types[layer_idx] == "full_attention": + k_states[layer_idx][:, start : start + chunk_len, :] = k_out[:, :chunk_len, :] + else: + k_states[layer_idx] = k_out + v_out = output_map.get("V_cache_out") + if v_out is not None: + if self.layer_types[layer_idx] == "full_attention": + v_states[layer_idx][:, start : start + chunk_len, :] = v_out[:, :chunk_len, :] + else: + v_states[layer_idx] = v_out + data = output_map["output"] + ensure_finite(f"prefill layer {layer_idx} output", data) + if verbose: + print( + f"prefill chunk={start // self.prefill_len} layer={layer_idx} " + f"shape_group={shape_group} history_len={history_len} {tensor_stats(data)}" + ) + last_hidden = data[:, chunk_len - 1 : chunk_len, :] + if return_states: + return k_states, v_states, last_hidden + return last_hidden + + def run_single_layer_decode_step( + self, + layer_idx: int, + hidden: np.ndarray, + position: int, + k_states: List[np.ndarray], + v_states: List[np.ndarray], + verbose: bool = False, + ) -> np.ndarray: + session = self.decoder_sessions[layer_idx] + layer_shapes = self.layer_decode_input_shapes[layer_idx] + layer_dtypes = self.layer_decode_input_dtypes[layer_idx] + indices = None + if "indices" in layer_shapes: + indices = np.zeros(layer_shapes["indices"], dtype=layer_dtypes["indices"]) + indices.reshape(-1)[0] = position + mask = None + if "mask" in layer_shapes: + if self.layer_types[layer_idx] == "full_attention" and len(layer_shapes["mask"]) == 3: + mask = np.full(layer_shapes["mask"], -65536.0, dtype=np.float32) + valid_past = min(position, layer_shapes["mask"][-1] - 1) + if valid_past > 0: + mask[:, :, :valid_past] = 0.0 + mask[:, :, -1:] = 0.0 + else: + mask = np.ones(layer_shapes["mask"], dtype=np.float32) + mask = mask.astype(layer_dtypes["mask"]) + outputs = session.run( + None, + self.make_feed( + layer_shapes, + { + "K_cache": k_states[layer_idx], + "V_cache": v_states[layer_idx], + **({"indices": indices} if indices is not None else {}), + "input": hidden, + **({"mask": mask} if mask is not None else {}), + }, + ), + shape_group=0, + ) + output_map = dict(zip(self.layer_decode_output_names[layer_idx], outputs)) + k_out = output_map.get("K_cache_out") + if k_out is not None: + if self.layer_types[layer_idx] == "full_attention" and k_states[layer_idx].shape != k_out.shape: + pos_end = position + k_out.shape[1] + k_states[layer_idx][:, position:pos_end, :] = k_out + else: + k_states[layer_idx] = k_out + v_out = output_map.get("V_cache_out") + if v_out is not None: + if self.layer_types[layer_idx] == "full_attention" and v_states[layer_idx].shape != v_out.shape: + pos_end = position + v_out.shape[1] + v_states[layer_idx][:, position:pos_end, :] = v_out + else: + v_states[layer_idx] = v_out + out = output_map["output"] + ensure_finite(f"decode step={position} layer={layer_idx} output", out) + if verbose: + print(f"decode step={position} layer={layer_idx} {tensor_stats(out)}") + return out + + def run_decode_step( + self, + hidden: np.ndarray, + position: int, + k_states: List[np.ndarray], + v_states: List[np.ndarray], + verbose: bool = False, + ) -> np.ndarray: + data = hidden + for layer_idx in range(len(self.decoder_sessions)): + data = self.run_single_layer_decode_step(layer_idx, data, position, k_states, v_states, verbose=verbose) + return data + + def decode_replay_prompt( + self, + token_ids: Sequence[int], + limit_prompt_tokens: Optional[int] = None, + verbose: bool = False, + ) -> Tuple[List[np.ndarray], List[np.ndarray], np.ndarray]: + if limit_prompt_tokens is not None: + token_ids = token_ids[:limit_prompt_tokens] + k_states, v_states = self.alloc_layer_states() + last_hidden = None + for pos, token_id in enumerate(token_ids): + last_hidden = self.run_decode_step(self.embed_token(int(token_id)), pos, k_states, v_states, verbose=verbose) + return k_states, v_states, last_hidden + + def run_post(self, hidden: np.ndarray) -> np.ndarray: + logits = self.get_post_session().run(None, {"input": hidden})[0] + ensure_finite("post logits", logits) + return logits + + def greedy_next_token(self, hidden: np.ndarray) -> int: + logits = self.run_post(hidden) + logits = np.asarray(logits, dtype=np.float32).reshape(-1) + return int(np.argmax(logits)) + + def generate(self, token_ids: Sequence[int], max_new_tokens: int, prompt_mode: str, verbose: bool = False): + if prompt_mode == "prefill": + k_states, v_states, last_hidden = self.run_prefill(token_ids, verbose=verbose, return_states=True) + else: + k_states, v_states, last_hidden = self.decode_replay_prompt(token_ids, verbose=verbose) + prompt_len = len(token_ids) + generated = [] + for step in range(max_new_tokens): + next_token = self.greedy_next_token(last_hidden) + generated.append(next_token) + print(f"gen step={step} token_id={next_token} piece={self.decode_tokens([next_token])!r}") + if next_token == self.eos_token_id: + break + last_hidden = self.run_decode_step( + self.embed_token(next_token), + prompt_len + step, + k_states, + v_states, + verbose=verbose, + ) + + print("generated_ids:", generated) + print("generated_text:", self.decode_tokens(generated)) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="MiniCPM-V-4.6 AX650 text-only axmodel Python runner/debugger" + ) + parser.add_argument("--hf-model", default=DEFAULT_HF_MODEL, help="Tokenizer/config path for AutoProcessor") + parser.add_argument("--axmodel-dir", required=True, help="Compiled axmodel directory") + parser.add_argument("--embed-bin", default=None, help="Embedding bf16 bin path") + parser.add_argument( + "--mode", + default="inspect", + choices=["inspect", "prefill", "decode_replay", "generate"], + help="Execution mode", + ) + parser.add_argument("--prompt", default="你好，请做一个简短自我介绍。", help="User prompt") + parser.add_argument("--prompt-file", default=None, help="Read user prompt from a UTF-8 text file") + parser.add_argument("--max-layers", type=int, default=None, help="Only load the first N decoder layers") + parser.add_argument( + "--limit-prompt-tokens", + type=int, + default=None, + help="Only consume the first N prompt tokens in decode_replay mode", + ) + parser.add_argument( + "--prompt-mode", + default="decode_replay", + choices=["decode_replay", "prefill"], + help="How to consume the prompt before generation", + ) + parser.add_argument("--max-new-tokens", type=int, default=16, help="Generation length for --mode generate") + parser.add_argument("--kv-cache-len", type=int, default=255, help="Decode KV cache length used at compile time") + parser.add_argument("--verbose", action="store_true", help="Print per-layer tensor stats") + return parser.parse_args() + + +def main(): + args = parse_args() + if args.prompt_file: + with open(args.prompt_file, "r", encoding="utf-8") as f: + args.prompt = f.read() + runner = MiniCPMTextAxModelRunner( + hf_model=args.hf_model, + axmodel_dir=args.axmodel_dir, + embed_bin=args.embed_bin, + max_layers=args.max_layers, + kv_cache_len=args.kv_cache_len, + ) + try: + token_ids = runner.tokenize_prompt(args.prompt) + print("prompt_token_count:", len(token_ids)) + print("prompt_token_ids:", token_ids) + print("prompt_template_repr:", runner.decode_tokens(token_ids).encode("unicode_escape").decode()) + + if args.mode == "inspect": + runner.inspect() + return + + if args.mode == "prefill": + hidden = runner.run_prefill(token_ids, verbose=args.verbose) + print("prefill_last_hidden:", tensor_stats(hidden)) + logits = runner.run_post(hidden) + print("post_logits:", tensor_stats(logits)) + print("greedy_next_token:", runner.greedy_next_token(hidden)) + return + + if args.mode == "decode_replay": + _, _, hidden = runner.decode_replay_prompt( + token_ids, + limit_prompt_tokens=args.limit_prompt_tokens, + verbose=args.verbose, + ) + print("decode_replay_last_hidden:", tensor_stats(hidden)) + logits = runner.run_post(hidden) + print("post_logits:", tensor_stats(logits)) + print("greedy_next_token:", runner.greedy_next_token(hidden)) + return + + runner.generate( + token_ids, + max_new_tokens=args.max_new_tokens, + prompt_mode=args.prompt_mode, + verbose=args.verbose, + ) + finally: + runner.close() + + +if __name__ == "__main__": + main() diff --git a/python/infer_torch.py b/python/infer_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..340d6658cfd8a6e0cc45b9e3c5c916e559dae24a --- /dev/null +++ b/python/infer_torch.py @@ -0,0 +1,93 @@ +import argparse + +import torch +from transformers import AutoModelForImageTextToText, AutoProcessor + + +DEFAULT_MODEL_PATH = ( + "/data/tmp/yongqiang/nfs/auto_model_deployment/" + "Minicpm-V-4.6-hf-original/MiniCPM-V-4.6" +) + + +def resolve_dtype(name: str): + if name == "auto": + return "auto" + return { + "float32": torch.float32, + "float16": torch.float16, + "bfloat16": torch.bfloat16, + }[name] + + +def resolve_device(name: str): + if name != "auto": + return name + return "cuda:0" if torch.cuda.is_available() else "cpu" + + +def main(): + parser = argparse.ArgumentParser(description="MiniCPM-V-4.6 official torch text-only inference") + parser.add_argument("--model-path", default=DEFAULT_MODEL_PATH) + parser.add_argument("--prompt", default="你好，请做一个简短自我介绍。") + parser.add_argument("--device", default="auto") + parser.add_argument("--dtype", default="auto", choices=["auto", "float32", "float16", "bfloat16"]) + parser.add_argument("--attn-implementation", default="eager") + parser.add_argument("--max-new-tokens", type=int, default=64) + parser.add_argument("--do-sample", action="store_true") + args = parser.parse_args() + + device = resolve_device(args.device) + dtype = resolve_dtype(args.dtype) + + processor = AutoProcessor.from_pretrained(args.model_path) + + load_kwargs = { + "attn_implementation": args.attn_implementation, + "device_map": device, + } + if dtype != "auto": + load_kwargs["dtype"] = dtype + + model = AutoModelForImageTextToText.from_pretrained(args.model_path, **load_kwargs).eval() + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": args.prompt}, + ], + } + ] + + inputs = processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + ).to(model.device) + + prompt_text = processor.tokenizer.decode(inputs.input_ids[0], skip_special_tokens=False) + print("model_path:", args.model_path) + print("device:", model.device) + print("input_ids.shape:", tuple(inputs.input_ids.shape)) + print("input_ids:", inputs.input_ids[0].tolist()) + print("prompt_template_repr:", prompt_text.encode("unicode_escape").decode()) + + with torch.inference_mode(): + generated_ids = model.generate( + **inputs, + max_new_tokens=args.max_new_tokens, + do_sample=args.do_sample, + ) + + trimmed_ids = generated_ids[0, inputs.input_ids.shape[-1] :] + output_text = processor.decode(trimmed_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) + + print("generated_ids:", trimmed_ids.tolist()) + print("output_text:", output_text) + + +if __name__ == "__main__": + main() diff --git a/python/minicpm_v46_tokenizer/.gitattributes b/python/minicpm_v46_tokenizer/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..52373fe24473b1aa44333d318f578ae6bf04b49b --- /dev/null +++ b/python/minicpm_v46_tokenizer/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/python/minicpm_v46_tokenizer/README.md b/python/minicpm_v46_tokenizer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..80a14d84d0ac0a0d00cd27f02adb8888d14d15eb --- /dev/null +++ b/python/minicpm_v46_tokenizer/README.md @@ -0,0 +1,498 @@ +--- +license: apache-2.0 +pipeline_tag: image-text-to-text +tags: +- minicpm-v +- multimodal +- On-Device Model +- lightweight +--- + +A Pocket-Sized MLLM for Ultra-Efficient Image and Video Understanding on Your Phone + +[GitHub](https://github.com/OpenBMB/MiniCPM-o) | [CookBook](https://github.com/OpenSQZ/MiniCPM-V-CookBook) | [Demo](https://huggingface.co/spaces/openbmb/MiniCPM-V-4.6-Demo) | +[Feishu (Lark)](https://raw.githubusercontent.com/openbmb/MiniCPM-V/main/assets/feishu_qrcode.png) + +## MiniCPM-V 4.6 + +**MiniCPM-V 4.6** is our most edge-deployment-friendly model to date. The model is built based on SigLIP2-400M and the Qwen3.5-0.8B LLM. It inherits the strong single-image, multi-image, and video understanding capabilities of MiniCPM-V family, while significantly improving computation efficiency. It also introduces mixed 4x/16x visual token compression. Notable features of MiniCPM-V 4.6 include: + +- 🔥 **Leading Foundation Capability.** + MiniCPM-V 4.6 scores 13 on the Artificial Analysis Intelligence Index benchmark, outperforming Qwen3.5-0.8B's score of 10 with 19x fewer token cost, and Qwen3.5-0.8B-Thinking's score of 11 with 43x fewer token cost. It also surpasses the larger Ministral 3 3B (score of 11). + +- 💪 **Strong Multimodal Capability.** + MiniCPM-V 4.6 outperforms Qwen3.5-0.8B on most vision-language understanding tasks, and reaches Qwen3.5 2B-level capability on many benchmarks including OpenCompass, RefCOCO, HallusionBench, MUIRBench, and OCRBench. +- 🚀 **Ultra-Efficient Architecture.** + Based on the latest technique in [LLaVA-UHD v4](https://github.com/THUMAI-Lab/LLaVA-UHD-v4), MiniCPM-V 4.6 reduces the visual encoding computation FLOPs by more than 50%. It enables MiniCPM-V 4.6 to achieve better efficiency to even smaller models, achieving ~1.5x token throughput compared to Qwen3.5-0.8B. + It also supports mixed 4x/16x visual token compression rate, allowing flexible switching between accuracy and speed. +- 📱 **Broad Mobile Platform Coverage.** + MiniCPM-V 4.6 can be deployed across all three mainstream mobile platforms — iOS, Android, and HarmonyOS. With every edge adaptation code open-sourced, developers can reproduce the on-device experience in [just a few steps](#deploy-minicpm-v-46-on-ios-android-and-harmonyos-platforms). +- 🛠️ **Developer Friendly.** + MiniCPM-V 4.6 is adapted to [inference frameworks](#inference-and-training) such as vLLM, SGLang, llama.cpp, Ollama, and supports [fine-tuning ecosystems](#inference-and-training) such as SWIFT and LLaMA-Factory. Developers can quickly customize models for new domains and tasks on consumer-grade GPUs. We provide multiple quantized variants across GGUF, BNB, AWQ, and GPTQ formats. + + +### Evaluation + +**Overall Performance (Instruct)** + +

+ +

+ + +
+Click to view MiniCPM-V 4.6-Thinking performance. + + +

+ +

+ + +
+ + +
+Click to view MiniCPM-V 4.6 inference efficiency results. + + +**High-Concurrency Throughput** + +

+ +

+ +**Single Request TTFT (ms)** + +

+ +

+ + +
+ + +### Examples + +#### Overall + +
+ +
+ +MiniCPM-V 4.6 can be deployed across three mainstream end-side platforms — **iOS, Android and HarmonyOS**. The clips below are raw screen recordings on phone devices without edition. + + + + + + + + + + + + +
iPhone
iPhone 17 Pro Max		Android
Redmi K70		HarmonyOS
HUAWEI nova 14
+ + +### Usages + +#### Inference with Transformers +##### Installation + +```bash +pip install "transformers[torch]>=5.7.0" torchvision torchcodec +``` + +> **Note on CUDA compatibility:** `torchcodec` (used for video decoding) may have compatibility issues with certain CUDA versions. For example, `torch>=2.11` bundles CUDA 13.1 by default, while environments with CUDA 12.x may encounter errors such as `RuntimeError: Could not load libtorchcodec`. Two workarounds: +> +> 1. **Replace `torchcodec` with `PyAV`** — supports both image and video inference without CUDA version constraints: +> ```bash +> pip install "transformers[torch]>=5.7.0" torchvision av +> ``` +> 2. **Pin the CUDA version** when installing torch to match your environment (e.g. CUDA 12.8): +> ```bash +> pip install "transformers>=5.7.0" torchvision torchcodec --index-url https://download.pytorch.org/whl/cu128 +> ``` + +##### Load Model + +```python +from transformers import AutoModelForImageTextToText, AutoProcessor + +model_id = "openbmb/MiniCPM-V-4.6" + +processor = AutoProcessor.from_pretrained(model_id) +model = AutoModelForImageTextToText.from_pretrained( + model_id, torch_dtype="auto", device_map="auto" +) + +# Flash Attention 2 is recommended for better acceleration and memory saving, +# especially in multi-image and video scenarios. +# model = AutoModelForImageTextToText.from_pretrained( +# model_id, +# torch_dtype=torch.bfloat16, +# attn_implementation="flash_attention_2", +# device_map="auto", +# ) +``` + +##### Image Inference + +```python +messages = [ + { + "role": "user", + "content": [ + {"type": "image", "url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}, + {"type": "text", "text": "What causes this phenomenon?"}, + ], + } +] + +downsample_mode = "16x" # Using `downsample_mode="4x"` for Finer Detail + +inputs = processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, + return_dict=True, return_tensors="pt", + downsample_mode=downsample_mode, + max_slice_nums=36, +).to(model.device) + +generated_ids = model.generate(**inputs, downsample_mode=downsample_mode, max_new_tokens=512) +generated_ids_trimmed = [ + out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) +] +output_text = processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False +) +print(output_text[0]) +``` + +##### Video Inference + +```python +messages = [ + { + "role": "user", + "content": [ + {"type": "video", "url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/football.mp4"}, + {"type": "text", "text": "Describe this video in detail. Follow the timeline and focus on on-screen text, interface changes, main actions, and scene changes."}, + ], + } +] + +downsample_mode = "16x" # Using `downsample_mode="4x"` for Finer Detail + +inputs = processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, + return_dict=True, return_tensors="pt", + downsample_mode=downsample_mode, + max_num_frames=128, + stack_frames=1, + max_slice_nums=1, + use_image_id=False, +).to(model.device) + +generated_ids = model.generate(**inputs, downsample_mode=downsample_mode, max_new_tokens=2048) +generated_ids_trimmed = [ + out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) +] +output_text = processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False +) +print(output_text[0]) +``` + +##### Advanced Parameters + +You can customize image/video processing by passing additional parameters to `apply_chat_template`: + +| Parameter | Default | Applies to | Description | +|-----------|---------|------------|-------------| +| `downsample_mode` | `"16x"` | Image & Video | Visual token downsampling. `"16x"` merges tokens for efficiency; `"4x"` keeps 4× more tokens for finer detail. Must also be passed to `generate()`. | +| `max_slice_nums` | `9` | Image & Video | Maximum number of slices when splitting a high-resolution image. Higher values preserve more detail for large images. Recommended: `36` for image, `1` for video. | +| `max_num_frames` | `128` | Video only | Maximum number of main frames sampled from the video. | +| `stack_frames` | `1` | Video only | Total sample points per second. `1` = main frame only (no stacking). `N` (N>1) = 1 main frame + N−1 sub-frames per second; the sub-frames are composited into a grid image and interleaved with main frames. Recommended setting is `1` for short videos, and `3` or `5` for long videos. | +| `use_image_id` | `True` | Image & Video | Whether to prepend `N` tags before each image/frame placeholder. Set `True` for image, `False` for video. | + +> **Note:** `downsample_mode` must be passed to **both** `apply_chat_template` (for correct placeholder count) and `generate` (for the vision encoder). All other parameters only need to be passed to `apply_chat_template`. + +##### Serving with `transformers serve` + +Hugging Face Transformers includes a lightweight OpenAI-compatible server for quick testing and moderate-load deployment. + +```bash +pip install "transformers[serving]>=5.7.0" +``` + +Start the server: + +```bash +transformers serve openbmb/MiniCPM-V-4.6 --port 8000 --host 0.0.0.0 --continuous-batching +``` + +Send a request: + +```bash +curl -s http://localhost:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ] + }] + }' +``` + +Tool calling example: + +```bash +curl -s http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "the weather of Beijing"} + ]}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"} + }, + "required": ["location"] + } + } + }] +}' +``` + +The model returns a natural-language explanation followed by a structured block embedded in the content field. Note that a dedicated tool call parser for this format has not yet been added to the transformers library, so the tool calls need to be extracted manually via regex for now. + +``` +{ + "id": "f4f09c7d-8045-4cb1-ade9-07aa5dee637d", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "I need to check the current weather for Beijing, so I will call the get_weather function.\n\n\n\n\nBeijing\n\n\n", + "role": "assistant" + } + } + ], + "created": 1778748859, + "model": "openbmb/MiniCPM-V-4.6@main", + "object": "chat.completion", + "usage": { + "completion_tokens": 47, + "prompt_tokens": 283, + "total_tokens": 330 + } +} +``` + +#### Handling Escaped Newlines in Model Outputs + +In some cases, the model might output escaped newline characters `\n` as string literals instead of actual newlines. To render the text correctly, especially in UI layers, you can use the following utility function. This function carefully replaces literal `\n` with real newlines while protecting scenarios where `\n` has specific semantic meaning. + +**Utility Function:** + +```python +import re + +_PATTERN = re.compile( + r'(```[\s\S]*?```' # fenced code blocks + r'|`[^`]+`' # inline code + r'|\$\$[\s\S]*?\$\$' # display math + r'|\$[^$]+\$' # inline math + r'|\\\([\s\S]*?\\\)' # \(...\) + r'|\\\[[\s\S]*?\\\]' # \[...\] + r')' + r'|(? str: + """ + Lightweight post-processing: Converts literal '\\n' to actual newlines, + while protecting code blocks, inline code, and LaTeX commands. + """ + if not isinstance(text, str) or "\\" not in text: + return text + return _PATTERN.sub(lambda m: m.group(1) or '\n', text) +``` + +#### Deploy MiniCPM-V 4.6 on iOS, Android, and HarmonyOS Platforms + +We have adapted MiniCPM-V 4.6 for deployment on **iOS, Android, and HarmonyOS** platforms, with **all edge adaptation code fully open-sourced**. Developers can reproduce the on-device experience in just a few steps. Visit our [edge deployment repository](https://github.com/OpenBMB/MiniCPM-V-edge-demo) for platform-specific build guides, or go to the [download page](https://github.com/OpenBMB/MiniCPM-V-edge-demo/blob/main/DOWNLOAD.md) to try pre-built apps directly. + + +#### Use MiniCPM-V 4.6 in Other Inference and Training Frameworks + +MiniCPM-V 4.6 supports multiple inference and training frameworks. Below are quick-start commands for each. For full details, see our [Cookbook](https://github.com/OpenSQZ/MiniCPM-V-CookBook). + +
+vLLM — Full Guide + +```bash +vllm serve openbmb/MiniCPM-V-4.6 \ + --port 8000 \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --default-chat-template-kwargs '{"enable_thinking": false}' +``` + +> **Note:** `--enable-auto-tool-choice` and `--tool-call-parser qwen3_coder` enable tool/function calling support. If you don't need tool use, you can omit these flags and simply run `vllm serve openbmb/MiniCPM-V-4.6`. + +```bash +curl -s http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ]}] +}' +``` + + +Tool calling example: + +```bash +curl -s http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "北京的天气"} + ]}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"} + }, + "required": ["location"] + } + } + }] +}' +``` + +
+ +
+SGLang — Full Guide + +```bash +python -m sglang.launch_server --model openbmb/MiniCPM-V-4.6 --port 30000 +``` + +```bash +curl -s http://localhost:30000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openbmb/MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ]}] +}' +``` + +
+ +
+llama.cpp — Full Guide + +```bash +llama-server -m MiniCPM-V-4.6-Q4_K_M.gguf --port 8080 +``` + +```bash +curl -s http://localhost:8080/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "MiniCPM-V-4.6", + "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}}, + {"type": "text", "text": "What causes this phenomenon?"} + ]}] +}' +``` + +
+ +
+Ollama — Full Guide + +```bash +ollama run minicpm-v-4.6 +``` + +In the interactive session, paste an image path or URL directly to chat with the model. + +
+ +
+LLaMA-Factory (Fine-tuning) — Full Guide + +```bash +llamafactory-cli train examples/train_lora/minicpmv4_6_lora_sft.yaml +``` + +
+ +
+ms-swift (Fine-tuning) — Full Guide + +```bash +swift sft --model_type minicpm-v-4_6 --dataset +``` + +
+ +## License + +#### Model License +* The MiniCPM-o/V model weights and code are open-sourced under the [Apache-2.0](https://github.com/OpenBMB/MiniCPM-V/blob/main/LICENSE) license. + +#### Statement +* As MLLMs, MiniCPM-o/V models generate content by learning a large number of multimodal corpora, but they cannot comprehend, express personal opinions, or make value judgements. Anything generated by MiniCPM-o/V models does not represent the views and positions of the model developers +* We will not be liable for any problems arising from the use of MiniCPM-o/V models, including but not limited to data security issues, risk of public opinion, or any risks and problems arising from the misdirection, misuse, dissemination, or misuse of the model. + + +## Technical Reports and Key Techniques Papers + +👏 Welcome to explore key techniques of MiniCPM-o/V and other multimodal projects of our team: + +**Technical Reports:** [MiniCPM-o 4.5](https://huggingface.co/papers/2604.27393) | [MiniCPM-V 4.5](https://arxiv.org/abs/2509.18154) | [MiniCPM-o 2.6](https://openbmb.notion.site/MiniCPM-o-2-6-A-GPT-4o-Level-MLLM-for-Vision-Speech-and-Multimodal-Live-Streaming-on-Your-Phone-185ede1b7a558042b5d5e45e6b237da9) | [MiniCPM-Llama3-V 2.5](https://arxiv.org/abs/2408.01800) | [MiniCPM-V 2.0](https://openbmb.vercel.app/minicpm-v-2) + +**Other Multimodal Projects:** [VisCPM](https://github.com/OpenBMB/VisCPM/tree/main) | [RLPR](https://github.com/OpenBMB/RLPR) | [RLHF-V](https://github.com/RLHF-V/RLHF-V) | [LLaVA-UHD](https://github.com/thunlp/LLaVA-UHD) | [RLAIF-V](https://github.com/RLHF-V/RLAIF-V) | [LLaVA-UHD-v4](https://arxiv.org/abs/2605.08985 ) + + +## Citation + +If you find our model/code/paper helpful, please consider citing our papers 📝 and staring us ⭐️！ + +```bib +@proceedings{yu2025minicpmv45cookingefficient, + title={MiniCPM-V 4.5: Cooking Efficient MLLMs via Architecture, Data, and Training Recipe}, + author={Tianyu Yu and Zefan Wang and Chongyi Wang and Fuwei Huang and Wenshuo Ma and Zhihui He and Tianchi Cai and Weize Chen and Yuxiang Huang and Yuanqian Zhao and others}, + year={2025}, + url={https://arxiv.org/abs/2509.18154}, +} + +@article{yao2024minicpm, + title={MiniCPM-V: A GPT-4V Level MLLM on Your Phone}, + author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others}, + journal={arXiv preprint arXiv:2408.01800}, + year={2024} +} +``` \ No newline at end of file diff --git a/python/minicpm_v46_tokenizer/chat_template.jinja b/python/minicpm_v46_tokenizer/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..f25b6ac32579e33b2d7041442e76d9137a0311fa --- /dev/null +++ b/python/minicpm_v46_tokenizer/chat_template.jinja @@ -0,0 +1,145 @@ +{%- if enable_thinking is not defined -%} + {%- set enable_thinking = false -%} +{%- endif -%} +{%- macro render_content(content, is_system_content=false) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is iterable and content is not mapping -%} + {%- set ns = namespace(parts=[]) -%} + {%- for item in content -%} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%} + {%- if is_system_content -%} + {{- raise_exception('System message cannot contain images.') -}} + {%- endif -%} + {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%} + {%- elif 'video' in item or item.type == 'video' -%} + {%- if is_system_content -%} + {{- raise_exception('System message cannot contain videos.') -}} + {%- endif -%} + {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%} + {%- elif 'text' in item -%} + {%- set ns.parts = ns.parts + [item.text] -%} + {%- else -%} + {{- raise_exception('Unexpected item type in content.') -}} + {%- endif -%} + {%- endfor -%} + {{- ns.parts | join('\n') -}} + {%- elif content is none or content is undefined -%} + {{- '' -}} + {%- else -%} + {{- raise_exception('Unexpected content type.') -}} + {%- endif -%} +{%- endmacro -%} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/python/minicpm_v46_tokenizer/config.json b/python/minicpm_v46_tokenizer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dbe7f74b10746936eb2637ba14f5bf5515b5721c --- /dev/null +++ b/python/minicpm_v46_tokenizer/config.json @@ -0,0 +1,90 @@ +{ + "architectures": [ + "MiniCPMV4_6ForConditionalGeneration" + ], + "bos_token_id": null, + "drop_vision_last_layer": false, + "eos_token_id": 248044, + "image_size": 1120, + "model_type": "minicpmv4_6", + "pad_token_id": null, + "tie_word_embeddings": true, + "transformers_version": "5.7.0", + "use_cache": true, + "vision_config": { + "attention_dropout": 0.0, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 980, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "minicpmv4_6_vision", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14 + }, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 3584, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 16, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "vocab_size": 248094, + "model_type": "qwen3_5_text", + "tie_word_embeddings": true + }, + "insert_layer_id": 6, + "image_token_id": 248056, + "video_token_id": 248057 +} diff --git a/python/minicpm_v46_tokenizer/generation_config.json b/python/minicpm_v46_tokenizer/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b80dd7cba268869847631d044ba360aeff64ba7c --- /dev/null +++ b/python/minicpm_v46_tokenizer/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 248045, + "do_sample": true, + "eos_token_id": [ + 248044, + 248046 + ], + "temperature": 0.7, + "top_k": 0, + "top_p": 1.0, + "repetition_penalty": 1.0, + "transformers_version": "5.7.0" +} diff --git a/python/minicpm_v46_tokenizer/preprocessor_config.json b/python/minicpm_v46_tokenizer/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8619291054b6d574c16d9903c0070aa61c73bd0 --- /dev/null +++ b/python/minicpm_v46_tokenizer/preprocessor_config.json @@ -0,0 +1,19 @@ +{ + "image_processor_type": "MiniCPMV4_6ImageProcessor", + "processor_class": "MiniCPMV4_6Processor", + "max_slice_nums": 9, + "scale_resolution": 448, + "patch_size": 14, + "use_image_id": true, + "slice_mode": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ] +} \ No newline at end of file diff --git a/python/minicpm_v46_tokenizer/tokenizer.json b/python/minicpm_v46_tokenizer/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..20e07d534707e3e879933e9f39dfb631ff1cbd9a --- /dev/null +++ b/python/minicpm_v46_tokenizer/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33861e37bb955af1e3f3061182b820f347eba2b9c2c1011c82794bf0d6e77b54 +size 19992481 diff --git a/python/minicpm_v46_tokenizer/tokenizer_config.json b/python/minicpm_v46_tokenizer/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9b6fab078fcc380bdd4ab6fa8d9ac3310d6709d9 --- /dev/null +++ b/python/minicpm_v46_tokenizer/tokenizer_config.json @@ -0,0 +1,42 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": "<|im_start|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": { + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "image_start_token": "", + "image_end_token": "", + "slice_start_token": "", + "slice_end_token": "", + "image_id_start_token": "", + "image_id_end_token": "" + }, + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "unk_token": "", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>", + "chat_template": "{%- if enable_thinking is not defined -%}\n {%- set enable_thinking = false -%}\n{%- endif -%}\n{%- macro render_content(content, is_system_content=false) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is iterable and content is not mapping -%}\n {%- set ns = namespace(parts=[]) -%}\n {%- for item in content -%}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n {%- if is_system_content -%}\n {{- raise_exception('System message cannot contain images.') -}}\n {%- endif -%}\n {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}\n {%- elif 'video' in item or item.type == 'video' -%}\n {%- if is_system_content -%}\n {{- raise_exception('System message cannot contain videos.') -}}\n {%- endif -%}\n {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}\n {%- elif 'text' in item -%}\n {%- set ns.parts = ns.parts + [item.text] -%}\n {%- else -%}\n {{- raise_exception('Unexpected item type in content.') -}}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.parts | join('\\n') -}}\n {%- elif content is none or content is undefined -%}\n {{- '' -}}\n {%- else -%}\n {{- raise_exception('Unexpected content type.') -}}\n {%- endif -%}\n{%- endmacro -%}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content)|trim %}\n {%- if not(content.startswith('') and content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n\\n\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n' }}\n {%- endif %}\n{%- endif %}\n", + "tokenizer_class": "Qwen2Tokenizer" +} \ No newline at end of file diff --git a/qwen3_5_text_p128_l0_together.axmodel b/qwen3_5_text_p128_l0_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..870e9a710667e97a4b75129e9b54770e16a0f73f --- /dev/null +++ b/qwen3_5_text_p128_l0_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a9ecabeb93b62bf2e26a305c148bcc3a7a2d9fe39ba2ab05e091e284081b64b +size 27682449 diff --git a/qwen3_5_text_p128_l10_together.axmodel b/qwen3_5_text_p128_l10_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..b36b2e21d4dc8935835e2361e9ac25a46ae9e4d4 --- /dev/null +++ b/qwen3_5_text_p128_l10_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6402aa3504b0b4778042e2e23b56765f326666a18598485dd40931e026516eb +size 27682081 diff --git a/qwen3_5_text_p128_l11_together.axmodel b/qwen3_5_text_p128_l11_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1cd5fada3ff3d42f05d0a39a1ccc8b21cd343e22 --- /dev/null +++ b/qwen3_5_text_p128_l11_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a0609ea0ed4a14e3873cd8d6db4b8efa370b31a6beb3e0ca9e21be54334e14d +size 24273573 diff --git a/qwen3_5_text_p128_l12_together.axmodel b/qwen3_5_text_p128_l12_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..2409efd864d5927bb534611d3cd962c52a023f95 --- /dev/null +++ b/qwen3_5_text_p128_l12_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855333dd407cbaec53bd34cdfccaaf369e9c881c4e8912e5216f5ca2ccb4156c +size 27682081 diff --git a/qwen3_5_text_p128_l13_together.axmodel b/qwen3_5_text_p128_l13_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..7731ba75bc3972c8dd3975fa975cb562d7a0d12d --- /dev/null +++ b/qwen3_5_text_p128_l13_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:800b592829ebfdd4e797a568217a07afbd0501ce65e159561db908996572cfc3 +size 27682177 diff --git a/qwen3_5_text_p128_l14_together.axmodel b/qwen3_5_text_p128_l14_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a4b3b87253faee69a9c86d22d35492d175ec4b8b --- /dev/null +++ b/qwen3_5_text_p128_l14_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feacb5c876ff67054bb9a50233956241cd7cedc035b5ce7ba059a033441c6a61 +size 27682081 diff --git a/qwen3_5_text_p128_l15_together.axmodel b/qwen3_5_text_p128_l15_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..d3a6f79b9b3f1b4e90e58dcc4e8b524ffdf439b2 --- /dev/null +++ b/qwen3_5_text_p128_l15_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0decfb1327156025b6e98a09b03ee353a2ba960db73ba00274e3b500ce486023 +size 24273573 diff --git a/qwen3_5_text_p128_l16_together.axmodel b/qwen3_5_text_p128_l16_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a5a05e9b6f706cfd194088ee39091894a48bb461 --- /dev/null +++ b/qwen3_5_text_p128_l16_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:484a2738f1b30aeaeeb65f1534cfc8ed8ac8f282e8bcbbd0744f1b2cd0895f23 +size 27682081 diff --git a/qwen3_5_text_p128_l17_together.axmodel b/qwen3_5_text_p128_l17_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a30347e6de2fb0ae575e7dfb33d96625eca40049 --- /dev/null +++ b/qwen3_5_text_p128_l17_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1191dd1abceb45c2eb6a5557c681bb3c28b4bfe3f03fe26c4a7931a3e63b3fe +size 27682081 diff --git a/qwen3_5_text_p128_l18_together.axmodel b/qwen3_5_text_p128_l18_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e6b8afa9b1e89c32d0e097c164848180129fa9c2 --- /dev/null +++ b/qwen3_5_text_p128_l18_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3686669d283b26a766f976654614a79b0a2f6ce3861397eb7eff9640d8cbd7c2 +size 27682081 diff --git a/qwen3_5_text_p128_l19_together.axmodel b/qwen3_5_text_p128_l19_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..70ae95368708a0abf71902a1d18f76a2b6a0da53 --- /dev/null +++ b/qwen3_5_text_p128_l19_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bfecc5dacdf943bd5c41d8344f60c85301702afeddfc05dc69595544334dce3 +size 24273573 diff --git a/qwen3_5_text_p128_l1_together.axmodel b/qwen3_5_text_p128_l1_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..526377e6f76e06e7c7f5cb259d4026c32fb2a588 --- /dev/null +++ b/qwen3_5_text_p128_l1_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c318f65070ec6456f256640f424327d557407e4fa89b6e46f8ec79077bc5cf67 +size 27682353 diff --git a/qwen3_5_text_p128_l20_together.axmodel b/qwen3_5_text_p128_l20_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e092c6f03ac3b01122cad430b25a1989200b611e --- /dev/null +++ b/qwen3_5_text_p128_l20_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c3974b9cbacc63d02474dc6bebef7cec276a427f38c6db47ffa8a3ca905ea69 +size 27682081 diff --git a/qwen3_5_text_p128_l21_together.axmodel b/qwen3_5_text_p128_l21_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..6f39ff2745d9bf3a714000f9c88ac085b6b74670 --- /dev/null +++ b/qwen3_5_text_p128_l21_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c828b6e0243e02f7c564875ccdacbfee6275272aef8f5d2a48ac51648b51104 +size 27682081 diff --git a/qwen3_5_text_p128_l22_together.axmodel b/qwen3_5_text_p128_l22_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..73cedcaae17830435f1d2980f045be7401369e20 --- /dev/null +++ b/qwen3_5_text_p128_l22_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bb656a1c19f4e47cc6f757b8afdc976d94f7716fb636468bcd01b36ad43971a +size 27682081 diff --git a/qwen3_5_text_p128_l23_together.axmodel b/qwen3_5_text_p128_l23_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..c9294091a23a537f4006ef5ffbf3527ad9cba5db --- /dev/null +++ b/qwen3_5_text_p128_l23_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdcfa32c4b8c3c6a52141814eff13c8497a064c0cfdb38811433ad8905b63495 +size 24273573 diff --git a/qwen3_5_text_p128_l2_together.axmodel b/qwen3_5_text_p128_l2_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..76c8f9da17c3c29fbb50ed41b89a6ad1fb42ebfc --- /dev/null +++ b/qwen3_5_text_p128_l2_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:553dc8a2c4825eed7c435b393853c4757e0fdbda198be70622eaf0a002ca5d3a +size 27682065 diff --git a/qwen3_5_text_p128_l3_together.axmodel b/qwen3_5_text_p128_l3_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..25e319ac09e0657f4e0e3bd3670e3f1328350d46 --- /dev/null +++ b/qwen3_5_text_p128_l3_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccfbe80d6db2ffd8611650ee29a7c582da9bc92a7f6423b0d1bccb432dfb3462 +size 24273493 diff --git a/qwen3_5_text_p128_l4_together.axmodel b/qwen3_5_text_p128_l4_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..b181430bb07c96ab9c5466e091436b3f1b723678 --- /dev/null +++ b/qwen3_5_text_p128_l4_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b59af8b13cc09897634f780511a2bf5a3125190339e4ebf5325158ee892e05ef +size 27682065 diff --git a/qwen3_5_text_p128_l5_together.axmodel b/qwen3_5_text_p128_l5_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..125be5cf84e838fd5babdb634851b22cec76ff22 --- /dev/null +++ b/qwen3_5_text_p128_l5_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b87fef59116bb884f56f7f7f624662535961a22c4038b86a82c309dc53f0f5 +size 27682161 diff --git a/qwen3_5_text_p128_l6_together.axmodel b/qwen3_5_text_p128_l6_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..f76123fb7416580ef654ecb6cc59eae820d63d63 --- /dev/null +++ b/qwen3_5_text_p128_l6_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cc549f5c19ff7b25184ec0bf2d97015a5340f191af15ec95c1ec51cf37574fe +size 27682193 diff --git a/qwen3_5_text_p128_l7_together.axmodel b/qwen3_5_text_p128_l7_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..5526797cf68c9adbd08487c3c4f30f42974bc4f5 --- /dev/null +++ b/qwen3_5_text_p128_l7_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e635c13ade2468dcded5d741f3f0892ada76f3c8a9443159925bf5c259cf273 +size 24274357 diff --git a/qwen3_5_text_p128_l8_together.axmodel b/qwen3_5_text_p128_l8_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1af85bce819a11b8d77a1200031e2464ce537086 --- /dev/null +++ b/qwen3_5_text_p128_l8_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93d45efec7319b66572b7c6a2776a9dd4642844f1974cf8a771f0d069507a880 +size 27682257 diff --git a/qwen3_5_text_p128_l9_together.axmodel b/qwen3_5_text_p128_l9_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e924fc160e7de0a71abbbd7761d2d20a1480584a --- /dev/null +++ b/qwen3_5_text_p128_l9_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c69dbf6827046d97d6d385c2372b5bc10ca128f908bf94fd9fba43d90abb85c +size 27682193 diff --git a/qwen3_5_text_post.axmodel b/qwen3_5_text_post.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..662b3b4e333bc0c0f6e43dbef34b050ebd791abe --- /dev/null +++ b/qwen3_5_text_post.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09ca16f8db69f2b8178b1119276e0dbe6e74c29bd9fe89b621d66e63d5a91d16 +size 277422568