Add Q5_0 and Q4_0 GGUF quantizations

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+paraformer-q4_0.gguf filter=lfs diff=lfs merge=lfs -text
+paraformer-q5_0.gguf filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 FunASR
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,84 @@
+---
+license: apache-2.0
+language:
+  - zh
+  - en
+base_model: funasr/paraformer-zh
+pipeline_tag: automatic-speech-recognition
+tags:
+  - paraformer
+  - funasr
+  - gguf
+  - quantized
+---
+
+# paraformer-zh GGUF
+
+Experimental low-memory GGUF quantizations of
+[`funasr/paraformer-zh`](https://huggingface.co/funasr/paraformer-zh).
+The vocabulary and CMVN tensors are embedded in each file.
+
+## Files
+
+| File | Size | Quantization | Intended use |
+| --- | ---: | --- | --- |
+| `paraformer-q5_0.gguf` | 149.7 MiB | Q5_0 matrices, remaining tensors F32 | Recommended low-memory balance |
+| `paraformer-q4_0.gguf` | 124.3 MiB | Q4_0 matrices, remaining tensors F32 | Experimental minimum-memory option |
+
+The original exporter only supports F32, F16, and Q8_0. These files were
+exported directly from the official FP32 checkpoint with an extended exporter;
+they were not requantized from Q8.
+
+## Measured behavior
+
+Measurements used the persistent CPU Paraformer GGUF runtime on an Intel
+Core i7-14700KF and a 30-second Mandarin/English mixed-language clip.
+
+| Quantization | Resident peak working set | Recognition time |
+| --- | ---: | ---: |
+| Q8_0 upstream baseline | 252.8 MiB | 996 ms |
+| Q5_0 | 176.5 MiB | 1,201 ms |
+| Q4_0 | 151.1 MiB | 1,009 ms |
+
+Across 59 clips from the same recording, Q5_0 and Q4_0 each matched the Q8_0
+token sequence exactly on 18 clips. Their aggregate token edit distance from
+Q8_0 was 8.8% and 10.1%, respectively. This is output drift relative to Q8,
+not a labeled word-error-rate measurement. Validate the quantization with your
+own language and acoustic conditions before replacing Q8 in quality-sensitive
+deployments.
+
+## Compatibility
+
+These files use the custom `paraformer` GGUF architecture implemented by the
+FunASR `runtime/llama.cpp/paraformer` runtime. They are not language models and
+cannot be loaded by general-purpose llama.cpp applications.
+
+The compatible persistent Windows runtime and its English BPE spacing patch
+are maintained in the
+[`VRChatVoiceInput`](https://github.com/AIRsLight/VRChatVoiceInput) project.
+
+## Source integrity
+
+The export input was the official `model.pt` file:
+
+```text
+size:   880502012 bytes
+sha256: 5bba782a5e9196166233b9ab12ba04cadff9ef9212b4ff6153ed9290ff679025
+```
+
+Exported files:
+
+```text
+paraformer-q5_0.gguf
+size:   156967168 bytes
+sha256: 1f2309eacd761c1f4184177c718321cd6ad3c07e7b17d6f796e5fb15565906ec
+
+paraformer-q4_0.gguf
+size:   130313216 bytes
+sha256: 992562722aa2c4e88158245a0fc5be0e1338c580db808a97c6262f6864317584
+```
+
+## License
+
+The base model and the FunASR exporter/runtime are provided under Apache-2.0.
+See `LICENSE` and the base model repository for details.

export_paraformer_low_memory_gguf.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Export Paraformer to GGUF with experimental Q5_0/Q4_0 support."""
+
+import argparse
+import glob
+import json
+import os
+import re
+
+import gguf
+import numpy as np
+import torch
+
+
+def parse_mvn(path):
+    blocks = [
+        np.array([float(x) for x in block.split()], np.float32)
+        for block in re.findall(r"\[([^\]]*)\]", open(path).read())
+    ]
+    vectors = [block for block in blocks if block.size > 1]
+    return vectors[0], vectors[1]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_pt", required=True)
+    parser.add_argument("--mvn", required=True)
+    parser.add_argument("--out", required=True)
+    parser.add_argument(
+        "--wtype",
+        default="q5_0",
+        choices=["f32", "f16", "q8_0", "q5_0", "q4_0"],
+    )
+    parser.add_argument("--tokens", default=None)
+    args = parser.parse_args()
+
+    state_dict = torch.load(args.model_pt, map_location="cpu")
+    state_dict = state_dict.get("state_dict", state_dict)
+    writer = gguf.GGUFWriter(args.out, "paraformer")
+    writer.add_uint32("pf.enc.output_size", 512)
+    writer.add_uint32("pf.enc.attention_heads", 4)
+    writer.add_uint32("pf.enc.num_blocks", 50)
+    writer.add_uint32("pf.enc.kernel_size", 11)
+    writer.add_uint32("pf.dec.num_blocks", 16)
+    writer.add_uint32("pf.dec.att_layer_num", 16)
+    writer.add_uint32("pf.dec.decoders3", 1)
+    writer.add_uint32("pf.dec.attention_heads", 4)
+    writer.add_uint32("pf.dec.kernel_size", 11)
+    writer.add_uint32("pf.vocab_size", 8404)
+
+    token_path = args.tokens or (
+        glob.glob(os.path.join(os.path.dirname(args.model_pt), "tokens.json")) + [None]
+    )[0]
+    if token_path and os.path.exists(token_path):
+        with open(token_path, encoding="utf-8") as token_file:
+            tokens = json.load(token_file)
+        writer.add_array("pf.vocab", tokens)
+        print(f"embedded pf.vocab ({len(tokens)} tokens) from {token_path}")
+    else:
+        print("WARNING: tokens.json not found - GGUF will have no vocabulary")
+
+    writer.add_float32("pf.predictor.tail_threshold", 0.45)
+    writer.add_float32("pf.predictor.threshold", 1.0)
+    shift, scale = parse_mvn(args.mvn)
+    writer.add_tensor("cmvn.shift", shift)
+    writer.add_tensor("cmvn.scale", scale)
+
+    from gguf import GGMLQuantizationType as QuantType
+    from gguf import quants
+
+    quant_types = {
+        "q8_0": QuantType.Q8_0,
+        "q5_0": QuantType.Q5_0,
+        "q4_0": QuantType.Q4_0,
+    }
+    quant_type = quant_types.get(args.wtype)
+    quant_block_size = (
+        quants._type_traits[quant_type].block_size if quant_type is not None else 1
+    )
+    tensor_count = 0
+    quantized_count = 0
+
+    for name, value in state_dict.items():
+        if not name.startswith(("encoder.", "decoder.", "predictor.")):
+            continue
+        if name == "decoder.embed.0.weight":
+            continue
+
+        array = value.detach().to(torch.float32).contiguous().numpy()
+        if name.endswith("fsmn_block.weight") and array.ndim == 3:
+            array = np.ascontiguousarray(array[:, 0, :].T)
+        elif (
+            args.wtype == "f16"
+            and array.ndim == 2
+            and "norm" not in name
+            and "cif_output" not in name
+        ):
+            array = array.astype(np.float16)
+
+        can_quantize = (
+            quant_type is not None
+            and array.ndim == 2
+            and "norm" not in name
+            and "fsmn_block" not in name
+            and "predictor" not in name
+            and array.shape[1] % quant_block_size == 0
+        )
+        if can_quantize:
+            writer.add_tensor(
+                name,
+                quants.quantize(array, quant_type),
+                raw_dtype=quant_type,
+            )
+            quantized_count += 1
+        else:
+            writer.add_tensor(name, array)
+        tensor_count += 1
+
+    print(
+        f"writing {tensor_count} tensors (+cmvn), {quantized_count} quantized "
+        f"as {args.wtype}, to {args.out}"
+    )
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file()
+    writer.close()
+    print(f"done: {args.out} ({os.path.getsize(args.out) / 1e6:.1f} MB)")
+
+
+if __name__ == "__main__":
+    main()

paraformer-q4_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992562722aa2c4e88158245a0fc5be0e1338c580db808a97c6262f6864317584
+size 130313216

paraformer-q5_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f2309eacd761c1f4184177c718321cd6ad3c07e7b17d6f796e5fb15565906ec
+size 156967168