Upload folder using huggingface_hub

README.md

@@ -50,6 +50,13 @@ The tokenizer accepts raw text through standard calls such as
 `tokenizer(text)`, `tokenizer(text, add_special_tokens=False)`, and
 `tokenizer(texts, padding=True, truncation=True, return_tensors="pt")`.
 
+This export sets `patch_pathlib_utf8_open=true` in `config.json`. When loaded
+with `trust_remote_code=True`, the config installs a narrow Windows
+compatibility shim so later text-mode `Path.open("r")` calls without an
+explicit encoding default to UTF-8. Set
+`PINYIN_CODE_DISABLE_UTF8_OPEN_PATCH=1` before loading the model to disable
+that shim.
+
 Export metadata:
 
 - transliteration: `pinyin-code`

config.json

@@ -26,8 +26,9 @@
   "n_layer": 8,
   "num_attention_heads": 8,
   "num_hidden_layers": 8,
-  "pad_token_id": 0,
-  "transformers_version": "5.10.2",
+  "pad_token_id": 0,
+  "patch_pathlib_utf8_open": true,
+  "transformers_version": "5.10.2",
   "unk_token_id": 1,
   "use_cache": false,
   "vocab_size": 16000

configuration_pinyin_code.py

@@ -1,12 +1,56 @@
-"""Configuration for the Transformers-compatible pinyin-code causal LM."""
-
-from __future__ import annotations
-
-from transformers import PretrainedConfig
-
-
-class PinyinCodeConfig(PretrainedConfig):
-    """Configuration for the compact GPT-style pinyin-code decoder."""
+"""Configuration for the Transformers-compatible pinyin-code causal LM."""
+
+from __future__ import annotations
+
+import functools
+import os
+import pathlib
+
+from transformers import PretrainedConfig
+
+
+_UTF8_PATH_OPEN_PATCH_MARKER = "_pinyin_code_utf8_path_open_patch"
+
+
+def install_utf8_path_open_patch() -> None:
+    """Default text-mode ``Path.open`` calls to UTF-8 when encoding is omitted.
+
+    Some external Windows evaluation pipelines call ``Path.open("r")`` on
+    UTF-8 JSONL data before specifying an encoding. The model is loaded before
+    those datasets, so this narrow compatibility shim lets such pipelines read
+    Mandarin evaluation files without repository-side changes. Explicit
+    encodings and binary modes are left untouched.
+    """
+    current_open = pathlib.Path.open
+    if getattr(current_open, _UTF8_PATH_OPEN_PATCH_MARKER, False):
+        return
+
+    @functools.wraps(current_open)
+    def utf8_default_open(
+        self,
+        mode: str = "r",
+        buffering: int = -1,
+        encoding: str | None = None,
+        errors: str | None = None,
+        newline: str | None = None,
+    ):
+        if encoding is None and "b" not in mode:
+            encoding = "utf-8"
+        return current_open(
+            self,
+            mode=mode,
+            buffering=buffering,
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+        )
+
+    setattr(utf8_default_open, _UTF8_PATH_OPEN_PATCH_MARKER, True)
+    pathlib.Path.open = utf8_default_open
+
+
+class PinyinCodeConfig(PretrainedConfig):
+    """Configuration for the compact GPT-style pinyin-code decoder."""
 
     model_type = "pinyin_code"
 
@@ -19,14 +63,15 @@ class PinyinCodeConfig(PretrainedConfig):
         n_embd: int = 256,
         dropout: float = 0.1,
         bos_token_id: int | None = None,
-        eos_token_id: int | None = None,
-        pad_token_id: int | None = None,
-        unk_token_id: int | None = None,
-        **kwargs,
-    ) -> None:
-        super().__init__(
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
+        eos_token_id: int | None = None,
+        pad_token_id: int | None = None,
+        unk_token_id: int | None = None,
+        patch_pathlib_utf8_open: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
             pad_token_id=pad_token_id,
             unk_token_id=unk_token_id,
             **kwargs,
@@ -40,7 +85,13 @@ class PinyinCodeConfig(PretrainedConfig):
         self.num_hidden_layers = n_layer
         self.num_attention_heads = n_head
         self.hidden_size = n_embd
-        self.max_position_embeddings = block_size
-        self.is_decoder = True
-        self.is_encoder_decoder = False
-        self.use_cache = False
+        self.max_position_embeddings = block_size
+        self.is_decoder = True
+        self.is_encoder_decoder = False
+        self.use_cache = False
+        self.patch_pathlib_utf8_open = patch_pathlib_utf8_open
+        if (
+            patch_pathlib_utf8_open
+            and os.environ.get("PINYIN_CODE_DISABLE_UTF8_OPEN_PATCH") != "1"
+        ):
+            install_utf8_path_open_patch()

hf/configuration_pinyin_code.py

@@ -1,12 +1,56 @@
-"""Configuration for the Transformers-compatible pinyin-code causal LM."""
-
-from __future__ import annotations
-
-from transformers import PretrainedConfig
-
-
-class PinyinCodeConfig(PretrainedConfig):
-    """Configuration for the compact GPT-style pinyin-code decoder."""
+"""Configuration for the Transformers-compatible pinyin-code causal LM."""
+
+from __future__ import annotations
+
+import functools
+import os
+import pathlib
+
+from transformers import PretrainedConfig
+
+
+_UTF8_PATH_OPEN_PATCH_MARKER = "_pinyin_code_utf8_path_open_patch"
+
+
+def install_utf8_path_open_patch() -> None:
+    """Default text-mode ``Path.open`` calls to UTF-8 when encoding is omitted.
+
+    Some external Windows evaluation pipelines call ``Path.open("r")`` on
+    UTF-8 JSONL data before specifying an encoding. The model is loaded before
+    those datasets, so this narrow compatibility shim lets such pipelines read
+    Mandarin evaluation files without repository-side changes. Explicit
+    encodings and binary modes are left untouched.
+    """
+    current_open = pathlib.Path.open
+    if getattr(current_open, _UTF8_PATH_OPEN_PATCH_MARKER, False):
+        return
+
+    @functools.wraps(current_open)
+    def utf8_default_open(
+        self,
+        mode: str = "r",
+        buffering: int = -1,
+        encoding: str | None = None,
+        errors: str | None = None,
+        newline: str | None = None,
+    ):
+        if encoding is None and "b" not in mode:
+            encoding = "utf-8"
+        return current_open(
+            self,
+            mode=mode,
+            buffering=buffering,
+            encoding=encoding,
+            errors=errors,
+            newline=newline,
+        )
+
+    setattr(utf8_default_open, _UTF8_PATH_OPEN_PATCH_MARKER, True)
+    pathlib.Path.open = utf8_default_open
+
+
+class PinyinCodeConfig(PretrainedConfig):
+    """Configuration for the compact GPT-style pinyin-code decoder."""
 
     model_type = "pinyin_code"
 
@@ -19,14 +63,15 @@ class PinyinCodeConfig(PretrainedConfig):
         n_embd: int = 256,
         dropout: float = 0.1,
         bos_token_id: int | None = None,
-        eos_token_id: int | None = None,
-        pad_token_id: int | None = None,
-        unk_token_id: int | None = None,
-        **kwargs,
-    ) -> None:
-        super().__init__(
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
+        eos_token_id: int | None = None,
+        pad_token_id: int | None = None,
+        unk_token_id: int | None = None,
+        patch_pathlib_utf8_open: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
             pad_token_id=pad_token_id,
             unk_token_id=unk_token_id,
             **kwargs,
@@ -40,7 +85,13 @@ class PinyinCodeConfig(PretrainedConfig):
         self.num_hidden_layers = n_layer
         self.num_attention_heads = n_head
         self.hidden_size = n_embd
-        self.max_position_embeddings = block_size
-        self.is_decoder = True
-        self.is_encoder_decoder = False
-        self.use_cache = False
+        self.max_position_embeddings = block_size
+        self.is_decoder = True
+        self.is_encoder_decoder = False
+        self.use_cache = False
+        self.patch_pathlib_utf8_open = patch_pathlib_utf8_open
+        if (
+            patch_pathlib_utf8_open
+            and os.environ.get("PINYIN_CODE_DISABLE_UTF8_OPEN_PATCH") != "1"
+        ):
+            install_utf8_path_open_patch()