Refresh GGUF quickstart runner

quickstart.py

@@ -24,9 +24,22 @@ def _build_parser() -> argparse.ArgumentParser:
         help="Start an interactive terminal chat instead of running a single prompt.",
     )
     parser.add_argument("--max-new-tokens", type=int, default=192)
+    parser.add_argument("--ctx-size", type=int, default=4096)
     parser.add_argument("--temperature", type=float, default=0.1)
     parser.add_argument("--threads", type=int, default=max(1, os.cpu_count() or 1))
     parser.add_argument("--llama-cli", default="", help="Optional explicit path to llama-cli.")
+    parser.add_argument(
+        "--cache-local",
+        action="store_true",
+        help="Copy the GGUF into LOCALAPPDATA before running. Useful when a runtime cannot read network paths.",
+    )
+    parser.add_argument("--reasoning", choices=("on", "off", "auto"), default="off")
+    parser.add_argument(
+        "--reasoning-format",
+        choices=("auto", "none", "deepseek", "deepseek-legacy"),
+        default="auto",
+    )
+    parser.add_argument("--reasoning-budget", type=int, default=None)
     return parser
 
 
@@ -40,8 +53,8 @@ def _preferred_gguf(root: Path) -> Path:
     return gguf_candidates[0]
 
 
-def _local_model_path(model_path: Path) -> Path:
-    if not str(model_path).startswith("\\"):
+def _local_model_path(model_path: Path, *, cache_local: bool = False) -> Path:
+    if not cache_local:
         return model_path
     local_app_data = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
     cache_dir = local_app_data / "tinyluminax" / "gguf-cache"
@@ -99,6 +112,7 @@ def _run_llama_cpp_python(
     system_prompt: str,
     user_prompt: str,
     max_new_tokens: int,
+    ctx_size: int,
     temperature: float,
     threads: int,
 ) -> str:
@@ -106,7 +120,7 @@ def _run_llama_cpp_python(
 
     llm = Llama(
         model_path=str(model_path),
-        n_ctx=8192,
+        n_ctx=ctx_size,
         n_threads=threads,
         n_gpu_layers=0,
         chat_format="chat_template.default",
@@ -130,30 +144,41 @@ def _run_llama_cli(
     system_prompt: str,
     user_prompt: str,
     max_new_tokens: int,
+    ctx_size: int,
     temperature: float,
     threads: int,
+    reasoning: str,
+    reasoning_format: str,
+    reasoning_budget: int | None,
 ) -> None:
+    command = [
+        str(llama_cli_path),
+        "-m",
+        str(model_path),
+        "-sys",
+        system_prompt,
+        "-p",
+        user_prompt,
+        "-cnv",
+        "-st",
+        "-n",
+        str(max_new_tokens),
+        "-c",
+        str(ctx_size),
+        "--reasoning",
+        reasoning,
+        "--temp",
+        str(temperature),
+        "--threads",
+        str(threads),
+        "--no-display-prompt",
+    ]
+    if reasoning_format != "auto":
+        command.extend(["--reasoning-format", reasoning_format])
+    if reasoning_budget is not None:
+        command.extend(["--reasoning-budget", str(reasoning_budget)])
     completed = subprocess.run(
-        [
-            str(llama_cli_path),
-            "-m",
-            str(model_path),
-            "-sys",
-            system_prompt,
-            "-p",
-            user_prompt,
-            "-cnv",
-            "-st",
-            "-n",
-            str(max_new_tokens),
-            "--reasoning",
-            "off",
-            "--temp",
-            str(temperature),
-            "--threads",
-            str(threads),
-            "--no-display-prompt",
-        ],
+        command,
         check=False,
         capture_output=True,
         text=True,
@@ -177,15 +202,19 @@ def _run_interactive_llama_cpp_python(
     model_path: Path,
     system_prompt: str,
     max_new_tokens: int,
+    ctx_size: int,
     temperature: float,
     threads: int,
     opening_prompt: str | None = None,
+    reasoning: str = "off",
+    reasoning_format: str = "auto",
+    reasoning_budget: int | None = None,
 ) -> None:
     from llama_cpp import Llama
 
     llm = Llama(
         model_path=str(model_path),
-        n_ctx=8192,
+        n_ctx=ctx_size,
         n_threads=threads,
         n_gpu_layers=0,
         chat_format="chat_template.default",
@@ -237,9 +266,13 @@ def _run_interactive_llama_cli(
     model_path: Path,
     system_prompt: str,
     max_new_tokens: int,
+    ctx_size: int,
     temperature: float,
     threads: int,
     opening_prompt: str | None = None,
+    reasoning: str = "off",
+    reasoning_format: str = "auto",
+    reasoning_budget: int | None = None,
 ) -> None:
     print("LumynaX interactive terminal chat")
     print("Interactive mode already uses llama-cli directly. Use Ctrl+C to exit.")
@@ -252,14 +285,20 @@ def _run_interactive_llama_cli(
         "-cnv",
         "-n",
         str(max_new_tokens),
+        "-c",
+        str(ctx_size),
         "--reasoning",
-        "off",
+        reasoning,
         "--temp",
         str(temperature),
         "--threads",
         str(threads),
         "--simple-io",
     ]
+    if reasoning_format != "auto":
+        command.extend(["--reasoning-format", reasoning_format])
+    if reasoning_budget is not None:
+        command.extend(["--reasoning-budget", str(reasoning_budget)])
     if opening_prompt and opening_prompt.strip():
         command.extend(["-p", opening_prompt.strip()])
     completed = subprocess.run(command, check=False)
@@ -293,23 +332,31 @@ def main() -> None:
                 )
             _run_interactive_llama_cli(
                 llama_cli_path=llama_cli_path,
-                model_path=source_model_path,
+                model_path=_local_model_path(source_model_path, cache_local=args.cache_local),
                 system_prompt=system_prompt,
                 opening_prompt=args.prompt,
                 max_new_tokens=args.max_new_tokens,
+                ctx_size=args.ctx_size,
                 temperature=args.temperature,
                 threads=args.threads,
+                reasoning=args.reasoning,
+                reasoning_format=args.reasoning_format,
+                reasoning_budget=args.reasoning_budget,
             )
             return
-        model_path = _local_model_path(source_model_path)
+        model_path = _local_model_path(source_model_path, cache_local=args.cache_local)
         try:
             _run_interactive_llama_cpp_python(
                 model_path=model_path,
                 system_prompt=system_prompt,
                 opening_prompt=args.prompt,
                 max_new_tokens=args.max_new_tokens,
+                ctx_size=args.ctx_size,
                 temperature=args.temperature,
                 threads=args.threads,
+                reasoning=args.reasoning,
+                reasoning_format=args.reasoning_format,
+                reasoning_budget=args.reasoning_budget,
             )
             return
         except Exception as exc:  # noqa: BLE001
@@ -325,12 +372,16 @@ def main() -> None:
             )
             _run_interactive_llama_cli(
                 llama_cli_path=llama_cli_path,
-                model_path=source_model_path,
+                model_path=model_path,
                 system_prompt=system_prompt,
                 opening_prompt=args.prompt,
                 max_new_tokens=args.max_new_tokens,
+                ctx_size=args.ctx_size,
                 temperature=args.temperature,
                 threads=args.threads,
+                reasoning=args.reasoning,
+                reasoning_format=args.reasoning_format,
+                reasoning_budget=args.reasoning_budget,
             )
             return
     if explicit_cli_requested:
@@ -341,15 +392,19 @@ def main() -> None:
             )
         _run_llama_cli(
             llama_cli_path=llama_cli_path,
-            model_path=source_model_path,
+            model_path=_local_model_path(source_model_path, cache_local=args.cache_local),
             system_prompt=system_prompt,
             user_prompt=single_prompt,
             max_new_tokens=args.max_new_tokens,
+            ctx_size=args.ctx_size,
             temperature=args.temperature,
             threads=args.threads,
+            reasoning=args.reasoning,
+            reasoning_format=args.reasoning_format,
+            reasoning_budget=args.reasoning_budget,
         )
         return
-    model_path = _local_model_path(source_model_path)
+    model_path = _local_model_path(source_model_path, cache_local=args.cache_local)
     try:
         print(
             _run_llama_cpp_python(
@@ -357,6 +412,7 @@ def main() -> None:
                 system_prompt=system_prompt,
                 user_prompt=single_prompt,
                 max_new_tokens=args.max_new_tokens,
+                ctx_size=args.ctx_size,
                 temperature=args.temperature,
                 threads=args.threads,
             ),
@@ -380,8 +436,12 @@ def main() -> None:
             system_prompt=system_prompt,
             user_prompt=single_prompt,
             max_new_tokens=args.max_new_tokens,
+            ctx_size=args.ctx_size,
             temperature=args.temperature,
             threads=args.threads,
+            reasoning=args.reasoning,
+            reasoning_format=args.reasoning_format,
+            reasoning_budget=args.reasoning_budget,
         )