Upload app.py

app.py

@@ -1,28 +1,39 @@
 from flask import Flask, render_template, request, jsonify
 from llama_cpp import Llama
-import re
+import os
 
 app = Flask(__name__)
 
-# Path to the local GGUF model weights
-MODEL_PATH = "models/oss_20b_gguf/gpt-oss-20b-Q2_K_L.gguf"  # update this path
+# Update this path to your downloaded model weight
+MODEL_PATH = "models/oss_20b_gguf/gpt-oss-20b-Q2_K_L.gguf"
 
-# Initialize model
-llm = Llama(
-    model_path=MODEL_PATH,
-    n_ctx=2048,
-    n_threads=8  # adjust based on your CPU
-)
+# Detect GPU automatically: if llama-cpp-python was compiled with CUDA/Metal and GPU layers can be offloaded
+# Adjust n_gpu_layers for your GPU memory; 20-40 for mid GPUs, 60-100 for higher VRAM, 0 = CPU only
+try:
+    print("Trying GPU offload...")
+    llm = Llama(
+        model_path=MODEL_PATH,
+        n_ctx=2048,
+        n_threads=os.cpu_count(),
+        n_gpu_layers=40  # increase or decrease based on your GPU memory
+    )
+    print("GPU initialized successfully.")
+except Exception as e:
+    print(f"GPU failed: {e}\nFalling back to CPU.")
+    llm = Llama(
+        model_path=MODEL_PATH,
+        n_ctx=2048,
+        n_threads=os.cpu_count(),
+        n_gpu_layers=0  # CPU only
+    )
 
-# Build adaptive prompt
 def build_prompt(history, user_text):
     system_prompt = (
-        "You are a helpful and adaptive assistant. Follow these rules strictly:\n"
-        "- If the user asks a simple or factual question, give a short, precise answer.\n"
-        "- If the user requests a story, essay, or letter, provide a longer, well-structured response.\n"
-        "- If the user asks for programming help or code, provide correct, complete, well-formatted code.\n"
-        "- Always keep answers clear, neat, and structured; use points when helpful.\n"
-        "- Output code inside proper Markdown code blocks with language tags for syntax highlighting.\n"
+        "You are a helpful assistant. Follow these:\n"
+        "- Simple Q: Short, precise.\n"
+        "- Story/letter/essay: Longer answer.\n"
+        "- Code: Complete, neat, Markdown fenced code with language tag.\n"
+        "- Use points when helpful.\n"
     )
     prompt = system_prompt + "\n\n"
     for turn in history:
@@ -42,7 +53,6 @@ def chat():
 
     prompt = build_prompt(history, user_message)
 
-    # Adjust max_tokens dynamically
     if any(word in user_message.lower() for word in ["story", "letter", "essay"]):
         max_out = 800
     elif any(word in user_message.lower() for word in ["code", "program", "script", "python", "java", "html", "c++"]):
@@ -57,10 +67,7 @@ def chat():
         stop=["\nUser:", "\nAssistant:"]
     )
 
-    text = resp["choices"][0]["text"].strip()
-
-    # Wrap fenced code blocks with copy button (handled in JS)
-    return jsonify({"response": text})
+    return jsonify({"response": resp["choices"][0]["text"].strip()})
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=5000, debug=True)