unsloth

/

Qwen3-Coder-30B-A3B-Instruct-GGUF

like 727

Follow

Unsloth AI 24.8k

I have posted about 150k tokens from a mod (https://github.com/sirskunkalot/PlanBuild)

and when asking the sourcebase it will respond with gibberish:

Streaming response:

Looking.9.%:
)月使用      1).月 =>月、
,:,,)
,
;
月       string?..

        Null =,

.;:)
9: %),: of月9
,
,
月基础属性
月  0.
",
.月.:9.;
      + EqualityComparer<月 Curve9S..)).;
.).,
,
),:,9,
.,:月..:..月 Func)
;


: =月,SS.,
.、 of     foreach  //) >
) | 0  }
 F =    => 0  ToString >  1：
, V1.月  1,IntPtr 1);
, return.9):float =月0.1月.900..9 with  //月 ushort  // > |)
  }
 1 = >;. and P1月.
.9, 't<br
;
 and =月Bench <|fim_suffix|>0
月 {
 |  0 | >VectorBy  = >
.月 411:.,,,0.., 00:月
月 1.)();
.AddTo<0: 1
,Buffer
：
:string =R月 ？
: Debug. of =1月, SystemB
91月.
)：

. > .P
 of
base  {
x1月:
莅  {
 >
 |1月月, Mathf
   //
. >System
)Z
,
<br istance2月月]11)
, typeof("  // BaseBase, Program
 and 1<br", new.<br1
<br  //0和社会 File, Unity.1.Space.;
       Copy月SS月1月).S.0],
               2;
();
, Get);
        this 0 the  =  1月月,B月.:


<<<<<<<月通行月月,base,e;
  varB.Invoke月
 =
00",

Instead with q4_K_M I have no issue, I suspect there is some issue with the OLLAMA_KV_CACHE_TYPE and the NL version.

import requests
import json
#Llama-3_3-Nemotron-Super-49B-v1_5-GGUF:q5_k_xl
#hf.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:q6_k
#hf.co/unsloth/Llama-3_3-Nemotron-Super-49B-v1_5-GGUF:iq4_nl
#hf.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:IQ4_NL
#hf.co/unsloth/Magistral-Small-2509-GGUF:Q6_K

#qwen3-coder:30b-a3b-q4_K_M
def send_prompt_with_file_ollama(user_input, filename, model="qwen3-coder:30b-a3b-q4_K_M",
                                 host="192.168.1.88", port=11437, output_path="output.md"):
    # Read file content
    try:
        with open(filename, 'r', encoding='utf-8') as f:
            file_content = f.read()
    except Exception as e:
        print(f"Error reading file: {e}")
        return

    # Combine prompt
    full_prompt = f"{user_input}\n\nHere is the file content:\n{file_content}"

    url = f"http://{host}:{port}/api/generate"
    payload = {
        "model": model,
        "prompt": full_prompt,
        "stream": True   # enables streaming chunks
    }

    streamed_output = ""
    print("Streaming response:\n")

    try:
        with requests.post(url, json=payload, stream=True) as response:
            response.raise_for_status()
            for line in response.iter_lines(decode_unicode=True):
                if not line.strip():
                    continue
                try:
                    data = json.loads(line)
                    # Ollama streams tokens in "response"
                    token = data.get("response", "")
                    if token:
                        print(token, end='', flush=True)
                        streamed_output += token
                    # When done, Ollama sends a JSON object with "done": true
                    if data.get("done", False):
                        break
                except json.JSONDecodeError:
                    print(f"\n[Warning: Couldn't decode line: {line}]\n")
    except Exception as e:
        print(f"Request failed: {e}")
        return

    # Save output to markdown file
    try:
        with open(output_path, 'w', encoding='utf-8') as f:
            f.write(streamed_output)
        print(f"\n\n--- Saved output to: {output_path} ---")
    except Exception as e:
        print(f"Error saving output to file: {e}")

    print("--- Done. Press Enter to exit. ---")
    input()


# Example usage
send_prompt_with_file_ollama(
    user_input="""give me a code analysis and how objects are related and what is the most nested part there""",
    filename="E:/python_projects/MinimalAIChatMod/dirs_2025-07-09_23-02-29.txt",
    output_path="E:/python_projects/MinimalAIChatMod/local_llm10.md"