{
"schema_version": 1,
"target": "MLX-Affine-6bit",
"source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
"validated_at": "2026-08-16T01:48:43.179373+00:00",
"structural": {
"passed": true,
"passes": [
"artifact directory exists",
"atomic build completion record",
"local SHA-256 manifest",
"all build-hashed files present (24)",
"all build payload SHA-256 hashes match",
"per-artifact quantization manifest",
"artifact manifest base model",
"artifact manifest source revision",
"artifact manifest declares vanilla quantization",
"plausible size 19.47 GB in [18, 24]",
"required sidecar config.json",
"required sidecar tokenizer_config.json",
"required sidecar generation_config.json",
"required sidecar preprocessor_config.json",
"required sidecar video_preprocessor_config.json",
"required sidecar chat_template.jinja",
"required sidecar tokenizer.json",
"required sidecar vocab.json",
"required sidecar merges.txt",
"required sidecar LICENSE",
"chat template byte-identical to source",
"generation_config.json semantically intact",
"preprocessor_config.json semantically intact",
"video_preprocessor_config.json semantically intact",
"tokenizer.json byte-identical to source",
"vocab.json byte-identical to source",
"merges.txt byte-identical to source",
"LICENSE byte-identical to source",
"tokenizer config preserves chat_template",
"tokenizer config preserves eos_token",
"tokenizer config preserves pad_token",
"tokenizer config preserves additional_special_tokens",
"official internal architecture id retained",
"MTP layer declaration retained",
"vision configuration retained",
"image special token id retained",
"video special token id retained",
"vision-start token id retained",
"vision-end token id retained",
"text architecture preserves num_hidden_layers",
"text architecture preserves hidden_size",
"text architecture preserves vocab_size",
"text architecture preserves max_position_embeddings",
"text architecture preserves full_attention_interval",
"text architecture preserves layer_types",
"text architecture preserves linear_conv_kernel_dim",
"text architecture preserves linear_key_head_dim",
"text architecture preserves linear_num_key_heads",
"text architecture preserves linear_num_value_heads",
"text architecture preserves linear_value_head_dim",
"full vision configuration retained",
"main safetensors readable",
"vision tensors retained (333)",
"main MLX bundle does not misplace MTP tensors",
"MTP drafter tensors retained (15)",
"MTP drafter model type",
"all floating tensors finite"
],
"failures": [],
"warnings": [],
"disk_size_bytes": 19467896991
},
"runtime": {
"runtime": "mlx-vlm",
"model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-5bit-Group128",
"artifact_role": "quantized_candidate",
"source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
"text_tests": [
{
"id": "fact",
"category": "factual",
"passed": true,
"failures": [],
"text": "The capital of France is Paris.",
"finish_reason": "stop",
"prompt_tokens": 24,
"generation_tokens": 8,
"generation_tps": 15.13961681517839,
"peak_memory_gb": 20.552993318,
"wall_seconds": 1.45040150010027,
"loop_score": 0.0
},
{
"id": "math",
"category": "mathematical_reasoning",
"passed": true,
"failures": [],
"text": "The user is asking a simple multiplication problem. A box has 6 rows of 7 blue marbles. I need to multiply 6 × 7 to find the total number of blue marbles.\n\n6 × 7 = 42\n\n\n6 × 7 = **42 blue marbles**",
"finish_reason": "stop",
"prompt_tokens": 36,
"generation_tokens": 66,
"generation_tps": 13.714017846291666,
"peak_memory_gb": 20.631333466,
"wall_seconds": 5.0787066670600325,
"loop_score": 0.0
},
{
"id": "code",
"category": "coding",
"passed": true,
"failures": [],
"text": "The user wants a Python function called `clamp(x, low, high)` that returns `x` limited to the inclusive range `[low, high]`. This is a straightforward function:\n- If x < low, return low\n- If x > high, return high\n- Otherwise, return x\n\nI can implement this using `max(low, min(x, high))` or with if/elif/else. The one-liner with max/min is clean and Pythonic.\n\nThey want one fenced code block.\n\n\n```python\ndef clamp(x, low, high):\n return max(low, min(x, high))\n```",
"finish_reason": "stop",
"prompt_tokens": 67,
"generation_tokens": 134,
"generation_tps": 13.666359743269183,
"peak_memory_gb": 20.736925975,
"wall_seconds": 10.213078666944057,
"loop_score": 0.0
},
{
"id": "multiturn",
"category": "multi_turn",
"passed": true,
"failures": [],
"text": "cedar",
"finish_reason": "stop",
"prompt_tokens": 50,
"generation_tokens": 3,
"generation_tps": 20.48029710717471,
"peak_memory_gb": 20.736925975,
"wall_seconds": 0.4163297500927001,
"loop_score": 0.0
},
{
"id": "thinking_off",
"category": "thinking_disabled",
"passed": true,
"failures": [],
"text": "Yes, 17 is prime.\n\n**Reason:** It has no divisors other than 1 and itself (it is not divisible by 2, 3, or any other integer less than its square root, $\\sqrt{17} \\approx 4.12$).",
"finish_reason": "stop",
"prompt_tokens": 25,
"generation_tokens": 61,
"generation_tps": 13.809376108219007,
"peak_memory_gb": 20.736925975,
"wall_seconds": 4.71690741693601,
"loop_score": 0.0
},
{
"id": "reasoning_low",
"category": "reasoning_effort_low",
"passed": true,
"failures": [],
"text": "Compare 3/5 and 5/9.\n\nCross-multiply: 3 × 9 = 27, and 5 × 5 = 25.\n\nSince 27 > 25, 3/5 > 5/9.\n\n\n3/5",
"finish_reason": "stop",
"prompt_tokens": 60,
"generation_tokens": 64,
"generation_tps": 13.746131205613972,
"peak_memory_gb": 20.736925975,
"wall_seconds": 4.935991333099082,
"loop_score": 0.0
},
{
"id": "reasoning_high",
"category": "reasoning_effort_high",
"passed": true,
"failures": [],
"text": "We need answer simple math. Need final. Find smallest positive integer divisible by 6,8,15 = lcm. Factor: 6=2*3, 8=2^3, 15=3*5. LCM=2^3*3*5=120. Explain briefly.\n\n\nThe smallest positive integer divisible by 6, 8, and 15 is their least common multiple (LCM).\n\nPrime factorizations:\n- 6 = 2 × 3\n- 8 = 2³\n- 15 = 3 × 5\n\nTake the highest power of each prime:\n- 2³\n- 3\n- 5\n\nSo the LCM is:\n\n\\[\n2^3 \\times 3 \\times 5 = 8 \\times 3 \\times 5 = 120\n\\]\n\n**Answer: 120**",
"finish_reason": "stop",
"prompt_tokens": 73,
"generation_tokens": 201,
"generation_tps": 13.607607756339537,
"peak_memory_gb": 20.756765927,
"wall_seconds": 15.182417042087764,
"loop_score": 0.0
},
{
"id": "long_form",
"category": "long_form",
"passed": true,
"failures": [],
"text": "Checksums serve as the foundational integrity mechanism for reproducible machine learning by providing a unique digital fingerprint for every artifact in the release pipeline. By calculating cryptographic hashes for raw datasets, preprocessed features, model weights, and dependency libraries, teams can verify that the exact same inputs and configurations are used during training, validation, and deployment. This ensures that any discrepancy between environments is immediately detectable, preventing subtle data drift or library version mismatches from silently altering model behavior and guaranteeing that the deployed model is bit-for-bit identical to the one that passed quality assurance.\n\nImmutable revisions extend this integrity by locking the entire state of the project into a specific, unchangeable snapshot, often managed through version control systems or specialized artifact registries. Unlike mutable branches where code and data can be altered after the fact, an immutable revision captures the precise combination of source code, hyperparameters, and data versions at a single point in time. This creates a definitive \"point-in-time\" record that allows engineers to reconstruct the exact experimental conditions, ensuring that a successful model run can be perfectly replicated without the risk of upstream changes inadvertently modifying the underlying logic or data structures.\n\nThe synergy between checksums and immutable revisions eliminates the \"it works on my machine\" problem by establishing a verifiable chain of custody from data ingestion to model inference. When a release is tagged with an immutable revision, the associated checksums act as a seal of authenticity, allowing automated systems to validate that the deployed environment matches the tested environment exactly. This rigorous verification process reduces the cognitive load on data scientists, as they no longer need to manually track which specific dataset version or library update caused a performance regression, thereby accelerating the debugging process and increasing confidence in production stability.\n\nUltimately, these practices transform machine learning releases from opaque, ad-hoc processes into auditable, engineering-grade software products. By enforcing strict immutability and continuous integrity checks, organizations can meet regulatory compliance requirements, facilitate seamless handoffs between data science and operations teams, and enable safe, incremental updates. This robust framework not only ensures that models behave predictably in production but also fosters a culture of transparency and accountability, where every decision and data transformation is traceable, verifiable, and reproducible across different teams and timeframes.",
"finish_reason": "stop",
"prompt_tokens": 31,
"generation_tokens": 453,
"generation_tps": 13.602199251258176,
"peak_memory_gb": 20.756765927,
"wall_seconds": 33.60679283295758,
"loop_score": 0.0
},
{
"id": "non_english",
"category": "non_english",
"passed": true,
"failures": [],
"text": "太陽光が大気中を通過する際、波長の短い青い光は他の色よりも強く散乱されます。そのため、空全体から青い光が四方八方に拡散して私たちの目に届くため、空は青く見えます。",
"finish_reason": "stop",
"prompt_tokens": 28,
"generation_tokens": 55,
"generation_tps": 13.808740223169782,
"peak_memory_gb": 20.756765927,
"wall_seconds": 4.2827439999673516,
"loop_score": 0.0
},
{
"id": "json",
"category": "structured_json",
"passed": true,
"failures": [],
"text": "```json\n{\n \"name\": \"sample\",\n \"primes\": [2, 3, 5],\n \"active\": true\n}\n```",
"finish_reason": "stop",
"prompt_tokens": 41,
"generation_tokens": 38,
"generation_tps": 13.920806636452872,
"peak_memory_gb": 20.756765927,
"wall_seconds": 2.994563875021413,
"loop_score": 0.0
}
],
"tool_tests": [
{
"id": "one_tool",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
}
],
"raw": "The user is asking to call weather_at for Paris in Celsius. This is a simple, direct request.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
},
{
"id": "parallel_tools",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
},
{
"name": "weather_at",
"arguments": {
"location": "Austin",
"units": "fahrenheit"
}
}
],
"raw": "The user is asking to call weather_at in parallel for Paris (Celsius) and Austin (Fahrenheit). Since these are independent calls, they can be executed in the same block.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n\n\n\n\nAustin\n\n\nfahrenheit\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
},
{
"id": "sequential_dependency",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
}
],
"raw": "The user is asking me to first call weather_at for Paris in Celsius, and then create an event with the title being exactly the returned condition string. Since the second call depends on the result of the first call, I need to wait for the first result before making the second call.\n\nLet me start with the weather call first.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n",
"finish_reason": "stop",
"dependent_raw": "The user wants me to create an event with the title exactly matching the returned condition string, which is \"sunny\".\n\n\n\n\n\nsunny\n\n\n"
},
{
"id": "nested_json",
"passed": true,
"failures": [],
"calls": [
{
"name": "add_event",
"arguments": {
"title": "Release review",
"attendees": [
"ana@example.test",
"li@example.test"
],
"metadata": {
"release": {
"model": "Qwen3.8-27B",
"slot": 21
}
}
}
}
],
"raw": "The user is asking to call add_event with specific parameters:\n- title: \"Release review\"\n- attendees: [\"ana@example.test\", \"li@example.test\"]\n- metadata: {\"release\":{\"model\":\"Qwen3.8-27B\",\"slot\":21}}\n\nThis is a straightforward function call request. Let's execute it.\n\n\n\n\n\nRelease review\n\n\n[\"ana@example.test\", \"li@example.test\"]\n\n\n{\"release\": {\"model\": \"Qwen3.8-27B\", \"slot\": 21}}\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
},
{
"id": "invalid_recovery",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
}
],
"raw": "The user is asking to call weather_at for Paris with units=kelvin. However, the schema specifies that units must be either \"celsius\" or \"fahrenheit\" (enum). Therefore, kelvin violates the schema. The user is asking to recover by using celsius.\n\nSo, I should call weather_at with location=Paris and units=celsius.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
}
],
"vision_tests": [
{
"id": "shapes_colors",
"passed": true,
"missing_patterns": [],
"text": "Based on the image provided, here are the three large shapes from left to right with their corresponding colors:\n\n1. **Red Square** \n - Shape: Square \n - Color: Red \n - Label below: “RED”\n\n2. **Blue Circle** \n - Shape: Circle \n - Color: Blue \n - Label below: “BLUE”\n\n3. **Green Triangle** \n - Shape: Triangle \n - Color: Green \n - Label below: “GREEN”\n\n---\n\n✅ Final Answer (left to right): \n**Red Square, Blue Circle, Green Triangle**",
"finish_reason": "stop"
},
{
"id": "printed_text",
"passed": true,
"missing_patterns": [],
"text": "VISION CHECK 27B",
"finish_reason": "stop"
},
{
"id": "chart",
"passed": true,
"missing_patterns": [],
"text": "The tallest chart bar is **C**, and the number printed above it is **135**.",
"finish_reason": "stop"
}
],
"mtp": {
"passed": true,
"drafter_kind": "mtp",
"output_equivalent_temperature_zero": true,
"accepted_drafts": 84,
"drafted_tokens": 88,
"acceptance_rate": 0.9545454545454546,
"baseline_tps": 13.703316514224614,
"mtp_tps": 14.343391542905652,
"speedup": 1.04670949751592,
"measured_improvement": true,
"baseline_wall_seconds": 9.637984917033464,
"mtp_wall_seconds": 9.147978458087891,
"advertise_acceleration": true
},
"warnings": [],
"phases": [
"mtp",
"text",
"tools",
"vision"
],
"validation_inputs": {
"prompts_sha256": "136a918e5fee962f2b52f8e520a0275fd5ec5569181e0f5fdf4910ee3c34d528",
"tools_sha256": "86ae46ebdbb0324c9672eec87e6b7f7683b0eb9c8c7169dcabdf646cf9bab122",
"image_sha256": "0b1ae6badbe19a6049305c36d165a34cdf36df00033842e265339b2b7f057295"
},
"metal_memory_policy": {
"device": {
"device_name": "Apple M5 Pro",
"max_recommended_working_set_size": 55662788608,
"memory_size": 68719476736,
"architecture": "applegpu_g17s",
"max_buffer_length": 41747087360,
"resource_limit": 499000
},
"cache_limit_bytes": 256000000,
"wired_limit_bytes": 54549532835,
"previous_cache_limit_bytes": 65283502899,
"previous_wired_limit_bytes": 0,
"warnings": []
}
},
"warnings": [],
"overall_passed": true,
"runtime_failures": [],
"quality": {
"schema_version": 1,
"comparison_type": "cross-runtime output agreement against pinned BF16 source",
"passed": true,
"source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
"embedding_model": {
"repo_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
"revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
"pooling": "attention-mask mean pooling followed by L2 normalization",
"maximum_tokens": 256
},
"thresholds": {
"mean_semantic_similarity": 0.55,
"per_case_severe_regression": 0.25
},
"baseline_valid": true,
"candidate_functional": true,
"semantic_gate_passed": true,
"validation_inputs_match": true,
"validation_inputs": {
"prompts_sha256": "136a918e5fee962f2b52f8e520a0275fd5ec5569181e0f5fdf4910ee3c34d528",
"tools_sha256": "86ae46ebdbb0324c9672eec87e6b7f7683b0eb9c8c7169dcabdf646cf9bab122",
"image_sha256": "0b1ae6badbe19a6049305c36d165a34cdf36df00033842e265339b2b7f057295"
},
"mean_semantic_similarity": 0.9368221938610077,
"exact_matches": 3,
"comparisons": [
{
"id": "fact",
"reference_passed": true,
"candidate_passed": true,
"exact_match": true,
"sequence_agreement": 1.0,
"semantic_similarity": 1.0,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "math",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.6446280991735537,
"semantic_similarity": 0.9139397144317627,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "code",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.5797101449275363,
"semantic_similarity": 0.9799442291259766,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "multiturn",
"reference_passed": true,
"candidate_passed": true,
"exact_match": true,
"sequence_agreement": 1.0,
"semantic_similarity": 1.0,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "thinking_off",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.9695290858725761,
"semantic_similarity": 0.9862716794013977,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "reasoning_low",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.6030150753768844,
"semantic_similarity": 0.7996187210083008,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "reasoning_high",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.6644295302013423,
"semantic_similarity": 0.9212576150894165,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "long_form",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.10660091047040972,
"semantic_similarity": 0.8248775005340576,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "non_english",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.7034482758620689,
"semantic_similarity": 0.942312479019165,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "json",
"reference_passed": true,
"candidate_passed": true,
"exact_match": true,
"sequence_agreement": 1.0,
"semantic_similarity": 1.0,
"severe_regression": false,
"candidate_loop_score": 0.0
}
],
"functional_results": {
"reference_text": {
"passed": 10,
"total": 10
},
"candidate_text": {
"passed": 10,
"total": 10
},
"reference_tools": {
"passed": 5,
"total": 5
},
"candidate_tools": {
"passed": 5,
"total": 5
},
"reference_vision": {
"passed": 3,
"total": 3
},
"candidate_vision": {
"passed": 3,
"total": 3
}
},
"measurements": {
"average_generation_tps": 14.54951526929673,
"peak_memory_gb": 20.756765927,
"artifact_bytes": 19467900026,
"maximum_prompt_tokens_tested": 73,
"loop_rate": 0.0
},
"warnings": [
"Semantic similarity is a measured embedding-model proxy, not ground-truth accuracy.",
"Raw-logit equality is unavailable across all target runtimes; exact functional gates and output agreement are used for portable release validation.",
"Sequence agreement is lexical and is reported diagnostically, not used as semantic accuracy."
]
},
"quality_failures": []
}