quality_report.json

{
  "schema_version": 1,
  "comparison_type": "cross-runtime output agreement against pinned BF16 source",
  "passed": true,
  "source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
  "embedding_model": {
    "repo_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
    "revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
    "pooling": "attention-mask mean pooling followed by L2 normalization",
    "maximum_tokens": 256
  },
  "thresholds": {
    "mean_semantic_similarity": 0.55,
    "per_case_severe_regression": 0.25
  },
  "baseline_valid": true,
  "candidate_functional": true,
  "semantic_gate_passed": true,
  "validation_inputs_match": true,
  "validation_inputs": {
    "prompts_sha256": "136a918e5fee962f2b52f8e520a0275fd5ec5569181e0f5fdf4910ee3c34d528",
    "tools_sha256": "86ae46ebdbb0324c9672eec87e6b7f7683b0eb9c8c7169dcabdf646cf9bab122",
    "image_sha256": "0b1ae6badbe19a6049305c36d165a34cdf36df00033842e265339b2b7f057295"
  },
  "mean_semantic_similarity": 0.9774516999721528,
  "exact_matches": 6,
  "comparisons": [
    {
      "id": "fact",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "math",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.5480093676814989,
      "semantic_similarity": 0.9332988262176514,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "code",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.32563025210084034,
      "semantic_similarity": 0.9706727266311646,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "multiturn",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "thinking_off",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.9695290858725761,
      "semantic_similarity": 0.9862716794013977,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "reasoning_low",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "reasoning_high",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "long_form",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": false,
      "sequence_agreement": 0.17132798748288675,
      "semantic_similarity": 0.8842736482620239,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "non_english",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0000001192092896,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    },
    {
      "id": "json",
      "reference_passed": true,
      "candidate_passed": true,
      "exact_match": true,
      "sequence_agreement": 1.0,
      "semantic_similarity": 1.0,
      "severe_regression": false,
      "candidate_loop_score": 0.0
    }
  ],
  "functional_results": {
    "reference_text": {
      "passed": 10,
      "total": 10
    },
    "candidate_text": {
      "passed": 10,
      "total": 10
    },
    "reference_tools": {
      "passed": 5,
      "total": 5
    },
    "candidate_tools": {
      "passed": 5,
      "total": 5
    },
    "reference_vision": {
      "passed": 3,
      "total": 3
    },
    "candidate_vision": {
      "passed": 3,
      "total": 3
    }
  },
  "measurements": {
    "average_generation_tps": 12.568759032272249,
    "peak_memory_gb": 23.639191525,
    "artifact_bytes": 21989153029,
    "maximum_prompt_tokens_tested": 73,
    "loop_rate": 0.0
  },
  "warnings": [
    "Semantic similarity is a measured embedding-model proxy, not ground-truth accuracy.",
    "Raw-logit equality is unavailable across all target runtimes; exact functional gates and output agreement are used for portable release validation.",
    "Sequence agreement is lexical and is reported diagnostically, not used as semantic accuracy."
  ]
}