from __future__ import annotations import base64 import textwrap def _encode(script: str) -> str: return base64.b64encode(script.encode("utf-8")).decode("ascii") UNIVERSAL_MODEL_CARD_WORKER_SCRIPT = r''' import ast import base64 import hashlib import hmac import hashlib import hmac import json import os import re import shutil import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path from textwrap import dedent TARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$") GIST_URL = "https://gist.github.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d" GIST_RAW_URL = "https://gist.githubusercontent.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d/raw" AUTO_SPACE_HARDWARE_CHOICES = {"zero-a10g", "cpu-basic", "t4-small", "t4-medium", "a10g-small", "a10g-large", "l4x1", "l40sx1"} DEFAULT_PREFERRED_SPACE_HARDWARE = "zero-a10g" DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large" DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2" MAX_PI_REPAIR_ATTEMPTS = 3 APP_VERSION = "v198.26.32" app_version = "v198.26.32" # Internal agent/recovery files may be needed inside the transient Pi # workspace, but they should not be published to the generated Space or shown # twice as generated application files. Canonical copies live in run_dir/logs, # run_dir/repair, and run_dir/traces. INTERNAL_WORKSPACE_ARTIFACT_NAMES = { "GOAL.md", "INCIDENT_BRIEF.md", "PI_DIAGNOSIS_GOAL.md", "DEPENDENCY_ERROR_BRIEF.md", "REPAIR_DECISION.json", "REPAIR_TASK_PACKET.json", "MEMORY_DIAGNOSIS_GOAL.md", "MEMORY_DIAGNOSIS.json", "REPAIR_GOAL.md", "REPAIR_BRIEF.md", "REPAIR_PLAN.md", "REPAIR_SUMMARY.md", "PI_SUMMARY.md", "PI_SOURCE_USAGE.json", "MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md", "APP_RUNTIME_CONTRACT.json", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "PI_TASK_PACKET.json", "RECIPE_AWARE_REPAIR_PACKET.json", "PATCH_REFUSAL.json", "TECHNICAL_BLOCKERS.json", "pi_feasibility_brief.json", "pi_implementation_plan.json", } INTERNAL_WORKSPACE_ARTIFACT_DIRS = {"analysis_inputs", "model_inspect", "refs", "logs", "traces", "repair", "artifacts", "snapshots", "checkpoints", "weights", "hf_cache", "model_cache"} def internal_workspace_upload_ignore_patterns() -> list[str]: patterns = [".git/*", ".cache/*", "**/.cache/*", "node_modules/*", "__pycache__/*", "*.pyc", ".pi/*", "**/.pi/*", "analysis_inputs/*", "**/analysis_inputs/*", "model_inspect/*", "**/model_inspect/*", "refs/*", "**/refs/*", "logs/*", "**/logs/*", "traces/*", "**/traces/*", "repair/*", "**/repair/*", "auth.json", "**/auth.json"] for name in sorted(INTERNAL_WORKSPACE_ARTIFACT_NAMES): patterns.extend([name, f"**/{name}"]) return patterns def internal_workspace_copy_ignore(_dir: str, names: list[str]) -> set[str]: ignored = {".git", ".cache", "node_modules", "__pycache__", ".pi", "analysis_inputs", "model_inspect", "refs", "logs", "traces", "repair", "artifacts", "snapshots", "checkpoints", "weights", "hf_cache", "model_cache"} ignored.update(name for name in names if name in INTERNAL_WORKSPACE_ARTIFACT_NAMES or name.endswith(".pyc")) return ignored def is_publishable_workspace_file(path: Path) -> bool: if path.name in INTERNAL_WORKSPACE_ARTIFACT_NAMES or path.name == "auth.json": return False if any(part in INTERNAL_WORKSPACE_ARTIFACT_DIRS for part in path.parts): return False if any(part in {".git", ".cache", "node_modules", "__pycache__"} for part in path.parts): return False if path.suffix == ".pyc": return False return True RUNTIME_PAYLOAD_BLOCKED_SUFFIXES = { ".safetensors", ".bin", ".pt", ".pth", ".ckpt", ".onnx", ".gguf", ".h5", ".pb", ".tflite", ".npz", ".npy", ".tar", ".zip", ".7z", ".rar", ".parquet", ".arrow", ".sqlite", ".db", } RUNTIME_PAYLOAD_ALLOWED_SUFFIXES = { "", ".py", ".txt", ".md", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".css", ".js", ".html", ".csv", ".tsv", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".mp3", ".wav", ".flac", ".ogg", ".mp4", ".webm", ".m4a", } RUNTIME_UPLOAD_DEFAULT_MAX_TOTAL_BYTES = 250 * 1024 * 1024 RUNTIME_UPLOAD_DEFAULT_MAX_FILE_BYTES = 50 * 1024 * 1024 def _runtime_upload_limit_from_env(name: str, default: int) -> int: raw = os.environ.get(name, "").strip() if not raw: return default try: value = int(raw) return value if value > 0 else default except Exception: return default def runtime_payload_file_reason(path: Path, *, size_bytes: int = 0, max_file_bytes: int | None = None) -> tuple[bool, str]: """Return whether a workspace file is safe to publish to the target Space repo. The Pi workspace may contain large audit/model-inspection artifacts that are useful in the run bucket but must never be committed to the generated Space. Model weights should be loaded at Space runtime through from_pretrained(), not uploaded through Git/LFS with the demo code. """ rel = Path(path) suffix = rel.suffix.lower() # Report model/weight artifacts as such even when they live under an audit dir. if suffix in RUNTIME_PAYLOAD_BLOCKED_SUFFIXES: return False, "blocked_heavy_or_model_suffix" if max_file_bytes is not None and size_bytes > max_file_bytes: return False, "file_too_large_for_runtime_payload" if not is_publishable_workspace_file(rel): return False, "internal_or_audit_artifact" if any(part in INTERNAL_WORKSPACE_ARTIFACT_DIRS for part in rel.parts): return False, "excluded_runtime_artifact_dir" if any(part.startswith(".") and part not in {".streamlit"} for part in rel.parts): return False, "hidden_or_cache_path" if suffix not in RUNTIME_PAYLOAD_ALLOWED_SUFFIXES: return False, "suffix_not_runtime_whitelisted" return True, "included" def build_runtime_upload_payload(workspace: Path, run_dir: Path, events_path: Path) -> tuple[Path, dict]: """Create a minimal, auditable runtime payload for Space upload. This deliberately separates the rich Pi/audit workspace from the deployable Space repo. The generated app can still download heavy model weights at runtime through Hugging Face Hub APIs, but the Space repo itself remains a small code package. """ required = ["app.py", "README.md", "requirements.txt"] for filename in required: if not (workspace / filename).exists(): raise RuntimeError(f"Missing required generated file: {filename}") payload_dir = run_dir / "runtime_upload_payload" if payload_dir.exists(): shutil.rmtree(payload_dir) payload_dir.mkdir(parents=True, exist_ok=True) max_total_bytes = _runtime_upload_limit_from_env("ASF_RUNTIME_UPLOAD_MAX_BYTES", RUNTIME_UPLOAD_DEFAULT_MAX_TOTAL_BYTES) max_file_bytes = _runtime_upload_limit_from_env("ASF_RUNTIME_UPLOAD_MAX_FILE_BYTES", RUNTIME_UPLOAD_DEFAULT_MAX_FILE_BYTES) manifest = { "schema_version": "runtime_upload_payload.v198_25_1", "source_workspace": str(workspace), "payload_dir": str(payload_dir), "max_total_bytes": max_total_bytes, "max_file_bytes": max_file_bytes, "files": [], "excluded_files": [], "total_bytes": 0, "file_count": 0, "blocked_heavy_file_count": 0, "payload_guard": "whitelist_runtime_only", } for src in sorted(workspace.rglob("*")): if not src.is_file(): continue try: rel = src.relative_to(workspace) except Exception: continue try: size = src.stat().st_size except Exception: size = 0 include, reason = runtime_payload_file_reason(rel, size_bytes=size, max_file_bytes=max_file_bytes) entry = {"path": str(rel), "size_bytes": size, "reason": reason} if not include: if reason in {"blocked_heavy_or_model_suffix", "file_too_large_for_runtime_payload"}: manifest["blocked_heavy_file_count"] += 1 manifest["excluded_files"].append(entry) continue dest = payload_dir / rel dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dest) manifest["files"].append(entry) manifest["total_bytes"] += size manifest["file_count"] = len(manifest["files"]) manifest["excluded_count"] = len(manifest["excluded_files"]) manifest_path = run_dir / "runtime_upload_payload_manifest.json" write_json(manifest_path, manifest) if manifest["total_bytes"] > max_total_bytes: payload = { "schema_version": "runtime_upload_payload_guard.v198_25_1", "failure_owner": "factory_packaging", "failure_class": "runtime_payload_too_large", "repair_candidate": False, "total_bytes": manifest["total_bytes"], "max_total_bytes": max_total_bytes, "file_count": manifest["file_count"], "manifest": str(manifest_path), "message": "Runtime upload payload exceeds the configured safety limit before HF upload.", } write_json(run_dir / "factory_upload_error.json", payload) append_event(events_path, "upload_files", "failed", "Runtime upload payload is too large; aborting before HF upload", payload) raise RuntimeError(f"runtime_payload_too_large: {manifest['total_bytes']} bytes > {max_total_bytes} bytes") append_event( events_path, "runtime_payload", "success", "Prepared whitelist runtime payload for generated Space upload", { "manifest": "runtime_upload_payload_manifest.json", "file_count": manifest["file_count"], "total_bytes": manifest["total_bytes"], "excluded_count": manifest["excluded_count"], "blocked_heavy_file_count": manifest["blocked_heavy_file_count"], }, ) return payload_dir, manifest def classify_space_upload_exception(exc: Exception, run_dir: Path, events_path: Path) -> dict: raw = str(exc) lower = raw.lower() storage_limit = "repository storage limit" in lower or "storage limit reached" in lower or "max: 1 gb" in lower or "/info/lfs/objects/batch" in lower payload = { "schema_version": "factory_upload_error.v198_25_1", "failure_owner": "factory_packaging" if storage_limit else "hf_platform_or_upload", "failure_class": "factory_upload_error" if storage_limit else "space_upload_error", "storage_limit_reached": bool(storage_limit), "repair_candidate": False, "recommended_action": "Upload only the whitelist runtime payload; keep model/audit artifacts out of the target Space repo." if storage_limit else "Inspect HF upload error and retry only if transient.", "error": redact_text(raw)[:4000] if "redact_text" in globals() else raw[:4000], } write_json(run_dir / "factory_upload_error.json", payload) append_event(events_path, "upload_files", "failed", "Generated Space upload failed before live validation", payload) return payload def now(): return datetime.now(timezone.utc).isoformat() def write_json(path: Path, payload: dict): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2, ensure_ascii=False, default=str) + "\n", encoding="utf-8") def read_json(path: Path, default=None): try: if path.exists(): return json.loads(path.read_text(encoding="utf-8")) except Exception: pass return default def _rel_to_workspace(workspace: Path, path: Path | str) -> str: try: pp = path if isinstance(path, Path) else workspace / str(path) return str(pp.relative_to(workspace)) except Exception: return str(path) def _safe_file_chars(path: Path, limit: int | None = None) -> int: try: text = path.read_text(encoding="utf-8", errors="ignore") if limit is not None: text = text[:limit] return len(text) except Exception: return 0 def estimate_pi_context_budget( run_dir: Path, workspace: Path, *, phase: str, task_type: str, direct_prompt: str, referenced_files: list[str] | None = None, max_context_chars: int = 60000, omitted_sections: list[str] | None = None, events_path: Path | None = None, artifact_dir: Path | None = None, ) -> dict: """Record what ASF made available to Pi for this call. Pi may still decide which files to open internally. This artifact therefore measures the *Factory-provided context surface*, not the provider's exact final prompt. It is intentionally approximate and stable enough to compare runs over time. """ referenced_files = referenced_files or [] omitted_sections = omitted_sections or [] file_entries = [] referenced_chars = 0 for rel in referenced_files: path = workspace / rel chars = _safe_file_chars(path) referenced_chars += chars file_entries.append({"path": rel, "chars": chars, "exists": path.exists()}) payload = { "schema_version": "pi_prompt_budget.v198_25", "phase": phase, "task_type": task_type, "direct_prompt_chars": len(direct_prompt or ""), "referenced_context_chars": referenced_chars, "estimated_total_context_chars": len(direct_prompt or "") + referenced_chars, "max_context_chars": max_context_chars, "context_budget_status": "ok" if len(direct_prompt or "") + referenced_chars <= max_context_chars else "large_context_observed", "blocking": False, "referenced_files": file_entries, "omitted_sections": omitted_sections, "note": "Measures Factory-provided prompt plus referenced files; Pi/provider internal context may differ.", } budgets_path = run_dir / "pi_prompt_budget.json" aggregate = read_json(budgets_path, {"schema_version": "pi_prompt_budget_aggregate.v198_25", "calls": []}) if not isinstance(aggregate, dict): aggregate = {"schema_version": "pi_prompt_budget_aggregate.v198_25", "calls": []} calls = aggregate.get("calls") if isinstance(aggregate.get("calls"), list) else [] calls.append(payload) aggregate["calls"] = calls aggregate["latest"] = payload write_json(budgets_path, aggregate) if artifact_dir: artifact_dir.mkdir(parents=True, exist_ok=True) write_json(artifact_dir / f"PI_PROMPT_BUDGET_{phase}.json", payload) if events_path: append_event(events_path, "pi_prompt_budget", "success", f"Recorded Pi prompt budget for {phase}", {"phase": phase, "task_type": task_type, "estimated_total_context_chars": payload["estimated_total_context_chars"], "status": payload["context_budget_status"]}) return payload def _sha256_text(text: str) -> str: try: return hashlib.sha256((text or "").encode("utf-8")).hexdigest() except Exception: return "" def _sha256_file(path: Path | None) -> str: try: if path and path.exists(): return hashlib.sha256(path.read_bytes()).hexdigest() except Exception: pass return "" def _pi_settings_model(default_model: str = "") -> tuple[str, dict]: settings_path = Path.home() / ".pi" / "agent" / "settings.json" try: settings = json.loads(settings_path.read_text(encoding="utf-8")) return settings.get("model") or default_model, settings except Exception: return default_model, {"model": default_model, "read_error": "could_not_read_settings"} def resolve_pi_call_identity(requested_model: str, output_text: str = "") -> dict: configured_model, settings = _pi_settings_model(requested_model) observed = extract_pi_models_from_text(output_text or "") if "extract_pi_models_from_text" in globals() else [] normalized_requested = normalize_pi_model_name(requested_model) if "normalize_pi_model_name" in globals() else (requested_model or "").lower() normalized_configured = normalize_pi_model_name(configured_model) if "normalize_pi_model_name" in globals() else (configured_model or "").lower() effective_model = "" for raw in observed: normalized = normalize_pi_model_name(raw) if "normalize_pi_model_name" in globals() else str(raw).lower() if normalized and normalized not in {normalized_requested, normalized_configured}: effective_model = raw break if not effective_model and observed: effective_model = observed[0] if not effective_model: effective_model = configured_model or requested_model mismatch = bool(effective_model and (normalize_pi_model_name(effective_model) if "normalize_pi_model_name" in globals() else effective_model.lower()) not in {normalized_requested, normalized_configured}) return { "requested_model": requested_model, "configured_model": configured_model, "effective_model": effective_model, "observed_models": observed, "provider": "huggingface", "mismatch": mismatch, "settings": settings, } def classify_pi_provider_drift(previous: dict | None, current: dict, *, phase: str = "") -> dict: previous = previous or {} prev_effective = normalize_pi_model_name(previous.get("effective_model") or "") if "normalize_pi_model_name" in globals() else str(previous.get("effective_model") or "").lower() curr_effective = normalize_pi_model_name(current.get("effective_model") or "") if "normalize_pi_model_name" in globals() else str(current.get("effective_model") or "").lower() provider_drift = bool(prev_effective and curr_effective and prev_effective != curr_effective) severity = "none" if provider_drift: if phase in {"repair_patch", "memory_patch"}: severity = "high" elif phase in {"diagnosis", "memory_diagnosis"}: severity = "moderate" else: severity = "minor" elif current.get("mismatch"): severity = "minor" return { "provider_drift_from_previous_call": provider_drift, "previous_effective_model": previous.get("effective_model") or "", "drift_severity": severity, "policy": "Worker validation remains authoritative; higher drift severity tightens repair diff gates and freezes diagnosis reinterpretation.", } def write_pi_call_fingerprint( run_dir: Path, *, phase: str, requested_model: str, prompt_text: str = "", output_text: str = "", task_packet_path: Path | None = None, events_path: Path | None = None, artifact_dir: Path | None = None, identity: dict | None = None, ) -> dict: identity = identity or resolve_pi_call_identity(requested_model, output_text) aggregate_path = run_dir / "pi_call_fingerprints.json" aggregate = read_json(aggregate_path, {"schema_version": "pi_call_fingerprints.v198_25", "calls": []}) if not isinstance(aggregate, dict): aggregate = {"schema_version": "pi_call_fingerprints.v198_25", "calls": []} calls = aggregate.get("calls") if isinstance(aggregate.get("calls"), list) else [] previous = calls[-1] if calls else {} drift = classify_pi_provider_drift(previous, identity, phase=phase) payload = { "schema_version": "pi_call_fingerprint.v198_25", "phase": phase, **identity, **drift, "prompt_hash": _sha256_text(prompt_text or ""), "task_packet_hash": _sha256_file(task_packet_path), "output_hash": _sha256_text(output_text or ""), "created_at": now(), } calls.append(payload) aggregate["calls"] = calls aggregate["latest"] = payload write_json(aggregate_path, aggregate) out_dir = artifact_dir or (run_dir / "pi") out_dir.mkdir(parents=True, exist_ok=True) write_json(out_dir / f"PI_CALL_FINGERPRINT_{phase}.json", payload) # Canonical alias for the most recent call. write_json(run_dir / "pi_call_fingerprint.json", payload) if events_path: append_event(events_path, "pi_call_fingerprint", "warning" if payload.get("drift_severity") in {"high", "critical"} else "success", "Recorded Pi call fingerprint and provider drift policy", {k: payload.get(k) for k in ["phase", "requested_model", "configured_model", "effective_model", "provider_drift_from_previous_call", "drift_severity"]}) return payload def failure_owner_from_classification(classification: dict | None = None) -> str: classification = classification or {} explicit = classification.get("failure_owner") or classification.get("owner") if explicit: return str(explicit) category = str(classification.get("category") or classification.get("failure_class") or "").lower() phase = str(classification.get("failure_phase") or "").lower() if category in {"smoke_schema_error", "validation_client_payload_error"}: return "factory_validation_client" if category in {"cuda_oom", "hardware_capacity", "hardware_runtime_blocker"}: return "hardware_capacity" if category in {"hf_platform", "infra_transient_scheduling", "logs_unavailable"} or "platform" in category: return "hf_platform" if category in {"dependency_error", "import_error", "model_loading_error"}: return "dependency" if category != "model_loading_error" else "model_loading" if "smoke" in phase and category in {"unknown", ""}: return "unknown" return "app_runtime" if category else "unknown" def write_log_evidence_packet(run_dir: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "", classification: dict | None = None, excerpt: str = "") -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) classification = classification or {} excerpt = excerpt or extract_actionable_error_excerpt("\n".join([failure_reason or "", build_log or "", runtime_log or ""]), max_lines=80) payload = { "schema_version": "log_evidence_packet.v198_25", "failure_owner": failure_owner_from_classification(classification), "failure_class": classification.get("category") or classification.get("failure_class") or "unknown", "failure_phase": classification.get("failure_phase") or "unknown", "logs_quality": classification.get("logs_quality") or "unknown", "first_actionable_error": excerpt[:4000], "traceback_excerpt": excerpt[-6000:], "full_logs_available": bool(build_log or runtime_log), "full_logs_path": ["logs/space_logs_build.txt", "logs/space_logs_runtime.txt", "logs/space_logs_run.txt"], "confidence": "high" if excerpt.strip() and classification.get("logs_quality") == "useful" else ("medium" if excerpt.strip() else "low"), "created_at": now(), } write_json(repair_dir / "LOG_EVIDENCE_PACKET.json", payload) return payload def write_frozen_repair_decision(workspace: Path, run_dir: Path, decision: dict, classification: dict | None = None) -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) payload = { "schema_version": "frozen_repair_decision.v198_25", "decision": decision or {}, "classification": classification or (decision or {}).get("classification") or {}, "root_cause": (decision or {}).get("classification", {}).get("category") or (classification or {}).get("category") or "unknown", "frozen": True, "policy": "Repair patch calls must implement this decision and must not reinterpret the run unless they write PATCH_REFUSAL.json without editing publishable files.", "created_at": now(), } write_json(repair_dir / "FROZEN_REPAIR_DECISION.json", payload) write_json(workspace / "FROZEN_REPAIR_DECISION.json", payload) return payload def is_deep_repair_category(category: str) -> bool: return str(category or "") in {"wrong_pipeline_class", "model_card_misunderstood", "architecture_mismatch", "wrong_task_type", "repeated_surgical_failures"} def validate_repair_diff_against_task_packet(diff_payload: dict, task_packet: dict, classification: dict | None, run_dir: Path, events_path: Path) -> tuple[bool, str, dict]: changed = set((diff_payload or {}).get("changed_files") or []) | set((diff_payload or {}).get("added_files") or []) | set((diff_payload or {}).get("removed_files") or []) allowed = set((task_packet or {}).get("allowed_files") or []) category = (classification or {}).get("category") or (task_packet or {}).get("failure_class") or "" task_type = (task_packet or {}).get("task_type") or "" if task_type == "dependency_repair": allowed.update({"requirements.txt", "requirements_policy.json"}) if task_type == "runtime_patch" or category in {"model_loading_error", "runtime_patch", "app_runtime_error", "publishable_app_py_syntax_error", "full_inference_media_contract_missing_generate_endpoint", "zero_gpu_full_inference_without_spaces_gpu_decorator"}: allowed.update({"app.py", "requirements.txt"}) # Pi sometimes writes advisory sidecar files during a bounded runtime patch. # Do not reject a useful app.py repair because of these non-publishable extras; # quarantine them before upload in repair_workspace_with_pi(). benign_sidecars = {"requirements_policy.json"} # Deterministic factory show_error patches may touch app.py after Pi; allow it for any repair but still record it. allowed.add("app.py") if "app.py" in changed and (run_dir / "repair" / "gradio_show_error_patch.json").exists() else None unexpected_all = sorted(changed - allowed) benign_unexpected = sorted([name for name in unexpected_all if name in benign_sidecars]) unexpected = sorted([name for name in unexpected_all if name not in benign_sidecars]) payload = { "schema_version": "repair_diff_gate.v198_26_28", "allowed_files": sorted(allowed), "changed_files": sorted(changed), "unexpected_files": unexpected, "benign_unexpected_files": benign_unexpected, "quarantine_before_upload": benign_unexpected, "task_type": task_type, "failure_class": category, "passed": not bool(unexpected), "policy": "Only non-publishable advisory sidecars may be quarantined; publishable app.py/requirements.txt repairs are judged by compile and integrity guards.", } write_json(run_dir / "repair" / "repair_diff_gate.json", payload) append_event(events_path, "repair_diff_gate", "success" if payload["passed"] else "failed", "Checked repair diff against Pi task packet allowed files", payload) if unexpected: return False, f"Repair modified files outside the bounded task packet: {', '.join(unexpected)}", payload if benign_unexpected: return True, f"Repair touched only publishable files plus quarantinable sidecars: {', '.join(benign_unexpected)}", payload return True, "Repair diff stayed within bounded task packet.", payload def quarantine_benign_repair_sidecars(workspace: Path, run_dir: Path, diff_gate_payload: dict, events_path: Path) -> dict: names = list((diff_gate_payload or {}).get("quarantine_before_upload") or []) payload = { "schema_version": "repair_sidecar_quarantine.v198_26_28", "requested_files": names, "quarantined_files": [], "missing_files": [], } if not names: return payload target_dir = run_dir / "repair" / "untrusted_extra" target_dir.mkdir(parents=True, exist_ok=True) for name in names: rel = Path(str(name)) if rel.is_absolute() or ".." in rel.parts: continue src = workspace / rel if src.exists() and src.is_file(): dst = target_dir / rel.name shutil.copy2(src, dst) try: src.unlink() except Exception: pass payload["quarantined_files"].append(str(rel)) else: payload["missing_files"].append(str(rel)) write_json(run_dir / "repair" / "repair_sidecar_quarantine.json", payload) append_event(events_path, "repair_sidecar_quarantine", "success", "Quarantined non-publishable Pi repair sidecars before upload", payload) return payload def extract_actionable_error_excerpt(text: str, max_lines: int = 120) -> str: """Return a compact error-centered excerpt for Pi repair packets. Full HF logs remain archived under logs/. Pi should normally receive the latest actionable traceback or OOM/import/dependency section rather than a giant raw log tail. """ raw = text or "" if not raw.strip(): return "" lines = raw.splitlines() markers = [ "traceback", "error", "exception", "modulenotfounderror", "importerror", "cuda out of memory", "outofmemoryerror", "no matching distribution", "could not find a version", "resolutionimpossible", "api_name", "unexpected keyword", "failed to load", "from_pretrained", ] chosen = len(lines) - 1 for idx in range(len(lines) - 1, -1, -1): low = lines[idx].lower() if any(m in low for m in markers): chosen = idx break half = max(10, max_lines // 2) start = max(0, chosen - half) end = min(len(lines), start + max_lines) start = max(0, end - max_lines) return "\n".join(lines[start:end])[-12000:] def write_actionable_error_artifacts(run_dir: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "", classification: dict | None = None) -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) combined = "\n".join([failure_reason or "", build_log or "", runtime_log or ""]) excerpt = extract_actionable_error_excerpt(combined, max_lines=120) (repair_dir / "actionable_error_excerpt.txt").write_text(excerpt, encoding="utf-8") signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification or {}) if "compute_failure_signature" in globals() else "unknown" log_packet = write_log_evidence_packet(run_dir, failure_reason, build_log, runtime_log, classification or {}, excerpt) payload = { "schema_version": "actionable_error_excerpt.v198_25", "failure_signature": signature, "classification": classification or {}, "failure_owner": log_packet.get("failure_owner"), "excerpt_chars": len(excerpt), "full_logs_archived": ["logs/space_logs_build.txt", "logs/space_logs_runtime.txt", "logs/space_logs_run.txt"], "log_evidence_packet": "repair/LOG_EVIDENCE_PACKET.json", } write_json(repair_dir / "error_signature.json", payload) return payload def _first_list_value(*values): for value in values: if isinstance(value, list): return value return None def _first_dict_value(*values): for value in values: if isinstance(value, dict): return value return None def load_parent_replay_source(parent_dir: Path, target_space_id: str, expected_output_type: str) -> dict: """Load the parent build's successful automatic smoke payload as replay truth. In linked Space Test replay mode, the validation worker must not invent a different request when the parent Build Run already proved a working request. The parent smoke artefacts are the source of truth; schema discovery is only used afterwards as a guardrail/retry mechanism. """ if not parent_dir or not parent_dir.exists(): return {} smoke = read_json(parent_dir / "tests" / "generation_smoke.json", {}) or {} retry_payload = read_json(parent_dir / "tests" / "generation_smoke_payload_retry.json", {}) or {} initial_payload = read_json(parent_dir / "tests" / "generation_smoke_payload.json", {}) or {} state = read_json(parent_dir / "state.json", {}) or {} state_smoke = state.get("generation_smoke") if isinstance(state.get("generation_smoke"), dict) else {} status_tokens = { str(smoke.get("status") or "").lower(), str(state.get("status") or "").lower(), str(state_smoke.get("status") or "").lower(), } parent_was_successful = bool(status_tokens & {"success", "full_inference_success", "completed", "complete"}) if not parent_was_successful: return {} api_name = ( smoke.get("api_name") or retry_payload.get("api_name") or initial_payload.get("api_name") or state_smoke.get("api_name") or "" ) test_args = _first_list_value( smoke.get("effective_args"), smoke.get("test_args"), retry_payload.get("test_args"), initial_payload.get("test_args"), state_smoke.get("effective_args"), state_smoke.get("test_args"), ) test_kwargs = _first_dict_value( smoke.get("effective_kwargs"), smoke.get("test_kwargs"), retry_payload.get("test_kwargs"), initial_payload.get("test_kwargs"), state_smoke.get("effective_kwargs"), state_smoke.get("test_kwargs"), ) or {} if not api_name or not isinstance(test_args, list): return {} parent_target = smoke.get("target_space") or state.get("target_space") or state_smoke.get("target_space") or "" return { "source": "parent_automatic_smoke", "parent_run_id": parent_dir.name, "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": test_args, "test_kwargs": test_kwargs, "expected_output_type": smoke.get("expected_output_type") or state_smoke.get("expected_output_type") or expected_output_type, "latency_seconds": smoke.get("latency_seconds") or state_smoke.get("latency_seconds"), "parent_smoke_status": smoke.get("status") or state_smoke.get("status") or state.get("status"), "initial_payload_present": bool(initial_payload), "retry_payload_present": bool(retry_payload), } def load_env_replay_source(target_space_id: str, expected_output_type: str) -> dict: """Replay source passed by backend through the validation Job env. The validation Job starts in a fresh workspace; the parent Build Run folder is usually not mounted locally. This env payload is therefore the canonical transport for replaying a successful parent automatic smoke request. """ raw = os.environ.get("PARENT_REPLAY_SOURCE_JSON") or "" if not raw.strip(): return {} try: data = json.loads(raw) except Exception as exc: return {"source_error": f"invalid_parent_replay_source_json:{exc}"} if not isinstance(data, dict): return {"source_error": "parent_replay_source_json_not_object"} args = data.get("test_args") kwargs = data.get("test_kwargs") or {} api_name = data.get("api_name") or "" if not api_name or not isinstance(args, list) or not isinstance(kwargs, dict): return {"source_error": "parent_replay_source_missing_api_args_or_kwargs", "raw_keys": sorted(data.keys())} parent_target = data.get("parent_target_space") or "" return { "source": data.get("source") or "parent_automatic_smoke_backend", "transport": "job_env", "parent_run_id": data.get("parent_run_id") or os.environ.get("PARENT_BUILD_RUN_ID", ""), "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": args, "test_kwargs": kwargs, "expected_output_type": data.get("expected_output_type") or expected_output_type or "any", "latency_seconds": data.get("latency_seconds"), "parent_smoke_status": data.get("parent_smoke_status") or "success", "backend_replay_source_used": True, } def parent_has_successful_manual_validation(parent_dir: Path) -> bool: manual = read_json(parent_dir / "manual_validation_status.json", {}) or {} if str(manual.get("status") or "").lower() == "success": return True linked = read_json(parent_dir / "linked_validations.json", {}) or {} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] success_tokens = {"success", "full_inference_success", "passed", "validated_after_manual_space_test"} return any(str(row.get("status") or row.get("effective_status") or "").lower() in success_tokens for row in validations if isinstance(row, dict)) def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None): path.parent.mkdir(parents=True, exist_ok=True) event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}} line = json.dumps(event, ensure_ascii=False) with path.open("a", encoding="utf-8") as f: f.write(line + "\n") print(line, flush=True) def write_live_status(run_dir: Path, *, stage: str, status: str = "running", message: str = "", data: dict | None = None): """Persist a tiny fast-changing status snapshot for the UI. The event log can be compacted or uploaded in batches. live_status.json is intentionally small and may be overwritten often so the web UI can show the current micro-stage without waiting for a full bundle refresh. """ payload = { "schema_version": "live_status.v1", "updated_at": now(), "run_id": os.environ.get("RUN_ID", ""), "stage": stage, "status": status, "message": message, "data": data or {}, } try: write_json(run_dir / "live_status.json", payload) except Exception: pass return payload def write_final_summary(run_dir: Path, final_state: dict, inference_gate: dict | None = None, generation_smoke: dict | None = None, *, status: str | None = None, message: str | None = None) -> dict: """Write the lightweight run card summary after terminalization. The backend can canonicalize from state/gate/smoke, but summary.json is the first file many list views and manual audits inspect. Keep it terminal and hardware-accurate once the worker knows the final verdict. """ gate = inference_gate or {} smoke = generation_smoke or {} final_status = status or gate.get("status") or final_state.get("status") or "unknown" promise_validation = gate.get("promise_validation") if isinstance(gate.get("promise_validation"), dict) else final_state.get("promise_validation") if isinstance(final_state.get("promise_validation"), dict) else {} promise_fulfilled = bool(promise_validation.get("promise_fulfilled") or gate.get("promise_fulfilled") or final_status == "full_inference_success") app_boot_validation_status = promise_validation.get("app_boot_validation_status") or gate.get("app_boot_validation_status") or "" promise_validation_status = promise_validation.get("promise_validation_status") or gate.get("promise_validation_status") or ("fulfilled" if promise_fulfilled else "") payload = { "run_id": final_state.get("run_id") or os.environ.get("RUN_ID"), "kind": final_state.get("kind") or "universal_model_card_builder", "status": final_status, "message": message or gate.get("message") or final_state.get("message") or "", "model_id": final_state.get("model_id") or os.environ.get("MODEL_ID", ""), "target_space": final_state.get("target_space") or os.environ.get("TARGET_SPACE_ID", ""), "target_space_url": final_state.get("target_space_url") or (f"https://huggingface.co/spaces/{final_state.get('target_space')}" if final_state.get("target_space") else ""), "selected_hardware": final_state.get("selected_hardware") or os.environ.get("SELECTED_HARDWARE", ""), "created_by": final_state.get("created_by") or os.environ.get("HF_USERNAME", ""), "bucket_source": final_state.get("bucket_source") or os.environ.get("BUCKET_SOURCE", ""), "health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")), "generation_smoke_passed": bool((((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success") and promise_fulfilled), "demo_usable_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("demo_usable_smoke_passed") is True) or smoke.get("demo_usable_smoke_passed") is True or smoke.get("status") == "demo_usable_smoke_passed"), "canonical_promise_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("canonical_promise_smoke_passed") is True) or smoke.get("canonical_promise_smoke_passed") is True or smoke.get("status") == "success"), "validation_level": smoke.get("validation_level") or "", "app_boot_validation_status": app_boot_validation_status, "promise_validation_status": promise_validation_status, "promise_fulfilled": promise_fulfilled, "ui_status": gate.get("ui_status") or final_state.get("ui_status") or final_status, "ui_badge": gate.get("ui_badge") or final_state.get("ui_badge") or "", "failure_type": "" if final_status == "full_inference_success" else (smoke.get("failure_type") or gate.get("failure_type") or (final_state.get("details") or {}).get("failure_type") or (final_state.get("details") or {}).get("repair_failure_type") or (final_state.get("repair_outcome") or {}).get("failure_type") or final_state.get("failure_type") or ""), "repair_outcome_status": (final_state.get("repair_outcome") or {}).get("post_repair_validation") or (final_state.get("details") or {}).get("repair_outcome_status") or "", "active_error": None if final_status == "full_inference_success" else (final_state.get("active_error") or ""), "validation_error": "" if final_status == "full_inference_success" else (final_state.get("validation_error") or ""), "job_exit_code": 0 if final_status == "full_inference_success" else final_state.get("job_exit_code", 1), "updated_at": now(), } write_json(run_dir / "summary.json", payload) return payload SAFE_SPACE_PYTHON_VERSIONS = {"3.10", "3.11", "3.12"} DEFAULT_SPACE_PYTHON_VERSION = "3.10" def normalize_space_python_version(value) -> tuple[str, bool, str]: raw = str(value or "").strip().strip("\"'") lowered = raw.lower().replace("python", "").strip() match = re.search(r"(3)\.(\d+)(?:\.\d+)?", lowered) if match: normalized = f"{match.group(1)}.{match.group(2)}" else: normalized = lowered if normalized in SAFE_SPACE_PYTHON_VERSIONS: return normalized, normalized != raw, "allowed" return DEFAULT_SPACE_PYTHON_VERSION, True, f"unsupported_or_ambiguous:{raw or 'missing'}" def requirements_has_package(lines: list[str], package: str) -> bool: wanted = package.lower().replace("_", "-") for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped: continue name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") if name == wanted: return True return False def workspace_app_imports_torch(workspace: Path) -> bool: app_path = workspace / "app.py" if not app_path.exists(): return False text = app_path.read_text(encoding="utf-8", errors="ignore") return bool(re.search(r"(?m)^\s*(import\s+torch\b|from\s+torch\b)", text)) def _artifact_entry(run_dir: Path, rel_path: str, *, kind: str = "file") -> dict: path = run_dir / rel_path if kind == "folder": present = path.exists() and any(child.is_file() for child in path.rglob("*")) else: present = path.exists() and path.is_file() payload = {"path": rel_path, "kind": kind, "present": bool(present)} if present and kind == "file": try: payload["size"] = path.stat().st_size except Exception: pass return payload def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, reason: str = "snapshot") -> dict: """Snapshot the artifacts that the worker actually wrote to the mounted bucket. The web UI should prefer this manifest over guessed paths. It is written during failures as well as success so a run that created a Space and then crashed can still expose traces, logs and reports in the dock. """ run_id = os.environ.get("RUN_ID", "") bucket_source = os.environ.get("BUCKET_SOURCE", "") artifacts = [ _artifact_entry(run_dir, "events.jsonl"), _artifact_entry(run_dir, "state.json"), _artifact_entry(run_dir, "summary.json"), _artifact_entry(run_dir, "live_status.json"), _artifact_entry(run_dir, "report.md"), _artifact_entry(run_dir, "model_analysis.json"), _artifact_entry(run_dir, "analysis_inputs/model_card.md"), _artifact_entry(run_dir, "analysis_inputs/model_card_source.json"), _artifact_entry(run_dir, "analysis_inputs/model_repo_tree.json"), _artifact_entry(run_dir, "analysis_inputs/prescan_summary.json"), _artifact_entry(run_dir, "analysis_inputs/source_policy.md"), _artifact_entry(run_dir, "MODEL_RECIPE.json"), _artifact_entry(run_dir, "CONTEXT_INDEX.json"), _artifact_entry(run_dir, "SOURCE_PRIORITY.md"), _artifact_entry(run_dir, "planning/pi_feasibility_brief.json"), _artifact_entry(run_dir, "planning/pi_implementation_plan.json"), _artifact_entry(run_dir, "planning/worker_plan_review.json"), _artifact_entry(run_dir, "planning/model_card_grounding_review.json"), _artifact_entry(run_dir, "planning/PI_SOURCE_USAGE.json"), _artifact_entry(run_dir, "hardware_strategy.json"), _artifact_entry(run_dir, "hardware_intent.json"), _artifact_entry(run_dir, "hardware_attempts.json"), _artifact_entry(run_dir, "output_type_resolution.json"), _artifact_entry(run_dir, "inference_gate.json"), _artifact_entry(run_dir, "repair_outcome.json"), _artifact_entry(run_dir, "space_runtime.json"), _artifact_entry(run_dir, "tests/generation_smoke.json"), _artifact_entry(run_dir, "tests/api_schema.json"), _artifact_entry(run_dir, "generated/TECHNICAL_BLOCKERS.json"), _artifact_entry(run_dir, "repair/INCIDENT_BRIEF.md"), _artifact_entry(run_dir, "repair/DEPENDENCY_ERROR_BRIEF.md"), _artifact_entry(run_dir, "repair/PI_DIAGNOSIS_GOAL.md"), _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS_PACKET.json"), _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS_GOAL.md"), _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS.json"), _artifact_entry(run_dir, "repair/REPAIR_DECISION.json"), _artifact_entry(run_dir, "repair/REPAIR_TASK_PACKET.json"), _artifact_entry(run_dir, "repair/RECIPE_AWARE_REPAIR_PACKET.json"), _artifact_entry(run_dir, "repair/PI_TASK_PACKET.json"), _artifact_entry(run_dir, "repair/FROZEN_REPAIR_DECISION.json"), _artifact_entry(run_dir, "repair/LOG_EVIDENCE_PACKET.json"), _artifact_entry(run_dir, "repair/PATCH_REFUSAL.json"), _artifact_entry(run_dir, "repair/actionable_error_excerpt.txt"), _artifact_entry(run_dir, "repair/error_signature.json"), _artifact_entry(run_dir, "repair/PI_PROMPT_BUDGET_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_PROMPT_BUDGET_memory_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_PROMPT_BUDGET_repair_patch.json"), _artifact_entry(run_dir, "pi_prompt_budget.json"), _artifact_entry(run_dir, "pi_call_fingerprints.json"), _artifact_entry(run_dir, "pi_call_fingerprint.json"), _artifact_entry(run_dir, "repair/PI_CALL_FINGERPRINT_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_CALL_FINGERPRINT_memory_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_CALL_FINGERPRINT_repair_patch.json"), _artifact_entry(run_dir, "repair/REPAIR_BRIEF.md"), _artifact_entry(run_dir, "repair/REPAIR_PLAN.md"), _artifact_entry(run_dir, "repair/REPAIR_SUMMARY.md"), _artifact_entry(run_dir, "repair/REPAIR_OUTCOME.json"), _artifact_entry(run_dir, "repair/BLOCKAGE.json"), _artifact_entry(run_dir, "logs/pi_live_output.txt"), _artifact_entry(run_dir, "logs/pi_output.txt"), _artifact_entry(run_dir, "logs/pi_diagnosis_output.txt"), _artifact_entry(run_dir, "logs/pi_repair_output.txt"), _artifact_entry(run_dir, "logs/space_logs_build.txt"), _artifact_entry(run_dir, "logs/space_logs_runtime.txt"), _artifact_entry(run_dir, "logs/space_logs_index.json"), _artifact_entry(run_dir, "logs/space_runtime_snapshot.json"), _artifact_entry(run_dir, "logs/space_log_diagnostics.json"), _artifact_entry(run_dir, "traces/redacted", kind="folder"), _artifact_entry(run_dir, "generated", kind="folder"), _artifact_entry(run_dir, "analysis_inputs", kind="folder"), _artifact_entry(run_dir, "repair", kind="folder"), _artifact_entry(run_dir, "logs", kind="folder"), ] payload = { "run_id": run_id, "bucket_source": bucket_source, "updated_at": now(), "reason": reason, "artifacts": artifacts, "present_paths": [a["path"] for a in artifacts if a.get("present")], } write_json(run_dir / "artifact_manifest.json", payload) if os.environ.get("ASF_EVAL_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"}: try: publish_eval_record(run_dir, phase=f"manifest:{reason}", events_path=events_path) except Exception: pass if events_path is not None: try: append_event(events_path, "artifact_manifest", "success", "Updated run artifact manifest", {"reason": reason, "present_count": len(payload["present_paths"])}) except Exception: pass return payload def update_state(run_dir: Path, patch: dict) -> dict: path = run_dir / "state.json" current = load_json_if_exists(path) if path.exists() else {} if not isinstance(current, dict): current = {} merged = {**current, **patch, "updated_at": now()} write_json(path, merged) return merged def redact_text(text: str | None) -> str: if not text: return "" value = text for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]: secret = os.environ.get(secret_name) if secret: value = value.replace(secret, "[REDACTED]") value = re.sub(r"Bearer\s+[A-Za-z0-9_\-.=]+", "Bearer [REDACTED]", value) value = re.sub(r"hf_[A-Za-z0-9_\-]{10,}", "hf_[REDACTED]", value) return value class AuthRefreshRequired(RuntimeError): """Raised when an OAuth/JWT token is expired or too close to expiry.""" def _decode_jwt_claims_unverified(token: str) -> dict: """Decode JWT claims without verification, only to inspect non-sensitive expiry metadata. This never returns or writes the raw token. Opaque PAT-style tokens are supported by returning an empty dict so they remain usable with an `unknown` expiry state. """ try: parts = (token or "").split(".") if len(parts) < 2: return {} payload = parts[1] payload += "=" * ((4 - len(payload) % 4) % 4) raw = base64.urlsafe_b64decode(payload.encode("utf-8")) data = json.loads(raw.decode("utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict: issued_at = now() if not token: return { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": False, "token_kind": "missing", "expiry_known": False, "status": "missing", "safe_for_phase": False, "minimum_required_seconds": minimum_required_seconds, } claims = _decode_jwt_claims_unverified(token) exp = claims.get("exp") if isinstance(claims, dict) else None iat = claims.get("iat") if isinstance(claims, dict) else None token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown") payload = { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": True, "token_kind": token_kind, "expiry_known": exp is not None, "minimum_required_seconds": int(minimum_required_seconds or 0), "token_value": "[REDACTED]", } if iat is not None: try: payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat() except Exception: pass if exp is None: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"}) return payload try: exp_int = int(exp) seconds_left = exp_int - int(time.time()) payload.update({ "expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(), "seconds_until_expiry": seconds_left, }) except Exception: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"}) return payload if seconds_left <= 0: payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"}) elif minimum_required_seconds and seconds_left < minimum_required_seconds: payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"}) else: payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"}) return payload def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict: payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds) payload["phase"] = phase safe_payload = {k: v for k, v in payload.items() if k != "token_value"} try: probes_dir = run_dir / "auth_probes" probes_dir.mkdir(parents=True, exist_ok=True) write_json(probes_dir / f"{phase}.json", safe_payload) write_json(run_dir / "auth_status.json", safe_payload) except Exception: pass status = str(payload.get("status") or "unknown") if events_path: event_status = "success" if payload.get("safe_for_phase") else "failed" if status == "unknown": event_status = "warning" append_event( events_path, "auth_probe", event_status, f"HF OAuth/token expiry check for {phase}: {status}", safe_payload, ) if raise_on_unsafe and not payload.get("safe_for_phase"): raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.") return payload def is_auth_expired_error(error: Exception | str) -> bool: text = str(error or "").lower() return any(marker in text for marker in [ "oauth token has expired", "exp claim timestamp check failed", "token has expired", "jwt expired", ]) def classify_repair_validation_error(error: Exception | str) -> dict: """Classify post-repair validation failures without losing the repair result. A repair can be correctly diagnosed, patched, and uploaded while the final validation is inconclusive because OAuth expired. Keep that distinct from a model/runtime repair failure so the UI and audit trail do not imply that Pi's patch necessarily failed. """ text = str(error or "") if isinstance(error, AuthRefreshRequired) or is_auth_expired_error(text): return { "post_repair_validation": "inconclusive_auth_expired", "failure_type": "repair_validation_inconclusive_auth", "terminal_status": "auth_refresh_required", "message": "Repair progress could not be validated because HF auth expired or is too close to expiry.", } return { "post_repair_validation": "failed", "failure_type": "repair_validation_failed", "terminal_status": "failed", "message": "Repair was applied, but the repaired Space did not pass live validation.", } def write_repair_outcome(run_dir: Path, events_path: Path | None = None, **updates) -> dict: """Write a cumulative repair outcome artifact. Keep it at the run root for list-view consumers and mirror it inside repair/ for manual audits. The payload is intentionally compact and redacted; full logs remain in logs/ and repair/ artifacts. """ root_path = run_dir / "repair_outcome.json" existing = load_json_if_exists(root_path) if root_path.exists() else {} if not isinstance(existing, dict): existing = {} payload = { "schema_version": "repair_outcome.v1", **existing, **{k: v for k, v in updates.items() if v is not None}, "updated_at": now(), } write_json(root_path, payload) try: write_json(run_dir / "repair" / "REPAIR_OUTCOME.json", payload) except Exception: pass if events_path: append_event( events_path, "repair_outcome", str(payload.get("post_repair_validation") or payload.get("repair_status") or payload.get("repair_decision") or "updated"), "Repair outcome artifact updated", payload, ) return payload def minimum_auth_seconds_for_phase(phase: str) -> int: env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_") raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "") if raw: try: return max(0, int(raw)) except Exception: pass defaults = { "before_space_create": 900, "before_upload": 900, "before_initial_validation": 1800, "before_repair": 1800, "before_repair_upload": 1800, "before_repair_validation": 1800, "before_linked_space_test": 1800, } return defaults.get(phase, 0) def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict: """Ensure Pi and HF tooling see the same private HF token safely. HF Jobs receive HF_TOKEN as a secret. Pi, huggingface_hub, gradio_client, and the hf CLI can use either HF_TOKEN or HUGGING_FACE_HUB_TOKEN depending on the tool/version, so expose both aliases inside the isolated Job process. Never write the token value to artifacts. """ token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "" if token: os.environ.setdefault("HF_TOKEN", token) os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token) os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation")) payload = { "hf_token_present": bool(token), "hf_token_length": len(token) if token else 0, "hugging_face_hub_token_alias_present": bool(os.environ.get("HUGGING_FACE_HUB_TOKEN")), "hf_username": os.environ.get("HF_USERNAME") or "", "bucket_source": os.environ.get("BUCKET_SOURCE") or "", "target_space_id": os.environ.get("TARGET_SPACE_ID") or "", "token_value": "[REDACTED]" if token else "", "token_kind": expiry.get("token_kind"), "expiry_known": expiry.get("expiry_known"), "expires_at": expiry.get("expires_at"), "seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"), "auth_risk": expiry.get("auth_risk"), "safe_for_long_build": bool(expiry.get("safe_for_phase")), "minimum_required_seconds": expiry.get("minimum_required_seconds"), } run_dir.mkdir(parents=True, exist_ok=True) write_json(run_dir / "token_context.json", payload) if events_path: append_event(events_path, "token_context", "success" if token else "warning", "Verified HF token context for Pi/HF operations", {k: v for k, v in payload.items() if k != "token_value"}) return payload def pi_tooling_context_note() -> str: return """HF tooling context: - You are running inside an HF Job with an HF_TOKEN secret configured. - For gated/private base models, LoRA adapters, PEFT adapters, or private Hub files, the generated Space must read `HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")` and pass `token=HF_TOKEN` to `from_pretrained`, `hf_hub_download`, `snapshot_download`, and `load_lora_weights` when those APIs are used. Never print the token. - For ZeroGPU/Diffusers/LoRA GPU apps, keep the app compatible with both HF ZeroGPU and fixed GPU fallback: import `spaces` before CUDA-touching imports, protect the heavy inference path with `@spaces.GPU`, and never leave a failed model load as a silent global `None` that later makes `/generate` fail with "pipeline not loaded". If using lazy/cache loading, surface real load errors clearly and keep `/generate` repairable. - The Factory exposes both HF_TOKEN and HUGGING_FACE_HUB_TOKEN aliases for huggingface_hub, gradio_client, hf CLI, and Pi provider access. - You may use the token through normal tools when needed to inspect the model/Space or validate private resources. - Never print, write, echo, or commit token values. Do not include Authorization headers in artifacts. - If a Hub operation fails with 401/403/quota/billing, report it as evidence instead of retrying blindly. """ def _agent_trace_json_safe(payload): try: return json.loads(redact_text(json.dumps(payload, ensure_ascii=False))) except Exception: return {"value": redact_text(str(payload))[:8000]} def write_agent_trace_record( run_dir: Path, *, phase: str, event: str, status: str = "info", message: str = "", data: dict | None = None, text: str | None = None, artifacts: list[str] | None = None, ): """Append one run-level agent trace record to the redacted journal only. Pi can be invoked several times during a run: initial build, blockage diagnosis, and optional repair patch. The individual Pi session files and stdout logs remain useful, but this journal is the canonical narrative tying those phases together into one auditable agent run. Security note: raw Pi session traces may contain credentials if a tool or assistant accidentally prints environment values. The Factory therefore does not persist RAW Pi traces to the run bucket. Only redacted traces are archived and exposed in the UI. """ try: payload = { "ts": now(), "phase": phase, "event": event, "status": status, "message": message, "data": data or {}, "artifacts": artifacts or [], } if text is not None: payload["text"] = text redacted_dir = run_dir / "traces" / "redacted" redacted_dir.mkdir(parents=True, exist_ok=True) redacted_payload = _agent_trace_json_safe(payload) redacted_line = json.dumps(redacted_payload, ensure_ascii=False) with (redacted_dir / "agent_trace.jsonl").open("a", encoding="utf-8") as f: f.write(redacted_line + "\n") except Exception: # Trace journaling must never break the build/repair path. pass def append_agent_trace_artifact(run_dir: Path, *, phase: str, event: str, artifact: str, text: str | None = None, status: str = "success", data: dict | None = None): tail = None if text is None else text[-20000:] write_agent_trace_record(run_dir, phase=phase, event=event, status=status, message=f"Agent phase wrote {artifact}", data=data or {}, text=tail, artifacts=[artifact]) def safe_details(details: dict | None) -> dict: if not details: return {} try: return json.loads(redact_text(json.dumps(details, ensure_ascii=False))) except Exception: return {"redacted_details": redact_text(str(details))[-4000:]} def exception_payload(exc: Exception) -> dict: """Extract the most useful redacted details from HF Hub / HTTP exceptions.""" payload = { "error": redact_text(str(exc))[:4000], "exception_type": type(exc).__name__, } response = getattr(exc, "response", None) if response is not None: status_code = getattr(response, "status_code", None) if status_code is not None: payload["status_code"] = status_code try: text_body = response.text except Exception: text_body = "" if text_body: payload["response_text"] = redact_text(text_body)[:4000] payload["hf_error_detail"] = redact_text(text_body)[:4000] try: json_body = response.json() except Exception: json_body = None if json_body is not None: payload["response_json"] = safe_details(json_body) if isinstance(json_body, dict): for key in ("error", "message", "detail", "details"): value = json_body.get(key) if value: payload["hf_error_message"] = redact_text(value if isinstance(value, str) else json.dumps(value, ensure_ascii=False))[:4000] break for attr in ("server_message", "request_id"): value = getattr(exc, attr, None) if value: payload[attr] = redact_text(str(value))[:1000] return payload def eval_enabled() -> bool: return os.environ.get("ASF_EVAL_RECORD_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"} def eval_safe_hash(value: str | None, prefix: str = "h") -> str: raw = (value or "").strip() salt = os.environ.get("ASF_EVAL_SALT", "").strip() if not raw: return "" if salt: digest = hmac.new(salt.encode("utf-8"), raw.encode("utf-8"), hashlib.sha256).hexdigest()[:24] else: digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] return f"{prefix}_{digest}" def eval_load_json(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8", errors="replace")) except Exception as exc: return {"_parse_error": str(exc)} def eval_load_events(path: Path) -> list[dict]: if not path.exists(): return [] events = [] for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): try: evt = json.loads(line) if isinstance(evt, dict): events.append(evt) except Exception: pass return events def eval_parse_ts(value: str | None) -> float | None: if not value: return None try: return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() except Exception: return None def eval_compact_timeline(events: list[dict]) -> list[dict]: stages: dict[str, dict] = {} order: list[str] = [] for evt in events: step = str(evt.get("step") or "unknown")[:80] status = str(evt.get("status") or "")[:80] ts = str(evt.get("ts") or "") tsv = eval_parse_ts(ts) if step not in stages: stages[step] = {"stage": step, "first_ts": ts, "last_ts": ts, "status": status, "event_count": 0} order.append(step) item = stages[step] item["last_ts"] = ts or item.get("last_ts") item["status"] = status or item.get("status") item["event_count"] = int(item.get("event_count") or 0) + 1 if tsv is not None: if "_first" not in item: item["_first"] = tsv item["_last"] = tsv timeline = [] for step in order: item = dict(stages[step]) first = item.pop("_first", None) last = item.pop("_last", None) if first is not None and last is not None and last >= first: item["duration_ms"] = int((last - first) * 1000) timeline.append(item) return timeline[-80:] def eval_redacted_tail(path: Path, limit: int = 2400) -> str: if os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() not in {"1", "true", "yes", "on"}: return "" try: return redact_text(path.read_text(encoding="utf-8", errors="replace"))[-limit:] except Exception: return "" def eval_len_if_list(value) -> int: return len(value) if isinstance(value, list) else 0 def eval_seconds_bucket(value) -> str: try: seconds = int(value) except Exception: return "unknown" if seconds < 0: return "expired" if seconds < 15 * 60: return "lt_15m" if seconds < 60 * 60: return "lt_1h" if seconds < 4 * 60 * 60: return "lt_4h" if seconds < 8 * 60 * 60: return "lt_8h" return "gte_8h" def eval_v191_plus_signals(run_dir: Path, analysis: dict, generation_smoke: dict, *, contract: dict | None = None, requirements_policy: dict | None = None, auth_status: dict | None = None, repair_outcome: dict | None = None, worker_plan_review: dict | None = None, grounding_review: dict | None = None) -> dict: """Return privacy-safe metrics for features added from v191.1 onward.""" analysis = analysis if isinstance(analysis, dict) else {} generation_smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = contract if isinstance(contract, dict) else {} requirements_policy = requirements_policy if isinstance(requirements_policy, dict) else {} auth_status = auth_status if isinstance(auth_status, dict) else {} repair_outcome = repair_outcome if isinstance(repair_outcome, dict) else {} worker_plan_review = worker_plan_review if isinstance(worker_plan_review, dict) else {} grounding_review = grounding_review if isinstance(grounding_review, dict) else {} build_risk = analysis.get("build_risk") if isinstance(analysis.get("build_risk"), dict) else {} kernel_strategy = analysis.get("kernel_strategy") if isinstance(analysis.get("kernel_strategy"), dict) else {} if not kernel_strategy and isinstance(analysis.get("metadata"), dict): kernel_strategy = analysis["metadata"].get("kernel_strategy") if isinstance(analysis["metadata"].get("kernel_strategy"), dict) else {} grounding_source = grounding_review.get("model_card_source") if isinstance(grounding_review.get("model_card_source"), dict) else {} return { "schema_version": "v191_plus_eval_signals.v1", "platform_dependency_policy": { "present": bool(requirements_policy), "status": requirements_policy.get("status") or "", "removed_platform_pin_count": eval_len_if_list(requirements_policy.get("removed_pins")), "normalized_platform_line_count": eval_len_if_list(requirements_policy.get("normalized_platform_lines")), "injected_platform_line_count": eval_len_if_list(requirements_policy.get("injected_platform_lines")), "torch_added": bool(requirements_policy.get("torch_added")), }, "auth_context": { "present": bool(auth_status), "status": auth_status.get("status") or "", "token_kind": auth_status.get("token_kind") or "", "expiry_known": bool(auth_status.get("expiry_known")), "seconds_until_expiry_bucket": eval_seconds_bucket(auth_status.get("seconds_until_expiry")), "safe_for_phase": bool(auth_status.get("safe_for_phase")), }, "model_scan": { "build_risk_level": build_risk.get("level") or analysis.get("build_risk_level") or "", "build_risk_signal_count": eval_len_if_list(build_risk.get("signals")), "build_risk_visibility_only": bool(build_risk.get("visibility_only", True)) if build_risk else True, "recommended_session_minutes": build_risk.get("recommended_session_minutes"), "kernel_strategy_present": bool(kernel_strategy), "native_kernel_detected": bool(kernel_strategy.get("native_kernel_detected") or kernel_strategy.get("detected")), "kernel_signal_count": eval_len_if_list(kernel_strategy.get("signals")), "kernel_candidate_count": eval_len_if_list(kernel_strategy.get("candidates")), }, "contract_validation": { "contract_present": bool(contract), "full_inference_implemented": bool(contract.get("full_inference_implemented")), "validation_level": contract.get("validation_level") or "", "requires_gpu": bool(contract.get("requires_gpu")), "blockers_count": int(contract.get("blockers_count") or 0) if str(contract.get("blockers_count") or "0").isdigit() else 0, "generation_smoke_status": generation_smoke.get("status") or "", "generation_smoke_skipped": str(generation_smoke.get("status") or "").lower() == "skipped", "generation_smoke_skip_reason": generation_smoke.get("skip_reason") or "", }, "repair_outcome": { "present": bool(repair_outcome), "repair_decision": repair_outcome.get("repair_decision") or repair_outcome.get("decision") or "", "patch_applied": bool(repair_outcome.get("patch_applied")), "upload_success": bool(repair_outcome.get("upload_success")), "post_repair_validation": repair_outcome.get("post_repair_validation") or "", "failure_type": repair_outcome.get("failure_type") or "", }, "planning": { "worker_plan_review_present": bool(worker_plan_review), "status": worker_plan_review.get("status") or "", "declared_strategy": worker_plan_review.get("declared_strategy") or "", "worker_recommendation": worker_plan_review.get("worker_recommendation") or "", "warning_count": eval_len_if_list(worker_plan_review.get("warnings")), }, "model_card_grounding": { "present": bool(grounding_review), "status": grounding_review.get("status") or "", "source_available": bool(grounding_review.get("source_available")), "model_card_present": bool(grounding_review.get("model_card_present")), "source": grounding_source.get("source") or "", "resolved_card_file": grounding_source.get("resolved_card_file") or "", "fallback_used": bool(grounding_source.get("fallback_used")), "pi_evidence_present": bool(grounding_review.get("pi_evidence_present")), "pi_evidence_count": int(grounding_review.get("pi_evidence_count") or 0), "warning_count": eval_len_if_list(grounding_review.get("warnings")), "warnings": [str(w)[:120] for w in (grounding_review.get("warnings") if isinstance(grounding_review.get("warnings"), list) else [])[:8]], }, } def eval_terminal_status(status: str, phase: str = "") -> bool: status = str(status or "").strip().lower() phase = str(phase or "").strip().lower() if phase in {"final", "failure"}: return True return status in { "done", "success", "failed", "cancelled", "canceled", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "demo_usable_full_promise_not_verified", "interactive_app_available_smoke_failed", "manual_test_required_smoke_failed", "partial_validation", "technical_blocker", "manual_hardware_required", } def eval_event_seen(events: list[dict], step: str, status: str | None = None) -> bool: for event in events or []: if str(event.get("step") or "") != step: continue if status is None or str(event.get("status") or "") == status: return True return False def eval_health_passed(inference_gate: dict, generation_smoke: dict, events: list[dict], status: str) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} if signals.get("health_passed") is True: return True if str(status or "") in {"full_inference_success", "full_inference_candidate_health_passed", "health_only"}: return True if eval_event_seen(events, "api_validation", "success") or eval_event_seen(events, "repair_validation", "success"): return True return False def eval_generation_smoke_passed(inference_gate: dict, generation_smoke: dict) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} return bool(signals.get("generation_smoke_passed") is True or generation_smoke.get("status") == "success") def eval_failure_summary(status: str, state: dict, inference_gate: dict, generation_smoke: dict, health_passed: bool, smoke_passed: bool) -> dict: failure_type = generation_smoke.get("failure_type") or inference_gate.get("failure_type") or "" failure_owner = generation_smoke.get("failure_owner") or inference_gate.get("failure_owner") or "" missing_argument = generation_smoke.get("missing_argument") or inference_gate.get("missing_argument") or "" reason = missing_argument or failure_type or str(state.get("message") or inference_gate.get("message") or "")[:240] if not failure_owner: if failure_type == "validator_request_error" or missing_argument: failure_owner = "factory_validator" elif smoke_passed: failure_owner = "" elif health_passed and str(generation_smoke.get("status") or "") == "failed": failure_owner = "unknown" return { "failure_type": failure_type, "failure_owner": failure_owner, "failure_reason": reason, "missing_argument": missing_argument, } def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: bool, full_inference_verified: bool, inference_gate: dict) -> str: status = str(status or "").strip() if status in {"cancelled", "canceled"}: return "cancelled" if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"): return "manual_action_required" if status in {"technical_blocker", "technical_blocker_boot_only"}: return "technical_blocker" if full_inference_verified: return "success" if status in {"full_inference_candidate_health_passed", "health_only"} or (health_passed and not smoke_passed): return "partial_validation" if phase == "failure" or status in {"failed", "error"}: return "failed" if eval_terminal_status(status, phase): return "partial_validation" if health_passed else "failed" return "running" def publish_eval_record(run_dir: Path, *, phase: str, events_path: Path | None = None) -> dict: """Write an anonymized cross-run evaluation record to the optional eval bucket. This intentionally stores structured metrics and hashed identifiers, not raw generated code, prompts, tokens, user bucket paths, or target Space IDs. """ if not eval_enabled(): return {"enabled": False} # Jobs only write the anonymized eval record back into the user's run bucket. # The ASF backend later publishes this record to the operator eval bucket # mounted on the ASF Space. User Jobs must never mount or write the private # operator archive directly. run_id = os.environ.get("RUN_ID", run_dir.name) events = eval_load_events(events_path or (run_dir / "events.jsonl")) state = eval_load_json(run_dir / "state.json") analysis = eval_load_json(run_dir / "model_analysis.json") hardware_strategy = eval_load_json(run_dir / "hardware_strategy.json") hardware_attempts = eval_load_json(run_dir / "hardware_attempts.json") inference_gate = eval_load_json(run_dir / "inference_gate.json") generation_smoke = eval_load_json(run_dir / "tests" / "generation_smoke.json") generation_smoke_retry = eval_load_json(run_dir / "tests" / "generation_smoke_payload_retry.json") endpoint_discovery_payload = eval_load_json(run_dir / "tests" / "gradio_endpoint_discovery.json") manual_validation_status = eval_load_json(run_dir / "manual_validation_status.json") linked_validations_payload = eval_load_json(run_dir / "linked_validations.json") space_runtime = eval_load_json(run_dir / "space_runtime.json") manifest = eval_load_json(run_dir / "artifact_manifest.json") repair_decision = eval_load_json(run_dir / "repair" / "REPAIR_DECISION.json") blockage = eval_load_json(run_dir / "repair" / "BLOCKAGE.json") repair_outcome = eval_load_json(run_dir / "repair_outcome.json") or eval_load_json(run_dir / "repair" / "REPAIR_OUTCOME.json") requirements_policy = eval_load_json(run_dir / "generated" / "requirements_policy.json") or eval_load_json(run_dir / "requirements_policy.json") auth_status = eval_load_json(run_dir / "auth_status.json") contract = eval_load_json(run_dir / "generated" / "INFERENCE_CONTRACT.json") worker_plan_review = eval_load_json(run_dir / "planning" / "worker_plan_review.json") grounding_review = eval_load_json(run_dir / "planning" / "model_card_grounding_review.json") or (worker_plan_review.get("model_card_grounding") if isinstance(worker_plan_review.get("model_card_grounding"), dict) else {}) pi_resolution = state.get("pi_model_resolution") if isinstance(state.get("pi_model_resolution"), dict) else {} status = str(state.get("status") or inference_gate.get("status") or phase or "unknown") failure_message = str(state.get("message") or (state.get("details") or {}).get("error") or "")[:1200] started_at = events[0].get("ts") if events else (state.get("created_at") or os.environ.get("LAUNCHED_AT") or "") finished_at = state.get("updated_at") or (events[-1].get("ts") if events else now()) model_id = os.environ.get("MODEL_ID") or state.get("model_id") or analysis.get("model_id") or "" target_space = state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" include_model_id = os.environ.get("ASF_EVAL_INCLUDE_MODEL_ID", "").strip().lower() in {"1", "true", "yes", "on"} anon_run_id = eval_safe_hash(run_id, "run") health_passed = eval_health_passed(inference_gate, generation_smoke, events, status) generation_smoke_passed = eval_generation_smoke_passed(inference_gate, generation_smoke) promise_validation = inference_gate.get("promise_validation") if isinstance(inference_gate.get("promise_validation"), dict) else {} promise_fulfilled = bool(promise_validation.get("promise_fulfilled") or inference_gate.get("promise_fulfilled") or status == "full_inference_success") full_inference_verified = bool(promise_fulfilled and status == "full_inference_success" and health_passed and generation_smoke_passed) process_completed = eval_terminal_status(status, phase) failure_summary = eval_failure_summary(status, state, inference_gate, generation_smoke, health_passed, generation_smoke_passed) verdict = eval_verdict(status, phase, health_passed, generation_smoke_passed, full_inference_verified, inference_gate) linked_rows = linked_validations_payload.get("validations") if isinstance(linked_validations_payload.get("validations"), list) else [] linked_success_rows = [row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() == "success"] manual_applied = str(manual_validation_status.get("status") or "").lower() == "success" or bool(linked_success_rows) effective_status = str((manual_validation_status if manual_applied else {}).get("effective_status") or (linked_success_rows[-1].get("effective_status") if linked_success_rows else "") or ("success" if full_inference_verified else verdict)) endpoint_names = endpoint_discovery_payload.get("discovered_api_names") or endpoint_discovery_payload.get("candidates") or [] smoke_retry_meta = generation_smoke.get("auto_retry") if isinstance(generation_smoke.get("auto_retry"), dict) else {} record = { "process_completed": process_completed, "verdict": verdict, "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "schema_version": "1.4", "app_version": os.environ.get("ASF_VERSION", "unknown"), "phase": phase, "anonymous_run_id": anon_run_id, "run_id_hash": eval_safe_hash(run_id, "run"), "anonymous_user_hash": eval_safe_hash(os.environ.get("HF_USERNAME"), "user"), "salt_configured": bool(os.environ.get("ASF_EVAL_SALT")), "run_kind": state.get("kind") or ("validate_existing_space" if os.environ.get("API_NAME") else "universal_model_card_builder"), "started_at": started_at, "finished_at": finished_at, "input_model": { "model_id_redacted": not include_model_id, "model_id": model_id if include_model_id else "", "model_hash": eval_safe_hash(model_id, "model"), "pipeline_tag": analysis.get("pipeline_tag"), "library_name": analysis.get("library_name"), "default_model_target": bool(analysis.get("default_model_target")), }, "config": { "implementation_mode": os.environ.get("IMPLEMENTATION_MODE", ""), "expected_output_type": os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "pi_model_requested": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "pi_model_effective": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "try_zero_gpu_first": os.environ.get("TRY_ZERO_GPU_FIRST", ""), "allow_fixed_gpu_fallback": os.environ.get("ALLOW_FIXED_GPU_FALLBACK", ""), "preferred_gpu": os.environ.get("PREFERRED_SPACE_HARDWARE", ""), "fallback_gpu": os.environ.get("FALLBACK_SPACE_HARDWARE", ""), }, "outcome": { "status": status, "verdict": verdict, "process_completed": process_completed, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "app_boot_validation_status": inference_gate.get("app_boot_validation_status") or "", "promise_validation_status": inference_gate.get("promise_validation_status") or promise_validation.get("promise_validation_status") or "", "promise_fulfilled": bool(promise_fulfilled), "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "message_redacted": redact_text(failure_message)[:1200], "target_space_hash": eval_safe_hash(target_space, "space"), "space_created": bool(target_space), "space_runtime_stage": space_runtime.get("stage"), "space_runtime_hardware": space_runtime.get("hardware") or space_runtime.get("requested_hardware"), }, "timeline": eval_compact_timeline(events), "hardware": { "selected": hardware_strategy.get("selected_hardware") or state.get("selected_hardware") or "", "strategy": hardware_strategy.get("strategy") or hardware_attempts.get("strategy") or "", "attempts": hardware_strategy.get("attempts") or hardware_attempts.get("attempts") or state.get("hardware_attempts") or [], "single_space_per_run": True, }, "pi": { "requested_model": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "effective_model": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "model_mismatch": bool(pi_resolution.get("mismatch")), "build_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_output.txt"), "diagnosis_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_diagnosis_output.txt"), }, "recovery": { "entered": any(str(e.get("step", "")).startswith(("repair", "failure_diagnosis", "pi_diagnosis", "technical_blocker")) for e in events), "decision": repair_decision.get("action") or repair_decision.get("decision") or "", "confidence": repair_decision.get("confidence") or "", "blockage": bool(blockage), "blockage_category": blockage.get("category") or blockage.get("reason") or "", }, "validation": { "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "generation_smoke_status": generation_smoke.get("status") or "", "latency_seconds": generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "expected_output_type": generation_smoke.get("expected_output_type") or os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "inference_gate_status": inference_gate.get("status") or "", "manual_hardware_required": bool(inference_gate.get("manual_hardware_required")), "failure_type": failure_summary.get("failure_type") or "", "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "missing_argument": failure_summary.get("missing_argument") or "", }, "automatic_outcome": { "status": status, "verdict": verdict, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, }, "effective_outcome": { "automatic_verdict": verdict, "automatic_status": status, "effective_verdict": effective_status, "effective_status": effective_status, "manual_validation_applied": manual_applied, "source": "linked_space_test" if manual_applied else "automatic_run", }, "linked_validation": { "present": bool(linked_rows or manual_validation_status), "count": len(linked_rows), "success_count": len(linked_success_rows), "failure_count": len([row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() in {"failed", "failure", "error"}]), "status": "success" if manual_applied else str(manual_validation_status.get("status") or ""), "mode": manual_validation_status.get("space_test_policy_mode") or "", "api_name": manual_validation_status.get("api_name") or generation_smoke.get("api_name") or "", "latency_seconds": manual_validation_status.get("latency_seconds") or generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "hardware_used_for_validation": manual_validation_status.get("hardware_used_for_validation") or "", }, "endpoint_discovery": { "required": bool(endpoint_discovery_payload.get("endpoint_discovery_required") or endpoint_discovery_payload), "succeeded": bool(endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint")), "selected_endpoint": endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint") or "", "candidate_count": len(endpoint_names) if isinstance(endpoint_names, list) else 0, "excluded_health_endpoint": True, }, "smoke_retry": { "retried": bool(generation_smoke_retry or smoke_retry_meta.get("retried")), "reason": generation_smoke_retry.get("retry_reason") or smoke_retry_meta.get("retry_reason") or smoke_retry_meta.get("reason") or "", "attempts": generation_smoke_retry.get("attempts") or smoke_retry_meta.get("attempts") or (2 if generation_smoke_retry else 1), "passed_after_retry": bool(generation_smoke_retry and generation_smoke.get("status") == "success"), }, "v191_plus": eval_v191_plus_signals( run_dir, analysis, generation_smoke, contract=contract, requirements_policy=requirements_policy, auth_status=auth_status, repair_outcome=repair_outcome, worker_plan_review=worker_plan_review, grounding_review=grounding_review, ), "artifacts": { "present_count": len(manifest.get("present_paths") or []), "present_paths": sorted(str(p) for p in (manifest.get("present_paths") or []))[:120], }, "privacy": { "tokens_stored": False, "raw_prompts_stored": False, "generated_code_stored": False, "target_space_redacted": True, "user_bucket_redacted": True, "redacted_tails_enabled": os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() in {"1", "true", "yes", "on"}, "linked_run_ids_redacted": True, "endpoint_schema_stored": False, "validation_args_stored": False, "model_card_raw_stored": False, "pi_evidence_text_stored": False, "requirements_txt_stored": False, "auth_token_stored": False, }, } day = (finished_at or now())[:10] try: yyyy, mm, dd = day.split("-") except Exception: yyyy, mm, dd = now()[:10].split("-") dest = run_dir / "eval_record.json" write_json(dest, record) events_dest = run_dir / "events_compact.jsonl" events_dest.write_text("".join(json.dumps(item, ensure_ascii=False) + "\n" for item in record["timeline"]), encoding="utf-8") return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"} def _env_truthy(*names: str) -> bool: for name in names: value = os.environ.get(name, "") if str(value).strip().lower() in {"1", "true", "yes", "on"}: return True return False def _hardware_is_gpuish(selected_hardware: str | None) -> bool: value = (selected_hardware or "").strip().lower() if not value: return False if value in {"cpu-basic", "cpu-upgrade", "default-cpu-or-existing"}: return False return any(marker in value for marker in ["gpu", "a10g", "a100", "h100", "l4", "l40", "t4", "zero-"]) def _gpu_partial_autopause_mode() -> str: raw = str(os.environ.get("ASF_GPU_PARTIAL_AUTOPAUSE_MODE") or "recommended").strip().lower() return raw if raw in {"off", "recommended", "strict"} else "recommended" def should_pause_generated_space(final_status: str | None, target_space: str | None, selected_hardware: str | None = None, *, keep_failed_spaces_running: bool | None = None, health_semantic_passed: bool | None = None) -> dict: """Decide whether a generated Space should be paused after a terminal outcome. v198.26.7 keeps GPU partial cleanup non-aggressive by default: a reachable partial can still be useful and should be validated via Space Test/minimal smoke before pausing. `ASF_GPU_PARTIAL_AUTOPAUSE_MODE=strict` restores automatic pause for GPU partials; `recommended` emits a cost warning only. """ status = (final_status or "").strip().lower() target = (target_space or "").strip() hardware = (selected_hardware or "").strip() keep_running = _env_truthy("ASF_KEEP_FAILED_SPACES_RUNNING", "KEEP_FAILED_SPACES_RUNNING") if keep_failed_spaces_running is None else bool(keep_failed_spaces_running) base = {"final_status": status, "selected_hardware": hardware, "health_semantic_passed": health_semantic_passed} if keep_running: return {"should_pause": False, "reason": "keep_failed_spaces_running_enabled", **base} if not target: return {"should_pause": False, "reason": "no_target_space", **base} if status in {"full_inference_success", "success"}: return {"should_pause": False, "reason": "successful_run", **base} gpuish = _hardware_is_gpuish(hardware) if status == "technical_blocker_boot_only": should = gpuish return {"should_pause": should, "reason": "technical_blocker_boot_only_gpu" if should else "technical_blocker_boot_only_cpu", **base} failed_statuses = { "failed", "build_failed", "runtime_failed", "repair_failed", "repair_patch_failed", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard", "validation_failed", "dependency_error", "manual_hardware_required", "full_inference_failed", "generation_smoke_failed", } if status in failed_statuses or status.endswith("_failed"): return {"should_pause": True, "reason": "terminal_failure", **base} partial_statuses = { "partial", "partial_validation", "health_only", "full_inference_candidate_health_passed", "completed_with_warnings", "interactive_app_available_smoke_failed", "manual_test_required_smoke_failed", "demo_usable_full_promise_not_verified", } if status in partial_statuses and gpuish: mode = _gpu_partial_autopause_mode() unhealthy = health_semantic_passed is False if mode == "strict": return {"should_pause": True, "reason": "partial_gpu_strict_autopause" if not unhealthy else "partial_gpu_pipeline_unhealthy_strict_autopause", "pause_recommended": True, "autopause_mode": mode, **base} return {"should_pause": False, "reason": "partial_gpu_pause_recommended" if unhealthy else "partial_gpu_cost_warning_manual_validation_recommended", "pause_recommended": True, "autopause_mode": mode, **base} return {"should_pause": False, "reason": "status_not_pauseable", **base} def pause_generated_space_best_effort(target_space: str, token: str | None, run_dir: Path, events_path: Path, *, final_status: str | None = None, selected_hardware: str | None = None, reason: str | None = None) -> dict: """Pause a generated Space best-effort and publish cleanup_status.json. This helper must never mask the primary run outcome. It catches all exceptions, stores a redacted status artifact, and emits cleanup events. """ cleanup = { "space_pause_attempted": False, "space_pause_success": False, "target_space": target_space or "", "final_status": final_status or "", "selected_hardware": selected_hardware or "", "reason": reason or "", "updated_at": now(), } if not target_space: cleanup.update({"skipped": True, "skip_reason": "no_target_space"}) write_json(run_dir / "cleanup_status.json", cleanup) return cleanup if not token: cleanup.update({"skipped": True, "skip_reason": "missing_token"}) append_event(events_path, "cleanup", "warning", "Skipped Space pause because no HF token is available", {"target_space": target_space, "final_status": final_status or ""}) write_json(run_dir / "cleanup_status.json", cleanup) return cleanup cleanup["space_pause_attempted"] = True append_event(events_path, "cleanup", "started", "Pausing generated Space after terminal non-success outcome", {"target_space": target_space, "final_status": final_status or "", "selected_hardware": selected_hardware or "", "reason": reason or ""}) try: from huggingface_hub import HfApi api = HfApi(token=token) runtime = api.pause_space(repo_id=target_space, token=token) cleanup.update({ "space_pause_success": True, "runtime_repr": repr(runtime)[:1000], "updated_at": now(), }) append_event(events_path, "cleanup", "success", "Generated Space paused after terminal non-success outcome", {"target_space": target_space, "final_status": final_status or "", "selected_hardware": selected_hardware or ""}) except Exception as exc: cleanup.update({ "space_pause_success": False, "error_type": type(exc).__name__, "error": redact_text(str(exc))[:2000], "manual_action_required": True, "updated_at": now(), }) append_event(events_path, "cleanup", "warning", "Could not pause generated Space after terminal outcome", {"target_space": target_space, "error_type": type(exc).__name__, "error": redact_text(str(exc))[:1000]}) write_json(run_dir / "cleanup_status.json", cleanup) return cleanup def run_final_cleanup_if_needed(run_dir: Path, events_path: Path, final_state: dict, *, token: str | None = None) -> dict: """Apply v197.4 final cleanup policy and merge the result into state.json.""" state = dict(final_state or {}) target_space = state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" final_status = state.get("status") or "" selected_hardware = state.get("selected_hardware") or os.environ.get("SELECTED_HARDWARE") or "" runtime_recovery = load_json_if_exists(run_dir / "runtime_recovery.json") if (run_dir / "runtime_recovery.json").exists() else {} if not isinstance(runtime_recovery, dict): runtime_recovery = {} gate = state.get("inference_gate") if isinstance(state.get("inference_gate"), dict) else {} signals = gate.get("implementation_signals") if isinstance(gate.get("implementation_signals"), dict) else {} health_semantic_passed = signals.get("health_semantic_passed") if health_semantic_passed is None: health_semantic_passed = signals.get("health_passed") if runtime_recovery.get("triggered") and runtime_recovery.get("recovery_attempted") and not runtime_recovery.get("recovery_exhausted"): decision = {"should_pause": False, "reason": "runtime_recovery_pending", "final_status": final_status, "selected_hardware": selected_hardware} else: decision = should_pause_generated_space(final_status, target_space, selected_hardware, health_semantic_passed=health_semantic_passed if isinstance(health_semantic_passed, bool) else None) cleanup = { "space_pause_attempted": False, "space_pause_success": False, "target_space": target_space, "final_status": final_status, "selected_hardware": selected_hardware, "decision": decision, "updated_at": now(), } if decision.get("should_pause"): cleanup = pause_generated_space_best_effort(target_space, token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "", run_dir, events_path, final_status=final_status, selected_hardware=selected_hardware, reason=decision.get("reason")) cleanup["decision"] = decision else: cleanup.update({"skipped": True, "skip_reason": decision.get("reason")}) write_json(run_dir / "cleanup_status.json", cleanup) append_event(events_path, "cleanup", "skipped", "No generated Space pause needed for final outcome", {"target_space": target_space, "final_status": final_status, "selected_hardware": selected_hardware, "reason": decision.get("reason")}) state["cleanup"] = cleanup write_json(run_dir / "state.json", state) return cleanup def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"): safe = safe_details(details) append_event(events_path, "failure", "failed", message, safe) existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {} if not isinstance(existing_state, dict): existing_state = {} repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {} if isinstance(repair_outcome, dict) and repair_outcome: safe.setdefault("repair_outcome_status", repair_outcome.get("post_repair_validation") or repair_outcome.get("repair_status") or "") safe.setdefault("repair_failure_type", repair_outcome.get("failure_type") or "") failure_state = { **existing_state, "run_id": os.environ.get("RUN_ID"), "kind": existing_state.get("kind") or "universal_model_card_builder", "status": status, "message": message, "updated_at": now(), "details": safe, "repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {}, } # Preserve the target Space when the worker fails after repository creation. target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" if target_space: failure_state["target_space"] = target_space failure_state["target_space_url"] = f"https://huggingface.co/spaces/{target_space}" write_json(run_dir / "state.json", failure_state) cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, failure_state) failure_state["cleanup"] = cleanup_status write_terminal_status_reconciliation(run_dir, events_path, failure_state, status=status, message=message, details=safe, job_exit_code=1) write_final_summary(run_dir, failure_state, {}, {}, status=status, message=message) live_failure_data = dict(safe) live_failure_data["cleanup"] = cleanup_status write_live_status(run_dir, stage="failure", status=status, message=message, data=live_failure_data) report = f"""# Agentic Space Factory — model Article Reproduction Report Status: **{status}** {message} Target Space: {failure_state.get('target_space_url') or failure_state.get('target_space') or 'not created / unknown'} ```json {json.dumps(safe, indent=2, ensure_ascii=False)} ``` """ (run_dir / "report.md").write_text(report, encoding="utf-8") write_artifact_manifest(run_dir, events_path=events_path, reason="failure") try: publish_eval_record(run_dir, phase="failure", events_path=events_path) except Exception: pass raise SystemExit(1) def run_cmd(cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600): result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) return result.returncode, redact_text(result.stdout) def sync_pi_traces(run_dir: Path, *, emit_event: bool = False, events_path: Path | None = None): """Best-effort incremental copy of Pi session JSONL traces into the run bucket.""" traces_dir = Path.home() / ".pi" / "agent" / "sessions" redacted_dir = run_dir / "traces" / "redacted" redacted_dir.mkdir(parents=True, exist_ok=True) count = 0 if traces_dir.exists(): for path in traces_dir.rglob("*.jsonl"): rel = path.relative_to(traces_dir) text = path.read_text(encoding="utf-8", errors="ignore") target_redacted = redacted_dir / rel target_redacted.parent.mkdir(parents=True, exist_ok=True) target_redacted.write_text(redact_text(text), encoding="utf-8") count += 1 if emit_event and events_path: append_event(events_path, "pi_traces", "running", "Synchronized Pi traces while Pi is running", {"count": count}) return count def run_cmd_streaming( cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600, live_log_path: Path | None = None, events_path: Path | None = None, run_dir: Path | None = None, step: str = "pi_run", trace_phase: str | None = None, ): """Run a command while incrementally writing stdout to the bucket. subprocess.run() only returns output at the end. Pi can work for many minutes, so stream stdout to logs/pi_live_output.txt and periodically copy Pi session JSONL traces into traces/redacted while the process is still alive. """ started = time.monotonic() trace_phase = trace_phase or step last_event = 0.0 last_trace_sync = 0.0 lines: list[str] = [] if live_log_path: live_log_path.parent.mkdir(parents=True, exist_ok=True) live_log_path.write_text("", encoding="utf-8") if run_dir: write_agent_trace_record(run_dir, phase=trace_phase, event="command_started", status="started", message="Pi command started", data={"step": step, "cwd": str(cwd) if cwd else ""}) proc = subprocess.Popen( cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, ) try: assert proc.stdout is not None while True: if timeout and time.monotonic() - started > timeout: proc.kill() message = f"Command timed out after {timeout}s" lines.append(message) if live_log_path: with live_log_path.open("a", encoding="utf-8") as f: f.write(message + "\n") return 124, redact_text("\n".join(lines)) line = proc.stdout.readline() if line: clean = redact_text(line.rstrip("\n")) lines.append(clean) if live_log_path: with live_log_path.open("a", encoding="utf-8") as f: f.write(clean + "\n") now_m = time.monotonic() if events_path and now_m - last_event > 8: output_tail = "\n".join(lines[-20:])[-2000:] append_event(events_path, step, "running", "Pi is still working", {"output_tail": output_tail}) if run_dir: write_agent_trace_record(run_dir, phase=trace_phase, event="stdout_tail", status="running", message="Pi is still working", text=output_tail, data={"step": step}) last_event = now_m if run_dir and now_m - last_trace_sync > 15: sync_pi_traces(run_dir, emit_event=False) last_trace_sync = now_m continue if proc.poll() is not None: remainder = proc.stdout.read() if proc.stdout else "" if remainder: for raw in remainder.splitlines(): clean = redact_text(raw) lines.append(clean) if live_log_path: with live_log_path.open("a", encoding="utf-8") as f: f.write(clean + "\n") break time.sleep(0.2) finally: if run_dir: sync_pi_traces(run_dir, emit_event=False) output = redact_text("\n".join(lines)) if run_dir: write_agent_trace_record(run_dir, phase=trace_phase, event="command_finished", status="success" if int(proc.returncode or 0) == 0 else "failed", message="Pi command finished", text=output[-20000:], data={"step": step, "returncode": int(proc.returncode or 0)}, artifacts=[str(live_log_path.relative_to(run_dir)) if live_log_path and str(live_log_path).startswith(str(run_dir)) else ""]) return int(proc.returncode or 0), output def install_python_deps(events_path: Path): append_event(events_path, "dependencies", "started", "Installing Python worker dependencies") code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0", "requests>=2.31.0"], timeout=600) if code != 0: append_event(events_path, "dependencies", "failed", "Python dependency installation failed", {"output_tail": out[-4000:]}) raise RuntimeError(out) append_event(events_path, "dependencies", "success", "Python worker dependencies installed") def ensure_node(events_path: Path): node = shutil.which("node") npm = shutil.which("npm") if node and npm: _, node_v = run_cmd([node, "--version"], timeout=30) _, npm_v = run_cmd([npm, "--version"], timeout=30) append_event(events_path, "node", "success", "Node/npm already available", {"node": node_v.strip(), "npm": npm_v.strip()}) return append_event(events_path, "node", "started", "Installing nodejs/npm through apt-get") code, out = run_cmd(["bash", "-lc", "apt-get update -qq && apt-get install -y -qq nodejs npm"], timeout=600) if code != 0: append_event(events_path, "node", "failed", "Could not install nodejs/npm", {"output_tail": out[-4000:]}) raise RuntimeError(out) append_event(events_path, "node", "success", "Installed nodejs/npm") def install_pi(events_path: Path): ensure_node(events_path) append_event(events_path, "pi_install", "started", "Installing Pi coding agent from npm") code, out = run_cmd(["npm", "install", "-g", "@mariozechner/pi-coding-agent"], timeout=900) if code != 0: append_event(events_path, "pi_install", "failed", "Pi npm installation failed", {"output_tail": out[-4000:]}) raise RuntimeError(out) code, version = run_cmd(["pi", "--version"], timeout=60) append_event(events_path, "pi_install", "success", "Pi installed", {"version_output": version.strip()[-300:]}) def configure_pi(events_path: Path, model: str): pi_dir = Path.home() / ".pi" / "agent" pi_dir.mkdir(parents=True, exist_ok=True) token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "" if token: os.environ.setdefault("HF_TOKEN", token) os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token) (pi_dir / "auth.json").write_text(json.dumps({"huggingface": {"type": "api_key", "key": token}}, indent=2), encoding="utf-8") (pi_dir / "settings.json").write_text(json.dumps({"model": model, "provider": "huggingface", "autoRun": True, "autoApply": True}, indent=2), encoding="utf-8") append_event(events_path, "pi_config", "success" if token else "warning", "Configured Pi with Hugging Face provider token context", {"model": model, "hf_token_present": bool(token), "token_value": "[REDACTED]" if token else ""}) def normalize_pi_model_name(value: str | None) -> str: raw = (value or "").strip().lower() # Pi/provider traces may report either a full Hub id (moonshotai/Kimi-K2-...) # or only the served model name (Kimi-K2-...). Compare on the model leaf so # provider fallbacks are detected without false positives from missing owners. if "/" in raw: raw = raw.rsplit("/", 1)[-1] return re.sub(r"[^a-z0-9]+", "", raw) def extract_pi_models_from_text(text: str) -> list[str]: """Best-effort extraction of assistant/model names from Pi stdout/session traces.""" if not text: return [] patterns = [ r"\bQwen/[A-Za-z0-9_.-]+", r"\bQwen(?:2(?:\.5)?|3)?[-_/A-Za-z0-9.]*Coder[-_/A-Za-z0-9.]*", r"\bmoonshotai/[A-Za-z0-9_.-]+", r"\bKimi[-_/A-Za-z0-9.]+", r"\bzai-org/[A-Za-z0-9_.-]+", r"\bGLM[-_/A-Za-z0-9.]+", r"\bdeepseek-ai/[A-Za-z0-9_.-]+", r"\bDeepSeek[-_/A-Za-z0-9.]+", r"\bClaude[-_/A-Za-z0-9.]+", r"\bGPT[-_/A-Za-z0-9.]+", r"\b[Mm]odel(?:\\s+used|\\s+selected|\\s*:)?\\s*[:=]?\\s*([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", ] found: list[str] = [] for pattern in patterns: for match in re.finditer(pattern, text, flags=re.IGNORECASE): value = match.group(1) if match.groups() else match.group(0) value = value.strip().strip('",` ') if value and value not in found: found.append(value) return found[:12] def detect_pi_model_resolution(requested_model: str, run_dir: Path, pi_output: str) -> dict: settings_path = Path.home() / ".pi" / "agent" / "settings.json" configured_model = requested_model try: settings = json.loads(settings_path.read_text(encoding="utf-8")) configured_model = settings.get("model") or requested_model except Exception: settings = {"model": requested_model, "read_error": "could_not_read_settings"} corpus = [pi_output or ""] for trace in (run_dir / "traces" / "redacted").rglob("*.jsonl") if (run_dir / "traces" / "redacted").exists() else []: try: corpus.append(trace.read_text(encoding="utf-8", errors="ignore")[:200000]) except Exception: pass observed = extract_pi_models_from_text("\n".join(corpus)) normalized_requested = normalize_pi_model_name(requested_model) normalized_configured = normalize_pi_model_name(configured_model) normalized_observed = [normalize_pi_model_name(x) for x in observed] effective_model = "" for raw, normalized in zip(observed, normalized_observed): if normalized and normalized != normalized_requested and normalized != normalized_configured: effective_model = raw break if not effective_model and observed: effective_model = observed[0] mismatch = bool(effective_model and normalize_pi_model_name(effective_model) not in {normalized_requested, normalized_configured}) payload = { "requested_model": requested_model, "configured_model": configured_model, "observed_models": observed, "effective_model": effective_model or configured_model, "provider": "huggingface", "mismatch": mismatch, "source": "published_pi_traces_after_sync", "settings": settings, } write_json(run_dir / "pi_model_resolution.json", payload) return payload def emit_pi_model_resolution(events_path: Path, resolution: dict): if resolution.get("mismatch"): append_event( events_path, "pi_model_resolution", "warning", "Pi assistant model appears to differ from the requested model", { "requested_model": resolution.get("requested_model"), "configured_model": resolution.get("configured_model"), "effective_model": resolution.get("effective_model"), "observed_models": resolution.get("observed_models", [])[:8], "provider": resolution.get("provider"), }, ) else: append_event( events_path, "pi_model_resolution", "success", "Pi assistant model matched the requested configuration", { "requested_model": resolution.get("requested_model"), "configured_model": resolution.get("configured_model"), "effective_model": resolution.get("effective_model"), "provider": resolution.get("provider"), }, ) def collect_pi_traces(run_dir: Path, events_path: Path): count = sync_pi_traces(run_dir, emit_event=False) write_agent_trace_record(run_dir, phase="initial_build", event="pi_sessions_collected", status="success", message="Synchronized redacted Pi session traces into the run-level trace folder", data={"count": count, "raw_published": False}, artifacts=["traces/redacted/agent_trace.jsonl"]) append_event(events_path, "traces", "success", "Collected redacted Pi traces", {"count": count, "agent_trace": "traces/redacted/agent_trace.jsonl", "raw_published": False}) write_artifact_manifest(run_dir, reason="pi_traces_collected") return count def sanitize_model_id(model_id: str) -> str: model_id = (model_id or DEFAULT_MODEL_ID).strip().replace("https://huggingface.co/", "") model_id = model_id.split("?", 1)[0].strip("/") if not re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", model_id): raise ValueError("MODEL_ID must look like owner/model-name") return model_id def expected_space_subdomain(target_space_id: str) -> str: owner, _, slug = str(target_space_id or "").partition("/") if not owner or not slug: return "" return re.sub(r"[^a-z0-9-]+", "-", f"{owner}-{slug}".lower()).strip("-") def gradio_client_identity_payload(client, target_space_id: str) -> dict: """Best-effort identity check for gradio_client target resolution. v198.26.7 restores the safe v198.25.3 behavior for valid Spaces: the repo identity returned by gradio_client (`space_id`/`space_name` or a repo-like `src`) is authoritative. Hugging Face may expose the running app through an `.hf.space` runtime host/alias that does not exactly match the repo slug, so a host alias is recorded as a warning and must not block validation when the repo identity matches the target. A mismatch is fatal only when the repo identity is absent/different and observed URLs clearly point to another Space. """ from urllib.parse import urlparse def _repo_like(value: str) -> str: raw = str(value or "").strip() if not raw or ".hf.space" in raw: return "" if raw.startswith("https://huggingface.co/spaces/"): raw = raw.split("/spaces/", 1)[1] raw = raw.split("?", 1)[0].strip("/") if raw.count("/") >= 1: owner, slug = raw.split("/", 1)[:2] if owner and slug: return f"{owner}/{slug}".lower() return "" observed_values = [] for attr in ("space_id", "space_name", "src", "app_url", "root_url", "src_url", "api_url"): try: value = getattr(client, attr, None) except Exception: value = None if value: observed_values.append({"attr": attr, "value": str(value)}) expected_repo = str(target_space_id or "").strip() expected_repo_l = expected_repo.lower() expected_subdomain = expected_space_subdomain(expected_repo) authoritative_repo_matches = [] authoritative_repo_mismatches = [] for item in observed_values: attr = item["attr"] repo = _repo_like(item["value"]) if not repo: continue record = {"attr": attr, "value": item["value"], "repo": repo} if expected_repo_l and repo == expected_repo_l: authoritative_repo_matches.append(record) elif attr in {"space_id", "space_name"}: authoritative_repo_mismatches.append(record) observed_hf_space_urls = [] mismatched_hf_space_urls = [] runtime_host_aliases = [] for item in observed_values: value = item["value"] if ".hf.space" not in value: continue parsed = urlparse(value if value.startswith(("http://", "https://")) else "https://" + value.lstrip("/")) host = (parsed.netloc or parsed.path.split("/", 1)[0]).lower() if not host.endswith(".hf.space"): continue subdomain = host.rsplit(".hf.space", 1)[0] record = {"attr": item["attr"], "value": value, "host": host, "subdomain": subdomain} observed_hf_space_urls.append(record) if expected_subdomain and subdomain != expected_subdomain: if authoritative_repo_matches: runtime_host_aliases.append(record) else: mismatched_hf_space_urls.append(record) mismatch = False reason = "" if authoritative_repo_matches: mismatch = False reason = "authoritative_repo_match" elif authoritative_repo_mismatches: mismatch = True reason = "authoritative_repo_mismatch" elif mismatched_hf_space_urls and not any(item.get("subdomain") == expected_subdomain for item in observed_hf_space_urls): mismatch = True reason = "hf_space_url_points_elsewhere_without_repo_match" else: mismatch = False reason = "no_conflicting_identity_observed" return { "schema_version": "gradio_client_identity.v198_26_7", "target_space_id": expected_repo, "expected_subdomain": expected_subdomain, "observed": observed_values, "authoritative_repo_matches": authoritative_repo_matches, "authoritative_repo_mismatches": authoritative_repo_mismatches, "observed_hf_space_urls": observed_hf_space_urls, "mismatched_hf_space_urls": mismatched_hf_space_urls if mismatch else [], "runtime_host_aliases": runtime_host_aliases, "runtime_host_alias_detected": bool(runtime_host_aliases), "mismatch": bool(mismatch), "identity_reason": reason, "failure_owner": "factory_validation_client" if mismatch else "", "failure_class": "space_identity_mismatch" if mismatch else "", } def write_gradio_client_identity(client, target_space_id: str, run_dir: Path, events_path: Path, *, phase: str) -> dict: payload = gradio_client_identity_payload(client, target_space_id) payload["phase"] = phase write_json(run_dir / "tests" / f"gradio_client_identity_{phase}.json", payload) if payload.get("mismatch"): append_event(events_path, "space_identity", "failed", "Gradio client resolved a different Space than the run target", payload) elif payload.get("runtime_host_alias_detected"): append_event(events_path, "space_identity", "warning", "Gradio client uses a Hugging Face runtime host alias for the target Space", payload) return payload def make_gradio_client(target_space_id: str, token: str, timeout_s: int | float | None = None): import inspect from gradio_client import Client params = inspect.signature(Client).parameters kwargs = {} if timeout_s and "httpx_kwargs" in params: kwargs["httpx_kwargs"] = {"timeout": float(timeout_s)} if "token" in params: return Client(target_space_id, token=token, **kwargs) if "hf_token" in params: return Client(target_space_id, hf_token=token, **kwargs) if "api_key" in params: return Client(target_space_id, api_key=token, **kwargs) if "headers" in params: return Client(target_space_id, headers={"Authorization": f"Bearer {token}"}, **kwargs) return Client(target_space_id, **kwargs) def api_names_from_schema(schema) -> list[str]: names: list[str] = [] if isinstance(schema, dict): endpoints = schema.get("named_endpoints") or schema.get("endpoints") or {} if isinstance(endpoints, dict): for key, value in endpoints.items(): if isinstance(key, str) and key.startswith("/"): names.append(key) if isinstance(value, dict): api_name = value.get("api_name") if isinstance(api_name, str) and api_name.startswith("/"): names.append(api_name) if isinstance(schema.get("dependencies"), list): for dep in schema["dependencies"]: if isinstance(dep, dict): api_name = dep.get("api_name") if isinstance(api_name, str): names.append(api_name if api_name.startswith("/") else f"/{api_name}") return list(dict.fromkeys(names)) def normalize_api_name(name: str | None) -> str: value = (name or "").strip() if not value: return "/generate" return value if value.startswith("/") else "/" + value def default_smoke_args(expected_output_type: str) -> list: expected = (expected_output_type or "any").lower() if expected == "image": return ["a cinematic robot cat astronaut, detailed, studio lighting"] if expected == "video": return ["a short cinematic shot of a robot cat astronaut walking on the moon"] if expected == "audio": return ["A calm voice saying hello from Agentic Space Factory."] return ["Explain the concept of agentic AI in one paragraph."] def endpoint_schema_for_api(schema, api_name: str): target = normalize_api_name(api_name) alternatives = {target, target.lstrip("/")} found = None def walk(obj): nonlocal found if found is not None: return if isinstance(obj, dict): for key, value in obj.items(): if isinstance(key, str) and key in alternatives and isinstance(value, (dict, list)): found = value return value = obj.get("api_name") or obj.get("apiName") if isinstance(value, str) and normalize_api_name(value) == target: found = obj return for value in obj.values(): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(schema) return found def endpoint_parameter_names(endpoint) -> list[str]: names: list[str] = [] def add(value): if not isinstance(value, str): return cleaned = value.strip() if cleaned and cleaned not in names: names.append(cleaned) def from_parameter(param): if isinstance(param, dict): for key in ["parameter_name", "parameterName", "name", "label"]: add(param.get(key)) component = param.get("component") or param.get("component_type") if isinstance(component, dict): for key in ["label", "name"]: add(component.get(key)) elif isinstance(param, str): add(param) def walk(obj): if isinstance(obj, dict): params = obj.get("parameters") or obj.get("inputs") if isinstance(params, list): for param in params: from_parameter(param) for value in obj.values(): if isinstance(value, (dict, list)): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(endpoint) return names def endpoint_parameters_for_smoke(endpoint) -> list[dict]: """Return Gradio endpoint parameters in call order with names/defaults. gradio_client schema formats vary across versions. Keep this permissive and preserve ordering because client.predict uses positional arguments. """ params = None def find_params(obj): nonlocal params if params is not None: return if isinstance(obj, dict): for key in ("parameters", "inputs"): value = obj.get(key) if isinstance(value, list) and value: params = value return for value in obj.values(): if isinstance(value, (dict, list)): find_params(value) elif isinstance(obj, list): for item in obj: find_params(item) find_params(endpoint) out = [] for i, param in enumerate(params or []): if isinstance(param, dict): component = param.get("component") or param.get("component_type") or {} if not isinstance(component, dict): component = {} name = ( param.get("parameter_name") or param.get("parameterName") or param.get("name") or param.get("label") or component.get("label") or component.get("name") or f"arg{i}" ) default = None has_default = False for key in ("parameter_default", "default", "value"): if key in param: default = param.get(key) has_default = True break if not has_default: for key in ("value", "default"): if key in component: default = component.get(key) has_default = True break if isinstance(default, str) and default.strip().lower() in {"none", "null", ""} and not param.get("parameter_has_default"): has_default = False default = None if "parameter_has_default" in param: has_default = bool(param.get("parameter_has_default")) raw_choices = param.get("choices") or param.get("options") or (param.get("api_info") or {}).get("choices") or component.get("choices") or [] choices = raw_choices if isinstance(raw_choices, list) else [] required = bool(param.get("required", not has_default)) out.append({ "name": str(name).strip(), "required": required, "has_default": has_default, "default": default, "component": component.get("type") or component.get("component") or param.get("component_type") or param.get("type") or "", "choices": choices, }) elif isinstance(param, str): out.append({"name": param.strip() or f"arg{i}", "required": True, "has_default": False, "default": None, "component": ""}) return out def merge_smoke_parameters_with_endpoint_schema(params: list[dict] | None, endpoint_params: list[dict] | None) -> list[dict]: """Merge contract smoke params with the live Gradio schema. Pi contracts may describe parameters but omit dropdown/radio choices. The live schema is authoritative for values accepted by ``client.predict``. Preserve contract names/defaults when useful, but always carry schema choices into the payload coercion step before the first smoke call. """ params = list(params or []) endpoint_params = list(endpoint_params or []) if not params: return endpoint_params if not endpoint_params: return params merged = [] max_len = max(len(params), len(endpoint_params)) for index in range(max_len): contract_param = params[index] if index < len(params) and isinstance(params[index], dict) else {} schema_param = endpoint_params[index] if index < len(endpoint_params) and isinstance(endpoint_params[index], dict) else {} row = {**schema_param, **contract_param} schema_choices = schema_param.get("choices") if isinstance(schema_param, dict) else [] contract_choices = contract_param.get("choices") if isinstance(contract_param, dict) else [] if isinstance(schema_choices, list) and schema_choices: row["choices"] = schema_choices elif isinstance(contract_choices, list) and contract_choices: row["choices"] = contract_choices if not row.get("name") and schema_param.get("name"): row["name"] = schema_param.get("name") merged.append(row) return merged def smoke_value_for_parameter(param: dict, expected_output_type: str): name = str(param.get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") component = str(param.get("component") or "").lower() expected = str(expected_output_type or "").strip().lower() file_kind = gradio_file_kind_for_param(param) if file_kind == "image": return "__ASF_SMOKE_IMAGE__" if file_kind == "file": return "__ASF_SMOKE_FILE__" if file_kind == "audio": return "https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac" if file_kind == "video": return "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/a_video.mp4" # For video models, prefer a cheap smoke payload over UI defaults. The goal # is to prove the pipeline runs, not to benchmark the best demo quality. if expected == "video": if name in {"height", "image_height"} or name.endswith("_height"): return 320 if name in {"width", "image_width"} or name.endswith("_width"): return 512 if name in {"num_frames", "frames", "n_frames", "frame_count"} or "num_frames" in name: return 17 if name in {"frame_rate", "fps"}: return 8 if name in {"num_inference_steps", "inference_steps", "steps", "num_steps"} or "step" in name: return 6 if param.get("has_default"): return param.get("default") if name in {"prompt", "text", "query", "input", "instruction"} or "prompt" in name and "negative" not in name: return default_smoke_args(expected_output_type)[0] if name in {"negative_prompt", "negative", "negative_text"} or "negative" in name: return "" if name in {"seed", "random_seed"} or name.endswith("_seed"): return 42 if name in {"height", "width", "image_height", "image_width"}: return 512 if name in {"num_inference_steps", "inference_steps", "steps", "num_steps"} or "step" in name: return 4 if name in {"guidance_scale", "cfg_scale", "scale"} or "guidance" in name: return 0.0 if "bool" in component or name.startswith("enable_") or name.startswith("use_"): return False if "number" in component or "slider" in component: return 0 return "" def _choice_scalar(choice): if isinstance(choice, dict): for key in ("value", "label", "name"): if key in choice: return choice.get(key) return next(iter(choice.values()), choice) if choice else choice if isinstance(choice, (list, tuple)) and choice: return choice[0] return choice def _choice_value_and_aliases(choice): """Return the API value and UI aliases for a Gradio choice. Gradio schemas may expose dropdown values as plain strings, dictionaries (label/value), or tuples. The validator must call predict() with the API value, but it should accept semantically equivalent UI labels coming from Pi contracts, for example ``1024x1024 (1:1)`` when the schema value is ``1024x1024``. """ aliases = [] actual = _choice_scalar(choice) if isinstance(choice, dict): for key in ("value", "label", "name"): if key in choice: aliases.append(choice.get(key)) aliases.extend(v for v in choice.values() if isinstance(v, (str, int, float))) elif isinstance(choice, (list, tuple)): aliases.extend(choice) else: aliases.append(choice) aliases.append(actual) return actual, aliases def _choice_text_aliases(value) -> set[str]: text = str(value or "").strip() if not text: return set() lowered = text.lower().replace("×", "x") lowered = re.sub(r"\s+", " ", lowered).strip() aliases = {lowered, lowered.replace(" ", "")} # Drop common UI-only annotation suffixes: ``1024x1024 (1:1)``, # ``1024x1024 - square``, ``1024x1024 / square``. Keep this as an # alias only; the actual value returned to Gradio remains the schema value. no_paren_suffix = re.sub(r"\s*\([^)]*\)\s*$", "", lowered).strip() if no_paren_suffix: aliases.add(no_paren_suffix) aliases.add(no_paren_suffix.replace(" ", "")) for sep in (" - ", " – ", " — ", " | "): if sep in lowered: head = lowered.split(sep, 1)[0].strip() if head: aliases.add(head) aliases.add(head.replace(" ", "")) # Dimension values are especially common for image/video dropdowns. # Normalize labels like ``1024 × 1024 (1:1)`` and ``size: 1024x1024`` # to the API value shape used by Gradio. for w, h in re.findall(r"(\d{2,5})\s*[xX]\s*(\d{2,5})", lowered): aliases.add(f"{w}x{h}") return {a for a in aliases if a} def _choice_alias_matches(value, choice) -> bool: value_aliases = _choice_text_aliases(value) _, aliases = _choice_value_and_aliases(choice) for alias in aliases: if value == alias: return True if _choice_text_aliases(alias) & value_aliases: return True return False def _coerce_to_schema_choice(value, param: dict): choices = param.get("choices") or [] if not isinstance(choices, list) or not choices: return value, False for raw_choice in choices: choice, _aliases = _choice_value_and_aliases(raw_choice) if value == choice: return choice, True value_text = str(value).strip() for raw_choice in choices: choice, _aliases = _choice_value_and_aliases(raw_choice) if str(choice).strip() == value_text: return choice, True for raw_choice in choices: choice, _aliases = _choice_value_and_aliases(raw_choice) if _choice_alias_matches(value, raw_choice): return choice, True return value, False def coerce_smoke_value(value, param: dict): name = str(param.get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") component = str(param.get("component") or "").lower() if value is None: return value choice_value, matched_choice = _coerce_to_schema_choice(value, param) if matched_choice: return choice_value if isinstance(value, str): stripped = value.strip() if name in {"height", "width", "image_height", "image_width", "seed", "random_seed", "num_inference_steps", "inference_steps", "steps", "num_steps"} or name.endswith("_seed") or "step" in name: try: return int(float(stripped)) except Exception: return value if name in {"guidance_scale", "cfg_scale", "scale"} or "guidance" in name or "slider" in component or "number" in component: try: number = float(stripped) return int(number) if number.is_integer() and name not in {"guidance_scale", "cfg_scale", "scale"} else number except Exception: return value return value def coerce_smoke_args_to_schema_choices(args: list, params: list[dict]) -> tuple[list, list[dict]]: corrected = list(args or []) changes: list[dict] = [] for index, param in enumerate(params or []): if index >= len(corrected): break before = corrected[index] after, matched = _coerce_to_schema_choice(before, param) if matched and after != before: corrected[index] = after changes.append({ "index": index, "name": str(param.get("name") or f"arg{index}"), "from": before, "to": after, "reason": "schema_choice_type_match", }) return corrected, changes def is_schema_choice_type_error(error: Exception | str) -> bool: text = str(error or "").lower() return "not in the list of choices" in text or ("value:" in text and "choices" in text) def parse_gradio_choice_error(error: Exception | str) -> dict: """Extract a Gradio choice/type mismatch from messages like: Value: 1024 is not in the list of choices: ['512', '768', '1024'] The automatic validator used to mark this retryable but could miss the actual retry when the contract payload did not carry choices. Keep this parser conservative and only use it to coerce to an exact listed choice. """ text = str(error or "") if not is_schema_choice_type_error(text): return {"matched": False} value = None choices = [] m = re.search(r"Value:\s*(.*?)\s+is not in the list of choices:\s*(\[.*?\])", text, flags=re.IGNORECASE | re.DOTALL) if m: value = m.group(1).strip().strip("'\"") raw_choices = m.group(2).strip() try: import ast parsed = ast.literal_eval(raw_choices) if isinstance(parsed, list): choices = parsed except Exception: choices = [c.strip().strip("'\"") for c in raw_choices.strip("[]").split(",") if c.strip()] if value is None: m_value = re.search(r"Value:\s*([^\n]+?)\s+(?:is|not)", text, flags=re.IGNORECASE) if m_value: value = m_value.group(1).strip().strip("'\"") return {"matched": bool(value is not None), "value": value, "choices": choices, "raw_error": text[:2000]} def coerce_smoke_args_from_choice_error(args: list, error: Exception | str, params: list[dict] | None = None) -> tuple[list, list[dict]]: """Repair validator-owned choice mismatches even when schema params lacked choices. This restores the golden-path behavior for Gradio Dropdown/Radio choices where the app exposes string choices such as "1024" but the factory payload sends numeric 1024. The retry must use the exact choice object returned by Gradio. """ parsed = parse_gradio_choice_error(error) if not parsed.get("matched"): return list(args or []), [] wanted = str(parsed.get("value") or "").strip() choices = parsed.get("choices") if isinstance(parsed.get("choices"), list) else [] exact_choice = None for choice in choices: if _choice_alias_matches(wanted, choice): exact_choice = _choice_value_and_aliases(choice)[0] break corrected = list(args or []) changes = [] # Prefer the rejected argument itself, then a parameter whose schema choices # can normalize the rejected UI label to an accepted API value. params = params or [] candidate_indexes = [i for i, value in enumerate(corrected) if _choice_text_aliases(value) & _choice_text_aliases(wanted)] if not candidate_indexes: for index, param in enumerate(params): if index >= len(corrected): break param_choices = param.get("choices") or [] if isinstance(param, dict) else [] if param_choices and any(_choice_alias_matches(wanted, c) for c in param_choices): candidate_indexes.append(index) for index in candidate_indexes[:1]: before = corrected[index] param_choices = params[index].get("choices") if index < len(params) and isinstance(params[index], dict) else [] after = exact_choice if after is None and param_choices: after, matched = _coerce_to_schema_choice(before, {"choices": param_choices}) if not matched: after = wanted if after is None: after = wanted if before != after: corrected[index] = after name = str((params[index].get("name") if index < len(params) and isinstance(params[index], dict) else None) or f"arg{index}") changes.append({ "index": index, "name": name, "from": before, "to": after, "reason": "parsed_gradio_choice_error_normalized_choice", "error_value": parsed.get("value"), "choices": choices[:20], }) break return corrected, changes def gradio_file_kind_for_param(param: dict) -> str: """Return image/file/video/audio when a Gradio schema parameter is file-backed.""" if not isinstance(param, dict): return "" name = str(param.get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") component = str(param.get("component") or "").lower() haystack = f"{name} {component}" if any(token in haystack for token in ["image", "gr.image"]): return "image" if any(token in haystack for token in ["video", "gr.video"]): return "video" if any(token in haystack for token in ["audio", "gr.audio"]): return "audio" if any(token in haystack for token in ["file", "upload", "filepath", "path"]): return "file" return "" def ensure_smoke_input_file(run_dir: Path, kind: str) -> str: smoke_dir = run_dir / "tests" / "smoke_inputs" smoke_dir.mkdir(parents=True, exist_ok=True) if kind == "image": path = smoke_dir / "asf_smoke_image.png" if not path.exists(): # 1x1 valid PNG. Small and deterministic; avoids remote URL schema drift. path.write_bytes(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=")) return str(path) path = smoke_dir / "asf_smoke_file.txt" if not path.exists(): path.write_text("Agentic Space Factory smoke input file.\n", encoding="utf-8") return str(path) def _is_url_like(value) -> bool: text = str(value or "").strip().lower() return text.startswith(("http://", "https://", "hf://", "s3://", "gs://", "data:")) def resolve_smoke_file_source(run_dir: Path, source, kind: str) -> tuple[str, dict]: """Resolve relative Gradio file/image smoke inputs before handle_file(). v198.26.2: generated apps commonly include example media such as `sample_image.png` in runtime_upload_payload/. Passing the bare relative string to gradio_client makes the validator fail before it reaches the Space. Resolve local payload/generated/test files first; only leave URLs untouched. """ info = {"schema_version": "smoke_file_source_resolution.v198_26_2", "kind": kind} if isinstance(source, dict): info.update({"action": "already_filedata_or_dict", "source_type": "dict"}) return source, info text = str(source or "").strip() placeholder = text in {"", "__ASF_SMOKE_IMAGE__", "__ASF_SMOKE_FILE__"} if kind in {"image", "file"} and (source is None or placeholder): fallback = ensure_smoke_input_file(run_dir, kind) info.update({"action": "created_fallback_smoke_input", "source": text, "resolved_path": fallback}) return fallback, info if not text or _is_url_like(text): info.update({"action": "url_or_empty_passthrough", "source": text}) return text, info path = Path(text).expanduser() candidates = [] if path.is_absolute(): candidates.append(path) else: candidates.extend([ run_dir / "runtime_upload_payload" / text, run_dir / "generated" / text, run_dir / text, run_dir / "tests" / "smoke_inputs" / text, ]) if path.exists(): candidates.append(path) for candidate in candidates: try: if candidate.exists() and candidate.is_file(): resolved = str(candidate.resolve()) info.update({"action": "resolved_local_file", "source": text, "resolved_path": resolved}) return resolved, info except Exception: continue info.update({ "action": "missing_local_file", "source": text, "searched_paths": [str(c) for c in candidates[:12]], }) raise FileNotFoundError( "ASF smoke input materialization failed: " f"{kind} input {text!r} was not found in runtime_upload_payload, generated, run root, or tests/smoke_inputs." ) def prepare_gradio_file_inputs(args: list, params: list[dict], run_dir: Path) -> tuple[list, list[dict]]: """Wrap file-like smoke inputs with gradio_client.handle_file(). Passing raw URL strings to gr.Image/File/Video/Audio can raise Pydantic ImageData/FileData validation errors in recent Gradio stacks. The worker owns that payload construction, so these are factory-validator errors, not generated-app repairs. """ from gradio_client import handle_file resolved = list(args or []) conversions: list[dict] = [] for index, param in enumerate(params or []): if index >= len(resolved): break kind = gradio_file_kind_for_param(param) if not kind: continue before = resolved[index] if isinstance(before, dict) and (before.get("meta") or {}).get("_type") == "gradio.FileData": continue try: source, resolution = resolve_smoke_file_source(run_dir, before, kind) resolved[index] = handle_file(str(source)) conversions.append({ "index": index, "name": str(param.get("name") or f"arg{index}"), "kind": kind, "from_type": type(before).__name__, "source": str(source)[:500], "action": "handle_file", "resolution": resolution, }) except Exception as exc: conversions.append({ "index": index, "name": str(param.get("name") or f"arg{index}"), "kind": kind, "from_type": type(before).__name__, "source": str(before)[:500], "action": "handle_file_failed", "error": str(exc)[:1000], }) raise return resolved, conversions def _smoke_param_name(param: dict) -> str: return str((param or {}).get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") def _minimal_numeric_value(name: str, value, *, expected_output_type: str = ""): """Return a safer minimal smoke value for expensive generation parameters. v198.26.7 introduces a second validation level: a minimal demo smoke can prove the Space is usable after a canonical/promise smoke OOM, but it must not be promoted to full inference success. """ output = str(expected_output_type or "").lower() try: current = float(value) except Exception: current = None if any(token in name for token in ("width", "height", "resolution", "size")): if current is None: return value return int(min(max(current, 1), 512)) if any(token in name for token in ("num_inference_steps", "inference_steps", "steps", "num_steps", "sampling_steps")) or name == "steps": if current is None: return value return int(min(max(current, 1), 4)) if any(token in name for token in ("duration", "seconds", "length")): if current is None: return value cap = 2 if output in {"video", "audio"} else 4 return int(min(max(current, 1), cap)) if any(token in name for token in ("frames", "num_frames")): if current is None: return value return int(min(max(current, 1), 8)) if any(token in name for token in ("guidance", "cfg_scale", "scale")): if current is None: return value return float(min(max(current, 0.0), 3.0)) return value def build_minimal_demo_smoke_args(args: list, params: list[dict], expected_output_type: str = "") -> tuple[list, list[dict]]: """Build a cheaper validation payload from the canonical smoke args. The canonical smoke remains the only path to `full_inference_success`. This helper is used after canonical OOM to check whether the deployed demo is at least usable with reduced parameters. """ minimal = list(args or []) changes: list[dict] = [] for index, param in enumerate(params or []): if index >= len(minimal): break name = _smoke_param_name(param) before = minimal[index] after = before if isinstance(before, bool): if any(token in name for token in ("upsample", "hires", "high_res", "enhance", "refiner", "randomize")): after = False elif isinstance(before, (int, float)) or (isinstance(before, str) and before.strip().replace('.', '', 1).isdigit()): after = _minimal_numeric_value(name, before, expected_output_type=expected_output_type) elif isinstance(before, str): lower = before.strip().lower() if any(token in name for token in ("mode", "quality", "preset")) and lower in {"quality", "high", "best", "default", "upscale", "upsample"}: after = "Fast" if before != "Fast" else before if after != before: minimal[index] = after changes.append({"index": index, "name": str(param.get("name") or f"arg{index}"), "from": before, "to": after, "reason": "minimal_demo_smoke"}) return minimal, changes def minimal_smoke_status_payload(payload: dict) -> dict: """Mark a successful minimal smoke without claiming full promise success.""" out = dict(payload or {}) out["schema_version"] = "generation_smoke_result.v198_26_7" out["status"] = "demo_usable_smoke_passed" out["demo_usable_smoke_passed"] = True out["canonical_promise_smoke_passed"] = False out["full_promise_smoke_passed"] = False out["validation_level"] = "minimal_demo_smoke" out["message"] = "A reduced minimal smoke generated output after the canonical promise smoke failed or OOMed. The demo is usable, but the full promise was not verified." return out def build_generation_smoke_args(endpoint, expected_output_type: str) -> tuple[list, list[dict]]: params = endpoint_parameters_for_smoke(endpoint) if not params: return default_smoke_args(expected_output_type), [] return [coerce_smoke_value(smoke_value_for_parameter(param, expected_output_type), param) for param in params], params def read_inference_contract(workspace: Path | None) -> dict: if not workspace: return {} path = workspace / "INFERENCE_CONTRACT.json" if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def read_demo_quality_contract(workspace: Path | None) -> dict: """Read Pi's demo-quality contract when present. v198.13: this lets the Worker validate the demo with Pi's canonical, model-card-aligned example instead of only guessing a generic payload. """ if not workspace: return {} path = workspace / "DEMO_QUALITY_CONTRACT.json" if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def _as_bool_false(value) -> bool: if value is False: return True if isinstance(value, str): return value.strip().lower() in {"false", "no", "0", "off"} return False def _read_workspace_json(workspace: Path | None, filename: str) -> dict: if not workspace: return {} path = workspace / filename if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8", errors="replace")) return data if isinstance(data, dict) else {} except Exception: return {} def contract_declares_no_full_inference(workspace: Path | None) -> dict: """Return a structured reason when Pi explicitly declared boot-only/no inference. v191.2: A machine-readable no-inference contract must be respected by the worker. In that case automatic generation smoke is skipped instead of inventing /generate fallback arguments that cannot exist. """ contract = read_inference_contract(workspace) blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json") validation_level = str(contract.get("validation_level") or "").strip().lower().replace("_", "-") primary = normalize_api_name(str(contract.get("primary_api_name") or "")) if contract else "" contract_full_false = _as_bool_false(contract.get("full_inference_implemented")) blocker_source = str(blockers.get("source") or "") if isinstance(blockers, dict) else "" heuristic_blocker_json = blocker_source == "worker_heuristic_from_PI_SUMMARY_or_app.py" blocker_full_false = False if heuristic_blocker_json else _as_bool_false(blockers.get("full_inference_implemented")) validation_boot_only = validation_level in {"boot-only", "health-only", "diagnostic-only", "info-only"} primary_absent = not primary or primary == "/health" blockers_count = blockers.get("blockers_count") if not isinstance(blockers_count, int): raw_blockers = blockers.get("blockers") blockers_count = len(raw_blockers) if isinstance(raw_blockers, list) else 0 declared = bool(contract_full_false or blocker_full_false or (validation_boot_only and primary_absent)) return { "declared": declared, "source": "inference_contract" if contract_full_false or validation_boot_only else "technical_blockers" if blocker_full_false else "", "full_inference_implemented": False if (contract_full_false or blocker_full_false) else contract.get("full_inference_implemented"), "validation_level": contract.get("validation_level"), "primary_api_name": contract.get("primary_api_name"), "blockers_count": blockers_count, "contract_present": bool(contract), "technical_blockers_present": bool(blockers), "technical_blockers_heuristic_only": bool(heuristic_blocker_json), } def _as_bool_true(value) -> bool: if value is True: return True if isinstance(value, str): return value.strip().lower() in {"true", "yes", "1", "on", "success", "passed", "ok"} return bool(value) if isinstance(value, (int, float)) else False def build_promise_validation_status(workspace: Path | None, validation: dict | None, generation_smoke: dict | None, *, status_hint: str = "") -> dict: """Separate app boot validation from model-card promise fulfillment. v198.20: a diagnostic/manual-action Space can boot and respond to Gradio, but that is not the same as fulfilling the model-card promise. This helper provides a stable machine-readable verdict that prevents hydrated Run Explorer/Active Run views from promoting diagnostic-only blockers to Success. """ contract = read_inference_contract(workspace) demo_contract = read_demo_quality_contract(workspace) blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json") validation = validation if isinstance(validation, dict) else {} generation_smoke = generation_smoke if isinstance(generation_smoke, dict) else {} status_hint = str(status_hint or "").strip() contract_no_full = contract_declares_no_full_inference(workspace) validation_level = str(contract.get("validation_level") or demo_contract.get("validation_level") or "").strip().lower().replace("_", "-") fallback_only = _as_bool_true(contract.get("fallback_or_diagnostic_only")) or _as_bool_true(demo_contract.get("fallback_or_diagnostic_only")) real_inference_false = _as_bool_false(contract.get("real_inference_implemented")) or _as_bool_false(demo_contract.get("real_inference_implemented")) full_inference_false = _as_bool_false(contract.get("full_inference_implemented")) or _as_bool_false(demo_contract.get("full_inference_implemented")) manual_required = _as_bool_true(contract.get("manual_hardware_required")) or "manual" in validation_level or "manual" in status_hint.lower() diagnostic_level = any(marker in validation_level for marker in ("diagnostic", "boot-only", "health-only", "info-only", "manual-hardware")) official_only_text = " ".join([ str(contract.get("reason") or ""), str(demo_contract.get("model_card_promise") or ""), str(demo_contract.get("primary_user_flow") or ""), str(demo_contract.get("promise_fulfillment_risk") or ""), str(blockers.get("reason") or ""), ]).lower() # v198.26.13: "official Space" by itself is not a blocker. Pi often # mentions the upstream official Space as provenance or comparison even when # the generated Space implements real inference. Treat only explicit # redirect/only/not-implemented formulations as official-only blockers. official_only = any(marker in official_only_text for marker in ( "only available in the official", "available only in the official", "use the official demo instead", "use the official space instead", "redirects to the official", "redirect to the official", "this space does not run inference", "this demo does not run inference", "does not generate real", "does not load model", "not generate real", "real inference is not implemented", "full inference is not implemented", "full inference deferred", )) health_passed = validation_health_passed(validation) or bool(generation_smoke.get("health_passed")) smoke_status_lower = str(generation_smoke.get("status") or "").lower() smoke_passed = smoke_status_lower == "success" or _as_bool_true(generation_smoke.get("ok")) demo_usable_smoke = smoke_status_lower == "demo_usable_smoke_passed" or _as_bool_true(generation_smoke.get("demo_usable_smoke_passed")) # A live canonical smoke with positive contracts is runtime proof. It must # override advisory/heuristic text such as a benign "official Space" mention. runtime_full_inference_proof = bool( smoke_passed and not contract_no_full.get("declared") and not fallback_only and not real_inference_false and not full_inference_false and not diagnostic_level and not manual_required ) if runtime_full_inference_proof: official_only = False no_full_reason = contract_no_full.get("declared") or fallback_only or real_inference_false or full_inference_false or diagnostic_level or official_only or manual_required if manual_required: promise_status = "deferred_manual_hardware" ui_status = "manual_hardware_required" badge = "Manual hardware required" reason = "Full inference is deferred until the user selects suitable Space hardware." elif no_full_reason: promise_status = "not_fulfilled_diagnostic_only" ui_status = "technical_blocker_boot_only" if health_passed or smoke_passed else "technical_blocker" badge = "Diagnostic Space" reason = "The Space may boot/respond, but the model-card promise is not fulfilled because real/full inference is not implemented." elif smoke_passed: promise_status = "fulfilled" ui_status = "full_inference_success" badge = "Full inference" reason = "A live generation smoke test passed for the promised demo flow." elif demo_usable_smoke: promise_status = "not_verified_demo_usable" ui_status = "demo_usable_full_promise_not_verified" badge = "Demo usable" reason = "A reduced minimal smoke generated output, but the canonical/full promise smoke was not verified." elif health_passed: promise_status = "not_verified" smoke_owner = str(generation_smoke.get("failure_owner") or "") if isinstance(generation_smoke, dict) else "" smoke_class = str(generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "") if isinstance(generation_smoke, dict) else "" if smoke_owner == "factory_validation_client" or smoke_class == "smoke_schema_error": ui_status = "interactive_app_available_smoke_failed" badge = "Manual test required" reason = "The Space boots, but ASF could not verify generation because the automatic smoke payload did not match the Gradio schema." else: ui_status = "partial_validation" badge = "Health only" reason = "The Space boots, but the promised generation flow was not verified." else: promise_status = "not_verified" ui_status = status_hint or "unknown" badge = "Not verified" reason = "No successful app boot or promise validation was observed." app_boot_status = "passed" if health_passed or smoke_passed else "failed" if str(validation.get("status") or "").lower() in {"failed", "error", "timeout"} else "unknown" return { "schema_version": "promise_validation.v198_20", "app_boot_validation_status": app_boot_status, "promise_validation_status": promise_status, "promise_fulfilled": promise_status == "fulfilled", "ui_status": ui_status, "ui_badge": badge, "reason": reason, "health_passed": bool(health_passed), "generation_smoke_passed": bool(smoke_passed), "diagnostic_only": bool(no_full_reason and not manual_required), "manual_hardware_required": bool(manual_required), "fallback_or_diagnostic_only": bool(fallback_only), "real_inference_implemented": False if real_inference_false else contract.get("real_inference_implemented", demo_contract.get("real_inference_implemented")), "full_inference_implemented": False if (full_inference_false or contract_no_full.get("declared")) else contract.get("full_inference_implemented", demo_contract.get("full_inference_implemented")), "validation_level": contract.get("validation_level") or demo_contract.get("validation_level") or "", "contract_declares_no_full_inference": contract_no_full, } def write_contract_skipped_generation_smoke(run_dir: Path, events_path: Path, expected_output_type: str, target_space_id: str, reason: dict | None = None) -> dict: reason = reason or {} payload = { "status": "skipped", "skip_reason": "contract_declared_no_full_inference", "reason": "Pi declared that full inference is not implemented; automatic generation smoke was skipped.", "target_space": target_space_id, "api_name": None, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "contract_aware_skip": True, "contract_reason": reason, "next_action": "Full inference is blocked. No generation endpoint exists. Review TECHNICAL_BLOCKERS.json / PI_SUMMARY.md or provide a dedicated implementation and hardware plan.", **measured_zero_gpu_recommendation(None), } record_generation_smoke_result(run_dir, payload, phase="generation_smoke") write_json(run_dir / "tests" / "payload_source.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "selected_api_name": None, "parent_smoke_payload_used": False}) write_json(run_dir / "tests" / "validation_engine.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "resolved_request_required_before_predict": False, "skipped_by_contract": True}) write_json(run_dir / "tests" / "resolved_validation_request.json", {"api_name": None, "test_args": [], "test_kwargs": {}, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "resolved_request_required_before_predict": False}) write_live_status(run_dir, stage="generation_smoke", status="skipped", message="Generation smoke skipped because Pi declared full inference unavailable", data=payload) append_event(events_path, "generation_smoke", "skipped", "Generation smoke skipped because Pi declared no full inference endpoint", payload) return payload def _param_semantic_name(param: dict, index: int, expected_output_type: str = "") -> str: """Best-effort semantic name for anonymous Gradio `param_N` schemas. v198.26.7: gradio_client can expose Textbox/Number inputs as param_0, param_1... while Pi's canonical smoke example uses semantic keys like prompt/width/height/steps/seed. A positional anonymous fallback prevents a valid prompt from being replaced by the schema-generated empty string. """ raw = str((param or {}).get("name") or "").strip().lower().replace("-", "_").replace(" ", "_") component = str((param or {}).get("component") or "").strip().lower() if raw and not re.fullmatch(r"param_\d+|arg\d+", raw): return raw expected = str(expected_output_type or "").strip().lower() if index == 0 and any(token in component for token in ("textbox", "text", "str")): return "prompt" if index == 0 and expected in {"text", "audio", "video", "image"}: return "prompt" common = ["prompt", "width", "height", "num_inference_steps", "seed", "guidance_scale"] if expected == "audio": common = ["prompt", "duration", "steps", "seed", "guidance_scale"] if expected == "video": common = ["prompt", "width", "height", "num_frames", "num_inference_steps", "seed"] if index < len(common): return common[index] return raw or f"arg{index}" def _canonical_key_aliases(name: str) -> list[str]: name = str(name or "").strip().lower().replace("-", "_").replace(" ", "_") aliases = { "prompt": ["prompt", "text", "query", "input", "instruction", "caption"], "negative_prompt": ["negative_prompt", "negative", "negative_text"], "width": ["width", "image_width", "w"], "height": ["height", "image_height", "h"], "num_inference_steps": ["num_inference_steps", "inference_steps", "steps", "num_steps", "sampling_steps"], "steps": ["steps", "num_inference_steps", "inference_steps", "num_steps", "sampling_steps"], "seed": ["seed", "random_seed"], "guidance_scale": ["guidance_scale", "cfg_scale", "scale", "guidance"], "duration": ["duration", "seconds", "length", "audio_length"], "num_frames": ["num_frames", "frames", "frame_count"], } return aliases.get(name, [name, name.replace("_", " "), name.replace("_", "-")]) def _canonical_dict_value_for_param(raw_args: dict, param: dict, index: int, generated: list, expected_output_type: str) -> tuple[object, str]: semantic = _param_semantic_name(param, index, expected_output_type) raw_lower = {str(k).strip().lower().replace("-", "_").replace(" ", "_"): k for k in raw_args.keys()} for candidate in _canonical_key_aliases(semantic): key = raw_lower.get(candidate.replace("-", "_").replace(" ", "_")) if key is not None: return raw_args[key], f"semantic:{candidate}" name = str((param or {}).get("name") or f"arg{index}") for candidate in (name, name.replace("_", " "), name.replace("_", "-"), name.lower(), name.lower().replace("_", " "), name.lower().replace("_", "-")): if candidate in raw_args: return raw_args[candidate], f"schema_name:{candidate}" values = list(raw_args.values()) if index < len(values) and re.fullmatch(r"param_\d+|arg\d+", str(name).strip().lower()): return values[index], "anonymous_positional_dict_order" return (generated[index] if index < len(generated) else smoke_value_for_parameter(param, expected_output_type)), "schema_generated_fallback" def detect_required_text_payload_resolution_issue(args: list, params: list[dict], canonical_example: dict | None = None) -> dict: """Detect when ASF resolved a required text prompt to empty before predict().""" non_empty_canonical = [] if isinstance(canonical_example, dict): raw_inputs, _ = _raw_inputs_from_canonical_smoke(canonical_example) if isinstance(raw_inputs, dict): for key, value in raw_inputs.items(): if isinstance(value, str) and value.strip(): non_empty_canonical.append({"key": str(key), "value_preview": value[:120]}) elif isinstance(raw_inputs, list): for i, value in enumerate(raw_inputs): if isinstance(value, str) and value.strip(): non_empty_canonical.append({"index": i, "value_preview": value[:120]}) for index, param in enumerate(params or []): if index >= len(args or []): continue name = str((param or {}).get("name") or f"arg{index}").strip() semantic = _param_semantic_name(param, index) component = str((param or {}).get("component") or "").lower() required = bool((param or {}).get("required", False)) value = args[index] is_text = any(token in component for token in ("textbox", "text", "str")) or semantic in {"prompt", "text", "query", "instruction"} if required and is_text and isinstance(value, str) and not value.strip() and non_empty_canonical: return { "schema_version": "smoke_payload_resolution_guard.v198_26_7", "detected": True, "failure_owner": "factory_validation_client", "failure_class": "smoke_payload_resolution_failed", "failure_type": "required_text_resolved_empty", "parameter_index": index, "parameter_name": name, "semantic_name": semantic, "canonical_non_empty_inputs": non_empty_canonical[:5], "recommended_action": "Fix canonical smoke dict-to-args resolution before calling gradio_client.predict; do not call the Space with an empty required prompt.", } return {"schema_version": "smoke_payload_resolution_guard.v198_26_7", "detected": False} def _canonical_smoke_dict_from_contracts(inference_contract: dict | None, demo_quality_contract: dict | None) -> tuple[dict | None, str]: """Return the preferred canonical smoke example and its source. Preference order for v198.21: 1. DEMO_QUALITY_CONTRACT.canonical_smoke_example (demo-quality promise) 2. INFERENCE_CONTRACT.canonical_smoke_example 3. legacy INFERENCE_CONTRACT.smoke_test """ demo_quality_contract = demo_quality_contract if isinstance(demo_quality_contract, dict) else {} inference_contract = inference_contract if isinstance(inference_contract, dict) else {} demo_example = demo_quality_contract.get("canonical_smoke_example") if isinstance(demo_example, dict) and demo_example: return demo_example, "demo_quality_contract_canonical_smoke_example" inference_example = inference_contract.get("canonical_smoke_example") if isinstance(inference_example, dict) and inference_example: return inference_example, "inference_contract_canonical_smoke_example" legacy_smoke = inference_contract.get("smoke_test") if isinstance(legacy_smoke, dict) and legacy_smoke: return legacy_smoke, "inference_contract_smoke_test" return None, "" def _raw_inputs_from_canonical_smoke(smoke: dict) -> tuple[object, dict]: """Normalize Pi-provided smoke examples across common field names.""" if not isinstance(smoke, dict): return [], {} raw_kwargs = smoke.get("kwargs", {}) if raw_kwargs is None or not isinstance(raw_kwargs, dict): raw_kwargs = {} for key in ("args", "inputs", "input", "values", "test_args"): if key in smoke: return smoke.get(key), raw_kwargs return [], raw_kwargs def canonical_smoke_example_payload(inference_contract: dict, demo_quality_contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None: smoke, source = _canonical_smoke_dict_from_contracts(inference_contract, demo_quality_contract) if not isinstance(smoke, dict): return None params = endpoint_parameters_for_smoke(endpoint) smoke_api = normalize_api_name(str(smoke.get("api_name") or smoke.get("endpoint") or api_name)) raw_args, raw_kwargs = _raw_inputs_from_canonical_smoke(smoke) generated, _ = build_generation_smoke_args(endpoint, expected_output_type) if isinstance(raw_args, list): args = [coerce_smoke_value(v, params[i] if i < len(params) else {}) for i, v in enumerate(raw_args)] if params and len(args) < len(params): args.extend(generated[len(args):]) return { "api_name": smoke_api, "test_args": args, "test_kwargs": raw_kwargs, "parameters": params, "source": source, "canonical_smoke_example_present": True, "canonical_smoke_reason": smoke.get("reason") or smoke.get("why_representative") or smoke.get("description") or "", } if isinstance(raw_args, dict): args = [] resolution = [] for i, param in enumerate(params): value, source_key = _canonical_dict_value_for_param(raw_args, param, i, generated, expected_output_type) args.append(coerce_smoke_value(value, param)) resolution.append({ "index": i, "name": str(param.get("name") or f"arg{i}"), "semantic_name": _param_semantic_name(param, i, expected_output_type), "source": source_key, }) if not params: args = list(raw_args.values()) return { "api_name": smoke_api, "test_args": args, "test_kwargs": raw_kwargs, "parameters": params, "source": source, "canonical_smoke_example_present": True, "canonical_smoke_reason": smoke.get("reason") or smoke.get("why_representative") or smoke.get("description") or "", "dict_resolution": resolution, } # Empty/unsupported inputs: canonical exists but cannot be transformed safely. return { "api_name": smoke_api, "test_args": generated, "test_kwargs": raw_kwargs, "parameters": params, "source": source + "_fallback_schema_args", "canonical_smoke_example_present": True, "canonical_smoke_unsupported_inputs": True, } def contract_smoke_test_payload(contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None: """Legacy compatibility wrapper for INFERENCE_CONTRACT.smoke_test.""" return canonical_smoke_example_payload(contract, {}, api_name, endpoint, expected_output_type) def _coerce_health_payload(value): if isinstance(value, dict): return value if isinstance(value, (list, tuple)) and value and isinstance(value[0], dict): return value[0] if isinstance(value, str): raw = value.strip() if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}: raw = raw[1:-1].strip() for parser in (json.loads, ast.literal_eval): try: parsed = parser(raw) except Exception: continue if isinstance(parsed, dict): return parsed if isinstance(parsed, (list, tuple)) and parsed and isinstance(parsed[0], dict): return parsed[0] return None def _semantic_health_payloads(validation: dict | None) -> list[tuple[str, dict]]: if not isinstance(validation, dict): return [] payloads: list[tuple[str, dict]] = [] for key in ("json", "result", "data", "health", "result_repr", "body", "text"): payload = _coerce_health_payload(validation.get(key)) if isinstance(payload, dict): payloads.append((key, payload)) nested = _coerce_health_payload(payload.get("json")) if isinstance(nested, dict): payloads.append((f"{key}.json", nested)) payloads.append(("validation", validation)) return payloads HEALTH_NEGATIVE_STATUS_MARKERS = { "unhealthy", "failed", "failure", "error", "not_ready", "not ready", "runtime_error", "unavailable", "not_available", "not available", "not_loaded", "not loaded", "model_not_loaded", "model not loaded", "model_unavailable", "pipeline_not_loaded", "pipeline not loaded", "pipeline_unavailable", "loading_failed", "load_failed", "initialization_failed", } HEALTH_POSITIVE_STATUS_MARKERS = {"healthy", "ok", "ready", "success", "passed"} MODEL_NOT_READY_TEXT_MARKERS = [ "unavailable", "not_ready", "not ready", "not_loaded", "not loaded", "model_not_loaded", "model not loaded", "model pipeline is not loaded", "model pipeline not loaded", "pipeline is not loaded", "pipeline not loaded", "pipeline_not_loaded", "model is not initialized", "model not initialized", "not initialized", "failed to load model", "model load failed", "loading failed", ] def _health_text_values(validation: dict | None) -> list[tuple[str, str]]: values: list[tuple[str, str]] = [] if not isinstance(validation, dict): return values for key in ("result_repr", "text", "body", "message", "error", "last_error", "load_error", "pipeline_error", "model_error"): value = validation.get(key) if value is not None: values.append((key, str(value))) for source, payload in _semantic_health_payloads(validation): for key in ("status", "health", "state", "model", "pipeline", "generation", "message", "error", "last_error", "load_error", "pipeline_error", "model_error"): value = payload.get(key) if isinstance(value, str): values.append((f"{source}.{key}", value)) return values def validation_health_semantics(validation: dict | None) -> dict: """Return semantic health details without equating reachability with readiness. v198.26.19: `/health` or Gradio `/health` transport success only proves the app is reachable. Values such as `unavailable`, `not_loaded`, or `Model pipeline is not loaded` mean the model is not ready and must not be counted as model-ready health or a completed repair. """ details = { "schema_version": "health_semantics.v198_26_30", "endpoint_reachable": False, "semantic_checked": False, "semantic_passed": None, "negative_markers": [], "positive_markers": [], "load_error": "", "last_error": "", "model_ready": None, "pipeline_ready": None, "model_family": "", "loader_strategy": "", "expected_output_type": "", "failure_class": "", "transport_status": "", } if not isinstance(validation, dict): return details method = str(validation.get("method") or "").lower() validator = str(validation.get("validator") or "").lower() api_name = normalize_api_name(str(validation.get("api_name") or "")) if validation.get("api_name") else "" transport_success = validation.get("status") == "success" details["transport_status"] = str(validation.get("status") or "") details["endpoint_reachable"] = bool(transport_success and (method in {"http_health", "gradio"} or validator in {"http_get_health", "gradio_client"} or api_name == "/health")) scaffold_markers = {"initial-scaffold", "initial_scaffold", "placeholder", "scaffold", "template"} for source, payload in _semantic_health_payloads(validation): for key in ("status", "health", "state", "model", "pipeline", "generation"): if key not in payload: continue value = str(payload.get(key) or "").strip().lower() # The top-level `status: success` is validator/transport status, not # model health. Only nested payloads may use `status` as a semantic # model readiness marker. if source == "validation" and key == "status": continue if value in HEALTH_NEGATIVE_STATUS_MARKERS: details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}={value}") elif value in HEALTH_POSITIVE_STATUS_MARKERS and key != "state": details["semantic_checked"] = True details["positive_markers"].append(f"{source}.{key}={value}") for key in ("stage", "runtime_stage", "phase"): if key in payload: value = str(payload.get(key) or "").strip().lower() if value in scaffold_markers or any(marker in value for marker in ("initial-scaffold", "initial_scaffold", "placeholder")): details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}={value}") for key in ("pipeline_ready", "pipeline_loaded", "model_ready", "model_loaded", "generation_ready"): if key in payload: details["semantic_checked"] = True if key in {"pipeline_ready", "pipeline_loaded"}: details["pipeline_ready"] = bool(payload.get(key)) if isinstance(payload.get(key), bool) else details.get("pipeline_ready") if payload.get(key) is False: details["negative_markers"].append(f"{source}.{key}=false") elif payload.get(key) is True: details["positive_markers"].append(f"{source}.{key}=true") for key in ("model_family", "loader_strategy", "expected_output_type"): value = payload.get(key) if value and not details.get(key): details[key] = str(value)[:200] for key in ("error", "last_error", "load_error", "pipeline_error", "model_error"): value = payload.get(key) if value: details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}=present") if not details["load_error"]: details["load_error"] = str(value)[:1000] if key == "last_error" and not details["last_error"]: details["last_error"] = str(value)[:1000] health_error_text = "\n".join(v for _, v in _health_text_values(validation)) try: if is_sdxl_lora_text_encoder_mismatch_text(health_error_text): details["failure_class"] = "sdxl_lora_text_encoder_mismatch" except Exception: pass for key, value in _health_text_values(validation): lowered = value.strip().lower().strip("'\"") if lowered in HEALTH_NEGATIVE_STATUS_MARKERS or any(marker in lowered for marker in MODEL_NOT_READY_TEXT_MARKERS): details["semantic_checked"] = True details["negative_markers"].append(f"{key}=model_not_ready") if not details["load_error"] and key in {"error", "last_error", "load_error", "pipeline_error", "model_error", "result_repr", "text", "message"}: details["load_error"] = value[:1000] if details["negative_markers"]: details["semantic_passed"] = False details["model_ready"] = False if details.get("pipeline_ready") is None: details["pipeline_ready"] = False elif details["semantic_checked"] and details["positive_markers"]: details["semantic_passed"] = True details["model_ready"] = True if details.get("pipeline_ready") is None: details["pipeline_ready"] = True return details def validation_endpoint_reachable(validation: dict | None) -> bool: if not isinstance(validation, dict) or validation.get("status") != "success": return False semantics = validation_health_semantics(validation) if semantics.get("endpoint_reachable"): return True method = str(validation.get("method") or "").lower() validator = str(validation.get("validator") or "").lower() api_name = normalize_api_name(str(validation.get("api_name") or "")) if validation.get("api_name") else "" return bool(method in {"http_health", "gradio"} or validator in {"http_get_health", "gradio_client"} or api_name == "/health") def validation_health_passed(validation: dict | None) -> bool: if not validation_endpoint_reachable(validation): return False semantics = validation_health_semantics(validation) if semantics.get("semantic_passed") is False: return False return True def repair_validation_outcome_from_validation(validation: dict | None, *, success_message: str) -> dict: """Classify post-repair live validation without overstating `/health`. A repair can make the Space reachable while the model remains unavailable. That is useful progress, but it is not a completed repair for full-inference demos; `/generate` still has to pass. """ semantics = validation_health_semantics(validation) if semantics.get("endpoint_reachable") and semantics.get("semantic_passed") is False: return { "post_repair_validation": "boot_validation_passed_model_unavailable", "failure_type": "model_unavailable_after_repair", "final_user_message": "Repair made the Space reachable, but /health reports the model unavailable/not ready; generation smoke must still pass before this is a full repair success.", "event_status": "warning", "event_message": "Repair reached boot-level validation, but model health is unavailable", "health_semantics": semantics, } return { "post_repair_validation": "success", "failure_type": "", "final_user_message": success_message, "event_status": "success", "event_message": success_message, "health_semantics": semantics, } def is_model_pipeline_not_loaded_text(text: str) -> bool: lowered = str(text or "").lower() direct_markers = [ "model pipeline is not loaded", "model pipeline not loaded", "pipeline is not loaded", "pipeline not loaded", "pipeline_not_loaded", "model_not_loaded", "model not loaded", "model is not initialized", "model not initialized", "pipe is none", "pipe is not loaded", ] if any(marker in lowered for marker in direct_markers): return True return "gpu may be unavailable" in lowered and ("pipeline" in lowered or "model" in lowered) def is_sdxl_lora_text_encoder_mismatch_text(text: str) -> bool: lowered = str(text or "").lower() if "cliptextmodel" in lowered and "text_model" in lowered: return True if "object has no attribute" in lowered and "text_model" in lowered and ("sdxl" in lowered or "lora" in lowered or "stable diffusion xl" in lowered): return True return False def is_hf_cache_permission_denied_text(text: str) -> bool: lowered = str(text or "").lower() if "permission denied" not in lowered and "errno 13" not in lowered: return False cache_markers = [ "'/data'", '"/data"', "/data/.cache", "hf_home", "hf_hub_cache", "transformers_cache", "diffusers_cache", "huggingface cache", "huggingface/hub", ] return any(marker in lowered for marker in cache_markers) def is_diffusers_duplicate_token_text(text: str) -> bool: lowered = str(text or "").lower() if "multiple values for keyword argument" not in lowered or "token" not in lowered: return False return any(marker in lowered for marker in ["from_pretrained", "diffusionpipeline", "pipeline", "diffusers", "zimagepipeline"]) def smoke_failure_signature(smoke: dict | None) -> str: if not isinstance(smoke, dict): return "" failure_class = str(smoke.get("failure_class") or smoke.get("failure_type") or "").strip().lower() failure_type = str(smoke.get("failure_type") or "").strip().lower() text = str(smoke.get("error") or smoke.get("result_repr") or smoke.get("result_info") or smoke.get("message") or smoke) lowered = text.lower() if is_sdxl_lora_text_encoder_mismatch_text(text): marker = "sdxl_lora_text_encoder_mismatch" elif is_diffusers_duplicate_token_text(text): marker = "duplicate_token" elif is_hf_cache_permission_denied_text(text): marker = "hf_cache_permission_denied" elif is_model_pipeline_not_loaded_text(text): marker = "model_pipeline_not_loaded" else: compact = re.sub(r"\s+", " ", lowered)[:300] marker = compact return "|".join(part for part in [failure_class, failure_type, marker] if part) def repair_post_smoke_validation_status(pre_repair_smoke: dict | None, post_repair_smoke: dict | None) -> dict: post = post_repair_smoke if isinstance(post_repair_smoke, dict) else {} if post.get("status") == "success": return {"post_repair_validation": "success", "same_error": False, "failure_type": "", "message": "Targeted smoke repair revalidated the Space."} pre_sig = smoke_failure_signature(pre_repair_smoke) post_sig = smoke_failure_signature(post) same = bool(pre_sig and post_sig and pre_sig == post_sig) failure_type = post.get("failure_type") or post.get("failure_class") or "generation_smoke_still_failed" if same: return { "post_repair_validation": "repair_failed_same_error", "same_error": True, "failure_type": failure_type, "message": "Targeted smoke repair uploaded, but the same generation error returned after revalidation.", "pre_repair_signature": pre_sig, "post_repair_signature": post_sig, } return { "post_repair_validation": "smoke_still_failed", "same_error": False, "failure_type": failure_type, "message": "Targeted smoke repair uploaded, but generation smoke still did not verify the primary demo flow.", "pre_repair_signature": pre_sig, "post_repair_signature": post_sig, } def is_cuda_oom_text(text: str) -> bool: lowered = str(text or "").lower() markers = [ "cuda out of memory", "torch.cuda.outofmemoryerror", "outofmemoryerror", "out of memory", "tried to allocate", "gpu memory", "reserved memory", "allocated memory", ] return ("cuda" in lowered or "gpu" in lowered or "outofmemory" in lowered) and any(marker in lowered for marker in markers) def is_placeholder_scaffold_text(text: str) -> bool: lowered = str(text or "").lower() markers = [ "initial scaffold", "initial-scaffold", "initial_scaffold", "pi should replace this", "model-specific inference path", "placeholder demo", "placeholder scaffold", "stage: initial-scaffold", ] return any(marker in lowered for marker in markers) def detect_placeholder_scaffold(workspace: Path) -> dict: """Detect a final runtime that still contains ASF/Pi placeholder scaffold text.""" app_path = workspace / "app.py" text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" detected = is_placeholder_scaffold_text(text) markers = [] lowered = text.lower() for marker in ["initial scaffold", "initial-scaffold", "pi should replace this", "model-specific inference path", "placeholder scaffold"]: if marker in lowered: markers.append(marker) return { "schema_version": "placeholder_scaffold_detection.v198_26_7", "detected": bool(detected), "failure_owner": "pi_generation" if detected else "", "failure_class": "placeholder_scaffold_deployed" if detected else "", "failure_type": "model_specific_runtime_missing" if detected else "", "repair_candidate": bool(detected), "markers": markers, "checked_file": "app.py", "recommended_action": "Replace the initial scaffold with a model-specific inference app before treating the Space as healthy." if detected else "", } def _extract_missing_executable(text: str) -> str: text = str(text or "") patterns = [ r"No such file or directory:\s*['\"]([^'\"]+)['\"]", r"\[Errno 2\].*?:\s*['\"]([^'\"]+)['\"]", r"FileNotFoundError:.*?['\"]([^'\"]+)['\"]", ] for pattern in patterns: match = re.search(pattern, text, flags=re.IGNORECASE | re.DOTALL) if match: candidate = (match.group(1) or "").strip() if candidate and "/" not in candidate and "\\" not in candidate: return candidate if candidate: return Path(candidate).name or candidate return "" def diagnose_generation_smoke_failure(error_or_result, *, inference_strategy: str = "", expected_output_type: str = "", phase: str = "exception") -> dict: """Classify failed automatic generation smoke tests without collapsing usable Spaces to failed. v198.13: smoke failures are no longer a single opaque `generation_smoke_error`. The diagnosis is intentionally conservative: it identifies validator-owned failures, missing runtime executables, and common model-init responses so the UI and repair loop can distinguish partial usable Spaces from terminal app/runtime failures. """ text = str(error_or_result or "") lowered = text.lower() strategy = str(inference_strategy or "").strip() evidence = [] if text: evidence.append(text[:1200]) if strategy: evidence.append(f"inference_strategy={strategy}") base = { "schema_version": "generation_smoke_diagnosis.v198_9", "smoke_passed": False, "phase": phase, "inference_strategy": strategy, "expected_output_type": expected_output_type or "", "failure_owner": "unknown", "failure_class": "generation_smoke_error", "failure_type": "generation_smoke_error", "actionability": "review_required", "repair_candidate": False, "recommended_action": "Inspect generation_smoke.json, Space logs, and retry via Space Test with adjusted inputs.", "evidence": evidence, } marker = "no value provided for required argument:" missing_executable = _extract_missing_executable(text) if "smoke_payload_resolution_failed" in lowered or "required_text_resolved_empty" in lowered or "prompt cannot be empty" in lowered: base.update({ "failure_owner": "factory_validation_client", "failure_class": "smoke_payload_resolution_failed", "failure_type": "required_text_resolved_empty", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Fix ASF canonical smoke payload resolution so required prompt/text inputs are non-empty before calling gradio_client; do not repair the generated Space.", }) elif is_placeholder_scaffold_text(text): base.update({ "failure_owner": "pi_generation", "failure_class": "placeholder_scaffold_deployed", "failure_type": "model_specific_runtime_missing", "actionability": "targeted_pi_repair", "repair_candidate": True, "recommended_action": "Run a targeted Pi repair to replace the initial scaffold with a model-specific inference app that returns the promised artifact.", }) elif "asf smoke input materialization failed" in lowered or "does not exist on local filesystem" in lowered: base.update({ "failure_owner": "factory_validation_client", "failure_class": "smoke_input_materialization_failed", "failure_type": "smoke_input_materialization_failed", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Resolve or create the media/file smoke input in the ASF validator before calling gradio_client; do not repair the generated Space.", }) elif is_smoke_schema_error_text(text): base.update({ "failure_owner": "factory_validation_client", "failure_class": "smoke_schema_error", "failure_type": "smoke_schema_error", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Fix ASF Gradio smoke payload construction for file/media inputs; do not repair the generated Space.", }) elif is_gradio_hidden_error_text(text): base.update({ "failure_owner": "app_runtime", "failure_class": "gradio_hidden_runtime_error", "failure_type": "gradio_hidden_runtime_error", "actionability": "deterministic_show_error_patch_then_retry", "repair_candidate": False, "recommended_action": "Apply deterministic demo.launch(show_error=True), redeploy, and rerun smoke to capture the real traceback before Pi repair.", }) elif is_schema_choice_type_error(text): base.update({ "failure_owner": "factory_validator", "failure_class": "validator_schema_choice_type_mismatch", "failure_type": "validator_schema_choice_type_mismatch", "actionability": "validator_self_repair", "retryable_with_schema_payload": True, "auto_retry_supported": True, "recommended_action": "Retry smoke with schema-correct choice values from Gradio view_api().", }) elif marker in lowered: missing = text.split(":", 1)[-1].strip().strip("'\"") base.update({ "failure_owner": "factory_validator", "failure_class": "validator_request_error", "failure_type": "validator_request_error", "actionability": "validator_payload_repair", "retryable_with_schema_payload": True, "missing_argument": missing, "recommended_action": "Resolve the required argument from the Gradio schema before retrying smoke validation.", }) elif missing_executable: base.update({ "failure_owner": "app_runtime", "failure_class": "missing_runtime_cli", "failure_type": "missing_runtime_cli", "missing_executable": missing_executable, "actionability": "repair_candidate", "repair_candidate": True, "recommended_action": "Run a targeted repair for the missing runtime CLI: fix the package dependency, PATH, or command invocation.", }) elif is_hf_cache_permission_denied_text(text): base.update({ "failure_owner": "factory_runtime_template", "failure_class": "hf_cache_permission_denied", "failure_type": "non_writable_hf_cache_path", "actionability": "deterministic_writable_cache_patch", "repair_candidate": True, "recommended_action": "Patch generated app cache env vars away from /data to a writable /tmp Hugging Face cache, redeploy, then re-run /generate.", }) elif is_diffusers_duplicate_token_text(text): base.update({ "failure_owner": "factory_runtime_template", "failure_class": "diffusers_duplicate_token_argument", "failure_type": "duplicate_from_pretrained_token_argument", "actionability": "deterministic_diffusers_token_patch", "repair_candidate": True, "recommended_action": "Patch generated Diffusers from_pretrained token handling to avoid passing token twice, redeploy, then re-run /generate.", }) elif is_sdxl_lora_text_encoder_mismatch_text(text): base.update({ "failure_owner": "model_runtime", "failure_class": "sdxl_lora_text_encoder_mismatch", "failure_type": "sdxl_base_or_loader_incompatible", "actionability": "sdxl_lora_loader_recipe_repair", "repair_candidate": True, "recommended_action": "Run a targeted SDXL-LoRA repair: verify the base SDXL loader, text encoders, LoRA weight_name/trigger words, dependency compatibility, then re-upload and validate /generate.", }) elif is_model_pipeline_not_loaded_text(text): base.update({ "failure_owner": "model_runtime", "failure_class": "model_pipeline_not_loaded", "failure_type": "model_not_loaded_after_boot", "actionability": "zerogpu_or_model_load_repair_review", "repair_candidate": True, "recommended_action": "Collect Space runtime logs, then run one targeted repair focused on model loading / ZeroGPU compatibility; re-upload and validate with /generate, not only /health.", }) elif any(m in lowered for m in ["timeout", "timed out", "read operation timed out"]): base.update({ "failure_owner": "infra_or_generated_space", "failure_class": "timeout", "failure_type": "timeout", "actionability": "retry_or_runtime_recovery", "retryable_with_longer_timeout": True, "recommended_action": "Retry with a longer timeout or apply runtime recovery if the Space startup/runtime state is inconclusive.", }) elif "modulenotfounderror" in lowered or "no module named" in lowered or "importerror" in lowered: base.update({ "failure_owner": "app_runtime", "failure_class": "missing_python_dependency", "failure_type": "missing_python_dependency", "actionability": "repair_candidate", "repair_candidate": True, "recommended_action": "Run a targeted dependency repair based on the missing Python package.", }) elif is_cuda_oom_text(text): base.update({ "failure_owner": "hardware", "failure_class": "gpu_oom", "failure_type": "insufficient_vram_or_payload_too_large", "actionability": "retry_minimal_smoke_or_larger_hardware", "repair_candidate": False, "recommended_action": "Do not trigger Pi repair for a pure CUDA OOM. Retry with a minimal smoke payload or use larger/ZeroGPU hardware before declaring the Space unusable.", }) elif any(m in lowered for m in ["traceback", "runtimeerror", "attributeerror", "valueerror", "cuda", "out of memory", "exception"]): base.update({ "failure_owner": "app_runtime", "failure_class": "app_runtime_error", "failure_type": "app_runtime_error", "actionability": "repair_candidate", "repair_candidate": True, "recommended_action": "Run a targeted repair if the traceback points to a minimal code or dependency fix.", }) elif any(m in lowered for m in ["invalid state", "runtime_error", "build_error"]): base.update({ "failure_owner": "generated_space", "failure_class": "space_runtime_state", "failure_type": "space_runtime_state", "actionability": "runtime_recovery_or_repair", "repair_candidate": True, "recommended_action": "Inspect runtime state/logs and apply runtime recovery before destructive repair if no traceback is definitive.", }) return base def classify_generation_smoke_error(error: Exception | str, *, inference_strategy: str = "", expected_output_type: str = "") -> dict: diagnosis = diagnose_generation_smoke_failure(error, inference_strategy=inference_strategy, expected_output_type=expected_output_type, phase="exception") payload = { "failure_type": diagnosis.get("failure_type") or diagnosis.get("failure_class") or "generation_smoke_error", "failure_owner": diagnosis.get("failure_owner") or "unknown", "failure_class": diagnosis.get("failure_class") or "generation_smoke_error", "actionability": diagnosis.get("actionability") or "review_required", "recommended_action": diagnosis.get("recommended_action") or "", } for key in ("retryable_with_schema_payload", "auto_retry_supported", "missing_argument", "retryable_with_longer_timeout", "repair_candidate", "missing_executable"): if key in diagnosis: payload[key] = diagnosis[key] return payload def result_contains_expected_output(result, expected: str): expected = (expected or "any").lower() info = {"result_type": type(result).__name__, "result_repr": repr(result)[:2000]} paths = [] type_hints = [] text_values = [] def visit(obj): type_hints.append(type(obj).__name__.lower()) if isinstance(obj, (str, Path)): value = str(obj) if value: paths.append(value) text_values.append(value) elif isinstance(obj, dict): for key, value in obj.items(): key_l = str(key).lower() if key_l in {"path", "name", "url", "file", "filepath"}: visit(value) elif key_l in {"mime_type", "mime", "type", "format"} and value: type_hints.append(str(value).lower()) elif key_l in {"image", "video", "audio", "text"}: type_hints.append(key_l) visit(value) elif isinstance(value, (dict, list, tuple)): visit(value) elif isinstance(obj, (list, tuple)): for item in obj: visit(item) else: # PIL Images, numpy arrays returned through Gradio objects, etc. module = getattr(type(obj), "__module__", "") if module: type_hints.append(module.lower()) visit(result) lower_paths = [str(p).lower() for p in paths] hints = " ".join(type_hints + lower_paths) info["detected_paths"] = paths[:20] info["type_hints"] = type_hints[:30] if expected == "any": return result is not None, info image_ext = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff") video_ext = (".mp4", ".mov", ".webm", ".avi", ".mkv") audio_ext = (".wav", ".mp3", ".flac", ".ogg", ".m4a") if expected == "text": if isinstance(result, str) and bool(result.strip()): return True, info return any(isinstance(x, str) and x.strip() and not Path(x).suffix.lower() in image_ext + video_ext + audio_ext for x in text_values), info if expected == "image": return any(p.endswith(image_ext) for p in lower_paths) or any(h in hints for h in ["image", "pil.", "png", "jpeg", "jpg", "webp"]), info if expected == "video": return any(p.endswith(video_ext) for p in lower_paths) or any(h in hints for h in ["video", "mp4", "webm", "moviepy"]), info if expected == "audio": return any(p.endswith(audio_ext) for p in lower_paths) or any(h in hints for h in ["audio", "wav", "soundfile"]), info return result is not None, info def copy_result_artifacts(result, run_dir: Path): artifacts = run_dir / "artifacts" artifacts.mkdir(parents=True, exist_ok=True) copied = [] def maybe_copy(obj): if isinstance(obj, (str, Path)): path = Path(str(obj)) if path.exists() and path.is_file(): target = artifacts / path.name try: shutil.copy2(path, target) copied.append(str(target)) except Exception: pass elif isinstance(obj, dict): for key in ["path", "name"]: if key in obj: maybe_copy(obj[key]) for value in obj.values(): if isinstance(value, (dict, list, tuple)): maybe_copy(value) elif isinstance(obj, (list, tuple)): for item in obj: maybe_copy(item) maybe_copy(result) return copied def inspect_video_artifacts(copied_paths: list[str]) -> dict: video_ext = {".mp4", ".mov", ".webm", ".avi", ".mkv"} videos = [] for item in copied_paths or []: path = Path(str(item)) if path.suffix.lower() not in video_ext: continue record = {"path": str(path), "extension": path.suffix.lower(), "exists": path.exists()} if path.exists(): try: record["size_bytes"] = path.stat().st_size record["container_valid"] = bool(path.stat().st_size > 0) except Exception as exc: record["inspection_error"] = str(exc)[:1000] videos.append(record) return { "schema_version": "video_artifact_inspection.v198_24", "videos": videos, "video_count": len(videos), "has_video_artifact": bool(videos), "quality_assessment": "not_evaluated", } def smoke_timeout_seconds(expected_output_type: str) -> int: expected = str(expected_output_type or "").strip().lower() if expected == "video": return int(os.environ.get("ASF_VIDEO_SMOKE_TIMEOUT_SECONDS", "900")) if expected == "audio": return int(os.environ.get("ASF_AUDIO_SMOKE_TIMEOUT_SECONDS", "420")) return int(os.environ.get("ASF_GENERATION_SMOKE_TIMEOUT_SECONDS", "300")) def is_smoke_schema_error_text(error_or_result) -> bool: text = str(error_or_result or "").lower() return any(marker in text for marker in [ "imagedata", "filedata", "videodata", "audiodata", "input should be a valid dictionary or instance of", "input_type=str", "input_value='http", 'input_value="http', ]) def is_gradio_hidden_error_text(error_or_result) -> bool: text = str(error_or_result or "").lower() return "has raised an exception" in text and "show_error=true" in text def record_generation_smoke_result(run_dir: Path, payload: dict, *, phase: str = "generation_smoke") -> dict: tests_dir = run_dir / "tests" tests_dir.mkdir(parents=True, exist_ok=True) history_path = tests_dir / "generation_smoke_history.json" history = read_json(history_path, {"schema_version": "generation_smoke_history.v198_24", "attempts": []}) or {"schema_version": "generation_smoke_history.v198_24", "attempts": []} attempts = history.get("attempts") if isinstance(history.get("attempts"), list) else [] entry = dict(payload or {}) entry.setdefault("recorded_at", now()) entry.setdefault("phase", phase) entry["attempt_index"] = len(attempts) + 1 attempts.append(entry) history = {"schema_version": "generation_smoke_history.v198_24", "latest_attempt_index": entry["attempt_index"], "latest_status": entry.get("status"), "attempts": attempts} write_json(history_path, history) write_json(tests_dir / "generation_smoke_latest.json", entry) write_json(tests_dir / "generation_smoke.json", payload) return entry def build_generation_smoke_failure_payload(target_space_id: str, expected_output_type: str, error, *, workspace: Path | None = None, phase: str = "exception") -> dict: contract = read_inference_contract(workspace) strategy = str((contract or {}).get("inference_strategy") or "") diagnosis = diagnose_generation_smoke_failure(error, inference_strategy=strategy, expected_output_type=expected_output_type, phase=phase) payload = { "status": "failed", "target_space": target_space_id, "expected_output_type": expected_output_type, "error": str(error)[:4000], "next_action": diagnosis.get("recommended_action") or "Use Prefill Space Test after the run finishes to retry with schema-adjusted arguments.", **measured_zero_gpu_recommendation(None), "diagnosis_path": "tests/generation_smoke_diagnosis.json", } payload.update(classify_generation_smoke_error(error, inference_strategy=strategy, expected_output_type=expected_output_type)) return payload def wait_for_video_smoke_settle(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, expected_output_type: str): if str(expected_output_type or "").strip().lower() != "video": return {"skipped": True, "reason": "not_video"} checks = int(os.environ.get("ASF_VIDEO_SMOKE_SETTLE_CHECKS", "3")) sleep_seconds = int(os.environ.get("ASF_VIDEO_SMOKE_SETTLE_SECONDS", "30")) history = [] for attempt in range(1, checks + 1): runtime = write_space_runtime(api, target_space_id, token, run_dir, events_path, attempt) stage = str((runtime or {}).get("stage") or "").upper() history.append(runtime) if "RUNNING_BUILDING" in stage or ("BUILDING" in stage and "RUNNING" in stage): append_event(events_path, "video_smoke_settle", "waiting", "Video Space is still settling after health passed; delaying heavy smoke", {"attempt": attempt, "runtime": runtime, "sleep_seconds": sleep_seconds}) time.sleep(sleep_seconds) continue append_event(events_path, "video_smoke_settle", "success", "Video Space runtime is stable enough for smoke", {"attempt": attempt, "runtime": runtime}) payload = {"schema_version": "video_smoke_settle.v198_24", "settled": True, "attempts": attempt, "history": history} write_json(run_dir / "tests" / "video_smoke_settle.json", payload) return payload payload = {"schema_version": "video_smoke_settle.v198_24", "settled": False, "attempts": checks, "history": history, "proceeding_after_max_checks": True} write_json(run_dir / "tests" / "video_smoke_settle.json", payload) append_event(events_path, "video_smoke_settle", "warning", "Proceeding with video smoke after settle budget expired", payload) return payload def ensure_gradio_launch_show_error(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload") -> dict: app_path = workspace / "app.py" payload = {"schema_version": "gradio_show_error_patch.v198_24", "reason": reason, "applied": False, "launch_calls_seen": 0, "launch_calls_patched": 0} if not app_path.exists(): payload["skip_reason"] = "missing_app_py" write_json(run_dir / "gradio_show_error_patch.json", payload) return payload source = app_path.read_text(encoding="utf-8", errors="ignore") try: tree = ast.parse(source) except Exception as exc: payload.update({"skip_reason": "app_py_parse_failed", "error": str(exc)[:1000]}) write_json(run_dir / "gradio_show_error_patch.json", payload) append_event(events_path, "gradio_show_error_patch", "warning", "Could not parse app.py to enforce show_error=True", payload) return payload calls = [] for node in ast.walk(tree): if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "launch": payload["launch_calls_seen"] += 1 if any(kw.arg == "show_error" for kw in node.keywords if kw.arg): continue if not hasattr(node, "end_lineno") or not hasattr(node, "end_col_offset"): continue calls.append(node) if not calls: write_json(run_dir / "gradio_show_error_patch.json", payload) return payload lines = source.splitlines(keepends=True) for node in sorted(calls, key=lambda n: (n.end_lineno, n.end_col_offset), reverse=True): if node.lineno == node.end_lineno: line_i = node.end_lineno - 1 line = lines[line_i] insert_at = node.end_col_offset - 1 insertion = "show_error=True" if not node.args and not node.keywords else ", show_error=True" lines[line_i] = line[:insert_at] + insertion + line[insert_at:] else: close_i = node.end_lineno - 1 close_line = lines[close_i] close_indent = re.match(r"^\s*", close_line).group(0) lines.insert(close_i, f"{close_indent} show_error=True,\n") payload["launch_calls_patched"] += 1 updated = "".join(lines) if updated != source: app_path.write_text(updated, encoding="utf-8") payload["applied"] = True write_json(run_dir / "gradio_show_error_patch.json", payload) append_event(events_path, "gradio_show_error_patch", "success", "Ensured generated Gradio launch uses show_error=True", payload) else: write_json(run_dir / "gradio_show_error_patch.json", payload) return payload def measured_zero_gpu_recommendation(latency_seconds: float | None): if latency_seconds is None: return { "observed_latency_seconds": None, "recommended_zero_gpu_duration_seconds": None, "recommendation_source": "not_measured", "recommendation_confidence": "none", } recommended = int(max(30, min(300, latency_seconds * 2 + 15))) return { "observed_latency_seconds": round(latency_seconds, 3), "recommended_zero_gpu_duration_seconds": recommended, "recommendation_source": "live_gradio_predict", "recommendation_confidence": "measured", "measurement_note": "Measured from a live gradio_client.predict call, including Gradio/API/network/result serialization overhead.", } def run_gradio_validation_harness(*, client, api_name: str, test_args: list, test_kwargs: dict, expected_output_type: str, run_dir: Path, events_path: Path, validation_mode: str, payload_source: str): """Shared v190.24 validation contract for automatic build smoke. Linked Space Test uses a mirrored helper with the same contract; both flows must resolve payload, write resolved_validation_request.json, coerce schema choices, then predict and verify output. """ resolved = { "api_name": api_name, "test_args": list(test_args or []), "test_kwargs": dict(test_kwargs or {}), "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "resolved_request_required_before_predict": True, } write_json(run_dir / "tests" / "resolved_validation_request.json", resolved) started = time.time() result = client.predict(*resolved["test_args"], api_name=api_name, **resolved["test_kwargs"]) latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) return result, latency, ok, info, resolved def run_generation_smoke(target_space_id: str, token: str, run_dir: Path, events_path: Path, expected_output_type: str, workspace: Path | None = None): api_name = "/generate" test_kwargs = {} timeout_s = smoke_timeout_seconds(expected_output_type) recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if workspace and (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") if not isinstance(recipe, dict): recipe = {} app_runtime_contract = load_app_runtime_contract(workspace, run_dir) if workspace else build_app_runtime_contract(recipe) append_event(events_path, "generation_smoke", "started", "Calling live generation endpoint for family-aware smoke", {"api_name": api_name, "expected_output_type": expected_output_type, "model_family": recipe.get("model_family"), "loader_strategy": recipe.get("loader_strategy"), "smoke_test_strategy": recipe.get("smoke_test_strategy"), "smoke_timeout_seconds": timeout_s}) client = make_gradio_client(target_space_id, token, timeout_s=timeout_s) identity = write_gradio_client_identity(client, target_space_id, run_dir, events_path, phase="generation_smoke") if identity.get("mismatch"): diagnosis = {"schema_version": "generation_smoke_diagnosis.v198_26_7", "smoke_passed": False, "phase": "space_identity", "expected_output_type": expected_output_type or "", "failure_owner": "factory_validation_client", "failure_class": "space_identity_mismatch", "failure_type": "space_identity_mismatch", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Do not validate a Space whose gradio_client URL does not match the target_space_id; fix Space identity propagation before retrying.", "evidence": [json.dumps(identity, ensure_ascii=False)[:1200]]} payload = {"status": "failed", "target_space": target_space_id, "api_name": api_name, "expected_output_type": expected_output_type, "error": "space_identity_mismatch", "failure_type": "space_identity_mismatch", "failure_class": "space_identity_mismatch", "failure_owner": "factory_validation_client", "actionability": "factory_fix_required", "repair_candidate": False, "space_identity": identity, **measured_zero_gpu_recommendation(None)} write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", diagnosis) record_generation_smoke_result(run_dir, payload, phase="generation_smoke") append_event(events_path, "generation_smoke", "failed", "Space identity mismatch before generation smoke", payload) return payload schema = client.view_api(return_format="dict") discovered = api_names_from_schema(schema) if api_name not in discovered and discovered: non_health = [name for name in discovered if name != "/health"] if non_health: api_name = non_health[0] endpoint = endpoint_schema_for_api(schema, api_name) contract = read_inference_contract(workspace) demo_quality_contract = read_demo_quality_contract(workspace) contract_primary = normalize_api_name(str(contract.get("primary_api_name") or "")) if contract else "" if contract_primary and contract_primary in discovered and contract_primary != "/health": api_name = contract_primary endpoint = endpoint_schema_for_api(schema, api_name) contract_payload = canonical_smoke_example_payload(contract, demo_quality_contract, api_name, endpoint, expected_output_type) if contract_payload and contract_payload.get("api_name") in discovered: api_name = contract_payload["api_name"] endpoint = endpoint_schema_for_api(schema, api_name) contract_payload = canonical_smoke_example_payload(contract, demo_quality_contract, api_name, endpoint, expected_output_type) endpoint_parameters = endpoint_parameter_names(endpoint) if contract_payload: test_args = contract_payload.get("test_args") or [] test_kwargs = contract_payload.get("test_kwargs") or {} contract_parameters = contract_payload.get("parameters") or [] smoke_parameters = merge_smoke_parameters_with_endpoint_schema(contract_parameters, endpoint_parameters_for_smoke(endpoint)) smoke_source = contract_payload.get("source") or "inference_contract_smoke_test" else: test_args, smoke_parameters = build_generation_smoke_args(endpoint, expected_output_type) smoke_source = "gradio_schema" if smoke_parameters else "default_fallback" raw_test_args, initial_choice_changes = coerce_smoke_args_to_schema_choices(test_args, smoke_parameters) test_args, file_input_conversions = prepare_gradio_file_inputs(raw_test_args, smoke_parameters, run_dir) canonical_example = None canonical_source = "" if contract_payload and contract_payload.get("canonical_smoke_example_present"): canonical_example, canonical_source = _canonical_smoke_dict_from_contracts(contract, demo_quality_contract) smoke_payload = {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "parameters": smoke_parameters, "source": smoke_source, "contract_present": bool(contract), "app_runtime_contract_present": bool(app_runtime_contract), "model_family": recipe.get("model_family") or app_runtime_contract.get("model_family"), "loader_strategy": recipe.get("loader_strategy") or app_runtime_contract.get("loader_strategy"), "smoke_test_strategy": recipe.get("smoke_test_strategy") or app_runtime_contract.get("smoke_test_strategy"), "demo_quality_contract_present": bool(demo_quality_contract), "canonical_smoke_example_present": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s} payload_resolution_guard = detect_required_text_payload_resolution_issue(test_args, smoke_parameters, canonical_example) smoke_payload["payload_resolution_guard"] = payload_resolution_guard if canonical_example: write_json(run_dir / "tests" / "canonical_smoke_example.json", {"source": canonical_source, "example": canonical_example, "resolved_api_name": api_name, "raw_resolved_args": raw_test_args, "resolved_args": test_args, "resolved_kwargs": test_kwargs, "expected_output_type": expected_output_type, "file_input_conversions": file_input_conversions, "dict_resolution": contract_payload.get("dict_resolution") if isinstance(contract_payload, dict) else [], "payload_resolution_guard": payload_resolution_guard}) write_json(run_dir / "tests" / "generation_smoke_payload.json", smoke_payload) write_json(run_dir / "tests" / "payload_source.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "payload_source": smoke_source, "selected_api_name": api_name, "parent_smoke_payload_used": False, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source}) write_json(run_dir / "tests" / "validation_engine.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "resolved_request_required_before_predict": True}) write_json(run_dir / "tests" / "resolved_validation_request.json", {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": smoke_source, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "schema_choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s, "payload_resolution_guard": payload_resolution_guard}) write_json(run_dir / "tests" / "schema_coercion.json", {"api_name": api_name, "changes": initial_choice_changes, "original_args": raw_test_args, "resolved_args": test_args, "file_input_conversions": file_input_conversions, "payload_resolution_guard": payload_resolution_guard}) write_json(run_dir / "tests" / "generation_api_schema.json", {"schema": schema, "api_names": discovered, "selected_api_name": api_name, "endpoint_parameters": endpoint_parameters, "smoke_parameters": smoke_parameters, "smoke_payload_source": smoke_source, "canonical_smoke_example_used": bool(canonical_example)}) if payload_resolution_guard.get("detected"): diagnosis = { "schema_version": "generation_smoke_diagnosis.v198_26_7", "smoke_passed": False, "phase": "payload_resolution", "inference_strategy": str(contract.get("inference_strategy") or ""), "expected_output_type": expected_output_type or "", "failure_owner": "factory_validation_client", "failure_class": "smoke_payload_resolution_failed", "failure_type": "required_text_resolved_empty", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": payload_resolution_guard.get("recommended_action"), "evidence": [json.dumps(payload_resolution_guard, ensure_ascii=False)[:1200]], } payload = { "status": "failed", "target_space": target_space_id, "api_name": api_name, "discovered_api_names": discovered, "endpoint_parameters": endpoint_parameters, "test_args": test_args, "test_kwargs": test_kwargs, "smoke_payload_source": smoke_source, "validation_level": "canonical_promise_smoke", "demo_usable_smoke_passed": False, "canonical_promise_smoke_passed": False, "full_promise_smoke_passed": False, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "payload_resolution_guard": payload_resolution_guard, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s, "next_action": diagnosis["recommended_action"], "expected_output_type": expected_output_type, "latency_seconds": None, "result_info": {}, "copied_artifacts": [], "validated_at": now(), "failure_type": diagnosis["failure_type"], "failure_class": diagnosis["failure_class"], "failure_owner": diagnosis["failure_owner"], "actionability": diagnosis["actionability"], "repair_candidate": False, **measured_zero_gpu_recommendation(None), } write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", diagnosis) record_generation_smoke_result(run_dir, payload, phase="generation_smoke") write_live_status(run_dir, stage="generation_smoke", status="failed", message="Automatic smoke payload resolution failed before calling Gradio", data=payload) append_event(events_path, "generation_smoke", "failed", "Automatic smoke payload resolution failed before calling Gradio", payload) return payload write_live_status(run_dir, stage="generation_smoke", status="running", message="Calling live generation endpoint", data={"api_name": api_name, "source": smoke_source, "canonical_smoke_example_used": bool(canonical_example)}) started = time.time() retry_info = {"attempts": 1, "choice_corrections": initial_choice_changes, "retried": False} minimal_smoke_attempt = {"attempted": False, "succeeded": False, "reason": ""} try: result = client.predict(*test_args, api_name=api_name, **test_kwargs) except Exception as first_error: if is_schema_choice_type_error(first_error): retry_params = smoke_parameters or endpoint_parameters_for_smoke(endpoint) corrected_args, retry_choice_changes = coerce_smoke_args_to_schema_choices(raw_test_args, retry_params) if not retry_choice_changes or corrected_args == raw_test_args: schema_params = endpoint_parameters_for_smoke(endpoint) corrected_args, retry_choice_changes = coerce_smoke_args_to_schema_choices(raw_test_args, schema_params) if retry_choice_changes: retry_params = schema_params if not retry_choice_changes or corrected_args == raw_test_args: parsed_args, parsed_changes = coerce_smoke_args_from_choice_error(raw_test_args, first_error, endpoint_parameters_for_smoke(endpoint)) if parsed_changes and parsed_args != raw_test_args: corrected_args, retry_choice_changes = parsed_args, parsed_changes retry_params = endpoint_parameters_for_smoke(endpoint) if retry_choice_changes and corrected_args != raw_test_args: append_event(events_path, "generation_smoke", "warning", "Retrying generation smoke with Gradio schema choice types", {"api_name": api_name, "error": str(first_error)[:1500], "choice_corrections": retry_choice_changes}) write_live_status(run_dir, stage="generation_smoke_retry", status="running", message="Retrying smoke test with schema-corrected choice values", data={"api_name": api_name, "choice_corrections": retry_choice_changes}) raw_test_args = corrected_args test_args, retry_file_conversions = prepare_gradio_file_inputs(raw_test_args, retry_params, run_dir) file_input_conversions = file_input_conversions + retry_file_conversions retry_info = {"attempts": 2, "choice_corrections": initial_choice_changes + retry_choice_changes, "file_input_conversions": file_input_conversions, "retried": True, "first_error": str(first_error)[:2000], "retry_reason": "schema_choice_type_mismatch", "self_repair": "schema_aware_payload_retry"} retry_artifact = {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "parameters": retry_params, **retry_info} write_json(run_dir / "tests" / "generation_smoke_payload_retry.json", retry_artifact) write_json(run_dir / "tests" / "validator_self_repair.json", {"schema_version": "validator_self_repair.v198_7", "triggered": True, "failure_owner": "factory_validator", "failure_type": "validator_schema_choice_type_mismatch", "first_error": str(first_error)[:2000], "action": "retry_with_schema_correct_choice_values", "choice_corrections": retry_choice_changes, "retry_payload_path": "tests/generation_smoke_payload_retry.json"}) result = client.predict(*test_args, api_name=api_name, **test_kwargs) else: write_json(run_dir / "tests" / "validator_self_repair.json", {"schema_version": "validator_self_repair.v198_7", "triggered": True, "failure_owner": "factory_validator", "failure_type": "validator_schema_choice_type_mismatch", "first_error": str(first_error)[:2000], "action": "no_safe_schema_correction_found", "choice_error": parse_gradio_choice_error(first_error)}) raise elif is_cuda_oom_text(str(first_error)): minimal_raw_args, minimal_changes = build_minimal_demo_smoke_args(raw_test_args, smoke_parameters or endpoint_parameters_for_smoke(endpoint), expected_output_type) if not minimal_changes or minimal_raw_args == raw_test_args: raise append_event(events_path, "generation_smoke", "warning", "Canonical promise smoke hit CUDA OOM; retrying reduced minimal demo smoke", {"api_name": api_name, "error": str(first_error)[:1500], "minimal_changes": minimal_changes}) write_live_status(run_dir, stage="minimal_demo_smoke", status="running", message="Retrying with reduced minimal smoke payload after canonical OOM", data={"api_name": api_name, "minimal_changes": minimal_changes}) minimal_params = smoke_parameters or endpoint_parameters_for_smoke(endpoint) minimal_args, minimal_file_conversions = prepare_gradio_file_inputs(minimal_raw_args, minimal_params, run_dir) minimal_smoke_attempt = {"attempted": True, "succeeded": False, "reason": "canonical_cuda_oom", "canonical_error": str(first_error)[:2000], "minimal_changes": minimal_changes} file_input_conversions = file_input_conversions + minimal_file_conversions retry_info = {"attempts": 2, "choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "retried": True, "first_error": str(first_error)[:2000], "retry_reason": "canonical_cuda_oom_minimal_demo_smoke", "minimal_changes": minimal_changes} minimal_artifact = {"api_name": api_name, "test_args": minimal_args, "raw_test_args": minimal_raw_args, "test_kwargs": test_kwargs, "parameters": minimal_params, "validation_level": "minimal_demo_smoke", **retry_info} write_json(run_dir / "tests" / "generation_smoke_payload_minimal.json", minimal_artifact) write_json(run_dir / "tests" / "minimal_demo_smoke_retry.json", {"schema_version": "minimal_demo_smoke_retry.v198_26_7", "triggered": True, "failure_owner": "hardware", "failure_type": "canonical_cuda_oom", "canonical_error": str(first_error)[:2000], "minimal_changes": minimal_changes, "retry_payload_path": "tests/generation_smoke_payload_minimal.json"}) try: result = client.predict(*minimal_args, api_name=api_name, **test_kwargs) raw_test_args = minimal_raw_args test_args = minimal_args smoke_parameters = minimal_params minimal_smoke_attempt["succeeded"] = True except Exception: # Keep the canonical OOM as the primary failure class. The # minimal retry is diagnostic only and must not hide the root # signal that the full promise payload exceeded capacity. raise first_error else: raise latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) copied = copy_result_artifacts(result, run_dir) video_inspection = inspect_video_artifacts(copied) if str(expected_output_type or "").strip().lower() == "video" else {} if video_inspection: write_json(run_dir / "tests" / "video_artifact_inspection.json", video_inspection) recommendation = measured_zero_gpu_recommendation(latency) smoke_diagnosis = {} if ok else diagnose_generation_smoke_failure(info.get("result_repr") or result, inference_strategy=str(contract.get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="output_verification") payload_status = "success" if ok else "failed" validation_level = "canonical_promise_smoke" if ok and minimal_smoke_attempt.get("attempted"): payload_status = "demo_usable_smoke_passed" validation_level = "minimal_demo_smoke" payload = { "status": payload_status, "target_space": target_space_id, "api_name": api_name, "discovered_api_names": discovered, "endpoint_parameters": endpoint_parameters, "test_args": test_args, "test_kwargs": test_kwargs, "smoke_payload_source": smoke_source, "validation_level": validation_level, "demo_usable_smoke_passed": bool(ok and minimal_smoke_attempt.get("attempted")), "canonical_promise_smoke_passed": bool(ok and not minimal_smoke_attempt.get("attempted")), "full_promise_smoke_passed": bool(ok and not minimal_smoke_attempt.get("attempted")), "minimal_smoke_attempt": minimal_smoke_attempt, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "auto_retry": retry_info, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s, "next_action": "" if ok else smoke_diagnosis.get("recommended_action") or "Use Prefill Space Test after the run finishes to retry with adjusted arguments.", "expected_output_type": expected_output_type, "latency_seconds": round(latency, 3), "result_info": info, "copied_artifacts": copied, "video_artifact_inspection": video_inspection, "validated_at": now(), **({ "failure_type": smoke_diagnosis.get("failure_type"), "failure_class": smoke_diagnosis.get("failure_class"), "failure_owner": smoke_diagnosis.get("failure_owner"), "actionability": smoke_diagnosis.get("actionability"), "repair_candidate": smoke_diagnosis.get("repair_candidate"), } if smoke_diagnosis else {}), **recommendation, } if smoke_diagnosis: write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, payload, phase="generation_smoke") if ok: if minimal_smoke_attempt.get("attempted"): payload = minimal_smoke_status_payload(payload) write_json(run_dir / "tests" / "generation_smoke.json", payload) write_json(run_dir / "tests" / "generation_smoke_latest.json", payload) write_live_status(run_dir, stage="minimal_demo_smoke", status="warning", message="Minimal demo smoke passed; full promise smoke was not verified", data={"latency_seconds": payload["latency_seconds"], "api_name": api_name, "auto_retry": retry_info}) append_event(events_path, "generation_smoke", "warning", "Minimal demo smoke passed after canonical OOM; full promise remains unverified", {"latency_seconds": payload["latency_seconds"], "recommended_zero_gpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "recommended_zerogpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "api_name": api_name, "auto_retry": retry_info, "validation_level": "minimal_demo_smoke"}) else: write_live_status(run_dir, stage="generation_smoke", status="success", message="Live generation smoke test passed", data={"latency_seconds": payload["latency_seconds"], "api_name": api_name, "auto_retry": retry_info}) append_event(events_path, "generation_smoke", "success", "Live generation smoke test passed and ZeroGPU timing was measured", {"latency_seconds": payload["latency_seconds"], "recommended_zero_gpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "recommended_zerogpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "api_name": api_name, "auto_retry": retry_info}) else: write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation returned an unexpected output type", data=payload) append_event(events_path, "generation_smoke", "failed", "Live generation returned an unexpected output type", payload) return payload def smoke_failure_repair_escalation_decision(generation_smoke: dict | None, *, validation: dict | None = None, contract: dict | None = None) -> dict: """Decide whether a failed generation smoke should trigger one targeted repair. v198.13: partial validation is useful for UI honesty, but a demo that boots yet cannot satisfy its primary flow should not stop there when the smoke failure is concrete and minimally repairable. This is deliberately bounded: validator-owned failures are handled by validator self-repair, model init without logs remains partial, and we only escalate actionable app/runtime failures such as a missing executable or missing Python dependency. """ smoke = generation_smoke if isinstance(generation_smoke, dict) else {} validation = validation if isinstance(validation, dict) else {} contract = contract if isinstance(contract, dict) else {} failure_class = str(smoke.get("failure_class") or smoke.get("failure_type") or "") failure_owner = str(smoke.get("failure_owner") or "") repair_candidate = bool(smoke.get("repair_candidate")) health_semantics = validation_health_semantics(validation) health_model_ready = validation_health_passed(validation) health_reachable = validation_endpoint_reachable(validation) actionable_classes = {"missing_runtime_cli", "missing_python_dependency", "app_runtime_error", "model_pipeline_not_loaded", "model_not_initialized", "hf_cache_permission_denied", "diffusers_duplicate_token_argument", "sdxl_lora_text_encoder_mismatch"} blocked_classes = {"validator_schema_choice_type_mismatch", "validator_request_error", "smoke_schema_error", "smoke_input_materialization_failed", "gradio_hidden_runtime_error", "gpu_oom", "insufficient_vram", "insufficient_vram_or_payload_too_large", "hardware_capacity", "cuda_out_of_memory"} strategy = str(contract.get("inference_strategy") or smoke.get("inference_strategy") or "") payload = { "schema_version": "generation_smoke_repair_escalation.v198_26_20", "triggered": False, "repair_candidate": repair_candidate, "health_passed": health_model_ready, "health_endpoint_reachable": health_reachable, "health_semantic_passed": health_semantics.get("semantic_passed"), "health_negative_markers": health_semantics.get("negative_markers") or [], "failure_class": failure_class, "failure_owner": failure_owner, "inference_strategy": strategy, "action": "keep_partial_validation", "reason": "not_actionable_or_not_safe_for_automatic_smoke_repair", } if str(smoke.get("status") or "") != "failed": payload["reason"] = "generation_smoke_not_failed" return payload if failure_class in blocked_classes or failure_owner in {"factory_validator", "factory_validation_client", "hardware"}: payload["reason"] = "handled_elsewhere_or_requires_logs_not_blind_repair" return payload if not health_reachable: payload["reason"] = "app_not_reachable_use_runtime_recovery_instead" return payload if not repair_candidate or failure_class not in actionable_classes: payload["reason"] = "smoke_failure_not_repair_candidate" return payload payload.update({ "triggered": True, "action": "targeted_patch_repair_before_final_partial", "reason": "space_reachable_but_primary_demo_flow_failed_with_actionable_smoke_error", "recommended_action": smoke.get("recommended_action") or "Run one targeted repair, redeploy, then re-run health and canonical smoke validation.", "missing_executable": smoke.get("missing_executable") or "", }) return payload def load_recipe_repair_context(workspace: Path | None, run_dir: Path | None) -> dict: """Load MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json for repair.""" workspace = Path(workspace) if workspace else None run_dir = Path(run_dir) if run_dir else None def load_named(name: str) -> tuple[dict, str]: candidates = [] if workspace: candidates.append(workspace / name) if run_dir: candidates.append(run_dir / name) candidates.append(run_dir / "analysis_inputs" / name) for candidate in candidates: try: if candidate.exists(): payload = load_json_if_exists(candidate) if isinstance(payload, dict): return payload, str(candidate) except Exception: continue return {}, "" recipe, recipe_source = load_named("MODEL_RECIPE.json") runtime_contract, runtime_source = load_named("APP_RUNTIME_CONTRACT.json") if not runtime_contract and recipe: try: runtime_contract = build_app_runtime_contract(recipe) runtime_source = "derived_from_MODEL_RECIPE.json" except Exception: runtime_contract = {} return { "schema_version": "recipe_repair_context.v198_26_32", "model_recipe": recipe, "model_recipe_source": recipe_source, "app_runtime_contract": runtime_contract, "app_runtime_contract_source": runtime_source, "recipe_available": bool(recipe), "runtime_contract_available": bool(runtime_contract), "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "artifact_role": recipe.get("artifact_role") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", "primary_endpoint": recipe.get("primary_endpoint") or runtime_contract.get("primary_endpoint") or "/generate", } def recipe_known_failure_catalog(recipe: dict | None = None, runtime_contract: dict | None = None) -> list[dict]: recipe = recipe if isinstance(recipe, dict) else {} runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} catalog = [] for item in recipe.get("known_failure_signatures") or []: if isinstance(item, dict): catalog.append(dict(item)) required = [ {"failure_class": "diffusers_duplicate_token_argument", "markers": ["got multiple values for keyword argument 'token'", "multiple values for keyword argument \"token\"", "duplicate token"], "repair_focus": "Remove duplicate token/use_auth_token injection from from_pretrained kwargs; do not change the model id or replace inference."}, {"failure_class": "hf_cache_permission_denied", "markers": ["permission denied", "/data", "HF_HOME", "huggingface cache", "read-only file system", "errno 13"], "repair_focus": "Move HF_HOME/HF_HUB_CACHE/TRANSFORMERS_CACHE/DIFFUSERS_CACHE to a writable Space path before importing/loading models."}, {"failure_class": "missing_spaces_gpu_decorator", "markers": ["missing @spaces.GPU", "ZeroGPU", "CUDA has been initialized before", "GPU task function", "@spaces.GPU"], "repair_focus": "Import spaces before CUDA-touching imports and protect the heavy inference function with @spaces.GPU while preserving fixed-GPU compatibility."}, {"failure_class": "validator_schema_choice_type_mismatch", "markers": ["not in the list of choices", "Invalid value", "Dropdown", "CheckboxGroup", "choice"], "repair_focus": "Respect the published Gradio schema and canonical smoke payload; do not patch the model for validator-owned payload coercion errors."}, {"failure_class": "sdxl_lora_text_encoder_mismatch", "markers": ["CLIPTextModel", "text_model", "has no attribute 'text_model'"], "repair_focus": "Treat the repo as an SDXL LoRA adapter: load the base SDXL pipeline first, then load_lora_weights(adapter_id, weight_name=...), and avoid loading the adapter as a standalone pipeline."}, {"failure_class": "flux_template_syntax_error", "markers": ["FluxPipeline", "jinja", "template", "syntax error", "unexpected", "chat_template"], "repair_focus": "Repair FLUX/prompt-template syntax without changing FluxPipeline loader semantics or replacing image generation with placeholder output."}, {"failure_class": "model_not_loaded_after_boot", "markers": ["model not loaded", "pipeline not loaded", "model_ready", "pipeline_ready", "last_error", "not initialized", "NoneType"], "repair_focus": "Fix loader initialization and expose the real load failure through /health.last_error; /generate must fail cleanly when the model is not ready."}, ] seen = {str(item.get("failure_class") or "") for item in catalog} for item in required: if item["failure_class"] not in seen: catalog.append(item) seen.add(item["failure_class"]) return catalog def match_recipe_known_failure_signature(error_text: str = "", recipe: dict | None = None, runtime_contract: dict | None = None, fallback_class: str = "") -> dict: text = str(error_text or "") low = text.lower() recipe = recipe if isinstance(recipe, dict) else {} runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} fallback_class = str(fallback_class or "") for sig in recipe_known_failure_catalog(recipe, runtime_contract): failure_class = str(sig.get("failure_class") or sig.get("category") or "") markers = [str(m) for m in (sig.get("markers") or sig.get("error_markers") or sig.get("signatures") or []) if str(m)] class_matches = bool(failure_class and fallback_class and failure_class == fallback_class) marker_matches = [m for m in markers if m.lower() in low] if failure_class == "sdxl_lora_text_encoder_mismatch" and is_sdxl_lora_text_encoder_mismatch_text(text): marker_matches = marker_matches or ["CLIPTextModel.text_model"] if class_matches or marker_matches: return { "schema_version": "recipe_known_failure_match.v198_26_32", "matched": True, "failure_class": failure_class or fallback_class, "matched_markers": marker_matches[:10], "repair_focus": sig.get("repair_focus") or sig.get("recommended_repair_focus") or sig.get("recommendation") or "Patch according to MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json.", "source_signature": sig, "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", } if fallback_class: return {"schema_version": "recipe_known_failure_match.v198_26_32", "matched": False, "failure_class": fallback_class, "matched_markers": [], "repair_focus": "No catalog signature matched; still use MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json as authoritative repair context.", "source_signature": {}, "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or ""} return {"schema_version": "recipe_known_failure_match.v198_26_32", "matched": False, "failure_class": "", "matched_markers": [], "repair_focus": ""} def recipe_family_repair_directives(recipe: dict | None = None, runtime_contract: dict | None = None, signature_match: dict | None = None) -> list[str]: recipe = recipe if isinstance(recipe, dict) else {} runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} signature_match = signature_match if isinstance(signature_match, dict) else {} family = str(recipe.get("model_family") or runtime_contract.get("model_family") or "") directives = [ "Use MODEL_RECIPE.json as the authoritative model-family plan, not a generic app repair brief.", "Preserve APP_RUNTIME_CONTRACT.json: /health must expose model_ready, pipeline_ready, model_family, loader_strategy, last_error, and expected_output_type.", "After patching, /generate must either return the expected output type or fail cleanly with the same underlying loader error visible in /health.last_error.", ] if signature_match.get("repair_focus"): directives.append("Known failure focus: " + str(signature_match.get("repair_focus"))) if family == "diffusers_sdxl_lora_adapter": directives.extend([ "This repo is an adapter. Do not call DiffusionPipeline.from_pretrained(adapter_id) as if it were a full pipeline.", "Load the SDXL base model from model_recipe.base_model_id, then call load_lora_weights(adapter_id, weight_name=adapter_weight_name when present).", "Keep the LoRA trigger words/prompt guidance from the model card when available.", ]) elif family == "diffusers_flux_pipeline": directives.extend([ "Keep FluxPipeline.from_pretrained semantics and the recipe-selected model id.", "Do not replace FLUX inference with a text-only or placeholder response; preserve image output.", ]) elif family == "gguf_llamacpp": directives.append("Use llama.cpp/llama-cpp-python style GGUF loading and text generation; do not attempt Transformers AutoModel loading for GGUF weights.") elif family == "onnx_runtime_model": directives.append("Use onnxruntime InferenceSession and preserve input/output tensor preprocessing; do not convert to a different runtime blindly.") elif family == "transformers_pipeline": directives.append("Use the Transformers pipeline/AutoModel strategy indicated by the recipe and preserve the task-specific output contract.") return directives def write_recipe_aware_repair_packet(workspace: Path, run_dir: Path, failure_reason: str = "", classification: dict | None = None, decision: dict | None = None) -> dict: ctx = load_recipe_repair_context(workspace, run_dir) recipe = ctx.get("model_recipe") or {} runtime_contract = ctx.get("app_runtime_contract") or {} category = str((classification or {}).get("category") or (classification or {}).get("failure_class") or (decision or {}).get("failure_class") or "") match = match_recipe_known_failure_signature(failure_reason, recipe, runtime_contract, fallback_class=category) packet = { "schema_version": "recipe_aware_repair_packet.v198_26_32", "authority": "MODEL_RECIPE.json + APP_RUNTIME_CONTRACT.json", **ctx, "known_failure_match": match, "family_repair_directives": recipe_family_repair_directives(recipe, runtime_contract, match), "repair_must_not_finish_not_started": True, "no_patch_policy": "If Pi cannot produce a publishable patch, ASF must either launch another more direct recipe-aware attempt while budget remains or finish with no_patch_produced_by_pi / repair_exhausted, never post_repair_validation=not_started.", } repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "RECIPE_AWARE_REPAIR_PACKET.json", packet) try: write_json(workspace / "RECIPE_AWARE_REPAIR_PACKET.json", packet) except Exception: pass return packet def recipe_repair_direct_followup_reason(run_dir: Path, terminal_status: str, failure_class: str = "") -> str: latest = _latest_repair_attempt_result(run_dir) return ( "Previous Pi repair did not produce an uploadable publishable patch.\n" f"Terminal status: {terminal_status}\n" f"Latest attempt result: {latest.get('post_repair_result') or ''}\n" f"Failure class: {failure_class or latest.get('failure_class') or ''}\n" "Relaunch with a more direct recipe-aware instruction: implement the known failure focus from RECIPE_AWARE_REPAIR_PACKET.json, modify only allowed publishable files, or write PATCH_REFUSAL.json with a concrete blocker." ) def should_relaunch_after_no_patch(run_dir: Path, terminal_status: str, remaining_patch_budget: int) -> dict: terminal_status = str(terminal_status or "") no_patch_statuses = {"no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard", "pi_repair_failed_before_upload"} latest = _latest_repair_attempt_result(run_dir) result = str(latest.get("post_repair_result") or "") result_is_no_patch = result in {"repair_noop", "expected_file_not_modified", "diff_gate_failed", "sanity_failed"} can_retry = remaining_patch_budget > 0 and (terminal_status in no_patch_statuses or result_is_no_patch) return {"schema_version": "recipe_repair_retry_decision.v198_26_32", "relaunch": bool(can_retry), "terminal_status": terminal_status, "remaining_patch_budget": int(remaining_patch_budget or 0), "latest_attempt": latest, "reason": "retry_with_more_direct_recipe_aware_brief" if can_retry else "repair_exhausted_or_not_retryable"} def write_smoke_repair_primary_error_packet(run_dir: Path, workspace: Path, generation_smoke: dict | None, *, failure_class: str = "") -> dict: smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = read_inference_contract(workspace) recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") if not isinstance(recipe, dict): recipe = {} repair_context = load_recipe_repair_context(workspace, run_dir) app_runtime_contract = repair_context.get("app_runtime_contract") or {} error_text = str(smoke.get("error") or smoke.get("result_info") or smoke.get("result_repr") or smoke)[:6000] payload = { "schema_version": "smoke_repair_primary_error.v198_26_27", "source": "generation_smoke", "failure_class": failure_class or smoke.get("failure_class") or smoke.get("failure_type") or "", "failure_type": smoke.get("failure_type") or "", "failure_owner": smoke.get("failure_owner") or "", "error": error_text, "expected_output_type": smoke.get("expected_output_type") or contract.get("expected_output_type") or "", "primary_api_name": smoke.get("api_name") or contract.get("primary_api_name") or contract.get("primary_endpoint") or "/generate", "contract": contract, "model_recipe": recipe, "app_runtime_contract": app_runtime_contract, "known_failure_match": match_recipe_known_failure_signature(error_text, recipe, app_runtime_contract, fallback_class=failure_class or smoke.get("failure_class") or smoke.get("failure_type") or ""), "sdxl_lora_text_encoder_mismatch": is_sdxl_lora_text_encoder_mismatch_text(error_text), "recommended_repair_focus": "sdxl_lora_base_loader_or_text_encoder_compatibility" if is_sdxl_lora_text_encoder_mismatch_text(error_text) else "model_loading_or_runtime_patch", } repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "SMOKE_PRIMARY_ERROR.json", payload) try: shutil.copy2(repair_dir / "SMOKE_PRIMARY_ERROR.json", workspace / "SMOKE_PRIMARY_ERROR.json") except Exception: pass return payload def _latest_repair_attempt_result(run_dir: Path) -> dict: payload = load_repair_attempts(run_dir) attempts = payload.get("attempts") if isinstance(payload.get("attempts"), list) else [] return attempts[-1] if attempts else {} def failed_pi_repair_terminal_status(run_dir: Path) -> dict: latest = _latest_repair_attempt_result(run_dir) result = str(latest.get("post_repair_result") or "").strip() mapping = { "repair_noop": ("no_patch_produced_by_pi", "Pi repair produced no publishable patch."), "pi_refused_patch": ("pi_refused_with_reason", "Pi refused to patch under the bounded repair policy."), "pi_exit_nonzero": ("repair_aborted_with_exception", "Pi repair command exited non-zero."), "diff_gate_failed": ("pi_patch_rejected_by_guard", "Pi produced a patch rejected by the repair diff guard."), "sanity_failed": ("pi_patch_rejected_by_guard", "Pi produced a patch rejected by sanity checks."), "expected_file_not_modified": ("no_relevant_patch_produced_by_pi", "Pi did not modify the expected publishable file."), } status, message = mapping.get(result, ("pi_repair_failed_before_upload", "Targeted Pi repair did not produce an uploadable patch.")) return {"post_repair_validation": status, "failure_type": status, "final_user_message": message, "latest_repair_attempt": latest} def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, token: str, implementation_mode: str, expected_output_type: str, validation: dict, generation_smoke: dict) -> tuple[dict, dict]: """Run one bounded targeted repair when the demo boots but its primary flow fails. The function returns the latest (validation, generation_smoke). It never loops: one repair, one upload, one health validation, one generation smoke retry. """ contract = read_inference_contract(workspace) escalation = smoke_failure_repair_escalation_decision(generation_smoke, validation=validation, contract=contract) write_json(run_dir / "tests" / "generation_smoke_repair_escalation.json", escalation) if not escalation.get("triggered"): return validation, generation_smoke failure_class = escalation.get("failure_class") or generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "generation_smoke_error" pre_repair_smoke = dict(generation_smoke) if isinstance(generation_smoke, dict) else {} try: collect_space_logs(target_space_id, token, run_dir, events_path) except Exception as log_exc: append_event(events_path, "generation_smoke_repair", "warning", "Could not refresh Space logs before targeted smoke repair", {"error": str(log_exc)[:1500]}) try: logs_dir = run_dir / "logs" latest_log_brief = "\n".join([ (logs_dir / "space_log_diagnostics.json").read_text(encoding="utf-8", errors="ignore")[:1500] if (logs_dir / "space_log_diagnostics.json").exists() else "", (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore")[-1500:] if (logs_dir / "space_logs_runtime.txt").exists() else "", (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore")[-1500:] if (logs_dir / "space_logs_build.txt").exists() else "", ]).strip() except Exception: latest_log_brief = "" cache_permission_evidence = "\n".join([ str(generation_smoke.get("error") or generation_smoke.get("result_info") or generation_smoke), latest_log_brief, ]) token_duplicate_evidence = cache_permission_evidence if failure_class == "diffusers_duplicate_token_argument" or is_diffusers_duplicate_token_text(token_duplicate_evidence): decision = { "action": "factory_patch_diffusers_token_guard", "confidence": "high", "source": "generation_smoke_repair_escalation.v198_26_20", "classification": { "category": "diffusers_duplicate_token_argument", "failure_phase": "generation_smoke", "recommendation": "Remove duplicate token injection from Diffusers from_pretrained kwargs, redeploy, then validate /generate.", "generation_smoke_failure": True, }, "constraints": [ "Do not replace the real inference app with diagnostics.", "Do not change the Gradio API contract or expected output type.", "Patch only duplicate token handling around Diffusers/model from_pretrained calls.", "After upload, success requires /generate to pass; /health alone is not full repair success.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Applying deterministic Diffusers token repair before Pi repair", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause="diffusers_duplicate_token_argument", repair_decision="factory_patch_diffusers_token_guard", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) token_patch = patch_workspace_for_diffusers_token_guard(workspace, run_dir, events_path, reason="generation_smoke_duplicate_token_repair", evidence=token_duplicate_evidence) if token_patch.get("applied"): write_repair_outcome(run_dir, events_path, patch_applied=True, deterministic_patch=token_patch, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading deterministic Diffusers token repair") try: write_auth_probe(run_dir, events_path, "before_diffusers_token_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Deterministic Diffusers token repair uploaded; revalidating health and demo smoke") except Exception as upload_exc: latest_smoke = dict(generation_smoke) latest_smoke.update({"status": "failed", "failure_type": "smoke_repair_upload_failed", "failure_class": "smoke_repair_upload_failed", "failure_owner": "factory_upload", "error": str(upload_exc)[:4000]}) write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="upload_failed", failure_type="smoke_repair_upload_failed", final_user_message="Deterministic Diffusers token repair was prepared but failed to upload; keeping partial validation.", validation_error=str(upload_exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_upload", "failed", "Deterministic Diffusers token repair upload failed; keeping partial validation", {"error": str(upload_exc)[:2000]}) return validation, latest_smoke try: write_auth_probe(run_dir, events_path, "before_diffusers_token_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_class = repair_post_smoke_validation_status(pre_repair_smoke, generation_smoke) post_status = post_class["post_repair_validation"] write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type=post_class.get("failure_type") or "", final_user_message=post_class.get("message") or "Deterministic Diffusers token repair validation completed.", post_repair_generation_smoke=generation_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Deterministic Diffusers token repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status"), "same_error": post_class.get("same_error")}) except Exception as exc: latest_smoke = build_generation_smoke_failure_payload(target_space_id, expected_output_type, exc, workspace=workspace, phase="post_diffusers_token_repair_exception") smoke_diagnosis = diagnose_generation_smoke_failure(exc, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="post_diffusers_token_repair_exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, latest_smoke, phase="post_diffusers_token_repair_exception") post_class = repair_post_smoke_validation_status(pre_repair_smoke, latest_smoke) generation_smoke = latest_smoke write_repair_outcome(run_dir, events_path, post_repair_validation=post_class["post_repair_validation"], failure_type=latest_smoke.get("failure_type") or post_class.get("failure_type") or "diffusers_token_repair_validation_failed", final_user_message=post_class.get("message") or "Deterministic Diffusers token repair validation failed; keeping partial validation.", validation_error=str(exc)[:4000], post_repair_generation_smoke=latest_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "failed", "Deterministic Diffusers token repair validation failed; keeping partial validation", {"error": str(exc)[:2000], "latest_failure_class": latest_smoke.get("failure_class"), "latest_failure_owner": latest_smoke.get("failure_owner"), "same_error": post_class.get("same_error")}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, deterministic_patch=token_patch, patch_applied=False, post_repair_validation="not_started", failure_type="diffusers_token_patch_not_applied") append_event(events_path, "generation_smoke_repair", "warning", "Diffusers duplicate-token repair was indicated but no deterministic token patch was applied; falling back to Pi repair", token_patch) if failure_class == "hf_cache_permission_denied" or is_hf_cache_permission_denied_text(cache_permission_evidence): decision = { "action": "factory_patch_writable_hf_cache", "confidence": "high", "source": "generation_smoke_repair_escalation.v198_26_20", "classification": { "category": "hf_cache_permission_denied", "failure_phase": "generation_smoke", "recommendation": "Rewrite generated app HF cache env vars away from /data to a writable /tmp cache, redeploy, then validate /generate.", "generation_smoke_failure": True, }, "constraints": [ "Do not replace the real inference app with diagnostics.", "Do not change the Gradio API contract or expected output type.", "Patch only Hugging Face cache environment paths and bootstrap writable cache directories.", "After upload, success requires /generate to pass; /health alone is not full repair success.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Applying deterministic writable HF cache repair before Pi repair", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause="hf_cache_permission_denied", repair_decision="factory_patch_writable_hf_cache", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) cache_patch = patch_workspace_for_writable_hf_cache(workspace, run_dir, events_path, reason="generation_smoke_permission_denied_repair", evidence=cache_permission_evidence) if cache_patch.get("applied"): write_repair_outcome(run_dir, events_path, patch_applied=True, deterministic_patch=cache_patch, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading deterministic writable cache repair") try: write_auth_probe(run_dir, events_path, "before_writable_cache_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Deterministic writable cache repair uploaded; revalidating health and demo smoke") except Exception as upload_exc: latest_smoke = dict(generation_smoke) latest_smoke.update({"status": "failed", "failure_type": "smoke_repair_upload_failed", "failure_class": "smoke_repair_upload_failed", "failure_owner": "factory_upload", "error": str(upload_exc)[:4000]}) write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="upload_failed", failure_type="smoke_repair_upload_failed", final_user_message="Deterministic writable cache repair was prepared but failed to upload; keeping partial validation.", validation_error=str(upload_exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_upload", "failed", "Deterministic writable cache repair upload failed; keeping partial validation", {"error": str(upload_exc)[:2000]}) return validation, latest_smoke try: write_auth_probe(run_dir, events_path, "before_writable_cache_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_class = repair_post_smoke_validation_status(pre_repair_smoke, generation_smoke) if 'pre_repair_smoke' in locals() else repair_post_smoke_validation_status(generation_smoke, generation_smoke) post_status = post_class["post_repair_validation"] write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type=post_class.get("failure_type") or "", final_user_message="Deterministic writable cache repair revalidated the Space." if post_status == "success" else post_class.get("message") or "Deterministic writable cache repair uploaded, but generation smoke still did not verify the primary demo flow.", post_repair_generation_smoke=generation_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Deterministic writable cache repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status"), "same_error": post_class.get("same_error")}) except Exception as exc: repair_class = classify_repair_validation_error(exc) latest_smoke = build_generation_smoke_failure_payload(target_space_id, expected_output_type, exc, workspace=workspace, phase="post_writable_cache_repair_exception") smoke_diagnosis = diagnose_generation_smoke_failure(exc, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="post_writable_cache_repair_exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, latest_smoke, phase="post_writable_cache_repair_exception") generation_smoke = latest_smoke write_repair_outcome(run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=latest_smoke.get("failure_type") or repair_class["failure_type"] or "writable_cache_repair_validation_failed", final_user_message=repair_class["message"], validation_error=str(exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_validation", "failed", "Deterministic writable cache repair validation failed; keeping partial validation", {"error": str(exc)[:2000], "latest_failure_class": latest_smoke.get("failure_class"), "latest_failure_owner": latest_smoke.get("failure_owner")}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, deterministic_patch=cache_patch, patch_applied=False, post_repair_validation="not_started", failure_type="writable_cache_patch_not_applied") append_event(events_path, "generation_smoke_repair", "warning", "Writable cache repair was indicated but no deterministic cache patch was applied; falling back to Pi repair", cache_patch) primary_error_packet = write_smoke_repair_primary_error_packet(run_dir, workspace, generation_smoke, failure_class=failure_class) recipe_repair_packet = write_recipe_aware_repair_packet(workspace, run_dir, str(generation_smoke.get("error") or generation_smoke.get("result_info") or generation_smoke), {"category": failure_class, "failure_phase": "generation_smoke"}, {"action": "patch_code", "classification": {"category": failure_class}}) failure_reason = ( "Automatic generation smoke failed after the Space became reachable.\n" f"Authoritative smoke error: {primary_error_packet.get('error') or ''}\n" f"Failure class: {failure_class}\n" f"Failure owner: {generation_smoke.get('failure_owner') or ''}\n" f"Actionability: {generation_smoke.get('actionability') or ''}\n" f"Recommended action: {generation_smoke.get('recommended_action') or ''}\n" f"Recipe-aware repair focus: {(primary_error_packet.get('known_failure_match') or {}).get('repair_focus') or ''}\n" f"Model family: {recipe_repair_packet.get('model_family') or ''}; loader: {recipe_repair_packet.get('loader_strategy') or ''}\n" f"Error/result: {generation_smoke.get('error') or generation_smoke.get('result_info') or generation_smoke}\n" f"Latest Space log brief: {latest_log_brief[:3000]}\n" ) decision = { "action": "patch_code", "confidence": "high" if failure_class in {"missing_runtime_cli", "missing_python_dependency"} else "medium", "source": "generation_smoke_repair_escalation.v198_26_20", "classification": { "category": failure_class, "failure_phase": "generation_smoke", "logs_quality": "useful", "recommendation": escalation.get("recommended_action") or generation_smoke.get("recommended_action") or "Patch the smallest concrete dependency or command issue, preserve the demo flow, redeploy, then re-run smoke validation.", "generation_smoke_failure": True, "smoke_primary_error_packet": "repair/SMOKE_PRIMARY_ERROR.json", "sdxl_lora_text_encoder_mismatch": failure_class == "sdxl_lora_text_encoder_mismatch", "missing_executable": generation_smoke.get("missing_executable") or "", "recipe_aware_repair_packet": "repair/RECIPE_AWARE_REPAIR_PACKET.json", "known_failure_match": primary_error_packet.get("known_failure_match") or {}, "model_family": recipe_repair_packet.get("model_family") or "", "loader_strategy": recipe_repair_packet.get("loader_strategy") or "", }, "family_repair_directives": recipe_repair_packet.get("family_repair_directives") or [], "constraints": [ "Preserve real inference and the existing Gradio API contract.", "Do not replace the model call with placeholders or diagnostics if the failure is repairable.", "Patch only the dependency, command invocation, model loading, or ZeroGPU compatibility issue required for the failed smoke path.", "For ZeroGPU, keep import spaces before CUDA-touching imports and keep the heavy inference function protected by @spaces.GPU while preserving fixed-GPU compatibility.", "After upload, the repair is only fully successful if /generate passes; /health alone is boot-level evidence.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Running one targeted repair because the Space is reachable but the primary demo smoke failed", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause=failure_class, repair_decision="patch_code", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger="generation_smoke_failure") if not repaired: terminal = failed_pi_repair_terminal_status(run_dir) write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation=terminal["post_repair_validation"], failure_type=terminal["failure_type"], final_user_message=terminal["final_user_message"], latest_repair_attempt=terminal.get("latest_repair_attempt"), pre_repair_generation_smoke=generation_smoke) append_event(events_path, "generation_smoke_repair", "failed", terminal["final_user_message"], {"failure_class": failure_class, "terminal_status": terminal["post_repair_validation"]}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading targeted smoke repair") try: write_auth_probe(run_dir, events_path, "before_smoke_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) except Exception as upload_exc: latest_smoke = dict(generation_smoke) latest_smoke.update({"status": "failed", "failure_type": "smoke_repair_upload_failed", "failure_class": "smoke_repair_upload_failed", "failure_owner": "factory_upload", "error": str(upload_exc)[:4000]}) write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="upload_failed", failure_type="smoke_repair_upload_failed", final_user_message="Targeted smoke repair was prepared but failed to upload; keeping partial validation.", validation_error=str(upload_exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_upload", "failed", "Targeted smoke repair upload failed; keeping partial validation", {"error": str(upload_exc)[:2000]}) return validation, latest_smoke write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Targeted smoke repair uploaded; revalidating health and demo smoke") try: write_auth_probe(run_dir, events_path, "before_smoke_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_class = repair_post_smoke_validation_status(pre_repair_smoke, generation_smoke) if 'pre_repair_smoke' in locals() else repair_post_smoke_validation_status(generation_smoke, generation_smoke) post_status = post_class["post_repair_validation"] write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type=post_class.get("failure_type") or "", final_user_message=post_class.get("message") or ("Targeted smoke repair revalidated the Space." if post_status == "success" else "Targeted smoke repair uploaded, but generation smoke still did not verify the primary demo flow."), post_repair_generation_smoke=generation_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Targeted smoke repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status"), "same_error": post_class.get("same_error")}) except Exception as exc: repair_class = classify_repair_validation_error(exc) latest_smoke = build_generation_smoke_failure_payload(target_space_id, expected_output_type, exc, workspace=workspace, phase="post_smoke_repair_exception") smoke_diagnosis = diagnose_generation_smoke_failure(exc, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="post_smoke_repair_exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, latest_smoke, phase="post_smoke_repair_exception") generation_smoke = latest_smoke write_repair_outcome(run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=latest_smoke.get("failure_type") or repair_class["failure_type"] or "smoke_repair_validation_failed", final_user_message=repair_class["message"], validation_error=str(exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_validation", "failed", "Targeted smoke repair validation failed; keeping partial validation", {"error": str(exc)[:2000], "latest_failure_class": latest_smoke.get("failure_class"), "latest_failure_owner": latest_smoke.get("failure_owner")}) return validation, generation_smoke def patch_workspace_for_diffusers_token_guard(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_guard", evidence: str = "") -> dict: """Remove duplicate Diffusers `token` injection patterns before upload/repair. The guard is intentionally deterministic and conservative: it only patches token handling around model/Diffusers `from_pretrained` calls when a mutable kwargs dict that is passed with `**...` also receives `token`/ `use_auth_token`, or when runtime evidence proves duplicate token injection. It covers generated variants such as `_load_kwargs["token"] = HF_TOKEN` plus `from_pretrained(..., **_load_kwargs, token=HF_TOKEN)`, which previously slipped through because the detector only looked for a variable literally named `kwargs`. """ result = { "schema_version": "diffusers_token_guard.v198_26_24", "reason": reason, "applied": False, "checked_file": "app.py", "detected_duplicate_token_risk": False, "evidence_matched": is_diffusers_duplicate_token_text(evidence or ""), "replacements": 0, "removed_lines": [], "notes": [], "kwargs_vars_with_token_assignment": [], "from_pretrained_star_kwargs_vars": [], "from_pretrained_explicit_token_args": False, "post_patch_duplicate_token_risk": False, } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "diffusers_token_guard.json", result) return result text = app_path.read_text(encoding="utf-8", errors="ignore") lower = text.lower() has_diffusers_pipeline = "diffusers" in lower or "pipeline" in lower or "from_pretrained" in lower def _from_pretrained_call_bodies(src: str) -> list[str]: bodies = [] marker = "from_pretrained" pos = 0 while True: idx = src.find(marker, pos) if idx < 0: break open_idx = src.find("(", idx + len(marker)) if open_idx < 0: break depth = 0 quote = None escape = False for j in range(open_idx, len(src)): ch = src[j] if quote: if escape: escape = False elif ch == "\\": escape = True elif ch == quote: quote = None continue if ch in {"'", '"'}: quote = ch continue if ch == "(": depth += 1 elif ch == ")": depth -= 1 if depth == 0: bodies.append(src[open_idx + 1:j]) pos = j + 1 break else: break return bodies bodies = _from_pretrained_call_bodies(text) star_vars = set() explicit_token_in_from_pretrained = False explicit_token_with_star = False for body in bodies: call_star_vars = set(re.findall(r"\*\*\s*([A-Za-z_]\w*)", body)) if call_star_vars: star_vars.update(call_star_vars) if re.search(r"(?:^|,)\s*(?:token|use_auth_token)\s*=", body): explicit_token_in_from_pretrained = True if call_star_vars: explicit_token_with_star = True result["from_pretrained_star_kwargs_vars"] = sorted(star_vars) result["from_pretrained_explicit_token_args"] = bool(explicit_token_in_from_pretrained) token_assignment_vars = set() assignment_patterns = [ r"([A-Za-z_]\w*)\s*\[\s*[\"'](?:token|use_auth_token)[\"']\s*\]\s*=", r"([A-Za-z_]\w*)\.update\s*\(\s*\{\s*[\"'](?:token|use_auth_token)[\"']\s*:", r"([A-Za-z_]\w*)\.update\s*\(\s*dict\s*\(\s*(?:token|use_auth_token)\s*=", ] for pattern in assignment_patterns: token_assignment_vars.update(re.findall(pattern, text)) result["kwargs_vars_with_token_assignment"] = sorted(token_assignment_vars) risky_star_vars = star_vars & token_assignment_vars result["detected_duplicate_token_risk"] = bool( has_diffusers_pipeline and ( result["evidence_matched"] or bool(risky_star_vars) or (explicit_token_with_star and bool(token_assignment_vars)) ) ) if not result["detected_duplicate_token_risk"] and not result["evidence_matched"]: write_json(run_dir / "diffusers_token_guard.json", result) return result original = text vars_to_clean = set(risky_star_vars) or set(token_assignment_vars) if result["evidence_matched"]: vars_to_clean |= set(token_assignment_vars) | set(star_vars) lines = text.splitlines(True) new_lines = [] i = 0 var_group = "|".join(re.escape(v) for v in sorted(vars_to_clean)) or r"[A-Za-z_]\w*" assignment_line_re = re.compile(rf"\b(?:{var_group})\s*\[\s*[\"'](?:token|use_auth_token)[\"']\s*\]\s*=") update_line_re = re.compile(rf"\b(?:{var_group})\.update\s*\(\s*(?:\{{\s*[\"'](?:token|use_auth_token)[\"']\s*:|dict\s*\(\s*(?:token|use_auth_token)\s*=)") one_line_if_re = re.compile(rf"^if\s+[^:]+:\s*(?:{var_group})\s*\[\s*[\"'](?:token|use_auth_token)[\"']\s*\]\s*=") while i < len(lines): line = lines[i] stripped = line.strip() if one_line_if_re.search(stripped): result["replacements"] += 1 result["removed_lines"].append(stripped[:200]) i += 1 continue if re.match(r"^\s*if\s+[^:]+:\s*$", line) and i + 1 < len(lines): next_line = lines[i + 1] if assignment_line_re.search(next_line) or update_line_re.search(next_line): result["replacements"] += 1 result["removed_lines"].append((line.strip() + " " + next_line.strip())[:200]) i += 2 continue if assignment_line_re.search(line) or update_line_re.search(line): result["replacements"] += 1 result["removed_lines"].append(stripped[:200]) i += 1 continue new_lines.append(line) i += 1 text = "".join(new_lines) # If a from_pretrained call combines **kwargs and an explicit token arg while # the kwargs dict had a token assignment, remove the explicit arg too. This # is safe for the risky generated pattern and avoids stale duplicate-token # failures after a partial line cleanup. if result["evidence_matched"] or explicit_token_with_star: token_arg_patterns = [ r",\s*(?:token|use_auth_token)\s*=\s*(?:HF_TOKEN|hf_token|token|hf_token_value|os\.environ\.get\([^\)]*\))\s*(?=,|\))", r"(?:token|use_auth_token)\s*=\s*(?:HF_TOKEN|hf_token|token|hf_token_value|os\.environ\.get\([^\)]*\))\s*,\s*", ] for pattern in token_arg_patterns: text, count = re.subn(pattern, "", text) result["replacements"] += count # Normalize accidental doubled commas introduced by argument removal. text, comma_count = re.subn(r",\s*,", ",", text) result["replacements"] += comma_count # Re-scan after patch so upload guards can fail closed if the deterministic # patch did not actually remove the duplicate-token risk. post_bodies = _from_pretrained_call_bodies(text) post_star_vars = set() post_explicit_token_with_star = False for body in post_bodies: call_star_vars = set(re.findall(r"\*\*\s*([A-Za-z_]\w*)", body)) post_star_vars.update(call_star_vars) if call_star_vars and re.search(r"(?:^|,)\s*(?:token|use_auth_token)\s*=", body): post_explicit_token_with_star = True post_assignment_vars = set() for pattern in assignment_patterns: post_assignment_vars.update(re.findall(pattern, text)) result["post_patch_duplicate_token_risk"] = bool((post_star_vars & post_assignment_vars) or (post_explicit_token_with_star and post_assignment_vars)) if text != original: app_path.write_text(text, encoding="utf-8") result["applied"] = True append_event(events_path, "diffusers_token_guard", "success", "Patched generated Diffusers token handling to avoid duplicate token injection", result) else: result["notes"].append("no_token_patch_needed") write_json(run_dir / "diffusers_token_guard.json", result) return result def normalize_runtime_expected_output_type(value: str) -> str: text = str(value or "").strip().lower() if not text: return "any" if "text_diagnostic" in text or "diagnostic" in text: return "text_diagnostic" if "image" in text or text in {"png", "jpg", "jpeg", "webp"}: return "image" if "video" in text or text in {"mp4", "webm", "mov"}: return "video" if "audio" in text or text in {"wav", "mp3", "flac"}: return "audio" if "text" in text: return "text" return text def archive_rejected_workspace(workspace: Path, run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None) -> dict: """Persist the publishable workspace that failed pre-upload integrity. v198.26.29 keeps blocked workspaces auditable and makes pre-upload failures repairable: the exact rejected payload is preserved before the worker fails closed or requests a targeted repair pass. """ rejected_dir = run_dir / "generated_rejected" if rejected_dir.exists(): shutil.rmtree(rejected_dir) rejected_dir.parent.mkdir(parents=True, exist_ok=True) payload = { "schema_version": "pre_upload_integrity_failure.v198_26_28", "reason": reason, "archived": False, "rejected_workspace": "generated_rejected", "guard_result": guard_result or {}, "created_at": now(), } try: if workspace.exists(): shutil.copytree(workspace, rejected_dir, ignore=internal_workspace_copy_ignore) payload["archived"] = True payload["files_sample"] = sorted(str(p.relative_to(rejected_dir)) for p in rejected_dir.rglob("*") if p.is_file())[:80] else: payload["error"] = "workspace_missing" except Exception as exc: payload["error"] = str(exc)[:1000] write_json(run_dir / "pre_upload_integrity_failure.json", payload) append_event(events_path, "pre_upload_integrity_failure", "failed", "Archived rejected publishable workspace before blocking upload", payload) return payload def write_pre_upload_repair_attempt(run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None, deterministic_repairs: list[dict] | None = None, pi_repair_required: bool = False) -> dict: """Record the repair trajectory for a pre-upload template failure. This artifact closes the visibility gap from v198.26.29: a pre-upload guard must not fail opaquely. It must say which deterministic repairs were tried and whether a targeted Pi repair would be required before a final failure. """ payload = { "schema_version": "pre_upload_repair_attempt.v198_26_28", "reason": reason, "guard_result": guard_result or {}, "deterministic_repairs": deterministic_repairs or [], "pi_repair_required": bool(pi_repair_required), "status": "pi_repair_required" if pi_repair_required else "deterministic_repairs_exhausted", "created_at": now(), } write_json(run_dir / "pre_upload_repair_attempt.json", payload) append_event(events_path, "pre_upload_repair_attempt", "warning", "Pre-upload template repair path recorded before blocking upload", payload) return payload def _ensure_spaces_import_early(app_text: str) -> tuple[str, bool]: if re.search(r"^\s*import\s+spaces\b", app_text, flags=re.M): return app_text, False lines = app_text.splitlines() insert_at = 0 # Keep shebang, encoding comments and __future__ imports first. Place # `import spaces` before torch/diffusers so ZeroGPU sees it before CUDA-ish # imports or module-level model setup. while insert_at < len(lines): stripped = lines[insert_at].strip() if not stripped or stripped.startswith("#!") or stripped.startswith("#") or stripped.startswith("from __future__ import"): insert_at += 1 continue break lines.insert(insert_at, "import spaces") return "\n".join(lines) + ("\n" if app_text.endswith("\n") else ""), True def _candidate_generate_function_names(app_text: str) -> list[str]: names: list[str] = [] def add(name: str): if name and name not in names and re.search(rf"^\s*(?:async\s+)?def\s+{re.escape(name)}\s*\(", app_text, flags=re.M): names.append(name) # Prefer Gradio event/API bindings that explicitly expose /generate. for match in re.finditer(r"fn\s*=\s*([A-Za-z_]\w*)[\s\S]{0,500}?api_name\s*=\s*[\"']/?generate[\"']", app_text): add(match.group(1)) for match in re.finditer(r"api_name\s*=\s*[\"']/?generate[\"'][\s\S]{0,500}?fn\s*=\s*([A-Za-z_]\w*)", app_text): add(match.group(1)) # Common Interface/ChatInterface/Blocks patterns. for match in re.finditer(r"(?:Interface|ChatInterface)\s*\([\s\S]{0,500}?fn\s*=\s*([A-Za-z_]\w*)", app_text): add(match.group(1)) for fallback in ("generate", "generate_image", "predict", "infer", "run_inference"): add(fallback) return names def _insert_decorator_before_function(app_text: str, func_name: str, decorator: str) -> tuple[str, bool]: lines = app_text.splitlines(True) def_re = re.compile(rf"^(?P\s*)(?:async\s+)?def\s+{re.escape(func_name)}\s*\(") for idx, line in enumerate(lines): match = def_re.match(line) if not match: continue # If the function already has a spaces.GPU decorator in its contiguous # decorator block, do nothing. start = idx while start > 0 and lines[start - 1].lstrip().startswith("@"): start -= 1 decorator_block = "".join(lines[start:idx]) if "spaces.GPU" in decorator_block or "spaces.gpu" in decorator_block.lower(): return app_text, False indent = match.group("indent") or "" lines.insert(start, f"{indent}{decorator}\n") return "".join(lines), True return app_text, False def _patch_common_python_syntax_defects(app_text: str) -> tuple[str, list[str]]: """Patch narrow syntax defects Pi has produced in real runs. This is intentionally conservative. v198.26.29 covers the observed `torch_dtype=DTYPE,,` class and repeated commas in function calls without attempting a broad Python formatter. """ patched = app_text changes: list[str] = [] if ",," in patched: while ",," in patched: patched = patched.replace(",,", ",") changes.append("collapsed_duplicate_commas") patched2 = re.sub(r",\s*(\))", r"\1", patched) if patched2 != patched: patched = patched2 changes.append("removed_trailing_comma_before_close_paren") # Pi has repeatedly emitted an unterminated guard such as: # try: # import spaces # def generate(...): # which fails with "expected 'except' or 'finally' block". Repair only this # narrow import-spaces guard by inserting a safe except before the next # top-level statement; do not attempt a broad Python formatter. lines = patched.splitlines(True) repaired_lines = None for idx, line in enumerate(lines): m = re.match(r"^(?P\s*)try:\s*(?:#.*)?$", line) if not m: continue base = m.group("indent") j = idx + 1 while j < len(lines) and not lines[j].strip(): j += 1 if j >= len(lines) or not re.match(rf"^{re.escape(base)}\s+import\s+spaces\b", lines[j]): continue has_except = False boundary = None k = j + 1 while k < len(lines): stripped = lines[k].strip() if not stripped: k += 1 continue if re.match(rf"^{re.escape(base)}except\b", lines[k]) or re.match(rf"^{re.escape(base)}finally\b", lines[k]): has_except = True break if re.match(rf"^{re.escape(base)}\S", lines[k]): boundary = k break k += 1 if has_except or boundary is None: continue indent = re.match(r"^(\s*)", lines[j]).group(1) insert = [f"{base}except Exception:\n", f"{indent}spaces = None\n"] repaired_lines = lines[:boundary] + insert + lines[boundary:] break if repaired_lines is not None: patched2 = "".join(repaired_lines) if patched2 != patched: patched = patched2 changes.append("closed_unterminated_import_spaces_try_block") return patched, changes def compile_workspace_app(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_compile_guard") -> dict: """Compile generated/app.py before any upload and patch tiny syntax defects. A Space should never be created/uploaded with syntactically invalid Python. The guard patches narrow, deterministic defects once, then records whether Pi repair is still required. """ app_path = workspace / "app.py" guard_path = run_dir / ("pre_upload_post_patch_compile_guard.json" if "post_patch" in str(reason) else "pre_upload_compile_guard.json") result = { "schema_version": "pre_upload_compile_guard.v198_26_28", "reason": reason, "checked_file": "app.py", "passed": False, "initial_passed": False, "patch_applied": False, "changes": [], "error": "", "post_patch_error": "", } if not app_path.exists(): result["error"] = "app_py_missing" write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "failed", "Generated app.py is missing before upload", result) return result app_text = app_path.read_text(encoding="utf-8", errors="ignore") try: compile(app_text, str(app_path), "exec") result["passed"] = True result["initial_passed"] = True write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "success", "Generated app.py compiled before upload", result) return result except SyntaxError as exc: result["error"] = f"{exc.__class__.__name__}: {exc}" patched, changes = _patch_common_python_syntax_defects(app_text) result["changes"] = changes if changes and patched != app_text: app_path.write_text(patched, encoding="utf-8") result["patch_applied"] = True try: compile(patched, str(app_path), "exec") result["passed"] = True write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "success", "Patched a narrow generated app.py syntax defect before upload", result) return result except SyntaxError as exc: result["post_patch_error"] = f"{exc.__class__.__name__}: {exc}" write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "failed", "Generated app.py did not compile before upload", result) return result def _has_generate_api_endpoint(app_text: str) -> bool: return bool(re.search(r"api_name\s*=\s*[\"']/?generate[\"']", app_text)) def _find_matching_paren(text: str, open_idx: int) -> int: depth = 0 quote = "" escape = False for idx in range(open_idx, len(text)): ch = text[idx] if quote: if escape: escape = False elif ch == "\\": escape = True elif ch == quote: quote = "" continue if ch in {"'", '"'}: quote = ch continue if ch == "(": depth += 1 elif ch == ")": depth -= 1 if depth == 0: return idx return -1 def _patch_api_name_into_call(app_text: str, fn_name: str) -> tuple[str, bool]: """Add api_name='generate' to the Gradio call that binds fn_name. Covers multiline `.click(...)`, `.submit(...)`, `.then(...)`, `.load(...)`, `gr.Interface(...)`, and `gr.ChatInterface(...)` calls where Pi forgot the API name even though the contract declares `/generate`. """ patterns = [ rf"fn\s*=\s*{re.escape(fn_name)}\b", rf"(?:gr\.)?(?:Interface|ChatInterface)\s*\(\s*{re.escape(fn_name)}\b", ] for pattern in patterns: for match in re.finditer(pattern, app_text): open_idx = app_text.rfind("(", 0, match.start()) if open_idx < 0: continue close_idx = _find_matching_paren(app_text, open_idx) if close_idx < 0: continue call_text = app_text[open_idx:close_idx] if "api_name" in call_text: # Replace api_name=None/False only if present. replaced = re.sub(r"api_name\s*=\s*(?:None|False|[\"']/?health[\"'])", "api_name=\"generate\"", call_text) if replaced != call_text: return app_text[:open_idx] + replaced + app_text[close_idx:], True continue insertion = ",\n api_name=\"generate\"" return app_text[:close_idx] + insertion + app_text[close_idx:], True return app_text, False def patch_workspace_for_missing_generate_endpoint(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_integrity_guard") -> dict: """Repair media contracts that forgot to expose Gradio `/generate`. If Pi produced a real `generate` function and bound it to a Gradio event but forgot `api_name='generate'`, this deterministic patch makes the Space validateable without waiting for a live runtime failure. """ result = { "schema_version": "generate_endpoint_repair.v198_26_27", "reason": reason, "applied": False, "checked_file": "app.py", "function_candidates": [], "patched_function": "", "notes": [], } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "generate_endpoint_repair.json", result) return result app_text = app_path.read_text(encoding="utf-8", errors="ignore") if _has_generate_api_endpoint(app_text): result["notes"].append("generate_endpoint_already_present") write_json(run_dir / "generate_endpoint_repair.json", result) return result candidates = _candidate_generate_function_names(app_text) result["function_candidates"] = candidates for name in candidates: patched, changed = _patch_api_name_into_call(app_text, name) if changed: app_path.write_text(patched, encoding="utf-8") result["applied"] = True result["patched_function"] = name append_event(events_path, "generate_endpoint_repair", "success", "Patched Gradio binding to expose /generate before upload", result) write_json(run_dir / "generate_endpoint_repair.json", result) return result result["notes"].append("no_patchable_gradio_binding_found") write_json(run_dir / "generate_endpoint_repair.json", result) return result def patch_workspace_for_missing_spaces_gpu_decorator(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_integrity_guard") -> dict: """Repair a generated ZeroGPU app that has /generate but missed @spaces.GPU. This is intentionally narrow and auditable: it adds `import spaces` early and decorates the function bound to Gradio `api_name="generate"`. If the target function cannot be identified, the guard remains fatal but archives the rejected workspace instead of publishing a non-actionable ZeroGPU app. """ result = { "schema_version": "spaces_gpu_decorator_repair.v198_26_27", "reason": reason, "applied": False, "checked_file": "app.py", "function_candidates": [], "decorated_function": "", "import_added": False, "notes": [], } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result app_text = app_path.read_text(encoding="utf-8", errors="ignore") if "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower(): result["notes"].append("spaces_gpu_decorator_already_present") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result candidates = _candidate_generate_function_names(app_text) result["function_candidates"] = candidates if not candidates: result["notes"].append("no_generate_function_candidate_found") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result patched, import_added = _ensure_spaces_import_early(app_text) for name in candidates: patched2, changed = _insert_decorator_before_function(patched, name, "@spaces.GPU(duration=120)") if changed: patched = patched2 result["decorated_function"] = name break if not result["decorated_function"]: result["notes"].append("candidate_function_not_decorated") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result app_path.write_text(patched, encoding="utf-8") result["applied"] = True result["import_added"] = import_added append_event(events_path, "spaces_gpu_decorator_repair", "success", "Patched ZeroGPU publishable app with missing @spaces.GPU decorator before upload", result) write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path: Path, *, hardware_intent: dict | None = None, expected_output_type: str = "", reason: str = "pre_upload", raise_on_failure: bool = True) -> dict: """Pre-upload integrity guard for publishable runtime templates. v198.26.29 extends the v198.26.24 repair-first policy with compile checks and endpoint repair. The worker should not fail-fast merely because Pi forgot `api_name='generate'` or emitted a narrow syntax typo; it should repair, re-check, then fail closed only when the app is still invalid. """ contract = read_inference_contract(workspace) app_path = workspace / "app.py" app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" hw_text = " ".join(str((hardware_intent or {}).get(k) or "") for k in ("preferred_hardware", "fallback_hardware", "recommended_hardware")) contract_hw = str(contract.get("recommended_target_space_hardware") or contract.get("recommended_hardware") or "") zero_target = "zero" in (hw_text + " " + contract_hw).lower() expected = normalize_runtime_expected_output_type(expected_output_type or contract.get("expected_output_type") or "") primary = normalize_api_name(str(contract.get("primary_api_name") or contract.get("primary_endpoint") or "/generate")) full_inference = contract.get("full_inference_implemented") is True diagnostic_output = normalize_runtime_expected_output_type(str(contract.get("expected_output_type") or "")) == "text_diagnostic" deterministic_repairs: list[dict] = [] compile_guard = compile_workspace_app(workspace, run_dir, events_path, reason=f"{reason}_compile_guard") deterministic_repairs.append({"type": "compile_guard", "result": compile_guard}) app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_generate = _has_generate_api_endpoint(app_text) has_spaces_gpu = "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower() generate_endpoint_patch = {"applied": False, "notes": ["not_run_until_app_py_compiles"]} spaces_gpu_patch = {"applied": False, "notes": ["not_run_until_app_py_compiles"]} cache_patch = {"applied": False, "notes": ["not_run_until_app_py_compiles"]} token_patch = {"applied": False, "post_patch_duplicate_token_risk": False, "detected_duplicate_token_risk": False, "notes": ["not_run_until_app_py_compiles"]} post_patch_compile_guard = compile_guard if compile_guard.get("passed"): cache_patch = patch_workspace_for_writable_hf_cache(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") token_patch = patch_workspace_for_diffusers_token_guard(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") deterministic_repairs.extend([ {"type": "writable_cache_patch", "result": cache_patch}, {"type": "diffusers_token_patch", "result": token_patch}, ]) # Re-read after deterministic cache/token patches. app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_generate = _has_generate_api_endpoint(app_text) has_spaces_gpu = "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower() generate_endpoint_patch = {"applied": False, "notes": ["not_needed"]} spaces_gpu_patch = {"applied": False, "notes": ["not_needed"]} if full_inference and expected in {"image", "video", "audio"} and primary == "/generate" and not has_generate: generate_endpoint_patch = patch_workspace_for_missing_generate_endpoint(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") deterministic_repairs.append({"type": "generate_endpoint_patch", "result": generate_endpoint_patch}) app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_generate = _has_generate_api_endpoint(app_text) # Syntax-first: never decorate or semantically patch an uncompilable app.py. # Once app.py compiles, a missing @spaces.GPU decorator on an otherwise # full inference ZeroGPU app is repairable before upload. if full_inference and zero_target and has_generate and not has_spaces_gpu: spaces_gpu_patch = patch_workspace_for_missing_spaces_gpu_decorator(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") deterministic_repairs.append({"type": "spaces_gpu_decorator_patch", "result": spaces_gpu_patch}) app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_spaces_gpu = "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower() has_generate = _has_generate_api_endpoint(app_text) # Compile once more after endpoint/decorator patches. post_patch_compile_guard = compile_workspace_app(workspace, run_dir, events_path, reason=f"{reason}_post_patch_compile_guard") deterministic_repairs.append({"type": "post_patch_compile_guard", "result": post_patch_compile_guard}) else: deterministic_repairs.extend([ {"type": "writable_cache_patch", "result": cache_patch}, {"type": "diffusers_token_patch", "result": token_patch}, {"type": "generate_endpoint_patch", "result": generate_endpoint_patch}, {"type": "spaces_gpu_decorator_patch", "result": spaces_gpu_patch}, {"type": "post_patch_compile_guard", "result": post_patch_compile_guard}, ]) recipe_alignment = recipe_runtime_alignment_guard(workspace, run_dir, events_path, reason=reason) app_contract_guard = app_runtime_contract_guard(workspace, run_dir, events_path, expected_output_type=expected_output_type, reason=reason) fatal = [] repairable_unresolved = [] if not compile_guard.get("passed") and not post_patch_compile_guard.get("passed"): repairable_unresolved.append("publishable_app_py_syntax_error") if token_patch.get("post_patch_duplicate_token_risk"): fatal.append("diffusers_duplicate_token_risk_after_guard") if token_patch.get("detected_duplicate_token_risk") and not token_patch.get("applied"): fatal.append("diffusers_duplicate_token_risk_unpatched") if full_inference and zero_target and not has_spaces_gpu: repairable_unresolved.append("zero_gpu_full_inference_without_spaces_gpu_decorator") if full_inference and expected in {"image", "video", "audio"} and not has_generate: repairable_unresolved.append("full_inference_media_contract_missing_generate_endpoint") for problem in app_contract_guard.get("repairable_unresolved_problems") or []: if problem not in repairable_unresolved: repairable_unresolved.append(problem) if expected in {"image", "video", "audio"} and diagnostic_output: fatal.append("media_expected_output_replaced_by_text_diagnostic") if full_inference and not app_path.exists(): fatal.append("publishable_app_py_missing") fatal.extend(repairable_unresolved) result = { "schema_version": "runtime_template_integrity_guard.v198_26_28", "reason": reason, "passed": not fatal, "fatal_problems": fatal, "repairable_unresolved_problems": repairable_unresolved, "problems": list(fatal), "zero_target": zero_target, "expected_output_type": expected, "primary_api_name": primary, "full_inference_implemented": full_inference, "has_generate_endpoint": has_generate, "has_spaces_gpu": has_spaces_gpu, "compile_guard": compile_guard, "post_patch_compile_guard": post_patch_compile_guard, "writable_cache_patch": cache_patch, "diffusers_token_patch": token_patch, "generate_endpoint_patch": generate_endpoint_patch, "spaces_gpu_decorator_patch": spaces_gpu_patch, "recipe_runtime_alignment": recipe_alignment, "app_runtime_contract_guard": app_contract_guard, } write_json(run_dir / "runtime_template_integrity_guard.json", result) append_event(events_path, "runtime_template_integrity_guard", "success" if result["passed"] else "failed", "Checked publishable runtime template integrity before upload", result) if fatal: write_pre_upload_repair_attempt(run_dir, events_path, reason="runtime_template_integrity_failed_before_upload", guard_result=result, deterministic_repairs=deterministic_repairs, pi_repair_required=bool(repairable_unresolved)) archive_rejected_workspace(workspace, run_dir, events_path, reason="runtime_template_integrity_failed_before_upload", guard_result=result) if raise_on_failure: raise RuntimeError("Runtime template integrity failed before upload after deterministic repair: " + ", ".join(fatal)) return result def _update_pre_upload_repair_attempt_status(run_dir: Path, **updates) -> dict: path = run_dir / "pre_upload_repair_attempt.json" payload = load_json_if_exists(path) if path.exists() else {} if not isinstance(payload, dict): payload = {} payload.update(updates) payload["updated_at"] = now() write_json(path, payload) return payload def _pre_upload_repair_failure_reason(guard_result: dict | None, workspace: Path) -> str: guard = guard_result if isinstance(guard_result, dict) else {} app_excerpt = _read_workspace_text_if_exists(workspace / "app.py")[:6000] return ( "Pre-upload runtime template integrity failed before Space upload.\n" f"Fatal problems: {', '.join(guard.get('fatal_problems') or guard.get('problems') or [])}.\n" f"Repairable unresolved problems: {', '.join(guard.get('repairable_unresolved_problems') or [])}.\n" f"Expected output type: {guard.get('expected_output_type') or ''}. Primary API: {guard.get('primary_api_name') or ''}. ZeroGPU target: {guard.get('zero_target')}.\n" "Patch the publishable app.py minimally so it compiles, exposes the declared /generate endpoint, preserves the media output contract, returns structured /health fields required by APP_RUNTIME_CONTRACT.json, and keeps @spaces.GPU when ZeroGPU is targeted.\n\n" "Current app.py excerpt:\n" + app_excerpt ) def ensure_runtime_template_uploadable_with_pi_repair( workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, implementation_mode: str, expected_output_type: str, hardware_intent: dict | None = None, ) -> dict: """Run pre-upload guard, then execute one Pi repair pass if unresolved defects remain. Guards must protect ASF from publishing broken apps, but they should not be terminal guillotines for repairable template defects. v198.26.29 guarantees that a `pi_repair_required` pre-upload state is followed by either a repaired workspace that passes the guard, or an explicit terminal repair outcome. """ guard = runtime_template_integrity_guard( workspace, run_dir, events_path, hardware_intent=hardware_intent, expected_output_type=expected_output_type, reason="pre_upload_repair_first", raise_on_failure=False, ) if guard.get("passed"): return guard unresolved = list(guard.get("repairable_unresolved_problems") or []) fatal = list(guard.get("fatal_problems") or guard.get("problems") or []) if not unresolved: _update_pre_upload_repair_attempt_status(run_dir, status="fatal_guard_failure_no_pi_repair", pi_repair_required=False, terminal=True) raise RuntimeError("Runtime template integrity failed before upload: " + ", ".join(fatal)) primary_problem = unresolved[0] decision = { "action": "patch_code", "confidence": "high", "source": "pre_upload_integrity_guard.v198_26_28", "classification": { "category": primary_problem, "failure_class": primary_problem, "failure_phase": "pre_upload_runtime_template_integrity", "logs_quality": "workspace_static_analysis", "recommendation": "Patch app.py so the generated Space is publishable: compile cleanly, expose the declared /generate endpoint, return structured /health fields, preserve the media output type, and include @spaces.GPU for ZeroGPU.", }, "constraints": [ "Patch only publishable runtime files needed to pass pre-upload integrity.", "Preserve the real model inference path and expected output type.", "Do not replace the app with diagnostics or placeholders.", "Keep /health and /generate API contracts coherent with APP_RUNTIME_CONTRACT.json and INFERENCE_CONTRACT.json.", ], } write_json(run_dir / "pre_upload_pi_repair_decision.json", decision) append_event(events_path, "pre_upload_pi_repair", "started", "Running Pi repair for unresolved pre-upload template integrity defects", {"problem": primary_problem, "fatal_problems": fatal}) failure_reason = _pre_upload_repair_failure_reason(guard, workspace) repair_attempt, _ = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi( workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger="pre_upload_integrity_failure", ) if not repaired: _update_pre_upload_repair_attempt_status(run_dir, status="pi_repair_failed_or_no_patch", pi_repair_required=True, terminal=True, repair_attempt=repair_attempt) archive_rejected_workspace(workspace, run_dir, events_path, reason="pre_upload_pi_repair_failed", guard_result=guard) append_event(events_path, "pre_upload_pi_repair", "failed", "Pi repair did not produce a publishable pre-upload patch", {"problem": primary_problem, "attempt": repair_attempt}) raise RuntimeError("Runtime template integrity failed before upload; Pi repair did not produce a publishable patch: " + ", ".join(fatal)) repaired_dir = run_dir / "generated_repaired" if repaired_dir.exists(): shutil.rmtree(repaired_dir) shutil.copytree(workspace, repaired_dir, ignore=internal_workspace_copy_ignore) guard_after = runtime_template_integrity_guard( workspace, run_dir, events_path, hardware_intent=hardware_intent, expected_output_type=expected_output_type, reason="pre_upload_after_pi_repair", raise_on_failure=False, ) if guard_after.get("passed"): _update_pre_upload_repair_attempt_status(run_dir, status="pi_repair_succeeded", pi_repair_required=False, terminal=True, repair_attempt=repair_attempt, post_repair_guard=guard_after) append_event(events_path, "pre_upload_pi_repair", "success", "Pi repaired the runtime template before upload", {"attempt": repair_attempt, "guard": guard_after}) return guard_after _update_pre_upload_repair_attempt_status(run_dir, status="pi_repair_did_not_clear_guard", pi_repair_required=True, terminal=True, repair_attempt=repair_attempt, post_repair_guard=guard_after) archive_rejected_workspace(workspace, run_dir, events_path, reason="pre_upload_pi_repair_did_not_clear_guard", guard_result=guard_after) append_event(events_path, "pre_upload_pi_repair", "failed", "Pi repair completed but pre-upload integrity still failed", {"attempt": repair_attempt, "guard": guard_after}) raise RuntimeError("Runtime template integrity failed before upload after Pi repair: " + ", ".join(guard_after.get("fatal_problems") or guard_after.get("problems") or fatal)) def patch_workspace_for_writable_hf_cache(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_guard", evidence: str = "") -> dict: """Patch generated apps that hard-code Hugging Face caches under `/data`. ZeroGPU/fixed-GPU Spaces do not always have a writable persistent `/data` volume. A generated app that sets HF_HOME/HF_HUB_CACHE/TRANSFORMERS_CACHE or DIFFUSERS_CACHE to `/data...` can boot the UI but fail model loading with `[Errno 13] Permission denied: '/data'`. This deterministic factory patch is intentionally narrow: it only rewrites HF cache environment paths to `/tmp/asf-hf-cache` and adds a small cache bootstrap. It does not change the model call, Gradio API, hardware target, or output contract. """ result = { "schema_version": "writable_hf_cache_guard.v198_26_20", "reason": reason, "applied": False, "checked_file": "app.py", "replacements": 0, "detected_non_writable_data_cache": False, "cache_root": "/tmp/asf-hf-cache", "notes": [], } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "writable_hf_cache_guard.json", result) return result text = app_path.read_text(encoding="utf-8", errors="ignore") combined = f"{text}\n{evidence or ''}" if "/data" not in combined and not is_hf_cache_permission_denied_text(combined): return result original = text result["detected_non_writable_data_cache"] = True env_paths = { "HF_HOME": "/tmp/asf-hf-cache", "HF_HUB_CACHE": "/tmp/asf-hf-cache/hub", "HUGGINGFACE_HUB_CACHE": "/tmp/asf-hf-cache/hub", "TRANSFORMERS_CACHE": "/tmp/asf-hf-cache/transformers", "DIFFUSERS_CACHE": "/tmp/asf-hf-cache/diffusers", } def _replace_with_count(pattern: str, repl_func): nonlocal text text, count = re.subn(pattern, repl_func, text, flags=re.IGNORECASE) result["replacements"] += count for env_name, safe_path in env_paths.items(): _replace_with_count( rf"(os\.environ\.setdefault\(\s*[\"']{env_name}[\"']\s*,\s*)[\"']/data[^\"']*[\"'](\s*\))", lambda m, safe_path=safe_path: f'{m.group(1)}"{safe_path}"{m.group(2)}', ) _replace_with_count( rf"(os\.environ\[\s*[\"']{env_name}[\"']\s*\]\s*=\s*)[\"']/data[^\"']*[\"']", lambda m, safe_path=safe_path: f'{m.group(1)}"{safe_path}"', ) _replace_with_count( rf"({env_name}\s*=\s*)[\"']/data[^\"']*[\"']", lambda m, safe_path=safe_path: f'{m.group(1)}"{safe_path}"', ) literal_replacements = [ ("/data/.cache/huggingface/hub", "/tmp/asf-hf-cache/hub"), ("/data/.cache/huggingface/transformers", "/tmp/asf-hf-cache/transformers"), ("/data/.cache/huggingface/diffusers", "/tmp/asf-hf-cache/diffusers"), ("/data/.cache/huggingface", "/tmp/asf-hf-cache"), ("/data/huggingface/hub", "/tmp/asf-hf-cache/hub"), ("/data/huggingface", "/tmp/asf-hf-cache"), ] for src, dst in literal_replacements: if src in text: count = text.count(src) text = text.replace(src, dst) result["replacements"] += count snippet = """ # ASF v198.26.20 writable Hugging Face cache guard. _ASF_HF_CACHE_ROOT = os.environ.get("ASF_HF_CACHE_DIR") or "/tmp/asf-hf-cache" os.makedirs(_ASF_HF_CACHE_ROOT, exist_ok=True) os.makedirs(os.path.join(_ASF_HF_CACHE_ROOT, "hub"), exist_ok=True) os.makedirs(os.path.join(_ASF_HF_CACHE_ROOT, "transformers"), exist_ok=True) os.makedirs(os.path.join(_ASF_HF_CACHE_ROOT, "diffusers"), exist_ok=True) os.environ.setdefault("HF_HOME", _ASF_HF_CACHE_ROOT) os.environ.setdefault("HF_HUB_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "hub")) os.environ.setdefault("HUGGINGFACE_HUB_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "hub")) os.environ.setdefault("TRANSFORMERS_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "transformers")) os.environ.setdefault("DIFFUSERS_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "diffusers")) """ if "writable Hugging Face cache guard" not in text: if not re.search(r"(?m)^\s*import\s+os\b", text): text = "import os\n" + text result["replacements"] += 1 lines = text.splitlines(True) insert_at = 0 for i, line in enumerate(lines): if re.match(r"\s*(import\s+\w+|from\s+\w+\s+import\s+)", line): insert_at = i + 1 lines.insert(insert_at, snippet) text = "".join(lines) result["replacements"] += 1 if text != original: app_path.write_text(text, encoding="utf-8") result["applied"] = True result["notes"].append("patched_hf_cache_paths_to_tmp") append_event(events_path, "writable_hf_cache_guard", "success", "Patched generated app to use writable /tmp Hugging Face caches instead of /data", result) else: result["notes"].append("no_publishable_cache_patch_needed") write_json(run_dir / "writable_hf_cache_guard.json", result) return result def space_subdomain_url(target_space_id: str) -> str: owner, name = target_space_id.split("/", 1) # This matches the common Spaces app URL pattern. Keep conservative: our # generated slugs are ASCII and hyphen-friendly. return f"https://{owner}-{name}.hf.space".replace("_", "-").lower() def runtime_to_dict(runtime) -> dict: payload = {} for attr in ["stage", "hardware", "requested_hardware", "sleep_time", "storage", "gc_timeout"]: value = getattr(runtime, attr, None) payload[attr] = getattr(value, "value", value) return {k: str(v) if v is not None else None for k, v in payload.items()} def write_space_runtime(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int | None = None) -> dict: try: runtime = api.get_space_runtime(repo_id=target_space_id, token=token) payload = runtime_to_dict(runtime) payload["attempt"] = attempt write_json(run_dir / "space_runtime.json", payload) return payload except Exception as exc: payload = {"error": str(exc)[:2000], "attempt": attempt} write_json(run_dir / "space_runtime.json", payload) append_event(events_path, "space_runtime", "warning", "Could not fetch Space runtime", payload) return payload RESTART_GUARDRAIL_STATE = { "restart": {"count": 0, "last_ts": 0.0}, "factory_reboot": {"count": 0, "last_ts": 0.0}, "same_code_upload": {"count": 0, "last_ts": 0.0}, } RESTART_GUARDRAIL_LIMITS = { "restart": 1, "factory_reboot": 1, "same_code_upload": 1, } RESTART_GUARDRAIL_COOLDOWN_SECONDS = int(os.environ.get("SPACE_FACTORY_RESTART_COOLDOWN_SECONDS", "240")) def runtime_stage_value(runtime_payload: dict | None) -> str: return str((runtime_payload or {}).get("stage") or (runtime_payload or {}).get("runtime_stage") or "").lower() def restart_guardrail_allows(action: str, events_path: Path, *, reason: str = "") -> bool: state = RESTART_GUARDRAIL_STATE.setdefault(action, {"count": 0, "last_ts": 0.0}) limit = RESTART_GUARDRAIL_LIMITS.get(action, 1) now = time.time() if state["count"] >= limit: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart budget exhausted", {"action": action, "limit": limit, "reason": reason}) return False if state["last_ts"] and now - state["last_ts"] < RESTART_GUARDRAIL_COOLDOWN_SECONDS: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart cooldown active", {"action": action, "cooldown_seconds": RESTART_GUARDRAIL_COOLDOWN_SECONDS, "elapsed_seconds": round(now - state["last_ts"], 1), "reason": reason}) return False state["count"] += 1 state["last_ts"] = now return True def safe_restart_space(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, factory_reboot: bool = False, reason: str = "", require_logs_checked: bool = True, allow_busy: bool = False) -> bool: """Restart a Space with strict guardrails. Hugging Face exposes HfApi.restart_space(repo_id=..., factory_reboot=True) for a factory rebuild and restart_space(..., factory_reboot=False) for a normal restart. The Factory only calls it through this wrapper so we do not create restart loops or mask actionable build errors. """ action = "factory_reboot" if factory_reboot else "restart" if require_logs_checked: logs_dir = run_dir / "logs" if not ((logs_dir / "space_logs_build.txt").exists() or (logs_dir / "space_logs_runtime.txt").exists()): append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: logs have not been collected yet", {"reason": reason}) return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"} and not allow_busy: append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: Space is already busy", {"runtime": runtime_before, "reason": reason}) return False if not restart_guardrail_allows(action, events_path, reason=reason): return False try: append_event(events_path, "factory_rebuild", "started", f"Requesting {'factory reboot' if factory_reboot else 'Space restart'} via Hugging Face API", {"runtime_before": runtime_before, "reason": reason}) result = api.restart_space(repo_id=target_space_id, token=token, factory_reboot=factory_reboot) payload = {"factory_reboot": factory_reboot, "reason": reason, "result": str(result)[:1000]} write_json(run_dir / ("space_factory_reboot.json" if factory_reboot else "space_restart.json"), payload) append_event(events_path, "factory_rebuild", "success", f"{'Factory reboot' if factory_reboot else 'Space restart'} requested", payload) return True except TypeError: # Older huggingface_hub versions may not expose factory_reboot. Do not # loop; fall back to a plain restart only for non-factory requests. if factory_reboot: append_event(events_path, "factory_rebuild", "warning", "Installed huggingface_hub does not support restart_space(factory_reboot=True)", {"reason": reason}) return False try: result = api.restart_space(repo_id=target_space_id, token=token) append_event(events_path, "factory_rebuild", "success", "Space restart requested", {"reason": reason, "result": str(result)[:1000]}) return True except Exception as exc: append_event(events_path, "factory_rebuild", "failed", "Space restart API call failed", {"reason": reason, **exception_payload(exc)}) return False except Exception as exc: append_event(events_path, "factory_rebuild", "failed", f"{'Factory reboot' if factory_reboot else 'Space restart'} API call failed", {"reason": reason, **exception_payload(exc)}) return False def safe_same_code_reupload(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, reason: str = "") -> bool: """Re-upload the current workspace once, with restart-loop guardrails.""" if not restart_guardrail_allows("same_code_upload", events_path, reason=reason): return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"}: append_event(events_path, "factory_rebuild", "warning", "Skipped same-code upload: Space is already building/starting", {"runtime": runtime_before, "reason": reason}) return False upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) append_event(events_path, "factory_rebuild", "success", "Same-code workspace uploaded under restart guardrails", {"reason": reason, "runtime_before": runtime_before}) return True def collect_space_logs(target_space_id: str, token: str, run_dir: Path, events_path: Path): """Collect Space build/runtime logs with explicit capability detection and an index. This is intentionally more structured than the old best-effort collector: - runtime state is always captured when possible; - runtime/build log collection first uses HfApi.get_space_logs when available; - if the installed huggingface_hub lacks that method, REST fallbacks are attempted; - legacy text files are still written, but availability/quality lives in space_logs_index.json. """ from huggingface_hub import HfApi import json as _json import sys as _sys try: import huggingface_hub as _hf_hub hub_version = getattr(_hf_hub, "__version__", None) except Exception: hub_version = None logs_dir = run_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) api = HfApi(token=token) collected_at = now() written = [] index = { "schema_version": "space_logs_fetch.v198_16", "target_space_id": target_space_id, "collected_at": collected_at, "huggingface_hub_version": hub_version, "python_version": _sys.version.split()[0], "capabilities": { "hfapi_get_space_runtime": hasattr(api, "get_space_runtime"), "hfapi_get_space_logs": hasattr(api, "get_space_logs"), "rest_fallback": True, }, "entries": {}, "hf_space_log_endpoints": { "run": f"https://huggingface.co/api/spaces/{target_space_id}/logs/run", "build": f"https://huggingface.co/api/spaces/{target_space_id}/logs/build", }, "legacy_runtime_filename": "logs/space_logs_runtime.txt", } def _tail(text: str, n: int = 1200) -> str: return (text or "")[-n:] def _stringify_log_response(out) -> str: if out is None: return "" if isinstance(out, (list, tuple)): return "\n".join(str(x) for x in out) if isinstance(out, (dict, list)): return _json.dumps(out, ensure_ascii=False, indent=2) return str(out) def _write_text(filename: str, text: str) -> int: path = logs_dir / filename path.write_text(text or "", encoding="utf-8") return path.stat().st_size if path.exists() else 0 def _entry(key: str, *, filename: str, available: bool, source: str, quality: str, reason: str = "", error: str = "", text: str = ""): size = _write_text(filename, text) payload = { "available": bool(available), "path": f"logs/{filename}", "source": source, "quality": quality, "reason": reason, "error": error[:1500] if error else "", "size_bytes": size, "tail": _tail(text), } index["entries"][key] = payload written.append({ "file": filename, "source": source, "available": bool(available), "quality": quality, "reason": reason, "error": error[:1000] if error else "", "returncode": 0 if available else 1, "tail": _tail(text, 1000), }) return payload def _unavailable_text(kind: str, reason: str, error: str = "") -> str: return ( f"[ASF_LOG_UNAVAILABLE]\n" f"kind={kind}\n" f"reason={reason}\n" f"target_space_id={target_space_id}\n" f"huggingface_hub_version={hub_version}\n" f"error={error[:1500] if error else ''}\n" "See logs/space_logs_index.json for structured availability metadata.\n" ) # Runtime state / snapshot. This is not a log stream, but it is critical context for # BUILD_ERROR/RUNTIME_ERROR triage and is always tracked separately from log streams. try: runtime_obj = api.get_space_runtime(repo_id=target_space_id, token=token) runtime_payload = runtime_to_dict(runtime_obj) runtime_payload.update({"target_space_id": target_space_id, "collected_at": collected_at}) runtime_text = _json.dumps(runtime_payload, indent=2, ensure_ascii=False) _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) # Legacy path retained because older reports/tests point to this file. _entry("runtime_state_legacy", filename="space_runtime_state.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) except Exception as exc: err = f"{type(exc).__name__}: {str(exc)[:1500]}" _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_snapshot", "runtime_snapshot_error", err)) _entry("runtime_state_legacy", filename="space_runtime_state.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_state", "runtime_snapshot_error", err)) def _collect_via_hfapi(kind: str): if not hasattr(api, "get_space_logs"): return None, "hub_method_unavailable", "" try: out = api.get_space_logs(repo_id=target_space_id, token=token, build=(kind == "build")) text = _stringify_log_response(out) if text.strip(): return text, "", "" return "", "empty_response", "" except Exception as exc: return None, "hfapi_error", f"{type(exc).__name__}: {str(exc)[:1500]}" def _space_log_endpoint(kind: str) -> str: # HF Space log streams are exposed under /logs/run and /logs/build. # "runtime" is an ASF legacy label for the file name; the Hub endpoint is "run". endpoint_kind = "build" if kind == "build" else "run" return f"https://huggingface.co/api/spaces/{target_space_id}/logs/{endpoint_kind}" def _read_streaming_response(response, *, max_bytes: int = 1_200_000) -> str: chunks = [] total = 0 try: iterator = response.iter_content(chunk_size=8192, decode_unicode=True) except Exception: return response.text or "" for chunk in iterator: if not chunk: continue if isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="ignore") chunks.append(str(chunk)) total += len(str(chunk).encode("utf-8", errors="ignore")) if total >= max_bytes: chunks.append("\n[ASF_LOG_TRUNCATED: max_bytes reached]\n") break return "".join(chunks) def _collect_via_rest(kind: str): # v198.16: use the real HF Hub Space log endpoints. The runtime stream is # /logs/run, not any legacy runtime endpoint and not the older query-string endpoint. # Wrong endpoints made Pi repair from generic RUNTIME_ERROR messages. try: import requests except Exception as exc: return None, "requests_unavailable", f"{type(exc).__name__}: {str(exc)[:1500]}" headers = {"Authorization": f"Bearer {token}", "Accept": "text/plain,application/json,*/*"} if token else {"Accept": "text/plain,application/json,*/*"} primary_url = _space_log_endpoint(kind) candidates = [primary_url] last_error = "" for url in candidates: try: response = requests.get(url, headers=headers, timeout=(10, 45), stream=True) body = _read_streaming_response(response) if response.ok and body.strip(): ctype = response.headers.get("content-type", "") return body, "", f"{url} ({response.status_code}, {ctype})" last_error = f"{url} returned {response.status_code}: {body[:500]}" except Exception as exc: last_error = f"{url} failed: {type(exc).__name__}: {str(exc)[:500]}" return None, "rest_logs_unavailable", last_error def _collect_stream(kind: str, filename: str): attempted = [] text, reason, error = _collect_via_hfapi(kind) attempted.append({"method": "HfApi.get_space_logs", "reason": reason, "error": error[:500] if error else ""}) if text is not None and text.strip(): return _entry(kind, filename=filename, available=True, source="HfApi.get_space_logs", quality="full", text=text) rest_text, rest_reason, rest_error = _collect_via_rest(kind) attempted.append({"method": "hf_rest_logs", "reason": rest_reason, "error": rest_error[:500] if rest_error else ""}) if rest_text is not None and rest_text.strip(): source = "hf_rest_logs_run_build_endpoint" if rest_error and rest_error.startswith("https://"): source = rest_error return _entry(kind, filename=filename, available=True, source=source, quality="full", text=rest_text) # Keep legacy files present, but explicitly mark them as unavailable rather than # pretending an SDK AttributeError is a Space runtime/build log. This also avoids # the obsolete CLI path that emitted "invalid choice: spaces". final_reason = rest_reason or reason or "logs_unavailable" final_error = rest_error or error or "" entry = _entry(kind, filename=filename, available=False, source="log_collection", quality="unavailable", reason=final_reason, error=final_error, text=_unavailable_text(kind, final_reason, final_error)) entry["attempted_methods"] = attempted return entry runtime_entry = _collect_stream("runtime", "space_logs_runtime.txt") build_entry = _collect_stream("build", "space_logs_build.txt") def _first_error_from(text: str) -> str: if not text: return "" lines = text.splitlines() lowered = [line.lower() for line in lines] markers = ("traceback", "exception", "error", "exit code", "failed", "runtimeerror", "attributeerror", "modulenotfounderror") for idx, line in enumerate(lowered): if any(marker in line for marker in markers): start = max(0, idx - 3) end = min(len(lines), idx + 12) return "\n".join(lines[start:end])[:4000] return "" runtime_text = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if runtime_entry.get("available") and (logs_dir / "space_logs_runtime.txt").exists() else "" build_text = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if build_entry.get("available") and (logs_dir / "space_logs_build.txt").exists() else "" # v198.16 canonical aliases matching the HF API endpoint names. Keep legacy # filenames for compatibility but make the correct run/build source explicit. (logs_dir / "space_logs_run.txt").write_text(runtime_text if runtime_entry.get("available") else _unavailable_text("run", runtime_entry.get("reason") or "run_logs_unavailable", runtime_entry.get("error") or ""), encoding="utf-8") (logs_dir / "space_logs_build.txt").write_text(build_text if build_entry.get("available") else _unavailable_text("build", build_entry.get("reason") or "build_logs_unavailable", build_entry.get("error") or ""), encoding="utf-8") first_error = _first_error_from(runtime_text) or _first_error_from(build_text) log_quality = "full" if runtime_entry.get("available") or build_entry.get("available") else "snapshot_only" if index["entries"].get("runtime_snapshot", {}).get("available") else "unavailable" diagnostics = { "schema_version": "1.0", "target_space_id": target_space_id, "collected_at": collected_at, "log_quality": log_quality, "run_logs_endpoint": _space_log_endpoint("runtime"), "build_logs_endpoint": _space_log_endpoint("build"), "run_logs_available": bool(runtime_entry.get("available")), "runtime_logs_available": bool(runtime_entry.get("available")), "build_logs_available": bool(build_entry.get("available")), "run_log_bytes": len(runtime_text.encode("utf-8", errors="ignore")), "build_log_bytes": len(build_text.encode("utf-8", errors="ignore")), "first_actionable_error": first_error, "first_error": first_error, "diagnosis_log_source": "run_or_build_logs" if first_error else ("runtime_snapshot" if log_quality == "snapshot_only" else "none"), } diagnostics_text = _json.dumps(diagnostics, ensure_ascii=False, indent=2) _entry("diagnostics", filename="space_log_diagnostics.json", available=True, source="asf_log_collector", quality=log_quality, text=diagnostics_text) index["log_quality"] = log_quality index["run_logs_available"] = bool(runtime_entry.get("available")) index["runtime_logs_available"] = bool(runtime_entry.get("available")) index["build_logs_available"] = bool(build_entry.get("available")) index["first_actionable_error"] = first_error[:4000] index["first_error"] = first_error[:1000] fetch_status = { "schema_version": "space_logs_fetch_status.v198_16", "space_id": target_space_id, "run_logs_endpoint": _space_log_endpoint("runtime"), "build_logs_endpoint": _space_log_endpoint("build"), "run_logs_fetched": bool(runtime_entry.get("available")), "build_logs_fetched": bool(build_entry.get("available")), "run_log_bytes": len(runtime_text.encode("utf-8", errors="ignore")), "build_log_bytes": len(build_text.encode("utf-8", errors="ignore")), "first_actionable_error_source": "run" if _first_error_from(runtime_text) else ("build" if _first_error_from(build_text) else "none"), "first_actionable_error": first_error[:4000], "pi_repair_allowed_from_logs": bool(first_error), } fetch_text = _json.dumps(fetch_status, ensure_ascii=False, indent=2) (logs_dir / "space_logs_fetch_status.json").write_text(fetch_text, encoding="utf-8") written.append({"file": "space_logs_fetch_status.json", "source": "asf_log_collector", "available": True, "quality": log_quality, "returncode": 0, "tail": _tail(fetch_text, 1000)}) actionable_brief = ( "# Actionable Space log brief\n\n" f"Space: `{target_space_id}`\n\n" f"Run logs endpoint: `{_space_log_endpoint('runtime')}`\n\n" f"Build logs endpoint: `{_space_log_endpoint('build')}`\n\n" f"Run logs fetched: `{bool(runtime_entry.get('available'))}`\n\n" f"Build logs fetched: `{bool(build_entry.get('available'))}`\n\n" "## First actionable error\n\n" "```text\n" + (first_error or "No actionable error found in fetched Space run/build logs.")[:4000] + "\n```\n" ) (run_dir / "repair").mkdir(parents=True, exist_ok=True) (run_dir / "repair" / "actionable_error_brief.md").write_text(actionable_brief, encoding="utf-8") index_text = _json.dumps(index, ensure_ascii=False, indent=2) (logs_dir / "space_logs_index.json").write_text(index_text, encoding="utf-8") written.append({"file": "space_logs_index.json", "source": "asf_log_collector", "available": True, "quality": log_quality, "returncode": 0, "tail": _tail(index_text, 1000)}) event_status = "success" if runtime_entry.get("available") or build_entry.get("available") else "warning" event_message = "Collected HF Space run/build logs and runtime snapshot" if event_status == "success" else "HF Space run/build log streams unavailable; runtime snapshot/index written" append_event(events_path, "space_logs", event_status, event_message, {"files": written, "log_quality": log_quality, "index": "logs/space_logs_index.json"}) write_artifact_manifest(run_dir, reason="space_logs_collected") return written def validate_http_health(target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int): import requests base_url = space_subdomain_url(target_space_id) url = base_url.rstrip("/") + "/health" headers = {"Authorization": f"Bearer {token}", "Accept": "application/json,text/plain,*/*"} response = requests.get(url, headers=headers, timeout=20) payload = { "status": "success" if response.ok else "failed", "attempt": attempt, "url": url, "status_code": response.status_code, "content_type": response.headers.get("content-type"), "text": response.text[:2000], } if response.ok: try: payload["json"] = response.json() except Exception: pass semantic = validation_health_semantics(payload | {"validator": "http_get_health"}) payload["health_endpoint_reachable"] = True payload["health_semantic_passed"] = semantic.get("semantic_passed") payload["health_semantics"] = semantic write_json(run_dir / "tests" / "http_health.json", payload) write_json(run_dir / "tests" / "test_result.json", payload | {"validator": "http_get_health"}) event_status = "success" if semantic.get("semantic_passed") is not False else "warning" event_message = "HTTP /health validation passed" if event_status == "success" else "HTTP /health is reachable but reported unhealthy/not-ready semantic state" append_event(events_path, "api_validation", event_status, event_message, {"attempt": attempt, "url": url, "status_code": response.status_code, "health_semantic_passed": semantic.get("semantic_passed")}) return payload | {"validator": "http_get_health"} raise RuntimeError(f"HTTP /health returned {response.status_code}: {response.text[:500]}") def validate_gradio_api(target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int): client = make_gradio_client(target_space_id, token) identity = write_gradio_client_identity(client, target_space_id, run_dir, events_path, phase="api_validation") if identity.get("mismatch"): raise RuntimeError(f"space_identity_mismatch: expected {target_space_id}, observed {identity.get('observed')}") schema = client.view_api(return_format="dict") write_json(run_dir / "tests" / "api_schema.json", schema if isinstance(schema, dict) else {"schema": str(schema)}) discovered = api_names_from_schema(schema) candidates = [] for name in ["/health", "/predict", "/greet"] + discovered: if name not in candidates: candidates.append(name) errors = [] for api_name in candidates: try: if api_name == "/greet": result = client.predict("Agentic Space Factory", api_name=api_name) else: result = client.predict(api_name=api_name) payload = {"status": "success", "attempt": attempt, "api_name": api_name, "discovered_api_names": discovered, "result_repr": repr(result)[:2000], "validator": "gradio_client"} if api_name == "/health": semantic = validation_health_semantics(payload) payload["health_endpoint_reachable"] = True payload["health_semantic_passed"] = semantic.get("semantic_passed") payload["health_semantics"] = semantic event_status = "success" if semantic.get("semantic_passed") is not False else "warning" event_message = "Gradio /health validation passed" if event_status == "success" else "Gradio /health is reachable but reported unavailable/not-ready semantic state" else: event_status = "success" event_message = "Gradio API validation passed" write_json(run_dir / "tests" / "test_result.json", payload) append_event(events_path, "api_validation", event_status, event_message, {"attempt": attempt, "api_name": api_name, "discovered_api_names": discovered, "health_semantic_passed": payload.get("health_semantic_passed")}) return payload except Exception as exc: errors.append({"api_name": api_name, "error": str(exc)[:1000]}) raise RuntimeError("; ".join(f"{e['api_name']}: {e['error']}" for e in errors[:5]) or "No callable API endpoints found") def latest_collected_build_log(run_dir: Path) -> str: path = run_dir / "logs" / "space_logs_build.txt" return path.read_text(encoding="utf-8", errors="ignore") if path.exists() else "" def runtime_stage_is_build_error(stage: str) -> bool: value = (stage or "").upper() return "BUILD_ERROR" in value or value in {"BUILDING_ERROR", "ERROR", "FAILED", "BUILD_FAILED"} def runtime_stage_is_busy(stage: str) -> bool: value = (stage or "").upper() return any(marker in value for marker in ["BUILDING", "BUILDQUEUED", "BUILD_QUEUED", "STARTING", "PENDING"]) def build_log_has_terminal_error(text: str) -> bool: low = (text or "").lower() terminal_markers = [ "--> error:", "error: resolutionimpossible", "resolutionimpossible", "no matching distribution found", "could not find a version that satisfies the requirement", "did not complete successfully", "process \"/bin/sh -c pip install", "cannot install", "conflicting dependencies", "metadata-generation-failed", "exit code: 1", ] return any(marker in low for marker in terminal_markers) def _parse_iso_timestamp_epoch(value: str | None) -> float | None: raw = str(value or "").strip() if not raw: return None try: if raw.endswith("Z"): raw = raw[:-1] + "+00:00" return datetime.fromisoformat(raw).timestamp() except Exception: return None def _log_block_timestamp_epoch(block: str) -> float | None: matches = re.findall(r'"timestamp"\s*:\s*"([^"]+)"', block or "") epochs = [_parse_iso_timestamp_epoch(item) for item in matches] epochs = [item for item in epochs if item is not None] return max(epochs) if epochs else None def get_last_runtime_upload_epoch(run_dir: Path) -> dict: payload = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {} return payload if isinstance(payload, dict) else {} def get_last_runtime_upload_at(run_dir: Path) -> str: payload = get_last_runtime_upload_epoch(run_dir) return str(payload.get("last_upload_completed_at") or payload.get("uploaded_at") or "") def filter_build_log_since(text: str, since_iso: str | None = None) -> str: """Return only build-log SSE blocks at or after the latest upload epoch. Hugging Face build-log streams can include previous failed build attempts for the same Space. v198.26.1 keeps those lines in archived logs but never lets them drive the active validation state after a later upload/rebuild. """ raw = text or "" since_epoch = _parse_iso_timestamp_epoch(since_iso) if since_epoch is None: return raw blocks = re.split(r"(\n\s*\n)", raw) kept: list[str] = [] stale_count = 0 for i in range(0, len(blocks), 2): block = blocks[i] sep = blocks[i + 1] if i + 1 < len(blocks) else "" if not block: continue ts = _log_block_timestamp_epoch(block) if ts is not None and ts + 0.001 < since_epoch: stale_count += 1 continue kept.append(block + sep) return "".join(kept) def build_log_freshness_payload(raw_text: str, fresh_text: str, since_iso: str | None = None) -> dict: raw_issue = extract_pip_dependency_issue(raw_text or "") fresh_issue = extract_pip_dependency_issue(fresh_text or "") raw_terminal = build_log_has_terminal_error(raw_text or "") fresh_terminal = build_log_has_terminal_error(fresh_text or "") return { "schema_version": "build_log_freshness.v198_25_3", "last_upload_completed_at": since_iso or "", "raw_bytes": len((raw_text or "").encode("utf-8", errors="ignore")), "fresh_bytes": len((fresh_text or "").encode("utf-8", errors="ignore")), "raw_terminal_error_detected": bool(raw_terminal), "fresh_terminal_error_detected": bool(fresh_terminal), "raw_dependency_issue_detected": bool(raw_issue), "fresh_dependency_issue_detected": bool(fresh_issue), "stale_build_error_ignored": bool((raw_terminal or raw_issue) and not (fresh_terminal or fresh_issue) and since_iso), } def latest_fresh_collected_build_log(run_dir: Path) -> str: return filter_build_log_since(latest_collected_build_log(run_dir), get_last_runtime_upload_at(run_dir)) def record_runtime_upload_epoch(run_dir: Path, events_path: Path | None, *, target_space_id: str, manifest: dict | None = None) -> dict: existing = get_last_runtime_upload_epoch(run_dir) history = existing.get("history") if isinstance(existing.get("history"), list) else [] sequence = int(existing.get("upload_sequence") or len(history) or 0) + 1 uploaded_at = now() entry = { "sequence": sequence, "phase": "initial_upload" if sequence == 1 else "repair_upload", "uploaded_at": uploaded_at, "target_space_id": target_space_id, "file_count": (manifest or {}).get("file_count"), "total_bytes": (manifest or {}).get("total_bytes"), } history.append(entry) payload = { "schema_version": "runtime_upload_epoch.v198_25_3", "target_space_id": target_space_id, "upload_sequence": sequence, "last_upload_completed_at": uploaded_at, "last_upload_phase": entry["phase"], "history": history[-20:], "updated_at": uploaded_at, } write_json(run_dir / "runtime_upload_epoch.json", payload) if events_path is not None: append_event(events_path, "runtime_upload_epoch", "success", "Recorded runtime upload epoch for stale build-log filtering", {k: payload[k] for k in ["upload_sequence", "last_upload_completed_at", "last_upload_phase", "target_space_id"]}) return payload def clear_active_repair_error_after_success(run_dir: Path, events_path: Path | None = None, *, final_gate_status: str = "full_inference_success") -> dict: outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {} if not isinstance(outcome, dict) or not outcome: return {} history = outcome.get("historical_failures") if isinstance(outcome.get("historical_failures"), list) else [] if outcome.get("failure_type") or outcome.get("validation_error") or str(outcome.get("post_repair_validation") or "").lower() not in {"", "success"}: history.append({ "phase": "pre_final_success", "failure_type": outcome.get("failure_type") or "", "validation_error": outcome.get("validation_error") or "", "post_repair_validation": outcome.get("post_repair_validation") or "", "recorded_at": now(), }) outcome.update({ "post_repair_validation": "success" if outcome.get("patch_applied") or outcome.get("upload_success") else outcome.get("post_repair_validation", ""), "failure_type": "", "validation_error": "", "final_gate_status": final_gate_status, "active_error_cleared_by_final_success": True, "historical_failures": history[-20:], "final_user_message": "Final full-inference gate passed; previous dependency/build errors are historical.", "updated_at": now(), }) write_json(run_dir / "repair_outcome.json", outcome) try: write_json(run_dir / "repair" / "REPAIR_OUTCOME.json", outcome) except Exception: pass if events_path is not None: append_event(events_path, "final_status_reconciliation", "success", "Cleared active repair error fields after final full-inference success", {"final_gate_status": final_gate_status, "historical_failure_count": len(history)}) return outcome def final_full_inference_success_recorded(run_dir: Path) -> bool: gate = load_json_if_exists(run_dir / "inference_gate.json") if (run_dir / "inference_gate.json").exists() else {} state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {} smoke = load_json_if_exists(run_dir / "tests" / "generation_smoke.json") if (run_dir / "tests" / "generation_smoke.json").exists() else {} signals = gate.get("implementation_signals") if isinstance(gate.get("implementation_signals"), dict) else {} return bool( (gate.get("status") == "full_inference_success" or state.get("status") == "full_inference_success") and (smoke.get("status") == "success" or signals.get("generation_smoke_passed") is True) ) def reconcile_final_success_state(run_dir: Path, events_path: Path | None, final_state: dict, inference_gate: dict | None = None, generation_smoke: dict | None = None) -> dict: gate = inference_gate or {} smoke = generation_smoke or {} status = str(gate.get("status") or final_state.get("status") or "") if status != "full_inference_success": return final_state clean = dict(final_state or {}) clean.update({ "status": "full_inference_success", "failure_type": "", "validation_error": "", "active_error": None, "job_exit_status": "success", "job_exit_code": 0, "recovered": bool(load_json_if_exists(run_dir / "repair_outcome.json")), "final_status_reconciled_at": now(), }) details = clean.get("details") if isinstance(clean.get("details"), dict) else {} if details: historical = clean.get("historical_failures") if isinstance(clean.get("historical_failures"), list) else [] if details.get("error") or details.get("failure_type") or details.get("validation_error"): historical.append({"phase": "pre_final_success_details", "details": details, "recorded_at": now()}) clean["historical_failures"] = historical[-20:] clean.pop("details", None) repair_outcome = clear_active_repair_error_after_success(run_dir, events_path, final_gate_status="full_inference_success") if repair_outcome: clean["repair_outcome"] = repair_outcome write_json(run_dir / "state.json", clean) write_json(run_dir / "final_status_reconciliation.json", { "schema_version": "final_status_reconciliation.v198_25_3", "status": "full_inference_success", "job_exit_code": 0, "active_errors_cleared": True, "health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")), "generation_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success"), "updated_at": now(), }) return clean def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None, final_state: dict, *, status: str, message: str = "", details: dict | None = None, job_exit_code: int | None = None) -> dict: """Persist a terminal truth artifact for every final outcome. v198.26.9: the UI must not have to infer failed/stopped states from a mix of state, repair_outcome and recent events. Success already writes final_status_reconciliation.json; failed/auth/cancelled terminal paths now write the same artifact so Active Run can converge deterministically. """ state = final_state or {} safe = safe_details(details or {}) runtime_upload_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {} space_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {} runtime_history = runtime_upload_epoch.get("history") if isinstance(runtime_upload_epoch.get("history"), list) else [] target_space = state.get("target_space") or state.get("target_space_id") or runtime_upload_epoch.get("target_space_id") or os.environ.get("TARGET_SPACE_ID", "") runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history) space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url"))) payload = { "schema_version": "final_status_reconciliation.v198_26_9", "status": status, "terminal": True, "message": message or state.get("message") or "", "job_exit_code": int(job_exit_code if job_exit_code is not None else (0 if status == "full_inference_success" else 1)), "target_space": target_space, "target_space_url": f"https://huggingface.co/spaces/{target_space}" if target_space else "", "space_created": bool(target_space and (runtime_uploaded or space_runtime_known)), "runtime_uploaded": bool(runtime_uploaded), "space_uploaded": bool(runtime_uploaded), "space_runtime_known": bool(space_runtime_known), "failure_type": safe.get("failure_type") or safe.get("repair_failure_type") or state.get("failure_type") or "", "failure_phase": safe.get("phase") or safe.get("failure_phase") or safe.get("step") or "", "details": safe, "updated_at": now(), } write_json(run_dir / "final_status_reconciliation.json", payload) if events_path is not None: append_event(events_path, "final_status_reconciliation", "success" if status == "full_inference_success" else "failed", "Wrote terminal final status reconciliation", {k: payload.get(k) for k in ("status", "job_exit_code", "target_space", "runtime_uploaded", "space_runtime_known", "failure_type")}) return payload def _normalize_dependency_package_name(value: str = "") -> str: name = re.split(r"[<>=!~;\[,\s]", str(value or "").strip(), 1)[0].strip().lower().replace("_", "-") aliases = { "huggingface-hub": "huggingface-hub", "huggingface_hub": "huggingface-hub", "gradio-client": "gradio-client", "gradio_client": "gradio-client", "opencv-python-headless": "opencv-python-headless", "opencv-python": "opencv-python", } return aliases.get(name, name) def extract_dependency_conflict_packages(text: str = "") -> list[str]: """Best-effort package extraction from pip resolver conflicts. This intentionally favors useful signal over perfect parsing. Pip emits several formats (plain text, JSON-escaped SSE `data:` lines, long candidate lists); the Factory only needs enough structure to route recovery safely. """ raw = text or "" packages: list[str] = [] def add(candidate: str = ""): name = _normalize_dependency_package_name(candidate) if name and re.match(r"^[a-z0-9][a-z0-9_.-]*$", name) and name not in packages: packages.append(name) for match in re.finditer(r"\b([A-Za-z0-9_.-]+)\s+(?:[A-Za-z0-9_.+-]+\s+)?depends on\s+([A-Za-z0-9_.-]+)", raw, flags=re.I): add(match.group(1)); add(match.group(2)) for match in re.finditer(r"The user requested\s+([A-Za-z0-9_.-]+)", raw, flags=re.I): add(match.group(1)) for match in re.finditer(r"Cannot install\s+(.+?)\s+because these package versions have conflicting dependencies", raw, flags=re.I | re.S): chunk = match.group(1) chunk = re.sub(r"-r\s+\S+\s+\(line\s+\d+\)", " ", chunk, flags=re.I) for token in re.split(r"[,\s]+|\band\b", chunk): token = token.strip(" .;:'\"`()[]{}") if token and not token.startswith("-") and token.lower() not in {"the", "user", "requested", "install"}: add(token) for match in re.finditer(r"(?:No matching distribution found for|Could not find a version that satisfies the requirement)\s+([^\s]+)", raw, flags=re.I): add(match.group(1)) return packages[:20] def detect_dependency_platform_conflict(text: str = "", packages: list[str] | None = None) -> dict: raw = text or "" low = raw.lower() packages = packages or extract_dependency_conflict_packages(raw) pkg_set = set(packages) hub_floor = bool(re.search(r"gradio\s+[A-Za-z0-9_.+-]*\s*depends on\s+huggingface[-_]hub\s*<\s*2(?:\.0)?\s*(?:,|and)\s*>=\s*1(?:\.2)?", low, flags=re.I)) transformers_hub_ceiling = bool(re.search(r"transformers\s+[A-Za-z0-9_.+-]*\s*depends on\s+huggingface[-_]hub\s*<\s*1(?:\.0)?", low, flags=re.I)) detected = hub_floor and transformers_hub_ceiling and {"gradio", "transformers", "huggingface-hub"}.issubset(pkg_set) return { "detected": bool(detected), "kind": "gradio_transformers_hub_range_conflict" if detected else "none", "packages": packages, "evidence": { "gradio_requires_hub_1_or_newer": hub_floor, "transformers_requires_hub_below_1": transformers_hub_ceiling, }, "recommended_action": "surgical_requirements_repair_or_dependency_platform_conflict" if detected else "standard_dependency_repair", } def write_dependency_issue_artifact(run_dir: Path, issue: dict, *, source: str = "build_logs", runtime_payload: dict | None = None) -> dict: payload = { "schema_version": "dependency_issue.v198_25_3", "created_at": now(), "source": source, "runtime": runtime_payload or {}, **(issue or {}), } if "failure_owner" not in payload: payload["failure_owner"] = "dependency" if "failure_class" not in payload: payload["failure_class"] = "dependency_resolution_conflict" if "repair_candidate" not in payload: payload["repair_candidate"] = True write_json(run_dir / "dependency_issue.json", payload) return payload def collect_space_build_logs_fast(target_space_id: str, token: str, run_dir: Path, events_path: Path, *, attempt: int = 0, reason: str = "", max_seconds: int = 12, max_bytes: int = 800_000) -> dict: """Fast, bounded read of the HF Space build log stream. The full log collector intentionally archives run/build streams, but HF's text/event-stream endpoint can stay open while BUILDING. During live API validation we only need enough build-log text to notice terminal pip/build errors. This probe exits as soon as a terminal marker is seen, max_bytes is reached, or a small time/read timeout is hit. """ logs_dir = run_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) status_path = logs_dir / "build_log_fast_probe_status.json" url = f"https://huggingface.co/api/spaces/{target_space_id}/logs/build" headers = {"Authorization": f"Bearer {token}", "Accept": "text/plain,application/json,*/*"} if token else {"Accept": "text/plain,application/json,*/*"} collected = "" terminal = False error = "" available = False source = f"{url} fast_probe" last_upload_at = get_last_runtime_upload_at(run_dir) started = time.time() try: import requests response = requests.get(url, headers=headers, timeout=(5, max(5, int(max_seconds))), stream=True) source = f"{url} ({response.status_code}, {response.headers.get('content-type', '')}) fast_probe" if not response.ok: try: collected = response.text[:4000] except Exception: collected = "" error = f"http_status={response.status_code}" else: chunks: list[str] = [] total = 0 available = True try: iterator = response.iter_content(chunk_size=8192, decode_unicode=True) except Exception: iterator = [] chunks.append(response.text or "") for chunk in iterator: if not chunk: if time.time() - started >= max_seconds: break continue if isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="ignore") chunks.append(str(chunk)) total += len(str(chunk).encode("utf-8", errors="ignore")) current = "".join(chunks) if build_log_has_terminal_error(filter_build_log_since(current, last_upload_at)): terminal = True break if total >= max_bytes: chunks.append("\n[ASF_FAST_BUILD_LOG_TRUNCATED: max_bytes reached]\n") break if time.time() - started >= max_seconds: chunks.append("\n[ASF_FAST_BUILD_LOG_TRUNCATED: max_seconds reached]\n") break collected = "".join(chunks) except Exception as exc: error = f"{type(exc).__name__}: {str(exc)[:1000]}" finally: try: if 'response' in locals(): response.close() except Exception: pass fresh_collected = filter_build_log_since(collected, last_upload_at) if collected.strip(): # Preserve accumulated full-ish build logs when multiple probes run, but # classify active errors only from fresh blocks after the latest upload. path = logs_dir / "space_logs_build.txt" previous = path.read_text(encoding="utf-8", errors="ignore") if path.exists() else "" if collected not in previous: combined = (previous + "\n" + collected).strip() if previous.strip() else collected path.write_text(combined[-2_000_000:], encoding="utf-8") issue = extract_pip_dependency_issue(fresh_collected) if fresh_collected.strip() else {} freshness = build_log_freshness_payload(collected, fresh_collected, last_upload_at) payload = { "schema_version": "build_log_fast_probe.v198_25_3", "target_space_id": target_space_id, "attempt": attempt, "reason": reason, "available": bool(available or collected.strip()), "terminal_error_detected": bool(build_log_has_terminal_error(fresh_collected)), "dependency_issue_detected": bool(issue), "dependency_issue": issue, "source": source, "error": error[:1500] if error else "", "bytes_read": len((collected or "").encode("utf-8", errors="ignore")), "fresh_bytes_read": len((fresh_collected or "").encode("utf-8", errors="ignore")), "last_upload_completed_at": last_upload_at, "freshness": freshness, "stale_build_error_ignored": bool(freshness.get("stale_build_error_ignored")), "elapsed_seconds": round(time.time() - started, 3), "created_at": now(), } status_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") if payload["terminal_error_detected"] or payload["dependency_issue_detected"]: append_event(events_path, "build_log_fast_probe", "warning", "Detected terminal build/dependency error from bounded build-log probe", payload) return payload def raise_if_collected_build_error(target_space_id: str, token: str, run_dir: Path, events_path: Path, *, attempt: int, reason: str = "", runtime_payload: dict | None = None): stage = str((runtime_payload or {}).get("stage") or "") # Fast path for BUILDING: never let the full log stream block the live # validation loop for tens of minutes when pip has already emitted a # terminal resolver error. fast_probe = collect_space_build_logs_fast(target_space_id, token, run_dir, events_path, attempt=attempt, reason=reason) build_log = latest_fresh_collected_build_log(run_dir) has_terminal_build_log = build_log_has_terminal_error(build_log) stage_is_terminal = runtime_stage_is_build_error(stage) # Once the Hub reports a terminal build stage, make one archival collection # attempt for better artifacts. During plain BUILDING we rely only on the # bounded fast probe to avoid stream stalls. if stage_is_terminal and not has_terminal_build_log: try: collect_space_logs(target_space_id, token, run_dir, events_path) build_log = latest_fresh_collected_build_log(run_dir) has_terminal_build_log = build_log_has_terminal_error(build_log) except Exception as exc: append_event(events_path, "space_logs", "warning", "Full log collection failed after terminal build stage; using fast probe evidence", {"error": f"{type(exc).__name__}: {str(exc)[:1000]}", "fast_probe": fast_probe}) if stage_is_terminal or has_terminal_build_log or fast_probe.get("terminal_error_detected"): issue = extract_pip_dependency_issue(build_log) if issue: issue = write_dependency_issue_artifact(run_dir, issue, source="fast_build_log_probe" if fast_probe.get("dependency_issue_detected") else "build_logs", runtime_payload=runtime_payload or {}) payload = { "schema_version": "build_error_observation.v198_25_3", "attempt": attempt, "reason": reason, "runtime": runtime_payload or {}, "fast_probe": fast_probe, "dependency_issue": issue, "failure_owner": "dependency" if issue else "space_build", "failure_class": (issue or {}).get("failure_class") or ("space_build_error" if stage_is_terminal else "build_log_terminal_error"), "repair_candidate": bool((issue or {}).get("repair_candidate", bool(issue))), "display_status": "dependency_error" if issue else "build_error", "effective_status": "dependency_error" if issue else "build_error", "tail": build_log[-4000:], "created_at": now(), } append_event(events_path, "api_validation", "failed", "Space build/dependency error detected; stopping wait and entering recovery", payload) write_json(run_dir / "build_error_observation.json", payload) if issue: raise RuntimeError(f"Space build dependency error detected from bounded build-log probe. {json.dumps(issue, ensure_ascii=False)[:1000]} See dependency_issue.json, build_error_observation.json, and logs/space_logs_build.txt") raise RuntimeError(f"Space build failed according to runtime/logs. See build_error_observation.json and logs/space_logs_build.txt") TRANSIENT_SCHEDULING_MARKERS = [ "scheduling failure", "not enough hardware capacity", "unable to schedule", "capacity unavailable", "hardware capacity", ] DEFINITIVE_APP_ERROR_MARKERS = [ "modulenotfounderror", "importerror", "syntaxerror", "typeerror", "attributeerror", "nameerror", "resolutionimpossible", "no matching distribution found", "could not find a version that satisfies the requirement", "cuda out of memory", "outofmemoryerror", "traceback (most recent call last)", ] STARTUP_STALL_MARKERS = [ "app_starting", "starting", "live health/api validation did not pass before timeout", "http /health returned 404", "gradio api failed", "timed out", "timeout", ] def _combined_runtime_failure_text(failure_reason: str = "", build_log: str = "", runtime_log: str = "", runtime_payload: dict | None = None) -> str: return f"{failure_reason}\n{build_log[-12000:]}\n{runtime_log[-12000:]}\n{json.dumps(runtime_payload or {}, ensure_ascii=False)}".lower() def has_definitive_app_error(failure_reason: str = "", build_log: str = "", runtime_log: str = "") -> bool: text = _combined_runtime_failure_text(failure_reason, build_log, runtime_log) if extract_pip_dependency_issue(text): return True return any(marker in text for marker in DEFINITIVE_APP_ERROR_MARKERS) def classify_runtime_recovery_need(failure_reason: str = "", build_log: str = "", runtime_log: str = "", runtime_payload: dict | None = None) -> dict: """Classify transient HF runtime states that deserve one recovery before Pi repair/fail. v198.5: HF Space scheduling/startup can stall without an actionable app traceback. In those cases the Factory should try one bounded factory reboot before declaring failure, calling Pi repair, or pausing the generated Space. """ text = _combined_runtime_failure_text(failure_reason, build_log, runtime_log, runtime_payload) stage = runtime_stage_value(runtime_payload or {}) definitive = has_definitive_app_error(failure_reason, build_log, runtime_log) scheduling = any(marker in text for marker in TRANSIENT_SCHEDULING_MARKERS) app_starting = "app_starting" in stage or stage == "starting" or "app_starting" in text startup_stall = app_starting or (any(marker in text for marker in STARTUP_STALL_MARKERS) and not definitive) if scheduling: return { "triggered": True, "failure_class": "infra_transient_scheduling", "reason": "Scheduling/capacity failure may be transient on Hugging Face infrastructure.", "recommended_action": "factory_rebuild_before_fail", "repair_should_be_deferred": True, "definitive_app_error": definitive, "runtime_stage": stage, } if startup_stall and not definitive: return { "triggered": True, "failure_class": "startup_stalled_no_definitive_traceback", "reason": "Space stayed in startup/health timeout without a definitive app traceback.", "recommended_action": "factory_rebuild_before_repair", "repair_should_be_deferred": True, "definitive_app_error": definitive, "runtime_stage": stage, } if startup_stall and definitive: return { "triggered": False, "failure_class": "startup_stalled_with_definitive_app_error", "reason": "Startup stalled, but logs contain an actionable app/dependency error; repair can proceed directly.", "recommended_action": "repair_direct", "repair_should_be_deferred": False, "definitive_app_error": definitive, "runtime_stage": stage, } return { "triggered": False, "failure_class": "not_recoverable_by_runtime_retry", "reason": "No transient scheduling/startup pattern detected.", "recommended_action": "normal_repair_protocol", "repair_should_be_deferred": False, "definitive_app_error": definitive, "runtime_stage": stage, } def write_runtime_recovery(run_dir: Path, payload: dict) -> dict: existing = load_json_if_exists(run_dir / "runtime_recovery.json") if (run_dir / "runtime_recovery.json").exists() else {} if not isinstance(existing, dict): existing = {} merged = {"schema_version": "runtime_recovery.v198_5", **existing, **payload, "updated_at": now()} write_json(run_dir / "runtime_recovery.json", merged) # startup_recovery.json remains as a convenience artifact for UI/debugging. if str(merged.get("failure_class") or "").startswith("startup") or "startup" in str(merged.get("failure_class") or ""): write_json(run_dir / "startup_recovery.json", merged) return merged def attempt_runtime_recovery_before_repair(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, failure_reason: str, build_log: str = "", runtime_log: str = "", runtime_payload: dict | None = None) -> dict: classification = classify_runtime_recovery_need(failure_reason, build_log, runtime_log, runtime_payload) recovery = write_runtime_recovery(run_dir, { **classification, "actions": [], "recovery_attempted": False, "recovery_exhausted": not classification.get("triggered"), }) if not classification.get("triggered"): return recovery append_event(events_path, "runtime_recovery", "started", "Attempting one HF runtime recovery before repair/fail", classification) write_repair_outcome( run_dir, events_path, pre_repair_recovery_attempted=True, pre_repair_recovery_action="factory_reboot", repair_deferred_until_after_rebuild=True, repair_decision="runtime_recovery_before_repair", post_repair_validation="pending", ) ok = safe_restart_space( api, target_space_id, token, run_dir, events_path, factory_reboot=True, reason=classification.get("failure_class") or "runtime_recovery", require_logs_checked=False, allow_busy=True, ) action = {"type": "factory_reboot", "attempt": 1, "requested": bool(ok)} if not ok: recovery = write_runtime_recovery(run_dir, { **classification, "actions": [action | {"result": "request_failed_or_guarded"}], "recovery_attempted": True, "recovery_exhausted": True, "result": "recovery_request_failed", }) append_event(events_path, "runtime_recovery", "failed", "HF runtime recovery request was blocked or failed", recovery) return recovery recovery = write_runtime_recovery(run_dir, { **classification, "actions": [action | {"result": "requested"}], "recovery_attempted": True, "recovery_exhausted": False, "result": "recovery_requested", }) append_event(events_path, "runtime_recovery", "success", "HF runtime recovery requested; revalidation should run before Pi repair", recovery) return recovery def validate_live_api(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, timeout_s: int = 900): append_event(events_path, "api_validation", "started", "Waiting for live HTTP /health or Gradio API to become available") deadline = time.time() + timeout_s attempt = 0 last_error = None runtime_error_count = 0 while time.time() < deadline: attempt += 1 runtime_payload = write_space_runtime(api, target_space_id, token, run_dir, events_path, attempt) stage = str(runtime_payload.get("stage") or "").upper() # Never keep waiting when the Hub already reports a BUILD_ERROR. This # is the critical observer loop: manual refresh should not be required. if runtime_stage_is_build_error(stage): raise_if_collected_build_error(target_space_id, token, run_dir, events_path, attempt=attempt, reason=f"runtime_stage={stage}", runtime_payload=runtime_payload) # While the Space is building/starting, periodically inspect build logs. # A terminal pip/build error in logs is enough to enter recovery even if # runtime state has not propagated yet. if attempt == 1 or attempt % 2 == 0 or any(marker in stage for marker in ["BUILD", "ERROR", "RUNTIME"]): raise_if_collected_build_error(target_space_id, token, run_dir, events_path, attempt=attempt, reason=f"runtime_stage={stage}", runtime_payload=runtime_payload) if "RUNTIME_ERROR" in stage: runtime_error_count += 1 last_error = f"Space runtime stage is {stage}" if runtime_error_count >= 2: raise RuntimeError(f"Space is in RUNTIME_ERROR. See logs/space_logs_runtime.txt and logs/space_logs_build.txt. Last runtime: {runtime_payload}") try: return validate_http_health(target_space_id, token, run_dir, events_path, attempt) except Exception as exc: last_error = f"HTTP /health failed: {exc}" try: return validate_gradio_api(target_space_id, token, run_dir, events_path, attempt) except Exception as exc: if "space_identity_mismatch" in str(exc): append_event(events_path, "api_validation", "failed", "Space identity mismatch is terminal; stopping live API wait", {"attempt": attempt, "runtime": runtime_payload, "error": str(exc)[:1500]}) raise last_error = (last_error or "") + f"; Gradio API failed: {exc}" append_event(events_path, "api_validation", "waiting", "Live health/API not ready yet", {"attempt": attempt, "runtime": runtime_payload, "error": last_error[-1500:] if last_error else None}) time.sleep(30) collect_space_logs(target_space_id, token, run_dir, events_path) raise RuntimeError(f"Live health/API validation did not pass before timeout: {last_error}") def is_auth_or_billing_like_error(error: str | None) -> bool: value = error or "" markers = [ "401", "402", "403", "Invalid username or password", "Unauthorized", "Repository Not Found", "payment", "billing", "quota", "grant", ] return any(marker.lower() in value.lower() for marker in markers) def request_hardware(api, target_space_id: str, hardware: str, token: str, events_path: Path, step: str, retries: int = 2): """Best-effort hardware request after Space creation. V23 tries hardware at create_repo time first. This function remains as a fallback for cases where a Space was created on CPU and the Hub later accepts a hardware switch. Auth/billing/quota errors are not retried. """ if not hardware: return {"phase": "post_create_request", "requested": False, "hardware": hardware, "ok": False, "error": "empty hardware"} last_error = None for attempt in range(1, retries + 1): try: runtime = api.request_space_hardware(repo_id=target_space_id, hardware=hardware, token=token) payload = { "phase": "post_create_request", "requested": True, "hardware": hardware, "ok": True, "attempt": attempt, "runtime_stage": getattr(getattr(runtime, "stage", None), "value", str(getattr(runtime, "stage", None))), "requested_hardware": getattr(runtime, "requested_hardware", None), "hardware_current": getattr(runtime, "hardware", None), } append_event(events_path, step, "success", f"Requested Space hardware {hardware}", payload) return payload except Exception as exc: exc_payload = exception_payload(exc) last_error = str(exc_payload.get("hf_error_message") or exc_payload.get("hf_error_detail") or exc_payload.get("error") or "")[:4000] auth_like = is_auth_or_billing_like_error(last_error) payload = {"phase": "post_create_request", "attempt": attempt, "hardware": hardware, "error": last_error, "manual_action_required": auth_like, **exc_payload} append_event(events_path, step, "failed" if auth_like or attempt == retries else "waiting", f"Could not request Space hardware {hardware}", payload) if auth_like: return {"phase": "post_create_request", "requested": True, "hardware": hardware, "ok": False, "attempts": attempt, "error": last_error, "manual_action_required": True, **exc_payload} if attempt < retries: time.sleep(8 * attempt) return {"phase": "post_create_request", "requested": True, "hardware": hardware, "ok": False, "attempts": retries, "error": last_error, "manual_action_required": False} def normalize_auto_space_hardware(value: str | None, default: str) -> str: candidate = (value or default).strip() return candidate if candidate in AUTO_SPACE_HARDWARE_CHOICES else default def build_hardware_sequence(preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, try_zero_gpu_first: bool = True) -> list[str]: sequence = [] preferred_hardware = normalize_auto_space_hardware(preferred_hardware, DEFAULT_PREFERRED_SPACE_HARDWARE) fallback_hardware = normalize_auto_space_hardware(fallback_hardware, DEFAULT_FALLBACK_SPACE_HARDWARE) planned = [] if try_zero_gpu_first: planned.append("zero-a10g") planned.append(preferred_hardware) if allow_fixed_gpu_fallback: planned.append(fallback_hardware) for hw in planned: value = (hw or "").strip() if value and value not in sequence: sequence.append(value) return sequence def is_repo_already_exists_error(error: str) -> bool: text = str(error or "").lower() return ("already" in text and ("created this space repo" in text or "already exists" in text or "repo already" in text or "name already" in text)) def collision_safe_target_space_id(target_space_id: str, attempt: int) -> str: owner, slug = target_space_id.split("/", 1) base = re.sub(r"[^A-Za-z0-9._-]+", "-", slug).strip(".-_") or "space-factory" suffix = f"-r{int(time.time()) % 100000}-{attempt}" max_slug_len = min(95, 96 - len(owner) - 1) return f"{owner}/{base[:max_slug_len - len(suffix)].rstrip('.-_')}{suffix}" def create_space_with_hardware_strategy(api, target_space_id: str, token: str, preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, events_path: Path, allow_auto_rename: bool = False, try_zero_gpu_first: bool = True): """Create a private Space and request hardware as early as possible. HF supports `space_hardware` directly on create_repo. This is the cleanest moment to request hardware because the Space does not need a second restart. If OAuth/billing/quota prevents automatic hardware selection, fall back to a normal private CPU Space and mark manual hardware as required. When the target Space name was generated automatically, a stale duplicate run id or a double-submit should not kill the build. In that one case, retry creation with a fresh suffix and return the final target Space id so the rest of the run, report and UI stay coherent. User-provided names still fail loudly if they already exist. """ sequence = build_hardware_sequence(preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first) attempts = [] current_target_space_id = target_space_id rename_attempt = 0 def maybe_rename_after_collision(error: str, phase: str, hardware: str | None = None) -> bool: nonlocal current_target_space_id, rename_attempt if not (allow_auto_rename and is_repo_already_exists_error(error) and rename_attempt < 5): return False previous = current_target_space_id rename_attempt += 1 current_target_space_id = collision_safe_target_space_id(previous, rename_attempt) append_event( events_path, "target_space_collision", "warning", "Auto-generated target Space already existed; retrying with a fresh generated name", {"phase": phase, "hardware": hardware, "previous_target_space": previous, "target_space": current_target_space_id}, ) return True def previous_create_attempt_for_current_target() -> bool: return any( str(item.get("target_space") or "") == current_target_space_id and str(item.get("phase") or "").startswith("create_repo_") for item in attempts if isinstance(item, dict) ) def adopt_existing_after_ambiguous_create(error: str, phase: str, hardware: str | None = None): """Keep one Space per run when a previous create may have succeeded. HF Hub calls can occasionally create the repo but still return an error to the client, or a hardware-at-creation ZeroGPU attempt can leave a CPU Space behind. In that situation the next hardware fallback sees "already exists" for the same target. That must not trigger an auto-rename, otherwise one run could create a second Space. """ if not (is_repo_already_exists_error(error) and previous_create_attempt_for_current_target()): return None payload = { "phase": "create_repo_existing_after_ambiguous_create", "hardware": hardware or "cpu-basic", "ok": True, "target_space": current_target_space_id, "source_error": error, "single_space_per_run": True, } append_event( events_path, "create_space", "warning", "Target Space already exists after an earlier create attempt; reusing it instead of generating another Space", payload, ) if hardware: hw_payload = request_hardware(api, current_target_space_id, hardware, token, events_path, "hardware_post_create_reuse") attempts.append({**payload, "post_create_hardware_request": hw_payload}) selected = hardware if hw_payload.get("ok") else "default-cpu-or-existing" manual = bool(hw_payload.get("manual_action_required") or not hw_payload.get("ok")) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": selected, "requested_sequence": sequence, "attempts": attempts, "manual_action_required": manual} attempts.append(payload) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": "default-cpu-or-existing", "requested_sequence": sequence, "attempts": attempts, "manual_action_required": True} for hardware in sequence: while True: try: append_event(events_path, "create_space_hardware", "started", f"Creating private Space with requested hardware {hardware}", {"target_space": current_target_space_id, "hardware": hardware}) api.create_repo( repo_id=current_target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, space_hardware=hardware, token=token, ) payload = {"phase": "create_repo_space_hardware", "hardware": hardware, "ok": True, "target_space": current_target_space_id} append_event(events_path, "create_space", "success", f"Private target Space created with requested hardware {hardware}", payload) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": hardware, "requested_sequence": sequence, "attempts": attempts + [payload], "manual_action_required": False} except Exception as exc: exc_payload = exception_payload(exc) error = str(exc_payload.get("hf_error_message") or exc_payload.get("hf_error_detail") or exc_payload.get("error") or "")[:4000] reused = adopt_existing_after_ambiguous_create(error, "create_repo_space_hardware", hardware) if reused: return reused if maybe_rename_after_collision(error, "create_repo_space_hardware", hardware): continue manual = is_auth_or_billing_like_error(error) payload = {"phase": "create_repo_space_hardware", "hardware": hardware, "ok": False, "error": error, "manual_action_required": manual, "target_space": current_target_space_id, **exc_payload} attempts.append(payload) append_event(events_path, "create_space_hardware", "failed", f"Could not create Space with requested hardware {hardware}", payload) break # Continue through the sequence: ZeroGPU quota/auth can fail while a fixed GPU # may still be worth trying. If fixed GPU also fails, we'll create CPU below. append_event(events_path, "create_space", "started", "Creating private target Space on default CPU after hardware-at-creation attempts failed", {"target_space": current_target_space_id}) while True: try: api.create_repo(repo_id=current_target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, token=token) cpu_payload = {"phase": "create_repo_default_cpu", "hardware": "cpu-basic", "ok": True, "target_space": current_target_space_id, "manual_action_required": True} append_event(events_path, "create_space", "success", "Private target Space created on default CPU; manual hardware selection may be required", cpu_payload) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": "default-cpu-or-existing", "requested_sequence": sequence, "attempts": attempts + [cpu_payload], "manual_action_required": True} except Exception as exc: exc_payload = exception_payload(exc) error = str(exc_payload.get("hf_error_message") or exc_payload.get("hf_error_detail") or exc_payload.get("error") or "")[:4000] reused = adopt_existing_after_ambiguous_create(error, "create_repo_default_cpu") if reused: return reused if maybe_rename_after_collision(error, "create_repo_default_cpu"): continue raise def _safe_json_object_from_workspace(path: Path) -> dict: payload = read_json(path, {}) return payload if isinstance(payload, dict) else {} def _first_truthy_string(*values) -> str | None: for value in values: if isinstance(value, str) and value.strip(): return value.strip() return None def _json_safe(value): try: json.dumps(value) return value except Exception: return str(value) def build_model_repo_tree_snapshot(model_id: str, siblings: list[str] | None, info=None) -> dict: """Create a compact, deterministic file-tree snapshot for Pi grounding. This is intentionally lightweight: it does not download model weights and it avoids making feasibility decisions. The goal is to give Pi structural evidence next to the model card. """ siblings = [str(s) for s in (siblings or []) if str(s)] top_level: list[str] = [] seen: set[str] = set() for name in siblings: first = name.split("/", 1)[0] display = first + "/" if "/" in name else first if display not in seen: seen.add(display) top_level.append(display) components = [] lower_names = "\n".join(siblings).lower() for label, needles in { "readme": ["readme.md"], "diffusers_model_index": ["model_index.json"], "transformer": ["transformer/", "transformer."], "text_encoder": ["text_encoder/"], "tokenizer": ["tokenizer/"], "vae": ["vae/"], "scheduler": ["scheduler/"], "lora": ["lora/"], "int8_weights": ["int8", "base_model_int8"], "whisper": ["whisper"], "vocal_separator": ["vocal_separator", "vocal-separator"], "custom_code_hint": [".py", "requirements", "environment.yml", "setup.py"], }.items(): if any(needle in lower_names for needle in needles): components.append(label) return { "schema_version": "model_repo_tree.v1", "model_id": model_id, "source": "HfApi.model_info(files_metadata=True).siblings", "file_count": len(siblings), "top_level_entries": top_level[:120], "sample_files": siblings[:240], "detected_components": components, "truncated": len(siblings) > 240, "created_at": now(), } def resolve_build_time_model_card(model_id: str, token: str | None, siblings: list[str] | None, info=None) -> dict: """Resolve the canonical model card used to ground Pi's build. Best effort and non-blocking: README.md is preferred. If it cannot be read, the worker still writes an explicit fallback file so the run remains auditable and Pi is told that the canonical card was unavailable. """ siblings = [str(s) for s in (siblings or []) if str(s)] has_readme = any(name.lower() == "readme.md" for name in siblings) source = { "schema_version": "model_card_source.v1", "model_id": model_id, "resolved_card_file": "README.md" if has_readme else None, "source": None, "used_for_pi_prompt": True, "fallback_used": False, "warnings": [], "created_at": now(), } text = "" if has_readme: try: from huggingface_hub import hf_hub_download readme_path = hf_hub_download(repo_id=model_id, filename="README.md", repo_type="model", token=token) raw = Path(readme_path).read_bytes() text = raw.decode("utf-8", errors="replace") source.update({ "source": "hf_hub_download", "card_bytes": len(raw), "card_sha256": hashlib.sha256(raw).hexdigest(), "fallback_used": False, }) except Exception as exc: source["fallback_used"] = True source["source"] = "fallback:model_info_metadata" source["warnings"].append(f"readme_download_failed:{type(exc).__name__}") else: source["fallback_used"] = True source["source"] = "fallback:model_info_metadata" source["warnings"].append("readme_not_listed_in_repo_siblings") if not text: card_data = getattr(info, "card_data", None) if info is not None else None tags = list(getattr(info, "tags", []) or []) if info is not None else [] pipeline_tag = getattr(info, "pipeline_tag", None) if info is not None else None library_name = getattr(info, "library_name", None) if info is not None else None text = dedent(f""" # Model Card Fallback for {model_id} The worker could not resolve README.md for this model during build-time grounding. Pi must treat this as an incomplete source and use Hub metadata/repo files only as fallback evidence. - pipeline_tag: {pipeline_tag} - library_name: {library_name} - tags: {tags[:80]} - card_data_present: {bool(card_data)} """).strip() + "\n" source.setdefault("card_bytes", len(text.encode("utf-8"))) source["card_sha256"] = hashlib.sha256(text.encode("utf-8")).hexdigest() return {"model_card.md": text, "model_card_source.json": source} def build_prescan_summary_from_analysis(model_analysis: dict | None, model_card_source: dict | None = None, repo_tree: dict | None = None) -> dict: model_analysis = model_analysis or {} model_card_source = model_card_source or {} repo_tree = repo_tree or {} return { "schema_version": "prescan_summary.v1", "model_id": model_analysis.get("model_id"), "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), "tags": (model_analysis.get("tags") or [])[:80], "preferred_hardware": model_analysis.get("preferred_hardware"), "fallback_hardware": model_analysis.get("fallback_hardware"), "implementation_mode": model_analysis.get("implementation_mode"), "model_card_source": { "source": model_card_source.get("source"), "resolved_card_file": model_card_source.get("resolved_card_file"), "fallback_used": model_card_source.get("fallback_used"), "warnings": model_card_source.get("warnings", []), }, "repo_tree": { "file_count": repo_tree.get("file_count"), "detected_components": repo_tree.get("detected_components", []), "top_level_entries": (repo_tree.get("top_level_entries") or [])[:80], }, "note": "Build-time grounding summary for Pi. Pre-build UI scan remains best-effort and non-blocking.", "created_at": now(), } def source_policy_markdown() -> str: return dedent(f""" # Build-time source policy `analysis_inputs/model_card.md` is the canonical model documentation for this build when it was successfully resolved from README.md. Use sources in this order: 1. `analysis_inputs/model_card.md` / README.md evidence. 2. `analysis_inputs/model_repo_tree.json` structural evidence. 3. Hub metadata and config/model_index hints. 4. Auto-generated Hugging Face snippets only as hints. Auto-generated “Use this model” snippets must not override README.md/model-card instructions. If README.md indicates custom code, torchrun, conda, ffmpeg, FlashAttention, xformers, multi-GPU, special weights, or a specific pipeline class, treat that as authoritative unless live Space logs prove otherwise. If README.md is unavailable, explicitly mention the fallback in `pi_feasibility_brief.json`. """).strip() + "\n" HF_SPACES_GIST_FALLBACK_SUMMARY = """# HF Spaces operational gist fallback summary The Worker could not provide a full remote gist snapshot. Use this built-in summary together with the original gist URL when web access is available. - Push early and validate on the real Hugging Face Space instead of relying on local assumptions. - Use live Space logs and gradio_client/API calls as the runtime source of truth. - ZeroGPU requires Gradio and a real @spaces.GPU-decorated inference function; never decorate a cheap health endpoint, and keep the same app valid on fixed GPU fallback hardware. - If the app does no local GPU work, recommend/use cpu-basic instead of ZeroGPU or a fixed GPU. - Proxy/API-only apps should use cpu-basic and must not wrap remote calls in @spaces.GPU. - Keep dependency fixes surgical: read the first actionable build error, patch minimally, rebuild, validate live. - Do not fake model outputs, static samples, or canned success responses. """ def fetch_hf_spaces_gist_snapshot() -> dict: """Best-effort snapshot of the HF Spaces operational gist for Pi grounding. This is intentionally non-blocking. The build must continue with a bundled fallback summary and URL instruction when the network, GitHub, or rate limits make the remote gist unavailable. """ fetched_at = now() if str(os.environ.get("ASF_DISABLE_GIST_FETCH") or "").strip().lower() in {"1", "true", "yes"}: return { "hf_spaces_operational_gist.md": HF_SPACES_GIST_FALLBACK_SUMMARY, "hf_spaces_gist_source.json": { "schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": False, "source": "disabled_by_env", "fallback_summary_included": True, "url_instruction_in_goal": True, "fetched_at": fetched_at, }, } try: import urllib.request req = urllib.request.Request(GIST_RAW_URL, headers={"User-Agent": "agentic-space-factory/198.1"}) with urllib.request.urlopen(req, timeout=5) as response: raw = response.read(512_000) text = raw.decode("utf-8", errors="replace").strip() if not text or len(text) < 200: raise ValueError("gist_raw_too_small") return { "hf_spaces_operational_gist.md": text + "\n", "hf_spaces_gist_source.json": { "schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": True, "source": "remote_raw_gist", "fallback_summary_included": False, "url_instruction_in_goal": False, "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), "bytes": len(raw), "fetched_at": fetched_at, }, } except Exception as exc: return { "hf_spaces_operational_gist.md": HF_SPACES_GIST_FALLBACK_SUMMARY, "hf_spaces_gist_source.json": { "schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": False, "source": "remote_failed", "fallback_summary_included": True, "url_instruction_in_goal": True, "error_type": type(exc).__name__, "error": str(exc)[:1000], "fetched_at": fetched_at, }, } def hf_spaces_guidance_goal_section() -> str: return f"""HF Spaces operational guidance / HF Spaces gist operational rules: - First read `analysis_inputs/hf_spaces_operational_gist.md`. It is the run-local snapshot or fallback summary of the HF Spaces operational guidance. - Also read `analysis_inputs/hf_spaces_gist_source.json` to know whether the full remote gist was fetched. - If `download_success=false` and you have web access, read the original gist directly: {GIST_URL} - In `pi_feasibility_brief.json` or `PI_SUMMARY.md`, include `gist_rules_used`: 1–3 concrete gist/fallback rules that affected your implementation choices. ASF-specific override to the gist: - Do not use Hugging Face Space dev mode. - Do not use SSH or rely on an interactive shell inside the target Space. - Do not propose in-Space file editing, in-Space pip installs, or manual dev-mode edits as the implementation or repair mechanism. - ASF is push-based and audit-based: Pi edits only the Worker workspace files; the Worker creates the Space, uploads snapshots, reads logs, and validates via Gradio/API. - Every implementation or repair must be representable as file changes in app.py, requirements.txt, README.md, or structured artifacts. HF Job vs target Space hardware: - The current HF Job hardware is only the environment running Worker and Pi. - It is not the target Space hardware. - Do not assume CUDA availability in the Job implies CUDA availability in the generated Space. - Do not choose the Space implementation strategy based on the Job hardware. - Recommend hardware only for the target Space, and let the Worker verify/finalize it. """ def hf_spaces_guidance_repair_section() -> str: return f"""HF Spaces operational guidance for repair: - Read `analysis_inputs/hf_spaces_operational_gist.md` if present, and `analysis_inputs/hf_spaces_gist_source.json` for source status. - If the local gist source says `download_success=false` and you have web access, read the original gist directly: {GIST_URL} - Use the gist/fallback log-first method: read the first actionable live build/runtime error, patch minimally, rebuild, and validate live. ASF-specific repair override: - Do not use dev-mode, SSH, in-Space editing, or in-Space pip installs. - Repairs must be file changes in the Worker workspace. - Preserve the original inference strategy whenever possible. - If the failure is UI-only, do not rewrite the inference backend. - If the failure is dependency-only, patch requirements minimally. - Never replace real inference with placeholders, canned text, static files, or broad exception swallowing. """ def pi_reference_dependency_goal_section() -> str: return """Reference Space Discovery — bounded and evidence-based: - Before implementing, check whether existing Hugging Face Spaces use the exact target model id, the same official organization/model family, or the same task + library stack. - Keep this exploration bounded: inspect up to 3 exact/reference Spaces and up to 3 same-family/same-stack Spaces when discoverable. Do not spend the whole build on search. - Prefer high-signal references: official Spaces, Spaces linked from the model card, Spaces from the model author/org, or Spaces that clearly load the same model id. - Inspect only implementation-relevant files when available: app.py, requirements.txt, README.md, hardware assumptions, Gradio API shape, @spaces.GPU usage, dependency pins, and known runtime workarounds. - Use reference Spaces as implementation guidance, not blind copy-paste. Do not copy secrets, tokens, private assumptions, analytics, unrelated UI, or unrelated business logic. - If no useful reference exists or web discovery is unavailable, continue with model card, repo tree, and deterministic evidence instead of blocking. - Write `REFERENCE_SPACE_FINDINGS.json` when possible. It should include: searched, exact_model_spaces, related_spaces, useful_patterns[], files_checked, applicability, risks, and no_reference_found_reason. Model Dependency Discovery — self-contained vs dependent repos: - Do not assume the target repository is a complete standalone model. Determine whether the repo is self-contained before selecting the implementation strategy. - Check `analysis_inputs/model_card.md`, repo tree/config names, model metadata, and any reference Spaces for external GitHub repos, auxiliary model ids, tokenizer/processor repos, VAE/text encoder/safety checker/guardrail models, custom Python packages, local paths in official examples, gated/private dependencies, and hardware/VRAM assumptions. - Explicitly recognize dependent model artifacts such as LoRA, QLoRA, PEFT adapters, DreamBooth LoRA, ControlNet, IP-Adapter, textual inversion, VAE/component repos, tokenizer/processor-only repos, and merged vs unmerged checkpoints. - Look for adapter/component signals such as `adapter_config.json`, `base_model`, `base_model_name_or_path`, `peft_type`, `lora_alpha`, `target_modules`, `adapter_model.safetensors`, `pytorch_lora_weights.safetensors`, `diffusers_lora`, `load_lora_weights`, `PeftModel`, `AutoPeftModelForCausalLM`, `ss_base_model_version`, "trained on", "based on", "requires", and "use with". - If the target is a LoRA/PEFT adapter or component repo, identify the required base model with evidence before attempting full inference. If the base model cannot be identified or is unavailable/gated, declare a technical blocker instead of pretending the adapter can run standalone. - Do not choose hardware from adapter size alone; include the base model and auxiliary components in VRAM/runtime reasoning. - Write `MODEL_DEPENDENCIES.json` when possible. It should include: self_contained, model_kind, base_model_required, base_model_id, adapter_application_method, dependencies[], missing_or_uncertain_dependencies[], and impact_on_strategy. ASF strategy and contract vocabulary: - Use `inference_strategy` consistently in pi_feasibility_brief.json, pi_implementation_plan.json, INFERENCE_CONTRACT.json, and PI_SUMMARY.md when available. - `inference_strategy` must be one of: diffusers_pipeline, transformers_pipeline, vllm_local_server, openai_compatible_local_api, tts_sdk_cpu, onnx_runtime, spaces_gpu, diagnostic_only, custom_subprocess_backend, unknown. - Include `strategy_reason` with concrete evidence from the model card, repo tree, reference Space, or dependency analysis. - Recommend target Space hardware with `recommended_target_space_hardware`: cpu-basic, zero-a10g, a10g-small, a10g-large, or manual_required. Include hardware_reason, requires_gpu, estimated_vram, and hardware_confidence when known. - Rules of thumb: diagnostic_only + requires_gpu=false -> cpu-basic; proxy/API-only -> cpu-basic; CPU-only SDK/ONNX/TTS -> cpu-basic; real local Diffusers/Torch CUDA inference -> ZeroGPU or fixed GPU; vLLM local server -> fixed GPU likely required; model exceeds available GPU capacity -> manual_required. - The Worker remains the final authority for hardware and validation. Your recommendation is advisory but should be evidence-backed. """ def pi_conservative_guidance_goal_section() -> str: return """Conservative implementation guidance — smallest working truthful Space: - Prefer the smallest working Space implementation that can truthfully validate the model over copying or vendoring a complex reference Space. - Reference Spaces are guidance, not authority. Use them to identify patterns, dependencies, hardware assumptions, API shapes, and known pitfalls; do not blindly copy their entire app structure, dependency pins, UI, analytics, or hardware choice. - Vendor a full reference app only when the model card or official inference path genuinely requires that app structure or package layout. Otherwise, build a minimal reproducible app around the required model call. - If a reference Space is complex but the model can be run with a smaller direct pipeline/SDK call, choose the smaller implementation and document what reference patterns were intentionally not copied. - Keep generated GPU apps ZeroGPU-first and fixed-GPU-compatible. If the model performs local GPU inference, prefer a real `@spaces.GPU`-decorated generation function for ZeroGPU, but write it so the same app.py remains valid when the Worker falls back to fixed GPU hardware because the current user has exhausted ZeroGPU quota. - Do not remove `@spaces.GPU` or refactor the app away from ZeroGPU merely because one run falls back to `a10g-large`. ZeroGPU quota/capacity is user/runtime state, not model evidence. - If importing `spaces` is optional in a local/fixed-GPU context, use a safe decorator fallback instead of crashing at import time; if CUDA is absent for a GPU-required model, show a clear UI error rather than faking output. - Expose `api_name="health"` in the top-level Gradio app whenever possible. If the app is multi-file or vendorized and health is registered in another file, document the exact file/function in INFERENCE_CONTRACT.json or PI_SUMMARY.md using `health_endpoint_location`. - Do not let reference discovery expand scope indefinitely. When evidence is mixed, prefer a conservative bootable implementation with honest blockers over a large fragile clone. """ def pi_demo_quality_goal_section() -> str: return """Demo quality and model-card promise fulfillment: - Your goal is not only to make the Space boot. Build a usable Gradio demo that truthfully reflects the model-card promise and lets a reviewer try the primary user flow. - Before finalizing the app, identify the `model_card_promise`: the task the model card claims the model can perform, the expected inputs, expected output type, and any important limitations. - Provide examples in the Gradio UI whenever full or partial real inference is implemented. Use `gr.Examples(...)` or an equivalent Gradio examples mechanism appropriate for the UI. - Examples must be task-appropriate and lightweight: text-to-image prompts for image models, text/voice/language examples for TTS, short prompts/lyrics for audio or music models, short messages for chat/text models, and realistic but small inputs for multimodal models. - Do not include fake example outputs. Examples are input presets; the output must still come from the model unless the app is explicitly diagnostic-only. - Provide a canonical smoke-test example that the Worker can later use for live validation. The example should use the same API endpoint exposed by the app and should be cheap enough for a first validation call. - The canonical smoke example should use schema-compatible values when known. For dropdown/radio choices, prefer the exact choice value shown in the Gradio UI schema, not a different Python type. - If full inference is impossible, still make the Space useful: provide a diagnostic Gradio page explaining the blocker, required hardware/dependencies, and at least one example input that would be used once the blocker is resolved. - Write `DEMO_QUALITY_CONTRACT.json` describing: model_card_promise, demo_task, primary_user_flow, examples_provided, examples[], canonical_smoke_example, real_inference_required, real_inference_implemented, fallback_or_diagnostic_only, limitations_disclosed, and promise_fulfillment_risk. - Include `canonical_smoke_example` either in `DEMO_QUALITY_CONTRACT.json` or `INFERENCE_CONTRACT.json`. It should include api_name, inputs, expected_output_type, and a short reason why this example is cheap and representative. - README.md must explain how to try the demo, list at least one example input, and disclose any limitations/blockers honestly. """ def pi_reference_dependency_repair_section() -> str: return """Reference/dependency awareness during repair: - Preserve the original inference_strategy and model dependency logic whenever possible. - If the app uses a LoRA/PEFT adapter, ControlNet, IP-Adapter, textual inversion, VAE/component repo, or another dependent artifact, preserve the base model loading/application path. Do not "fix" the app by loading the adapter/component as if it were standalone. - If repair evidence shows a missing base model, auxiliary model, external package, or gated/private dependency, document it in REPAIR_SUMMARY.md and TECHNICAL_BLOCKERS.json when full inference cannot be safely restored. - For UI-only or dependency-only failures, do not rewrite reference-Space-derived inference code unless the first actionable error requires it. """ def prepare_build_analysis_inputs(model_id: str, token: str | None, siblings: list[str] | None, info, model_analysis: dict | None = None) -> dict: card_bundle = resolve_build_time_model_card(model_id, token, siblings, info) repo_tree = build_model_repo_tree_snapshot(model_id, siblings, info) source = card_bundle["model_card_source.json"] prescan = build_prescan_summary_from_analysis(model_analysis or {}, source, repo_tree) gist_snapshot = fetch_hf_spaces_gist_snapshot() return { "model_card.md": card_bundle["model_card.md"], "model_card_source.json": source, "model_repo_tree.json": repo_tree, "prescan_summary.json": prescan, "source_policy.md": source_policy_markdown(), "hf_spaces_operational_gist.md": gist_snapshot["hf_spaces_operational_gist.md"], "hf_spaces_gist_source.json": gist_snapshot["hf_spaces_gist_source.json"], } def write_analysis_inputs_dir(base_dir: Path, analysis_inputs: dict | None) -> dict: inputs = analysis_inputs or {} target = base_dir / "analysis_inputs" target.mkdir(parents=True, exist_ok=True) written = [] defaults = { "model_card.md": "# Model Card Unavailable\n\nNo build-time model card was provided to this workspace.\n", "model_card_source.json": {"schema_version": "model_card_source.v1", "status": "missing", "fallback_used": True, "used_for_pi_prompt": True, "warnings": ["analysis_inputs_missing"]}, "model_repo_tree.json": {"schema_version": "model_repo_tree.v1", "status": "missing", "top_level_entries": [], "sample_files": [], "detected_components": []}, "prescan_summary.json": {"schema_version": "prescan_summary.v1", "status": "missing"}, "source_policy.md": source_policy_markdown(), "hf_spaces_operational_gist.md": HF_SPACES_GIST_FALLBACK_SUMMARY, "hf_spaces_gist_source.json": {"schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": False, "source": "analysis_inputs_missing", "fallback_summary_included": True, "url_instruction_in_goal": True, "fetched_at": now()}, } for name, default in defaults.items(): value = inputs.get(name, default) path = target / name if isinstance(value, (dict, list)): path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") else: path.write_text(str(value), encoding="utf-8") written.append(f"analysis_inputs/{name}") return {"dir": str(target), "written": written} def _extract_model_card_evidence_items(*objects) -> list: items = [] for obj in objects: if not isinstance(obj, dict): continue for key in ("model_card_evidence", "evidence"): value = obj.get(key) if isinstance(value, list): for item in value: if isinstance(item, dict): claim = str(item.get("claim") or item.get("name") or item.get("summary") or "").strip() evidence = str(item.get("evidence") or item.get("quote") or item.get("text") or "").strip() source = str(item.get("source") or "").strip() if claim or evidence: items.append({"claim": claim, "evidence": evidence[:500], "source": source}) elif isinstance(item, str) and item.strip(): items.append({"claim": item.strip()[:240], "evidence": item.strip()[:500], "source": "unspecified"}) return items[:12] def review_model_card_grounding(workspace: Path, brief: dict, plan: dict) -> dict: source = read_json(workspace / "analysis_inputs" / "model_card_source.json", {}) or {} card_path = workspace / "analysis_inputs" / "model_card.md" card_present = card_path.exists() and card_path.stat().st_size > 0 source_available = card_present and not bool(source.get("fallback_used")) and source.get("source") not in {None, "missing"} evidence_items = _extract_model_card_evidence_items(brief, plan) warnings = [] if not card_present: warnings.append("model_card_input_missing") elif source.get("fallback_used"): warnings.append("model_card_source_fallback_used") if not evidence_items: warnings.append("pi_model_card_evidence_missing") status = "ok" if source_available and evidence_items else ("partial" if card_present else "missing") return { "schema_version": "model_card_grounding_review.v1", "status": status, "non_blocking": True, "source_available": bool(source_available), "model_card_present": bool(card_present), "model_card_source": { "source": source.get("source"), "resolved_card_file": source.get("resolved_card_file"), "fallback_used": source.get("fallback_used"), "warnings": source.get("warnings", []), }, "pi_evidence_present": bool(evidence_items), "pi_evidence_count": len(evidence_items), "pi_evidence_sample": evidence_items[:5], "warnings": warnings, "created_at": now(), } def write_pi_planning_review(workspace: Path, run_dir: Path, events_path: Path, model_analysis: dict | None = None, implementation_mode: str = "") -> dict: """Persist Pi planning artifacts and a non-blocking worker review. v191.9 intentionally makes planning visibility-only. Pi is asked to produce a feasibility brief and implementation plan, but older/partial Pi behavior must not fail a run that would otherwise work. The worker records whether those artifacts exist, mirrors them under planning/, and keeps the final authority in the existing deterministic gates. """ model_analysis = model_analysis or {} planning_dir = run_dir / "planning" planning_dir.mkdir(parents=True, exist_ok=True) brief_path = workspace / "pi_feasibility_brief.json" plan_path = workspace / "pi_implementation_plan.json" source_usage_path = workspace / "PI_SOURCE_USAGE.json" brief = _safe_json_object_from_workspace(brief_path) plan = _safe_json_object_from_workspace(plan_path) source_usage = _safe_json_object_from_workspace(source_usage_path) brief_present = bool(brief) plan_present = bool(plan) source_usage_present = bool(source_usage) if brief_present: write_json(planning_dir / "pi_feasibility_brief.json", brief) else: write_json(planning_dir / "pi_feasibility_brief.json", { "schema_version": "pi_feasibility_brief.v1", "status": "missing", "non_blocking": True, "message": "Pi did not produce pi_feasibility_brief.json; continuing with existing deterministic worker gates.", }) if plan_present: write_json(planning_dir / "pi_implementation_plan.json", plan) else: write_json(planning_dir / "pi_implementation_plan.json", { "schema_version": "pi_implementation_plan.v1", "status": "missing", "non_blocking": True, "message": "Pi did not produce pi_implementation_plan.json; continuing with existing deterministic worker gates.", }) if source_usage_present: write_json(planning_dir / "PI_SOURCE_USAGE.json", source_usage) else: write_json(planning_dir / "PI_SOURCE_USAGE.json", { "schema_version": "pi_source_usage.v198_25", "status": "missing", "non_blocking": True, "message": "Pi did not produce PI_SOURCE_USAGE.json; continuing with deterministic worker gates.", }) declared_strategy = _first_truthy_string( brief.get("recommended_strategy"), brief.get("strategy"), plan.get("strategy"), plan.get("implementation_strategy"), ) or "unspecified" should_attempt_full = brief.get("should_attempt_full_inference") if should_attempt_full is None: should_attempt_full = plan.get("should_attempt_full_inference") worker_recommendation = "continue_existing_flow" warnings: list[str] = [] if not brief_present: warnings.append("missing_pi_feasibility_brief") if not plan_present: warnings.append("missing_pi_implementation_plan") if not source_usage_present: warnings.append("missing_pi_source_usage") grounding_review = review_model_card_grounding(workspace, brief, plan) warnings.extend(grounding_review.get("warnings", [])) if declared_strategy in {"boot_only_blocker", "boot_only_blocker_or_manual_refactor", "manual_hardware_required"} or should_attempt_full is False: worker_recommendation = "respect_plan_if_contract_declares_blocker" elif declared_strategy in {"full_inference", "proceed_full_inference", "attempt_full_inference"} or should_attempt_full is True: worker_recommendation = "proceed_full_inference_with_existing_gates" review = { "schema_version": "worker_plan_review.v1", "status": "ok" if brief_present and plan_present else "partial", "non_blocking": True, "implementation_mode": implementation_mode, "model_id": model_analysis.get("model_id"), "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), "planning_artifacts_present": { "pi_feasibility_brief": brief_present, "pi_implementation_plan": plan_present, "pi_source_usage": source_usage_present, }, "pi_source_usage": source_usage if source_usage_present else {"status": "missing", "non_blocking": True}, "declared_strategy": declared_strategy, "should_attempt_full_inference": should_attempt_full, "worker_recommendation": worker_recommendation, "warnings": warnings, "model_card_grounding": grounding_review, "authority_note": "Pi planning/source usage is advisory in v198.25; deterministic worker gates, INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation remain authoritative.", "created_at": now(), } write_json(planning_dir / "worker_plan_review.json", review) write_json(planning_dir / "model_card_grounding_review.json", grounding_review) append_event(events_path, "pi_planning_review", review["status"], "Pi feasibility and implementation planning reviewed", review) return review def _recipe_file_names(repo_tree: dict | None, model_analysis: dict | None = None) -> list[str]: repo_tree = repo_tree if isinstance(repo_tree, dict) else {} model_analysis = model_analysis if isinstance(model_analysis, dict) else {} names: list[str] = [] for key in ("sample_files", "top_level_entries"): values = repo_tree.get(key) if isinstance(values, list): names.extend(str(v) for v in values if str(v).strip()) siblings = model_analysis.get("siblings") if isinstance(siblings, list): names.extend(str(v) for v in siblings if str(v).strip()) seen: set[str] = set() out: list[str] = [] for name in names: clean = str(name).strip().strip("/") if clean and clean not in seen: seen.add(clean) out.append(clean) return out def _recipe_extract_base_model(card: str, model_analysis: dict | None = None) -> str: text = card or "" candidates = [] # Model card frontmatter often contains `base_model: owner/name`. for pattern in [ r"(?im)^\s*base_model\s*:\s*['\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", r"(?im)^\s*base[_ -]?model[_ -]?id\s*:\s*['\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", r"(?i)base model(?:\s+is|\s*:)?\s+[`'\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", r"(?i)based on\s+[`'\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", ]: candidates.extend(re.findall(pattern, text)) if isinstance(model_analysis, dict): for key in ("base_model", "base_model_id"): value = model_analysis.get(key) if isinstance(value, str): candidates.append(value) for value in candidates: clean = str(value or "").strip().strip("`'\".,;)") if "/" in clean: return clean return "" def _recipe_extract_trigger_words(card: str) -> list[str]: text = card or "" triggers: list[str] = [] for pattern in [ r"(?i)trigger\s+word[s]?\s*[:\-]?\s*[`'\"]?([^`'\"\n,;]+)", r"(?i)trigger\s*[:\-]?\s*[`'\"]?([^`'\"\n,;]+)", ]: for match in re.findall(pattern, text): for part in re.split(r"[,/]|\band\b", str(match)): clean = part.strip().strip("`'\".:- ") if clean and len(clean) <= 80 and clean not in triggers: triggers.append(clean) # Common LoRA trigger tokens are often shown in examples but not labelled. for token in re.findall(r"\b(?:sdxl|flux|lora)[-_][A-Za-z0-9_.-]{3,}\b", text, flags=re.IGNORECASE): if token not in triggers: triggers.append(token) return triggers[:8] def _recipe_weight_files(files: list[str]) -> list[str]: return [name for name in files if name.lower().endswith((".safetensors", ".ckpt", ".bin", ".pt", ".pth", ".gguf", ".onnx"))] def _recipe_first_file(files: list[str], suffixes: tuple[str, ...]) -> str: for name in files: if name.lower().endswith(suffixes): return name return "" def classify_model_recipe_family(model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> dict: """Return the worker-owned model-family decision used to constrain Pi. This is metadata/readme/repo-tree based and intentionally conservative. It does not decide terminal feasibility by itself; it gives Pi and later guards an authoritative recipe to compare against. """ model_analysis = model_analysis or {} analysis_inputs = analysis_inputs or {} card = str(analysis_inputs.get("model_card.md") or "") card_low = card.lower() tags = [str(t).lower() for t in (model_analysis.get("tags") or []) if str(t).strip()] pipeline_tag = str(model_analysis.get("pipeline_tag") or "").lower() library_name = str(model_analysis.get("library_name") or "").lower() repo_tree = analysis_inputs.get("model_repo_tree.json") if isinstance(analysis_inputs.get("model_repo_tree.json"), dict) else {} components = [str(c).lower() for c in (repo_tree.get("detected_components") or [])] files = _recipe_file_names(repo_tree, model_analysis) lower_files = [f.lower() for f in files] weight_files = _recipe_weight_files(files) has_model_index = any(f.endswith("model_index.json") or f == "model_index.json" for f in lower_files) or "diffusers_model_index" in components has_diffusers_components = bool({"transformer", "text_encoder", "tokenizer", "vae", "scheduler"} & set(components)) has_lora = ( "lora" in components or "lora" in tags or "adapter" in tags or "peft" in tags or "lora" in card_low or any("lora" in f for f in lower_files) ) has_sdxl = any(x in card_low for x in ["sdxl", "stable diffusion xl", "stable-diffusion-xl"]) or any("sdxl" in t for t in tags) or "xl" in model_id.lower() has_flux = "flux" in model_id.lower() or "fluxpipeline" in card_low or any("flux" in t for t in tags) has_gguf = any(f.endswith(".gguf") for f in lower_files) or "gguf" in tags has_onnx = any(f.endswith(".onnx") for f in lower_files) or "onnx" in tags has_single_file_checkpoint = any(f.endswith((".safetensors", ".ckpt")) for f in lower_files) and not has_model_index and not has_diffusers_components expected = "image" if pipeline_tag in {"text-to-image", "image-to-image"} or "text-to-image" in tags or "diffusers" in tags else "text" if pipeline_tag in {"text-to-video", "image-to-video"}: expected = "video" elif pipeline_tag in {"text-to-audio", "text-to-speech", "audio-to-audio"}: expected = "audio" elif pipeline_tag in {"automatic-speech-recognition"}: expected = "text" family = "generic_model" artifact_role = "unknown" loader_strategy = "inspect_model_card_then_select_loader" base_model_id = _recipe_extract_base_model(card, model_analysis) adapter_id = "" weight_name = _recipe_first_file(weight_files, (".safetensors", ".bin", ".pt", ".pth")) evidence: list[str] = [] known_failure_signatures: list[dict] = [] smoke_strategy = "generic_minimal_smoke" required_code_markers: list[str] = [] guard_expectations: list[str] = [] if has_gguf: family = "gguf_llamacpp" artifact_role = "quantized_runtime" loader_strategy = "llama_cpp_or_compatible_gguf_runtime" expected = "text" if expected == "text" else expected smoke_strategy = "short_text_generation_smoke" required_code_markers = ["gguf_runtime_or_llama_cpp_loader", "/generate"] evidence.append("repo_contains_gguf_weight") elif has_onnx: family = "onnx_runtime_model" artifact_role = "onnx_graph" loader_strategy = "onnxruntime_session_or_task_specific_pipeline" smoke_strategy = "task_specific_onnx_smoke" required_code_markers = ["onnxruntime_or_transformers_onnx_loader", "/generate"] evidence.append("repo_contains_onnx_weight") elif (library_name == "diffusers" or "diffusers" in tags or pipeline_tag in {"text-to-image", "image-to-image", "text-to-video", "image-to-video"}) and has_lora and has_sdxl and not has_model_index: family = "diffusers_sdxl_lora_adapter" artifact_role = "adapter" adapter_id = model_id loader_strategy = "sdxl_base_from_pretrained_or_single_file_then_load_lora_weights" expected = "image" smoke_strategy = "sdxl_lora_trigger_word_low_steps_image_smoke" required_code_markers = ["StableDiffusionXLPipeline", "load_lora_weights", "/generate", "@spaces.GPU_if_zero_gpu"] guard_expectations = ["base_model_resolved", "adapter_loaded", "health_exposes_model_error"] known_failure_signatures.append({"match": "CLIPTextModel object has no attribute text_model", "failure_class": "sdxl_lora_text_encoder_mismatch", "repair_hint": "base_or_loader_incompatible; try alternate SDXL base loader or dependency matrix"}) evidence.extend(["lora_marker_present", "sdxl_marker_present", "no_diffusers_model_index"]) elif (library_name == "diffusers" or "diffusers" in tags or pipeline_tag in {"text-to-image", "image-to-image", "text-to-video", "image-to-video"}) and has_flux: family = "diffusers_flux_pipeline" artifact_role = "full_model" loader_strategy = "FluxPipeline.from_pretrained" expected = "image" smoke_strategy = "diffusers_image_low_steps_schema_aware_smoke" required_code_markers = ["FluxPipeline", "/generate", "@spaces.GPU_if_zero_gpu"] guard_expectations = ["health_exposes_model_error", "generate_endpoint_exposed"] known_failure_signatures.append({"match": "FluxPipeline", "failure_class": "diffusers_flux_runtime_error", "repair_hint": "preserve FluxPipeline.from_pretrained and reduce smoke steps/resolution"}) evidence.append("flux_marker_present") elif library_name == "diffusers" or "diffusers" in tags or pipeline_tag in {"text-to-image", "image-to-image", "text-to-video", "image-to-video"}: family = "diffusers_full_pipeline" if has_model_index or has_diffusers_components else ("diffusers_single_file_checkpoint" if has_single_file_checkpoint else "diffusers_pipeline_or_checkpoint") artifact_role = "full_model" if has_model_index or has_diffusers_components else ("single_file_checkpoint" if has_single_file_checkpoint else "unknown_diffusers_artifact") loader_strategy = "DiffusionPipeline_or_family_specific_from_pretrained" if family != "diffusers_single_file_checkpoint" else "family_pipeline.from_single_file" expected = "image" if expected == "text" else expected smoke_strategy = "diffusers_media_low_cost_smoke" required_code_markers = ["diffusers_pipeline_loader", "/generate"] if expected in {"image", "video"}: required_code_markers.append("@spaces.GPU_if_zero_gpu") evidence.append("diffusers_or_media_task_marker_present") elif library_name == "transformers" or "transformers" in tags or pipeline_tag: family = "transformers_pipeline" artifact_role = "full_model_or_processor_bundle" loader_strategy = "transformers_pipeline_or_auto_model_for_task" smoke_strategy = "task_specific_transformers_smoke" required_code_markers = ["transformers_loader", "/generate_or_predict"] evidence.append("transformers_or_pipeline_tag_marker_present") hardware = { "preferred": "zero-a10g" if expected in {"image", "video", "audio"} or family.startswith("diffusers") else (model_analysis.get("preferred_hardware") or "cpu-basic"), "fallback": "a10g-large" if expected in {"image", "video", "audio"} or family.startswith("diffusers") else (model_analysis.get("fallback_hardware") or "cpu-basic"), "try_zero_gpu_first": bool(expected in {"image", "video", "audio"} or family.startswith("diffusers")), "allow_fixed_gpu_fallback": bool(expected in {"image", "video", "audio"} or family.startswith("diffusers")), } return { "model_family": family, "artifact_role": artifact_role, "loader_strategy": loader_strategy, "base_model_id": base_model_id, "adapter_id": adapter_id, "adapter_weight_name": weight_name if artifact_role == "adapter" else "", "primary_weight_name": weight_name if artifact_role != "adapter" else "", "trigger_words": _recipe_extract_trigger_words(card), "expected_output_type": expected, "primary_endpoint": "/generate" if expected in {"image", "video", "audio", "text"} else "/predict", "health_endpoint": "/health", "health_contract": { "endpoint": "/health", "required_fields": ["status", "model_ready", "pipeline_ready", "model_family", "loader_strategy", "last_error", "expected_output_type"], "cheap": True, "must_not_load_model": True, "model_ready_false_exposes_last_error": True, }, "generation_contract": { "endpoint": "/generate" if expected in {"image", "video", "audio", "text"} else "/predict", "must_fail_cleanly_when_not_ready": True, "must_surface_model_load_error": True, "no_placeholder_success": True, }, "hardware_strategy": hardware, "smoke_test_strategy": smoke_strategy, "required_code_markers": required_code_markers, "guard_expectations": guard_expectations, "known_failure_signatures": known_failure_signatures, "evidence": evidence, "repo_weight_files_sample": weight_files[:20], } APP_RUNTIME_HEALTH_REQUIRED_FIELDS = ["status", "model_ready", "pipeline_ready", "model_family", "loader_strategy", "last_error", "expected_output_type"] def build_app_runtime_contract(recipe: dict | None) -> dict: recipe = recipe if isinstance(recipe, dict) else {} health_contract = recipe.get("health_contract") if isinstance(recipe.get("health_contract"), dict) else {} generation_contract = recipe.get("generation_contract") if isinstance(recipe.get("generation_contract"), dict) else {} required_fields = health_contract.get("required_fields") if isinstance(health_contract.get("required_fields"), list) else APP_RUNTIME_HEALTH_REQUIRED_FIELDS return { "schema_version": "app_runtime_contract.v198_26_30", "authority": "worker_required_generated_app_contract", "model_family": recipe.get("model_family") or "generic_model", "artifact_role": recipe.get("artifact_role") or "unknown", "loader_strategy": recipe.get("loader_strategy") or "inspect_model_card_then_select_loader", "expected_output_type": recipe.get("expected_output_type") or "any", "primary_endpoint": recipe.get("primary_endpoint") or generation_contract.get("endpoint") or "/generate", "health_endpoint": recipe.get("health_endpoint") or health_contract.get("endpoint") or "/health", "required_health_fields": [str(x) for x in required_fields if str(x).strip()], "health_must_be_cheap": True, "health_must_not_load_model": True, "generate_must_fail_cleanly_when_not_ready": True, "generate_must_surface_model_load_error": True, "generate_must_not_return_placeholder_success": True, "smoke_test_strategy": recipe.get("smoke_test_strategy") or "generic_minimal_smoke", "known_failure_signatures": recipe.get("known_failure_signatures") or [], "notes": [ "/health must return a structured payload, not only a transport OK.", "When the model or pipeline is unavailable, /health must set model_ready=false or pipeline_ready=false and include last_error.", "/generate must expose concrete model-load/runtime errors instead of TypeError/None placeholders.", ], } def load_app_runtime_contract(workspace: Path, run_dir: Path | None = None) -> dict: for base in [workspace, run_dir]: if not base: continue path = base / "APP_RUNTIME_CONTRACT.json" if path.exists(): payload = load_json_if_exists(path) if isinstance(payload, dict) and payload: return payload recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else {} if (not isinstance(recipe, dict) or not recipe) and run_dir and (run_dir / "MODEL_RECIPE.json").exists(): recipe = load_json_if_exists(run_dir / "MODEL_RECIPE.json") return build_app_runtime_contract(recipe if isinstance(recipe, dict) else {}) def _app_text_has_health_endpoint(app_text: str) -> bool: low = str(app_text or "").lower() return bool('/health' in low or 'api_name="health"' in low or "api_name='health'" in low or re.search(r"def\s+health\s*\(", app_text or "")) def _app_text_mentions_primary_endpoint(app_text: str, endpoint: str) -> bool: endpoint = normalize_api_name(endpoint or "/generate") if endpoint == "/generate": return _has_generate_api_endpoint(app_text) bare = endpoint.lstrip("/") return bool(endpoint in app_text or f'api_name="{bare}"' in app_text or f"api_name='{bare}'" in app_text or f'api_name="{endpoint}"' in app_text or f"api_name='{endpoint}'" in app_text) def app_runtime_contract_guard(workspace: Path, run_dir: Path, events_path: Path, *, expected_output_type: str = "", reason: str = "pre_upload") -> dict: """Verify the v198.26.30 family-aware generated app contract before upload. This guard is intentionally static and cheap. It does not prove inference success; it proves Pi produced an app that exposes the right health and generation surfaces for the live validator and future recipe-aware repair. """ app_path = workspace / "app.py" app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" contract = read_inference_contract(workspace) runtime_contract = load_app_runtime_contract(workspace, run_dir) recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") if not isinstance(recipe, dict): recipe = {} required_health_fields = runtime_contract.get("required_health_fields") if isinstance(runtime_contract.get("required_health_fields"), list) else APP_RUNTIME_HEALTH_REQUIRED_FIELDS required_health_fields = [str(x) for x in required_health_fields if str(x).strip()] missing_health_fields = [field for field in required_health_fields if field not in app_text] expected = normalize_runtime_expected_output_type(expected_output_type or contract.get("expected_output_type") or runtime_contract.get("expected_output_type") or recipe.get("expected_output_type") or "") primary = normalize_api_name(str(contract.get("primary_api_name") or contract.get("primary_endpoint") or runtime_contract.get("primary_endpoint") or recipe.get("primary_endpoint") or "/generate")) full_inference = contract.get("full_inference_implemented") is True blockers = load_json_if_exists(workspace / "TECHNICAL_BLOCKERS.json") if (workspace / "TECHNICAL_BLOCKERS.json").exists() else {} try: blockers_count = len(blockers.get("blockers") or []) if isinstance(blockers, dict) and isinstance(blockers.get("blockers"), list) else int(contract.get("blockers_count") or 0) except Exception: blockers_count = 0 has_health = _app_text_has_health_endpoint(app_text) has_primary = _app_text_mentions_primary_endpoint(app_text, primary) not_ready_markers = ["last_error", "model_ready", "pipeline_ready", "not ready", "not_ready", "not loaded", "not_loaded", "load_error", "pipeline_error", "model_error"] has_not_ready_error_path = any(marker in app_text.lower() for marker in not_ready_markers) problems: list[str] = [] repairable: list[str] = [] if not has_health: problems.append("app_runtime_contract_missing_health_endpoint") repairable.append("app_runtime_contract_missing_health_endpoint") if missing_health_fields: problems.append("app_runtime_contract_missing_structured_health_fields") repairable.append("app_runtime_contract_missing_structured_health_fields") if full_inference and primary and primary != "/health" and not has_primary: problems.append("app_runtime_contract_missing_primary_generation_endpoint") repairable.append("app_runtime_contract_missing_primary_generation_endpoint") if full_inference and not has_not_ready_error_path: problems.append("app_runtime_contract_missing_not_ready_error_path") repairable.append("app_runtime_contract_missing_not_ready_error_path") if expected in {"image", "video", "audio"} and full_inference and primary == "/health": problems.append("app_runtime_contract_primary_endpoint_is_health_for_media") repairable.append("app_runtime_contract_primary_endpoint_is_health_for_media") payload = { "schema_version": "app_runtime_contract_guard.v198_26_30", "reason": reason, "passed": not problems, "problems": problems, "repairable_unresolved_problems": repairable, "model_family": runtime_contract.get("model_family") or recipe.get("model_family") or contract.get("model_family") or "", "artifact_role": runtime_contract.get("artifact_role") or recipe.get("artifact_role") or contract.get("artifact_role") or "", "loader_strategy": runtime_contract.get("loader_strategy") or recipe.get("loader_strategy") or contract.get("loader_strategy") or "", "expected_output_type": expected, "primary_endpoint": primary, "health_endpoint": runtime_contract.get("health_endpoint") or "/health", "required_health_fields": required_health_fields, "missing_health_fields": missing_health_fields, "has_health_endpoint": has_health, "has_primary_endpoint": has_primary, "has_not_ready_error_path": has_not_ready_error_path, "full_inference_implemented": full_inference, "blockers_count": blockers_count, "technical_blockers_present": bool(blockers_count), "runtime_contract": runtime_contract, } write_json(run_dir / "app_runtime_contract_guard.json", payload) append_event(events_path, "app_runtime_contract_guard", "success" if payload["passed"] else "failed", "Checked family-aware /health and /generate app contract", payload) return payload def recipe_runtime_alignment_guard(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload") -> dict: """Check generated app shape against the worker-owned MODEL_RECIPE. v198.26.30 keeps this recipe alignment diagnostic non-fatal; APP_RUNTIME_CONTRACT.json is enforced separately by the pre-upload guard. """ recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else {} if not isinstance(recipe, dict) or not recipe: recipe = load_json_if_exists(run_dir / "MODEL_RECIPE.json") if (run_dir / "MODEL_RECIPE.json").exists() else {} app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore") if (workspace / "app.py").exists() else "" family = str(recipe.get("model_family") or "") expected = normalize_runtime_expected_output_type(str(recipe.get("expected_output_type") or "")) mismatches: list[str] = [] satisfied: list[str] = [] if family == "diffusers_sdxl_lora_adapter": for marker, label in [("load_lora_weights", "missing_lora_loader"), ("StableDiffusionXLPipeline", "missing_sdxl_pipeline"), ("api_name=\"generate\"", "missing_generate_endpoint")]: alt = marker.replace('"', "'") if marker in app_text or alt in app_text: satisfied.append(label.replace("missing_", "has_")) else: mismatches.append(label) if not recipe.get("base_model_id"): mismatches.append("sdxl_lora_base_model_unresolved") elif family == "diffusers_flux_pipeline": if "FluxPipeline" in app_text: satisfied.append("has_flux_pipeline") else: mismatches.append("missing_flux_pipeline") if expected in {"image", "video", "audio"} and not _has_generate_api_endpoint(app_text): mismatches.append("media_recipe_missing_generate_endpoint") payload = { "schema_version": "recipe_runtime_alignment_guard.v198_26_30", "reason": reason, "non_blocking": True, "recipe_present": bool(recipe), "model_family": family, "expected_output_type": expected, "mismatches": mismatches, "satisfied": satisfied, "passed": not mismatches, } write_json(run_dir / "recipe_runtime_alignment_guard.json", payload) append_event(events_path, "recipe_runtime_alignment_guard", "success" if not mismatches else "warning", "Checked generated runtime against worker-owned model recipe", payload) return payload def build_model_recipe(model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> dict: model_analysis = model_analysis or {} analysis_inputs = analysis_inputs or {} card = str(analysis_inputs.get("model_card.md") or "") card_low = card.lower() repo_tree = analysis_inputs.get("model_repo_tree.json") if isinstance(analysis_inputs.get("model_repo_tree.json"), dict) else {} family = classify_model_recipe_family(model_id, model_analysis, analysis_inputs) dep_hints = [] for pkg in ["diffusers", "transformers", "sentencepiece", "protobuf", "accelerate", "safetensors", "peft", "imageio", "imageio-ffmpeg", "torchvision", "spaces", "gradio", "onnxruntime", "llama-cpp-python"]: if pkg.lower() in card_low or pkg in json.dumps(repo_tree)[:20000].lower(): dep_hints.append(pkg) # Family-derived dependencies are authoritative hints even when the card omits them. if str(family.get("model_family")) == "diffusers_sdxl_lora_adapter": for pkg in ["diffusers", "safetensors", "peft", "transformers", "accelerate", "spaces"]: if pkg not in dep_hints: dep_hints.append(pkg) if str(family.get("model_family")) == "diffusers_flux_pipeline": for pkg in ["diffusers", "transformers", "accelerate", "safetensors", "spaces"]: if pkg not in dep_hints: dep_hints.append(pkg) pipeline_hints = [] for name in re.findall(r"\b[A-Za-z0-9_]*(?:Pipeline|Model|Processor|Tokenizer)\b", card)[:40]: if name not in pipeline_hints: pipeline_hints.append(name) if family.get("model_family") == "diffusers_sdxl_lora_adapter" and "StableDiffusionXLPipeline" not in pipeline_hints: pipeline_hints.insert(0, "StableDiffusionXLPipeline") if family.get("model_family") == "diffusers_flux_pipeline" and "FluxPipeline" not in pipeline_hints: pipeline_hints.insert(0, "FluxPipeline") constraints = [] if "divisible by 32" in card_low or "divisible_by_32" in card_low: constraints.append("resolution_divisible_by_32") if "8k+1" in card_low or "8 *" in card_low and "frames" in card_low: constraints.append("frames_8k_plus_1") if "bf16" in card_low or "bfloat16" in card_low: constraints.append("bf16_or_bfloat16_hint") if "fp16" in card_low or "float16" in card_low: constraints.append("fp16_hint") return { "schema_version": "model_recipe.v198_26_30", "authority": "worker_authoritative_runtime_planning", "purpose": "Worker-owned model-family recipe. Pi must implement code consistent with this recipe or explicitly write TECHNICAL_BLOCKERS.json; live validation remains the final success authority.", "model_id": model_id, "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), "tags": model_analysis.get("tags", [])[:50] if isinstance(model_analysis.get("tags"), list) else [], "repo_components": repo_tree.get("detected_components", []), "recommended_pipeline_or_class_hints": pipeline_hints[:20], "dependency_hints": dep_hints, "input_constraints_hints": constraints, "model_family": family.get("model_family"), "artifact_role": family.get("artifact_role"), "loader_strategy": family.get("loader_strategy"), "base_model_id": family.get("base_model_id"), "adapter_id": family.get("adapter_id"), "adapter_weight_name": family.get("adapter_weight_name"), "primary_weight_name": family.get("primary_weight_name"), "trigger_words": family.get("trigger_words", []), "expected_output_type": family.get("expected_output_type"), "primary_endpoint": family.get("primary_endpoint"), "health_endpoint": family.get("health_endpoint"), "hardware_strategy": family.get("hardware_strategy"), "smoke_test_strategy": family.get("smoke_test_strategy"), "required_code_markers": family.get("required_code_markers", []), "guard_expectations": family.get("guard_expectations", []), "known_failure_signatures": family.get("known_failure_signatures", []), "app_runtime_contract": build_app_runtime_contract(family), "recipe_evidence": family.get("evidence", []), "repo_weight_files_sample": family.get("repo_weight_files_sample", []), "source_priority": ["MODEL_RECIPE.json", "analysis_inputs/model_card.md", "analysis_inputs/model_repo_tree.json", "analysis_inputs/source_policy.md", "analysis_inputs/hf_spaces_operational_gist.md"], "warning": "Pi must implement a runtime matching this recipe. If the model card contradicts it, write the contradiction in pi_feasibility_brief.json and do not silently change model family or output type.", } def build_context_index(analysis_inputs: dict | None = None) -> dict: analysis_inputs = analysis_inputs or {} files = [] for name, description in [ ("analysis_inputs/model_card.md", "Canonical README/model card resolved by the worker when available."), ("analysis_inputs/model_repo_tree.json", "Model repository file tree and detected components."), ("analysis_inputs/prescan_summary.json", "Deterministic worker prescan summary."), ("analysis_inputs/source_policy.md", "Source priority and snippet distrust policy."), ("analysis_inputs/hf_spaces_operational_gist.md", "HF Spaces operational guidance snapshot/fallback."), ("MODEL_RECIPE.json", "Compact worker-generated guide. Start here, then verify in full sources."), ("APP_RUNTIME_CONTRACT.json", "Mandatory generated-app contract for structured /health, /generate error behavior, and family-aware smoke validation."), ("SOURCE_PRIORITY.md", "Short source ordering guide for Pi."), ]: files.append({"path": name, "description": description}) return {"schema_version": "context_index.v198_25", "purpose": "Keep Pi fully armed at initial build while making source navigation explicit.", "files": files} def write_initial_pi_context_artifacts(workspace: Path, model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> list[str]: recipe = build_model_recipe(model_id, model_analysis, analysis_inputs) index = build_context_index(analysis_inputs) app_runtime_contract = build_app_runtime_contract(recipe) source_priority = """# Source priority for Pi Start with `MODEL_RECIPE.json`. It is the worker-owned runtime plan for model family, artifact role, loader strategy, expected output type, endpoint, smoke strategy, and hardware. Then apply `APP_RUNTIME_CONTRACT.json`, which is the mandatory generated-app contract for structured `/health`, `/generate` error behavior, and family-aware smoke validation. Treat both as authoritative unless the model card/repo tree directly contradicts them; if so, write the contradiction explicitly in planning artifacts. Verify critical facts in this order: 1. `MODEL_RECIPE.json` worker-owned runtime plan and required code markers. 2. `APP_RUNTIME_CONTRACT.json` mandatory generated-app health/generation contract. 3. `analysis_inputs/model_card.md` official/model-card instructions and examples. 4. `analysis_inputs/model_repo_tree.json` repo structure, configs, component names, weights. 5. `analysis_inputs/source_policy.md` trust rules for snippets and generated examples. 6. `analysis_inputs/hf_spaces_operational_gist.md` HF Spaces operational constraints. For initial build, you are expected to use the rich context. For later repairs, ASF will provide narrower task packets. """ write_json(workspace / "MODEL_RECIPE.json", recipe) write_json(workspace / "APP_RUNTIME_CONTRACT.json", app_runtime_contract) write_json(workspace / "CONTEXT_INDEX.json", index) (workspace / "SOURCE_PRIORITY.md").write_text(source_priority, encoding="utf-8") return ["MODEL_RECIPE.json", "APP_RUNTIME_CONTRACT.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md"] def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None): workspace.mkdir(parents=True, exist_ok=True) model_analysis = model_analysis or {} pipeline_tag = model_analysis.get("pipeline_tag") library_name = model_analysis.get("library_name") tags = model_analysis.get("tags", [])[:40] siblings = model_analysis.get("siblings", [])[:60] analysis_inputs_result = write_analysis_inputs_dir(workspace, analysis_inputs) initial_context_artifacts = write_initial_pi_context_artifacts(workspace, model_id, model_analysis, analysis_inputs) initial_recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if not isinstance(initial_recipe, dict): initial_recipe = {} app_py = f"""import gradio as gr from huggingface_hub import model_info, list_repo_files MODEL_ID = {model_id!r} TARGET_SPACE_ID = {target_space_id!r} MODEL_FAMILY = {str(initial_recipe.get("model_family") or "generic_model")!r} LOADER_STRATEGY = {str(initial_recipe.get("loader_strategy") or "inspect_model_card_then_select_loader")!r} EXPECTED_OUTPUT_TYPE = {str(initial_recipe.get("expected_output_type") or "any")!r} LAST_ERROR = "Initial scaffold has not loaded the model yet. Pi must replace it with a model-specific runtime or write TECHNICAL_BLOCKERS.json." def health(): return {{ "status": "unavailable", "model_ready": False, "pipeline_ready": False, "model_family": MODEL_FAMILY, "loader_strategy": LOADER_STRATEGY, "last_error": LAST_ERROR, "expected_output_type": EXPECTED_OUTPUT_TYPE, "model_id": MODEL_ID, "target_space_id": TARGET_SPACE_ID, "stage": "initial-scaffold", "note": "Pi should replace this scaffold with a model-specific demo while preserving this structured health contract.", }} def generate(prompt="Hello from Agentic Space Factory"): raise RuntimeError(LAST_ERROR) with gr.Blocks(title="Generated Model Space — Agentic Space Factory") as demo: gr.Markdown("# Generated Model Space — Agentic Space Factory") gr.Markdown(f"Private generated Space for `{{MODEL_ID}}`.") gr.JSON(label="Health", value=health(), every=None) gr.Button("Health check").click(fn=health, inputs=None, outputs=gr.JSON(), api_name="health") prompt = gr.Textbox(label="Input", value="Hello from Agentic Space Factory") output = gr.Textbox(label="Output") prompt.submit(fn=generate, inputs=prompt, outputs=output, api_name="generate") gr.Button("Run").click(fn=generate, inputs=prompt, outputs=output, api_name="generate") if __name__ == "__main__": demo.launch() """ (workspace / "app.py").write_text(app_py, encoding="utf-8") req = """gradio>=6.0.0 huggingface_hub>=0.34.0,<2.0.0 spaces transformers>=4.45.0,<6.0.0 diffusers accelerate safetensors torch pillow numpy requests """ (workspace / "requirements.txt").write_text(req, encoding="utf-8") readme = f"""--- title: Generated Model Space sdk: gradio app_file: app.py python_version: "3.10" suggested_hardware: {preferred_hardware or fallback_hardware or "cpu-basic"} short_description: "Generated model demo" --- # Generated Model Space — Agentic Space Factory Private generated Space for `{model_id}`. This Space is created by Agentic Space Factory. It should remain private until manually reviewed. """ (workspace / "README.md").write_text(readme, encoding="utf-8") analysis_json = json.dumps({"pipeline_tag": pipeline_tag, "library_name": library_name, "tags": tags, "siblings": siblings}, indent=2, ensure_ascii=False) goal = f"""You are Pi running inside a Hugging Face Job for Agentic Space Factory. Goal: build the best possible private Hugging Face Space demo for an arbitrary model card. MODEL_ID: {model_id} TARGET_SPACE_ID: {target_space_id} IMPLEMENTATION_MODE: {implementation_mode} MODEL_METADATA: ```json {analysis_json} ``` BUILD-TIME MODEL ANALYSIS INPUTS: - `analysis_inputs/model_card.md`: canonical model card / README resolved by the worker when available. - `analysis_inputs/model_card_source.json`: source, hash, fallback status, and warnings for the model card. - `analysis_inputs/model_repo_tree.json`: structural file-tree snapshot from Hub metadata. - `analysis_inputs/prescan_summary.json`: compact deterministic worker summary. - `analysis_inputs/source_policy.md`: source priority and snippet distrust policy. - `analysis_inputs/hf_spaces_operational_gist.md`: run-local HF Spaces operational gist snapshot or fallback summary. - `analysis_inputs/hf_spaces_gist_source.json`: fetch status, URL fallback, and hash/source metadata for the gist guidance. Before writing code, first read `MODEL_RECIPE.json`, `APP_RUNTIME_CONTRACT.json`, `CONTEXT_INDEX.json`, and `SOURCE_PRIORITY.md`, then read `analysis_inputs/model_card.md`, `analysis_inputs/model_repo_tree.json`, `analysis_inputs/source_policy.md`, `analysis_inputs/hf_spaces_operational_gist.md`, and `analysis_inputs/hf_spaces_gist_source.json`. `MODEL_RECIPE.json` is the worker-owned runtime plan for model family, artifact role, loader strategy, output type, endpoint, health contract, smoke test strategy, and hardware. `APP_RUNTIME_CONTRACT.json` is the mandatory generated-app contract for structured `/health`, `/generate` not-ready behavior, and family-aware validation. Treat it as authoritative unless you can cite a direct contradiction from the model card or repo tree. If you disagree with the recipe, write the contradiction explicitly in `pi_feasibility_brief.json`; do not silently change family, output type, or endpoint. Treat the model card as canonical for model facts when it was resolved from README.md. Auto-generated Hugging Face “Use this model” snippets are hints only and must not override README.md/model-card instructions. {hf_spaces_guidance_goal_section()} {pi_reference_dependency_goal_section()} {pi_conservative_guidance_goal_section()} {pi_demo_quality_goal_section()} {pi_tooling_context_note()} Non-negotiable safety and product constraints: - The target Space must remain private. - Do not delete any user resources. - Do not print secrets or tokens. - Work only inside the current workspace. - The wrapper will create the private Space, request allowed hardware best-effort, upload files, and validate the live app. Do not create/delete repos yourself in this builder worker. - Preserve a cheap health endpoint named `health` with `api_name="health"`. It must not load weights, run GPU work, or download large files. If model state is exposed, use explicit values such as `model_ready: true/false`; never report `ok` when the pipeline is unavailable. - Platform dependency policy: Gradio, Gradio Client, Hugging Face Hub, Spaces, and hf_xet are owned by Agentic Space Factory / the Hugging Face Spaces runtime. Do not pin or downgrade them. Never write `gradio==...`, `gradio<=...`, `gradio~=...`, `gradio-client==...`, `gradio_client==...`, or stale exact pins for `huggingface_hub`, `spaces`, or `hf_xet`. - If generated code needs an older Gradio API, update the code to modern Gradio instead of downgrading Gradio. - Follow the gist dependency method for model-specific packages: pin only when useful to reduce pip resolver backtracking or when the model card/build logs explicitly require a version; do not cargo-cult every version from examples. - Do not pin torch or torchaudio unless unavoidable and explicitly justified; Spaces provide the PyTorch stack. If torchvision is needed, prefer leaving it unpinned so it can resolve against the managed torch runtime. - If model code is not pip-installable, vendor the necessary code into the Space repo instead of referencing local paths, editable installs, or clone-time side effects in requirements.txt. - Do not run local venv installs, broad `pip install --dry-run`, or dependency checks as proof of compatibility; only the live Space build/logs are authoritative. Use `python -m py_compile app.py` only as a cheap syntax check. - Use huggingface_hub>=0.34.0,<2.0.0 unless a newer Transformers path requires a compatible newer range. The worker will normalize platform-owned dependency lines before upload. - README.md frontmatter must remain valid; if it uses short_description, it must be 60 characters or fewer. Planning contract: - Before committing to a full implementation strategy, produce `pi_feasibility_brief.json` and `pi_implementation_plan.json`. These files are advisory artifacts for the worker and must not contain secrets. - `pi_feasibility_brief.json` should include: complexity, expected_runtime, zerogpu_feasible, requires_custom_code, multi_gpu_risk, dependency_risk, recommended_strategy, `inference_strategy`, strategy_reason, minimum_hardware, recommended_target_space_hardware, hardware_reason, should_attempt_full_inference, evidence, `gist_rules_used`, and `model_card_evidence` with 2–5 concrete facts from `analysis_inputs/model_card.md` when available. - `pi_implementation_plan.json` should include: selected_strategy, `inference_strategy`, intended_files, endpoints, dependency_strategy, hardware_strategy, recommended_target_space_hardware, smoke_test_plan, canonical_smoke_example, demo_examples_plan, fallback_plan, blocker_policy, `gist_rules_used` if not already in the brief, and any README/model-card facts that materially affect implementation choices. - If the model appears high-risk or not portable, say so in the plan rather than silently attempting a fragile fake success. The worker will still use INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation as the final authority. Implementation contract: - If IMPLEMENTATION_MODE is `full-inference-gated`, you are not allowed to silently replace generation with a placeholder and call it success. - Try to implement the closest real inference path for the model card using evidence from `analysis_inputs/model_card.md`, model metadata, config files, and repo files. If a generic snippet conflicts with the README/model-card, the README/model-card wins. - You may choose an appropriate Gradio UI for the task: text, image, audio, video, multimodal, embeddings, classification, etc. - If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint. - Your app must conform to `MODEL_RECIPE.json`: preserve `model_family`, `artifact_role`, `loader_strategy`, `expected_output_type`, `primary_endpoint`, and `smoke_test_strategy`. For adapters such as SDXL LoRA, implement the base-model + adapter relationship explicitly; do not load an adapter repo as if it were a standalone full model. - Your app must conform to `APP_RUNTIME_CONTRACT.json`: expose `/health` with literal structured fields `status`, `model_ready`, `pipeline_ready`, `model_family`, `loader_strategy`, `last_error`, and `expected_output_type`. Health must be cheap and must not trigger model loading or GPU work. - `/generate` must fail cleanly when the model/pipeline is not ready: raise or return a structured error containing the concrete load/runtime error, not a vague `TypeError`, `None`, or UI-only message. If the model load hits `CLIPTextModel.text_model` or another family-specific known error, surface that exact error in both `/health.last_error` and `/generate`. - If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function and preserve fixed-GPU compatibility. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement. If a pipeline load fails, `/generate` should raise the concrete load error instead of a vague silent `None` state. - If the model requires special dependencies, include them only when needed and document risks. - Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels/Kernel Hub where relevant, Transformers AttentionInterface, Diffusers attention processors, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs. - Native kernel policy: if the model card or logs mention flash-attn, custom attention kernels, xformers, Triton kernels, fused ops, or CUDA/C++ extensions, do not blindly add source-built native packages to requirements.txt. Prefer runtime-compatible drop-in backends or compatible wheels first. Use `kernels` only when it directly matches the needed operation and the target hardware/runtime is plausible; document the selected backend in PI_SUMMARY.md and INFERENCE_CONTRACT.json. - If real inference is impossible or unsafe in a Space, write TECHNICAL_BLOCKERS.json with concrete evidence for every blocker. Deliverables: - pi_feasibility_brief.json must summarize feasibility and risk before/alongside implementation, including `model_card_evidence` when model-card evidence is available. - pi_implementation_plan.json must summarize the selected implementation plan, endpoints, dependencies, target Space hardware recommendation, and validation approach. - REFERENCE_SPACE_FINDINGS.json should summarize bounded reference Space discovery when possible; if none were found or web discovery is unavailable, say so without blocking. - MODEL_DEPENDENCIES.json should summarize whether the repo is standalone or dependent, including LoRA/PEFT/adapter/component/base-model dependencies when present. - DEMO_QUALITY_CONTRACT.json should summarize the model-card promise, the primary user flow, Gradio examples, canonical_smoke_example, whether real inference is implemented, and any promise fulfillment risks. - app.py must boot on Hugging Face Spaces. - app.py must expose health/api_name="health" and return the structured `APP_RUNTIME_CONTRACT.json` fields. - If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic. - If real generation is not implemented, write TECHNICAL_BLOCKERS.json with: full_inference_implemented=false, blockers[], evidence[], minimum_runtime, and suggested_next_step. - Write INFERENCE_CONTRACT.json with: full_inference_implemented, inference_strategy, health_endpoint, primary_api_name, expected_output_type, validation_level, requires_gpu, recommended_target_space_hardware, estimated_vram, blockers_count, model_family, artifact_role, loader_strategy, app_runtime_contract_version, health_required_fields, generate_not_ready_behavior, and recipe_conformance_notes. - README.md must explain the runtime strategy, task, limitations, how to test, and at least one example input when the demo is not purely diagnostic. - Write PI_SOURCE_USAGE.json with: used_model_card, used_official_code_examples, used_repo_tree, used_hf_spaces_gist, critical_claims_verified[], and any skipped_sources[]. This is an audit artifact only; it must not block implementation. - Write a concise PI_SUMMARY.md with what you changed, whether full inference is implemented, and any `gist_rules_used` if they are not already captured in JSON. """ (workspace / "GOAL.md").write_text(goal, encoding="utf-8") return ["app.py", "requirements.txt", "README.md", "GOAL.md"] + initial_context_artifacts + analysis_inputs_result.get("written", []) def sanitize_readme_metadata(workspace: Path, events_path: Path): """Ensure the generated Space README always keeps valid HF metadata. Pi is allowed to rewrite README.md, but Spaces require a valid YAML frontmatter block for sdk/app_file. If Pi drops it, the Space can fail with a Hub configuration error before the app even builds. Treat this as a factory invariant, not a Pi preference. """ readme_path = workspace / "README.md" if not readme_path.exists(): readme_path.write_text("# Generated Model Space\n", encoding="utf-8") text = readme_path.read_text(encoding="utf-8", errors="ignore") body = text metadata = {} changed = False if text.startswith("---"): parts = text.split("---", 2) if len(parts) >= 3: _, frontmatter, body = parts for line in frontmatter.splitlines(): if ":" not in line: continue key, value = line.split(":", 1) metadata[key.strip()] = value.strip() else: body = text changed = True else: changed = True raw_python_version = metadata.get("python_version") or DEFAULT_SPACE_PYTHON_VERSION python_version, python_version_changed, python_version_reason = normalize_space_python_version(raw_python_version) required = { "title": metadata.get("title") or "Generated Model Space", "sdk": "gradio", "app_file": "app.py", "python_version": python_version, } if python_version_changed: changed = True suggested = metadata.get("suggested_hardware") if suggested: required["suggested_hardware"] = suggested short = metadata.get("short_description") or "Generated model demo" if len(short.strip('"\'')) > 60: short = "Generated model demo" required["short_description"] = short ordered_keys = ["title", "sdk", "app_file", "python_version", "suggested_hardware", "short_description"] lines = [] for key in ordered_keys: if key in required and required[key]: value = str(required[key]).strip() if key in {"title", "short_description", "python_version"} and not (value.startswith('"') or value.startswith("'")): value = json.dumps(value, ensure_ascii=False) lines.append(f"{key}: {value}") normalized_body = body.lstrip("\n") or "# Generated Model Space\n" new_text = "---\n" + "\n".join(lines) + "\n---\n\n" + normalized_body if new_text != text: readme_path.write_text(new_text, encoding="utf-8") append_event( events_path, "metadata_sanitize", "success", "Ensured README Space metadata", { "metadata_keys": [k for k in ordered_keys if k in required], "python_version_before": str(raw_python_version).strip().strip("\"'"), "python_version_after": python_version, "python_version_normalized": bool(python_version_changed), "python_version_reason": python_version_reason, }, ) def _read_workspace_text_if_exists(path: Path, *, limit: int = 20000) -> str: try: if path.exists() and path.is_file(): text = path.read_text(encoding="utf-8", errors="ignore") return text[-limit:] except Exception: return "" return "" def detect_transformers5_dependency_requirement(text: str = "") -> dict: """Detect concrete evidence that a dependency requires Transformers 5.x. This is deliberately narrow: ASF should not globally allow every generated app to float to an unknown major release, but it must not reintroduce a `<5.0.0` upper bound after pip logs or repair artifacts prove a package needs `transformers>=5.0.0`. """ raw = text or "" low = raw.lower() if "transformers>=5" not in low and "transformers >=5" not in low and "transformers 5" not in low: return {} has_blocking_upper_bound = bool(re.search(r"transformers[^\n#]*<\s*5(?:\.0(?:\.0)?)?", raw, flags=re.I)) has_requires_phrase = any( phrase in low for phrase in [ "requires transformers>=5", "depends on transformers>=5", "cosmos-guardrail", "requires transformers >=5", ] ) if has_requires_phrase or has_blocking_upper_bound: return { "package": "transformers", "requires": "transformers>=5.0.0", "conflicting_upper_bound": has_blocking_upper_bound, "evidence": raw[-2000:], } return {} def workspace_allows_transformers_major5(workspace: Path, extra_evidence: str = "") -> dict: """Return evidence allowing ASF to relax its default Transformers <5 guard. Sources are intentionally local/auditable artifacts from the current repair pass. This makes the upload-time sanitizer idempotent: once a dependency repair brief/summary documents the concrete Transformers 5 requirement, a later upload sanitize pass must preserve the relaxed requirement. """ chunks = [extra_evidence or ""] for name in [ "DEPENDENCY_ERROR_BRIEF.md", "REPAIR_PLAN.md", "REPAIR_SUMMARY.md", "requirements_policy.json", "requirements.txt", ]: chunks.append(_read_workspace_text_if_exists(workspace / name)) evidence = "\n".join(chunks) issue = detect_transformers5_dependency_requirement(evidence) if issue: issue["source"] = "repair_artifacts_or_dependency_logs" return issue def relax_transformers_upper_bound_for_repair(workspace: Path, events_path: Path, evidence_text: str = "") -> bool: """Apply the narrow Cosmos-style deterministic requirements fix. Only removes `<5` from a Transformers requirement when concrete local logs or repair artifacts show a dependency requiring Transformers 5.x. This function does not add Transformers if absent; the regular sanitizer handles the base policy line using the same evidence. """ issue = workspace_allows_transformers_major5(workspace, evidence_text) if not issue: return False req_path = workspace / "requirements.txt" if not req_path.exists(): return False before = req_path.read_text(encoding="utf-8", errors="ignore") changed = False out_lines: list[str] = [] for line in before.splitlines(): stripped = line.strip() name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") if stripped and not stripped.startswith("#") else "" if name == "transformers" and re.search(r"<\s*5(?:\.0(?:\.0)?)?", line, flags=re.I): # Keep any lower-bound/extras/comments simple and deterministic for now. out_lines.append("transformers>=4.51.0") changed = True else: out_lines.append(line) if not changed: return False after = "\n".join(out_lines).rstrip() + "\n" req_path.write_text(after, encoding="utf-8") append_event( events_path, "requirements_repair", "success", "Relaxed Transformers <5 upper bound because dependency logs require Transformers 5.x", {"issue": issue, "from_contains": "<5", "to": "transformers>=4.51.0"}, ) return True def workspace_publishable_fingerprint(root: Path) -> dict: """Return a hash map for files that matter to the generated Space. Internal Pi/repair artifacts are excluded so a repair cannot count as applied merely because REPAIR_PLAN.md or REPAIR_SUMMARY.md was written. """ if not root.exists(): return {} result: dict[str, str] = {} for path in sorted(root.rglob("*")): if not path.is_file(): continue try: rel = path.relative_to(root) except Exception: continue if not is_publishable_workspace_file(rel): continue try: result[str(rel)] = hashlib.sha256(path.read_bytes()).hexdigest() except Exception: result[str(rel)] = "" return result def write_repair_diff_artifact(run_dir: Path, events_path: Path, before_dir: Path, after_root: Path, *, category: str = "", expected_files: list[str] | None = None) -> dict: before = workspace_publishable_fingerprint(before_dir) after = workspace_publishable_fingerprint(after_root) before_keys = set(before) after_keys = set(after) changed = sorted(k for k in before_keys & after_keys if before.get(k) != after.get(k)) added = sorted(after_keys - before_keys) removed = sorted(before_keys - after_keys) expected_files = expected_files or [] expected_touched = sorted(set(expected_files) & set(changed + added + removed)) payload = { "schema_version": "1.0", "category": category, "changed_files": changed, "added_files": added, "removed_files": removed, "has_publishable_diff": bool(changed or added or removed), "expected_files": expected_files, "expected_files_touched": expected_touched, } repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "repair_diff.json", payload) append_event( events_path, "repair_diff", "success" if payload["has_publishable_diff"] else "failed", "Computed publishable before/after repair diff", payload, ) return payload def expected_repair_files_from_classification(classification: dict | None = None, dependency_issue: dict | None = None) -> list[str]: category = (classification or {}).get("category") or "" files: list[str] = [] if category == "dependency_error" or dependency_issue: files.extend(["requirements.txt", "requirements_policy.json"]) if category in {"publishable_app_py_syntax_error", "full_inference_media_contract_missing_generate_endpoint", "zero_gpu_full_inference_without_spaces_gpu_decorator", "pre_upload_runtime_template_integrity", "sdxl_lora_text_encoder_mismatch", "model_pipeline_not_loaded", "model_loading_error"}: files.append("app.py") return list(dict.fromkeys(files)) def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path, dependency_evidence: str = ""): """Normalize broad known-dangerous base and platform dependencies before upload. Do not try to solve every dependency conflict here. Do not try to solve every model-specific dependency conflict here. Pi is responsible for reading concrete build logs and patching requirements when a Space build fails. The Factory only prevents obviously unsafe broad ranges and owns the Gradio/HF platform stack so old generated pins cannot conflict with the current Spaces runtime; concrete pip conflicts remain Pi repair work. """ req_path = workspace / "requirements.txt" if not req_path.exists(): return raw = req_path.read_text(encoding="utf-8", errors="ignore") lines = [line.rstrip() for line in raw.splitlines()] prefix_lines: list[str] = [] package_lines: list[str] = [] for line in lines: stripped = line.strip() if not stripped: continue if stripped.startswith("--") or stripped.startswith("-f "): prefix_lines.append(line) else: package_lines.append(line) # Minimal base pins only. Do not globally pin diffusers here: newer model # cards may legitimately need a recent Diffusers release. If diffusers causes # a pip conflict, Pi is responsible for concrete build-log repairs. Pi must repair from the concrete build log. transformers5_issue = workspace_allows_transformers_major5(workspace, dependency_evidence) keep_transformers_policy, transformers_policy_reason = workspace_should_keep_transformers_policy(workspace, dependency_evidence) default_policy: dict[str, str] = { # Spaces currently installs platform Gradio with a modern hub floor. Keep # the app-side hub range compatible with that platform stack instead of # forcing pip to backtrack into a known impossible 4.x Transformers path. "huggingface-hub": "huggingface_hub>=1.2.0,<2.0.0", "transformers": "transformers>=4.51.0", } transformers_policy = "transformers>=5.0.0" if transformers5_issue else default_policy["transformers"] policy: dict[str, str] = { "huggingface-hub": default_policy["huggingface-hub"], } if keep_transformers_policy: policy["transformers"] = transformers_policy # Platform-owned dependencies are normalized more aggressively than model # dependencies. Gradio/HF runtime packages must stay current with Spaces and # Agentic Space Factory validation; Pi should patch app.py for modern Gradio # rather than pinning or downgrading this stack. platform_policy: dict[str, str] = { "gradio": "gradio", "gradio-client": "gradio-client", "huggingface-hub": policy["huggingface-hub"], "spaces": "spaces>=0.30", "hf-xet": "hf_xet", } aliases = { "huggingface_hub": "huggingface-hub", "huggingface-hub": "huggingface-hub", "transformers": "transformers", "gradio": "gradio", "gradio-client": "gradio-client", "gradio_client": "gradio-client", "spaces": "spaces", "hf-xet": "hf-xet", "hf_xet": "hf-xet", } platform_owned = set(platform_policy) seen_policy: set[str] = set() seen_platform: set[str] = set() filtered: list[str] = [] changed = False removed_platform_pins: list[str] = [] normalized_platform_lines: list[dict] = [] for line in package_lines: stripped = line.strip() if stripped.startswith("#") or "://" in stripped or stripped.startswith((".", "/")): filtered.append(line) continue name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") canonical = aliases.get(name) if canonical in platform_owned: normalized = platform_policy[canonical] if stripped != normalized: changed = True removed_platform_pins.append(stripped) normalized_platform_lines.append({"from": stripped, "to": normalized, "package": canonical}) if canonical not in seen_platform: filtered.append(normalized) seen_platform.add(canonical) if canonical in policy: seen_policy.add(canonical) else: changed = True continue if canonical in policy: if stripped != policy[canonical]: changed = True if canonical not in seen_policy: filtered.append(policy[canonical]) seen_policy.add(canonical) else: changed = True continue if canonical == "transformers" and not keep_transformers_policy: changed = True normalized_platform_lines.append({"from": stripped, "to": "removed", "package": "transformers", "reason": transformers_policy_reason}) continue filtered.append(line) stable_policy_lines: list[str] = [] for canonical in ["huggingface-hub", "transformers"]: if canonical in policy and canonical not in seen_policy: stable_policy_lines.append(policy[canonical]) changed = True injected_platform_lines: list[str] = [] app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore").lower() if (workspace / "app.py").exists() else "" if ("import gradio" in app_text or "from gradio" in app_text or "gr." in app_text) and "gradio" not in seen_platform: injected_platform_lines.append(platform_policy["gradio"]) seen_platform.add("gradio") changed = True if ("gradio_client" in app_text or "from gradio_client" in app_text) and "gradio-client" not in seen_platform: injected_platform_lines.append(platform_policy["gradio-client"]) seen_platform.add("gradio-client") changed = True torch_added = False if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines + injected_platform_lines, "torch"): stable_policy_lines.append("torch>=2.0.0") torch_added = True changed = True app_deps_text = "" deps_path = workspace / "MODEL_DEPENDENCIES.json" if deps_path.exists(): app_deps_text = deps_path.read_text(encoding="utf-8", errors="ignore").lower() lora_peft_added = False if any(x in (app_text + "\n" + app_deps_text) for x in ["load_lora_weights", "peft_type", "lora_adapter", "diffusers_lora", "adapter_config.json"]): candidate_lines = filtered + stable_policy_lines + injected_platform_lines if not requirements_has_package(candidate_lines, "peft"): stable_policy_lines.append("peft") lora_peft_added = True changed = True new_lines = prefix_lines + stable_policy_lines + injected_platform_lines + filtered new = "\n".join(line for line in new_lines if line.strip()) + "\n" if new != raw: changed = True if changed: req_path.write_text(new, encoding="utf-8") policy_payload = { "schema_version": "1.0", "status": "normalized", "platform_owned_dependencies": ["gradio", "gradio-client", "huggingface_hub", "spaces", "hf_xet"], "removed_pins": removed_platform_pins, "normalized_platform_lines": normalized_platform_lines, "injected_platform_lines": injected_platform_lines, "base_policy": {"huggingface_hub": policy["huggingface-hub"], "transformers": policy.get("transformers")}, "transformers_policy_applied": bool(keep_transformers_policy), "transformers_policy_reason": transformers_policy_reason, "transformers_upper_bound_relaxed": bool(transformers5_issue), "transformers_upper_bound_relaxation_evidence": transformers5_issue, "base_policy_reason": "Avoid forcing Transformers 4.x under modern Gradio/HF Hub platform constraints; do not inject or preserve Transformers for CPU-only/TTS/ONNX apps without code or dependency evidence; require torch when app.py imports torch.", "torch_added": torch_added, "torch_policy": "torch>=2.0.0", "torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present", "peft_added_for_diffusers_lora": lora_peft_added, "peft_policy": "peft is required by Diffusers load_lora_weights for LoRA adapters", "model_dependency_policy": "Model-specific pins are preserved unless they hit a known deterministic guardrail; concrete conflicts remain Pi repair work from build logs.", "reason": "Gradio/HF runtime dependencies are owned by Agentic Space Factory / Spaces runtime and must not be downgraded by generated requirements.", } write_json(workspace / "requirements_policy.json", policy_payload) append_event( events_path, "requirements_sanitize", "success", "Normalized platform-owned Gradio/HF dependencies and broad base dependencies; concrete pip conflicts remain Pi repair work", policy_payload, ) def useful_log_signals(text: str) -> list[str]: low = (text or "").lower() signals = [] patterns = { "python_exception": ["traceback", "modulenotfounderror", "importerror", "runtimeerror", "valueerror", "typeerror"], "dependency_resolution": ["resolutionimpossible", "no matching distribution", "could not find a version", "pip subprocess", "metadata-generation-failed"], "gradio_api": ["api_name", "endpoint", "unexpected keyword", "too many arguments", "not enough arguments", "no api found"], "model_loading": ["from_pretrained", "model_index", "safetensors", "does not appear to have", "failed to load", "pipeline"], "hardware_memory": ["cuda out of memory", "outofmemoryerror", "oom", "not enough memory"], "auth_access": ["401", "403", "gated", "unauthorized", "forbidden", "repository not found"], "zerogpu": ["zerogpu", "spaces.gpu", "duration"], } for name, needles in patterns.items(): if any(n in low for n in needles): signals.append(name) return signals def log_quality(build_log: str = "", runtime_log: str = "", failure_reason: str = "") -> str: combined = f"{build_log}\n{runtime_log}\n{failure_reason}".strip() low = combined.lower() if not combined or len(combined) < 80: return "empty" no_reason_markers = [ "no logs", "logs are empty", "no reason", "unknown error", "runtime error", "build error", "error: none", "status=error", ] if any(m in low for m in no_reason_markers) and not useful_log_signals(combined): return "no_reason" if useful_log_signals(combined): return "useful" return "partial" def classify_repair_failure(failure_reason: str, build_log: str = "", runtime_log: str = "") -> dict: """Classify a failed live validation before asking Pi to choose an action. This classifier is deliberately conservative. It gives Pi and the Factory evidence quality and a suggested class, but the diagnosis step must still make a bounded decision before any patch is allowed. """ text = f"{failure_reason}\n{build_log[-6000:]}\n{runtime_log[-6000:]}".lower() quality = log_quality(build_log, runtime_log, failure_reason) phase = "unknown" if "build" in text or "pip" in text or "container" in text: phase = "space_build" if "runtime" in text or "health" in text or "client.predict" in text or "api" in text: phase = "space_runtime" if "generation" in text or "smoke" in text or "expected output" in text: phase = "api_validation" checks = [ ("cuda_oom", ["cuda out of memory", "outofmemoryerror", "oom", "not enough memory"], "Prefer hardware/manual action or memory reduction; do not fake inference."), ("dependency_error", ["resolutionimpossible", "could not find a version", "no matching distribution", "dependency conflict", "pip subprocess", "metadata-generation-failed"], "Fix requirements with compatible, modern pins."), ("import_error", ["modulenotfounderror", "importerror", "cannot import name"], "Fix imports or requirements without hiding the error."), ("gradio_api_mismatch", ["no api found", "api_name", "endpoint", "unexpected keyword", "too many arguments", "not enough arguments"], "Preserve the expected Gradio endpoint and align input/output schema."), ("model_loading_error", ["from_pretrained", "safetensors", "model_index", "pipeline", "failed to load", "does not appear to have"], "Re-read the model card usage and fix the pipeline loading path."), ("hf_auth_error", ["401", "403", "gated", "unauthorized", "forbidden", "repository not found"], "Do not bypass auth; report gated/private access if needed."), ("space_boot_timeout", ["timed out", "timeout", "space did not become ready", "sleeping"], "Inspect logs first; if logs are empty, wait/rebuild same code before patching."), ("wrong_output_type", ["wrong output", "expected output", "invalid output", "not an image", "not text"], "Return the expected output type from real inference."), ("zero_gpu_duration_error", ["duration", "spaces.gpu", "zerogpu", "zero gpu"], "Tune @spaces.GPU(duration=...) from observed smoke-test latency when available."), ] category = "unknown_runtime_error" recommendation = "Use the Space logs, Gradio API schema and smoke error to choose the smallest truthful action." for code, needles, rec in checks: if any(n in text for n in needles): category = code recommendation = rec break gradio_kwarg_issue = extract_gradio_unexpected_kwarg(f"{failure_reason}\\n{build_log}\\n{runtime_log}") if gradio_kwarg_issue: category = "gradio_component_api_error" phase = "space_runtime" quality = "useful" recommendation = "Patch the unsupported Gradio component keyword without changing the inference backend." if quality in {"empty", "no_reason"} and category == "unknown_runtime_error": category = "hf_runtime_flake" if phase != "space_build" else "hf_build_flake" recommendation = "Logs are not actionable. Prefer wait/inspect or a same-code factory rebuild before any code patch." dependency_issue = extract_pip_dependency_issue(f"{failure_reason}\n{build_log}\n{runtime_log}") failure_class = category if dependency_issue: category = "dependency_error" phase = "space_build" quality = "useful" failure_class = dependency_issue.get("failure_class") or "dependency_resolution_conflict" if dependency_issue.get("platform_conflict", {}).get("detected"): recommendation = "Structural dependency platform conflict detected. Attempt only requirements.txt / requirements_policy.json repair, or declare dependency_platform_conflict without app-code edits." else: recommendation = "Patch requirements.txt / requirements_policy.json from the first pip resolver error; do not rebuild same code first." return { "category": category, "failure_class": failure_class, "failure_phase": phase, "logs_quality": quality, "signals": useful_log_signals(text), "recommendation": recommendation, "dependency_issue": dependency_issue, "gradio_kwarg_issue": gradio_kwarg_issue, "repair_type": gradio_kwarg_issue.get("repair_type", "") if gradio_kwarg_issue else "", "gist_alignment": "read first actionable pip error, patch minimally, rebuild, validate live" if dependency_issue else "standard blockage diagnosis", } def workspace_file_inventory(workspace: Path, max_files: int = 80) -> list[str]: files = [] for path in sorted(workspace.rglob("*")): if not path.is_file(): continue if any(part in {".git", ".cache", "node_modules", "__pycache__"} for part in path.parts): continue try: files.append(str(path.relative_to(workspace))) except Exception: files.append(path.name) if len(files) >= max_files: break return files def write_incident_brief(workspace: Path, run_dir: Path, *, target_space_id: str, model_id: str, pi_model: str, failure_reason: str, build_log: str, runtime_log: str, classification: dict, implementation_mode: str, expected_output_type: str, iteration: int, budgets: dict | None = None) -> str: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) inventory = workspace_file_inventory(workspace) budgets = budgets or {} brief = f"""# Agentic Space Factory incident brief This is a blockage diagnosis pass for an existing generated Hugging Face Space. Pi must use the HF Spaces gist method: read logs first, act once, choose the cheapest useful iteration rung, and verify on the live Space. ## Target - Model ID: `{model_id}` - Target Space: `{target_space_id}` - Pi model: `{pi_model}` - Implementation mode: `{implementation_mode}` - Expected output type: `{expected_output_type}` - Diagnosis iteration: `{iteration}` - HF Space run logs endpoint: `https://huggingface.co/api/spaces/{target_space_id}/logs/run` - HF Space build logs endpoint: `https://huggingface.co/api/spaces/{target_space_id}/logs/build` ## Current classifier hints ```json {json.dumps(classification, indent=2, ensure_ascii=False)} ``` ## Dependency build error policy If `dependency_issue` is non-empty, this is an actionable pip build failure. Prefer `patch_code` with a minimal `requirements.txt` change. Do not choose `factory_rebuild_same_code` until a dependency patch has been attempted. ## HF tooling/token context ```json {json.dumps(load_json_if_exists(run_dir / "token_context.json"), indent=2, ensure_ascii=False)} ``` ## Remaining action budget ```json {json.dumps(budgets, indent=2, ensure_ascii=False)} ``` ## Observed failure ```text {failure_reason[:5000]} ``` ## Actionable error excerpt ```text {extract_actionable_error_excerpt(failure_reason + chr(10) + build_log + chr(10) + runtime_log, max_lines=120)} ``` Full logs remain archived under `logs/space_logs_build.txt`, `logs/space_logs_runtime.txt`, and `logs/space_logs_run.txt`. Do not inspect full logs unless this excerpt is insufficient. ## Workspace files ```text {chr(10).join(inventory)} ``` ## Allowed decisions You must choose exactly one action: - `wait_for_logs`: logs are empty/late; wait and collect again. - `inspect_more_logs`: enough uncertainty remains; collect logs/API status again without changing code. - `factory_rebuild_same_code`: likely HF build/runtime flake; re-upload the same workspace to force a rebuild, then revalidate. - `patch_code`: logs are actionable and point to a code/dependency/API/model-loading issue. - `request_manual_hardware`: issue is hardware/quota/restricted GPU/memory and should not be patched away. - `declare_technical_blocker`: no safe automated action remains. ## Hard rules - Do not choose `patch_code` when logs are empty or no-reason. The Factory must first fetch real HF Space logs from `/logs/run` or `/logs/build`, or use a concrete validator/API-schema signal. - Do not fake inference or replace model output with placeholders. - Do not patch hardware/quota/authorization failures as code bugs. - If logs are actionable, identify the first real error, not only the last line. - If using `factory_rebuild_same_code`, do not change files. ## Required output Write `REPAIR_DECISION.json` in the workspace root with this schema: ```json {{ "action": "wait_for_logs|inspect_more_logs|factory_rebuild_same_code|patch_code|request_manual_hardware|declare_technical_blocker", "confidence": "low|medium|high", "reason": "short explanation", "evidence": ["specific evidence"], "patch_allowed": false, "requires_manual_hardware": false }} ``` """ (repair_dir / "INCIDENT_BRIEF.md").write_text(brief, encoding="utf-8") (workspace / "INCIDENT_BRIEF.md").write_text(brief, encoding="utf-8") write_json(repair_dir / "classification.json", classification) return brief def extract_json_object(text: str) -> dict: if not text: return {} candidates = [] fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", text, flags=re.S) candidates.extend(fenced) first = text.find("{") last = text.rfind("}") if first != -1 and last != -1 and last > first: candidates.append(text[first:last + 1]) for candidate in candidates: try: value = json.loads(candidate) if isinstance(value, dict): return value except Exception: continue return {} def extract_pip_dependency_issue(text: str = "") -> dict: """Extract the first actionable pip dependency resolver/build error. v198.26.1: this is intentionally generic. The worker does not try to be a package-resolution agent; it makes dependency failures first-class evidence, extracts the likely packages, distinguishes structural platform conflicts, and constrains any Pi repair to dependency files. """ raw = text or "" low = raw.lower() issue: dict = {} patterns = [ ("no_matching_distribution", r"Could not find a version that satisfies the requirement\s+([^\s]+)(?:\s+\(from[^\n]*\))?"), ("no_matching_distribution", r"No matching distribution found for\s+([^\s]+)"), ("resolution_impossible", r"ERROR:\s*Cannot install[^\n]{0,2000}?conflicting dependencies[^\n]*"), ("resolution_impossible", r"ResolutionImpossible[^\n]*"), ("metadata_generation_failed", r"metadata-generation-failed[^\n]*"), ("metadata_generation_failed", r"Preparing metadata .*? did not run successfully[^\n]*"), ("wheel_build_failed", r"Failed building wheel for\s+([^\s]+)"), ("subprocess_exited", r"subprocess-exited-with-error[^\n]*"), ("subprocess_exited", r"pip subprocess.*?did not run successfully[^\n]*"), ("requires_python", r"requires-python[^\n]*(?:not satisfied|unsupported|incompatible)[^\n]*"), ] for kind, pattern in patterns: match = re.search(pattern, raw, flags=re.I | re.S) if match: requirement = match.group(1).strip(" ,.;:\")'") if match.groups() else "" package = _normalize_dependency_package_name(requirement) issue = { "schema_version": "dependency_issue.v198_25_3", "kind": kind, "requirement": requirement, "package": package, "line": match.group(0)[:1500], } break generic_markers = [ "could not find a version", "no matching distribution", "resolutionimpossible", "metadata-generation-failed", "subprocess-exited-with-error", "failed building wheel", "requires-python", "conflicting dependencies", "cannot install", ] if not issue and any(marker in low for marker in generic_markers): issue = {"schema_version": "dependency_issue.v198_25_3", "kind": "dependency_resolution", "requirement": "", "package": "", "line": raw[-2000:]} if issue: packages = extract_dependency_conflict_packages(raw) if issue.get("package") and issue["package"] not in packages: packages.insert(0, issue["package"]) platform_conflict = detect_dependency_platform_conflict(raw, packages) failure_class = "dependency_platform_conflict" if platform_conflict.get("detected") else "dependency_resolution_conflict" issue.update({ "category": "dependency_error", "failure_owner": "dependency", "failure_class": failure_class, "actionable": True, "repair_candidate": True, "app_code_repair_allowed": False, "recommended_action": "patch_requirements" if not platform_conflict.get("detected") else "surgical_requirements_repair_or_declare_platform_conflict", "recommended_patch_scope": ["requirements.txt", "requirements_policy.json"], "allowed_files": ["requirements.txt", "requirements_policy.json"], "forbidden_files": ["app.py", "README.md", "INFERENCE_CONTRACT.json", "DEMO_QUALITY_CONTRACT.json"], "packages": packages, "platform_conflict": platform_conflict, "summary": "pip dependency resolution/build failure during Space build", }) return issue def write_dependency_error_brief(workspace: Path, run_dir: Path, events_path: Path, issue: dict, build_log: str = "", runtime_log: str = "", failure_reason: str = "") -> str: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) brief = f"""# Dependency build error brief The Space build failed during dependency installation. This brief does **not** replace the HF Spaces gist. It is a focused evidence appendix for applying the gist method: read logs first, find the first actionable build error, make one minimal requirements patch, rebuild, then validate live. ## First dependency error ```json {json.dumps(issue, indent=2, ensure_ascii=False)} ``` ## Gist-aligned repair guidance - Read the first real pip resolver error, not only the last log line. - Use the cheapest useful iteration rung from the gist. - For actionable pip errors, the cheapest useful action is normally a minimal `requirements.txt` / dependency pin patch, followed by rebuild and live validation. - Do not use `factory_rebuild_same_code` before a dependency patch when the build log already contains a concrete pip resolver error. - Do not change inference behavior to work around dependency resolution. - Keep the patch minimal and explain why the chosen versions are compatible. ## Build log tail ```text {build_log[-12000:]} ``` ## Runtime log tail ```text {runtime_log[-4000:]} ``` ## Failure reason ```text {failure_reason[:4000]} ``` """ (repair_dir / "DEPENDENCY_ERROR_BRIEF.md").write_text(brief, encoding="utf-8") (workspace / "DEPENDENCY_ERROR_BRIEF.md").write_text(brief, encoding="utf-8") append_event(events_path, "failure_diagnosis", "warning", "Detected pip dependency build error; applying gist log-first method", {"dependency_issue": issue, "artifact": "repair/DEPENDENCY_ERROR_BRIEF.md"}) write_agent_trace_record(run_dir, phase="diagnosis", event="dependency_error_detected", status="warning", message="Detected pip dependency build error; applying gist log-first method", data={"dependency_issue": issue}, artifacts=["repair/DEPENDENCY_ERROR_BRIEF.md"]) return brief def known_dependency_guardrail_needed(workspace: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "") -> bool: text = f"{failure_reason}\n{build_log}\n{runtime_log}" low = text.lower() issue = extract_pip_dependency_issue(text) # Keep automatic deterministic fixes extremely narrow. General pip conflicts # must be handed to Pi with DEPENDENCY_ERROR_BRIEF.md. The Cosmos-style # Transformers 5 conflict is safe to repair mechanically because ASF itself # owns the incompatible `<5` upper bound. if detect_transformers5_dependency_requirement(text): return True if issue and issue.get("package") == "safetensors" and "transformers-5" in low and "safetensors>=0.8.0" in low: return True return False def apply_dependency_guardrail_repair(workspace: Path, run_dir: Path, events_path: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "") -> bool: """Apply deterministic requirements repair for known pip resolver failures. This handles build failures before Pi can produce a useful runtime patch. """ if not known_dependency_guardrail_needed(workspace, failure_reason, build_log, runtime_log): return False evidence_text = f"{failure_reason}\n{build_log}\n{runtime_log}" before = (workspace / "requirements.txt").read_text(encoding="utf-8", errors="ignore") if (workspace / "requirements.txt").exists() else "" relax_transformers_upper_bound_for_repair(workspace, events_path, evidence_text) normalize_requirements_for_modern_hub(workspace, events_path, dependency_evidence=evidence_text) after = (workspace / "requirements.txt").read_text(encoding="utf-8", errors="ignore") if (workspace / "requirements.txt").exists() else "" repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) summary = """# Deterministic dependency repair The Space build failed during `pip install` because the resolver selected a known Transformers 5.x path requiring unavailable `safetensors>=0.8.0`. The Factory applied only the narrow known trap guardrail: - `transformers>=4.51.0,<5.0.0` - `huggingface_hub>=0.34.0,<2.0.0` Other dependency conflicts are passed to Pi through `DEPENDENCY_ERROR_BRIEF.md`. No inference code was changed. """ (repair_dir / "REPAIR_SUMMARY.md").write_text(summary, encoding="utf-8") (repair_dir / "REPAIR_PLAN.md").write_text("# Deterministic dependency repair plan\n\nNormalize requirements and re-upload the same workspace for a clean Space rebuild.\n", encoding="utf-8") write_agent_trace_record( run_dir, phase="repair_patch", event="dependency_guardrail_repair", status="success", message="Factory applied deterministic requirements repair for safetensors>=0.8.0 resolver failure", data={"before": before[-2000:], "after": after[-2000:]}, artifacts=["repair/REPAIR_SUMMARY.md", "repair/REPAIR_PLAN.md"], ) append_event( events_path, "repair_patch", "success", "Applied deterministic dependency guardrail for pip resolver build error", { "category": "dependency_error", "transformers": "evidence_backed_policy", "note": "Known dependency conflicts are repaired only from concrete build-log evidence; Transformers <5 is relaxed when logs prove a dependency requires Transformers 5.x.", }, ) return before != after def fallback_decision_from_classifier(classification: dict, budgets: dict | None = None) -> dict: budgets = budgets or {} category = classification.get("category") or "unknown_runtime_error" quality = classification.get("logs_quality") or "partial" if category in {"hf_auth_error"}: action = "declare_technical_blocker" elif category == "cuda_oom": action = "request_manual_hardware" elif quality in {"empty", "no_reason"}: if budgets.get("wait_for_logs", 0) > 0: action = "wait_for_logs" elif budgets.get("factory_rebuild_same_code", 0) > 0: action = "factory_rebuild_same_code" else: action = "declare_technical_blocker" elif category in {"dependency_error", "import_error", "gradio_api_mismatch", "model_loading_error", "wrong_output_type", "zero_gpu_duration_error", "space_boot_timeout"}: action = "patch_code" if budgets.get("patch_code", 0) > 0 else "declare_technical_blocker" elif budgets.get("factory_rebuild_same_code", 0) > 0: action = "factory_rebuild_same_code" elif budgets.get("patch_code", 0) > 0 and quality == "useful": action = "patch_code" else: action = "declare_technical_blocker" return { "action": action, "confidence": "low", "reason": f"Factory fallback decision from classifier category={category}, logs_quality={quality}.", "evidence": classification.get("signals") or [], "patch_allowed": action == "patch_code", "requires_manual_hardware": action == "request_manual_hardware", "source": "factory_fallback", } def normalize_repair_decision(decision: dict, classification: dict, budgets: dict | None = None) -> dict: budgets = budgets or {} allowed = {"wait_for_logs", "inspect_more_logs", "factory_rebuild_same_code", "patch_code", "request_manual_hardware", "declare_technical_blocker"} action = str(decision.get("action") or "").strip().lower().replace("-", "_") if action not in allowed: decision = fallback_decision_from_classifier(classification, budgets) action = decision["action"] quality = classification.get("logs_quality") or "partial" category = classification.get("category") or "unknown_runtime_error" overrides = [] if action == "patch_code" and quality in {"empty", "no_reason"}: # v198.16: do not burn Pi repair budget from a generic BUILD_ERROR/RUNTIME_ERROR # when the real /logs/run or /logs/build streams were not fetched or contain no # actionable traceback. Pi can patch only after useful logs or another concrete # non-log validator signal upgrades quality. overrides.append("patch_code_blocked_without_actionable_hf_space_logs") if budgets.get("wait_for_logs", 0) > 0: action = "wait_for_logs" elif budgets.get("inspect_more_logs", 0) > 0: action = "inspect_more_logs" elif budgets.get("factory_rebuild_same_code", 0) > 0: action = "factory_rebuild_same_code" else: action = "declare_technical_blocker" if category == "dependency_error" and quality == "useful" and action in {"wait_for_logs", "inspect_more_logs", "factory_rebuild_same_code"}: overrides.append("gist_log_first_dependency_error_needs_minimal_requirements_patch") # The gist says read the first actionable error and act once surgically. # For a concrete pip resolver error, a same-code rebuild/wait is not the # cheapest useful rung. Ask Pi to patch requirements minimally instead. action = "patch_code" if budgets.get("patch_code", 0) > 0 else "declare_technical_blocker" if action in budgets and budgets.get(action, 0) <= 0: overrides.append(f"budget_exhausted:{action}") action = "declare_technical_blocker" if action == "patch_code" and category in {"hf_auth_error"}: overrides.append("auth_failure_not_patchable") action = "declare_technical_blocker" if action == "patch_code" and category == "cuda_oom": # The agent may sometimes reduce memory pressure, but default to manual # hardware unless the logs are useful and it explicitly gave evidence. evidence = " ".join(str(x).lower() for x in decision.get("evidence") or []) if "dtype" not in evidence and "offload" not in evidence and "steps" not in evidence: overrides.append("oom_needs_hardware_or_explicit_memory_plan") action = "request_manual_hardware" normalized = { "action": action, "confidence": str(decision.get("confidence") or "low"), "reason": str(decision.get("reason") or "No reason provided.")[:2000], "evidence": list(decision.get("evidence") or [])[:20], "patch_allowed": action == "patch_code", "requires_manual_hardware": action == "request_manual_hardware", "factory_overrides": overrides, "classification": classification, } return normalized def build_repair_task_packet(classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None, *, workspace: Path | None = None, run_dir: Path | None = None) -> dict: category = (classification or {}).get("category") or "unknown_runtime_error" signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification or {}) evidence = [] for item in (decision or {}).get("evidence") or []: evidence.append(str(item)[:500]) if not evidence: excerpt = extract_actionable_error_excerpt("\n".join([failure_reason or "", build_log or "", runtime_log or ""]), max_lines=40) if excerpt: evidence.append(excerpt[-2000:]) task_type = "runtime_patch" allowed_files = ["app.py", "requirements.txt"] expected_fix = "Apply the smallest code/dependency patch that addresses the classified failure." patch_mode = "pi_targeted_patch" recipe_packet = write_recipe_aware_repair_packet(workspace, run_dir, failure_reason, classification, decision) if workspace is not None and run_dir is not None else {"known_failure_match": match_recipe_known_failure_signature(failure_reason, {}, {}, fallback_class=category)} known_failure_match = recipe_packet.get("known_failure_match") if isinstance(recipe_packet, dict) else {} matched_class = str((known_failure_match or {}).get("failure_class") or category) if matched_class and matched_class != category: category = matched_class if category == "dependency_error": task_type = "dependency_repair" allowed_files = ["requirements.txt", "requirements_policy.json"] expected_fix = "Patch the first concrete dependency issue minimally in dependency files only. Do not edit app.py for pip resolver conflicts." elif category in {"import_error"}: task_type = "import_or_missing_dependency" allowed_files = ["app.py", "requirements.txt"] expected_fix = "Fix imports or add the missing dependency without changing the model or replacing real inference." elif category in {"gradio_api_mismatch", "gradio_component_api_error"}: task_type = "gradio_api_contract" allowed_files = ["app.py"] expected_fix = "Expose/repair the expected Gradio endpoint and modern Gradio component API without changing the model." elif category == "wrong_output_type": task_type = "output_contract_repair" allowed_files = ["app.py"] expected_fix = "Return the expected output type from the existing real inference path." elif category == "model_loading_error": task_type = "model_loading_repair" allowed_files = ["app.py", "requirements.txt"] expected_fix = "Fix the pipeline/model loading path using repo/model-card evidence while preserving real inference." elif category == "zero_gpu_duration_error": task_type = "zerogpu_duration_repair" allowed_files = ["app.py", "INFERENCE_CONTRACT.json", "README.md"] expected_fix = "Adjust @spaces.GPU duration from observed inference latency without changing inference semantics." elif category == "cuda_oom": task_type = "memory_optimization_patch" allowed_files = ["app.py"] expected_fix = "Only apply concrete memory optimizations supported by MEMORY_DIAGNOSIS.json evidence." patch_mode = "pi_memory_patch_after_diagnosis" elif category == "sdxl_lora_text_encoder_mismatch": task_type = "recipe_aware_sdxl_lora_loader_repair" allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] expected_fix = "Repair the SDXL LoRA adapter loader according to MODEL_RECIPE.json: base SDXL pipeline first, then load_lora_weights(adapter_id, weight_name), with health.last_error preserving load failures." elif category == "flux_template_syntax_error": task_type = "recipe_aware_flux_runtime_repair" allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] expected_fix = "Patch FLUX runtime/template handling while preserving FluxPipeline image generation and the family-aware runtime contract." elif category == "missing_spaces_gpu_decorator": task_type = "zerogpu_decorator_repair" allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json"] expected_fix = "Add/restore the spaces import and @spaces.GPU protection around heavy inference without touching model semantics." elif category == "model_not_loaded_after_boot": task_type = "runtime_loader_contract_repair" allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] expected_fix = "Fix model initialization so /health reports readiness accurately and /generate fails cleanly instead of raising vague NoneType/model-not-loaded errors." elif category == "validator_schema_choice_type_mismatch": task_type = "validator_payload_or_schema_alignment" allowed_files = ["app.py", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] expected_fix = "Align the published Gradio schema and canonical smoke payload without changing model semantics; do not patch if this is validator-owned." packet = { "schema_version": "repair_task_packet.v198_26_32", "task_type": task_type, "repair_mode": "deep_repair_reanalysis" if is_deep_repair_category(category) else "surgical_repair", "patch_mode": patch_mode, "failure_class": category, "failure_owner": failure_owner_from_classification(classification or {}), "failure_signature": signature, "logs_quality": (classification or {}).get("logs_quality"), "failure_phase": (classification or {}).get("failure_phase"), "authoritative_error": evidence[0] if evidence else "", "evidence": evidence[:5], "allowed_files": allowed_files, "expected_fix": expected_fix, "do_not_reinterpret": True, "success_criteria": [ "The Space should boot past the current failure signature.", "Real inference must be preserved; no placeholder/canned outputs.", "The expected Gradio health and generation/API contract must remain available.", ], "forbidden_changes": [ "Do not replace the model id.", "Do not remove real inference.", "Do not hide failures behind broad try/except placeholder responses.", "Do not turn the Space into diagnostic-only unless writing an explicit blocker.", "Do not publish or push anything from Pi.", ] + (["For dependency_repair, do not edit app.py; write PATCH_REFUSAL.json if the conflict cannot be resolved safely from dependency files only."] if task_type == "dependency_repair" else []), "full_context_files_available_if_needed": ["REPAIR_BRIEF.md", "INCIDENT_BRIEF.md", "DEPENDENCY_ERROR_BRIEF.md"], "default_context_files": ["REPAIR_TASK_PACKET.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], "refusal_mode": "If the frozen decision appears wrong or the failure belongs to the Factory, write PATCH_REFUSAL.json and do not edit publishable files.", "recipe_aware_repair_packet": "RECIPE_AWARE_REPAIR_PACKET.json" if workspace is not None and run_dir is not None else "", "model_recipe_summary": { "model_family": (recipe_packet.get("model_family") if isinstance(recipe_packet, dict) else "") or "", "artifact_role": (recipe_packet.get("artifact_role") if isinstance(recipe_packet, dict) else "") or "", "loader_strategy": (recipe_packet.get("loader_strategy") if isinstance(recipe_packet, dict) else "") or "", "expected_output_type": (recipe_packet.get("expected_output_type") if isinstance(recipe_packet, dict) else "") or "", "primary_endpoint": (recipe_packet.get("primary_endpoint") if isinstance(recipe_packet, dict) else "") or "/generate", }, "known_failure_match": known_failure_match or {}, "family_repair_directives": (recipe_packet.get("family_repair_directives") if isinstance(recipe_packet, dict) else []) or [], } if decision: packet["diagnosis_decision"] = decision return packet def write_repair_task_packet(workspace: Path, run_dir: Path, classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None) -> dict: packet = build_repair_task_packet(classification, failure_reason, build_log, runtime_log, decision, workspace=workspace, run_dir=run_dir) repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "REPAIR_TASK_PACKET.json", packet) write_json(workspace / "REPAIR_TASK_PACKET.json", packet) write_json(repair_dir / "PI_TASK_PACKET.json", packet) write_json(workspace / "PI_TASK_PACKET.json", packet) return packet def detect_memory_features_from_app(workspace: Path) -> dict: app_text = _read_workspace_text_if_exists(workspace / "app.py") low = app_text.lower() def has_any(*needles): return any(n in low for n in needles) size_match = re.findall(r"(?:width|height)\s*=\s*(\d{3,4})", app_text) steps_match = re.findall(r"(?:num_inference_steps|steps)\s*=\s*(\d{1,3})", app_text) return { "torch_dtype_bfloat16": has_any("torch.bfloat16", "dtype=torch.bfloat16"), "torch_dtype_float16": has_any("torch.float16", "torch_dtype=torch.float16", "variant=\"fp16\"", "variant='fp16'"), "uses_cpu_offload": has_any("enable_model_cpu_offload", "enable_sequential_cpu_offload"), "uses_attention_slicing": has_any("enable_attention_slicing", "enable_vae_slicing", "enable_vae_tiling"), "moves_pipeline_to_cuda": has_any(".to(\"cuda\")", ".to('cuda')"), "detected_sizes": size_match[:10], "detected_steps": steps_match[:10], } def write_memory_diagnosis_packet(workspace: Path, run_dir: Path, *, model_id: str, target_space_id: str, failure_reason: str, build_log: str, runtime_log: str, classification: dict, selected_hardware: str = "") -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) excerpt = extract_actionable_error_excerpt("\n".join([failure_reason or "", build_log or "", runtime_log or ""]), max_lines=120) packet = { "schema_version": "memory_diagnosis_packet.v198_25", "task_type": "memory_diagnosis", "model_id": model_id, "target_space_id": target_space_id, "selected_hardware": selected_hardware, "failure_class": (classification or {}).get("category"), "failure_phase": (classification or {}).get("failure_phase"), "logs_quality": (classification or {}).get("logs_quality"), "error_excerpt": excerpt[-4000:], "detected_memory_features": detect_memory_features_from_app(workspace), "app_py_relevant_excerpt": _read_workspace_text_if_exists(workspace / "app.py")[:12000], "requirements_excerpt": _read_workspace_text_if_exists(workspace / "requirements.txt")[:4000], } write_json(repair_dir / "MEMORY_DIAGNOSIS_PACKET.json", packet) write_json(workspace / "MEMORY_DIAGNOSIS_PACKET.json", packet) return packet def normalize_memory_diagnosis(raw: dict, classification: dict) -> dict: raw = raw if isinstance(raw, dict) else {} allowed_classes = {"hardware_capacity", "fixable_memory_optimization", "inference_parameter_too_heavy", "memory_leak_or_cache", "unknown"} cls = str(raw.get("classification") or raw.get("memory_classification") or "unknown").strip().lower() if cls not in allowed_classes: cls = "unknown" rec = str(raw.get("recommended_action") or "request_manual_hardware").strip().lower() if rec not in {"patch_code", "request_manual_hardware", "inspect_more_logs", "declare_technical_blocker"}: rec = "request_manual_hardware" if cls in {"hardware_capacity", "unknown"} else "patch_code" return { "schema_version": "memory_diagnosis.v198_23", "classification": cls, "confidence": str(raw.get("confidence") or "low"), "evidence": list(raw.get("evidence") or [])[:10], "recommended_action": rec, "patch_opportunities": list(raw.get("patch_opportunities") or raw.get("allowed_patch_types") or [])[:12], "classification_source": "pi_memory_diagnosis", "base_failure_classification": classification, } def diagnose_memory_with_pi(workspace: Path, run_dir: Path, events_path: Path, pi_model: str, target_space_id: str, model_id: str, failure_reason: str, classification: dict) -> dict: logs_dir = run_dir / "logs" build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else "" runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_memory_diagnosis_packet(workspace, run_dir, model_id=model_id, target_space_id=target_space_id, failure_reason=failure_reason, build_log=build_log, runtime_log=runtime_log, classification=classification) goal = f"""You are Pi in MEMORY DIAGNOSIS MODE for Agentic Space Factory. Read `MEMORY_DIAGNOSIS_PACKET.json` first. You are not allowed to edit files in this step. Your task is to decide whether the CUDA OOM is most likely: - `hardware_capacity`: the selected hardware is too small even for a reasonable implementation. - `fixable_memory_optimization`: the app has concrete fixable memory issues such as missing bf16/fp16, missing offload/slicing/tiling, duplicate pipeline loads, excessive default steps/resolution, or cache misuse. - `inference_parameter_too_heavy`: defaults are too large but the implementation is otherwise plausible. - `memory_leak_or_cache`: repeated calls leak or retain tensors. - `unknown`: evidence is insufficient. Return only `MEMORY_DIAGNOSIS.json` with this schema: ```json {{ "classification": "hardware_capacity|fixable_memory_optimization|inference_parameter_too_heavy|memory_leak_or_cache|unknown", "confidence": "low|medium|high", "evidence": ["specific evidence from app/log excerpt"], "recommended_action": "patch_code|request_manual_hardware|inspect_more_logs|declare_technical_blocker", "patch_opportunities": ["set_bfloat16", "enable_model_cpu_offload", "reduce_default_steps", "reduce_default_resolution", "clear_cuda_cache"] }} ``` Do not decide final run status. Do not write blockers. Do not patch code. """ (workspace / "MEMORY_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / "MEMORY_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") estimate_pi_context_budget(run_dir, workspace, phase="memory_diagnosis", task_type="cuda_oom_memory_diagnosis", direct_prompt=goal, referenced_files=["MEMORY_DIAGNOSIS_PACKET.json", "app.py", "requirements.txt"], max_context_chars=28000, omitted_sections=["full_build_logs", "full_runtime_logs", "REPAIR_BRIEF.md"], events_path=events_path, artifact_dir=repair_dir) append_event(events_path, "pi_memory_diagnosis", "started", "Running Pi bounded memory diagnosis for CUDA OOM", {"model": pi_model}) write_agent_trace_record(run_dir, phase="memory_diagnosis", event="command_started", status="started", message="Pi memory diagnosis started", data={"model": pi_model}, artifacts=["repair/MEMORY_DIAGNOSIS_PACKET.json", "repair/MEMORY_DIAGNOSIS_GOAL.md"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=900) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_memory_diagnosis_output.txt").write_text(out, encoding="utf-8") write_pi_call_fingerprint(run_dir, phase="memory_diagnosis", requested_model=pi_model, prompt_text=goal, output_text=out, task_packet_path=workspace / "MEMORY_DIAGNOSIS_PACKET.json", events_path=events_path, artifact_dir=repair_dir) raw = load_json_if_exists(workspace / "MEMORY_DIAGNOSIS.json") if (workspace / "MEMORY_DIAGNOSIS.json").exists() else {} if not raw: raw = extract_json_object(out) diagnosis = normalize_memory_diagnosis(raw, classification) write_json(repair_dir / "MEMORY_DIAGNOSIS.json", diagnosis) write_json(workspace / "MEMORY_DIAGNOSIS.json", diagnosis) append_agent_trace_artifact(run_dir, phase="memory_diagnosis", event="memory_diagnosis_output", artifact="logs/pi_memory_diagnosis_output.txt", text=out, status="success" if code == 0 else "warning", data={"returncode": code, "diagnosis": diagnosis}) append_event(events_path, "pi_memory_diagnosis", "success" if code == 0 else "warning", f"Pi memory diagnosis classified OOM as {diagnosis.get('classification')}", {"diagnosis": diagnosis}) return diagnosis def decision_from_memory_diagnosis(memory_diagnosis: dict, classification: dict, budgets: dict | None = None) -> dict: budgets = budgets or {} cls = (memory_diagnosis or {}).get("classification") or "unknown" rec = (memory_diagnosis or {}).get("recommended_action") or "request_manual_hardware" action = rec if rec in {"patch_code", "request_manual_hardware", "inspect_more_logs", "declare_technical_blocker"} else "request_manual_hardware" if action == "patch_code" and budgets.get("patch_code", 0) <= 0: action = "declare_technical_blocker" if action == "patch_code" and cls not in {"fixable_memory_optimization", "inference_parameter_too_heavy", "memory_leak_or_cache"}: action = "request_manual_hardware" return { "action": action, "confidence": (memory_diagnosis or {}).get("confidence") or "low", "reason": f"Pi memory diagnosis classified CUDA OOM as {cls}.", "evidence": (memory_diagnosis or {}).get("evidence") or [], "patch_allowed": action == "patch_code", "requires_manual_hardware": action == "request_manual_hardware", "source": "pi_memory_diagnosis", "memory_diagnosis": memory_diagnosis, "classification": classification, } def diagnose_failure_with_pi(workspace: Path, run_dir: Path, events_path: Path, pi_model: str, target_space_id: str, model_id: str, failure_reason: str, implementation_mode: str, expected_output_type: str, iteration: int, budgets: dict) -> dict: logs_dir = run_dir / "logs" build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else "" runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) classification = classify_repair_failure(failure_reason, build_log, runtime_log) dependency_issue = classification.get("dependency_issue") or {} if dependency_issue: write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, failure_reason) write_actionable_error_artifacts(run_dir, failure_reason, build_log, runtime_log, classification) if classification.get("category") == "cuda_oom" and classification.get("logs_quality") == "useful": memory_diagnosis = diagnose_memory_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, classification) raw_memory_decision = decision_from_memory_diagnosis(memory_diagnosis, classification, budgets) decision = normalize_repair_decision(raw_memory_decision, classification, budgets) write_json(repair_dir / "REPAIR_DECISION.json", decision) (workspace / "REPAIR_DECISION.json").write_text(json.dumps(decision, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") write_agent_trace_record(run_dir, phase="diagnosis", event="repair_decision", status="success", message=f"Pi memory diagnosis selected action: {decision.get('action')}", data={"decision": decision}, artifacts=["repair/MEMORY_DIAGNOSIS.json", "repair/REPAIR_DECISION.json"]) append_event(events_path, "repair_decision", "success", f"Pi memory diagnosis selected action: {decision.get('action')}", {"decision": decision}) return decision append_event(events_path, "failure_diagnosis", "started", "Preparing Pi blockage diagnosis from Space status and logs", {"iteration": iteration, **classification}) brief = write_incident_brief( workspace, run_dir, target_space_id=target_space_id, model_id=model_id, pi_model=pi_model, failure_reason=failure_reason, build_log=build_log, runtime_log=runtime_log, classification=classification, implementation_mode=implementation_mode, expected_output_type=expected_output_type, iteration=iteration, budgets=budgets, ) goal = f"""You are Pi in BLOCKAGE DIAGNOSIS MODE for Agentic Space Factory. First read `INCIDENT_BRIEF.md`. {hf_spaces_guidance_repair_section()} {pi_reference_dependency_repair_section()} {pi_tooling_context_note()} You are not allowed to edit code in this diagnosis step. Your task is to decide the next action for the Factory. Use the gist method: read logs first, identify the first actionable error, use the cheapest useful iteration rung, and require a live Gradio/API validation before success. When the first error is dependency-related, classify it with the ASF dependency contract in mind: Gradio/HF runtime packages are platform-owned, model-specific pins need evidence, torch/torchaudio are managed by Spaces, and non-pip-installable model code should be vendored. Write `REPAIR_DECISION.json` exactly as requested in INCIDENT_BRIEF.md. Do not patch files during this step. """ (workspace / "PI_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / "PI_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") append_event(events_path, "pi_diagnosis", "started", "Running Pi diagnosis decision before any repair action", {"model": pi_model, "iteration": iteration}) estimate_pi_context_budget(run_dir, workspace, phase="diagnosis", task_type="blockage_decision", direct_prompt=goal, referenced_files=["INCIDENT_BRIEF.md", "DEPENDENCY_ERROR_BRIEF.md"], max_context_chars=42000, omitted_sections=["full_logs", "REPAIR_BRIEF.md", "workspace_source_dump"], events_path=events_path, artifact_dir=repair_dir) write_agent_trace_record(run_dir, phase="diagnosis", event="command_started", status="started", message="Pi blockage diagnosis started", data={"model": pi_model, "iteration": iteration}, artifacts=["repair/INCIDENT_BRIEF.md", "repair/LOG_EVIDENCE_PACKET.json", "repair/PI_DIAGNOSIS_GOAL.md", "repair/PI_PROMPT_BUDGET_diagnosis.json"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=900) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_diagnosis_output.txt").write_text(out, encoding="utf-8") write_pi_call_fingerprint(run_dir, phase="diagnosis", requested_model=pi_model, prompt_text=goal, output_text=out, events_path=events_path, artifact_dir=repair_dir) append_agent_trace_artifact(run_dir, phase="diagnosis", event="diagnosis_output", artifact="logs/pi_diagnosis_output.txt", text=out, status="success" if code == 0 else "failed", data={"returncode": code}) raw_decision = {} decision_path = workspace / "REPAIR_DECISION.json" if decision_path.exists(): raw_decision = load_json_if_exists(decision_path) if not raw_decision: raw_decision = extract_json_object(out) if code != 0: append_event(events_path, "pi_diagnosis", "warning", "Pi diagnosis returned non-zero; using conservative factory fallback", {"returncode": code, "output_tail": out[-3000:]}) if not raw_decision: raw_decision = fallback_decision_from_classifier(classification, budgets) decision = normalize_repair_decision(raw_decision, classification, budgets) write_frozen_repair_decision(workspace, run_dir, decision, classification) write_json(repair_dir / "REPAIR_DECISION.json", decision) (workspace / "REPAIR_DECISION.json").write_text(json.dumps(decision, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") write_agent_trace_record(run_dir, phase="diagnosis", event="repair_decision", status="success", message=f"Pi diagnosis selected action: {decision.get('action')}", data={"decision": decision}, artifacts=["repair/REPAIR_DECISION.json", "repair/FROZEN_REPAIR_DECISION.json"]) append_event(events_path, "repair_decision", "success", f"Pi diagnosis selected action: {decision.get('action')}", {"decision": decision}) return decision def write_blockage_artifact(workspace: Path, run_dir: Path, events_path: Path, decision: dict, failure_reason: str, *, status: str = "technical_blocker"): blocker = { "full_inference_implemented": False, "source": "agentic_space_factory_blockage_protocol", "status": status, "decision": decision, "failure_reason": failure_reason[:4000], "blockers": [ { "type": decision.get("action") or status, "claim": decision.get("reason") or "Automated recovery could not safely continue.", "evidence": decision.get("evidence") or [], "severity": "blocking", } ], "suggested_next_step": "Inspect repair/INCIDENT_BRIEF.md and repair/REPAIR_DECISION.json, then retry manually or adjust hardware/settings if appropriate.", } (workspace / "TECHNICAL_BLOCKERS.json").write_text(json.dumps(blocker, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") gen_dir = run_dir / "generated" gen_dir.mkdir(parents=True, exist_ok=True) (gen_dir / "TECHNICAL_BLOCKERS.json").write_text(json.dumps(blocker, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") write_json(run_dir / "repair" / "BLOCKAGE.json", blocker) append_event(events_path, "technical_blocker", "failed", "Automated recovery stopped with an auditable blocker", {"decision": decision}) def write_repair_brief(workspace: Path, run_dir: Path, *, target_space_id: str, model_id: str, pi_model: str, failure_reason: str, build_log: str, runtime_log: str, classification: dict, implementation_mode: str, expected_output_type: str, decision: dict | None = None) -> str: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) inventory = workspace_file_inventory(workspace) decision = decision or {} brief = f"""# Agentic Space Factory repair brief ## Scope This is a structured patch pass for an existing generated Hugging Face Space. A separate Pi diagnosis already decided that `patch_code` is justified. Do not rebuild from scratch unless the current architecture is impossible to boot. ## Diagnosis decision ```json {json.dumps(decision, indent=2, ensure_ascii=False)} ``` ## Original target - Model ID: `{model_id}` - Target Space: `{target_space_id}` - Pi model: `{pi_model}` - Implementation mode: `{implementation_mode}` - Expected output type: `{expected_output_type}` ## Failure classification - Category: `{classification.get('category', 'unknown_runtime_error')}` - Logs quality: `{classification.get('logs_quality', 'partial')}` - Failure phase: `{classification.get('failure_phase', 'unknown')}` - Recommended strategy: {classification.get('recommendation', '')} ## HF tooling/token context ```json {json.dumps(load_json_if_exists(run_dir / "token_context.json"), indent=2, ensure_ascii=False)} ``` ## Observed failure ```text {failure_reason[:5000]} ``` ## Actionable error excerpt ```text {extract_actionable_error_excerpt(failure_reason + chr(10) + build_log + chr(10) + runtime_log, max_lines=120)} ``` Full logs remain archived under `logs/space_logs_build.txt`, `logs/space_logs_runtime.txt`, and `logs/space_logs_run.txt`. Use them only if the focused excerpt is insufficient. ## Workspace files ```text {chr(10).join(inventory)} ``` ## Repair constraints - Patch the current workspace; do not change the product goal. - Make the smallest patch that fixes the classified failure. - Preserve real inference for Strict inference mode. - Do not replace inference with static placeholders, fake images, canned text, or swallowed exceptions. - Do not change the model ID unless you write a technical blocker explaining why. - Preserve or restore a cheap `health` endpoint. - Preserve the expected Gradio API endpoint when possible. - Keep README metadata valid and short_description <= 60 chars. - Platform dependency policy: keep Gradio / Gradio Client / Hugging Face Hub / Spaces modern and worker-owned. Do not downgrade Gradio or add stale exact pins. If the failure is a pip resolver conflict, first attempt requirements-only repair; do not edit app.py unless REPAIR_TASK_PACKET.json explicitly allows it. - For model-specific dependencies, follow the gist method: patch the first concrete resolver error minimally, preserve justified model-card pins, and do not pin torch/torchaudio unless unavoidable and explicitly justified. If torchvision is needed, prefer leaving it unpinned so it resolves against the managed torch runtime. - If model code is not pip-installable, vendor the necessary code into the Space repo instead of relying on local paths, editable installs, or clone-time side effects in requirements.txt. - Do not prove dependency fixes with local venv installs or broad pip dry-runs; Space build logs are authoritative. Keep local checks cheap, such as `python -m py_compile app.py`. - Native kernel policy: for flash-attn/custom attention/xformers/Triton/fused CUDA errors, do not blindly add source-built packages. First consider PyTorch SDPA, compatible prebuilt wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors. If no compatible fallback exists, declare a blocker rather than forcing a fragile build. ## Required repair artifacts Before modifying files, write `REPAIR_PLAN.md` with: - root cause - strategy - files to change - risk level - whether hardware/manual action is required After modifying files, write `REPAIR_SUMMARY.md` with: - files changed - why the patch is minimal - how it preserves real inference - how to validate it """ (repair_dir / "REPAIR_BRIEF.md").write_text(brief, encoding="utf-8") (workspace / "REPAIR_BRIEF.md").write_text(brief, encoding="utf-8") write_json(repair_dir / "classification.json", classification) return brief def extract_gradio_unexpected_kwarg(text: str) -> dict: """Extract `gr.(unexpected_kwarg=...)` evidence from Gradio startup errors.""" raw = text or "" patterns = [ r"(?P[A-Za-z_][A-Za-z0-9_]*)\.__init__\(\) got an unexpected keyword argument [\"'](?P[A-Za-z_][A-Za-z0-9_]*)[\"']", r"unexpected keyword argument [\"'](?P[A-Za-z_][A-Za-z0-9_]*)[\"']", ] for pattern in patterns: m = re.search(pattern, raw, flags=re.IGNORECASE) if m: return { "component": m.groupdict().get("component") or "", "kwarg": m.group("kwarg"), "repair_type": "ui_component_api_repair", } return {} def infer_inference_strategy(workspace: Path) -> dict: """Infer the generated app's inference strategy from publishable files. This is intentionally evidence-based and conservative. It replaces the old one-size Diffusers/Z-Image marker check used after repairs. The Worker still rejects obvious fake outputs, but it should not reject valid non-Diffusers architectures such as vLLM local servers, OpenAI-compatible local APIs, or CPU TTS SDKs. """ app_path = workspace / "app.py" req_path = workspace / "requirements.txt" contract_path = workspace / "INFERENCE_CONTRACT.json" app_text_raw = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" req_text_raw = req_path.read_text(encoding="utf-8", errors="ignore") if req_path.exists() else "" app = app_text_raw.lower() req = req_text_raw.lower() contract = load_json_if_exists(contract_path) if contract_path.exists() else {} contract_full = contract.get("full_inference_implemented") expected_output = str(contract.get("expected_output_type") or "").lower() strategies: list[dict] = [] def add(name: str, markers: list[str], confidence: str = "medium"): present = [m for m in markers if m.lower() in app or m.lower() in req] if present: strategies.append({"name": name, "markers_present": present, "confidence": confidence}) if contract_full is False or expected_output in {"text_diagnostic", "diagnostic", "none"}: strategies.append({"name": "diagnostic_only", "markers_present": ["INFERENCE_CONTRACT.full_inference_implemented=false"], "confidence": "high"}) add("vllm_local_server", ["vllm serve", "vllm", "subprocess.popen", "/v1/audio/speech", "/v1/chat/completions", "/v1/completions", "--omni", "openai"], "high") add("openai_compatible_local_api", ["/v1/audio/speech", "/v1/chat/completions", "/v1/completions", "httpx.post", "requests.post", "openai"], "high") # .to("cuda") is a GPU marker, not a Diffusers marker. Treating it as # Diffusers caused Transformers/custom-code audio apps to be mislabeled. diffusers_core_markers = ["diffusionpipeline", "from diffusers", "zimagepipeline", "stablediffusion", "stable diffusion", "diffusers."] diffusers_present = [m for m in diffusers_core_markers if m in app or m in req] if diffusers_present: cuda_present = [m for m in [".to(\"cuda\")", ".to('cuda')"] if m in app] strategies.append({"name": "diffusers_pipeline", "markers_present": diffusers_present + cuda_present, "confidence": "high"}) add("transformers_pipeline", ["from_pretrained", "automodelfor", "automodel", "pipeline(", ".generate(", "transformers"], "medium") add("tts_sdk_cpu", ["supertonic", "tts(", "tts.synthesize", ".synthesize(", "save_audio", "gr.audio"], "high") add("onnx_runtime", ["onnxruntime", "inferencesession", ".onnx"], "medium") add("spaces_gpu", ["@spaces.gpu", "spaces.gpu", ".to(\"cuda\")", ".to('cuda')"], "medium") by_name = {s["name"]: s for s in strategies} contract_strategy = str(contract.get("inference_strategy") or contract.get("strategy") or "").strip().lower() if contract_strategy in by_name and contract_strategy not in {"spaces_gpu"}: primary = by_name[contract_strategy] else: order = ["diagnostic_only", "vllm_local_server", "openai_compatible_local_api", "diffusers_pipeline", "transformers_pipeline", "tts_sdk_cpu", "onnx_runtime", "spaces_gpu"] primary = next((by_name[name] for name in order if name in by_name), None) if not primary: primary = {"name": "unknown", "markers_present": [], "confidence": "low"} return { "schema_version": "1.0", "primary": primary.get("name"), "confidence": primary.get("confidence"), "markers_present": primary.get("markers_present") or [], "strategies": strategies, "contract_full_inference_implemented": contract_full, "contract_expected_output_type": expected_output, } def strategy_has_real_inference_path(strategy: dict) -> bool: primary = strategy.get("primary") if primary == "diagnostic_only": return True return primary in { "vllm_local_server", "openai_compatible_local_api", "diffusers_pipeline", "transformers_pipeline", "tts_sdk_cpu", "onnx_runtime", "spaces_gpu", } and bool(strategy.get("markers_present")) def _as_bool_true(value) -> bool: if value is True: return True if isinstance(value, str): return value.strip().lower() in {"true", "yes", "1", "on"} return False def _workspace_text_blob(workspace: Path | None, filenames: list[str]) -> str: if not workspace: return "" chunks = [] for name in filenames: path = workspace / name if path.exists() and path.is_file(): try: chunks.append(path.read_text(encoding="utf-8", errors="ignore")) except Exception: pass return "\n".join(chunks) def _read_workspace_text(path: Path) -> str: try: return path.read_text(encoding="utf-8", errors="ignore") except Exception: return "" def _contract_string_value(workspace: Path | None, *keys: str) -> str: contract = read_inference_contract(workspace) for key in keys: value = contract.get(key) if isinstance(value, str) and value.strip(): return value.strip() return "" def _contract_full_inference_value(workspace: Path | None): contract = read_inference_contract(workspace) return contract.get("full_inference_implemented") def _hard_gpu_markers_in_text(text: str) -> list[str]: low = (text or "").lower() markers = { "@spaces.GPU": ["@spaces.gpu", "spaces.gpu("], ".to(cuda)": [".to(\"cuda\")", ".to('cuda')"], "torch.cuda": ["torch.cuda", "cuda.is_available"], "device=cuda": ["device=\"cuda\"", "device='cuda'", "device_map=\"cuda\"", "device_map='cuda'"], "vllm": ["vllm serve", "import vllm", "from vllm"], } found = [] for name, needles in markers.items(): if any(n in low for n in needles): found.append(name) return found def workspace_hard_gpu_markers(workspace: Path | None) -> list[str]: return _hard_gpu_markers_in_text(_workspace_text_blob(workspace, ["app.py", "requirements.txt"])) def app_imports_transformers(workspace: Path | None) -> bool: text = _workspace_text_blob(workspace, ["app.py"]) if workspace else "" return bool(re.search(r"(?m)^\s*(import\s+transformers\b|from\s+transformers\b)", text)) def workspace_has_dependency_evidence_for_transformers(workspace: Path | None, dependency_evidence: str = "") -> bool: evidence = "\n".join([ dependency_evidence or "", _workspace_text_blob(workspace, ["INFERENCE_CONTRACT.json", "TECHNICAL_BLOCKERS.json", "MODEL_DEPENDENCIES.json", "REFERENCE_SPACE_FINDINGS.json"]), ]).lower() return "transformers" in evidence and any(x in evidence for x in ["requires", "dependency", "from_pretrained", "automodel", "diffusers", "cosmos"]) def workspace_should_keep_transformers_policy(workspace: Path | None, dependency_evidence: str = "") -> tuple[bool, str]: strategy = infer_inference_strategy(workspace) if workspace else {"primary": "unknown"} primary = strategy.get("primary") if workspace_allows_transformers_major5(workspace, dependency_evidence): return True, "transformers5_dependency_evidence" if app_imports_transformers(workspace): return True, "app_imports_transformers" if primary in {"transformers_pipeline", "diffusers_pipeline", "vllm_local_server", "openai_compatible_local_api"}: return True, f"strategy_{primary}" if workspace_has_dependency_evidence_for_transformers(workspace, dependency_evidence): return True, "model_dependency_evidence" return False, f"not_needed_for_strategy_{primary or 'unknown'}" def verify_workspace_health_endpoint(workspace: Path, run_dir: Path | None = None, events_path: Path | None = None) -> dict: """Verify cheap Gradio health endpoint across app.py and vendored modules. v198.4 keeps the pre-upload health gate, but stops assuming that complex vendorized apps register every endpoint in app.py. The endpoint is accepted when a publishable Python file contains api_name="health" (or equivalent) and the nearby code does not show obvious GPU/model-loading work. """ contract = read_inference_contract(workspace) contract_declares = bool(str(contract.get("health_endpoint") or contract.get("health_api_name") or "").strip().lower() in {"health", "/health", "true", "yes"}) candidates = [] app_py_found = False patterns = [r"api_name\s*=\s*[\"']health[\"']", r"/health\b", r"def\s+health\s*\("] heavy_needles = ["@spaces.gpu", "spaces.gpu", ".to(\"cuda\")", ".to('cuda')", "torch.cuda", "from_pretrained", "load_model", "generate(", "synthesize(", "pipeline("] for py in sorted(workspace.rglob("*.py")): try: rel = py.relative_to(workspace).as_posix() except Exception: rel = py.name parts = set(py.parts) if any(part in {".git", "__pycache__", ".venv", "venv", "node_modules"} for part in parts): continue text = _read_workspace_text(py) if not any(re.search(pat, text, flags=re.IGNORECASE) for pat in patterns): continue low = text.lower() health_pos = low.find('api_name="health"') if health_pos < 0: health_pos = low.find("api_name='health'") if health_pos < 0: health_pos = low.find("/health") if health_pos < 0: health_pos = low.find("def health") window = low[max(0, health_pos - 800): health_pos + 800] if health_pos >= 0 else low[:1600] heavy = [n for n in heavy_needles if n in window] candidate = {"file": rel, "heavy_markers_near_endpoint": heavy, "accepted": not heavy} candidates.append(candidate) if rel == "app.py" and candidate["accepted"]: app_py_found = True accepted = [c for c in candidates if c.get("accepted")] result = { "schema_version": "health_verification.v198_4", "contract_declares_health": contract_declares, "app_py_health_found": app_py_found, "workspace_health_found": bool(accepted), "accepted_files": [c["file"] for c in accepted], "rejected_candidates": [c for c in candidates if not c.get("accepted")], "decision": "accept" if accepted else "reject", "health_cheap_confidence": "high" if app_py_found else ("medium" if accepted else "low"), } if run_dir: write_json(run_dir / "health_verification.json", result) if events_path: append_event(events_path, "health_verification", "success" if accepted else "failed", "Verified generated health endpoint across publishable Python files", result) return result def _workspace_actionability_signals(workspace: Path | None) -> dict: """Inspect the publishable workspace without trusting diagnostics-only contracts. v198.26.19: manual-hardware output is only actionable if the Space that is uploaded already contains the full-inference candidate. A diagnostic wrapper must never be the publishable payload for "change hardware then validate". """ contract = read_inference_contract(workspace) app_text = _workspace_text_blob(workspace, ["app.py"]) if workspace else "" app_low = app_text.lower() primary_api = str(contract.get("primary_api_name") or contract.get("api_name") or "").strip() expected_output_type = str(contract.get("expected_output_type") or "").strip() recommended_hw = str(contract.get("recommended_target_space_hardware") or contract.get("recommended_hardware") or contract.get("minimum_hardware") or "").strip() has_spaces_import = bool(re.search(r"(?m)^\s*(import\s+spaces\b|from\s+spaces\b)", app_text)) has_spaces_gpu = "@spaces.gpu" in app_low or "spaces.gpu(" in app_low has_generate_endpoint = bool( re.search(r"api_name\s*=\s*[\"']/?generate[\"']", app_text, flags=re.IGNORECASE) or primary_api.strip().lower() in {"generate", "/generate"} ) diagnostic_only = bool( contract.get("fallback_or_diagnostic_only") is True or str(contract.get("inference_strategy") or "").strip().lower() in {"manual_hardware_actionable_space", "diagnostic_only", "boot_only_blocker"} or expected_output_type.lower() == "text_diagnostic" or contract.get("full_inference_implemented") is False ) zero_target = "zero" in " ".join([ recommended_hw.lower(), str((contract or {}).get("target_hardware") or "").lower(), ]) return { "schema_version": "workspace_actionability_signals.v198_26_19", "full_inference_implemented": contract.get("full_inference_implemented"), "real_inference_implemented": contract.get("real_inference_implemented"), "requires_gpu": contract.get("requires_gpu"), "expected_output_type": expected_output_type, "recommended_target_space_hardware": recommended_hw, "primary_api_name": primary_api, "has_spaces_import": has_spaces_import, "has_spaces_gpu": has_spaces_gpu, "has_generate_endpoint": has_generate_endpoint, "diagnostic_only": diagnostic_only, "zero_target": zero_target, } def manual_hardware_publish_actionability_check(workspace: Path, hardware_intent: dict | None, manual_block: dict | None = None, run_dir: Path | None = None, events_path: Path | None = None) -> dict: signals = _workspace_actionability_signals(workspace) hw_text = " ".join(str((hardware_intent or {}).get(k) or "") for k in ("preferred_hardware", "fallback_hardware", "recommended_hardware")) zero_target = signals.get("zero_target") or ("zero" in hw_text.lower()) full_candidate = bool(signals.get("full_inference_implemented") is True and not signals.get("diagnostic_only") and signals.get("has_generate_endpoint")) problems = [] if not full_candidate: problems.append("publishable_workspace_is_not_full_inference_candidate") if zero_target and not signals.get("has_spaces_gpu"): problems.append("zero_gpu_target_without_spaces_gpu_decorator") if signals.get("expected_output_type") == "text_diagnostic": problems.append("publishable_workspace_expected_output_type_is_text_diagnostic") if not signals.get("has_generate_endpoint"): problems.append("publishable_workspace_missing_generate_endpoint") actionable = not problems result = { "schema_version": "manual_hardware_actionability_check.v198_26_19", "actionable": actionable, "problems": problems, "zero_target": bool(zero_target), "signals": signals, "hardware_intent": hardware_intent or {}, "manual_block": manual_block or {}, "policy": "publish_full_inference_candidate" if actionable else "block_non_actionable_manual_hardware_payload", } if run_dir: write_json(run_dir / "manual_hardware_actionability_check.json", result) if events_path: append_event(events_path, "manual_hardware_actionability_check", "success" if actionable else "failed", "Verified manual-hardware Space payload actionability before upload", result) return result def should_stop_before_cpu_deploy_for_manual_hardware(workspace: Path, hardware_intent: dict) -> dict: contract_full = _contract_full_inference_value(workspace) requires_gpu = _contract_requires_gpu_value(workspace) gpu_markers = workspace_hard_gpu_markers(workspace) triggered = bool( isinstance(hardware_intent, dict) and hardware_intent.get("manual_hardware_required") is True and contract_full is True and requires_gpu is True and gpu_markers ) return { "schema_version": "manual_hardware_block.v198_26_19", "triggered": triggered, "reason": "Full GPU app requires manual hardware; publish the full-inference candidate so the Space is actionable after hardware selection." if triggered else "automatic_deployment_allowed", "contract_full_inference_implemented": contract_full, "contract_requires_gpu": requires_gpu, "gpu_markers": gpu_markers, "hardware_intent": hardware_intent, "recommended_action": "publish_full_inference_candidate_for_manual_hardware" if triggered else "continue", "space_required_for_manual_hardware": triggered, "diagnostic_wrapper_allowed": False, } def prepare_manual_hardware_actionable_workspace(workspace: Path, run_dir: Path, events_path: Path, *, model_id: str, target_space_id: str, hardware_intent: dict, manual_block: dict) -> dict: """Preserve, but do not replace, the full-inference candidate for manual hardware. v198.26.19 changes the product contract: Manual hardware required is only acceptable when the published Space already contains the real `/generate` app. The old CPU diagnostic wrapper made the requested user action useless on ZeroGPU/fixed GPU, so diagnostics now stay as run artifacts only. """ original_dir = run_dir / "generated_full_inference_candidate" if original_dir.exists(): shutil.rmtree(original_dir) shutil.copytree(workspace, original_dir, ignore=internal_workspace_copy_ignore) recommended = (hardware_intent or {}).get("recommended_hardware") or (hardware_intent or {}).get("preferred_hardware") or _contract_string_value(workspace, "recommended_target_space_hardware", "recommended_hardware") or "manual GPU hardware" reason = manual_block.get("reason") or (hardware_intent or {}).get("reason") or "Manual hardware selection is required before full inference can be run." markers = manual_block.get("gpu_markers") or [] markers_text = "\n".join(f"- {m}" for m in markers[:20]) or "- GPU-only inference markers detected" intent_json = json.dumps(hardware_intent or {}, indent=2, ensure_ascii=False) block_json = json.dumps(manual_block or {}, indent=2, ensure_ascii=False) actionability = manual_hardware_publish_actionability_check(workspace, hardware_intent, manual_block, run_dir, events_path) report_template = """# Manual hardware required ASF preserved and deployed the full-inference candidate for `{model_id}`. **Target Space:** `{target_space_id}` **Recommended hardware:** {recommended} **Reason:** {reason} ## What to do 1. Open this Space's **Settings** tab on Hugging Face. 2. Change **Hardware** to the recommended GPU or a stronger compatible GPU. 3. Restart/rebuild the Space. 4. Re-run ASF validation / Space Test after the hardware change. The published Space is expected to remain the real full-inference app. ASF did not replace it with a CPU diagnostic wrapper. ## Detected GPU markers {markers_text} ## Actionability check ```json {actionability_json} ``` ## Hardware intent ```json {intent_json} ``` ## Manual block ```json {block_json} ``` """.format( model_id=model_id, target_space_id=target_space_id, recommended=recommended, reason=reason, markers_text=markers_text, actionability_json=json.dumps(actionability, indent=2, ensure_ascii=False), intent_json=intent_json, block_json=block_json, ) (run_dir / "manual_hardware_actionable_space.md").write_text(report_template, encoding="utf-8") payload = { "schema_version": "manual_hardware_actionable_space.v198_26_19", "space_created_required": True, "diagnostic_app_uploaded": False, "full_inference_candidate_preserved": str(original_dir.relative_to(run_dir)), "full_inference_candidate_deployed": True, "published_payload": "full_inference_candidate_workspace", "target_space_id": target_space_id, "recommended_hardware": recommended, "reason": reason, "hardware_intent": hardware_intent, "manual_hardware_block": manual_block, "actionability_check": actionability, "status": "manual_hardware_required_actionable" if actionability.get("actionable") else "manual_hardware_not_actionable", } write_json(run_dir / "manual_hardware_actionable_space.json", payload) append_event(events_path, "manual_hardware_actionable_space", "success" if actionability.get("actionable") else "failed", "Prepared manual-hardware action artifacts while preserving the full-inference Space payload", payload) return payload def contract_effective_output_type(workspace: Path | None, fallback_expected_output_type: str = "any") -> dict: """Separate the source model task output from the deployed app contract output. A diagnostic-only app for a text-to-audio model should not keep reporting `audio` as the effective smoke output. In that mode the Worker validates a bootable diagnostic app, so the contract output (usually text_diagnostic) is the source of truth. """ fallback = (fallback_expected_output_type or "any").strip() or "any" contract = read_inference_contract(workspace) blocker_reason = contract_declares_no_full_inference(workspace) contract_output = str(contract.get("expected_output_type") or "").strip() use_contract = bool(blocker_reason.get("declared") and contract_output) return { "schema_version": "1.0", "model_expected_output_type": fallback, "contract_effective_output_type": contract_output or fallback, "effective_expected_output_type": contract_output if use_contract else fallback, "contract_output_applied": use_contract, "contract_declared_no_full_inference": bool(blocker_reason.get("declared")), "source": "inference_contract" if use_contract else "model_task_or_env", } def _contract_requires_gpu_value(workspace: Path | None) -> bool | None: contract = read_inference_contract(workspace) blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json") for value in [contract.get("requires_gpu"), blockers.get("requires_gpu")]: if _as_bool_true(value): return True if _as_bool_false(value): return False return None def _too_large_for_standard_single_gpu(workspace: Path | None, model_analysis: dict | None = None) -> dict: """Detect documented hardware-capacity blockers without guessing silently. This is intentionally conservative: it requires explicit blocker language mentioning unavailable/insufficient hardware, very high VRAM, GB200, or multi-GPU requirements. It prevents the Worker from optimistically burning an A10G on a Space that Pi already documented as too large. """ model_analysis = model_analysis or {} text = "\n".join([ _workspace_text_blob(workspace, ["TECHNICAL_BLOCKERS.json", "PI_SUMMARY.md", "README.md", "INFERENCE_CONTRACT.json"]), json.dumps(model_analysis, ensure_ascii=False), ]).lower() if not text.strip(): return {"detected": False, "markers": []} markers = [] checks = { "gb200": "gb200" in text, "multi_gpu": any(x in text for x in ["multi-gpu", "multi gpu", "multiple gpu", "2-gpu", "4-gpu"]), "very_high_vram": bool(re.search(r"(1[0-9]{2}|[2-9][0-9]{2})\s*(gb|gib)", text)) and "vram" in text, "a10g_insufficient": "a10g" in text and any(x in text for x in ["insufficient", "too small", "not enough", "cannot", "won't fit", "will not fit"]), "manual_hardware": any(x in text for x in ["manual hardware", "manual_hardware_required", "requires manual", "not available automatically"]), } markers = [name for name, present in checks.items() if present] # v198.20: do not treat a vague "A10G insufficient" note as a hard # manual-hardware blocker by itself. ZeroGPU is the first-class HF path for # many large demos, and quota/capacity failures for the current user should # not make Pi/Worker rewrite or pre-block a ZeroGPU-compatible app. Require # stronger evidence such as explicit manual hardware, multi-GPU, GB200, or # very high VRAM before stopping automatic deployment. hard_markers = [m for m in markers if m in {"gb200", "multi_gpu", "very_high_vram", "manual_hardware"}] return {"detected": bool(hard_markers), "markers": markers, "hard_markers": hard_markers, "a10g_insufficient_only_is_not_manual_block": markers == ["a10g_insufficient"]} def compute_hardware_intent( workspace: Path, model_analysis: dict | None, preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, try_zero_gpu_first: bool, ) -> dict: """Choose hardware from the generated app contract, not just the model task. v197.3 keeps this conservative: - diagnostic-only contracts with no GPU requirement use cpu-basic; - CPU-only TTS/ONNX apps use cpu-basic; - vLLM/Diffusers/GPU apps keep the existing GPU sequence; - explicit large-hardware blockers avoid A10G optimism and deploy a CPU diagnostic/manual-action Space instead. """ preferred = normalize_auto_space_hardware(preferred_hardware, DEFAULT_PREFERRED_SPACE_HARDWARE) fallback = normalize_auto_space_hardware(fallback_hardware, DEFAULT_FALLBACK_SPACE_HARDWARE) strategy = infer_inference_strategy(workspace) contract_skip = contract_declares_no_full_inference(workspace) requires_gpu = _contract_requires_gpu_value(workspace) recommended_hw = _contract_string_value(workspace, "recommended_target_space_hardware", "recommended_hardware", "minimum_hardware").lower() contract_strategy = _contract_string_value(workspace, "inference_strategy", "strategy").lower() too_large = _too_large_for_standard_single_gpu(workspace, model_analysis) publishable_text = _workspace_text_blob(workspace, ["app.py", "requirements.txt"]).lower() hard_gpu_markers = workspace_hard_gpu_markers(workspace) has_gpu_markers = bool(hard_gpu_markers) has_torch_or_diffusers = any(x in publishable_text for x in ["import torch", "from torch", "diffusers", "diffusionpipeline", "zimagepipeline"]) has_vllm = "vllm" in publishable_text or strategy.get("primary") == "vllm_local_server" or contract_strategy == "vllm_local_server" cpu_contract = (requires_gpu is False and recommended_hw == "cpu-basic") or contract_strategy in {"tts_sdk_cpu", "onnx_runtime"} cpu_tts = (strategy.get("primary") in {"tts_sdk_cpu", "onnx_runtime"} or contract_strategy in {"tts_sdk_cpu", "onnx_runtime"}) and not has_gpu_markers and not has_vllm and (requires_gpu is not True) intent = { "schema_version": "1.0", "intent": "gpu_best_effort", "preferred_hardware": preferred, "fallback_hardware": fallback, "allow_fixed_gpu_fallback": bool(allow_fixed_gpu_fallback), "try_zero_gpu_first": bool(try_zero_gpu_first), "manual_hardware_required": False, "reason": "Default full-inference hardware strategy preserved.", "confidence": "medium", "strategy": strategy, "signals": { "contract_declared_no_full_inference": bool(contract_skip.get("declared")), "contract_requires_gpu": requires_gpu, "too_large_for_standard_single_gpu": too_large, "has_gpu_markers": has_gpu_markers, "hard_gpu_markers": hard_gpu_markers, "has_torch_or_diffusers": has_torch_or_diffusers, "has_vllm": has_vllm, "contract_strategy": contract_strategy, "recommended_target_space_hardware": recommended_hw, "cpu_contract": cpu_contract, "cpu_tts_or_onnx": cpu_tts, }, } if contract_skip.get("declared") and requires_gpu is not True: intent.update({ "intent": "diagnostic_cpu_basic", "preferred_hardware": "cpu-basic", "fallback_hardware": "cpu-basic", "allow_fixed_gpu_fallback": False, "try_zero_gpu_first": False, "reason": "Diagnostic-only contract declares no full inference and no GPU requirement; deploy the bootable diagnostic Space on CPU.", "confidence": "high", }) elif cpu_tts or (cpu_contract and not has_gpu_markers and not has_vllm and not has_torch_or_diffusers): intent.update({ "intent": "cpu_basic", "preferred_hardware": "cpu-basic", "fallback_hardware": "cpu-basic", "allow_fixed_gpu_fallback": False, "try_zero_gpu_first": False, "reason": "CPU-only contract/strategy (TTS/ONNX/lazy SDK) has no hard GPU markers; prefer cpu-basic over broad generated-text heuristics.", "confidence": "high", "decision_source": "cpu_only_contract_override" if cpu_contract else "cpu_tts_or_onnx_strategy", }) elif too_large.get("detected"): # v198.26.19: a manual-hardware requirement must not demote a real GPU # app to cpu-basic or diagnostic output. Keep the user-action path # pointed at the original GPU candidate so changing hardware can make # the published Space validatable. manual_preferred = normalize_auto_space_hardware(recommended_hw or preferred, preferred) if manual_preferred == "cpu-basic" and (has_gpu_markers or has_torch_or_diffusers or requires_gpu is True): manual_preferred = preferred if preferred != "cpu-basic" else DEFAULT_PREFERRED_SPACE_HARDWARE manual_fallback = fallback if fallback != "cpu-basic" else DEFAULT_FALLBACK_SPACE_HARDWARE intent.update({ "intent": "manual_hardware_required", "preferred_hardware": manual_preferred, "fallback_hardware": manual_fallback, "allow_fixed_gpu_fallback": True, "try_zero_gpu_first": bool("zero" in manual_preferred or "zero" in preferred or try_zero_gpu_first), "manual_hardware_required": True, "reason": "Generated artifacts document hardware requirements beyond automatic capacity; publish the full-inference candidate and require manual hardware only if automatic assignment fails.", "confidence": "high", "decision_source": "manual_hardware_preserve_full_candidate", }) elif has_vllm or has_gpu_markers or has_torch_or_diffusers or requires_gpu is True: intent.update({ "intent": "gpu_best_effort", "reason": "Generated app contains GPU/vLLM/Diffusers/Torch inference markers; preserve ZeroGPU/fixed-GPU fallback strategy.", "confidence": "high" if (has_vllm or has_gpu_markers) else "medium", }) return intent def apply_hardware_intent( intent: dict, preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, try_zero_gpu_first: bool, ) -> tuple[str, str, bool, bool]: if not isinstance(intent, dict): return preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first return ( intent.get("preferred_hardware") or preferred_hardware, intent.get("fallback_hardware") or fallback_hardware, bool(intent.get("allow_fixed_gpu_fallback")) if "allow_fixed_gpu_fallback" in intent else allow_fixed_gpu_fallback, bool(intent.get("try_zero_gpu_first")) if "try_zero_gpu_first" in intent else try_zero_gpu_first, ) def sanity_check_repair_workspace(workspace: Path, implementation_mode: str, classification: dict | None = None, before_dir: Path | None = None, failure_context: str = "") -> tuple[bool, str]: app_path = workspace / "app.py" req_path = workspace / "requirements.txt" readme_path = workspace / "README.md" for required in [app_path, req_path, readme_path]: if not required.exists() or not required.read_text(encoding="utf-8", errors="ignore").strip(): return False, f"Repair removed or emptied required file: {required.name}" try: compile(app_path.read_text(encoding="utf-8", errors="ignore"), str(app_path), "exec") except SyntaxError as exc: return False, f"Repair app.py still does not compile: {exc.__class__.__name__}: {exc}" if implementation_mode == "full-inference-gated": app_text = app_path.read_text(encoding="utf-8", errors="ignore") app_low = app_text.lower() # Only block obvious fake-return implementations. Do not fail a valid # targeted runtime patch just because a comment/README still contains # words like "placeholder". The previous broad string scan blocked the # exact kind of repair Pi should be allowed to do. hard_fake_patterns = [ r"return\s+[\"'](?:placeholder|dummy image|fake inference|not implemented|canned response)", r"def\s+placeholder\s*\(", r"gr\.button\([^\n]*placeholder", ] if any(re.search(pattern, app_low) for pattern in hard_fake_patterns): return False, "Strict inference repair appears to introduce executable placeholder/fake inference markers." # Legacy anchor: real_inference_markers used to be a Diffusers-only list. # v197.2 keeps the phrase for old regression tests but replaces the # behavior with strategy-aware inference detection. real_inference_markers = [] strategy_before = infer_inference_strategy(before_dir) if before_dir and before_dir.exists() else {} strategy_after = infer_inference_strategy(workspace) ui_kwarg = extract_gradio_unexpected_kwarg(failure_context) if ui_kwarg: kwarg = ui_kwarg.get("kwarg") or "" if kwarg and re.search(rf"\b{re.escape(kwarg)}\s*=", app_text): return False, f"UI component repair did not remove unsupported Gradio kwarg: {kwarg}" before_primary = strategy_before.get("primary") after_primary = strategy_after.get("primary") if before_primary and before_primary != "unknown" and after_primary == "unknown": return False, "UI-only repair removed the previously detected inference strategy." return True, f"UI component repair sanity passed; preserved inference strategy {after_primary or before_primary or 'unknown'}." if not strategy_has_real_inference_path(strategy_after): return False, f"Strict inference repair no longer shows a real model inference path for strategy {strategy_after.get('primary')}." if strategy_before.get("primary") and strategy_before.get("primary") not in {"unknown", "diagnostic_only"}: if strategy_after.get("primary") == "unknown": return False, "Strict inference repair removed the previously detected inference strategy." return True, "Repair workspace sanity checks passed." def _compact_error_text(text: str, limit: int = 2400) -> str: return re.sub(r"\s+", " ", (text or "").strip())[:limit] def compute_failure_signature(failure_reason: str = "", build_log: str = "", runtime_log: str = "", generation_smoke: dict | None = None, classification: dict | None = None) -> str: """Return a stable, small signature used to avoid repeating failed repairs. This is intentionally conservative: keep the actionable exception/package/CLI, strip timestamps/noise, and prefer structured smoke/classification evidence. """ classification = classification or {} if isinstance(generation_smoke, dict): failure_class = generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "" if failure_class: missing = generation_smoke.get("missing_executable") or generation_smoke.get("missing_module") or "" return f"{failure_class}:{missing}" if missing else str(failure_class) err = generation_smoke.get("error") or generation_smoke.get("message") or "" else: err = "" combined = "\n".join(str(x or "") for x in [err, failure_reason, build_log, runtime_log]) low = combined.lower() patterns = [ (r"peft backend is required for this method", "missing_python_dependency_peft"), (r"modulenotfounderror:\s*no module named ['\"]([^'\"]+)['\"]", "missing_python_dependency"), (r"no such file or directory:\s*['\"]([^'\"]+)['\"]", "missing_runtime_cli"), (r"value:\s*([^\n]+?)\s+is not in the list of choices", "validator_schema_choice_type_mismatch"), (r"requires\s+([a-zA-Z0-9_.-]+)\s*([<>=!~]+[^\s,;]+)", "dependency_conflict"), (r"(cuda out of memory|outofmemoryerror)", "cuda_oom"), (r"(scheduling failure|not enough hardware capacity|unable to schedule)", "infra_transient_scheduling"), (r"(model not initialized)", "model_not_initialized"), (r"(typeerror|importerror|syntaxerror|runtimeerror|attributeerror):\s*([^\n]+)", "app_runtime_error"), ] for pattern, prefix in patterns: m = re.search(pattern, combined, re.IGNORECASE) if m: detail = m.group(1) if m.groups() else m.group(0) if prefix == "missing_python_dependency_peft": return "missing_python_dependency:peft" return f"{prefix}:{_compact_error_text(detail, 220)}" cat = classification.get("category") or classification.get("failure_class") or classification.get("failure_type") or "unknown" return f"{cat}:{_compact_error_text(combined, 280)}" def read_latest_space_log_text(run_dir: Path) -> tuple[str, str, str]: """Read the most recently collected Space run/build/actionable logs. Post-repair validation failures must be diagnosed from refreshed HF Space logs, not from the pre-repair build_log/runtime_log variables. """ def _read(rel: str, limit: int = 12000) -> str: path = run_dir / rel try: if path.exists(): return path.read_text(encoding="utf-8", errors="ignore")[-limit:] except Exception: return "" return "" runtime = _read("logs/space_logs_run.txt") or _read("logs/space_logs_runtime.txt") build = _read("logs/space_logs_build.txt") brief = _read("repair/actionable_error_brief.md", 6000) return build, runtime, brief def compute_latest_failure_signature(run_dir: Path, failure_reason: str = "", classification: dict | None = None) -> str: latest_build, latest_runtime, latest_brief = read_latest_space_log_text(run_dir) return compute_failure_signature( "\n".join(x for x in [failure_reason, latest_brief] if x), latest_build, latest_runtime, classification=classification or {}, ) def apply_deterministic_diffusers_lora_dependency_policy(workspace: Path, run_dir: Path, events_path: Path) -> dict: """Ensure Diffusers LoRA apps include PEFT before build/runtime. Diffusers `load_lora_weights()` raises `ValueError: PEFT backend is required for this method` when PEFT is absent. This is deterministic and should not require Pi to discover after one failed rebuild. """ app_path = workspace / "app.py" req_path = workspace / "requirements.txt" app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" deps_text = "" deps_path = workspace / "MODEL_DEPENDENCIES.json" if deps_path.exists(): deps_text = deps_path.read_text(encoding="utf-8", errors="ignore") combined = (app_text + "\n" + deps_text).lower() needs_peft = any(x in combined for x in ["load_lora_weights", "peft_type", "lora_adapter", "diffusers_lora", "adapter_config.json"]) result = { "schema_version": "diffusers_lora_dependency_policy.v198_21", "needs_peft": bool(needs_peft), "peft_present_before": False, "peft_added": False, } if not needs_peft: write_json(run_dir / "diffusers_lora_dependency_policy.json", result) return result if not req_path.exists(): req_path.write_text("peft\n", encoding="utf-8") result.update({"peft_added": True}) else: raw = req_path.read_text(encoding="utf-8", errors="ignore") lines = raw.splitlines() has_peft = requirements_has_package(lines, "peft") result["peft_present_before"] = bool(has_peft) if not has_peft: req_path.write_text(raw.rstrip() + "\npeft\n", encoding="utf-8") result["peft_added"] = True write_json(run_dir / "diffusers_lora_dependency_policy.json", result) if result["peft_added"]: append_event(events_path, "requirements_sanitize", "success", "Added peft for Diffusers LoRA load_lora_weights support", result) return result def load_repair_attempts(run_dir: Path) -> dict: path = run_dir / "repair" / "repair_attempts.json" try: if path.exists(): payload = json.loads(path.read_text(encoding="utf-8")) if isinstance(payload, dict): payload.setdefault("attempts", []) payload.setdefault("max_pi_repair_attempts", MAX_PI_REPAIR_ATTEMPTS) return payload except Exception: pass return { "schema_version": "repair_attempts.v198_21", "max_pi_repair_attempts": MAX_PI_REPAIR_ATTEMPTS, "attempts_used": 0, "remaining_attempts": MAX_PI_REPAIR_ATTEMPTS, "attempts": [], "repeated_failure_signatures": [], "repair_budget_exhausted": False, } def write_repair_attempts(run_dir: Path, payload: dict) -> dict: payload = dict(payload or {}) attempts = payload.get("attempts") if isinstance(payload.get("attempts"), list) else [] payload["schema_version"] = "repair_attempts.v198_21" payload["max_pi_repair_attempts"] = int(payload.get("max_pi_repair_attempts") or MAX_PI_REPAIR_ATTEMPTS) payload["attempts_used"] = len([a for a in attempts if a.get("consumes_pi_budget", True)]) payload["remaining_attempts"] = max(0, payload["max_pi_repair_attempts"] - payload["attempts_used"]) payload["repair_budget_exhausted"] = payload["remaining_attempts"] <= 0 write_json(run_dir / "repair" / "repair_attempts.json", payload) write_json(run_dir / "repair_budget.json", { "schema_version": "repair_budget.v198_21", "max_pi_repair_attempts": payload["max_pi_repair_attempts"], "pi_repair_attempts_used": payload["attempts_used"], "remaining_pi_repair_attempts": payload["remaining_attempts"], "validator_self_repairs_do_not_consume_pi_budget": True, "runtime_recovery_does_not_consume_pi_budget": True, "repair_budget_exhausted": payload["repair_budget_exhausted"], "last_failure_signature": (attempts[-1].get("new_failure_signature") or attempts[-1].get("failure_signature")) if attempts else "", }) return payload def render_repair_history_md(payload: dict) -> str: attempts = payload.get("attempts") if isinstance(payload.get("attempts"), list) else [] lines = ["# Repair history", "", f"Max Pi repair attempts: {payload.get('max_pi_repair_attempts', MAX_PI_REPAIR_ATTEMPTS)}", f"Attempts used: {len(attempts)}", ""] if not attempts: lines.append("No previous Pi repair attempts have been made.") for attempt in attempts: n = attempt.get("attempt") or "?" lines += [ f"## Attempt {n}", f"Trigger: {attempt.get('trigger') or ''}", f"Failure class: {attempt.get('failure_class') or ''}", f"Failure signature: {attempt.get('failure_signature') or ''}", f"Repair decision: {attempt.get('repair_decision') or ''}", f"Files changed: {', '.join(attempt.get('files_changed') or [])}", f"Diff applied: {attempt.get('diff_applied')}", f"Post-repair result: {attempt.get('post_repair_result') or ''}", f"New failure signature: {attempt.get('new_failure_signature') or ''}", "", ] return "\n".join(lines).rstrip() + "\n" def write_repair_history(run_dir: Path, payload: dict) -> None: write_repair_attempts(run_dir, payload) (run_dir / "repair").mkdir(parents=True, exist_ok=True) (run_dir / "repair" / "REPAIR_HISTORY.md").write_text(render_repair_history_md(payload), encoding="utf-8") def next_pi_repair_attempt(run_dir: Path) -> tuple[int, dict]: payload = load_repair_attempts(run_dir) attempts_used = len([a for a in payload.get("attempts", []) if a.get("consumes_pi_budget", True)]) return attempts_used + 1, payload def same_failure_signature_repeated(run_dir: Path, signature: str, *, threshold: int = 2) -> bool: if not signature: return False payload = load_repair_attempts(run_dir) signatures = [] for attempt in payload.get("attempts", []): for key in ("failure_signature", "new_failure_signature"): value = attempt.get(key) if value: signatures.append(str(value)) return signatures.count(str(signature)) >= threshold def append_repair_attempt_record(run_dir: Path, *, attempt: int, trigger: str, failure_class: str, failure_signature: str, repair_decision: str = "", files_changed: list[str] | None = None, diff_applied: bool | None = None, post_repair_result: str = "", new_failure_signature: str = "", consumes_pi_budget: bool = True, notes: str = "") -> dict: payload = load_repair_attempts(run_dir) attempts = payload.setdefault("attempts", []) existing_index = None for idx, item in enumerate(attempts): if int(item.get("attempt") or -1) == int(attempt): existing_index = idx break record = { "attempt": int(attempt), "trigger": trigger, "failure_class": failure_class, "failure_signature": failure_signature, "repair_decision": repair_decision, "files_changed": files_changed or [], "diff_applied": diff_applied, "post_repair_result": post_repair_result, "new_failure_signature": new_failure_signature, "consumes_pi_budget": bool(consumes_pi_budget), "notes": notes, "updated_at": now(), } if existing_index is None: attempts.append(record) else: merged = dict(attempts[existing_index]) merged.update({k: v for k, v in record.items() if v not in (None, "", [])}) attempts[existing_index] = merged write_repair_history(run_dir, payload) return payload def repair_attempt_context_section(run_dir: Path, attempt: int, failure_signature: str, classification: dict, decision: dict | None) -> str: payload = load_repair_attempts(run_dir) max_attempts = int(payload.get("max_pi_repair_attempts") or MAX_PI_REPAIR_ATTEMPTS) remaining_after_this = max(0, max_attempts - attempt) history = render_repair_history_md(payload) repeated_note = "" if same_failure_signature_repeated(run_dir, failure_signature, threshold=1): repeated_note = "\nWARNING: A similar failure signature already appeared earlier. Do not repeat the previous patch. Choose a different minimal fix or declare a blocker.\n" return f""" Repair attempt context: - This is Pi repair attempt {attempt} of {max_attempts}. - Remaining Pi repair attempts after this one: {remaining_after_this}. - Current failure signature: {failure_signature or 'unknown'}. - Current failure class: {classification.get('category') or classification.get('failure_class') or 'unknown'}. - Current repair decision/action: {(decision or {}).get('action') or (decision or {}).get('repair_decision') or 'patch_code'}. {repeated_note} Previous repair history: {history} Rules for this attempt: - Do not repeat or undo previous patches unless the history shows they were wrong. - Preserve the current inference strategy and demo promise. - Make the smallest publishable file change that addresses the current failure signature. - If the same failure would require a broad rewrite or unsupported hardware/manual access, declare a blocker instead of patching blindly. """.strip() def repair_workspace_with_pi(workspace: Path, run_dir: Path, events_path: Path, pi_model: str, target_space_id: str, model_id: str, failure_reason: str, implementation_mode: str = "full-inference-gated", expected_output_type: str = "any", decision: dict | None = None, repair_attempt: int | None = None, repair_trigger: str = "live_validation_failure"): """Structured contextual patch pass, bounded by the multi-attempt Pi repair budget.""" logs_dir = run_dir / "logs" build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else "" runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" repair_dir = run_dir / "repair" before_dir = repair_dir / "before" after_dir = repair_dir / "after" if before_dir.exists(): shutil.rmtree(before_dir) repair_dir.mkdir(parents=True, exist_ok=True) shutil.copytree(workspace, before_dir, ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__", "*.pyc")) classification = (decision or {}).get("classification") or classify_repair_failure(failure_reason, build_log, runtime_log) failure_signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification) if failure_owner_from_classification(classification) == "factory_validation_client": append_event(events_path, "repair_policy", "failed", "Refusing Pi repair because failure belongs to the ASF validation client", {"failure_owner": "factory_validation_client", "category": classification.get("category"), "failure_signature": failure_signature}) return False if repair_attempt is None: repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) else: _repair_payload = load_repair_attempts(run_dir) if repair_attempt > int(_repair_payload.get("max_pi_repair_attempts") or MAX_PI_REPAIR_ATTEMPTS): append_event(events_path, "repair_budget", "failed", "Pi repair budget exhausted before launching another repair attempt", {"attempt": repair_attempt, "max_pi_repair_attempts": _repair_payload.get("max_pi_repair_attempts")}) write_repair_history(run_dir, _repair_payload) return False if same_failure_signature_repeated(run_dir, failure_signature, threshold=2): append_event(events_path, "repair_budget", "failed", "Repeated failure signature reached the safe repair limit; refusing another identical patch", {"failure_signature": failure_signature, "attempt": repair_attempt}) write_repair_history(run_dir, _repair_payload) return False append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or classification.get("failure_class") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", post_repair_result="started") append_event(events_path, "repair_diagnosis", "success", "Patch repair is allowed by Pi diagnosis", {"category": classification.get("category"), "decision": decision or {}, "attempt": repair_attempt, "failure_signature": failure_signature}) write_actionable_error_artifacts(run_dir, failure_reason, build_log, runtime_log, classification) frozen_decision = write_frozen_repair_decision(workspace, run_dir, decision or {"action": "patch_code", "classification": classification}, classification) log_packet_path = run_dir / "repair" / "LOG_EVIDENCE_PACKET.json" if log_packet_path.exists(): shutil.copy2(log_packet_path, workspace / "LOG_EVIDENCE_PACKET.json") repair_task_packet = write_repair_task_packet(workspace, run_dir, classification, failure_reason, build_log, runtime_log, decision) brief = write_repair_brief( workspace, run_dir, target_space_id=target_space_id, model_id=model_id, pi_model=pi_model, failure_reason=failure_reason, build_log=build_log, runtime_log=runtime_log, classification=classification, implementation_mode=implementation_mode, expected_output_type=expected_output_type, decision=decision, ) append_event(events_path, "repair_task_packet", "success", "Repair task packet generated for targeted Pi patch", {"task_type": repair_task_packet.get("task_type"), "allowed_files": repair_task_packet.get("allowed_files"), "packet": "repair/REPAIR_TASK_PACKET.json"}) append_event(events_path, "repair_brief", "success", "Repair brief generated for minimal patch", {"category": classification.get("category"), "brief": "repair/REPAIR_BRIEF.md"}) goal = f"""You are Pi in STRUCTURED REPAIR MODE / STRUCTURED PATCH REPAIR MODE for Agentic Space Factory. First read `REPAIR_TASK_PACKET.json`, `RECIPE_AWARE_REPAIR_PACKET.json`, `FROZEN_REPAIR_DECISION.json`, and `LOG_EVIDENCE_PACKET.json`. `MODEL_RECIPE.json` and `APP_RUNTIME_CONTRACT.json` are authoritative for model-family repair. Do not reinterpret the whole run. Only read `REPAIR_BRIEF.md`, `INCIDENT_BRIEF.md`, and `DEPENDENCY_ERROR_BRIEF.md` if the packet is insufficient or the packet explicitly requests deep_repair_reanalysis. {hf_spaces_guidance_repair_section()} {pi_reference_dependency_repair_section()} {pi_tooling_context_note()} {repair_attempt_context_section(run_dir, repair_attempt, failure_signature, classification, decision)} You are continuing the same build run, not starting a separate project. This patch is allowed only because the diagnosis decision selected `patch_code`. Implement the frozen decision exactly. If you believe the frozen decision is wrong, write `PATCH_REFUSAL.json` with `action=refuse_patch` and do not edit publishable files. If `DEPENDENCY_ERROR_BRIEF.md` exists, treat it as evidence for the gist method: identify the first pip error and patch dependency files minimally. Do not modify inference code or app.py unless REPAIR_TASK_PACKET.json explicitly allows it. If safe dependency-only repair is impossible, write PATCH_REFUSAL.json. Follow the ASF/gist dependency contract during repair: Gradio/HF runtime packages are platform-owned, model-specific pins must be evidence-backed, torch/torchaudio should remain managed by Spaces, and non-pip-installable model code should be vendored instead of referenced through fragile local paths. For native kernel failures such as flash-attn/custom attention/xformers/Triton/fused CUDA ops, do not blindly add source-built native packages. Prefer PyTorch SDPA, compatible wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors when they match the required operation. Use `kernels` only when directly justified and document the backend in REPAIR_SUMMARY.md / INFERENCE_CONTRACT.json. Use the available HF token context to inspect private Hub resources when needed, but never print or persist token values. Critical method: 1. Diagnose the root cause from the brief and current files. 2. Write `REPAIR_PLAN.md` before editing files. 3. Apply the smallest patch possible. 4. Write `REPAIR_SUMMARY.md` after editing. 5. Preserve real inference. Never fake outputs just to satisfy validation. Failure category: {classification.get('category')} Recommended strategy: {classification.get('recommendation')} Model family: {repair_task_packet.get('model_recipe_summary', {}).get('model_family') or ''} Loader strategy: {repair_task_packet.get('model_recipe_summary', {}).get('loader_strategy') or ''} Known failure focus: {(repair_task_packet.get('known_failure_match') or {}).get('repair_focus') or ''} Family directives: {chr(10).join('- ' + str(x) for x in (repair_task_packet.get('family_repair_directives') or []))} Hard constraints: - Do not rebuild from scratch unless you clearly justify it in REPAIR_PLAN.md. - Do not change MODEL_ID `{model_id}` unless you write a technical blocker. - Do not remove the expected Gradio/API contract. - For Strict inference, do not replace inference with placeholders, static sample files, canned text, or broad try/except blocks that hide failures. - Keep or restore a cheap health endpoint with api_name="health". - Keep README metadata valid. - Do not publish anything. Work only in the current workspace. Required deliverables: REPAIR_PLAN.md, patched files, REPAIR_SUMMARY.md. """ (workspace / "REPAIR_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / "REPAIR_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / f"REPAIR_GOAL_attempt_{repair_attempt}.md").write_text(goal, encoding="utf-8") write_repair_history(run_dir, load_repair_attempts(run_dir)) append_event(events_path, "repair_plan", "started", "Running Pi minimal patch repair", {"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type")}) estimate_pi_context_budget(run_dir, workspace, phase="repair_patch", task_type=repair_task_packet.get("task_type") or "runtime_patch", direct_prompt=goal, referenced_files=["REPAIR_TASK_PACKET.json", "RECIPE_AWARE_REPAIR_PACKET.json", "MODEL_RECIPE.json", "APP_RUNTIME_CONTRACT.json", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], max_context_chars=32000, omitted_sections=["full_build_logs", "full_runtime_logs", "full_report", "full_pi_output"], events_path=events_path, artifact_dir=repair_dir) write_agent_trace_record(run_dir, phase="repair_patch", event="command_started", status="started", message="Pi structured patch repair started", data={"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type"), "decision": decision or {}}, artifacts=["repair/REPAIR_TASK_PACKET.json", "repair/RECIPE_AWARE_REPAIR_PACKET.json", "repair/FROZEN_REPAIR_DECISION.json", "repair/LOG_EVIDENCE_PACKET.json", "repair/REPAIR_GOAL.md", "repair/REPAIR_DECISION.json", "repair/PI_PROMPT_BUDGET_repair_patch.json"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=1500) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_repair_output.txt").write_text(out, encoding="utf-8") write_pi_call_fingerprint(run_dir, phase="repair_patch", requested_model=pi_model, prompt_text=goal, output_text=out, task_packet_path=workspace / "REPAIR_TASK_PACKET.json", events_path=events_path, artifact_dir=repair_dir) append_agent_trace_artifact(run_dir, phase="repair_patch", event="repair_output", artifact="logs/pi_repair_output.txt", text=out, status="success" if code == 0 else "failed", data={"returncode": code}) if code != 0: append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", post_repair_result="pi_exit_nonzero", notes=out[-1000:]) append_event(events_path, "repair_patch", "failed", "Pi patch repair returned a non-zero exit code", {"returncode": code, "output_tail": out[-3000:], "attempt": repair_attempt}) return False refusal_path = workspace / "PATCH_REFUSAL.json" if refusal_path.exists(): refusal = load_json_if_exists(refusal_path) or {} write_json(repair_dir / "PATCH_REFUSAL.json", refusal) append_event(events_path, "repair_refusal", "failed", "Pi refused to patch under the bounded repair policy", {"refusal": refusal, "attempt": repair_attempt}) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision="refuse_patch", post_repair_result="pi_refused_patch", notes=str(refusal)[:1000]) return False plan_path = workspace / "REPAIR_PLAN.md" summary_path = workspace / "REPAIR_SUMMARY.md" if not plan_path.exists(): plan_path.write_text(f"# Repair plan\n\nPi did not create a separate plan file. Classified category: `{classification.get('category')}`.\n\nStrategy: {classification.get('recommendation')}\n", encoding="utf-8") if not summary_path.exists(): summary_path.write_text("# Repair summary\n\nPi completed a structured repair pass but did not write a separate summary file. See logs/pi_repair_output.txt.\n", encoding="utf-8") shutil.copy2(plan_path, repair_dir / "REPAIR_PLAN.md") shutil.copy2(summary_path, repair_dir / "REPAIR_SUMMARY.md") write_agent_trace_record(run_dir, phase="repair_patch", event="repair_plan_summary", status="success", message="Repair plan and summary artifacts are available", data={}, artifacts=["repair/REPAIR_PLAN.md", "repair/REPAIR_SUMMARY.md"]) append_event(events_path, "repair_plan", "success", "Repair plan and summary artifacts are available", {"plan": "repair/REPAIR_PLAN.md", "summary": "repair/REPAIR_SUMMARY.md"}) dependency_evidence = f"{failure_reason}\n{build_log}\n{runtime_log}\n" + _read_workspace_text_if_exists(workspace / "REPAIR_PLAN.md") + "\n" + _read_workspace_text_if_exists(workspace / "REPAIR_SUMMARY.md") dependency_issue = extract_pip_dependency_issue(dependency_evidence) if detect_transformers5_dependency_requirement(dependency_evidence): dependency_issue = dependency_issue or {"category": "dependency_error", "package": "transformers"} relax_transformers_upper_bound_for_repair(workspace, events_path, dependency_evidence) normalize_requirements_for_modern_hub(workspace, events_path, dependency_evidence=dependency_evidence) ensure_gradio_launch_show_error(workspace, run_dir, events_path, reason=f"repair_attempt_{repair_attempt}") diff_payload = write_repair_diff_artifact( run_dir, events_path, before_dir, workspace, category=classification.get("category") or "", expected_files=expected_repair_files_from_classification(classification, dependency_issue), ) try: write_json(repair_dir / f"repair_diff_attempt_{repair_attempt}.json", diff_payload) except Exception: pass diff_gate_ok, diff_gate_message, diff_gate_payload = validate_repair_diff_against_task_packet(diff_payload, repair_task_packet, classification, run_dir, events_path) if not diff_gate_ok: append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("changed_files") or [], diff_applied=True, post_repair_result="diff_gate_failed", notes=diff_gate_message) append_event(events_path, "repair_patch", "failed", diff_gate_message, diff_gate_payload) return False quarantine_benign_repair_sidecars(workspace, run_dir, diff_gate_payload, events_path) if not diff_payload.get("has_publishable_diff"): write_agent_trace_record( run_dir, phase="repair_patch", event="repair_noop", status="failed", message="Pi repair produced no publishable workspace diff; refusing redeploy", data=diff_payload, artifacts=["repair/repair_diff.json", "repair/REPAIR_PLAN.md", "repair/REPAIR_SUMMARY.md"], ) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=False, post_repair_result="repair_noop") append_event(events_path, "repair_patch", "failed", "Structured repair produced no publishable workspace diff; refusing redeploy", diff_payload) return False expected_files = diff_payload.get("expected_files") or [] if expected_files and not diff_payload.get("expected_files_touched"): write_agent_trace_record( run_dir, phase="repair_patch", event="repair_expected_file_not_modified", status="failed", message="Pi repair did not modify the expected publishable file; refusing redeploy", data=diff_payload, artifacts=["repair/repair_diff.json", "repair/REPAIR_PLAN.md", "repair/REPAIR_SUMMARY.md"], ) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=True, post_repair_result="expected_file_not_modified") append_event(events_path, "repair_patch", "failed", "Structured repair did not modify the expected publishable file; refusing redeploy", diff_payload) return False strategy_before = infer_inference_strategy(before_dir) strategy_after = infer_inference_strategy(workspace) write_json(repair_dir / "inference_strategy_before.json", strategy_before) write_json(repair_dir / "inference_strategy_after.json", strategy_after) append_event( events_path, "repair_strategy", "success", "Detected inference strategy before and after repair", {"before": strategy_before, "after": strategy_after}, ) sanity_context = f"{failure_reason}\\n{build_log}\\n{runtime_log}\\n" + _read_workspace_text_if_exists(workspace / "REPAIR_PLAN.md") + "\\n" + _read_workspace_text_if_exists(workspace / "REPAIR_SUMMARY.md") ok, sanity_message = sanity_check_repair_workspace(workspace, implementation_mode, classification, before_dir, sanity_context) if not ok: append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=True, post_repair_result="sanity_failed", notes=sanity_message) append_event(events_path, "repair_patch", "failed", sanity_message, {"category": classification.get("category"), "strategy_before": strategy_before, "strategy_after": strategy_after}) return False append_event(events_path, "repair_patch", "success", sanity_message, {"category": classification.get("category"), "strategy_before": strategy_before, "strategy_after": strategy_after}) if after_dir.exists(): shutil.rmtree(after_dir) shutil.copytree(workspace, after_dir, ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__", "*.pyc")) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=True, post_repair_result="patch_ready_for_upload") append_event(events_path, "repair", "success", "Structured repair patch completed; ready to re-upload and revalidate", {"output_tail": out[-3000:], "category": classification.get("category"), "attempt": repair_attempt, "failure_signature": failure_signature}) return True def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, token: str, failure_reason: str, implementation_mode: str, expected_output_type: str): """Let Pi diagnose the blockage, then execute one bounded Factory action at a time. This is the core blockage protocol. Pi decides among allowed actions, but the Factory validates budgets and refuses unsafe patches without actionable logs. """ budgets = {"wait_for_logs": 1, "inspect_more_logs": 1, "factory_rebuild_same_code": 1, "patch_code": MAX_PI_REPAIR_ATTEMPTS} write_repair_history(run_dir, load_repair_attempts(run_dir)) current_error = failure_reason logs_dir = run_dir / "logs" fast_dependency_issue = load_json_if_exists(run_dir / "dependency_issue.json") if (run_dir / "dependency_issue.json").exists() else extract_pip_dependency_issue(current_error) if fast_dependency_issue: append_event(events_path, "space_logs", "skipped", "Using bounded dependency-error evidence; full log collection is deferred to avoid blocking recovery", {"reason": "fast_dependency_issue", "failure_class": fast_dependency_issue.get("failure_class")}) else: collect_space_logs(target_space_id, token, run_dir, events_path) build_log = latest_fresh_collected_build_log(run_dir) or ((logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() and not fast_dependency_issue else "") runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" gated_fast_path = classify_gated_access_error_from_logs(run_dir, model_id=model_id) if gated_fast_path.get("detected"): token_patch = patch_workspace_for_gated_runtime_token(workspace, run_dir, events_path) # If the generated app was not token-aware, this is an actionable Factory patch. # Return a clear failure to the caller so the bounded repair loop can upload the # patched workspace instead of burning a 900s Pi diagnosis on an already-classified 401. write_repair_outcome( run_dir, events_path, repair_trigger="gated_model_access_error", root_cause="gated_or_private_model_access", repair_decision="factory_patch_token_handling" if token_patch.get("applied") else "blocked_gated_access_or_secret_issue", patch_applied=bool(token_patch.get("applied")), upload_success=False, post_repair_validation="not_started", final_user_message="Detected a gated/private base-model access error from Space logs before Pi diagnosis.", validation_error=gated_fast_path.get("evidence_tail", "")[:4000], ) if token_patch.get("applied"): append_event(events_path, "gated_access_fast_path", "success", "Detected gated/private access error and patched app token handling before Pi diagnosis", {"category": gated_fast_path.get("category"), "token_value_logged": False}) configure_space_runtime_secrets(api, target_space_id, token, run_dir, events_path) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) return validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) append_event(events_path, "gated_access_fast_path", "failed", "Detected gated/private access error but app already looked token-aware; manual access/secret issue remains", {"category": gated_fast_path.get("category"), "token_value_logged": False}) write_blockage_artifact(workspace, run_dir, events_path, {"action": "declare_technical_blocker", "classification": gated_fast_path, "patch_allowed": False}, current_error, status="gated_model_access_required") raise RuntimeError("Gated/private model access failed in Space runtime; verify Space secrets and gated model license access.") latest_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {} if not isinstance(latest_runtime, dict): latest_runtime = {} recovery = attempt_runtime_recovery_before_repair( api, workspace, target_space_id, token, run_dir, events_path, failure_reason=current_error, build_log=build_log, runtime_log=runtime_log, runtime_payload=latest_runtime, ) if recovery.get("triggered") and recovery.get("recovery_attempted") and not recovery.get("recovery_exhausted"): append_event(events_path, "repair_validation", "started", "Revalidating after runtime recovery before Pi repair") try: write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) repair_validation = repair_validation_outcome_from_validation(validation, success_message="Runtime recovery resolved the Space startup/scheduling blockage.") write_runtime_recovery(run_dir, {**recovery, "recovery_exhausted": False, "result": "recovered", "post_recovery_validation": repair_validation["post_repair_validation"], "health_semantics": repair_validation.get("health_semantics")}) write_repair_outcome(run_dir, events_path, post_repair_validation=repair_validation["post_repair_validation"], failure_type=repair_validation["failure_type"], final_user_message=repair_validation["final_user_message"], post_repair_health_semantics=repair_validation.get("health_semantics")) append_event(events_path, "runtime_recovery", repair_validation["event_status"], repair_validation["event_message"]) return validation except Exception as exc: current_error = f"{current_error}\n\nRuntime recovery did not resolve validation: {str(exc)[:4000]}" collect_space_logs(target_space_id, token, run_dir, events_path) write_runtime_recovery(run_dir, {**recovery, "recovery_exhausted": True, "result": "post_recovery_validation_failed", "post_recovery_validation": "failed", "validation_error": str(exc)[:4000]}) write_repair_outcome(run_dir, events_path, post_repair_validation="failed_after_runtime_recovery", failure_type="runtime_recovery_exhausted", final_user_message="Runtime recovery was attempted before repair, but validation still failed.", validation_error=str(exc)[:4000]) append_event(events_path, "runtime_recovery", "failed", "Runtime recovery did not resolve validation; continuing to Pi diagnosis", {"error": str(exc)[:4000]}) build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else build_log runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else runtime_log dependency_issue = extract_pip_dependency_issue(f"{current_error}\n{build_log}\n{runtime_log}") if dependency_issue: write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, current_error) write_repair_outcome( run_dir, events_path, repair_trigger="dependency_resolution_error", root_cause="dependency_resolution_error", initial_error=str(current_error)[:4000], patch_applied=False, upload_success=False, post_repair_validation="not_started", ) if apply_dependency_guardrail_repair(workspace, run_dir, events_path, current_error, build_log, runtime_log): write_repair_outcome( run_dir, events_path, repair_trigger="dependency_resolution_error", repair_decision="deterministic_dependency_guardrail", patch_applied=True, upload_success=False, post_repair_validation="not_started", ) append_event(events_path, "factory_rebuild", "started", "Re-uploading dependency-guardrailed workspace after pip resolver build error") if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="dependency_guardrail_repair"): write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="not_started", failure_type="repair_upload_blocked_by_guardrail") raise RuntimeError("Dependency guardrail rebuild skipped by restart guardrails") write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "factory_rebuild", "success", "Dependency-guardrailed workspace uploaded; revalidating live Space") append_event(events_path, "repair_validation", "started", "Revalidating after deterministic dependency repair") write_live_status(run_dir, stage="live_validation", status="running", message="Waiting for Space runtime and health endpoint", data={"target_space": target_space_id}) try: write_auth_probe(run_dir, events_path, "before_initial_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) repair_validation = repair_validation_outcome_from_validation(validation, success_message="Deterministic dependency repair resolved the build blockage.") write_repair_outcome(run_dir, events_path, post_repair_validation=repair_validation["post_repair_validation"], failure_type=repair_validation["failure_type"], final_user_message=repair_validation["final_user_message"], post_repair_health_semantics=repair_validation.get("health_semantics")) append_event(events_path, "repair_validation", repair_validation["event_status"], repair_validation["event_message"]) return validation except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome( run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"], final_user_message=repair_class["message"], validation_error=str(exc)[:4000], ) if repair_class["terminal_status"] == "auth_refresh_required": append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]}) raise AuthRefreshRequired(repair_class["message"]) current_error = f"{current_error}\n\nDependency guardrail rebuild did not resolve validation: {str(exc)[:4000]}" collect_space_logs(target_space_id, token, run_dir, events_path) append_event(events_path, "repair_validation", "failed", "Dependency guardrail rebuild did not resolve validation; falling back to Pi diagnosis", {"error": str(exc)[:4000]}) for iteration in range(1, 5): decision = diagnose_failure_with_pi( workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, iteration, budgets, ) action = decision.get("action") if action in {"wait_for_logs", "inspect_more_logs"}: if budgets.get(action, 0) <= 0: decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + [f"budget_exhausted:{action}"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError(f"Automated recovery stopped after {action} budget was exhausted") budgets[action] -= 1 wait_seconds = int(os.environ.get("SPACE_FACTORY_FAILURE_LOG_WAIT_SECONDS", "45")) append_event(events_path, "wait_for_logs", "started", "Waiting for delayed HF Space logs before changing code", {"seconds": wait_seconds, "decision": decision}) time.sleep(max(1, wait_seconds)) collect_space_logs(target_space_id, token, run_dir, events_path) append_event(events_path, "wait_for_logs", "success", "Collected Space logs after wait", {"remaining_budget": budgets}) current_error = f"{current_error}\n\nAfter wait_for_logs/inspect_more_logs, validation is still considered failed; re-diagnose with refreshed logs." continue if action == "factory_rebuild_same_code": if budgets.get(action, 0) <= 0: decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + ["budget_exhausted:factory_rebuild_same_code"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError("Automated recovery stopped after factory rebuild budget was exhausted") budgets[action] -= 1 append_event(events_path, "factory_rebuild", "started", "Re-uploading the same workspace to force a same-code Space rebuild", {"decision": decision}) if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="pi_decision_factory_rebuild_same_code"): decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + ["same_code_upload_blocked_by_restart_guardrail"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError("Same-code rebuild skipped by restart guardrails") append_event(events_path, "factory_rebuild", "success", "Same-code workspace re-uploaded; revalidating live Space") append_event(events_path, "repair_validation", "started", "Revalidating after same-code factory rebuild") try: write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) repair_validation = repair_validation_outcome_from_validation(validation, success_message="Same-code factory rebuild resolved the blockage.") write_repair_outcome(run_dir, events_path, repair_decision="factory_rebuild_same_code", patch_applied=False, upload_success=True, post_repair_validation=repair_validation["post_repair_validation"], failure_type=repair_validation["failure_type"], final_user_message=repair_validation["final_user_message"], post_repair_health_semantics=repair_validation.get("health_semantics")) append_event(events_path, "repair_validation", repair_validation["event_status"], repair_validation["event_message"]) return validation except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome( run_dir, events_path, repair_decision="factory_rebuild_same_code", patch_applied=False, upload_success=True, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"], final_user_message=repair_class["message"], validation_error=str(exc)[:4000], ) if repair_class["terminal_status"] == "auth_refresh_required": append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]}) raise AuthRefreshRequired(repair_class["message"]) current_error = f"{current_error}\n\nSame-code factory rebuild did not resolve validation: {str(exc)[:4000]}" collect_space_logs(target_space_id, token, run_dir, events_path) append_event(events_path, "repair_validation", "failed", "Same-code factory rebuild did not resolve validation; re-diagnosing", {"error": str(exc)[:4000]}) continue if action == "patch_code": if budgets.get(action, 0) <= 0: decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + ["budget_exhausted:patch_code"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError("Automated recovery stopped after patch budget was exhausted") budgets[action] -= 1 append_event(events_path, "repair", "started", "Pi diagnosis allows a minimal code patch", {"decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure", root_cause=(decision.get("classification") or {}).get("category") or "live_validation_failure", repair_decision="patch_code", decision=decision, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure") if not repaired: terminal = failed_pi_repair_terminal_status(run_dir) retry_decision = should_relaunch_after_no_patch(run_dir, terminal["post_repair_validation"], budgets.get(action, 0)) write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation=terminal["post_repair_validation"], failure_type=terminal["failure_type"], final_user_message=terminal["final_user_message"], latest_repair_attempt=terminal.get("latest_repair_attempt"), recipe_repair_retry=retry_decision) append_event(events_path, "repair_patch", "failed", terminal["final_user_message"], {"decision": decision, "retry_decision": retry_decision}) if retry_decision.get("relaunch"): current_error = current_error + "\n\n" + recipe_repair_direct_followup_reason(run_dir, terminal["post_repair_validation"], (decision.get("classification") or {}).get("category") or "") append_event(events_path, "repair_patch", "started", "Relaunching Pi with a more direct recipe-aware repair brief because budget remains", retry_decision) continue write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="repair_exhausted") append_event(events_path, "failure", "failed", "Structured patch repair exhausted without an uploadable patch", {"decision": decision, "terminal_status": terminal["post_repair_validation"]}) raise RuntimeError(terminal["final_user_message"]) write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "repair_upload", "started", "Uploading repaired workspace") write_auth_probe(run_dir, events_path, "before_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) configure_space_runtime_secrets(api, target_space_id, token, run_dir, events_path) patch_workspace_for_gated_runtime_token(workspace, run_dir, events_path) apply_deterministic_diffusers_lora_dependency_policy(workspace, run_dir, events_path) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "repair_upload", "success", "Repaired workspace uploaded") append_event(events_path, "repair_validation", "started", "Revalidating repaired Space") try: write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) repair_validation = repair_validation_outcome_from_validation(validation, success_message="Repaired Space passed live API validation.") append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure", failure_class=(decision.get("classification") or {}).get("category") or "unknown", failure_signature=compute_failure_signature(current_error, build_log, runtime_log, classification=(decision.get("classification") or {})), repair_decision="patch_code", post_repair_result=repair_validation["post_repair_validation"], new_failure_signature="") write_repair_outcome(run_dir, events_path, post_repair_validation=repair_validation["post_repair_validation"], failure_type=repair_validation["failure_type"], final_user_message=repair_validation["final_user_message"], post_repair_health_semantics=repair_validation.get("health_semantics")) append_event(events_path, "repair_validation", repair_validation["event_status"], repair_validation["event_message"], {"repair_attempt": repair_attempt}) return validation except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome( run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"], final_user_message=repair_class["message"], validation_error=str(exc)[:4000], ) if repair_class["terminal_status"] == "auth_refresh_required": append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]}) raise AuthRefreshRequired(repair_class["message"]) # Refresh Space logs before computing the next signature. v198.21 # fixes a stale-signature bug where the loop kept the original # cuda_oom signature even after the repaired Space surfaced a new # actionable error such as "PEFT backend is required". collect_space_logs(target_space_id, token, run_dir, events_path) new_signature = compute_latest_failure_signature(run_dir, str(exc), classification=repair_class) previous_signature = compute_failure_signature(current_error, build_log, runtime_log, classification=(decision.get("classification") or {})) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure", failure_class=(decision.get("classification") or {}).get("category") or "unknown", failure_signature=previous_signature, repair_decision="patch_code", post_repair_result=repair_class["post_repair_validation"], new_failure_signature=new_signature) latest_build, latest_runtime, latest_brief = read_latest_space_log_text(run_dir) current_error = f"{current_error}\n\nPatch repair attempt {repair_attempt} did not resolve validation: {str(exc)[:4000]}\n\nRefreshed post-repair actionable error:\n{latest_brief[:3000]}" append_event(events_path, "repair_validation", "failed", "Repair attempted, refreshed Space logs, and continuing if repair budget remains for a distinct new failure", {"error": str(exc)[:4000], "repair_attempt": repair_attempt, "previous_failure_signature": previous_signature, "new_failure_signature": new_signature, "remaining_patch_budget": budgets.get("patch_code", 0)}) if budgets.get("patch_code", 0) > 0 and new_signature and new_signature != previous_signature and not same_failure_signature_repeated(run_dir, new_signature, threshold=2): continue if budgets.get("patch_code", 0) > 0 and not same_failure_signature_repeated(run_dir, new_signature, threshold=2): continue write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair_budget_exhausted") append_event(events_path, "failure", "failed", "Run failed after structured repair budget was exhausted or failure repeated", {"repair_error": str(exc)[:4000], "repair_attempt": repair_attempt, "new_failure_signature": new_signature}) raise if action == "request_manual_hardware": write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="manual_hardware_required") append_event(events_path, "manual_hardware_required", "failed", "Pi diagnosis requested manual hardware or quota action", {"decision": decision}) raise RuntimeError("Pi diagnosis requested manual hardware or quota action") write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) append_event(events_path, "failure", "failed", "Pi diagnosis declared a technical blocker", {"decision": decision}) raise RuntimeError("Pi diagnosis declared a technical blocker") final_decision = {"action": "declare_technical_blocker", "reason": "Blockage protocol exhausted all bounded iterations.", "evidence": [], "patch_allowed": False} write_blockage_artifact(workspace, run_dir, events_path, final_decision, current_error) append_event(events_path, "failure", "failed", "Blockage protocol exhausted all bounded iterations") raise RuntimeError("Blockage protocol exhausted all bounded iterations") def configure_space_runtime_secrets(api, target_space_id: str, token: str, run_dir: Path, events_path: Path) -> dict: """Propagate the Job HF token into the generated Space as runtime secrets. Gated base models and private adapters are fetched by the Space runtime, not by the Factory Job. A token that works in the Job is therefore not enough: the generated Space must receive HF_TOKEN/HUGGING_FACE_HUB_TOKEN as secrets. Security invariant: never persist the token value, only secret names/status. """ result = { "schema_version": "space_secrets.v198_20", "target_space_id": target_space_id, "space_secrets_configured": False, "secret_names": [], "token_value_logged": False, "errors": [], } if not token: result["errors"].append({"secret": "HF_TOKEN", "error": "missing_job_token"}) write_json(run_dir / "space_secrets_status.json", result) append_event(events_path, "space_secrets", "failed", "Cannot configure generated Space secrets: Job HF token is missing", {k: v for k, v in result.items() if k != "errors"}) return result for name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]: try: api.add_space_secret(repo_id=target_space_id, key=name, value=token, token=token) result["secret_names"].append(name) except Exception as exc: result["errors"].append({"secret": name, **exception_payload(exc)}) result["space_secrets_configured"] = bool(result["secret_names"]) write_json(run_dir / "space_secrets_status.json", result) append_event( events_path, "space_secrets", "success" if result["space_secrets_configured"] else "failed", "Configured generated Space runtime secrets for gated/private model access" if result["space_secrets_configured"] else "Failed to configure generated Space runtime secrets", {"target_space_id": target_space_id, "secret_names": result["secret_names"], "token_value_logged": False, "error_count": len(result["errors"])}, ) return result def _ensure_os_import_and_hf_token_var(app_text: str) -> tuple[str, bool]: changed = False if "import os" not in app_text and "from os import" not in app_text: lines = app_text.splitlines() insert_at = 0 while insert_at < len(lines) and (lines[insert_at].startswith("#!") or lines[insert_at].strip().startswith("#") or not lines[insert_at].strip()): insert_at += 1 lines.insert(insert_at, "import os") app_text = "\n".join(lines) + ("\n" if app_text.endswith("\n") else "") changed = True if "HF_TOKEN = os.environ.get(" not in app_text and "HUGGING_FACE_HUB_TOKEN" not in app_text.split("\n", 40): marker = "HF_TOKEN = os.environ.get(\"HF_TOKEN\") or os.environ.get(\"HUGGING_FACE_HUB_TOKEN\")\n" lines = app_text.splitlines() insert_at = 0 # Place after import block when possible. for i, line in enumerate(lines[:80]): stripped = line.strip() if stripped.startswith("import ") or stripped.startswith("from ") or not stripped: insert_at = i + 1 continue break lines.insert(insert_at, marker.rstrip("\n")) app_text = "\n".join(lines) + ("\n" if app_text.endswith("\n") else "") changed = True return app_text, changed def _inject_token_kwarg_in_simple_call(text: str, call_name: str) -> tuple[str, int]: """Add token=HF_TOKEN to simple non-nested calls when missing. This intentionally avoids complex AST rewriting. It is good enough for the common Diffusers/HF Hub patterns Pi generates, and it refuses calls that already pass token/use_auth_token. """ import re as _re count = 0 pattern = _re.compile(rf"({call_name}\s*\()(?P[^()]{{0,2500}}?)(\))", _re.DOTALL) def repl(match): nonlocal count args = match.group("args") if "token=" in args or "use_auth_token=" in args: return match.group(0) if "HF_TOKEN" in args: return match.group(0) stripped = args.rstrip() if not stripped: new_args = "token=HF_TOKEN" elif "\n" in args: new_args = stripped + ",\n token=HF_TOKEN," else: new_args = stripped + ", token=HF_TOKEN" count += 1 return match.group(1) + new_args + match.group(3) return pattern.sub(repl, text), count def patch_workspace_for_gated_runtime_token(workspace: Path, run_dir: Path, events_path: Path) -> dict: """Make generated apps gated-repo aware without exposing token values. Pi often detects LoRA/base-model dependencies correctly but forgets that the runtime Space needs to pass its secret token into Diffusers/HF Hub loaders. This patch adds HF_TOKEN/HUGGING_FACE_HUB_TOKEN env lookup and injects token=HF_TOKEN into common loader calls when absent. """ app_path = workspace / "app.py" deps = read_json(workspace / "MODEL_DEPENDENCIES.json", {}) if (workspace / "MODEL_DEPENDENCIES.json").exists() else {} app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" combined = json.dumps(deps, ensure_ascii=False).lower() + "\n" + app_text.lower() gated_or_adapter = any(x in combined for x in ["gated", "private", "lora", "adapter", "base_model", "base_model_id", "black-forest-labs/flux.1-dev"]) loader_present = any(x in app_text for x in ["from_pretrained(", "load_lora_weights(", "hf_hub_download(", "snapshot_download("]) result = { "schema_version": "gated_runtime_token_patch.v198_20", "applied": False, "reason": "not_applicable", "gated_or_adapter_detected": bool(gated_or_adapter), "loader_present": bool(loader_present), "calls_patched": {}, "token_value_logged": False, } if not app_path.exists() or not loader_present or not gated_or_adapter: write_json(run_dir / "gated_runtime_token_patch.json", result) return result new_text, changed = _ensure_os_import_and_hf_token_var(app_text) total = 0 for call in ["from_pretrained", "load_lora_weights", "hf_hub_download", "snapshot_download"]: new_text, n = _inject_token_kwarg_in_simple_call(new_text, call) if n: result["calls_patched"][call] = n total += n if changed or total: app_path.write_text(new_text, encoding="utf-8") result.update({"applied": True, "reason": "patched_hf_loader_token_kwargs"}) append_event(events_path, "gated_runtime_token", "success", "Patched generated app to pass Space HF_TOKEN into gated/private HF Hub loaders", {"calls_patched": result["calls_patched"], "token_value_logged": False}) else: result["reason"] = "already_token_aware" append_event(events_path, "gated_runtime_token", "success", "Generated app already appears token-aware for gated/private loaders", {"token_value_logged": False}) write_json(run_dir / "gated_runtime_token_patch.json", result) return result def classify_gated_access_error_from_logs(run_dir: Path, *, model_id: str = "") -> dict: """Detect clear gated/private Hub access failures before invoking Pi diagnosis.""" text = "" for rel in ["repair/actionable_error_brief.md", "logs/space_logs_run.txt", "logs/space_logs_runtime.txt", "logs/space_logs_build.txt"]: path = run_dir / rel if path.exists(): text += "\n" + path.read_text(encoding="utf-8", errors="ignore")[-8000:] lower = text.lower() is_auth = ("401 client error" in lower or "403 client error" in lower or "unauthorized" in lower or "gated repo" in lower or "restricted repo" in lower) is_hf_model = "huggingface.co/" in lower or "hf.co/" in lower or "model_index.json" in lower or "resolve/main" in lower if not (is_auth and is_hf_model): return {"detected": False} # Keep only redacted/actionable context; never include token values. brief = text[-4000:] payload = { "schema_version": "gated_access_fast_path.v198_20", "detected": True, "category": "gated_or_private_model_access", "model_id": model_id, "reason": "HF Hub 401/403/private/gated access error found in Space run/build logs", "recommended_action": "ensure Space secrets HF_TOKEN/HUGGING_FACE_HUB_TOKEN are configured and app loaders pass token=HF_TOKEN; if Job token also cannot access the repo, user must accept license/request access", "token_value_logged": False, "evidence_tail": redact_text(brief)[:4000], } write_json(run_dir / "gated_access_fast_path.json", payload) return payload def upload_workspace(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path): runtime_template_integrity_guard(workspace, run_dir, events_path, reason="upload_workspace_preflight") sanitize_readme_metadata(workspace, events_path) normalize_requirements_for_modern_hub(workspace, events_path) append_event(events_path, "upload_files", "started", "Preparing whitelist runtime payload for generated Space upload") gen_dir = run_dir / "generated" if gen_dir.exists(): shutil.rmtree(gen_dir) shutil.copytree(workspace, gen_dir, ignore=internal_workspace_copy_ignore) scaffold_detection = detect_placeholder_scaffold(workspace) write_json(run_dir / "placeholder_scaffold_detection.json", scaffold_detection) if scaffold_detection.get("detected"): append_event(events_path, "placeholder_scaffold", "warning", "Generated runtime still contains initial scaffold markers", scaffold_detection) payload_dir, manifest = build_runtime_upload_payload(workspace, run_dir, events_path) try: api.upload_folder( folder_path=str(payload_dir), repo_id=target_space_id, repo_type="space", token=token, ignore_patterns=internal_workspace_upload_ignore_patterns(), ) except Exception as exc: classify_space_upload_exception(exc, run_dir, events_path) raise uploaded_files = sorted(item["path"] for item in manifest.get("files", [])) upload_epoch = record_runtime_upload_epoch(run_dir, events_path, target_space_id=target_space_id, manifest=manifest) append_event( events_path, "upload_files", "success", "Uploaded whitelist runtime payload folder", { "file_count": len(uploaded_files), "files_sample": uploaded_files[:50], "total_bytes": manifest.get("total_bytes"), "excluded_count": manifest.get("excluded_count"), "payload_manifest": "runtime_upload_payload_manifest.json", "upload_epoch": upload_epoch, }, ) def load_json_if_exists(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8", errors="replace")) except Exception as exc: return {"parse_error": str(exc), "raw_tail": path.read_text(encoding="utf-8", errors="replace")[-2000:]} def infer_generation_gate(workspace: Path, implementation_mode: str, validation: dict, generation_smoke: dict | None, run_dir: Path, events_path: Path) -> dict: """Classify the run separately from process success. /health passing means the Space boots. It does not mean the generated Space performs model inference. In full-inference-gated mode we require either an actual implementation signal or a machine-readable blocker report. """ app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore") if (workspace / "app.py").exists() else "" summary_text = (workspace / "PI_SUMMARY.md").read_text(encoding="utf-8", errors="ignore") if (workspace / "PI_SUMMARY.md").exists() else "" req_text = (workspace / "requirements.txt").read_text(encoding="utf-8", errors="ignore") if (workspace / "requirements.txt").exists() else "" blockers_path = workspace / "TECHNICAL_BLOCKERS.json" blockers = load_json_if_exists(blockers_path) scaffold_detection = detect_placeholder_scaffold(workspace) combined = (app_text + "\n" + summary_text).lower() blocked_markers = [ "full generation is not implemented", "full generation is intentionally not wired", "full inference is blocked", "returns a detailed diagnostic", "diagnostic report instead", "diagnostic demo", "only available in the official", "available only in the official", "use the official demo instead", "use the official space instead", "redirects to the official", "redirect to the official", "this space does not run inference", "this demo does not run inference", "does not generate real", "does not load model weights", "does not generate real audio", "true inference is available in the official", "placeholder generator", "placeholder generation", "info-only", "not implemented", "cannot run in this environment", "out of scope", ] promise_validation = build_promise_validation_status(workspace, validation, generation_smoke) contract_no_full = bool((promise_validation.get("contract_declares_no_full_inference") or {}).get("declared")) diagnostic_contract = bool(promise_validation.get("diagnostic_only") or promise_validation.get("manual_hardware_required")) smoke_ok = isinstance(generation_smoke, dict) and generation_smoke.get("status") == "success" minimal_smoke_ok = isinstance(generation_smoke, dict) and (generation_smoke.get("status") == "demo_usable_smoke_passed" or generation_smoke.get("demo_usable_smoke_passed") is True) full_inference_requested = implementation_mode in {"full-inference-gated", "full-inference-attempt"} promise_fulfilled = bool(promise_validation.get("promise_fulfilled")) # v198.26.13 contradiction guard: when a canonical live generation smoke # succeeds and the machine-readable contract does not declare no-full # inference, a broad/heuristic diagnostic promise must not manufacture a # blocker. This keeps metrics as evidence while preserving real explicit # blockers (contract_no_full/manual/scaffold). runtime_smoke_contract_positive = bool( full_inference_requested and smoke_ok and not contract_no_full and not promise_validation.get("manual_hardware_required") ) if runtime_smoke_contract_positive and diagnostic_contract and not blockers: promise_validation = { **promise_validation, "promise_validation_status": "fulfilled", "promise_fulfilled": True, "ui_status": "full_inference_success", "ui_badge": "Full inference", "diagnostic_only": False, "reason": "A canonical live generation smoke test passed; heuristic diagnostic wording was downgraded.", "v198_26_10_runtime_smoke_overrode_diagnostic": True, } diagnostic_contract = False promise_fulfilled = True heuristic_marker_detected = any(m in combined for m in blocked_markers) scaffold_detected = bool(scaffold_detection.get("detected")) blocker_source = str(blockers.get("source") or "") if isinstance(blockers, dict) else "" heuristic_blocker_json = bool(blockers) and blocker_source == "worker_heuristic_from_PI_SUMMARY_or_app.py" heuristic_blocker_detected = bool(heuristic_marker_detected or heuristic_blocker_json) contract_blocker_detected = bool(contract_no_full or diagnostic_contract) hard_blocker_detected = bool((bool(blockers) and not heuristic_blocker_json) or contract_blocker_detected or scaffold_detected) blocker_detected = bool(hard_blocker_detected or heuristic_blocker_detected) strong_full_inference_success = bool(full_inference_requested and smoke_ok and promise_fulfilled) recommendation = generation_smoke if isinstance(generation_smoke, dict) else measured_zero_gpu_recommendation(None) health_semantics = validation_health_semantics(validation) health_ok = validation_health_passed(validation) implementation_signals = { "has_spaces_gpu": "@spaces.GPU" in app_text, "has_torch": "torch" in req_text or "import torch" in app_text, "has_diffusers": "diffusers" in req_text or "diffusers" in app_text, "has_video_output_hint": any(x in app_text.lower() for x in ["gr.video", "video", ".mp4", "ffmpeg"]), "health_passed": health_ok, "health_endpoint_reachable": bool(health_semantics.get("endpoint_reachable")), "health_semantic_passed": health_semantics.get("semantic_passed"), "health_semantic_negative_markers": health_semantics.get("negative_markers") or [], "health_load_error": health_semantics.get("load_error") or "", "placeholder_scaffold_detected": scaffold_detected, "generation_smoke_passed": smoke_ok, "demo_usable_smoke_passed": minimal_smoke_ok, "canonical_promise_smoke_passed": smoke_ok, "zero_gpu_duration_measured": recommendation.get("recommendation_confidence") == "measured", } non_blocking_warnings = [] blocking_evidence = [] if heuristic_blocker_detected: non_blocking_warnings.append({ "type": "heuristic_text_warning", "message": "Generated artifacts mention blocker-like language, but this is treated as advisory unless runtime validation fails.", "source": "TECHNICAL_BLOCKERS.json" if heuristic_blocker_json else "PI_SUMMARY.md_or_app.py", "non_blocking": True, }) if scaffold_detected: blocking_evidence.append({ "type": "placeholder_scaffold_deployed", "source": "app.py", "overridable_by_live_smoke": True, "details": scaffold_detection, }) if hard_blocker_detected: blocking_evidence.append({ "type": "contract_or_explicit_blocker", "source": "INFERENCE_CONTRACT.json_or_TECHNICAL_BLOCKERS.json_or_app.py", "overridable_by_live_smoke": True, }) if strong_full_inference_success: status = "full_inference_success" if blocker_detected: message = "Space boots and a live generation smoke test passed. Blocker-like text was detected but downgraded to a non-blocking warning because real inference succeeded." non_blocking_warnings.append({ "type": "runtime_success_overrode_blocker_text", "message": "Live generation success and fulfilled promise take precedence over contradictory blocker heuristics.", "non_blocking": True, }) else: message = "Space boots and a live generation smoke test passed. ZeroGPU duration recommendation was measured from real inference." elif blocker_detected: if promise_validation.get("manual_hardware_required"): status = "manual_hardware_required" message = "Space is actionable, but the model-card promise is deferred until suitable hardware is selected in Space Settings." elif promise_validation.get("diagnostic_only") or (isinstance(generation_smoke, dict) and generation_smoke.get("skip_reason") == "contract_declared_no_full_inference"): status = "technical_blocker_boot_only" if isinstance(generation_smoke, dict) and generation_smoke.get("skip_reason") == "contract_declared_no_full_inference": message = "Generation smoke was skipped by contract because full inference was declared unavailable; this is not a full inference success." else: message = "Space boots/responds as a diagnostic app, but real/full model inference is not implemented; this is not a full inference success." elif scaffold_detected: status = "placeholder_scaffold_deployed" message = "Space boots, but the published app still contains the initial scaffold instead of model-specific inference." elif hard_blocker_detected: status = "technical_blocker" message = "Space boots, but full model inference was not implemented. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md." else: status = "full_inference_candidate_health_passed" message = "Space boots, but live generation smoke did not verify full inference. Blocker-like text was detected and kept as a non-blocking warning until runtime proof exists." elif full_inference_requested: if minimal_smoke_ok: status = "demo_usable_full_promise_not_verified" message = "Space generated output with a reduced minimal smoke payload after the canonical promise smoke failed or exceeded capacity. The demo is usable, but full promise validation remains unverified." else: status = "" message = "" smoke_failure_owner = str((generation_smoke or {}).get("failure_owner") or "") if isinstance(generation_smoke, dict) else "" smoke_failure_class = str((generation_smoke or {}).get("failure_class") or (generation_smoke or {}).get("failure_type") or "") if isinstance(generation_smoke, dict) else "" if not status and health_ok and (smoke_failure_owner == "factory_validation_client" or smoke_failure_class in {"smoke_schema_error", "smoke_input_materialization_failed"}): status = "interactive_app_available_smoke_failed" message = "Space boots and exposes the expected API, but ASF could not verify generation because the automatic smoke payload did not match or materialize for the Gradio schema." elif not status and health_ok and smoke_failure_class in {"timeout", "gradio_hidden_runtime_error"}: status = "manual_test_required_smoke_failed" message = "Space boots and exposes the expected API, but automatic generation smoke did not complete or did not expose a traceback; manual Space Test is recommended." elif not status and not health_ok: status = "partial_validation" if health_semantics.get("endpoint_reachable") and health_semantics.get("semantic_passed") is False: message = "The health endpoint is reachable, but the model reported unhealthy/not-ready status; generation was not verified." else: message = "Live generation smoke did not produce a verified output, and semantic app health was not confirmed." elif not status: status = "full_inference_candidate_health_passed" message = "Space boots, but live generation smoke test did not produce a verified output. ZeroGPU duration recommendation was not measured." else: status = "health_only" message = "Safe scaffold health validation passed. Full inference was not requested." if non_blocking_warnings: warnings_payload = { "schema_version": "inference_warnings.v198_23", "source": "infer_generation_gate", "status": status, "warnings": non_blocking_warnings, } try: (run_dir / "generated").mkdir(parents=True, exist_ok=True) write_json(workspace / "INFERENCE_WARNINGS.json", warnings_payload) write_json(run_dir / "generated" / "INFERENCE_WARNINGS.json", warnings_payload) except Exception: pass if blocker_detected and not blockers and not strong_full_inference_success and hard_blocker_detected: blockers = { "full_inference_implemented": False, "source": "worker_contract_or_diagnostic_blocker", "blockers": [ { "type": "contract_or_diagnostic_blocker", "claim": "Machine-readable contracts state that full inference is blocked/not implemented or generation returns diagnostics/placeholders.", "evidence": "See INFERENCE_CONTRACT.json, DEMO_QUALITY_CONTRACT.json, PI_SUMMARY.md and app.py in generated artifacts.", "severity": "blocking", } ], "required_investigations_for_next_run": [ "Check whether PyTorch SDPA can replace flash-attn calls.", "Check whether HF Kernels flash-attn2/3/4 can replace required flash-attn APIs.", "Verify whether 2-GPU context parallelism is strictly required or can be reduced to a single-GPU smoke test.", ], } (run_dir / "generated").mkdir(parents=True, exist_ok=True) write_json(workspace / "TECHNICAL_BLOCKERS.json", blockers) write_json(run_dir / "generated" / "TECHNICAL_BLOCKERS.json", blockers) gate = { "status": status, "message": message, "implementation_mode": implementation_mode, "blocker_detected": blocker_detected, "hard_blocker_detected": hard_blocker_detected, "heuristic_blocker_detected": heuristic_blocker_detected, "strong_full_inference_success": strong_full_inference_success, "decision_basis": "live_generation_smoke_passed" if strong_full_inference_success else "hard_blocker" if hard_blocker_detected else "heuristic_warning" if heuristic_blocker_detected else "health_or_smoke_status", "blocking_evidence": [] if strong_full_inference_success else blocking_evidence, "non_blocking_warnings": non_blocking_warnings, "implementation_signals": implementation_signals, "validation_method": validation.get("method"), "generation_smoke": generation_smoke, "app_boot_validation_status": promise_validation.get("app_boot_validation_status"), "promise_validation_status": promise_validation.get("promise_validation_status"), "promise_fulfilled": bool(promise_validation.get("promise_fulfilled")), "ui_status": promise_validation.get("ui_status") if promise_validation.get("ui_status") not in {"full_inference_success", "partial_validation"} else status, "ui_badge": promise_validation.get("ui_badge"), "promise_validation": promise_validation, "zero_gpu_duration_recommendation": { "observed_latency_seconds": recommendation.get("observed_latency_seconds"), "recommended_zero_gpu_duration_seconds": recommendation.get("recommended_zero_gpu_duration_seconds"), "recommendation_source": recommendation.get("recommendation_source"), "recommendation_confidence": recommendation.get("recommendation_confidence"), "measurement_note": recommendation.get("measurement_note"), }, "placeholder_scaffold_detection": scaffold_detection, "blockers": blockers, } write_json(run_dir / "inference_gate.json", gate) append_event(events_path, "inference_gate", status, message, gate) return gate def main(): run_id = os.environ["RUN_ID"] hf_username = os.environ.get("HF_USERNAME", "unknown") bucket_source = os.environ.get("BUCKET_SOURCE", "unknown") output_root = Path(os.environ.get("OUTPUT_ROOT", "/output")) target_space_id = os.environ.get("TARGET_SPACE_ID", "") target_space_auto_generated = os.environ.get("TARGET_SPACE_AUTO_GENERATED", "false").lower() in {"1", "true", "yes", "on"} model_id = sanitize_model_id(os.environ.get("MODEL_ID", DEFAULT_MODEL_ID)) pi_model = os.environ.get("PI_MODEL", "Qwen/Qwen3-Coder-Next") preferred_hardware = normalize_auto_space_hardware(os.environ.get("PREFERRED_SPACE_HARDWARE"), DEFAULT_PREFERRED_SPACE_HARDWARE) fallback_hardware = normalize_auto_space_hardware(os.environ.get("FALLBACK_SPACE_HARDWARE"), DEFAULT_FALLBACK_SPACE_HARDWARE) try_zero_gpu_first = os.environ.get("TRY_ZERO_GPU_FIRST", "true").lower() in {"1", "true", "yes", "on"} allow_fixed_gpu_fallback = os.environ.get("ALLOW_FIXED_GPU_FALLBACK", "true").lower() in {"1", "true", "yes", "on"} implementation_mode = os.environ.get("IMPLEMENTATION_MODE", "full-inference-attempt") expected_output_type = os.environ.get("EXPECTED_OUTPUT_TYPE", "any") token = os.environ.get("HF_TOKEN") run_dir = output_root / "runs" / run_id events_path = run_dir / "events.jsonl" state_path = run_dir / "state.json" workspace = Path("/tmp/universal_workspace") ensure_hf_token_context(run_dir, events_path) pi_model_resolution = {"requested_model": pi_model, "configured_model": pi_model, "effective_model": pi_model, "provider": "huggingface", "mismatch": False} append_event(events_path, "bootstrap", "started", "Universal model-card builder worker started", {"model_id": model_id, "target_space_id": target_space_id}) write_json(state_path, {"run_id": run_id, "kind": "universal_model_card_builder", "status": "running", "message": "Attempting Universal model-card builder Space creation", "model_id": model_id, "pi_model": pi_model, "pi_model_resolution": pi_model_resolution, "target_space": target_space_id, "created_by": hf_username, "bucket_source": bucket_source, "created_at": now(), "updated_at": now()}) if not token: fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets") if not TARGET_RE.match(target_space_id): fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space_id": target_space_id}) try: install_python_deps(events_path) from huggingface_hub import HfApi api = HfApi(token=token) whoami = api.whoami(token=token) append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")}) write_auth_probe(run_dir, events_path, "before_space_create", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_space_create"), raise_on_unsafe=True) append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id}) info = api.model_info(model_id, token=token, files_metadata=True) siblings = [getattr(s, "rfilename", "") for s in (info.siblings or [])] analysis = {"model_id": model_id, "pipeline_tag": getattr(info, "pipeline_tag", None), "library_name": getattr(info, "library_name", None), "tags": list(getattr(info, "tags", []) or [])[:100], "siblings": siblings[:160], "default_model_target": model_id == DEFAULT_MODEL_ID, "preferred_hardware": preferred_hardware, "fallback_hardware": fallback_hardware, "allow_fixed_gpu_fallback": allow_fixed_gpu_fallback, "implementation_mode": implementation_mode} analysis_inputs = prepare_build_analysis_inputs(model_id, token, siblings, info, analysis) analysis["model_card_source"] = analysis_inputs.get("model_card_source.json", {}) analysis["model_repo_tree"] = { "file_count": analysis_inputs.get("model_repo_tree.json", {}).get("file_count"), "detected_components": analysis_inputs.get("model_repo_tree.json", {}).get("detected_components", []), } write_json(run_dir / "model_analysis.json", analysis) write_analysis_inputs_dir(run_dir, analysis_inputs) append_event(events_path, "model_analysis", "success", "Model metadata and build-time grounding inputs fetched", {"pipeline_tag": analysis["pipeline_tag"], "library_name": analysis["library_name"], "model_card_source": analysis["model_card_source"].get("source"), "model_card_fallback_used": analysis["model_card_source"].get("fallback_used")}) create_initial_workspace(workspace, model_id, target_space_id, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, implementation_mode, analysis, analysis_inputs) for context_name in ["MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md"]: try: if (workspace / context_name).exists(): shutil.copy2(workspace / context_name, run_dir / context_name) except Exception: pass append_event(events_path, "workspace", "success", "Prepared universal model-card workspace", {"files": sorted(p.name for p in workspace.iterdir())}) install_pi(events_path) configure_pi(events_path, pi_model) append_event(events_path, "pi_run", "started", "Running Pi on universal model-card workspace", {"model": pi_model}) (run_dir / "logs").mkdir(parents=True, exist_ok=True) initial_goal_text = (workspace / "GOAL.md").read_text(encoding="utf-8") estimate_pi_context_budget( run_dir, workspace, phase="initial_build", task_type="generate_gradio_demo", direct_prompt=initial_goal_text, referenced_files=[ "MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md", "analysis_inputs/model_card.md", "analysis_inputs/model_repo_tree.json", "analysis_inputs/source_policy.md", "analysis_inputs/hf_spaces_operational_gist.md", "analysis_inputs/hf_spaces_gist_source.json", ], max_context_chars=90000, omitted_sections=["previous_repair_history", "space_runtime_logs", "space_build_logs"], events_path=events_path, artifact_dir=run_dir, ) code, pi_out = run_cmd_streaming( ["pi", "-p", initial_goal_text], cwd=workspace, timeout=2400, live_log_path=run_dir / "logs" / "pi_live_output.txt", events_path=events_path, run_dir=run_dir, step="pi_run", trace_phase="initial_build", ) (run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8") if code != 0: append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]}) collect_pi_traces(run_dir, events_path) fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]}) append_event(events_path, "pi_run", "success", "Pi completed universal model-card workspace pass", {"output_tail": pi_out[-2000:]}) # Pi sessions are the source of truth for the assistant/model actually used. # Sync them first, then resolve requested/configured/observed model identity. collect_pi_traces(run_dir, events_path) pi_model_resolution = detect_pi_model_resolution(pi_model, run_dir, pi_out) write_pi_call_fingerprint(run_dir, phase="initial_build", requested_model=pi_model, prompt_text=initial_goal_text, output_text=pi_out, events_path=events_path, artifact_dir=run_dir, identity=pi_model_resolution) emit_pi_model_resolution(events_path, pi_model_resolution) if not (workspace / "PI_SUMMARY.md").exists(): (workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8") write_pi_planning_review(workspace, run_dir, events_path, analysis, implementation_mode) app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore") if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text: append_event(events_path, "pi_verification", "failed", "app.py does not appear to expose /health; injecting safe health endpoint is not implemented") fail(run_dir, events_path, "Pi output did not preserve a /health endpoint") early_app_contract_guard = app_runtime_contract_guard(workspace, run_dir, events_path, expected_output_type=expected_output_type, reason="post_pi_pre_integrity") append_event(events_path, "pi_verification", "success" if early_app_contract_guard.get("passed") else "warning", "Pi output preserved health endpoint; structured app contract will be enforced before upload", {"app_runtime_contract_passed": early_app_contract_guard.get("passed"), "problems": early_app_contract_guard.get("problems")}) output_type_resolution = contract_effective_output_type(workspace, expected_output_type) effective_expected_output_type = output_type_resolution.get("effective_expected_output_type") or expected_output_type write_json(run_dir / "output_type_resolution.json", output_type_resolution) append_event(events_path, "output_type_resolution", "success", "Resolved effective output type from generated contract", output_type_resolution) hardware_intent = compute_hardware_intent( workspace, analysis, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first, ) write_json(run_dir / "hardware_intent.json", hardware_intent) manual_block = should_stop_before_cpu_deploy_for_manual_hardware(workspace, hardware_intent) write_json(run_dir / "manual_hardware_block.json", manual_block) manual_hardware_actionable = False manual_hardware_actionable_payload = {} if manual_block.get("triggered"): append_event(events_path, "manual_hardware_required", "warning", "Manual hardware required; creating an actionable Space instead of stopping before Space creation", manual_block) manual_hardware_actionable_payload = prepare_manual_hardware_actionable_workspace( workspace, run_dir, events_path, model_id=model_id, target_space_id=target_space_id, hardware_intent=hardware_intent, manual_block=manual_block, ) manual_hardware_actionable = True # v198.26.19 keeps the full-inference app as the publishable payload. # Re-resolve the output type only as a guardrail: it should remain # the candidate's original type (for example image), not text_diagnostic. output_type_resolution = contract_effective_output_type(workspace, expected_output_type) effective_expected_output_type = output_type_resolution.get("effective_expected_output_type") or expected_output_type write_json(run_dir / "output_type_resolution.json", output_type_resolution) append_event(events_path, "output_type_resolution", "success", "Resolved effective output type from preserved full-inference candidate", output_type_resolution) preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first = apply_hardware_intent( hardware_intent, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first, ) append_event(events_path, "hardware_intent", "success", "Resolved contract-aware hardware intent", hardware_intent) append_event(events_path, "hardware_strategy", "started", "Creating Space with contract-aware hardware strategy", {"preferred_hardware": preferred_hardware, "fallback_hardware": fallback_hardware, "allow_fixed_gpu_fallback": allow_fixed_gpu_fallback, "try_zero_gpu_first": try_zero_gpu_first, "hardware_intent": hardware_intent}) hardware_strategy = create_space_with_hardware_strategy( api, target_space_id, token, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, events_path, allow_auto_rename=target_space_auto_generated, try_zero_gpu_first=try_zero_gpu_first, ) target_space_id = hardware_strategy.get("target_space_id") or target_space_id selected_hardware = hardware_strategy.get("selected_hardware") or "default-cpu-or-existing" hardware_attempts = list(hardware_strategy.get("attempts") or []) requested_hardware_sequence = list(hardware_strategy.get("requested_sequence") or []) update_state(run_dir, { "run_id": run_id, "kind": "universal_model_card_builder", "status": "running", "message": "Private Space created; uploading generated workspace", "model_id": model_id, "pi_model": pi_model, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "selected_hardware": selected_hardware, "hardware_intent": hardware_intent, "hardware_attempts": hardware_attempts, "requested_hardware_sequence": requested_hardware_sequence, "model_expected_output_type": output_type_resolution.get("model_expected_output_type"), "contract_effective_output_type": output_type_resolution.get("contract_effective_output_type"), "expected_output_type": effective_expected_output_type, "created_by": hf_username, "bucket_source": bucket_source, }) write_artifact_manifest(run_dir, reason="space_created") # Upload after create. If create_repo(space_hardware=...) succeeded, the build # starts directly on the requested hardware. If it fell back to CPU, the run # remains valid but will be marked manual_hardware_required when inference # signals indicate GPU is needed. write_auth_probe(run_dir, events_path, "before_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_upload"), raise_on_unsafe=True) configure_space_runtime_secrets(api, target_space_id, token, run_dir, events_path) patch_workspace_for_gated_runtime_token(workspace, run_dir, events_path) ensure_gradio_launch_show_error(workspace, run_dir, events_path, reason="initial_pre_upload") apply_deterministic_diffusers_lora_dependency_policy(workspace, run_dir, events_path) patch_workspace_for_writable_hf_cache(workspace, run_dir, events_path, reason="initial_pre_upload_guard") if manual_hardware_actionable: actionability_check = manual_hardware_publish_actionability_check(workspace, hardware_intent, manual_block, run_dir, events_path) if not actionability_check.get("actionable"): write_json(run_dir / "candidate_deployment_guard.json", actionability_check) fail(run_dir, events_path, "Manual hardware path is not actionable: publishable workspace is not the full-inference candidate") ensure_runtime_template_uploadable_with_pi_repair( workspace, run_dir, events_path, pi_model=pi_model, target_space_id=target_space_id, model_id=model_id, implementation_mode=implementation_mode, expected_output_type=effective_expected_output_type, hardware_intent=hardware_intent, ) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_artifact_manifest(run_dir, reason="workspace_uploaded") if selected_hardware == "default-cpu-or-existing": append_event(events_path, "hardware", "warning", "Automatic hardware-at-creation failed; Space is on default CPU unless user changes it manually", {"attempts": hardware_attempts}) write_json(run_dir / "hardware_attempts.json", {"selected_hardware": selected_hardware, "requested_sequence": requested_hardware_sequence, "attempts": hardware_attempts, "strategy": "create_repo_space_hardware_first", "try_zero_gpu_first": try_zero_gpu_first, "hardware_intent": hardware_intent}) write_json(run_dir / "hardware_strategy.json", {"selected_hardware": selected_hardware, "requested_sequence": requested_hardware_sequence, "attempts": hardware_attempts, "manual_action_required": selected_hardware == "default-cpu-or-existing" or bool(hardware_intent.get("manual_hardware_required")), "strategy": "create_repo_space_hardware_first", "try_zero_gpu_first": try_zero_gpu_first, "hardware_intent": hardware_intent}) manual_hardware_waiting_for_user = bool(manual_hardware_actionable and selected_hardware == "default-cpu-or-existing") if manual_hardware_waiting_for_user: validation = { "status": "skipped", "api_name": "/health", "reason": "manual_hardware_required_before_live_validation", "message": "Full-inference candidate was published, but automatic hardware assignment failed. Waiting for user to select the recommended hardware before validation.", "semantic_details": {"endpoint_reachable": None, "semantic_passed": None, "model_ready": None}, } write_json(run_dir / "tests" / "test_result.json", validation) append_event(events_path, "validation", "skipped", validation["message"], validation) else: try: validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) except Exception as validation_error: append_event(events_path, "failure_detected", "warning", "Initial live validation failed; entering Pi blockage diagnosis protocol", {"error": str(validation_error)[:2000]}) validation = recover_after_live_validation_failure( api, workspace, run_dir, events_path, pi_model=pi_model, target_space_id=target_space_id, model_id=model_id, token=token, failure_reason=str(validation_error), implementation_mode=implementation_mode, expected_output_type=effective_expected_output_type, ) generation_smoke = None if implementation_mode in {"full-inference-gated", "full-inference-attempt"}: if manual_hardware_waiting_for_user: generation_smoke = measured_zero_gpu_recommendation(None) | { "status": "skipped", "expected_output_type": effective_expected_output_type, "reason": "manual_hardware_required_before_generation_smoke", "message": "Full-inference candidate is deployed; select the recommended hardware before running /generate validation.", } write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke) append_event(events_path, "generation_smoke", "skipped", generation_smoke["message"], generation_smoke) else: contract_skip_reason = contract_declares_no_full_inference(workspace) if contract_skip_reason.get("declared"): generation_smoke = write_contract_skipped_generation_smoke(run_dir, events_path, effective_expected_output_type, target_space_id, contract_skip_reason) else: try: wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, effective_expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, effective_expected_output_type, workspace=workspace) except Exception as smoke_error: write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation smoke test failed", data={"error": str(smoke_error)[:2000]}) smoke_diagnosis = diagnose_generation_smoke_failure(smoke_error, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=effective_expected_output_type, phase="exception") generation_smoke = build_generation_smoke_failure_payload(target_space_id, effective_expected_output_type, smoke_error, workspace=workspace, phase="exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, generation_smoke, phase="generation_smoke_exception") append_event(events_path, "generation_smoke", "failed", "Live generation smoke test failed; ZeroGPU duration was not measured", generation_smoke) else: generation_smoke = measured_zero_gpu_recommendation(None) | {"status": "skipped", "expected_output_type": effective_expected_output_type} write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke) if isinstance(generation_smoke, dict) and generation_smoke.get("status") == "failed": validation, generation_smoke = attempt_generation_smoke_repair( api, workspace, run_dir, events_path, pi_model=pi_model, target_space_id=target_space_id, model_id=model_id, token=token, implementation_mode=implementation_mode, expected_output_type=effective_expected_output_type, validation=validation, generation_smoke=generation_smoke, ) inference_gate = infer_generation_gate(workspace, implementation_mode, validation, generation_smoke, run_dir, events_path) if isinstance(inference_gate, dict): inference_gate["hardware_intent"] = hardware_intent if hardware_intent.get("manual_hardware_required"): inference_gate["manual_hardware_required"] = True inference_gate["hardware_blocker_reason"] = hardware_intent.get("reason") write_json(run_dir / "inference_gate.json", inference_gate) # If the generated app looks like real GPU inference but automatic # hardware requests failed, classify the run honestly as needing manual # hardware instead of pretending CPU/default hardware is enough. the existing-Space validation workflow # can then smoke-test generation after the user sets a GPU manually. manual_hw_required = selected_hardware == "default-cpu-or-existing" and inference_gate.get("status") not in {"technical_blocker", "technical_blocker_boot_only", "health_only"} and ( inference_gate.get("implementation_signals", {}).get("has_spaces_gpu") or inference_gate.get("implementation_signals", {}).get("has_torch") or any((a.get("manual_action_required") for a in hardware_attempts if isinstance(a, dict))) ) if manual_hw_required: inference_gate = dict(inference_gate) inference_gate["status"] = "manual_hardware_required" inference_gate["message"] = "Space was generated and boots, but automatic ZeroGPU/fixed-GPU assignment failed. Set hardware manually, then run the existing-Space validation workflow." inference_gate["manual_hardware_required"] = True inference_gate["hardware_attempts"] = hardware_attempts write_json(run_dir / "inference_gate.json", inference_gate) append_event(events_path, "inference_gate", "manual_hardware_required", inference_gate["message"], inference_gate) if manual_hardware_actionable and manual_hardware_waiting_for_user: inference_gate = dict(inference_gate) inference_gate["status"] = "manual_hardware_required" inference_gate["message"] = "Full-inference candidate was published, but automatic hardware assignment failed. Open the Space settings, select the recommended GPU, restart, then re-run validation." inference_gate["manual_hardware_required"] = True inference_gate["manual_hardware_actionable"] = True inference_gate["manual_hardware_actionable_space"] = manual_hardware_actionable_payload inference_gate["hardware_attempts"] = hardware_attempts inference_gate["published_payload"] = "full_inference_candidate_workspace" write_json(run_dir / "inference_gate.json", inference_gate) append_event(events_path, "inference_gate", "manual_hardware_required", inference_gate["message"], inference_gate) collect_pi_traces(run_dir, events_path) final_state = { "run_id": run_id, "kind": "universal_model_card_builder", "status": inference_gate["status"], "message": inference_gate["message"], "model_id": model_id, "pi_model": pi_model, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "selected_hardware": selected_hardware, "hardware_intent": hardware_intent, "hardware_attempts": hardware_attempts, "model_expected_output_type": output_type_resolution.get("model_expected_output_type"), "contract_effective_output_type": output_type_resolution.get("contract_effective_output_type"), "expected_output_type": effective_expected_output_type, "output_type_resolution": output_type_resolution, "pi_model_resolution": pi_model_resolution, "validation": validation, "generation_smoke": generation_smoke, "inference_gate": inference_gate, "app_boot_validation_status": inference_gate.get("app_boot_validation_status"), "promise_validation_status": inference_gate.get("promise_validation_status"), "promise_fulfilled": bool(inference_gate.get("promise_fulfilled")), "promise_validation": inference_gate.get("promise_validation") or {}, "ui_status": inference_gate.get("ui_status") or inference_gate.get("status"), "ui_badge": inference_gate.get("ui_badge") or "", "manual_hardware_actionable": bool(manual_hardware_actionable), "manual_hardware_actionable_space": manual_hardware_actionable_payload, "updated_at": now(), "created_by": hf_username, "bucket_source": bucket_source, } write_json(state_path, final_state) cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, final_state, token=token) final_state["cleanup"] = cleanup_status final_state = reconcile_final_success_state(run_dir, events_path, final_state, inference_gate, generation_smoke) write_final_summary(run_dir, final_state, inference_gate, generation_smoke) write_live_status(run_dir, stage="done", status=inference_gate["status"], message=inference_gate["message"], data={"target_space": target_space_id, "selected_hardware": selected_hardware, "generation_smoke_status": (generation_smoke or {}).get("status"), "cleanup": cleanup_status, "job_exit_code": final_state.get("job_exit_code")}) report = f"""# Agentic Space Factory — Universal Model-Card Builder Report Run ID: `{run_id}` Status: **{inference_gate['status']}** {inference_gate['message']} Target Space: https://huggingface.co/spaces/{target_space_id} Model: `{model_id}` ## Pi assistant model ```json {json.dumps(pi_model_resolution, indent=2, ensure_ascii=False)} ``` ## Hardware Selected/requested hardware: `{selected_hardware}` Hardware intent: ```json {json.dumps(hardware_intent, indent=2, ensure_ascii=False)} ``` Hardware changes are best-effort with OAuth. If requests fail with 401/auth/billing errors, set the Space hardware manually and rerun validation. ```json {json.dumps(hardware_attempts, indent=2, ensure_ascii=False)} ``` ## Health validation The wrapper validated the live Space using HTTP `/health` first, with Gradio Client as fallback. This only proves bootability. ```json {json.dumps(validation, indent=2, ensure_ascii=False)} ``` ## Live generation smoke / ZeroGPU duration ```json {json.dumps(generation_smoke, indent=2, ensure_ascii=False)} ``` ## Next action {(generation_smoke or {}).get('next_action') or ('The generated Space is running but generation was not fully verified. Use Prefill Space Test to retry with adjusted arguments.' if inference_gate.get('status') == 'full_inference_candidate_health_passed' else 'No manual action is required for this verdict.')} ## Full-inference gate ```json {json.dumps(inference_gate, indent=2, ensure_ascii=False)} ``` ## Pi summary {(workspace / 'PI_SUMMARY.md').read_text(encoding='utf-8', errors='ignore') if (workspace / 'PI_SUMMARY.md').exists() else 'No PI_SUMMARY.md was produced.'} ## Safety - The target Space was created private. - No public publication was attempted. - Only redacted Pi traces are archived. Raw Pi traces are not published to the bucket. - If fallback fixed GPU was used or selected manually, review billing/hardware settings manually after the run. """ (run_dir / "report.md").write_text(report, encoding="utf-8") append_event(events_path, "report_write", "success", "Wrote report.md") write_artifact_manifest(run_dir, events_path=events_path, reason="final") try: publish_eval_record(run_dir, phase="final", events_path=events_path) append_event(events_path, "anonymous_eval", "success", "Published anonymized evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False}) except Exception as eval_exc: append_event(events_path, "anonymous_eval", "warning", "Could not publish anonymized evaluation record", {"error": str(eval_exc)[:1000]}) append_event(events_path, "done", inference_gate["status"], "Universal model-card builder completed", {"target_space": target_space_id, "selected_hardware": selected_hardware, "gate_status": inference_gate["status"]}) except SystemExit: raise except Exception as exc: try: collect_pi_traces(run_dir, events_path) except Exception: pass if final_full_inference_success_recorded(run_dir): existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {} existing_gate = load_json_if_exists(run_dir / "inference_gate.json") if (run_dir / "inference_gate.json").exists() else {} existing_smoke = load_json_if_exists(run_dir / "tests" / "generation_smoke.json") if (run_dir / "tests" / "generation_smoke.json").exists() else {} clean_state = reconcile_final_success_state(run_dir, events_path, existing_state, existing_gate, existing_smoke) append_event(events_path, "final_status_reconciliation", "warning", "Suppressed post-success finalizer exception; final full-inference gate remains authoritative", {"error": str(exc)[:2000], "job_exit_code": 0}) write_final_summary(run_dir, clean_state, existing_gate, existing_smoke, status="full_inference_success", message=clean_state.get("message") or existing_gate.get("message")) try: write_artifact_manifest(run_dir, events_path=events_path, reason="final_success_exception_suppressed") except Exception: pass return terminal_status = "auth_refresh_required" if isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc) else "failed" details = {"error": str(exc)} if terminal_status == "auth_refresh_required": details["failure_type"] = "auth_refresh_required" repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {} if isinstance(repair_outcome, dict) and repair_outcome: details["repair_outcome"] = repair_outcome details["failure_type"] = repair_outcome.get("failure_type") or details.get("failure_type", "") fail(run_dir, events_path, "Universal model-card builder worker failed", details, status=terminal_status) if __name__ == "__main__": main() ''' VALIDATE_EXISTING_SPACE_WORKER_SCRIPT = r''' import ast import base64 import json import os import re import shutil import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path TARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$") def is_sdxl_lora_text_encoder_mismatch_text(text: str) -> bool: lowered = str(text or "").lower() if "cliptextmodel" in lowered and "text_model" in lowered: return True if "object has no attribute" in lowered and "text_model" in lowered and ("sdxl" in lowered or "lora" in lowered or "stable diffusion xl" in lowered): return True return False def is_hf_cache_permission_denied_text(text: str) -> bool: lowered = str(text or "").lower() if "permission denied" not in lowered and "errno 13" not in lowered: return False return any(marker in lowered for marker in ["'/data'", '"/data"', "/data/.cache", "hf_home", "hf_hub_cache", "transformers_cache", "diffusers_cache", "huggingface cache", "huggingface/hub"]) def is_model_pipeline_not_loaded_text(text: str) -> bool: lowered = str(text or "").lower() markers = ["model pipeline is not loaded", "model pipeline not loaded", "pipeline is not loaded", "pipeline not loaded", "pipeline_not_loaded", "model_not_loaded", "model not loaded", "model is not initialized", "model not initialized", "pipe is none", "pipe is not loaded"] return any(marker in lowered for marker in markers) or ("gpu may be unavailable" in lowered and ("pipeline" in lowered or "model" in lowered)) def is_diffusers_duplicate_token_text(text: str) -> bool: lowered = str(text or "").lower() if "multiple values for keyword argument" not in lowered or "token" not in lowered: return False return any(marker in lowered for marker in ["from_pretrained", "diffusionpipeline", "pipeline", "diffusers", "zimagepipeline"]) def smoke_failure_signature(smoke: dict | None) -> str: if not isinstance(smoke, dict): return "" failure_class = str(smoke.get("failure_class") or smoke.get("failure_type") or "").strip().lower() failure_type = str(smoke.get("failure_type") or "").strip().lower() text = str(smoke.get("error") or smoke.get("result_repr") or smoke.get("result_info") or smoke.get("message") or smoke) lowered = text.lower() if is_sdxl_lora_text_encoder_mismatch_text(text): marker = "sdxl_lora_text_encoder_mismatch" elif is_diffusers_duplicate_token_text(text): marker = "duplicate_token" elif is_hf_cache_permission_denied_text(text): marker = "hf_cache_permission_denied" elif is_model_pipeline_not_loaded_text(text): marker = "model_pipeline_not_loaded" else: marker = re.sub(r"\s+", " ", lowered)[:300] return "|".join(part for part in [failure_class, failure_type, marker] if part) def repair_post_smoke_validation_status(pre_repair_smoke: dict | None, post_repair_smoke: dict | None) -> dict: post = post_repair_smoke if isinstance(post_repair_smoke, dict) else {} if post.get("status") == "success": return {"post_repair_validation": "success", "same_error": False, "failure_type": "", "message": "Targeted smoke repair revalidated the Space."} pre_sig = smoke_failure_signature(pre_repair_smoke) post_sig = smoke_failure_signature(post) same = bool(pre_sig and post_sig and pre_sig == post_sig) failure_type = post.get("failure_type") or post.get("failure_class") or "generation_smoke_still_failed" if same: return {"post_repair_validation": "repair_failed_same_error", "same_error": True, "failure_type": failure_type, "message": "Targeted smoke repair uploaded, but the same generation error returned after revalidation.", "pre_repair_signature": pre_sig, "post_repair_signature": post_sig} return {"post_repair_validation": "smoke_still_failed", "same_error": False, "failure_type": failure_type, "message": "Targeted smoke repair uploaded, but generation smoke still did not verify the primary demo flow.", "pre_repair_signature": pre_sig, "post_repair_signature": post_sig} def now(): return datetime.now(timezone.utc).isoformat() def write_json(path: Path, payload: dict): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2, ensure_ascii=False, default=str) + "\n", encoding="utf-8") def read_json(path: Path, default=None): try: if path.exists(): return json.loads(path.read_text(encoding="utf-8")) except Exception: pass return default def write_live_status(run_dir: Path, *, stage: str, status: str = "running", message: str = "", data: dict | None = None): """Persist a tiny status snapshot from the validation worker. v190.32: this helper must exist inside the linked-validation worker script itself. Earlier validation Jobs called write_live_status() after API discovery but the helper was only present in the build worker, causing a NameError before payload resolution. """ payload = { "schema_version": "live_status.v1", "updated_at": now(), "run_id": os.environ.get("RUN_ID", ""), "stage": stage, "status": status, "message": message, "data": data or {}, } try: write_json(run_dir / "live_status.json", payload) except Exception: pass return payload def _first_list_value(*values): for value in values: if isinstance(value, list): return value return None def _first_dict_value(*values): for value in values: if isinstance(value, dict): return value return None def load_parent_replay_source(parent_dir: Path, target_space_id: str, expected_output_type: str) -> dict: """Load the parent build's successful automatic smoke payload as replay truth. In linked Space Test replay mode, the validation worker must not invent a different request when the parent Build Run already proved a working request. The parent smoke artefacts are the source of truth; schema discovery is only used afterwards as a guardrail/retry mechanism. """ if not parent_dir or not parent_dir.exists(): return {} smoke = read_json(parent_dir / "tests" / "generation_smoke.json", {}) or {} retry_payload = read_json(parent_dir / "tests" / "generation_smoke_payload_retry.json", {}) or {} initial_payload = read_json(parent_dir / "tests" / "generation_smoke_payload.json", {}) or {} state = read_json(parent_dir / "state.json", {}) or {} state_smoke = state.get("generation_smoke") if isinstance(state.get("generation_smoke"), dict) else {} status_tokens = { str(smoke.get("status") or "").lower(), str(state.get("status") or "").lower(), str(state_smoke.get("status") or "").lower(), } parent_was_successful = bool(status_tokens & {"success", "full_inference_success", "completed", "complete"}) if not parent_was_successful: return {} api_name = ( smoke.get("api_name") or retry_payload.get("api_name") or initial_payload.get("api_name") or state_smoke.get("api_name") or "" ) test_args = _first_list_value( smoke.get("effective_args"), smoke.get("test_args"), retry_payload.get("test_args"), initial_payload.get("test_args"), state_smoke.get("effective_args"), state_smoke.get("test_args"), ) test_kwargs = _first_dict_value( smoke.get("effective_kwargs"), smoke.get("test_kwargs"), retry_payload.get("test_kwargs"), initial_payload.get("test_kwargs"), state_smoke.get("effective_kwargs"), state_smoke.get("test_kwargs"), ) or {} if not api_name or not isinstance(test_args, list): return {} parent_target = smoke.get("target_space") or state.get("target_space") or state_smoke.get("target_space") or "" return { "source": "parent_automatic_smoke", "parent_run_id": parent_dir.name, "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": test_args, "test_kwargs": test_kwargs, "expected_output_type": smoke.get("expected_output_type") or state_smoke.get("expected_output_type") or expected_output_type, "latency_seconds": smoke.get("latency_seconds") or state_smoke.get("latency_seconds"), "parent_smoke_status": smoke.get("status") or state_smoke.get("status") or state.get("status"), "initial_payload_present": bool(initial_payload), "retry_payload_present": bool(retry_payload), } def load_env_replay_source(target_space_id: str, expected_output_type: str) -> dict: """Replay source passed by backend through the validation Job env. The validation Job starts in a fresh workspace; the parent Build Run folder is usually not mounted locally. This env payload is therefore the canonical transport for replaying a successful parent automatic smoke request. """ raw = os.environ.get("PARENT_REPLAY_SOURCE_JSON") or "" if not raw.strip(): return {} try: data = json.loads(raw) except Exception as exc: return {"source_error": f"invalid_parent_replay_source_json:{exc}"} if not isinstance(data, dict): return {"source_error": "parent_replay_source_json_not_object"} args = data.get("test_args") kwargs = data.get("test_kwargs") or {} api_name = data.get("api_name") or "" if not api_name or not isinstance(args, list) or not isinstance(kwargs, dict): return {"source_error": "parent_replay_source_missing_api_args_or_kwargs", "raw_keys": sorted(data.keys())} parent_target = data.get("parent_target_space") or "" return { "source": data.get("source") or "parent_automatic_smoke_backend", "transport": "job_env", "parent_run_id": data.get("parent_run_id") or os.environ.get("PARENT_BUILD_RUN_ID", ""), "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": args, "test_kwargs": kwargs, "expected_output_type": data.get("expected_output_type") or expected_output_type or "any", "latency_seconds": data.get("latency_seconds"), "parent_smoke_status": data.get("parent_smoke_status") or "success", "backend_replay_source_used": True, } def parent_has_successful_manual_validation(parent_dir: Path) -> bool: manual = read_json(parent_dir / "manual_validation_status.json", {}) or {} if str(manual.get("status") or "").lower() == "success": return True linked = read_json(parent_dir / "linked_validations.json", {}) or {} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] success_tokens = {"success", "full_inference_success", "passed", "validated_after_manual_space_test"} return any(str(row.get("status") or row.get("effective_status") or "").lower() in success_tokens for row in validations if isinstance(row, dict)) def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None): path.parent.mkdir(parents=True, exist_ok=True) event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}} line = json.dumps(event, ensure_ascii=False) with path.open("a", encoding="utf-8") as f: f.write(line + "\n") print(line, flush=True) SAFE_SPACE_PYTHON_VERSIONS = {"3.10", "3.11", "3.12"} DEFAULT_SPACE_PYTHON_VERSION = "3.10" def normalize_space_python_version(value) -> tuple[str, bool, str]: raw = str(value or "").strip().strip("\"'") lowered = raw.lower().replace("python", "").strip() match = re.search(r"(3)\.(\d+)(?:\.\d+)?", lowered) if match: normalized = f"{match.group(1)}.{match.group(2)}" else: normalized = lowered if normalized in SAFE_SPACE_PYTHON_VERSIONS: return normalized, normalized != raw, "allowed" return DEFAULT_SPACE_PYTHON_VERSION, True, f"unsupported_or_ambiguous:{raw or 'missing'}" def requirements_has_package(lines: list[str], package: str) -> bool: wanted = package.lower().replace("_", "-") for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped: continue name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") if name == wanted: return True return False def workspace_app_imports_torch(workspace: Path) -> bool: app_path = workspace / "app.py" if not app_path.exists(): return False text = app_path.read_text(encoding="utf-8", errors="ignore") return bool(re.search(r"(?m)^\s*(import\s+torch\b|from\s+torch\b)", text)) def redact_text(text: str | None) -> str: if not text: return "" value = text for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]: secret = os.environ.get(secret_name) if secret: value = value.replace(secret, "[REDACTED]") value = re.sub(r"Bearer\s+[A-Za-z0-9_\-.=]+", "Bearer [REDACTED]", value) value = re.sub(r"hf_[A-Za-z0-9_\-]{10,}", "hf_[REDACTED]", value) return value def eval_enabled() -> bool: return os.environ.get("ASF_EVAL_RECORD_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"} def eval_safe_hash(value: str | None, prefix: str = "h") -> str: raw = (value or "").strip() salt = os.environ.get("ASF_EVAL_SALT", "").strip() if not raw: return "" if salt: digest = hmac.new(salt.encode("utf-8"), raw.encode("utf-8"), hashlib.sha256).hexdigest()[:24] else: digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] return f"{prefix}_{digest}" def eval_load_json(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8", errors="replace")) except Exception as exc: return {"_parse_error": str(exc)} def eval_load_events(path: Path) -> list[dict]: if not path.exists(): return [] events = [] for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): try: evt = json.loads(line) if isinstance(evt, dict): events.append(evt) except Exception: pass return events def eval_parse_ts(value: str | None) -> float | None: if not value: return None try: return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() except Exception: return None def eval_compact_timeline(events: list[dict]) -> list[dict]: stages: dict[str, dict] = {} order: list[str] = [] for evt in events: step = str(evt.get("step") or "unknown")[:80] status = str(evt.get("status") or "")[:80] ts = str(evt.get("ts") or "") tsv = eval_parse_ts(ts) if step not in stages: stages[step] = {"stage": step, "first_ts": ts, "last_ts": ts, "status": status, "event_count": 0} order.append(step) item = stages[step] item["last_ts"] = ts or item.get("last_ts") item["status"] = status or item.get("status") item["event_count"] = int(item.get("event_count") or 0) + 1 if tsv is not None: if "_first" not in item: item["_first"] = tsv item["_last"] = tsv timeline = [] for step in order: item = dict(stages[step]) first = item.pop("_first", None) last = item.pop("_last", None) if first is not None and last is not None and last >= first: item["duration_ms"] = int((last - first) * 1000) timeline.append(item) return timeline[-80:] def eval_redacted_tail(path: Path, limit: int = 2400) -> str: if os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() not in {"1", "true", "yes", "on"}: return "" try: return redact_text(path.read_text(encoding="utf-8", errors="replace"))[-limit:] except Exception: return "" def eval_len_if_list(value) -> int: return len(value) if isinstance(value, list) else 0 def eval_seconds_bucket(value) -> str: try: seconds = int(value) except Exception: return "unknown" if seconds < 0: return "expired" if seconds < 15 * 60: return "lt_15m" if seconds < 60 * 60: return "lt_1h" if seconds < 4 * 60 * 60: return "lt_4h" if seconds < 8 * 60 * 60: return "lt_8h" return "gte_8h" def eval_v191_plus_signals(run_dir: Path, analysis: dict, generation_smoke: dict, *, contract: dict | None = None, requirements_policy: dict | None = None, auth_status: dict | None = None, repair_outcome: dict | None = None, worker_plan_review: dict | None = None, grounding_review: dict | None = None) -> dict: """Return privacy-safe metrics for features added from v191.1 onward.""" analysis = analysis if isinstance(analysis, dict) else {} generation_smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = contract if isinstance(contract, dict) else {} requirements_policy = requirements_policy if isinstance(requirements_policy, dict) else {} auth_status = auth_status if isinstance(auth_status, dict) else {} repair_outcome = repair_outcome if isinstance(repair_outcome, dict) else {} worker_plan_review = worker_plan_review if isinstance(worker_plan_review, dict) else {} grounding_review = grounding_review if isinstance(grounding_review, dict) else {} build_risk = analysis.get("build_risk") if isinstance(analysis.get("build_risk"), dict) else {} kernel_strategy = analysis.get("kernel_strategy") if isinstance(analysis.get("kernel_strategy"), dict) else {} if not kernel_strategy and isinstance(analysis.get("metadata"), dict): kernel_strategy = analysis["metadata"].get("kernel_strategy") if isinstance(analysis["metadata"].get("kernel_strategy"), dict) else {} grounding_source = grounding_review.get("model_card_source") if isinstance(grounding_review.get("model_card_source"), dict) else {} return { "schema_version": "v191_plus_eval_signals.v1", "platform_dependency_policy": { "present": bool(requirements_policy), "status": requirements_policy.get("status") or "", "removed_platform_pin_count": eval_len_if_list(requirements_policy.get("removed_pins")), "normalized_platform_line_count": eval_len_if_list(requirements_policy.get("normalized_platform_lines")), "injected_platform_line_count": eval_len_if_list(requirements_policy.get("injected_platform_lines")), "torch_added": bool(requirements_policy.get("torch_added")), }, "auth_context": { "present": bool(auth_status), "status": auth_status.get("status") or "", "token_kind": auth_status.get("token_kind") or "", "expiry_known": bool(auth_status.get("expiry_known")), "seconds_until_expiry_bucket": eval_seconds_bucket(auth_status.get("seconds_until_expiry")), "safe_for_phase": bool(auth_status.get("safe_for_phase")), }, "model_scan": { "build_risk_level": build_risk.get("level") or analysis.get("build_risk_level") or "", "build_risk_signal_count": eval_len_if_list(build_risk.get("signals")), "build_risk_visibility_only": bool(build_risk.get("visibility_only", True)) if build_risk else True, "recommended_session_minutes": build_risk.get("recommended_session_minutes"), "kernel_strategy_present": bool(kernel_strategy), "native_kernel_detected": bool(kernel_strategy.get("native_kernel_detected") or kernel_strategy.get("detected")), "kernel_signal_count": eval_len_if_list(kernel_strategy.get("signals")), "kernel_candidate_count": eval_len_if_list(kernel_strategy.get("candidates")), }, "contract_validation": { "contract_present": bool(contract), "full_inference_implemented": bool(contract.get("full_inference_implemented")), "validation_level": contract.get("validation_level") or "", "requires_gpu": bool(contract.get("requires_gpu")), "blockers_count": int(contract.get("blockers_count") or 0) if str(contract.get("blockers_count") or "0").isdigit() else 0, "generation_smoke_status": generation_smoke.get("status") or "", "generation_smoke_skipped": str(generation_smoke.get("status") or "").lower() == "skipped", "generation_smoke_skip_reason": generation_smoke.get("skip_reason") or "", }, "repair_outcome": { "present": bool(repair_outcome), "repair_decision": repair_outcome.get("repair_decision") or repair_outcome.get("decision") or "", "patch_applied": bool(repair_outcome.get("patch_applied")), "upload_success": bool(repair_outcome.get("upload_success")), "post_repair_validation": repair_outcome.get("post_repair_validation") or "", "failure_type": repair_outcome.get("failure_type") or "", }, "planning": { "worker_plan_review_present": bool(worker_plan_review), "status": worker_plan_review.get("status") or "", "declared_strategy": worker_plan_review.get("declared_strategy") or "", "worker_recommendation": worker_plan_review.get("worker_recommendation") or "", "warning_count": eval_len_if_list(worker_plan_review.get("warnings")), }, "model_card_grounding": { "present": bool(grounding_review), "status": grounding_review.get("status") or "", "source_available": bool(grounding_review.get("source_available")), "model_card_present": bool(grounding_review.get("model_card_present")), "source": grounding_source.get("source") or "", "resolved_card_file": grounding_source.get("resolved_card_file") or "", "fallback_used": bool(grounding_source.get("fallback_used")), "pi_evidence_present": bool(grounding_review.get("pi_evidence_present")), "pi_evidence_count": int(grounding_review.get("pi_evidence_count") or 0), "warning_count": eval_len_if_list(grounding_review.get("warnings")), "warnings": [str(w)[:120] for w in (grounding_review.get("warnings") if isinstance(grounding_review.get("warnings"), list) else [])[:8]], }, } def eval_terminal_status(status: str, phase: str = "") -> bool: status = str(status or "").strip().lower() phase = str(phase or "").strip().lower() if phase in {"final", "failure"}: return True return status in { "done", "success", "failed", "cancelled", "canceled", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "demo_usable_full_promise_not_verified", "interactive_app_available_smoke_failed", "manual_test_required_smoke_failed", "partial_validation", "technical_blocker", "manual_hardware_required", } def eval_event_seen(events: list[dict], step: str, status: str | None = None) -> bool: for event in events or []: if str(event.get("step") or "") != step: continue if status is None or str(event.get("status") or "") == status: return True return False def eval_health_passed(inference_gate: dict, generation_smoke: dict, events: list[dict], status: str) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} if signals.get("health_passed") is True: return True if str(status or "") in {"full_inference_success", "full_inference_candidate_health_passed", "health_only"}: return True if eval_event_seen(events, "api_validation", "success") or eval_event_seen(events, "repair_validation", "success"): return True return False def eval_generation_smoke_passed(inference_gate: dict, generation_smoke: dict) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} return bool(signals.get("generation_smoke_passed") is True or generation_smoke.get("status") == "success") def eval_failure_summary(status: str, state: dict, inference_gate: dict, generation_smoke: dict, health_passed: bool, smoke_passed: bool) -> dict: failure_type = generation_smoke.get("failure_type") or inference_gate.get("failure_type") or "" failure_owner = generation_smoke.get("failure_owner") or inference_gate.get("failure_owner") or "" missing_argument = generation_smoke.get("missing_argument") or inference_gate.get("missing_argument") or "" reason = missing_argument or failure_type or str(state.get("message") or inference_gate.get("message") or "")[:240] if not failure_owner: if failure_type == "validator_request_error" or missing_argument: failure_owner = "factory_validator" elif smoke_passed: failure_owner = "" elif health_passed and str(generation_smoke.get("status") or "") == "failed": failure_owner = "unknown" return { "failure_type": failure_type, "failure_owner": failure_owner, "failure_reason": reason, "missing_argument": missing_argument, } def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: bool, full_inference_verified: bool, inference_gate: dict) -> str: status = str(status or "").strip() if status in {"cancelled", "canceled"}: return "cancelled" if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"): return "manual_action_required" if status in {"technical_blocker", "technical_blocker_boot_only"}: return "technical_blocker" if full_inference_verified: return "success" if status in {"full_inference_candidate_health_passed", "health_only"} or (health_passed and not smoke_passed): return "partial_validation" if phase == "failure" or status in {"failed", "error"}: return "failed" if eval_terminal_status(status, phase): return "partial_validation" if health_passed else "failed" return "running" def publish_eval_record(run_dir: Path, *, phase: str, events_path: Path | None = None) -> dict: """Write an anonymized cross-run evaluation record to the optional eval bucket. This intentionally stores structured metrics and hashed identifiers, not raw generated code, prompts, tokens, user bucket paths, or target Space IDs. """ if not eval_enabled(): return {"enabled": False} # Jobs only write the anonymized eval record back into the user's run bucket. # The ASF backend later publishes this record to the operator eval bucket # mounted on the ASF Space. User Jobs must never mount or write the private # operator archive directly. run_id = os.environ.get("RUN_ID", run_dir.name) events = eval_load_events(events_path or (run_dir / "events.jsonl")) state = eval_load_json(run_dir / "state.json") analysis = eval_load_json(run_dir / "model_analysis.json") hardware_strategy = eval_load_json(run_dir / "hardware_strategy.json") hardware_attempts = eval_load_json(run_dir / "hardware_attempts.json") inference_gate = eval_load_json(run_dir / "inference_gate.json") generation_smoke = eval_load_json(run_dir / "tests" / "generation_smoke.json") generation_smoke_retry = eval_load_json(run_dir / "tests" / "generation_smoke_payload_retry.json") endpoint_discovery_payload = eval_load_json(run_dir / "tests" / "gradio_endpoint_discovery.json") manual_validation_status = eval_load_json(run_dir / "manual_validation_status.json") linked_validations_payload = eval_load_json(run_dir / "linked_validations.json") space_runtime = eval_load_json(run_dir / "space_runtime.json") manifest = eval_load_json(run_dir / "artifact_manifest.json") repair_decision = eval_load_json(run_dir / "repair" / "REPAIR_DECISION.json") blockage = eval_load_json(run_dir / "repair" / "BLOCKAGE.json") repair_outcome = eval_load_json(run_dir / "repair_outcome.json") or eval_load_json(run_dir / "repair" / "REPAIR_OUTCOME.json") requirements_policy = eval_load_json(run_dir / "generated" / "requirements_policy.json") or eval_load_json(run_dir / "requirements_policy.json") auth_status = eval_load_json(run_dir / "auth_status.json") contract = eval_load_json(run_dir / "generated" / "INFERENCE_CONTRACT.json") worker_plan_review = eval_load_json(run_dir / "planning" / "worker_plan_review.json") grounding_review = eval_load_json(run_dir / "planning" / "model_card_grounding_review.json") or (worker_plan_review.get("model_card_grounding") if isinstance(worker_plan_review.get("model_card_grounding"), dict) else {}) pi_resolution = state.get("pi_model_resolution") if isinstance(state.get("pi_model_resolution"), dict) else {} status = str(state.get("status") or inference_gate.get("status") or phase or "unknown") failure_message = str(state.get("message") or (state.get("details") or {}).get("error") or "")[:1200] started_at = events[0].get("ts") if events else (state.get("created_at") or os.environ.get("LAUNCHED_AT") or "") finished_at = state.get("updated_at") or (events[-1].get("ts") if events else now()) model_id = os.environ.get("MODEL_ID") or state.get("model_id") or analysis.get("model_id") or "" target_space = state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" include_model_id = os.environ.get("ASF_EVAL_INCLUDE_MODEL_ID", "").strip().lower() in {"1", "true", "yes", "on"} anon_run_id = eval_safe_hash(run_id, "run") health_passed = eval_health_passed(inference_gate, generation_smoke, events, status) generation_smoke_passed = eval_generation_smoke_passed(inference_gate, generation_smoke) promise_validation = inference_gate.get("promise_validation") if isinstance(inference_gate.get("promise_validation"), dict) else {} promise_fulfilled = bool(promise_validation.get("promise_fulfilled") or inference_gate.get("promise_fulfilled") or status == "full_inference_success") full_inference_verified = bool(promise_fulfilled and status == "full_inference_success" and health_passed and generation_smoke_passed) process_completed = eval_terminal_status(status, phase) failure_summary = eval_failure_summary(status, state, inference_gate, generation_smoke, health_passed, generation_smoke_passed) verdict = eval_verdict(status, phase, health_passed, generation_smoke_passed, full_inference_verified, inference_gate) linked_rows = linked_validations_payload.get("validations") if isinstance(linked_validations_payload.get("validations"), list) else [] linked_success_rows = [row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() == "success"] manual_applied = str(manual_validation_status.get("status") or "").lower() == "success" or bool(linked_success_rows) effective_status = str((manual_validation_status if manual_applied else {}).get("effective_status") or (linked_success_rows[-1].get("effective_status") if linked_success_rows else "") or ("success" if full_inference_verified else verdict)) endpoint_names = endpoint_discovery_payload.get("discovered_api_names") or endpoint_discovery_payload.get("candidates") or [] smoke_retry_meta = generation_smoke.get("auto_retry") if isinstance(generation_smoke.get("auto_retry"), dict) else {} record = { "process_completed": process_completed, "verdict": verdict, "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "schema_version": "1.4", "app_version": os.environ.get("ASF_VERSION", "unknown"), "phase": phase, "anonymous_run_id": anon_run_id, "run_id_hash": eval_safe_hash(run_id, "run"), "anonymous_user_hash": eval_safe_hash(os.environ.get("HF_USERNAME"), "user"), "salt_configured": bool(os.environ.get("ASF_EVAL_SALT")), "run_kind": state.get("kind") or "linked_space_validation", "started_at": started_at, "finished_at": finished_at, "input_model": { "model_id_redacted": not include_model_id, "model_id": model_id if include_model_id else "", "model_hash": eval_safe_hash(model_id, "model"), "pipeline_tag": analysis.get("pipeline_tag"), "library_name": analysis.get("library_name"), "default_model_target": bool(analysis.get("default_model_target")), }, "config": { "implementation_mode": os.environ.get("IMPLEMENTATION_MODE", ""), "expected_output_type": os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "pi_model_requested": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "pi_model_effective": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "try_zero_gpu_first": os.environ.get("TRY_ZERO_GPU_FIRST", ""), "allow_fixed_gpu_fallback": os.environ.get("ALLOW_FIXED_GPU_FALLBACK", ""), "preferred_gpu": os.environ.get("PREFERRED_SPACE_HARDWARE", ""), "fallback_gpu": os.environ.get("FALLBACK_SPACE_HARDWARE", ""), }, "outcome": { "status": status, "verdict": verdict, "process_completed": process_completed, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "app_boot_validation_status": inference_gate.get("app_boot_validation_status") or "", "promise_validation_status": inference_gate.get("promise_validation_status") or promise_validation.get("promise_validation_status") or "", "promise_fulfilled": bool(promise_fulfilled), "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "message_redacted": redact_text(failure_message)[:1200], "target_space_hash": eval_safe_hash(target_space, "space"), "space_created": bool(target_space), "space_runtime_stage": space_runtime.get("stage"), "space_runtime_hardware": space_runtime.get("hardware") or space_runtime.get("requested_hardware"), }, "timeline": eval_compact_timeline(events), "hardware": { "selected": hardware_strategy.get("selected_hardware") or state.get("selected_hardware") or "", "strategy": hardware_strategy.get("strategy") or hardware_attempts.get("strategy") or "", "attempts": hardware_strategy.get("attempts") or hardware_attempts.get("attempts") or state.get("hardware_attempts") or [], "single_space_per_run": True, }, "pi": { "requested_model": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "effective_model": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "model_mismatch": bool(pi_resolution.get("mismatch")), "build_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_output.txt"), "diagnosis_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_diagnosis_output.txt"), }, "recovery": { "entered": any(str(e.get("step", "")).startswith(("repair", "failure_diagnosis", "pi_diagnosis", "technical_blocker")) for e in events), "decision": repair_decision.get("action") or repair_decision.get("decision") or "", "confidence": repair_decision.get("confidence") or "", "blockage": bool(blockage), "blockage_category": blockage.get("category") or blockage.get("reason") or "", }, "validation": { "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "generation_smoke_status": generation_smoke.get("status") or "", "latency_seconds": generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "expected_output_type": generation_smoke.get("expected_output_type") or os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "inference_gate_status": inference_gate.get("status") or "", "manual_hardware_required": bool(inference_gate.get("manual_hardware_required")), "failure_type": failure_summary.get("failure_type") or "", "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "missing_argument": failure_summary.get("missing_argument") or "", }, "automatic_outcome": { "status": status, "verdict": verdict, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, }, "effective_outcome": { "automatic_verdict": verdict, "automatic_status": status, "effective_verdict": effective_status, "effective_status": effective_status, "manual_validation_applied": manual_applied, "source": "linked_space_test" if manual_applied else "automatic_run", }, "linked_validation": { "present": bool(linked_rows or manual_validation_status), "count": len(linked_rows), "success_count": len(linked_success_rows), "failure_count": len([row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() in {"failed", "failure", "error"}]), "status": "success" if manual_applied else str(manual_validation_status.get("status") or ""), "mode": manual_validation_status.get("space_test_policy_mode") or "", "api_name": manual_validation_status.get("api_name") or generation_smoke.get("api_name") or "", "latency_seconds": manual_validation_status.get("latency_seconds") or generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "hardware_used_for_validation": manual_validation_status.get("hardware_used_for_validation") or "", }, "endpoint_discovery": { "required": bool(endpoint_discovery_payload.get("endpoint_discovery_required") or endpoint_discovery_payload), "succeeded": bool(endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint")), "selected_endpoint": endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint") or "", "candidate_count": len(endpoint_names) if isinstance(endpoint_names, list) else 0, "excluded_health_endpoint": True, }, "smoke_retry": { "retried": bool(generation_smoke_retry or smoke_retry_meta.get("retried")), "reason": generation_smoke_retry.get("retry_reason") or smoke_retry_meta.get("retry_reason") or smoke_retry_meta.get("reason") or "", "attempts": generation_smoke_retry.get("attempts") or smoke_retry_meta.get("attempts") or (2 if generation_smoke_retry else 1), "passed_after_retry": bool(generation_smoke_retry and generation_smoke.get("status") == "success"), }, "v191_plus": eval_v191_plus_signals( run_dir, analysis, generation_smoke, contract=contract, requirements_policy=requirements_policy, auth_status=auth_status, repair_outcome=repair_outcome, worker_plan_review=worker_plan_review, grounding_review=grounding_review, ), "artifacts": { "present_count": len(manifest.get("present_paths") or []), "present_paths": sorted(str(p) for p in (manifest.get("present_paths") or []))[:120], }, "privacy": { "tokens_stored": False, "raw_prompts_stored": False, "generated_code_stored": False, "target_space_redacted": True, "user_bucket_redacted": True, "redacted_tails_enabled": os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() in {"1", "true", "yes", "on"}, "linked_run_ids_redacted": True, "endpoint_schema_stored": False, "validation_args_stored": False, "model_card_raw_stored": False, "pi_evidence_text_stored": False, "requirements_txt_stored": False, "auth_token_stored": False, }, } day = (finished_at or now())[:10] try: yyyy, mm, dd = day.split("-") except Exception: yyyy, mm, dd = now()[:10].split("-") dest = run_dir / "eval_record.json" write_json(dest, record) events_dest = run_dir / "events_compact.jsonl" events_dest.write_text("".join(json.dumps(item, ensure_ascii=False) + "\n" for item in record["timeline"]), encoding="utf-8") return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"} class AuthRefreshRequired(RuntimeError): """Raised when an OAuth/JWT token is expired or too close to expiry.""" def _decode_jwt_claims_unverified(token: str) -> dict: """Decode JWT claims without verification, only to inspect non-sensitive expiry metadata. This never returns or writes the raw token. Opaque PAT-style tokens are supported by returning an empty dict so they remain usable with an `unknown` expiry state. """ try: parts = (token or "").split(".") if len(parts) < 2: return {} payload = parts[1] payload += "=" * ((4 - len(payload) % 4) % 4) raw = base64.urlsafe_b64decode(payload.encode("utf-8")) data = json.loads(raw.decode("utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict: issued_at = now() if not token: return { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": False, "token_kind": "missing", "expiry_known": False, "status": "missing", "safe_for_phase": False, "minimum_required_seconds": minimum_required_seconds, } claims = _decode_jwt_claims_unverified(token) exp = claims.get("exp") if isinstance(claims, dict) else None iat = claims.get("iat") if isinstance(claims, dict) else None token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown") payload = { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": True, "token_kind": token_kind, "expiry_known": exp is not None, "minimum_required_seconds": int(minimum_required_seconds or 0), "token_value": "[REDACTED]", } if iat is not None: try: payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat() except Exception: pass if exp is None: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"}) return payload try: exp_int = int(exp) seconds_left = exp_int - int(time.time()) payload.update({ "expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(), "seconds_until_expiry": seconds_left, }) except Exception: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"}) return payload if seconds_left <= 0: payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"}) elif minimum_required_seconds and seconds_left < minimum_required_seconds: payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"}) else: payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"}) return payload def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict: payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds) payload["phase"] = phase safe_payload = {k: v for k, v in payload.items() if k != "token_value"} try: probes_dir = run_dir / "auth_probes" probes_dir.mkdir(parents=True, exist_ok=True) write_json(probes_dir / f"{phase}.json", safe_payload) write_json(run_dir / "auth_status.json", safe_payload) except Exception: pass status = str(payload.get("status") or "unknown") if events_path: event_status = "success" if payload.get("safe_for_phase") else "failed" if status == "unknown": event_status = "warning" append_event( events_path, "auth_probe", event_status, f"HF OAuth/token expiry check for {phase}: {status}", safe_payload, ) if raise_on_unsafe and not payload.get("safe_for_phase"): raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.") return payload def is_auth_expired_error(error: Exception | str) -> bool: text = str(error or "").lower() return any(marker in text for marker in [ "oauth token has expired", "exp claim timestamp check failed", "token has expired", "jwt expired", ]) def minimum_auth_seconds_for_phase(phase: str) -> int: env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_") raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "") if raw: try: return max(0, int(raw)) except Exception: pass defaults = { "before_space_create": 900, "before_upload": 900, "before_initial_validation": 1800, "before_repair": 1800, "before_repair_upload": 1800, "before_repair_validation": 1800, "before_linked_space_test": 1800, } return defaults.get(phase, 0) def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict: """Ensure Pi and HF tooling see the same private HF token safely. HF Jobs receive HF_TOKEN as a secret. Pi, huggingface_hub, gradio_client, and the hf CLI can use either HF_TOKEN or HUGGING_FACE_HUB_TOKEN depending on the tool/version, so expose both aliases inside the isolated Job process. Never write the token value to artifacts. """ token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "" if token: os.environ.setdefault("HF_TOKEN", token) os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token) os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation")) payload = { "hf_token_present": bool(token), "hf_token_length": len(token) if token else 0, "hugging_face_hub_token_alias_present": bool(os.environ.get("HUGGING_FACE_HUB_TOKEN")), "hf_username": os.environ.get("HF_USERNAME") or "", "bucket_source": os.environ.get("BUCKET_SOURCE") or "", "target_space_id": os.environ.get("TARGET_SPACE_ID") or "", "token_value": "[REDACTED]" if token else "", "token_kind": expiry.get("token_kind"), "expiry_known": expiry.get("expiry_known"), "expires_at": expiry.get("expires_at"), "seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"), "auth_risk": expiry.get("auth_risk"), "safe_for_long_build": bool(expiry.get("safe_for_phase")), "minimum_required_seconds": expiry.get("minimum_required_seconds"), } run_dir.mkdir(parents=True, exist_ok=True) write_json(run_dir / "token_context.json", payload) if events_path: append_event(events_path, "token_context", "success" if token else "warning", "Verified HF token context for Pi/HF operations", {k: v for k, v in payload.items() if k != "token_value"}) return payload def pi_tooling_context_note() -> str: return """HF tooling context: - You are running inside an HF Job with an HF_TOKEN secret configured. - For gated/private base models, LoRA adapters, PEFT adapters, or private Hub files, the generated Space must read `HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")` and pass `token=HF_TOKEN` to `from_pretrained`, `hf_hub_download`, `snapshot_download`, and `load_lora_weights` when those APIs are used. Never print the token. - For ZeroGPU/Diffusers/LoRA GPU apps, keep the app compatible with both HF ZeroGPU and fixed GPU fallback: import `spaces` before CUDA-touching imports, protect the heavy inference path with `@spaces.GPU`, and never leave a failed model load as a silent global `None` that later makes `/generate` fail with "pipeline not loaded". If using lazy/cache loading, surface real load errors clearly and keep `/generate` repairable. - The Factory exposes both HF_TOKEN and HUGGING_FACE_HUB_TOKEN aliases for huggingface_hub, gradio_client, hf CLI, and Pi provider access. - You may use the token through normal tools when needed to inspect the model/Space or validate private resources. - Never print, write, echo, or commit token values. Do not include Authorization headers in artifacts. - If a Hub operation fails with 401/403/quota/billing, report it as evidence instead of retrying blindly. """ def run_cmd(cmd: list[str], *, env: dict | None = None, timeout: int = 120): result = subprocess.run(cmd, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) return result.returncode, redact_text(result.stdout) def install_deps(events_path: Path): append_event(events_path, "dependencies", "started", "Installing validation dependencies") code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0", "requests>=2.31.0"], timeout=600) if code != 0: append_event(events_path, "dependencies", "failed", "Dependency installation failed", {"output_tail": out[-4000:]}) raise RuntimeError(out) append_event(events_path, "dependencies", "success", "Validation dependencies installed") def make_gradio_client(target_space_id: str, token: str, timeout_s: int | float | None = None): import inspect from gradio_client import Client params = inspect.signature(Client).parameters kwargs = {} if timeout_s and "httpx_kwargs" in params: kwargs["httpx_kwargs"] = {"timeout": float(timeout_s)} if "token" in params: return Client(target_space_id, token=token, **kwargs) if "hf_token" in params: return Client(target_space_id, hf_token=token, **kwargs) if "api_key" in params: return Client(target_space_id, api_key=token, **kwargs) if "headers" in params: return Client(target_space_id, headers={"Authorization": f"Bearer {token}"}, **kwargs) return Client(target_space_id, **kwargs) def api_names_from_schema(schema) -> list[str]: names: list[str] = [] def add(name): if not isinstance(name, str) or not name: return if not name.startswith("/"): name = "/" + name if name not in names: names.append(name) def walk(obj): if isinstance(obj, dict): for k, v in obj.items(): if k in {"api_name", "apiName"}: add(v) if isinstance(k, str) and k.startswith("/"): add(k) walk(v) elif isinstance(obj, list): for item in obj: walk(item) walk(schema) return names def normalize_api_name(name: str | None) -> str: value = (name or "").strip() if not value: return "/generate" return value if value.startswith("/") else "/" + value HEALTH_NEGATIVE_STATUS_MARKERS = { "unhealthy", "failed", "failure", "error", "not_ready", "not ready", "runtime_error", "unavailable", "not_available", "not available", "not_loaded", "not loaded", "model_not_loaded", "model not loaded", "model_unavailable", "pipeline_not_loaded", "pipeline not loaded", "pipeline_unavailable", "loading_failed", "load_failed", "initialization_failed", } HEALTH_POSITIVE_STATUS_MARKERS = {"healthy", "ok", "ready", "success", "passed"} MODEL_NOT_READY_TEXT_MARKERS = [ "unavailable", "not_ready", "not ready", "not_loaded", "not loaded", "model_not_loaded", "model not loaded", "model pipeline is not loaded", "model pipeline not loaded", "pipeline is not loaded", "pipeline not loaded", "pipeline_not_loaded", "model is not initialized", "model not initialized", "not initialized", "failed to load model", "model load failed", "loading failed", ] def _coerce_health_payload(value): if isinstance(value, dict): return value if isinstance(value, (list, tuple)) and value and isinstance(value[0], dict): return value[0] if isinstance(value, str): raw = value.strip() if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}: raw = raw[1:-1].strip() for parser in (json.loads, ast.literal_eval): try: parsed = parser(raw) except Exception: continue if isinstance(parsed, dict): return parsed if isinstance(parsed, (list, tuple)) and parsed and isinstance(parsed[0], dict): return parsed[0] return None def _semantic_health_payloads(validation: dict | None) -> list[tuple[str, dict]]: if not isinstance(validation, dict): return [] payloads: list[tuple[str, dict]] = [] for key in ("json", "result", "data", "health", "result_repr", "body", "text"): payload = _coerce_health_payload(validation.get(key)) if isinstance(payload, dict): payloads.append((key, payload)) nested = _coerce_health_payload(payload.get("json")) if isinstance(nested, dict): payloads.append((f"{key}.json", nested)) payloads.append(("validation", validation)) return payloads def validation_health_semantics(validation: dict | None) -> dict: details = {"schema_version": "health_semantics.v198_26_30", "endpoint_reachable": False, "semantic_checked": False, "semantic_passed": None, "negative_markers": [], "positive_markers": [], "load_error": "", "last_error": "", "model_ready": None, "pipeline_ready": None, "model_family": "", "loader_strategy": "", "expected_output_type": "", "failure_class": "", "transport_status": ""} if not isinstance(validation, dict): return details method = str(validation.get("method") or "").lower() validator = str(validation.get("validator") or "").lower() api_name = normalize_api_name(str(validation.get("api_name") or "")) if validation.get("api_name") else "" details["transport_status"] = str(validation.get("status") or "") details["endpoint_reachable"] = bool(validation.get("status") == "success" and (method in {"http_health", "gradio"} or validator in {"http_get_health", "gradio_client"} or api_name == "/health")) for source, payload in _semantic_health_payloads(validation): for key in ("status", "health", "state", "model", "pipeline", "generation"): if key not in payload: continue value = str(payload.get(key) or "").strip().lower() if source == "validation" and key == "status": continue if value in HEALTH_NEGATIVE_STATUS_MARKERS: details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}={value}") elif value in HEALTH_POSITIVE_STATUS_MARKERS and key != "state": details["semantic_checked"] = True details["positive_markers"].append(f"{source}.{key}={value}") for key in ("pipeline_ready", "pipeline_loaded", "model_ready", "model_loaded", "generation_ready"): if key in payload: details["semantic_checked"] = True if key in {"pipeline_ready", "pipeline_loaded"}: details["pipeline_ready"] = bool(payload.get(key)) if isinstance(payload.get(key), bool) else details.get("pipeline_ready") if payload.get(key) is False: details["negative_markers"].append(f"{source}.{key}=false") elif payload.get(key) is True: details["positive_markers"].append(f"{source}.{key}=true") for key in ("model_family", "loader_strategy", "expected_output_type"): value = payload.get(key) if value and not details.get(key): details[key] = str(value)[:200] for key in ("error", "last_error", "load_error", "pipeline_error", "model_error"): if payload.get(key): details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}=present") if not details["load_error"]: details["load_error"] = str(payload.get(key))[:1000] if key == "last_error" and not details["last_error"]: details["last_error"] = str(payload.get(key))[:1000] health_error_text = "\n".join(str(payload.get(k) or "") for _, payload in _semantic_health_payloads(validation) for k in ("last_error", "error", "load_error", "pipeline_error", "model_error")) try: if is_sdxl_lora_text_encoder_mismatch_text(health_error_text): details["failure_class"] = "sdxl_lora_text_encoder_mismatch" except Exception: pass for key in ("result_repr", "text", "message", "error", "last_error", "load_error", "pipeline_error", "model_error"): value = validation.get(key) if value is None: continue lowered = str(value).strip().lower().strip("'\"") if lowered in HEALTH_NEGATIVE_STATUS_MARKERS or any(marker in lowered for marker in MODEL_NOT_READY_TEXT_MARKERS): details["semantic_checked"] = True details["negative_markers"].append(f"{key}=model_not_ready") if not details["load_error"]: details["load_error"] = str(value)[:1000] if details["negative_markers"]: details["semantic_passed"] = False details["model_ready"] = False if details.get("pipeline_ready") is None: details["pipeline_ready"] = False elif details["semantic_checked"] and details["positive_markers"]: details["semantic_passed"] = True details["model_ready"] = True if details.get("pipeline_ready") is None: details["pipeline_ready"] = True return details def validation_endpoint_reachable(validation: dict | None) -> bool: if not isinstance(validation, dict) or validation.get("status") != "success": return False semantics = validation_health_semantics(validation) if semantics.get("endpoint_reachable"): return True method = str(validation.get("method") or "").lower() validator = str(validation.get("validator") or "").lower() api_name = normalize_api_name(str(validation.get("api_name") or "")) if validation.get("api_name") else "" return bool(method in {"http_health", "gradio"} or validator in {"http_get_health", "gradio_client"} or api_name == "/health") def validation_health_passed(validation: dict | None) -> bool: if not validation_endpoint_reachable(validation): return False semantics = validation_health_semantics(validation) if semantics.get("semantic_passed") is False: return False return True def endpoint_schema_for_api(schema, api_name: str): target = normalize_api_name(api_name) alternatives = {target, target.lstrip("/")} found = None def walk(obj): nonlocal found if found is not None: return if isinstance(obj, dict): for key, value in obj.items(): if isinstance(key, str) and key in alternatives and isinstance(value, (dict, list)): found = value return value = obj.get("api_name") or obj.get("apiName") if isinstance(value, str) and normalize_api_name(value) == target: found = obj return for value in obj.values(): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(schema) return found def endpoint_parameter_names(endpoint) -> list[str]: names: list[str] = [] def add(value): if not isinstance(value, str): return cleaned = value.strip() if cleaned and cleaned not in names: names.append(cleaned) def from_parameter(param): if isinstance(param, dict): for key in ["parameter_name", "parameterName", "name", "label"]: add(param.get(key)) component = param.get("component") or param.get("component_type") if isinstance(component, dict): for key in ["label", "name"]: add(component.get(key)) elif isinstance(param, str): add(param) def walk(obj): if isinstance(obj, dict): params = obj.get("parameters") or obj.get("inputs") if isinstance(params, list): for param in params: from_parameter(param) for value in obj.values(): if isinstance(value, (dict, list)): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(endpoint) return names def sanitize_kwargs_for_schema(api_name: str, schema, kwargs: dict): endpoint = endpoint_schema_for_api(schema, api_name) parameter_names = endpoint_parameter_names(endpoint) if not parameter_names: return kwargs, {}, [] allowed = set(parameter_names) sanitized = {key: value for key, value in kwargs.items() if key in allowed} dropped = {key: value for key, value in kwargs.items() if key not in allowed} return sanitized, dropped, parameter_names def endpoint_parameter_objects(endpoint) -> list: params: list = [] def add_param(param): if isinstance(param, (dict, str)): params.append(param) def walk(obj): if isinstance(obj, dict): direct = obj.get("parameters") or obj.get("inputs") or obj.get("input_components") or obj.get("components") if isinstance(direct, list): for param in direct: add_param(param) return for value in obj.values(): if isinstance(value, (dict, list)): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(endpoint) return params def normalize_schema_param(param, index: int = 0) -> dict: """Return a dict-like Gradio parameter object for mixed schema entries. Some gradio_client view_api() payloads expose endpoint parameters as a heterogeneous list containing both rich dict objects and bare strings like "prompt" or "resolution". The validation worker must never call .get() directly on those bare strings; otherwise linked validation can fail during payload resolution before a request is written. """ if isinstance(param, dict): return param if isinstance(param, str): cleaned = param.strip() or f"arg{index}" return {"name": cleaned, "label": cleaned, "parameter_name": cleaned, "raw_schema_param_type": "str"} return {"name": f"arg{index}", "label": str(param), "raw_schema_param_type": type(param).__name__} def _safe_component_dict(param: dict) -> dict: component = param.get("component") if isinstance(param, dict) else {} return component if isinstance(component, dict) else {} def _schema_text(value) -> str: if value is None: return "" if isinstance(value, (str, int, float, bool)): return str(value) if isinstance(value, dict): for key in ["label", "name", "type", "component", "component_type", "type_name", "class_name", "__type__", "datatype", "python_type", "value"]: text = _schema_text(value.get(key)) if text: return text api_info = value.get("api_info") if isinstance(value.get("api_info"), dict) else {} for key in ["type", "component", "label"]: text = _schema_text(api_info.get(key)) if text: return text return "" def _schema_name(param, index: int) -> str: param = normalize_schema_param(param, index) for key in ["parameter_name", "parameterName", "name", "label", "id"]: text = _schema_text(param.get(key)) if text: return text component = _safe_component_dict(param) for key in ["label", "name"]: text = _schema_text(component.get(key)) if text: return text return f"arg{index}" def _schema_component(param) -> str: param = normalize_schema_param(param) return " ".join(filter(None, [ _schema_text(param.get("component")), _schema_text(param.get("component_type")), _schema_text(param.get("type")), _schema_text(param.get("datatype")), _schema_text(param.get("api_info")), ])).lower() def _choice_scalar(choice): if isinstance(choice, dict): for key in ("value", "label", "name"): if key in choice: return choice.get(key) return next(iter(choice.values()), choice) if choice else choice if isinstance(choice, (list, tuple)) and choice: return choice[0] return choice def _choice_value_and_aliases(choice): aliases = [] actual = _choice_scalar(choice) if isinstance(choice, dict): for key in ("value", "label", "name"): if key in choice: aliases.append(choice.get(key)) aliases.extend(v for v in choice.values() if isinstance(v, (str, int, float))) elif isinstance(choice, (list, tuple)): aliases.extend(choice) else: aliases.append(choice) aliases.append(actual) return actual, aliases def _choice_text_aliases(value) -> set[str]: text = str(value or "").strip() if not text: return set() lowered = text.lower().replace("×", "x") lowered = re.sub(r"\s+", " ", lowered).strip() aliases = {lowered, lowered.replace(" ", "")} no_paren_suffix = re.sub(r"\s*\([^)]*\)\s*$", "", lowered).strip() if no_paren_suffix: aliases.add(no_paren_suffix) aliases.add(no_paren_suffix.replace(" ", "")) for sep in (" - ", " – ", " — ", " | "): if sep in lowered: head = lowered.split(sep, 1)[0].strip() if head: aliases.add(head) aliases.add(head.replace(" ", "")) for w, h in re.findall(r"(\d{2,5})\s*[xX]\s*(\d{2,5})", lowered): aliases.add(f"{w}x{h}") return {a for a in aliases if a} def _choice_alias_matches(value, choice) -> bool: value_aliases = _choice_text_aliases(value) _, aliases = _choice_value_and_aliases(choice) for alias in aliases: if value == alias: return True if _choice_text_aliases(alias) & value_aliases: return True return False def _coerce_to_schema_choice(value, param: dict): choices = param.get("choices") or [] if not isinstance(choices, list) or not choices: return value, False for raw_choice in choices: choice, _aliases = _choice_value_and_aliases(raw_choice) if value == choice: return choice, True value_text = str(value).strip() for raw_choice in choices: choice, _aliases = _choice_value_and_aliases(raw_choice) if str(choice).strip() == value_text: return choice, True for raw_choice in choices: choice, _aliases = _choice_value_and_aliases(raw_choice) if _choice_alias_matches(value, raw_choice): return choice, True return value, False def _schema_choice_values(param) -> list: """Return Gradio choice values while preserving their runtime type when possible.""" param = normalize_schema_param(param) api_info = param.get("api_info") if isinstance(param.get("api_info"), dict) else {} component = _safe_component_dict(param) raw = param.get("choices") or param.get("options") or api_info.get("choices") or component.get("choices") or [] if not isinstance(raw, list): return [] out = [] for choice in raw: if isinstance(choice, dict): if "value" in choice: out.append(choice.get("value")) continue if "label" in choice: out.append(choice.get("label")) continue if isinstance(choice, (list, tuple)) and choice: out.append(choice[0]) continue out.append(choice) return out def _schema_choices(param) -> list: out = [] for choice in _schema_choice_values(param): text = _schema_text(choice) if text: out.append(text) return out def _schema_prefers_string_choice(param) -> bool: param = normalize_schema_param(param) component = _schema_component(param) name = _schema_name(param, 0).lower() if any(token in component for token in ["dropdown", "radio", "choice", "select"]): return True api_info = param.get("api_info") if isinstance(param.get("api_info"), dict) else {} type_hint = " ".join([_schema_text(api_info.get(k)).lower() for k in ["type", "python_type", "datatype"]]) return "string" in type_hint and any(token in name for token in ["width", "height", "size", "resolution"]) def _schema_default(param): param = normalize_schema_param(param) sentinel = object() for key in ["default", "value", "example_input"]: value = param.get(key, sentinel) if value is not sentinel: return value api_info = param.get("api_info") if isinstance(param.get("api_info"), dict) else {} if "default" in api_info: return api_info.get("default") component = _safe_component_dict(param) if "value" in component: return component.get("value") return None def _coerce_number(value, fallback): try: if value is None or value == "": return fallback number = float(value) return int(number) if number.is_integer() else number except Exception: return fallback def autofill_value_for_schema_param(param, index: int): name = _schema_name(param, index).lower().replace(" ", "_") component = _schema_component(param) default = _schema_default(param) choices = _schema_choices(param) if default is not None and not isinstance(default, (dict, list)): return default if choices: return choices[0] if any(token in component for token in ["file", "upload", "audio", "video"]): raise ValueError(f"Cannot auto-fill required file/media input: {_schema_name(param, index)}") if "image" in component and not any(token in name for token in ["prompt", "negative"]): raise ValueError(f"Cannot auto-fill required image input: {_schema_name(param, index)}") if "negative" in name and "prompt" in name: return "" if "prompt" in name or "text" in component or "textbox" in component: return "a cinematic robot cat astronaut, detailed, studio lighting" if "random_seed" in name: return True if "checkbox" in component or "bool" in component: return False if "seed" in name: return 42 if "height" in name or "width" in name or "resolution" in name: return 1024 if "step" in name or "num_inference" in name: return 9 if "guidance" in name or "cfg" in name: return 0.0 if "slider" in component or "number" in component or "int" in component or "float" in component: return _coerce_number(default, 0) return "test" def _schema_file_kind(param, index: int) -> str: name = _schema_name(param, index).lower().replace(" ", "_").replace("-", "_") component = _schema_component(param) haystack = f"{name} {component}" if "image" in haystack: return "image" if "video" in haystack: return "video" if "audio" in haystack: return "audio" if any(token in haystack for token in ["file", "upload", "filepath", "path"]): return "file" return "" def _ensure_validation_input_file(run_dir: Path, kind: str) -> str: smoke_dir = run_dir / "tests" / "smoke_inputs" smoke_dir.mkdir(parents=True, exist_ok=True) if kind == "image": path = smoke_dir / "asf_smoke_image.png" if not path.exists(): path.write_bytes(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=")) return str(path) path = smoke_dir / "asf_smoke_file.txt" if not path.exists(): path.write_text("Agentic Space Factory validation input file.\n", encoding="utf-8") return str(path) def prepare_validation_file_inputs(api_name: str, schema, args: list, run_dir: Path) -> tuple[list, list[dict]]: from gradio_client import handle_file params = endpoint_parameter_objects(endpoint_schema_for_api(schema, api_name)) resolved = list(args or []) conversions = [] for index, param in enumerate(params or []): if index >= len(resolved): break kind = _schema_file_kind(param, index) if not kind: continue before = resolved[index] if isinstance(before, dict) and (before.get("meta") or {}).get("_type") == "gradio.FileData": continue source = before if kind in {"image", "file"} and (source is None or str(source).strip() == ""): source = _ensure_validation_input_file(run_dir, kind) try: resolved[index] = handle_file(str(source)) conversions.append({"index": index, "name": _schema_name(param, index), "kind": kind, "source": str(source)[:500], "action": "handle_file"}) except Exception as exc: conversions.append({"index": index, "name": _schema_name(param, index), "kind": kind, "source": str(source)[:500], "action": "handle_file_failed", "error": str(exc)[:1000]}) raise return resolved, conversions def build_args_from_gradio_schema(api_name: str, schema, current_args: list) -> tuple[list, dict]: endpoint = endpoint_schema_for_api(schema, api_name) params = endpoint_parameter_objects(endpoint) if not params: return current_args, {"args_were_autofilled": False, "args_source": "provided_or_default", "endpoint_parameters": endpoint_parameter_names(endpoint)} if isinstance(current_args, list) and len(current_args) >= len(params): return current_args, {"args_were_autofilled": False, "args_source": "provided", "endpoint_parameters": [_schema_name(p, i) for i, p in enumerate(params)]} generated = [] for index, param in enumerate(params): if isinstance(current_args, list) and index < len(current_args): generated.append(current_args[index]) else: generated.append(autofill_value_for_schema_param(param, index)) return generated, {"args_were_autofilled": True, "args_source": "gradio_schema_auto_fill", "endpoint_parameters": [_schema_name(p, i) for i, p in enumerate(params)]} def coerce_validation_args_to_schema_choices(api_name: str, schema, args: list) -> tuple[list, list[dict]]: params = endpoint_parameter_objects(endpoint_schema_for_api(schema, api_name)) corrected = list(args or []) changes = [] for index, param in enumerate(params or []): if index >= len(corrected): break choices = _schema_choice_values(param) before = corrected[index] changed = False if choices: after, matched = _coerce_to_schema_choice(before, {"choices": choices}) if matched: changed = True if after != before: corrected[index] = after changes.append({"index": index, "name": _schema_name(param, index), "from": before, "to": after, "reason": "schema_choice_type_match"}) # Some Gradio schemas do not expose the choice list reliably but still # identify Dropdown/Radio string inputs. In replay mode, numeric-looking # smoke args (1024) must be normalized to strings ("1024") for these # components before predict(), otherwise Gradio rejects the call as a # choice/type mismatch. if not changed and not choices and _schema_prefers_string_choice(param) and not isinstance(before, str) and before is not None: corrected[index] = str(before) changes.append({"index": index, "name": _schema_name(param, index), "from": before, "to": corrected[index], "reason": "dropdown_string_type_normalization"}) return corrected, changes def is_schema_choice_type_error(error) -> bool: text = str(error or "").lower() return "not in the list of choices" in text or ("value:" in text and "choices" in text) def endpoint_candidate_score(name: str, schema) -> int: normalized = normalize_api_name(name) if normalized in {"/health", "/load", "/reset", "/clear"}: return -100 endpoint = endpoint_schema_for_api(schema, normalized) params = endpoint_parameter_objects(endpoint) score = 0 lname = normalized.lower() if lname == "/generate": score += 100 elif lname == "/predict": score += 80 elif "generate" in lname: score += 70 elif "predict" in lname or "infer" in lname: score += 55 else: score += 20 if params: score += min(len(params), 8) return score def select_validation_api_name(requested_api_name: str, schema) -> tuple[str, list[dict]]: discovered = api_names_from_schema(schema) normalized_requested = normalize_api_name(requested_api_name) if str(requested_api_name or "").strip() else "" names = list(dict.fromkeys(discovered or [])) candidates = [] if normalized_requested: candidates.append(normalized_requested) candidates.extend(["/generate", "/predict"]) candidates.extend(names) candidates = [normalize_api_name(c) for c in dict.fromkeys(candidates) if c] rows = [] for name in candidates: score = endpoint_candidate_score(name, schema) rows.append({"api_name": name, "score": score, "is_requested": name == normalized_requested, "is_discovered": name in names}) valid_rows = [r for r in rows if r["score"] > -100] if not valid_rows: raise RuntimeError(f"No suitable Gradio prediction endpoint found. Discovered endpoints: {names}") if normalized_requested and any(r["api_name"] == normalized_requested and r["is_discovered"] for r in valid_rows): selected = normalized_requested else: selected = sorted(valid_rows, key=lambda r: (r["score"], r["is_discovered"]), reverse=True)[0]["api_name"] return selected, rows def runtime_to_dict(runtime) -> dict: payload = {} for attr in ["stage", "hardware", "requested_hardware", "sleep_time", "storage", "gc_timeout"]: value = getattr(runtime, attr, None) payload[attr] = getattr(value, "value", value) return {k: str(v) if v is not None else None for k, v in payload.items()} def write_space_runtime(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int | None = None) -> dict: try: runtime = api.get_space_runtime(repo_id=target_space_id, token=token) payload = runtime_to_dict(runtime) payload["attempt"] = attempt write_json(run_dir / "space_runtime.json", payload) return payload except Exception as exc: payload = {"error": str(exc)[:2000], "attempt": attempt} write_json(run_dir / "space_runtime.json", payload) append_event(events_path, "space_runtime", "warning", "Could not fetch Space runtime", payload) return payload RESTART_GUARDRAIL_STATE = { "restart": {"count": 0, "last_ts": 0.0}, "factory_reboot": {"count": 0, "last_ts": 0.0}, "same_code_upload": {"count": 0, "last_ts": 0.0}, } RESTART_GUARDRAIL_LIMITS = { "restart": 1, "factory_reboot": 1, "same_code_upload": 1, } RESTART_GUARDRAIL_COOLDOWN_SECONDS = int(os.environ.get("SPACE_FACTORY_RESTART_COOLDOWN_SECONDS", "240")) def runtime_stage_value(runtime_payload: dict | None) -> str: return str((runtime_payload or {}).get("stage") or (runtime_payload or {}).get("runtime_stage") or "").lower() def restart_guardrail_allows(action: str, events_path: Path, *, reason: str = "") -> bool: state = RESTART_GUARDRAIL_STATE.setdefault(action, {"count": 0, "last_ts": 0.0}) limit = RESTART_GUARDRAIL_LIMITS.get(action, 1) now = time.time() if state["count"] >= limit: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart budget exhausted", {"action": action, "limit": limit, "reason": reason}) return False if state["last_ts"] and now - state["last_ts"] < RESTART_GUARDRAIL_COOLDOWN_SECONDS: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart cooldown active", {"action": action, "cooldown_seconds": RESTART_GUARDRAIL_COOLDOWN_SECONDS, "elapsed_seconds": round(now - state["last_ts"], 1), "reason": reason}) return False state["count"] += 1 state["last_ts"] = now return True def safe_restart_space(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, factory_reboot: bool = False, reason: str = "", require_logs_checked: bool = True, allow_busy: bool = False) -> bool: """Restart a Space with strict guardrails. Hugging Face exposes HfApi.restart_space(repo_id=..., factory_reboot=True) for a factory rebuild and restart_space(..., factory_reboot=False) for a normal restart. The Factory only calls it through this wrapper so we do not create restart loops or mask actionable build errors. """ action = "factory_reboot" if factory_reboot else "restart" if require_logs_checked: logs_dir = run_dir / "logs" if not ((logs_dir / "space_logs_build.txt").exists() or (logs_dir / "space_logs_runtime.txt").exists()): append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: logs have not been collected yet", {"reason": reason}) return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"} and not allow_busy: append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: Space is already busy", {"runtime": runtime_before, "reason": reason}) return False if not restart_guardrail_allows(action, events_path, reason=reason): return False try: append_event(events_path, "factory_rebuild", "started", f"Requesting {'factory reboot' if factory_reboot else 'Space restart'} via Hugging Face API", {"runtime_before": runtime_before, "reason": reason}) result = api.restart_space(repo_id=target_space_id, token=token, factory_reboot=factory_reboot) payload = {"factory_reboot": factory_reboot, "reason": reason, "result": str(result)[:1000]} write_json(run_dir / ("space_factory_reboot.json" if factory_reboot else "space_restart.json"), payload) append_event(events_path, "factory_rebuild", "success", f"{'Factory reboot' if factory_reboot else 'Space restart'} requested", payload) return True except TypeError: # Older huggingface_hub versions may not expose factory_reboot. Do not # loop; fall back to a plain restart only for non-factory requests. if factory_reboot: append_event(events_path, "factory_rebuild", "warning", "Installed huggingface_hub does not support restart_space(factory_reboot=True)", {"reason": reason}) return False try: result = api.restart_space(repo_id=target_space_id, token=token) append_event(events_path, "factory_rebuild", "success", "Space restart requested", {"reason": reason, "result": str(result)[:1000]}) return True except Exception as exc: append_event(events_path, "factory_rebuild", "failed", "Space restart API call failed", {"reason": reason, **exception_payload(exc)}) return False except Exception as exc: append_event(events_path, "factory_rebuild", "failed", f"{'Factory reboot' if factory_reboot else 'Space restart'} API call failed", {"reason": reason, **exception_payload(exc)}) return False def safe_same_code_reupload(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, reason: str = "") -> bool: """Re-upload the current workspace once, with restart-loop guardrails.""" if not restart_guardrail_allows("same_code_upload", events_path, reason=reason): return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"}: append_event(events_path, "factory_rebuild", "warning", "Skipped same-code upload: Space is already building/starting", {"runtime": runtime_before, "reason": reason}) return False upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) append_event(events_path, "factory_rebuild", "success", "Same-code workspace uploaded under restart guardrails", {"reason": reason, "runtime_before": runtime_before}) return True def collect_space_logs(target_space_id: str, token: str, run_dir: Path, events_path: Path): """Collect Space build/runtime logs with explicit capability detection and an index. This is intentionally more structured than the old best-effort collector: - runtime state is always captured when possible; - runtime/build log collection first uses HfApi.get_space_logs when available; - if the installed huggingface_hub lacks that method, REST fallbacks are attempted; - legacy text files are still written, but availability/quality lives in space_logs_index.json. """ from huggingface_hub import HfApi import json as _json import sys as _sys try: import huggingface_hub as _hf_hub hub_version = getattr(_hf_hub, "__version__", None) except Exception: hub_version = None logs_dir = run_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) api = HfApi(token=token) collected_at = now() written = [] index = { "schema_version": "1.0", "target_space_id": target_space_id, "collected_at": collected_at, "huggingface_hub_version": hub_version, "python_version": _sys.version.split()[0], "capabilities": { "hfapi_get_space_runtime": hasattr(api, "get_space_runtime"), "hfapi_get_space_logs": hasattr(api, "get_space_logs"), "rest_fallback": True, }, "entries": {}, } def _tail(text: str, n: int = 1200) -> str: return (text or "")[-n:] def _stringify_log_response(out) -> str: if out is None: return "" if isinstance(out, (list, tuple)): return "\n".join(str(x) for x in out) if isinstance(out, (dict, list)): return _json.dumps(out, ensure_ascii=False, indent=2) return str(out) def _write_text(filename: str, text: str) -> int: path = logs_dir / filename path.write_text(text or "", encoding="utf-8") return path.stat().st_size if path.exists() else 0 def _entry(key: str, *, filename: str, available: bool, source: str, quality: str, reason: str = "", error: str = "", text: str = ""): size = _write_text(filename, text) payload = { "available": bool(available), "path": f"logs/{filename}", "source": source, "quality": quality, "reason": reason, "error": error[:1500] if error else "", "size_bytes": size, "tail": _tail(text), } index["entries"][key] = payload written.append({ "file": filename, "source": source, "available": bool(available), "quality": quality, "reason": reason, "error": error[:1000] if error else "", "returncode": 0 if available else 1, "tail": _tail(text, 1000), }) return payload def _unavailable_text(kind: str, reason: str, error: str = "") -> str: return ( f"[ASF_LOG_UNAVAILABLE]\n" f"kind={kind}\n" f"reason={reason}\n" f"target_space_id={target_space_id}\n" f"huggingface_hub_version={hub_version}\n" f"error={error[:1500] if error else ''}\n" "See logs/space_logs_index.json for structured availability metadata.\n" ) # Runtime state / snapshot. This is not a log stream, but it is critical context for # BUILD_ERROR/RUNTIME_ERROR triage and is always tracked separately from log streams. try: runtime_obj = api.get_space_runtime(repo_id=target_space_id, token=token) runtime_payload = runtime_to_dict(runtime_obj) runtime_payload.update({"target_space_id": target_space_id, "collected_at": collected_at}) runtime_text = _json.dumps(runtime_payload, indent=2, ensure_ascii=False) _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) # Legacy path retained because older reports/tests point to this file. _entry("runtime_state_legacy", filename="space_runtime_state.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) except Exception as exc: err = f"{type(exc).__name__}: {str(exc)[:1500]}" _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_snapshot", "runtime_snapshot_error", err)) _entry("runtime_state_legacy", filename="space_runtime_state.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_state", "runtime_snapshot_error", err)) def _collect_via_hfapi(kind: str): if not hasattr(api, "get_space_logs"): return None, "hub_method_unavailable", "" try: out = api.get_space_logs(repo_id=target_space_id, token=token, build=(kind == "build")) text = _stringify_log_response(out) if text.strip(): return text, "", "" return "", "empty_response", "" except Exception as exc: return None, "hfapi_error", f"{type(exc).__name__}: {str(exc)[:1500]}" def _space_log_endpoint(kind: str) -> str: # HF Space log streams are exposed under /logs/run and /logs/build. # "runtime" is an ASF legacy label for the file name; the Hub endpoint is "run". endpoint_kind = "build" if kind == "build" else "run" return f"https://huggingface.co/api/spaces/{target_space_id}/logs/{endpoint_kind}" def _read_streaming_response(response, *, max_bytes: int = 1_200_000) -> str: chunks = [] total = 0 try: iterator = response.iter_content(chunk_size=8192, decode_unicode=True) except Exception: return response.text or "" for chunk in iterator: if not chunk: continue if isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="ignore") chunks.append(str(chunk)) total += len(str(chunk).encode("utf-8", errors="ignore")) if total >= max_bytes: chunks.append("\n[ASF_LOG_TRUNCATED: max_bytes reached]\n") break return "".join(chunks) def _collect_via_rest(kind: str): # v198.16: use the real HF Hub Space log endpoints. The runtime stream is # /logs/run, not any legacy runtime endpoint and not the older query-string endpoint. # Wrong endpoints made Pi repair from generic RUNTIME_ERROR messages. try: import requests except Exception as exc: return None, "requests_unavailable", f"{type(exc).__name__}: {str(exc)[:1500]}" headers = {"Authorization": f"Bearer {token}", "Accept": "text/plain,application/json,*/*"} if token else {"Accept": "text/plain,application/json,*/*"} primary_url = _space_log_endpoint(kind) candidates = [primary_url] last_error = "" for url in candidates: try: response = requests.get(url, headers=headers, timeout=(10, 45), stream=True) body = _read_streaming_response(response) if response.ok and body.strip(): ctype = response.headers.get("content-type", "") return body, "", f"{url} ({response.status_code}, {ctype})" last_error = f"{url} returned {response.status_code}: {body[:500]}" except Exception as exc: last_error = f"{url} failed: {type(exc).__name__}: {str(exc)[:500]}" return None, "rest_logs_unavailable", last_error def _collect_stream(kind: str, filename: str): attempted = [] text, reason, error = _collect_via_hfapi(kind) attempted.append({"method": "HfApi.get_space_logs", "reason": reason, "error": error[:500] if error else ""}) if text is not None and text.strip(): return _entry(kind, filename=filename, available=True, source="HfApi.get_space_logs", quality="full", text=text) rest_text, rest_reason, rest_error = _collect_via_rest(kind) attempted.append({"method": "hf_rest_logs", "reason": rest_reason, "error": rest_error[:500] if rest_error else ""}) if rest_text is not None and rest_text.strip(): source = "hf_rest_logs" if rest_error and rest_error.startswith("https://"): source = rest_error return _entry(kind, filename=filename, available=True, source=source, quality="full", text=rest_text) # Keep legacy files present, but explicitly mark them as unavailable rather than # pretending an SDK AttributeError is a Space runtime/build log. This also avoids # the obsolete CLI path that emitted "invalid choice: spaces". final_reason = rest_reason or reason or "logs_unavailable" final_error = rest_error or error or "" entry = _entry(kind, filename=filename, available=False, source="log_collection", quality="unavailable", reason=final_reason, error=final_error, text=_unavailable_text(kind, final_reason, final_error)) entry["attempted_methods"] = attempted return entry runtime_entry = _collect_stream("runtime", "space_logs_runtime.txt") build_entry = _collect_stream("build", "space_logs_build.txt") def _first_error_from(text: str) -> str: if not text: return "" lines = text.splitlines() lowered = [line.lower() for line in lines] markers = ("traceback", "exception", "error", "exit code", "failed", "runtimeerror", "attributeerror", "modulenotfounderror") for idx, line in enumerate(lowered): if any(marker in line for marker in markers): start = max(0, idx - 3) end = min(len(lines), idx + 12) return "\n".join(lines[start:end])[:4000] return "" runtime_text = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if runtime_entry.get("available") and (logs_dir / "space_logs_runtime.txt").exists() else "" build_text = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if build_entry.get("available") and (logs_dir / "space_logs_build.txt").exists() else "" first_error = _first_error_from(runtime_text) or _first_error_from(build_text) log_quality = "full" if runtime_entry.get("available") or build_entry.get("available") else "snapshot_only" if index["entries"].get("runtime_snapshot", {}).get("available") else "unavailable" diagnostics = { "schema_version": "1.0", "target_space_id": target_space_id, "collected_at": collected_at, "log_quality": log_quality, "runtime_logs_available": bool(runtime_entry.get("available")), "build_logs_available": bool(build_entry.get("available")), "first_error": first_error, "diagnosis_log_source": "runtime_or_build_logs" if first_error else ("runtime_snapshot" if log_quality == "snapshot_only" else "none"), } diagnostics_text = _json.dumps(diagnostics, ensure_ascii=False, indent=2) _entry("diagnostics", filename="space_log_diagnostics.json", available=True, source="asf_log_collector", quality=log_quality, text=diagnostics_text) index["log_quality"] = log_quality index["runtime_logs_available"] = bool(runtime_entry.get("available")) index["build_logs_available"] = bool(build_entry.get("available")) index["first_error"] = first_error[:1000] index_text = _json.dumps(index, ensure_ascii=False, indent=2) (logs_dir / "space_logs_index.json").write_text(index_text, encoding="utf-8") written.append({"file": "space_logs_index.json", "source": "asf_log_collector", "available": True, "quality": log_quality, "returncode": 0, "tail": _tail(index_text, 1000)}) event_status = "success" if runtime_entry.get("available") or build_entry.get("available") else "warning" event_message = "Collected Space logs and runtime snapshot" if event_status == "success" else "Space log streams unavailable; runtime snapshot/index written" append_event(events_path, "space_logs", event_status, event_message, {"files": written, "log_quality": log_quality, "index": "logs/space_logs_index.json"}) write_artifact_manifest(run_dir, reason="space_logs_collected") return written def smoke_failure_repair_escalation_decision(generation_smoke: dict | None, *, validation: dict | None = None, contract: dict | None = None) -> dict: """Decide whether a failed generation smoke should trigger one targeted repair. v198.13: partial validation is useful for UI honesty, but a demo that boots yet cannot satisfy its primary flow should not stop there when the smoke failure is concrete and minimally repairable. This is deliberately bounded: validator-owned failures are handled by validator self-repair, model init without logs remains partial, and we only escalate actionable app/runtime failures such as a missing executable or missing Python dependency. """ smoke = generation_smoke if isinstance(generation_smoke, dict) else {} validation = validation if isinstance(validation, dict) else {} contract = contract if isinstance(contract, dict) else {} failure_class = str(smoke.get("failure_class") or smoke.get("failure_type") or "") failure_owner = str(smoke.get("failure_owner") or "") repair_candidate = bool(smoke.get("repair_candidate")) health_semantics = validation_health_semantics(validation) health_model_ready = validation_health_passed(validation) health_reachable = validation_endpoint_reachable(validation) actionable_classes = {"missing_runtime_cli", "missing_python_dependency", "app_runtime_error", "model_pipeline_not_loaded", "model_not_initialized", "hf_cache_permission_denied", "diffusers_duplicate_token_argument", "sdxl_lora_text_encoder_mismatch"} blocked_classes = {"validator_schema_choice_type_mismatch", "validator_request_error", "smoke_schema_error", "smoke_input_materialization_failed", "gradio_hidden_runtime_error", "gpu_oom", "insufficient_vram", "insufficient_vram_or_payload_too_large", "hardware_capacity", "cuda_out_of_memory"} strategy = str(contract.get("inference_strategy") or smoke.get("inference_strategy") or "") payload = { "schema_version": "generation_smoke_repair_escalation.v198_26_20", "triggered": False, "repair_candidate": repair_candidate, "health_passed": health_model_ready, "health_endpoint_reachable": health_reachable, "health_semantic_passed": health_semantics.get("semantic_passed"), "health_negative_markers": health_semantics.get("negative_markers") or [], "failure_class": failure_class, "failure_owner": failure_owner, "inference_strategy": strategy, "action": "keep_partial_validation", "reason": "not_actionable_or_not_safe_for_automatic_smoke_repair", } if str(smoke.get("status") or "") != "failed": payload["reason"] = "generation_smoke_not_failed" return payload if failure_class in blocked_classes or failure_owner in {"factory_validator", "factory_validation_client", "hardware"}: payload["reason"] = "handled_elsewhere_or_requires_logs_not_blind_repair" return payload if not health_reachable: payload["reason"] = "app_not_reachable_use_runtime_recovery_instead" return payload if not repair_candidate or failure_class not in actionable_classes: payload["reason"] = "smoke_failure_not_repair_candidate" return payload payload.update({ "triggered": True, "action": "targeted_patch_repair_before_final_partial", "reason": "space_reachable_but_primary_demo_flow_failed_with_actionable_smoke_error", "recommended_action": smoke.get("recommended_action") or "Run one targeted repair, redeploy, then re-run health and canonical smoke validation.", "missing_executable": smoke.get("missing_executable") or "", }) return payload def load_recipe_repair_context(workspace: Path | None, run_dir: Path | None) -> dict: """Load MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json for repair.""" workspace = Path(workspace) if workspace else None run_dir = Path(run_dir) if run_dir else None def load_named(name: str) -> tuple[dict, str]: candidates = [] if workspace: candidates.append(workspace / name) if run_dir: candidates.append(run_dir / name) candidates.append(run_dir / "analysis_inputs" / name) for candidate in candidates: try: if candidate.exists(): payload = load_json_if_exists(candidate) if isinstance(payload, dict): return payload, str(candidate) except Exception: continue return {}, "" recipe, recipe_source = load_named("MODEL_RECIPE.json") runtime_contract, runtime_source = load_named("APP_RUNTIME_CONTRACT.json") if not runtime_contract and recipe: try: runtime_contract = build_app_runtime_contract(recipe) runtime_source = "derived_from_MODEL_RECIPE.json" except Exception: runtime_contract = {} return { "schema_version": "recipe_repair_context.v198_26_32", "model_recipe": recipe, "model_recipe_source": recipe_source, "app_runtime_contract": runtime_contract, "app_runtime_contract_source": runtime_source, "recipe_available": bool(recipe), "runtime_contract_available": bool(runtime_contract), "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "artifact_role": recipe.get("artifact_role") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", "primary_endpoint": recipe.get("primary_endpoint") or runtime_contract.get("primary_endpoint") or "/generate", } def recipe_known_failure_catalog(recipe: dict | None = None, runtime_contract: dict | None = None) -> list[dict]: recipe = recipe if isinstance(recipe, dict) else {} runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} catalog = [] for item in recipe.get("known_failure_signatures") or []: if isinstance(item, dict): catalog.append(dict(item)) required = [ {"failure_class": "diffusers_duplicate_token_argument", "markers": ["got multiple values for keyword argument 'token'", "multiple values for keyword argument \"token\"", "duplicate token"], "repair_focus": "Remove duplicate token/use_auth_token injection from from_pretrained kwargs; do not change the model id or replace inference."}, {"failure_class": "hf_cache_permission_denied", "markers": ["permission denied", "/data", "HF_HOME", "huggingface cache", "read-only file system", "errno 13"], "repair_focus": "Move HF_HOME/HF_HUB_CACHE/TRANSFORMERS_CACHE/DIFFUSERS_CACHE to a writable Space path before importing/loading models."}, {"failure_class": "missing_spaces_gpu_decorator", "markers": ["missing @spaces.GPU", "ZeroGPU", "CUDA has been initialized before", "GPU task function", "@spaces.GPU"], "repair_focus": "Import spaces before CUDA-touching imports and protect the heavy inference function with @spaces.GPU while preserving fixed-GPU compatibility."}, {"failure_class": "validator_schema_choice_type_mismatch", "markers": ["not in the list of choices", "Invalid value", "Dropdown", "CheckboxGroup", "choice"], "repair_focus": "Respect the published Gradio schema and canonical smoke payload; do not patch the model for validator-owned payload coercion errors."}, {"failure_class": "sdxl_lora_text_encoder_mismatch", "markers": ["CLIPTextModel", "text_model", "has no attribute 'text_model'"], "repair_focus": "Treat the repo as an SDXL LoRA adapter: load the base SDXL pipeline first, then load_lora_weights(adapter_id, weight_name=...), and avoid loading the adapter as a standalone pipeline."}, {"failure_class": "flux_template_syntax_error", "markers": ["FluxPipeline", "jinja", "template", "syntax error", "unexpected", "chat_template"], "repair_focus": "Repair FLUX/prompt-template syntax without changing FluxPipeline loader semantics or replacing image generation with placeholder output."}, {"failure_class": "model_not_loaded_after_boot", "markers": ["model not loaded", "pipeline not loaded", "model_ready", "pipeline_ready", "last_error", "not initialized", "NoneType"], "repair_focus": "Fix loader initialization and expose the real load failure through /health.last_error; /generate must fail cleanly when the model is not ready."}, ] seen = {str(item.get("failure_class") or "") for item in catalog} for item in required: if item["failure_class"] not in seen: catalog.append(item) seen.add(item["failure_class"]) return catalog def match_recipe_known_failure_signature(error_text: str = "", recipe: dict | None = None, runtime_contract: dict | None = None, fallback_class: str = "") -> dict: text = str(error_text or "") low = text.lower() recipe = recipe if isinstance(recipe, dict) else {} runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} fallback_class = str(fallback_class or "") for sig in recipe_known_failure_catalog(recipe, runtime_contract): failure_class = str(sig.get("failure_class") or sig.get("category") or "") markers = [str(m) for m in (sig.get("markers") or sig.get("error_markers") or sig.get("signatures") or []) if str(m)] class_matches = bool(failure_class and fallback_class and failure_class == fallback_class) marker_matches = [m for m in markers if m.lower() in low] if failure_class == "sdxl_lora_text_encoder_mismatch" and is_sdxl_lora_text_encoder_mismatch_text(text): marker_matches = marker_matches or ["CLIPTextModel.text_model"] if class_matches or marker_matches: return { "schema_version": "recipe_known_failure_match.v198_26_32", "matched": True, "failure_class": failure_class or fallback_class, "matched_markers": marker_matches[:10], "repair_focus": sig.get("repair_focus") or sig.get("recommended_repair_focus") or sig.get("recommendation") or "Patch according to MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json.", "source_signature": sig, "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", } if fallback_class: return {"schema_version": "recipe_known_failure_match.v198_26_32", "matched": False, "failure_class": fallback_class, "matched_markers": [], "repair_focus": "No catalog signature matched; still use MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json as authoritative repair context.", "source_signature": {}, "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or ""} return {"schema_version": "recipe_known_failure_match.v198_26_32", "matched": False, "failure_class": "", "matched_markers": [], "repair_focus": ""} def recipe_family_repair_directives(recipe: dict | None = None, runtime_contract: dict | None = None, signature_match: dict | None = None) -> list[str]: recipe = recipe if isinstance(recipe, dict) else {} runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} signature_match = signature_match if isinstance(signature_match, dict) else {} family = str(recipe.get("model_family") or runtime_contract.get("model_family") or "") directives = [ "Use MODEL_RECIPE.json as the authoritative model-family plan, not a generic app repair brief.", "Preserve APP_RUNTIME_CONTRACT.json: /health must expose model_ready, pipeline_ready, model_family, loader_strategy, last_error, and expected_output_type.", "After patching, /generate must either return the expected output type or fail cleanly with the same underlying loader error visible in /health.last_error.", ] if signature_match.get("repair_focus"): directives.append("Known failure focus: " + str(signature_match.get("repair_focus"))) if family == "diffusers_sdxl_lora_adapter": directives.extend([ "This repo is an adapter. Do not call DiffusionPipeline.from_pretrained(adapter_id) as if it were a full pipeline.", "Load the SDXL base model from model_recipe.base_model_id, then call load_lora_weights(adapter_id, weight_name=adapter_weight_name when present).", "Keep the LoRA trigger words/prompt guidance from the model card when available.", ]) elif family == "diffusers_flux_pipeline": directives.extend([ "Keep FluxPipeline.from_pretrained semantics and the recipe-selected model id.", "Do not replace FLUX inference with a text-only or placeholder response; preserve image output.", ]) elif family == "gguf_llamacpp": directives.append("Use llama.cpp/llama-cpp-python style GGUF loading and text generation; do not attempt Transformers AutoModel loading for GGUF weights.") elif family == "onnx_runtime_model": directives.append("Use onnxruntime InferenceSession and preserve input/output tensor preprocessing; do not convert to a different runtime blindly.") elif family == "transformers_pipeline": directives.append("Use the Transformers pipeline/AutoModel strategy indicated by the recipe and preserve the task-specific output contract.") return directives def write_recipe_aware_repair_packet(workspace: Path, run_dir: Path, failure_reason: str = "", classification: dict | None = None, decision: dict | None = None) -> dict: ctx = load_recipe_repair_context(workspace, run_dir) recipe = ctx.get("model_recipe") or {} runtime_contract = ctx.get("app_runtime_contract") or {} category = str((classification or {}).get("category") or (classification or {}).get("failure_class") or (decision or {}).get("failure_class") or "") match = match_recipe_known_failure_signature(failure_reason, recipe, runtime_contract, fallback_class=category) packet = { "schema_version": "recipe_aware_repair_packet.v198_26_32", "authority": "MODEL_RECIPE.json + APP_RUNTIME_CONTRACT.json", **ctx, "known_failure_match": match, "family_repair_directives": recipe_family_repair_directives(recipe, runtime_contract, match), "repair_must_not_finish_not_started": True, "no_patch_policy": "If Pi cannot produce a publishable patch, ASF must either launch another more direct recipe-aware attempt while budget remains or finish with no_patch_produced_by_pi / repair_exhausted, never post_repair_validation=not_started.", } repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "RECIPE_AWARE_REPAIR_PACKET.json", packet) try: write_json(workspace / "RECIPE_AWARE_REPAIR_PACKET.json", packet) except Exception: pass return packet def recipe_repair_direct_followup_reason(run_dir: Path, terminal_status: str, failure_class: str = "") -> str: latest = _latest_repair_attempt_result(run_dir) return ( "Previous Pi repair did not produce an uploadable publishable patch.\n" f"Terminal status: {terminal_status}\n" f"Latest attempt result: {latest.get('post_repair_result') or ''}\n" f"Failure class: {failure_class or latest.get('failure_class') or ''}\n" "Relaunch with a more direct recipe-aware instruction: implement the known failure focus from RECIPE_AWARE_REPAIR_PACKET.json, modify only allowed publishable files, or write PATCH_REFUSAL.json with a concrete blocker." ) def should_relaunch_after_no_patch(run_dir: Path, terminal_status: str, remaining_patch_budget: int) -> dict: terminal_status = str(terminal_status or "") no_patch_statuses = {"no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard", "pi_repair_failed_before_upload"} latest = _latest_repair_attempt_result(run_dir) result = str(latest.get("post_repair_result") or "") result_is_no_patch = result in {"repair_noop", "expected_file_not_modified", "diff_gate_failed", "sanity_failed"} can_retry = remaining_patch_budget > 0 and (terminal_status in no_patch_statuses or result_is_no_patch) return {"schema_version": "recipe_repair_retry_decision.v198_26_32", "relaunch": bool(can_retry), "terminal_status": terminal_status, "remaining_patch_budget": int(remaining_patch_budget or 0), "latest_attempt": latest, "reason": "retry_with_more_direct_recipe_aware_brief" if can_retry else "repair_exhausted_or_not_retryable"} def write_smoke_repair_primary_error_packet(run_dir: Path, workspace: Path, generation_smoke: dict | None, *, failure_class: str = "") -> dict: smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = read_inference_contract(workspace) recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") if not isinstance(recipe, dict): recipe = {} error_text = str(smoke.get("error") or smoke.get("result_info") or smoke.get("result_repr") or smoke)[:6000] payload = { "schema_version": "smoke_repair_primary_error.v198_26_27", "source": "generation_smoke", "failure_class": failure_class or smoke.get("failure_class") or smoke.get("failure_type") or "", "failure_type": smoke.get("failure_type") or "", "failure_owner": smoke.get("failure_owner") or "", "error": error_text, "expected_output_type": smoke.get("expected_output_type") or contract.get("expected_output_type") or "", "primary_api_name": smoke.get("api_name") or contract.get("primary_api_name") or contract.get("primary_endpoint") or "/generate", "contract": contract, "model_recipe": recipe, "sdxl_lora_text_encoder_mismatch": is_sdxl_lora_text_encoder_mismatch_text(error_text), "recommended_repair_focus": "sdxl_lora_base_loader_or_text_encoder_compatibility" if is_sdxl_lora_text_encoder_mismatch_text(error_text) else "model_loading_or_runtime_patch", } repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "SMOKE_PRIMARY_ERROR.json", payload) try: shutil.copy2(repair_dir / "SMOKE_PRIMARY_ERROR.json", workspace / "SMOKE_PRIMARY_ERROR.json") except Exception: pass return payload def _latest_repair_attempt_result(run_dir: Path) -> dict: payload = load_repair_attempts(run_dir) attempts = payload.get("attempts") if isinstance(payload.get("attempts"), list) else [] return attempts[-1] if attempts else {} def failed_pi_repair_terminal_status(run_dir: Path) -> dict: latest = _latest_repair_attempt_result(run_dir) result = str(latest.get("post_repair_result") or "").strip() mapping = { "repair_noop": ("no_patch_produced_by_pi", "Pi repair produced no publishable patch."), "pi_refused_patch": ("pi_refused_with_reason", "Pi refused to patch under the bounded repair policy."), "pi_exit_nonzero": ("repair_aborted_with_exception", "Pi repair command exited non-zero."), "diff_gate_failed": ("pi_patch_rejected_by_guard", "Pi produced a patch rejected by the repair diff guard."), "sanity_failed": ("pi_patch_rejected_by_guard", "Pi produced a patch rejected by sanity checks."), "expected_file_not_modified": ("no_relevant_patch_produced_by_pi", "Pi did not modify the expected publishable file."), } status, message = mapping.get(result, ("pi_repair_failed_before_upload", "Targeted Pi repair did not produce an uploadable patch.")) return {"post_repair_validation": status, "failure_type": status, "final_user_message": message, "latest_repair_attempt": latest} def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, token: str, implementation_mode: str, expected_output_type: str, validation: dict, generation_smoke: dict) -> tuple[dict, dict]: """Run one bounded targeted repair when the demo boots but its primary flow fails. The function returns the latest (validation, generation_smoke). It never loops: one repair, one upload, one health validation, one generation smoke retry. """ contract = read_inference_contract(workspace) escalation = smoke_failure_repair_escalation_decision(generation_smoke, validation=validation, contract=contract) write_json(run_dir / "tests" / "generation_smoke_repair_escalation.json", escalation) if not escalation.get("triggered"): return validation, generation_smoke failure_class = escalation.get("failure_class") or generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "generation_smoke_error" pre_repair_smoke = dict(generation_smoke) if isinstance(generation_smoke, dict) else {} try: collect_space_logs(target_space_id, token, run_dir, events_path) except Exception as log_exc: append_event(events_path, "generation_smoke_repair", "warning", "Could not refresh Space logs before targeted smoke repair", {"error": str(log_exc)[:1500]}) try: logs_dir = run_dir / "logs" latest_log_brief = "\n".join([ (logs_dir / "space_log_diagnostics.json").read_text(encoding="utf-8", errors="ignore")[:1500] if (logs_dir / "space_log_diagnostics.json").exists() else "", (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore")[-1500:] if (logs_dir / "space_logs_runtime.txt").exists() else "", (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore")[-1500:] if (logs_dir / "space_logs_build.txt").exists() else "", ]).strip() except Exception: latest_log_brief = "" cache_permission_evidence = "\n".join([ str(generation_smoke.get("error") or generation_smoke.get("result_info") or generation_smoke), latest_log_brief, ]) token_duplicate_evidence = cache_permission_evidence if failure_class == "diffusers_duplicate_token_argument" or is_diffusers_duplicate_token_text(token_duplicate_evidence): decision = { "action": "factory_patch_diffusers_token_guard", "confidence": "high", "source": "generation_smoke_repair_escalation.v198_26_20", "classification": { "category": "diffusers_duplicate_token_argument", "failure_phase": "generation_smoke", "recommendation": "Remove duplicate token injection from Diffusers from_pretrained kwargs, redeploy, then validate /generate.", "generation_smoke_failure": True, }, "constraints": [ "Do not replace the real inference app with diagnostics.", "Do not change the Gradio API contract or expected output type.", "Patch only duplicate token handling around Diffusers/model from_pretrained calls.", "After upload, success requires /generate to pass; /health alone is not full repair success.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Applying deterministic Diffusers token repair before Pi repair", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause="diffusers_duplicate_token_argument", repair_decision="factory_patch_diffusers_token_guard", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) token_patch = patch_workspace_for_diffusers_token_guard(workspace, run_dir, events_path, reason="generation_smoke_duplicate_token_repair", evidence=token_duplicate_evidence) if token_patch.get("applied"): write_repair_outcome(run_dir, events_path, patch_applied=True, deterministic_patch=token_patch, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading deterministic Diffusers token repair") try: write_auth_probe(run_dir, events_path, "before_diffusers_token_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Deterministic Diffusers token repair uploaded; revalidating health and demo smoke") except Exception as upload_exc: latest_smoke = dict(generation_smoke) latest_smoke.update({"status": "failed", "failure_type": "smoke_repair_upload_failed", "failure_class": "smoke_repair_upload_failed", "failure_owner": "factory_upload", "error": str(upload_exc)[:4000]}) write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="upload_failed", failure_type="smoke_repair_upload_failed", final_user_message="Deterministic Diffusers token repair was prepared but failed to upload; keeping partial validation.", validation_error=str(upload_exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_upload", "failed", "Deterministic Diffusers token repair upload failed; keeping partial validation", {"error": str(upload_exc)[:2000]}) return validation, latest_smoke try: write_auth_probe(run_dir, events_path, "before_diffusers_token_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_class = repair_post_smoke_validation_status(pre_repair_smoke, generation_smoke) post_status = post_class["post_repair_validation"] write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type=post_class.get("failure_type") or "", final_user_message=post_class.get("message") or "Deterministic Diffusers token repair validation completed.", post_repair_generation_smoke=generation_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Deterministic Diffusers token repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status"), "same_error": post_class.get("same_error")}) except Exception as exc: latest_smoke = build_generation_smoke_failure_payload(target_space_id, expected_output_type, exc, workspace=workspace, phase="post_diffusers_token_repair_exception") smoke_diagnosis = diagnose_generation_smoke_failure(exc, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="post_diffusers_token_repair_exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, latest_smoke, phase="post_diffusers_token_repair_exception") post_class = repair_post_smoke_validation_status(pre_repair_smoke, latest_smoke) generation_smoke = latest_smoke write_repair_outcome(run_dir, events_path, post_repair_validation=post_class["post_repair_validation"], failure_type=latest_smoke.get("failure_type") or post_class.get("failure_type") or "diffusers_token_repair_validation_failed", final_user_message=post_class.get("message") or "Deterministic Diffusers token repair validation failed; keeping partial validation.", validation_error=str(exc)[:4000], post_repair_generation_smoke=latest_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "failed", "Deterministic Diffusers token repair validation failed; keeping partial validation", {"error": str(exc)[:2000], "latest_failure_class": latest_smoke.get("failure_class"), "latest_failure_owner": latest_smoke.get("failure_owner"), "same_error": post_class.get("same_error")}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, deterministic_patch=token_patch, patch_applied=False, post_repair_validation="not_started", failure_type="diffusers_token_patch_not_applied") append_event(events_path, "generation_smoke_repair", "warning", "Diffusers duplicate-token repair was indicated but no deterministic token patch was applied; falling back to Pi repair", token_patch) if failure_class == "hf_cache_permission_denied" or is_hf_cache_permission_denied_text(cache_permission_evidence): decision = { "action": "factory_patch_writable_hf_cache", "confidence": "high", "source": "generation_smoke_repair_escalation.v198_26_20", "classification": { "category": "hf_cache_permission_denied", "failure_phase": "generation_smoke", "recommendation": "Rewrite generated app HF cache env vars away from /data to a writable /tmp cache, redeploy, then validate /generate.", "generation_smoke_failure": True, }, "constraints": [ "Do not replace the real inference app with diagnostics.", "Do not change the Gradio API contract or expected output type.", "Patch only Hugging Face cache environment paths and bootstrap writable cache directories.", "After upload, success requires /generate to pass; /health alone is not full repair success.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Applying deterministic writable HF cache repair before Pi repair", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause="hf_cache_permission_denied", repair_decision="factory_patch_writable_hf_cache", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) cache_patch = patch_workspace_for_writable_hf_cache(workspace, run_dir, events_path, reason="generation_smoke_permission_denied_repair", evidence=cache_permission_evidence) if cache_patch.get("applied"): write_repair_outcome(run_dir, events_path, patch_applied=True, deterministic_patch=cache_patch, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading deterministic writable cache repair") try: write_auth_probe(run_dir, events_path, "before_writable_cache_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Deterministic writable cache repair uploaded; revalidating health and demo smoke") except Exception as upload_exc: latest_smoke = dict(generation_smoke) latest_smoke.update({"status": "failed", "failure_type": "smoke_repair_upload_failed", "failure_class": "smoke_repair_upload_failed", "failure_owner": "factory_upload", "error": str(upload_exc)[:4000]}) write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="upload_failed", failure_type="smoke_repair_upload_failed", final_user_message="Deterministic writable cache repair was prepared but failed to upload; keeping partial validation.", validation_error=str(upload_exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_upload", "failed", "Deterministic writable cache repair upload failed; keeping partial validation", {"error": str(upload_exc)[:2000]}) return validation, latest_smoke try: write_auth_probe(run_dir, events_path, "before_writable_cache_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_class = repair_post_smoke_validation_status(pre_repair_smoke, generation_smoke) if 'pre_repair_smoke' in locals() else repair_post_smoke_validation_status(generation_smoke, generation_smoke) post_status = post_class["post_repair_validation"] write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type=post_class.get("failure_type") or "", final_user_message="Deterministic writable cache repair revalidated the Space." if post_status == "success" else post_class.get("message") or "Deterministic writable cache repair uploaded, but generation smoke still did not verify the primary demo flow.", post_repair_generation_smoke=generation_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Deterministic writable cache repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status"), "same_error": post_class.get("same_error")}) except Exception as exc: repair_class = classify_repair_validation_error(exc) latest_smoke = build_generation_smoke_failure_payload(target_space_id, expected_output_type, exc, workspace=workspace, phase="post_writable_cache_repair_exception") smoke_diagnosis = diagnose_generation_smoke_failure(exc, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="post_writable_cache_repair_exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, latest_smoke, phase="post_writable_cache_repair_exception") generation_smoke = latest_smoke write_repair_outcome(run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=latest_smoke.get("failure_type") or repair_class["failure_type"] or "writable_cache_repair_validation_failed", final_user_message=repair_class["message"], validation_error=str(exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_validation", "failed", "Deterministic writable cache repair validation failed; keeping partial validation", {"error": str(exc)[:2000], "latest_failure_class": latest_smoke.get("failure_class"), "latest_failure_owner": latest_smoke.get("failure_owner")}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, deterministic_patch=cache_patch, patch_applied=False, post_repair_validation="not_started", failure_type="writable_cache_patch_not_applied") append_event(events_path, "generation_smoke_repair", "warning", "Writable cache repair was indicated but no deterministic cache patch was applied; falling back to Pi repair", cache_patch) primary_error_packet = write_smoke_repair_primary_error_packet(run_dir, workspace, generation_smoke, failure_class=failure_class) failure_reason = ( "Automatic generation smoke failed after the Space became reachable.\n" f"Authoritative smoke error: {primary_error_packet.get('error') or ''}\n" f"Failure class: {failure_class}\n" f"Failure owner: {generation_smoke.get('failure_owner') or ''}\n" f"Actionability: {generation_smoke.get('actionability') or ''}\n" f"Recommended action: {generation_smoke.get('recommended_action') or ''}\n" f"Error/result: {generation_smoke.get('error') or generation_smoke.get('result_info') or generation_smoke}\n" f"Latest Space log brief: {latest_log_brief[:3000]}\n" ) decision = { "action": "patch_code", "confidence": "high" if failure_class in {"missing_runtime_cli", "missing_python_dependency"} else "medium", "source": "generation_smoke_repair_escalation.v198_26_20", "classification": { "category": failure_class, "failure_phase": "generation_smoke", "logs_quality": "useful", "recommendation": escalation.get("recommended_action") or generation_smoke.get("recommended_action") or "Patch the smallest concrete dependency or command issue, preserve the demo flow, redeploy, then re-run smoke validation.", "generation_smoke_failure": True, "smoke_primary_error_packet": "repair/SMOKE_PRIMARY_ERROR.json", "sdxl_lora_text_encoder_mismatch": failure_class == "sdxl_lora_text_encoder_mismatch", "missing_executable": generation_smoke.get("missing_executable") or "", }, "constraints": [ "Preserve real inference and the existing Gradio API contract.", "Do not replace the model call with placeholders or diagnostics if the failure is repairable.", "Patch only the dependency, command invocation, model loading, or ZeroGPU compatibility issue required for the failed smoke path.", "For ZeroGPU, keep import spaces before CUDA-touching imports and keep the heavy inference function protected by @spaces.GPU while preserving fixed-GPU compatibility.", "After upload, the repair is only fully successful if /generate passes; /health alone is boot-level evidence.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Running one targeted repair because the Space is reachable but the primary demo smoke failed", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause=failure_class, repair_decision="patch_code", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger="generation_smoke_failure") if not repaired: terminal = failed_pi_repair_terminal_status(run_dir) write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation=terminal["post_repair_validation"], failure_type=terminal["failure_type"], final_user_message=terminal["final_user_message"], latest_repair_attempt=terminal.get("latest_repair_attempt"), pre_repair_generation_smoke=generation_smoke) append_event(events_path, "generation_smoke_repair", "failed", terminal["final_user_message"], {"failure_class": failure_class, "terminal_status": terminal["post_repair_validation"]}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading targeted smoke repair") try: write_auth_probe(run_dir, events_path, "before_smoke_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) except Exception as upload_exc: latest_smoke = dict(generation_smoke) latest_smoke.update({"status": "failed", "failure_type": "smoke_repair_upload_failed", "failure_class": "smoke_repair_upload_failed", "failure_owner": "factory_upload", "error": str(upload_exc)[:4000]}) write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="upload_failed", failure_type="smoke_repair_upload_failed", final_user_message="Targeted smoke repair was prepared but failed to upload; keeping partial validation.", validation_error=str(upload_exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_upload", "failed", "Targeted smoke repair upload failed; keeping partial validation", {"error": str(upload_exc)[:2000]}) return validation, latest_smoke write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Targeted smoke repair uploaded; revalidating health and demo smoke") try: write_auth_probe(run_dir, events_path, "before_smoke_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_class = repair_post_smoke_validation_status(pre_repair_smoke, generation_smoke) if 'pre_repair_smoke' in locals() else repair_post_smoke_validation_status(generation_smoke, generation_smoke) post_status = post_class["post_repair_validation"] write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type=post_class.get("failure_type") or "", final_user_message=post_class.get("message") or ("Targeted smoke repair revalidated the Space." if post_status == "success" else "Targeted smoke repair uploaded, but generation smoke still did not verify the primary demo flow."), post_repair_generation_smoke=generation_smoke, repair_same_error=post_class.get("same_error"), pre_repair_signature=post_class.get("pre_repair_signature"), post_repair_signature=post_class.get("post_repair_signature")) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Targeted smoke repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status"), "same_error": post_class.get("same_error")}) except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome(run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"] or "smoke_repair_validation_failed", final_user_message=repair_class["message"], validation_error=str(exc)[:4000]) append_event(events_path, "generation_smoke_repair_validation", "failed", "Targeted smoke repair validation failed; keeping partial validation", {"error": str(exc)[:2000]}) return validation, generation_smoke def patch_workspace_for_diffusers_token_guard(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_guard", evidence: str = "") -> dict: """Remove duplicate Diffusers `token` injection patterns before upload/repair. The guard is intentionally deterministic and conservative: it only patches token handling around model/Diffusers `from_pretrained` calls when a mutable kwargs dict that is passed with `**...` also receives `token`/ `use_auth_token`, or when runtime evidence proves duplicate token injection. It covers generated variants such as `_load_kwargs["token"] = HF_TOKEN` plus `from_pretrained(..., **_load_kwargs, token=HF_TOKEN)`, which previously slipped through because the detector only looked for a variable literally named `kwargs`. """ result = { "schema_version": "diffusers_token_guard.v198_26_24", "reason": reason, "applied": False, "checked_file": "app.py", "detected_duplicate_token_risk": False, "evidence_matched": is_diffusers_duplicate_token_text(evidence or ""), "replacements": 0, "removed_lines": [], "notes": [], "kwargs_vars_with_token_assignment": [], "from_pretrained_star_kwargs_vars": [], "from_pretrained_explicit_token_args": False, "post_patch_duplicate_token_risk": False, } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "diffusers_token_guard.json", result) return result text = app_path.read_text(encoding="utf-8", errors="ignore") lower = text.lower() has_diffusers_pipeline = "diffusers" in lower or "pipeline" in lower or "from_pretrained" in lower def _from_pretrained_call_bodies(src: str) -> list[str]: bodies = [] marker = "from_pretrained" pos = 0 while True: idx = src.find(marker, pos) if idx < 0: break open_idx = src.find("(", idx + len(marker)) if open_idx < 0: break depth = 0 quote = None escape = False for j in range(open_idx, len(src)): ch = src[j] if quote: if escape: escape = False elif ch == "\\": escape = True elif ch == quote: quote = None continue if ch in {"'", '"'}: quote = ch continue if ch == "(": depth += 1 elif ch == ")": depth -= 1 if depth == 0: bodies.append(src[open_idx + 1:j]) pos = j + 1 break else: break return bodies bodies = _from_pretrained_call_bodies(text) star_vars = set() explicit_token_in_from_pretrained = False explicit_token_with_star = False for body in bodies: call_star_vars = set(re.findall(r"\*\*\s*([A-Za-z_]\w*)", body)) if call_star_vars: star_vars.update(call_star_vars) if re.search(r"(?:^|,)\s*(?:token|use_auth_token)\s*=", body): explicit_token_in_from_pretrained = True if call_star_vars: explicit_token_with_star = True result["from_pretrained_star_kwargs_vars"] = sorted(star_vars) result["from_pretrained_explicit_token_args"] = bool(explicit_token_in_from_pretrained) token_assignment_vars = set() assignment_patterns = [ r"([A-Za-z_]\w*)\s*\[\s*[\"'](?:token|use_auth_token)[\"']\s*\]\s*=", r"([A-Za-z_]\w*)\.update\s*\(\s*\{\s*[\"'](?:token|use_auth_token)[\"']\s*:", r"([A-Za-z_]\w*)\.update\s*\(\s*dict\s*\(\s*(?:token|use_auth_token)\s*=", ] for pattern in assignment_patterns: token_assignment_vars.update(re.findall(pattern, text)) result["kwargs_vars_with_token_assignment"] = sorted(token_assignment_vars) risky_star_vars = star_vars & token_assignment_vars result["detected_duplicate_token_risk"] = bool( has_diffusers_pipeline and ( result["evidence_matched"] or bool(risky_star_vars) or (explicit_token_with_star and bool(token_assignment_vars)) ) ) if not result["detected_duplicate_token_risk"] and not result["evidence_matched"]: write_json(run_dir / "diffusers_token_guard.json", result) return result original = text vars_to_clean = set(risky_star_vars) or set(token_assignment_vars) if result["evidence_matched"]: vars_to_clean |= set(token_assignment_vars) | set(star_vars) lines = text.splitlines(True) new_lines = [] i = 0 var_group = "|".join(re.escape(v) for v in sorted(vars_to_clean)) or r"[A-Za-z_]\w*" assignment_line_re = re.compile(rf"\b(?:{var_group})\s*\[\s*[\"'](?:token|use_auth_token)[\"']\s*\]\s*=") update_line_re = re.compile(rf"\b(?:{var_group})\.update\s*\(\s*(?:\{{\s*[\"'](?:token|use_auth_token)[\"']\s*:|dict\s*\(\s*(?:token|use_auth_token)\s*=)") one_line_if_re = re.compile(rf"^if\s+[^:]+:\s*(?:{var_group})\s*\[\s*[\"'](?:token|use_auth_token)[\"']\s*\]\s*=") while i < len(lines): line = lines[i] stripped = line.strip() if one_line_if_re.search(stripped): result["replacements"] += 1 result["removed_lines"].append(stripped[:200]) i += 1 continue if re.match(r"^\s*if\s+[^:]+:\s*$", line) and i + 1 < len(lines): next_line = lines[i + 1] if assignment_line_re.search(next_line) or update_line_re.search(next_line): result["replacements"] += 1 result["removed_lines"].append((line.strip() + " " + next_line.strip())[:200]) i += 2 continue if assignment_line_re.search(line) or update_line_re.search(line): result["replacements"] += 1 result["removed_lines"].append(stripped[:200]) i += 1 continue new_lines.append(line) i += 1 text = "".join(new_lines) # If a from_pretrained call combines **kwargs and an explicit token arg while # the kwargs dict had a token assignment, remove the explicit arg too. This # is safe for the risky generated pattern and avoids stale duplicate-token # failures after a partial line cleanup. if result["evidence_matched"] or explicit_token_with_star: token_arg_patterns = [ r",\s*(?:token|use_auth_token)\s*=\s*(?:HF_TOKEN|hf_token|token|hf_token_value|os\.environ\.get\([^\)]*\))\s*(?=,|\))", r"(?:token|use_auth_token)\s*=\s*(?:HF_TOKEN|hf_token|token|hf_token_value|os\.environ\.get\([^\)]*\))\s*,\s*", ] for pattern in token_arg_patterns: text, count = re.subn(pattern, "", text) result["replacements"] += count # Normalize accidental doubled commas introduced by argument removal. text, comma_count = re.subn(r",\s*,", ",", text) result["replacements"] += comma_count # Re-scan after patch so upload guards can fail closed if the deterministic # patch did not actually remove the duplicate-token risk. post_bodies = _from_pretrained_call_bodies(text) post_star_vars = set() post_explicit_token_with_star = False for body in post_bodies: call_star_vars = set(re.findall(r"\*\*\s*([A-Za-z_]\w*)", body)) post_star_vars.update(call_star_vars) if call_star_vars and re.search(r"(?:^|,)\s*(?:token|use_auth_token)\s*=", body): post_explicit_token_with_star = True post_assignment_vars = set() for pattern in assignment_patterns: post_assignment_vars.update(re.findall(pattern, text)) result["post_patch_duplicate_token_risk"] = bool((post_star_vars & post_assignment_vars) or (post_explicit_token_with_star and post_assignment_vars)) if text != original: app_path.write_text(text, encoding="utf-8") result["applied"] = True append_event(events_path, "diffusers_token_guard", "success", "Patched generated Diffusers token handling to avoid duplicate token injection", result) else: result["notes"].append("no_token_patch_needed") write_json(run_dir / "diffusers_token_guard.json", result) return result def normalize_runtime_expected_output_type(value: str) -> str: text = str(value or "").strip().lower() if not text: return "any" if "text_diagnostic" in text or "diagnostic" in text: return "text_diagnostic" if "image" in text or text in {"png", "jpg", "jpeg", "webp"}: return "image" if "video" in text or text in {"mp4", "webm", "mov"}: return "video" if "audio" in text or text in {"wav", "mp3", "flac"}: return "audio" if "text" in text: return "text" return text def archive_rejected_workspace(workspace: Path, run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None) -> dict: """Persist the publishable workspace that failed pre-upload integrity. v198.26.29 keeps blocked workspaces auditable and makes pre-upload failures repairable: the exact rejected payload is preserved before the worker fails closed or requests a targeted repair pass. """ rejected_dir = run_dir / "generated_rejected" if rejected_dir.exists(): shutil.rmtree(rejected_dir) rejected_dir.parent.mkdir(parents=True, exist_ok=True) payload = { "schema_version": "pre_upload_integrity_failure.v198_26_28", "reason": reason, "archived": False, "rejected_workspace": "generated_rejected", "guard_result": guard_result or {}, "created_at": now(), } try: if workspace.exists(): shutil.copytree(workspace, rejected_dir, ignore=internal_workspace_copy_ignore) payload["archived"] = True payload["files_sample"] = sorted(str(p.relative_to(rejected_dir)) for p in rejected_dir.rglob("*") if p.is_file())[:80] else: payload["error"] = "workspace_missing" except Exception as exc: payload["error"] = str(exc)[:1000] write_json(run_dir / "pre_upload_integrity_failure.json", payload) append_event(events_path, "pre_upload_integrity_failure", "failed", "Archived rejected publishable workspace before blocking upload", payload) return payload def write_pre_upload_repair_attempt(run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None, deterministic_repairs: list[dict] | None = None, pi_repair_required: bool = False) -> dict: """Record the repair trajectory for a pre-upload template failure. This artifact closes the visibility gap from v198.26.29: a pre-upload guard must not fail opaquely. It must say which deterministic repairs were tried and whether a targeted Pi repair would be required before a final failure. """ payload = { "schema_version": "pre_upload_repair_attempt.v198_26_28", "reason": reason, "guard_result": guard_result or {}, "deterministic_repairs": deterministic_repairs or [], "pi_repair_required": bool(pi_repair_required), "status": "pi_repair_required" if pi_repair_required else "deterministic_repairs_exhausted", "created_at": now(), } write_json(run_dir / "pre_upload_repair_attempt.json", payload) append_event(events_path, "pre_upload_repair_attempt", "warning", "Pre-upload template repair path recorded before blocking upload", payload) return payload def _ensure_spaces_import_early(app_text: str) -> tuple[str, bool]: if re.search(r"^\s*import\s+spaces\b", app_text, flags=re.M): return app_text, False lines = app_text.splitlines() insert_at = 0 # Keep shebang, encoding comments and __future__ imports first. Place # `import spaces` before torch/diffusers so ZeroGPU sees it before CUDA-ish # imports or module-level model setup. while insert_at < len(lines): stripped = lines[insert_at].strip() if not stripped or stripped.startswith("#!") or stripped.startswith("#") or stripped.startswith("from __future__ import"): insert_at += 1 continue break lines.insert(insert_at, "import spaces") return "\n".join(lines) + ("\n" if app_text.endswith("\n") else ""), True def _candidate_generate_function_names(app_text: str) -> list[str]: names: list[str] = [] def add(name: str): if name and name not in names and re.search(rf"^\s*(?:async\s+)?def\s+{re.escape(name)}\s*\(", app_text, flags=re.M): names.append(name) # Prefer Gradio event/API bindings that explicitly expose /generate. for match in re.finditer(r"fn\s*=\s*([A-Za-z_]\w*)[\s\S]{0,500}?api_name\s*=\s*[\"']/?generate[\"']", app_text): add(match.group(1)) for match in re.finditer(r"api_name\s*=\s*[\"']/?generate[\"'][\s\S]{0,500}?fn\s*=\s*([A-Za-z_]\w*)", app_text): add(match.group(1)) # Common Interface/ChatInterface/Blocks patterns. for match in re.finditer(r"(?:Interface|ChatInterface)\s*\([\s\S]{0,500}?fn\s*=\s*([A-Za-z_]\w*)", app_text): add(match.group(1)) for fallback in ("generate", "generate_image", "predict", "infer", "run_inference"): add(fallback) return names def _insert_decorator_before_function(app_text: str, func_name: str, decorator: str) -> tuple[str, bool]: lines = app_text.splitlines(True) def_re = re.compile(rf"^(?P\s*)(?:async\s+)?def\s+{re.escape(func_name)}\s*\(") for idx, line in enumerate(lines): match = def_re.match(line) if not match: continue # If the function already has a spaces.GPU decorator in its contiguous # decorator block, do nothing. start = idx while start > 0 and lines[start - 1].lstrip().startswith("@"): start -= 1 decorator_block = "".join(lines[start:idx]) if "spaces.GPU" in decorator_block or "spaces.gpu" in decorator_block.lower(): return app_text, False indent = match.group("indent") or "" lines.insert(start, f"{indent}{decorator}\n") return "".join(lines), True return app_text, False def _patch_common_python_syntax_defects(app_text: str) -> tuple[str, list[str]]: """Patch narrow syntax defects Pi has produced in real runs. This is intentionally conservative. v198.26.29 covers the observed `torch_dtype=DTYPE,,` class and repeated commas in function calls without attempting a broad Python formatter. """ patched = app_text changes: list[str] = [] if ",," in patched: while ",," in patched: patched = patched.replace(",,", ",") changes.append("collapsed_duplicate_commas") patched2 = re.sub(r",\s*(\))", r"\1", patched) if patched2 != patched: patched = patched2 changes.append("removed_trailing_comma_before_close_paren") # Pi has repeatedly emitted an unterminated guard such as: # try: # import spaces # def generate(...): # which fails with "expected 'except' or 'finally' block". Repair only this # narrow import-spaces guard by inserting a safe except before the next # top-level statement; do not attempt a broad Python formatter. lines = patched.splitlines(True) repaired_lines = None for idx, line in enumerate(lines): m = re.match(r"^(?P\s*)try:\s*(?:#.*)?$", line) if not m: continue base = m.group("indent") j = idx + 1 while j < len(lines) and not lines[j].strip(): j += 1 if j >= len(lines) or not re.match(rf"^{re.escape(base)}\s+import\s+spaces\b", lines[j]): continue has_except = False boundary = None k = j + 1 while k < len(lines): stripped = lines[k].strip() if not stripped: k += 1 continue if re.match(rf"^{re.escape(base)}except\b", lines[k]) or re.match(rf"^{re.escape(base)}finally\b", lines[k]): has_except = True break if re.match(rf"^{re.escape(base)}\S", lines[k]): boundary = k break k += 1 if has_except or boundary is None: continue indent = re.match(r"^(\s*)", lines[j]).group(1) insert = [f"{base}except Exception:\n", f"{indent}spaces = None\n"] repaired_lines = lines[:boundary] + insert + lines[boundary:] break if repaired_lines is not None: patched2 = "".join(repaired_lines) if patched2 != patched: patched = patched2 changes.append("closed_unterminated_import_spaces_try_block") return patched, changes def compile_workspace_app(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_compile_guard") -> dict: """Compile generated/app.py before any upload and patch tiny syntax defects. A Space should never be created/uploaded with syntactically invalid Python. The guard patches narrow, deterministic defects once, then records whether Pi repair is still required. """ app_path = workspace / "app.py" guard_path = run_dir / ("pre_upload_post_patch_compile_guard.json" if "post_patch" in str(reason) else "pre_upload_compile_guard.json") result = { "schema_version": "pre_upload_compile_guard.v198_26_28", "reason": reason, "checked_file": "app.py", "passed": False, "initial_passed": False, "patch_applied": False, "changes": [], "error": "", "post_patch_error": "", } if not app_path.exists(): result["error"] = "app_py_missing" write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "failed", "Generated app.py is missing before upload", result) return result app_text = app_path.read_text(encoding="utf-8", errors="ignore") try: compile(app_text, str(app_path), "exec") result["passed"] = True result["initial_passed"] = True write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "success", "Generated app.py compiled before upload", result) return result except SyntaxError as exc: result["error"] = f"{exc.__class__.__name__}: {exc}" patched, changes = _patch_common_python_syntax_defects(app_text) result["changes"] = changes if changes and patched != app_text: app_path.write_text(patched, encoding="utf-8") result["patch_applied"] = True try: compile(patched, str(app_path), "exec") result["passed"] = True write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "success", "Patched a narrow generated app.py syntax defect before upload", result) return result except SyntaxError as exc: result["post_patch_error"] = f"{exc.__class__.__name__}: {exc}" write_json(guard_path, result) append_event(events_path, "pre_upload_compile_guard", "failed", "Generated app.py did not compile before upload", result) return result def _has_generate_api_endpoint(app_text: str) -> bool: return bool(re.search(r"api_name\s*=\s*[\"']/?generate[\"']", app_text)) def _find_matching_paren(text: str, open_idx: int) -> int: depth = 0 quote = "" escape = False for idx in range(open_idx, len(text)): ch = text[idx] if quote: if escape: escape = False elif ch == "\\": escape = True elif ch == quote: quote = "" continue if ch in {"'", '"'}: quote = ch continue if ch == "(": depth += 1 elif ch == ")": depth -= 1 if depth == 0: return idx return -1 def _patch_api_name_into_call(app_text: str, fn_name: str) -> tuple[str, bool]: """Add api_name='generate' to the Gradio call that binds fn_name. Covers multiline `.click(...)`, `.submit(...)`, `.then(...)`, `.load(...)`, `gr.Interface(...)`, and `gr.ChatInterface(...)` calls where Pi forgot the API name even though the contract declares `/generate`. """ patterns = [ rf"fn\s*=\s*{re.escape(fn_name)}\b", rf"(?:gr\.)?(?:Interface|ChatInterface)\s*\(\s*{re.escape(fn_name)}\b", ] for pattern in patterns: for match in re.finditer(pattern, app_text): open_idx = app_text.rfind("(", 0, match.start()) if open_idx < 0: continue close_idx = _find_matching_paren(app_text, open_idx) if close_idx < 0: continue call_text = app_text[open_idx:close_idx] if "api_name" in call_text: # Replace api_name=None/False only if present. replaced = re.sub(r"api_name\s*=\s*(?:None|False|[\"']/?health[\"'])", "api_name=\"generate\"", call_text) if replaced != call_text: return app_text[:open_idx] + replaced + app_text[close_idx:], True continue insertion = ",\n api_name=\"generate\"" return app_text[:close_idx] + insertion + app_text[close_idx:], True return app_text, False def patch_workspace_for_missing_generate_endpoint(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_integrity_guard") -> dict: """Repair media contracts that forgot to expose Gradio `/generate`. If Pi produced a real `generate` function and bound it to a Gradio event but forgot `api_name='generate'`, this deterministic patch makes the Space validateable without waiting for a live runtime failure. """ result = { "schema_version": "generate_endpoint_repair.v198_26_27", "reason": reason, "applied": False, "checked_file": "app.py", "function_candidates": [], "patched_function": "", "notes": [], } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "generate_endpoint_repair.json", result) return result app_text = app_path.read_text(encoding="utf-8", errors="ignore") if _has_generate_api_endpoint(app_text): result["notes"].append("generate_endpoint_already_present") write_json(run_dir / "generate_endpoint_repair.json", result) return result candidates = _candidate_generate_function_names(app_text) result["function_candidates"] = candidates for name in candidates: patched, changed = _patch_api_name_into_call(app_text, name) if changed: app_path.write_text(patched, encoding="utf-8") result["applied"] = True result["patched_function"] = name append_event(events_path, "generate_endpoint_repair", "success", "Patched Gradio binding to expose /generate before upload", result) write_json(run_dir / "generate_endpoint_repair.json", result) return result result["notes"].append("no_patchable_gradio_binding_found") write_json(run_dir / "generate_endpoint_repair.json", result) return result def patch_workspace_for_missing_spaces_gpu_decorator(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_integrity_guard") -> dict: """Repair a generated ZeroGPU app that has /generate but missed @spaces.GPU. This is intentionally narrow and auditable: it adds `import spaces` early and decorates the function bound to Gradio `api_name="generate"`. If the target function cannot be identified, the guard remains fatal but archives the rejected workspace instead of publishing a non-actionable ZeroGPU app. """ result = { "schema_version": "spaces_gpu_decorator_repair.v198_26_27", "reason": reason, "applied": False, "checked_file": "app.py", "function_candidates": [], "decorated_function": "", "import_added": False, "notes": [], } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result app_text = app_path.read_text(encoding="utf-8", errors="ignore") if "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower(): result["notes"].append("spaces_gpu_decorator_already_present") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result candidates = _candidate_generate_function_names(app_text) result["function_candidates"] = candidates if not candidates: result["notes"].append("no_generate_function_candidate_found") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result patched, import_added = _ensure_spaces_import_early(app_text) for name in candidates: patched2, changed = _insert_decorator_before_function(patched, name, "@spaces.GPU(duration=120)") if changed: patched = patched2 result["decorated_function"] = name break if not result["decorated_function"]: result["notes"].append("candidate_function_not_decorated") write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result app_path.write_text(patched, encoding="utf-8") result["applied"] = True result["import_added"] = import_added append_event(events_path, "spaces_gpu_decorator_repair", "success", "Patched ZeroGPU publishable app with missing @spaces.GPU decorator before upload", result) write_json(run_dir / "spaces_gpu_decorator_repair.json", result) return result def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path: Path, *, hardware_intent: dict | None = None, expected_output_type: str = "", reason: str = "pre_upload", raise_on_failure: bool = True) -> dict: """Pre-upload integrity guard for publishable runtime templates. v198.26.29 extends the v198.26.24 repair-first policy with compile checks and endpoint repair. The worker should not fail-fast merely because Pi forgot `api_name='generate'` or emitted a narrow syntax typo; it should repair, re-check, then fail closed only when the app is still invalid. """ contract = read_inference_contract(workspace) app_path = workspace / "app.py" app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" hw_text = " ".join(str((hardware_intent or {}).get(k) or "") for k in ("preferred_hardware", "fallback_hardware", "recommended_hardware")) contract_hw = str(contract.get("recommended_target_space_hardware") or contract.get("recommended_hardware") or "") zero_target = "zero" in (hw_text + " " + contract_hw).lower() expected = normalize_runtime_expected_output_type(expected_output_type or contract.get("expected_output_type") or "") primary = normalize_api_name(str(contract.get("primary_api_name") or contract.get("primary_endpoint") or "/generate")) full_inference = contract.get("full_inference_implemented") is True diagnostic_output = normalize_runtime_expected_output_type(str(contract.get("expected_output_type") or "")) == "text_diagnostic" deterministic_repairs: list[dict] = [] compile_guard = compile_workspace_app(workspace, run_dir, events_path, reason=f"{reason}_compile_guard") deterministic_repairs.append({"type": "compile_guard", "result": compile_guard}) app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_generate = _has_generate_api_endpoint(app_text) has_spaces_gpu = "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower() generate_endpoint_patch = {"applied": False, "notes": ["not_run_until_app_py_compiles"]} spaces_gpu_patch = {"applied": False, "notes": ["not_run_until_app_py_compiles"]} cache_patch = {"applied": False, "notes": ["not_run_until_app_py_compiles"]} token_patch = {"applied": False, "post_patch_duplicate_token_risk": False, "detected_duplicate_token_risk": False, "notes": ["not_run_until_app_py_compiles"]} post_patch_compile_guard = compile_guard if compile_guard.get("passed"): cache_patch = patch_workspace_for_writable_hf_cache(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") token_patch = patch_workspace_for_diffusers_token_guard(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") deterministic_repairs.extend([ {"type": "writable_cache_patch", "result": cache_patch}, {"type": "diffusers_token_patch", "result": token_patch}, ]) # Re-read after deterministic cache/token patches. app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_generate = _has_generate_api_endpoint(app_text) has_spaces_gpu = "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower() generate_endpoint_patch = {"applied": False, "notes": ["not_needed"]} spaces_gpu_patch = {"applied": False, "notes": ["not_needed"]} if full_inference and expected in {"image", "video", "audio"} and primary == "/generate" and not has_generate: generate_endpoint_patch = patch_workspace_for_missing_generate_endpoint(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") deterministic_repairs.append({"type": "generate_endpoint_patch", "result": generate_endpoint_patch}) app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_generate = _has_generate_api_endpoint(app_text) # Syntax-first: never decorate or semantically patch an uncompilable app.py. # Once app.py compiles, a missing @spaces.GPU decorator on an otherwise # full inference ZeroGPU app is repairable before upload. if full_inference and zero_target and has_generate and not has_spaces_gpu: spaces_gpu_patch = patch_workspace_for_missing_spaces_gpu_decorator(workspace, run_dir, events_path, reason=f"{reason}_integrity_guard") deterministic_repairs.append({"type": "spaces_gpu_decorator_patch", "result": spaces_gpu_patch}) app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" has_spaces_gpu = "@spaces.GPU" in app_text or "@spaces.gpu" in app_text.lower() has_generate = _has_generate_api_endpoint(app_text) # Compile once more after endpoint/decorator patches. post_patch_compile_guard = compile_workspace_app(workspace, run_dir, events_path, reason=f"{reason}_post_patch_compile_guard") deterministic_repairs.append({"type": "post_patch_compile_guard", "result": post_patch_compile_guard}) else: deterministic_repairs.extend([ {"type": "writable_cache_patch", "result": cache_patch}, {"type": "diffusers_token_patch", "result": token_patch}, {"type": "generate_endpoint_patch", "result": generate_endpoint_patch}, {"type": "spaces_gpu_decorator_patch", "result": spaces_gpu_patch}, {"type": "post_patch_compile_guard", "result": post_patch_compile_guard}, ]) fatal = [] repairable_unresolved = [] if not compile_guard.get("passed") and not post_patch_compile_guard.get("passed"): repairable_unresolved.append("publishable_app_py_syntax_error") if token_patch.get("post_patch_duplicate_token_risk"): fatal.append("diffusers_duplicate_token_risk_after_guard") if token_patch.get("detected_duplicate_token_risk") and not token_patch.get("applied"): fatal.append("diffusers_duplicate_token_risk_unpatched") if full_inference and zero_target and not has_spaces_gpu: repairable_unresolved.append("zero_gpu_full_inference_without_spaces_gpu_decorator") if full_inference and expected in {"image", "video", "audio"} and not has_generate: repairable_unresolved.append("full_inference_media_contract_missing_generate_endpoint") if expected in {"image", "video", "audio"} and diagnostic_output: fatal.append("media_expected_output_replaced_by_text_diagnostic") if full_inference and not app_path.exists(): fatal.append("publishable_app_py_missing") fatal.extend(repairable_unresolved) result = { "schema_version": "runtime_template_integrity_guard.v198_26_28", "reason": reason, "passed": not fatal, "fatal_problems": fatal, "repairable_unresolved_problems": repairable_unresolved, "problems": list(fatal), "zero_target": zero_target, "expected_output_type": expected, "primary_api_name": primary, "full_inference_implemented": full_inference, "has_generate_endpoint": has_generate, "has_spaces_gpu": has_spaces_gpu, "compile_guard": compile_guard, "post_patch_compile_guard": post_patch_compile_guard, "writable_cache_patch": cache_patch, "diffusers_token_patch": token_patch, "generate_endpoint_patch": generate_endpoint_patch, "spaces_gpu_decorator_patch": spaces_gpu_patch, } write_json(run_dir / "runtime_template_integrity_guard.json", result) append_event(events_path, "runtime_template_integrity_guard", "success" if result["passed"] else "failed", "Checked publishable runtime template integrity before upload", result) if fatal: write_pre_upload_repair_attempt(run_dir, events_path, reason="runtime_template_integrity_failed_before_upload", guard_result=result, deterministic_repairs=deterministic_repairs, pi_repair_required=bool(repairable_unresolved)) archive_rejected_workspace(workspace, run_dir, events_path, reason="runtime_template_integrity_failed_before_upload", guard_result=result) if raise_on_failure: raise RuntimeError("Runtime template integrity failed before upload after deterministic repair: " + ", ".join(fatal)) return result def _update_pre_upload_repair_attempt_status(run_dir: Path, **updates) -> dict: path = run_dir / "pre_upload_repair_attempt.json" payload = load_json_if_exists(path) if path.exists() else {} if not isinstance(payload, dict): payload = {} payload.update(updates) payload["updated_at"] = now() write_json(path, payload) return payload def _pre_upload_repair_failure_reason(guard_result: dict | None, workspace: Path) -> str: guard = guard_result if isinstance(guard_result, dict) else {} app_excerpt = _read_workspace_text_if_exists(workspace / "app.py")[:6000] return ( "Pre-upload runtime template integrity failed before Space upload.\n" f"Fatal problems: {', '.join(guard.get('fatal_problems') or guard.get('problems') or [])}.\n" f"Repairable unresolved problems: {', '.join(guard.get('repairable_unresolved_problems') or [])}.\n" f"Expected output type: {guard.get('expected_output_type') or ''}. Primary API: {guard.get('primary_api_name') or ''}. ZeroGPU target: {guard.get('zero_target')}.\n" "Patch the publishable app.py minimally so it compiles, exposes the declared /generate endpoint, preserves the media output contract, and keeps @spaces.GPU when ZeroGPU is targeted.\n\n" "Current app.py excerpt:\n" + app_excerpt ) def ensure_runtime_template_uploadable_with_pi_repair( workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, implementation_mode: str, expected_output_type: str, hardware_intent: dict | None = None, ) -> dict: """Run pre-upload guard, then execute one Pi repair pass if unresolved defects remain. Guards must protect ASF from publishing broken apps, but they should not be terminal guillotines for repairable template defects. v198.26.29 guarantees that a `pi_repair_required` pre-upload state is followed by either a repaired workspace that passes the guard, or an explicit terminal repair outcome. """ guard = runtime_template_integrity_guard( workspace, run_dir, events_path, hardware_intent=hardware_intent, expected_output_type=expected_output_type, reason="pre_upload_repair_first", raise_on_failure=False, ) if guard.get("passed"): return guard unresolved = list(guard.get("repairable_unresolved_problems") or []) fatal = list(guard.get("fatal_problems") or guard.get("problems") or []) if not unresolved: _update_pre_upload_repair_attempt_status(run_dir, status="fatal_guard_failure_no_pi_repair", pi_repair_required=False, terminal=True) raise RuntimeError("Runtime template integrity failed before upload: " + ", ".join(fatal)) primary_problem = unresolved[0] decision = { "action": "patch_code", "confidence": "high", "source": "pre_upload_integrity_guard.v198_26_28", "classification": { "category": primary_problem, "failure_class": primary_problem, "failure_phase": "pre_upload_runtime_template_integrity", "logs_quality": "workspace_static_analysis", "recommendation": "Patch app.py so the generated Space is publishable: compile cleanly, expose the declared /generate endpoint, preserve the media output type, and include @spaces.GPU for ZeroGPU.", }, "constraints": [ "Patch only publishable runtime files needed to pass pre-upload integrity.", "Preserve the real model inference path and expected output type.", "Do not replace the app with diagnostics or placeholders.", "Keep /health and /generate API contracts coherent with INFERENCE_CONTRACT.json.", ], } write_json(run_dir / "pre_upload_pi_repair_decision.json", decision) append_event(events_path, "pre_upload_pi_repair", "started", "Running Pi repair for unresolved pre-upload template integrity defects", {"problem": primary_problem, "fatal_problems": fatal}) failure_reason = _pre_upload_repair_failure_reason(guard, workspace) repair_attempt, _ = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi( workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger="pre_upload_integrity_failure", ) if not repaired: _update_pre_upload_repair_attempt_status(run_dir, status="pi_repair_failed_or_no_patch", pi_repair_required=True, terminal=True, repair_attempt=repair_attempt) archive_rejected_workspace(workspace, run_dir, events_path, reason="pre_upload_pi_repair_failed", guard_result=guard) append_event(events_path, "pre_upload_pi_repair", "failed", "Pi repair did not produce a publishable pre-upload patch", {"problem": primary_problem, "attempt": repair_attempt}) raise RuntimeError("Runtime template integrity failed before upload; Pi repair did not produce a publishable patch: " + ", ".join(fatal)) repaired_dir = run_dir / "generated_repaired" if repaired_dir.exists(): shutil.rmtree(repaired_dir) shutil.copytree(workspace, repaired_dir, ignore=internal_workspace_copy_ignore) guard_after = runtime_template_integrity_guard( workspace, run_dir, events_path, hardware_intent=hardware_intent, expected_output_type=expected_output_type, reason="pre_upload_after_pi_repair", raise_on_failure=False, ) if guard_after.get("passed"): _update_pre_upload_repair_attempt_status(run_dir, status="pi_repair_succeeded", pi_repair_required=False, terminal=True, repair_attempt=repair_attempt, post_repair_guard=guard_after) append_event(events_path, "pre_upload_pi_repair", "success", "Pi repaired the runtime template before upload", {"attempt": repair_attempt, "guard": guard_after}) return guard_after _update_pre_upload_repair_attempt_status(run_dir, status="pi_repair_did_not_clear_guard", pi_repair_required=True, terminal=True, repair_attempt=repair_attempt, post_repair_guard=guard_after) archive_rejected_workspace(workspace, run_dir, events_path, reason="pre_upload_pi_repair_did_not_clear_guard", guard_result=guard_after) append_event(events_path, "pre_upload_pi_repair", "failed", "Pi repair completed but pre-upload integrity still failed", {"attempt": repair_attempt, "guard": guard_after}) raise RuntimeError("Runtime template integrity failed before upload after Pi repair: " + ", ".join(guard_after.get("fatal_problems") or guard_after.get("problems") or fatal)) def patch_workspace_for_writable_hf_cache(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload_guard", evidence: str = "") -> dict: """Patch generated apps that hard-code Hugging Face caches under `/data`. ZeroGPU/fixed-GPU Spaces do not always have a writable persistent `/data` volume. A generated app that sets HF_HOME/HF_HUB_CACHE/TRANSFORMERS_CACHE or DIFFUSERS_CACHE to `/data...` can boot the UI but fail model loading with `[Errno 13] Permission denied: '/data'`. This deterministic factory patch is intentionally narrow: it only rewrites HF cache environment paths to `/tmp/asf-hf-cache` and adds a small cache bootstrap. It does not change the model call, Gradio API, hardware target, or output contract. """ result = { "schema_version": "writable_hf_cache_guard.v198_26_20", "reason": reason, "applied": False, "checked_file": "app.py", "replacements": 0, "detected_non_writable_data_cache": False, "cache_root": "/tmp/asf-hf-cache", "notes": [], } app_path = workspace / "app.py" if not app_path.exists(): result["notes"].append("app_py_missing") write_json(run_dir / "writable_hf_cache_guard.json", result) return result text = app_path.read_text(encoding="utf-8", errors="ignore") combined = f"{text}\n{evidence or ''}" if "/data" not in combined and not is_hf_cache_permission_denied_text(combined): return result original = text result["detected_non_writable_data_cache"] = True env_paths = { "HF_HOME": "/tmp/asf-hf-cache", "HF_HUB_CACHE": "/tmp/asf-hf-cache/hub", "HUGGINGFACE_HUB_CACHE": "/tmp/asf-hf-cache/hub", "TRANSFORMERS_CACHE": "/tmp/asf-hf-cache/transformers", "DIFFUSERS_CACHE": "/tmp/asf-hf-cache/diffusers", } def _replace_with_count(pattern: str, repl_func): nonlocal text text, count = re.subn(pattern, repl_func, text, flags=re.IGNORECASE) result["replacements"] += count for env_name, safe_path in env_paths.items(): _replace_with_count( rf"(os\.environ\.setdefault\(\s*[\"']{env_name}[\"']\s*,\s*)[\"']/data[^\"']*[\"'](\s*\))", lambda m, safe_path=safe_path: f'{m.group(1)}"{safe_path}"{m.group(2)}', ) _replace_with_count( rf"(os\.environ\[\s*[\"']{env_name}[\"']\s*\]\s*=\s*)[\"']/data[^\"']*[\"']", lambda m, safe_path=safe_path: f'{m.group(1)}"{safe_path}"', ) _replace_with_count( rf"({env_name}\s*=\s*)[\"']/data[^\"']*[\"']", lambda m, safe_path=safe_path: f'{m.group(1)}"{safe_path}"', ) literal_replacements = [ ("/data/.cache/huggingface/hub", "/tmp/asf-hf-cache/hub"), ("/data/.cache/huggingface/transformers", "/tmp/asf-hf-cache/transformers"), ("/data/.cache/huggingface/diffusers", "/tmp/asf-hf-cache/diffusers"), ("/data/.cache/huggingface", "/tmp/asf-hf-cache"), ("/data/huggingface/hub", "/tmp/asf-hf-cache/hub"), ("/data/huggingface", "/tmp/asf-hf-cache"), ] for src, dst in literal_replacements: if src in text: count = text.count(src) text = text.replace(src, dst) result["replacements"] += count snippet = """ # ASF v198.26.20 writable Hugging Face cache guard. _ASF_HF_CACHE_ROOT = os.environ.get("ASF_HF_CACHE_DIR") or "/tmp/asf-hf-cache" os.makedirs(_ASF_HF_CACHE_ROOT, exist_ok=True) os.makedirs(os.path.join(_ASF_HF_CACHE_ROOT, "hub"), exist_ok=True) os.makedirs(os.path.join(_ASF_HF_CACHE_ROOT, "transformers"), exist_ok=True) os.makedirs(os.path.join(_ASF_HF_CACHE_ROOT, "diffusers"), exist_ok=True) os.environ.setdefault("HF_HOME", _ASF_HF_CACHE_ROOT) os.environ.setdefault("HF_HUB_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "hub")) os.environ.setdefault("HUGGINGFACE_HUB_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "hub")) os.environ.setdefault("TRANSFORMERS_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "transformers")) os.environ.setdefault("DIFFUSERS_CACHE", os.path.join(_ASF_HF_CACHE_ROOT, "diffusers")) """ if "writable Hugging Face cache guard" not in text: if not re.search(r"(?m)^\s*import\s+os\b", text): text = "import os\n" + text result["replacements"] += 1 lines = text.splitlines(True) insert_at = 0 for i, line in enumerate(lines): if re.match(r"\s*(import\s+\w+|from\s+\w+\s+import\s+)", line): insert_at = i + 1 lines.insert(insert_at, snippet) text = "".join(lines) result["replacements"] += 1 if text != original: app_path.write_text(text, encoding="utf-8") result["applied"] = True result["notes"].append("patched_hf_cache_paths_to_tmp") append_event(events_path, "writable_hf_cache_guard", "success", "Patched generated app to use writable /tmp Hugging Face caches instead of /data", result) else: result["notes"].append("no_publishable_cache_patch_needed") write_json(run_dir / "writable_hf_cache_guard.json", result) return result def space_subdomain_url(target_space_id: str) -> str: owner, name = target_space_id.split("/", 1) return f"https://{owner}-{name}.hf.space".replace("_", "-").lower() def validate_http_health(target_space_id: str, token: str, run_dir: Path, attempt: int): import requests url = space_subdomain_url(target_space_id).rstrip("/") + "/health" headers = {"Authorization": f"Bearer {token}", "Accept": "application/json,text/plain,*/*"} response = requests.get(url, headers=headers, timeout=20) payload = { "status": "success" if response.ok else "failed", "attempt": attempt, "url": url, "status_code": response.status_code, "content_type": response.headers.get("content-type"), "text": response.text[:2000], } if response.ok: try: payload["json"] = response.json() except Exception: pass semantic = validation_health_semantics(payload | {"validator": "http_get_health"}) payload["health_endpoint_reachable"] = True payload["health_semantic_passed"] = semantic.get("semantic_passed") payload["health_semantics"] = semantic write_json(run_dir / "tests" / "http_health.json", payload) return payload raise RuntimeError(f"HTTP /health returned {response.status_code}: {response.text[:500]}") def wait_until_live(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, timeout_s: int = 1800): append_event(events_path, "live_wait", "started", "Waiting for existing Space to become live") deadline = time.time() + timeout_s attempt = 0 last_error = None while time.time() < deadline: attempt += 1 runtime_payload = write_space_runtime(api, target_space_id, token, run_dir, events_path, attempt) stage = str(runtime_payload.get("stage") or "").upper() if "RUNTIME_ERROR" in stage: collect_space_logs(target_space_id, token, run_dir, events_path) last_error = f"Space is in RUNTIME_ERROR: {runtime_payload}" append_event(events_path, "live_wait", "waiting", "Space is in runtime error; still waiting in case hardware was changed manually", {"attempt": attempt, "runtime": runtime_payload}) time.sleep(30) continue try: health = validate_http_health(target_space_id, token, run_dir, attempt) health_semantic_passed = ((health or {}).get("health_semantics") or {}).get("semantic_passed") append_event(events_path, "live_wait", "success" if health_semantic_passed is not False else "warning", "HTTP /health is live" if health_semantic_passed is not False else "HTTP /health is reachable but model is unavailable/not ready", {"attempt": attempt, "health_semantic_passed": health_semantic_passed}) return {"validator": "http_health", "health": health, "runtime": runtime_payload} except Exception as http_exc: last_error = f"HTTP health failed: {http_exc}" try: client = make_gradio_client(target_space_id, token) schema = client.view_api(return_format="dict") names = api_names_from_schema(schema) write_json(run_dir / "tests" / "api_schema.json", {"schema": schema, "api_names": names}) if names: append_event(events_path, "live_wait", "success", "Gradio API schema is live", {"attempt": attempt, "api_names": names}) return {"validator": "gradio_schema", "api_names": names, "runtime": runtime_payload} except Exception as gr_exc: last_error = (last_error or "") + f"; Gradio schema failed: {gr_exc}" append_event(events_path, "live_wait", "waiting", "Space not live yet", {"attempt": attempt, "runtime": runtime_payload, "error": last_error[-1500:] if last_error else None}) time.sleep(30) collect_space_logs(target_space_id, token, run_dir, events_path) raise RuntimeError(f"Space did not become live before timeout: {last_error}") def parse_json_env(name: str, default): value = os.environ.get(name) if not value: return default try: return json.loads(value) except Exception as exc: raise ValueError(f"Invalid JSON for {name}: {exc}") def result_contains_expected_output(result, expected_output_type: str) -> tuple[bool, dict]: expected = (expected_output_type or "any").lower().strip() info = {"expected_output_type": expected, "result_type": type(result).__name__, "result_repr": repr(result)[:2000]} paths = [] type_hints = [] text_values = [] def visit(obj): type_hints.append(type(obj).__name__.lower()) if isinstance(obj, (str, Path)): value = str(obj) if value: paths.append(value) text_values.append(value) elif isinstance(obj, dict): for key, value in obj.items(): key_l = str(key).lower() if key_l in {"path", "name", "url", "file", "filepath"}: visit(value) elif key_l in {"mime_type", "mime", "type", "format"} and value: type_hints.append(str(value).lower()) elif key_l in {"image", "video", "audio", "text"}: type_hints.append(key_l) visit(value) elif isinstance(value, (dict, list, tuple)): visit(value) elif isinstance(obj, (list, tuple)): for item in obj: visit(item) else: module = getattr(type(obj), "__module__", "") if module: type_hints.append(module.lower()) visit(result) lower_paths = [str(p).lower() for p in paths] hints = " ".join(type_hints + lower_paths) info["detected_paths"] = paths[:20] info["type_hints"] = type_hints[:30] if expected == "any": return result is not None, info image_ext = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff") video_ext = (".mp4", ".mov", ".webm", ".avi", ".mkv") audio_ext = (".wav", ".mp3", ".flac", ".ogg", ".m4a") if expected == "text": if isinstance(result, str) and bool(result.strip()): return True, info return any(isinstance(x, str) and x.strip() and Path(x).suffix.lower() not in image_ext + video_ext + audio_ext for x in text_values), info if expected == "image": return any(p.endswith(image_ext) for p in lower_paths) or any(h in hints for h in ["image", "pil.", "png", "jpeg", "jpg", "webp", "gradio.data_classes.filedata"]), info if expected == "video": return any(p.endswith(video_ext) for p in lower_paths) or any(h in hints for h in ["video", "mp4", "webm", "moviepy"]), info if expected == "audio": return any(p.endswith(audio_ext) for p in lower_paths) or any(h in hints for h in ["audio", "wav", "soundfile"]), info return result is not None, info def copy_result_artifacts(result, run_dir: Path): artifacts = run_dir / "artifacts" artifacts.mkdir(parents=True, exist_ok=True) copied = [] def maybe_copy(obj): if isinstance(obj, (str, Path)): path = Path(str(obj)) if path.exists() and path.is_file(): target = artifacts / path.name try: shutil.copy2(path, target) copied.append(str(target)) except Exception: pass elif isinstance(obj, dict): for key in ["path", "name"]: if key in obj: maybe_copy(obj[key]) for value in obj.values(): if isinstance(value, (dict, list, tuple)): maybe_copy(value) elif isinstance(obj, (list, tuple)): for item in obj: maybe_copy(item) maybe_copy(result) return copied def run_gradio_validation_harness(*, client, schema, api_name: str, test_args: list, test_kwargs: dict, expected_output_type: str, run_dir: Path, events_path: Path, validation_mode: str, payload_source: str, replay_source_used: bool = False): """Common Gradio validation gate used by linked Space Test modes. v190.24 contract: automatic build smoke and manual linked validation must share the same endpoint → payload resolution → schema coercion → predict → output verification sequence. This helper is intentionally small in the validation worker; build smoke mirrors the same artefacts and coercion contract in run_generation_smoke(). """ resolved = { "api_name": api_name, "test_args": list(test_args or []), "test_kwargs": dict(test_kwargs or {}), "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "replay_source_used": bool(replay_source_used), "resolved_request_required_before_predict": True, } write_json(run_dir / "tests" / "resolved_validation_request.json", resolved) started = time.time() result = client.predict(*resolved["test_args"], api_name=api_name, **resolved["test_kwargs"]) latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) return result, latency, ok, info, resolved def smoke_generate(target_space_id: str, token: str, run_dir: Path, events_path: Path): """Run the linked Space Test validation smoke test with hard diagnostics. v190.32 rule: a validation run may fail, but it must not fail opaquely. The worker writes validation_engine / preflight diagnostics before any fragile Gradio operation, and it writes a terminal diagnosis if it exits before payload resolution or predict(). """ tests_dir = run_dir / "tests" tests_dir.mkdir(parents=True, exist_ok=True) # Compatibility anchors for historical release tests: # write_json(run_dir / "tests" / "validation_launch_payload.json", validation_launch_payload) # write_json(run_dir / "tests" / "payload_source.json", payload_source_record) # write_json(run_dir / "tests" / "replay_source.json", replay_source) # write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight) app_version = "v198.26.32" engine_version = "unified_gradio_validation_harness_v198_25_3" parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip() validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked" requested_api_name = (os.environ.get("API_NAME") or "").strip() expected_output_type = (os.environ.get("EXPECTED_OUTPUT_TYPE") or "any").strip() validation_launch_payload = parse_json_env("VALIDATION_LAUNCH_PAYLOAD_JSON", {}) if not isinstance(validation_launch_payload, dict): validation_launch_payload = {"parse_error": "VALIDATION_LAUNCH_PAYLOAD_JSON was not a JSON object"} env_replay_source_present = bool(os.environ.get("PARENT_REPLAY_SOURCE_JSON")) replay_mode = str(validation_mode or "").strip().lower() == "replay" initial_payload_source = "parent_automatic_smoke" if (replay_mode and env_replay_source_present) else ( os.environ.get("PAYLOAD_SOURCE") or validation_launch_payload.get("payload_source") or "provided" ) worker_version = { "app_version": app_version, "validation_engine_version": engine_version, "worker_kind": "linked_space_validation", "parent_build_run_id": parent_build_run_id, "has_env_replay_source": env_replay_source_present, "has_validation_launch_payload": bool(os.environ.get("VALIDATION_LAUNCH_PAYLOAD_JSON")), "written_at": now(), } write_json(tests_dir / "worker_version.json", worker_version) write_json(tests_dir / "validation_launch_payload.json", validation_launch_payload) write_json(tests_dir / "validation_engine.json", { "schema_version": "1.1", "validation_engine": "unified_gradio_validation_harness", "validation_engine_version": engine_version, "validation_mode": validation_mode, "payload_source": initial_payload_source, "parent_smoke_payload_used": False, "backend_launch_payload_present": bool(validation_launch_payload), "backend_launch_payload_source": validation_launch_payload.get("payload_source") if isinstance(validation_launch_payload, dict) else "", "resolved_request_required_before_predict": True, "stage": "initialized", "initialized_at": now(), }) write_json(tests_dir / "validation_preflight.json", { "schema_version": "1.1", "stage": "initialized", "parent_build_run_id": parent_build_run_id, "target_space_id": target_space_id, "validation_mode": validation_mode, "payload_source": initial_payload_source, "requested_api_name": requested_api_name, "expected_output_type": expected_output_type, "has_env_replay_source": env_replay_source_present, "has_validation_launch_payload": bool(validation_launch_payload), "launchable": None, "written_at": now(), }) stage = "initialized" schema = {} discovered = [] api_name = requested_api_name or "" test_args = [] safe_kwargs = {} schema_choice_corrections = [] replay_source = {} autofill_meta = {} dropped_kwargs = [] endpoint_parameters = [] original_test_args = [] file_input_conversions = [] def write_failure(exc, *, failure_type: str | None = None, stage_override: str | None = None): nonlocal stage, api_name, discovered, test_args, safe_kwargs, schema_choice_corrections, replay_source, dropped_kwargs, endpoint_parameters, expected_output_type message = str(exc) fail_stage = stage_override or stage or "unknown" if not failure_type: if fail_stage in {"initialized", "client_init", "endpoint_discovery"}: failure_type = "validation_engine_startup_failed" elif fail_stage in {"payload_resolution", "payload"}: failure_type = "payload_resolution_failed" else: failure_type = "validation_worker_failed" diagnosis = { "schema_version": "1.1", "failure_type": failure_type, "stage": fail_stage, "message": message[:4000], "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "validation_mode": validation_mode, "payload_source": initial_payload_source, "api_name": api_name or requested_api_name, "selected_endpoint": api_name, "discovered_api_names": discovered, "resolved_request_written": (tests_dir / "resolved_validation_request.json").exists(), "validation_engine_written": (tests_dir / "validation_engine.json").exists(), "schema_choice_corrections": schema_choice_corrections, "file_input_conversions": file_input_conversions, "replay_source_used": bool(replay_source), "parent_smoke_payload_used": bool(replay_source), "recommended_action": "Inspect worker logs and validation_launch_payload; the worker stopped before payload resolution." if fail_stage != "predict" else "Inspect Space runtime and resolved_validation_request.", "written_at": now(), } failure_payload = { "status": "failed", "failure_type": failure_type, "stage": fail_stage, "target_space": target_space_id, "api_name": api_name or requested_api_name, "discovered_api_names": discovered, "test_args": test_args, "test_kwargs": safe_kwargs, "effective_args": test_args, "effective_kwargs": safe_kwargs, "schema_choice_corrections": schema_choice_corrections, "replay_source_used": bool(replay_source), "parent_smoke_payload_used": bool(replay_source), "ignored_test_kwargs": dropped_kwargs, "endpoint_parameters": endpoint_parameters, "expected_output_type": expected_output_type, "error": message[:4000], "validated_at": now(), } write_json(tests_dir / "validation_failure_diagnosis.json", diagnosis) if not (tests_dir / "generation_smoke.json").exists(): write_json(tests_dir / "generation_smoke.json", failure_payload) if not (tests_dir / "test_result.json").exists(): write_json(tests_dir / "test_result.json", failure_payload) write_live_status(run_dir, stage=fail_stage, status="failed", message=diagnosis["message"], data=diagnosis) append_event(events_path, fail_stage, "failed", f"Linked validation failed at {fail_stage}", diagnosis) return failure_payload try: stage = "parse_payload" test_args = parse_json_env("TEST_ARGS_JSON", ["a cinematic robot cat astronaut, detailed, studio lighting"]) test_kwargs = parse_json_env("TEST_KWARGS_JSON", {}) if not isinstance(test_args, list): raise ValueError("TEST_ARGS_JSON must be a JSON list") if not isinstance(test_kwargs, dict): raise ValueError("TEST_KWARGS_JSON must be a JSON object") stage = "client_init" client = make_gradio_client(target_space_id, token) write_live_status(run_dir, stage="endpoint_discovery", status="running", message="Discovering Gradio endpoints before validation", data={"requested_api_name": requested_api_name or ""}) append_event(events_path, "endpoint_discovery", "started", "Discovering Gradio endpoints before validation", {"requested_api_name": requested_api_name or ""}) stage = "endpoint_discovery" schema = client.view_api(return_format="dict") discovered = api_names_from_schema(schema) parent_dir = run_dir.parent / parent_build_run_id if parent_build_run_id else None replay_source = (load_env_replay_source(target_space_id, expected_output_type) or load_parent_replay_source(parent_dir, target_space_id, expected_output_type)) if replay_mode else {} if replay_source.get("source_error"): write_json(tests_dir / "replay_source_error.json", replay_source) replay_source = {} preferred_api_name = replay_source.get("api_name") or requested_api_name api_name, endpoint_candidates = select_validation_api_name(preferred_api_name, schema) write_json(tests_dir / "gradio_endpoint_discovery.json", {"requested_api_name": requested_api_name or "", "preferred_api_name": preferred_api_name or "", "discovered_api_names": discovered, "selected_api_name": api_name, "endpoint_discovery_required": os.environ.get("ENDPOINT_DISCOVERY_REQUIRED") == "true", "replay_source_used": bool(replay_source)}) write_json(tests_dir / "gradio_schema.json", {"schema": schema, "api_names": discovered}) write_json(tests_dir / "endpoint_candidates.json", {"candidates": endpoint_candidates}) write_json(tests_dir / "selected_endpoint.json", {"api_name": api_name}) append_event(events_path, "endpoint_discovery", "success", "Selected Gradio endpoint for linked validation", {"api_name": api_name, "discovered_api_names": discovered}) write_live_status(run_dir, stage="endpoint_discovery", status="success", message=f"Selected Gradio endpoint {api_name}", data={"api_name": api_name, "discovered_api_names": discovered}) stage = "payload_resolution" if replay_source: test_args = list(replay_source.get("test_args") or []) test_kwargs = dict(replay_source.get("test_kwargs") or {}) expected_output_type = str(replay_source.get("expected_output_type") or expected_output_type or "any") safe_kwargs, dropped_kwargs, endpoint_parameters = sanitize_kwargs_for_schema(api_name, schema, test_kwargs) autofill_meta = {"args_were_autofilled": False, "args_source": "parent_automatic_smoke", "endpoint_parameters": endpoint_parameter_names(endpoint_schema_for_api(schema, api_name))} if autofill_meta.get("endpoint_parameters"): endpoint_parameters = autofill_meta.get("endpoint_parameters") original_test_args = list(test_args or []) test_args, schema_choice_corrections = coerce_validation_args_to_schema_choices(api_name, schema, test_args) write_json(tests_dir / "replay_source.json", {**replay_source, "selected_api_name": api_name, "using_parent_smoke_payload_as_source_of_truth": True, "schema_choice_corrections": schema_choice_corrections}) payload_source = "parent_automatic_smoke" append_event(events_path, "payload", "success", "Replayed validation payload from parent automatic smoke", {"api_name": api_name, "parent_build_run_id": parent_build_run_id, "payload_source": payload_source, "choice_corrections": schema_choice_corrections}) write_live_status(run_dir, stage="payload", status="success", message="Payload replayed from parent automatic smoke", data={"api_name": api_name, "parent_build_run_id": parent_build_run_id, "choice_corrections": schema_choice_corrections}) else: safe_kwargs, dropped_kwargs, endpoint_parameters = sanitize_kwargs_for_schema(api_name, schema, test_kwargs) test_args, autofill_meta = build_args_from_gradio_schema(api_name, schema, test_args) if autofill_meta.get("endpoint_parameters"): endpoint_parameters = autofill_meta.get("endpoint_parameters") original_test_args = list(test_args or []) test_args, schema_choice_corrections = coerce_validation_args_to_schema_choices(api_name, schema, test_args) payload_source = os.environ.get("PAYLOAD_SOURCE") or autofill_meta.get("args_source") or validation_launch_payload.get("payload_source") or "provided" test_args, file_input_conversions = prepare_validation_file_inputs(api_name, schema, test_args, run_dir) payload_source_record = { # Compatibility anchor: write_json(run_dir / "tests" / "payload_source.json", payload_source_record) # Compatibility anchor: write_json(run_dir / "tests" / "validation_engine.json", payload_source_record) # Compatibility anchor: write_json(run_dir / "tests" / "resolved_validation_request.json", resolved_request) "schema_version": "1.1", "validation_engine": "unified_gradio_validation_harness", "validation_engine_version": engine_version, "validation_mode": validation_mode, "payload_source": payload_source, "parent_smoke_payload_used": bool(replay_source), "backend_launch_payload_present": bool(validation_launch_payload), "backend_launch_payload_source": validation_launch_payload.get("payload_source") if isinstance(validation_launch_payload, dict) else "", "endpoint_discovery_used": True, "selected_api_name": api_name, } write_json(tests_dir / "payload_source.json", payload_source_record) write_json(tests_dir / "validation_engine.json", {**payload_source_record, "resolved_request_required_before_predict": True, "stage": "payload_resolved", "updated_at": now()}) validation_preflight = { "schema_version": "1.1", "stage": "payload_resolved", "parent_build_run_id": parent_build_run_id, "target_space_id": target_space_id, "validation_mode": validation_mode, "payload_source": payload_source, "requested_api_name": requested_api_name, "selected_api_name": api_name, "expected_output_type": expected_output_type, "endpoint_discovery_required": os.environ.get("ENDPOINT_DISCOVERY_REQUIRED") == "true", "discovered_api_names": discovered, "launchable": True, } validation_payload = {"api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "effective_args": test_args, "effective_kwargs": safe_kwargs, "original_test_args": original_test_args, "schema_choice_corrections": schema_choice_corrections, "file_input_conversions": file_input_conversions, "endpoint_discovery": {"requested_api_name": requested_api_name, "selected_api_name": api_name, "discovered_api_names": discovered}, **autofill_meta} resolved_request = { "api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "schema_choice_corrections": schema_choice_corrections, "replay_source_used": bool(replay_source), "parent_smoke_payload_used": bool(replay_source), } write_json(tests_dir / "validation_preflight.json", validation_preflight) write_json(tests_dir / "validation_payload.json", validation_payload) write_json(tests_dir / "resolved_validation_request.json", resolved_request) write_json(tests_dir / "schema_coercion.json", {"api_name": api_name, "changes": schema_choice_corrections, "original_args": original_test_args, "resolved_args": test_args}) write_json(tests_dir / "api_schema.json", {"schema": schema, "api_names": discovered, "selected_api_name": api_name, "endpoint_parameters": endpoint_parameters}) if schema_choice_corrections: append_event(events_path, "payload", "success", "Normalized validation payload to the Gradio endpoint schema", {"api_name": api_name, "choice_corrections": schema_choice_corrections}) write_live_status(run_dir, stage="payload", status="success", message="Payload normalized to Gradio schema", data={"api_name": api_name, "choice_corrections": schema_choice_corrections}) elif not replay_source: append_event(events_path, "payload", "success", "Resolved validation payload", {"api_name": api_name, "payload_source": payload_source}) write_live_status(run_dir, stage="payload", status="success", message="Payload resolved", data={"api_name": api_name, "payload_source": payload_source}) if autofill_meta.get("args_were_autofilled"): append_event(events_path, "payload", "success", "Filled validation arguments from discovered Gradio schema", {"api_name": api_name, "args_source": autofill_meta.get("args_source"), "endpoint_parameters": endpoint_parameters, "effective_args": test_args}) if dropped_kwargs: append_event(events_path, "generation_smoke", "warning", "Ignored unsupported validation kwargs for this Gradio endpoint", {"api_name": api_name, "ignored_kwargs": sorted(dropped_kwargs), "endpoint_parameters": endpoint_parameters}) append_event(events_path, "generation_smoke", "started", "Calling live generation endpoint", {"api_name": api_name, "expected_output_type": expected_output_type, "replay_source_used": bool(replay_source)}) stage = "predict" started = time.time() try: result = client.predict(*test_args, api_name=api_name, **safe_kwargs) except Exception as exc: message = str(exc) if is_schema_choice_type_error(exc): corrected_args, retry_choice_corrections = coerce_validation_args_to_schema_choices(api_name, schema, test_args) if not retry_choice_corrections or corrected_args == test_args: parsed_args, parsed_changes = coerce_smoke_args_from_choice_error(test_args, exc, endpoint_parameter_objects(endpoint_schema_for_api(schema, api_name))) if parsed_changes and parsed_args != test_args: corrected_args, retry_choice_corrections = parsed_args, parsed_changes if retry_choice_corrections and corrected_args != test_args: append_event(events_path, "generation_smoke", "warning", "Retrying validation with Gradio schema choice types", {"api_name": api_name, "error": message[:1500], "choice_corrections": retry_choice_corrections}) test_args = corrected_args write_json(tests_dir / "generation_smoke_payload_retry.json", {"api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "retry_reason": "schema_choice_type_mismatch", "choice_corrections": retry_choice_corrections}) write_json(tests_dir / "resolved_validation_request.json", {"api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "schema_choice_corrections": schema_choice_corrections + retry_choice_corrections, "retry_reason": "schema_choice_type_mismatch"}) write_json(tests_dir / "schema_coercion.json", {"api_name": api_name, "changes": schema_choice_corrections + retry_choice_corrections, "original_args": original_test_args, "resolved_args": test_args, "retried": True}) result = client.predict(*test_args, api_name=api_name, **safe_kwargs) else: raise elif safe_kwargs and "not a valid key-word argument" in message: append_event(events_path, "generation_smoke", "warning", "Retrying validation without keyword arguments after Gradio rejected kwargs", {"api_name": api_name, "error": message[:1500], "dropped_kwargs": sorted(safe_kwargs)}) safe_kwargs = {} result = client.predict(*test_args, api_name=api_name) else: failure_type = "predict_failed" if is_schema_choice_type_error(message): failure_type = "schema_choice_type_error_after_coercion" elif "not a valid key-word argument" in message: failure_type = "kwargs_rejected" write_failure(exc, failure_type=failure_type, stage_override="predict") raise latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) copied = copy_result_artifacts(result, run_dir) video_inspection = {"schema_version": "video_artifact_inspection.v198_24", "videos": [], "video_count": 0, "has_video_artifact": False, "quality_assessment": "not_evaluated"} if str(expected_output_type or "").lower() == "video": videos = [] for item in copied or []: path = Path(str(item)) if path.suffix.lower() in {".mp4", ".mov", ".webm", ".avi", ".mkv"}: record = {"path": str(path), "extension": path.suffix.lower(), "exists": path.exists()} if path.exists(): record["size_bytes"] = path.stat().st_size record["container_valid"] = bool(path.stat().st_size > 0) videos.append(record) video_inspection.update({"videos": videos, "video_count": len(videos), "has_video_artifact": bool(videos)}) write_json(tests_dir / "video_artifact_inspection.json", video_inspection) payload = { "status": "success" if ok else "failed", "target_space": target_space_id, "api_name": api_name, "discovered_api_names": discovered, "test_args": test_args, "test_kwargs": safe_kwargs, "effective_args": test_args, "effective_kwargs": safe_kwargs, "args_source": autofill_meta.get("args_source"), "args_were_autofilled": bool(autofill_meta.get("args_were_autofilled")), "original_test_kwargs": test_kwargs, "ignored_test_kwargs": dropped_kwargs, "endpoint_parameters": endpoint_parameters, "expected_output_type": expected_output_type, "latency_seconds": round(latency, 3), "observed_latency_seconds": round(latency, 3), "result_info": info, "copied_artifacts": copied, "video_artifact_inspection": video_inspection, "recommended_zero_gpu_duration_seconds": int(max(30, min(300, latency * 2 + 15))), "recommendation_source": "live_gradio_predict", "recommendation_confidence": "measured", "measurement_note": "Measured from a live gradio_client.predict call, including Gradio/API/network/result serialization overhead.", "validated_at": now(), } write_json(tests_dir / "generation_smoke.json", payload) write_json(tests_dir / "test_result.json", payload) if ok: append_event(events_path, "generation_smoke", "success", "Live generation smoke test passed", {"latency_seconds": payload["latency_seconds"], "recommended_zero_gpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "recommended_zerogpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "copied_artifacts": copied[:5]}) return payload append_event(events_path, "generation_smoke", "failed", "Live generation returned an unexpected output type", payload) write_failure(RuntimeError("Generation smoke test failed: unexpected output type"), failure_type="unexpected_output_type", stage_override="output_verification") raise RuntimeError("Generation smoke test failed: unexpected output type") except Exception as exc: # If a deeper block already wrote the diagnostic, keep it and simply re-raise. if not (tests_dir / "validation_failure_diagnosis.json").exists(): write_failure(exc) raise def main(): run_id = os.environ["RUN_ID"] username = os.environ.get("HF_USERNAME", "unknown") output_root = Path(os.environ.get("OUTPUT_ROOT", "/output")) target_space_id = os.environ["TARGET_SPACE_ID"].strip() parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip() token = os.environ.get("HF_TOKEN") run_dir = output_root / "runs" / run_id events_path = run_dir / "events.jsonl" state_path = run_dir / "state.json" append_event(events_path, "bootstrap", "started", "Existing Space validation worker started", {"target_space_id": target_space_id}) write_json(state_path, {"run_id": run_id, "kind": "linked_space_validation", "status": "running", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "created_by": username, "updated_at": now()}) if not token: raise RuntimeError("HF_TOKEN is missing") ensure_hf_token_context(run_dir, events_path) write_auth_probe(run_dir, events_path, "before_linked_space_test", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_linked_space_test"), raise_on_unsafe=True) if not TARGET_RE.match(target_space_id): raise ValueError("TARGET_SPACE_ID must look like owner/space-name") if not parent_build_run_id: raise ValueError("PARENT_BUILD_RUN_ID is required: Space Test validations must be linked to a Build Run.") try: install_deps(events_path) from huggingface_hub import HfApi api = HfApi(token=token) whoami = api.whoami(token=token) append_event(events_path, "auth", "success", "Authenticated inside validation Job", {"whoami_name": whoami.get("name")}) live = wait_until_live(api, target_space_id, token, run_dir, events_path, timeout_s=int(os.environ.get("LIVE_TIMEOUT_SECONDS", "1800"))) smoke = smoke_generate(target_space_id, token, run_dir, events_path) final_state = { "run_id": run_id, "kind": "linked_space_validation", "status": "full_inference_success", "message": "Existing Space passed live health/schema validation and generation smoke test.", "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "live_validation": live, "generation_smoke": smoke, "updated_at": now(), } final_state["parent_build_run_id"] = parent_build_run_id final_state["source_kind"] = "build_run_prefill" write_json(state_path, final_state) write_final_summary(run_dir, final_state, {}, smoke, status="full_inference_success", message=final_state.get("message")) if parent_build_run_id: parent_dir = output_root / "runs" / parent_build_run_id parent_dir.mkdir(parents=True, exist_ok=True) runtime = live.get("runtime") if isinstance(live, dict) else {} validation_hardware = (runtime or {}).get("hardware") or (runtime or {}).get("requested_hardware") or "unknown" api_name_used = smoke.get("api_name") or os.environ.get("API_NAME") or "/generate" expected_output_type_used = smoke.get("expected_output_type") or os.environ.get("EXPECTED_OUTPUT_TYPE") or "any" effective_status_on_success = os.environ.get("EFFECTIVE_STATUS_ON_SUCCESS") or "validated_after_space_test" space_test_policy_mode = os.environ.get("SPACE_TEST_POLICY_MODE") or "complete" manual_status = { "schema_version": "post_build_validation.v198_26_7", "status": "success", "effective_status": effective_status_on_success, "legacy_effective_status": "validated_after_manual_space_test", "space_test_policy_mode": space_test_policy_mode, "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "api_name": api_name_used, "expected_output_type": expected_output_type_used, "latency_seconds": smoke.get("latency_seconds"), "observed_latency_seconds": smoke.get("observed_latency_seconds") or smoke.get("latency_seconds"), "recommended_zero_gpu_duration_seconds": smoke.get("recommended_zero_gpu_duration_seconds"), "recommendation_source": "linked_space_test", "recommendation_hardware": validation_hardware, "hardware_used_for_validation": validation_hardware, "validated_at": now(), "applies_to_target_space": True, } write_json(parent_dir / "post_build_validation_status.json", manual_status) write_json(parent_dir / "manual_validation_status.json", manual_status) linked_path = parent_dir / "linked_validations.json" try: linked = json.loads(linked_path.read_text(encoding="utf-8")) except Exception: linked = {"parent_build_run_id": parent_build_run_id, "validations": []} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] validations.append(manual_status) linked = {"parent_build_run_id": parent_build_run_id, "effective_status": effective_status_on_success, "validations": validations, "updated_at": now()} write_json(linked_path, linked) report = f"""# Agentic Space Factory — Existing Space Validation Report Status: **full_inference_success** Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_id}) ## Generation smoke test ```json {json.dumps(smoke, indent=2, ensure_ascii=False)} ``` ## Notes - This validation is intended for Spaces whose hardware was set manually after generation. - Latency is measured from the live Gradio endpoint call. - The recommended ZeroGPU duration is a rough estimate from this live run, not a guarantee. """ (run_dir / "report.md").write_text(report, encoding="utf-8") append_event(events_path, "report_write", "success", "Wrote report.md") try: publish_eval_record(run_dir, phase="final", events_path=events_path) append_event(events_path, "anonymous_eval", "success", "Published anonymized validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False}) except Exception as eval_exc: append_event(events_path, "anonymous_eval", "warning", "Could not publish anonymized validation evaluation record", {"error": str(eval_exc)[:1000]}) # Rewrite the terminal summary at the very end of the success path. # v194: launch metadata can leave summary.json as running if the # final lightweight summary is not the last writer visible in the # bucket sync. Keep manual audits and list views terminal-consistent. write_final_summary(run_dir, final_state, {}, smoke, status="full_inference_success", message=final_state.get("message")) append_event(events_path, "summary_write", "success", "Wrote terminal validation summary", {"status": "full_inference_success"}) append_event(events_path, "done", "full_inference_success", "Existing Space validation completed", {"latency_seconds": smoke.get("latency_seconds")}) except Exception as exc: collect_space_logs(target_space_id, token or "", run_dir, events_path) auth_blocked = isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc) details = {"error": str(exc)[:4000]} if auth_blocked: details["status_reason"] = "auth_refresh_required" details["failure_type"] = "auth_refresh_required" terminal_status = "auth_refresh_required" if auth_blocked else "failed" failure_state = {"run_id": run_id, "kind": "linked_space_validation", "status": terminal_status, "message": "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else str(exc), "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "details": details, "updated_at": now()} write_json(state_path, failure_state) write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed") if parent_build_run_id: parent_dir = output_root / "runs" / parent_build_run_id parent_dir.mkdir(parents=True, exist_ok=True) failed_status = {"schema_version": "post_build_validation.v198_26_7", "status": terminal_status, "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "details": details, "updated_at": now(), "effective_status": "unchanged"} linked_path = parent_dir / "linked_validations.json" linked = read_json(linked_path, {"parent_build_run_id": parent_build_run_id, "validations": []}) or {"parent_build_run_id": parent_build_run_id, "validations": []} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] validations.append(failed_status) linked = {"parent_build_run_id": parent_build_run_id, "validations": validations, "updated_at": now()} if parent_has_successful_manual_validation(parent_dir): linked["effective_status"] = "validated_after_space_test" linked["legacy_effective_status"] = "validated_after_manual_space_test" append_event(events_path, "linked_validation", "warning", "Linked validation failed but parent already has a successful manual validation; parent effective status is preserved", {"parent_build_run_id": parent_build_run_id}) else: write_json(parent_dir / "manual_validation_status.json", failed_status) write_json(linked_path, linked) (run_dir / "report.md").write_text(f"# Existing Space Validation Failed\n\n```json\n{json.dumps(details, indent=2, ensure_ascii=False)}\n```\n", encoding="utf-8") append_event(events_path, "failure", "failed", "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else "Existing Space validation failed", details) try: publish_eval_record(run_dir, phase="failure", events_path=events_path) append_event(events_path, "anonymous_eval", "success", "Published anonymized failed validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False}) except Exception as eval_exc: append_event(events_path, "anonymous_eval", "warning", "Could not publish anonymized failed validation evaluation record", {"error": str(eval_exc)[:1000]}) # v194: also rewrite failure summary as the final visible file update # for linked validations, so a failed validation cannot remain listed # as running after the worker has already terminalized state.json. write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed") append_event(events_path, "summary_write", "success", "Wrote terminal failed validation summary", {"status": terminal_status}) raise SystemExit(1) if __name__ == "__main__": main() ''' def universal_model_card_worker_script() -> str: """Return the universal model-card builder worker script source.""" return UNIVERSAL_MODEL_CARD_WORKER_SCRIPT def validate_existing_space_worker_script() -> str: """Return the existing-Space validation worker script source.""" return VALIDATE_EXISTING_SPACE_WORKER_SCRIPT def encoded_universal_model_card_worker_script() -> str: """Return the base64-encoded universal model-card builder worker script. Kept for backwards compatibility in tests/imports. New Jobs read the worker from WORKER_SCRIPT_PATH to avoid oversized env/argv payloads. """ return _encode(UNIVERSAL_MODEL_CARD_WORKER_SCRIPT) def encoded_validate_existing_space_worker_script() -> str: """Return the base64-encoded existing-Space validation worker script. Kept for backwards compatibility in tests/imports. New Jobs read the worker from WORKER_SCRIPT_PATH to avoid oversized env/argv payloads. """ return _encode(VALIDATE_EXISTING_SPACE_WORKER_SCRIPT) def python_decode_and_run_command() -> list[str]: """Small command list for `run_job`. The worker script is stored in the run bucket and exposed through the Job volume as WORKER_SCRIPT_PATH. This avoids putting the large worker source in argv/env, which can make HF Jobs fail before Python starts with `argument list too long`. WORKER_SCRIPT_B64 remains as a tiny compatibility fallback for older tests/manual launch paths. """ runner = textwrap.dedent( """ import base64, os, pathlib, subprocess, sys script_path = os.environ.get('WORKER_SCRIPT_PATH') if script_path: path = pathlib.Path(script_path) if not path.exists(): raise SystemExit(f'Worker script not found at WORKER_SCRIPT_PATH={script_path}') else: script_b64 = os.environ.get('WORKER_SCRIPT_B64') if not script_b64: raise SystemExit('Missing WORKER_SCRIPT_PATH') path = pathlib.Path('/tmp/space_factory_worker.py') path.write_text(base64.b64decode(script_b64).decode('utf-8'), encoding='utf-8') raise SystemExit(subprocess.call([sys.executable, str(path)])) """ ).strip() return ["python", "-c", runner]