from __future__ import annotations import base64 import textwrap def _encode(script: str) -> str: return base64.b64encode(script.encode("utf-8")).decode("ascii") UNIVERSAL_MODEL_CARD_WORKER_SCRIPT = r''' import ast import base64 import hashlib import hmac import hashlib import hmac import json import os import re import shutil import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path from textwrap import dedent TARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$") GIST_URL = "https://gist.github.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d" GIST_RAW_URL = "https://gist.githubusercontent.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d/raw" AUTO_SPACE_HARDWARE_CHOICES = {"zero-a10g", "cpu-basic", "t4-small", "t4-medium", "a10g-small", "a10g-large", "l4x1", "l40sx1"} DEFAULT_PREFERRED_SPACE_HARDWARE = "zero-a10g" DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large" DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2" MAX_PI_REPAIR_ATTEMPTS = 3 APP_VERSION = "v198.26.9" app_version = "v198.26.9" # Internal agent/recovery files may be needed inside the transient Pi # workspace, but they should not be published to the generated Space or shown # twice as generated application files. Canonical copies live in run_dir/logs, # run_dir/repair, and run_dir/traces. INTERNAL_WORKSPACE_ARTIFACT_NAMES = { "GOAL.md", "INCIDENT_BRIEF.md", "PI_DIAGNOSIS_GOAL.md", "DEPENDENCY_ERROR_BRIEF.md", "REPAIR_DECISION.json", "REPAIR_TASK_PACKET.json", "MEMORY_DIAGNOSIS_GOAL.md", "MEMORY_DIAGNOSIS.json", "REPAIR_GOAL.md", "REPAIR_BRIEF.md", "REPAIR_PLAN.md", "REPAIR_SUMMARY.md", "PI_SUMMARY.md", "PI_SOURCE_USAGE.json", "MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "PI_TASK_PACKET.json", "PATCH_REFUSAL.json", "TECHNICAL_BLOCKERS.json", "pi_feasibility_brief.json", "pi_implementation_plan.json", } INTERNAL_WORKSPACE_ARTIFACT_DIRS = {"analysis_inputs", "model_inspect", "refs", "logs", "traces", "repair", "artifacts", "snapshots", "checkpoints", "weights", "hf_cache", "model_cache"} def internal_workspace_upload_ignore_patterns() -> list[str]: patterns = [".git/*", ".cache/*", "**/.cache/*", "node_modules/*", "__pycache__/*", "*.pyc", ".pi/*", "**/.pi/*", "analysis_inputs/*", "**/analysis_inputs/*", "model_inspect/*", "**/model_inspect/*", "refs/*", "**/refs/*", "logs/*", "**/logs/*", "traces/*", "**/traces/*", "repair/*", "**/repair/*", "auth.json", "**/auth.json"] for name in sorted(INTERNAL_WORKSPACE_ARTIFACT_NAMES): patterns.extend([name, f"**/{name}"]) return patterns def internal_workspace_copy_ignore(_dir: str, names: list[str]) -> set[str]: ignored = {".git", ".cache", "node_modules", "__pycache__", ".pi", "analysis_inputs", "model_inspect", "refs", "logs", "traces", "repair", "artifacts", "snapshots", "checkpoints", "weights", "hf_cache", "model_cache"} ignored.update(name for name in names if name in INTERNAL_WORKSPACE_ARTIFACT_NAMES or name.endswith(".pyc")) return ignored def is_publishable_workspace_file(path: Path) -> bool: if path.name in INTERNAL_WORKSPACE_ARTIFACT_NAMES or path.name == "auth.json": return False if any(part in INTERNAL_WORKSPACE_ARTIFACT_DIRS for part in path.parts): return False if any(part in {".git", ".cache", "node_modules", "__pycache__"} for part in path.parts): return False if path.suffix == ".pyc": return False return True RUNTIME_PAYLOAD_BLOCKED_SUFFIXES = { ".safetensors", ".bin", ".pt", ".pth", ".ckpt", ".onnx", ".gguf", ".h5", ".pb", ".tflite", ".npz", ".npy", ".tar", ".zip", ".7z", ".rar", ".parquet", ".arrow", ".sqlite", ".db", } RUNTIME_PAYLOAD_ALLOWED_SUFFIXES = { "", ".py", ".txt", ".md", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".css", ".js", ".html", ".csv", ".tsv", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".mp3", ".wav", ".flac", ".ogg", ".mp4", ".webm", ".m4a", } RUNTIME_UPLOAD_DEFAULT_MAX_TOTAL_BYTES = 250 * 1024 * 1024 RUNTIME_UPLOAD_DEFAULT_MAX_FILE_BYTES = 50 * 1024 * 1024 def _runtime_upload_limit_from_env(name: str, default: int) -> int: raw = os.environ.get(name, "").strip() if not raw: return default try: value = int(raw) return value if value > 0 else default except Exception: return default def runtime_payload_file_reason(path: Path, *, size_bytes: int = 0, max_file_bytes: int | None = None) -> tuple[bool, str]: """Return whether a workspace file is safe to publish to the target Space repo. The Pi workspace may contain large audit/model-inspection artifacts that are useful in the run bucket but must never be committed to the generated Space. Model weights should be loaded at Space runtime through from_pretrained(), not uploaded through Git/LFS with the demo code. """ rel = Path(path) suffix = rel.suffix.lower() # Report model/weight artifacts as such even when they live under an audit dir. if suffix in RUNTIME_PAYLOAD_BLOCKED_SUFFIXES: return False, "blocked_heavy_or_model_suffix" if max_file_bytes is not None and size_bytes > max_file_bytes: return False, "file_too_large_for_runtime_payload" if not is_publishable_workspace_file(rel): return False, "internal_or_audit_artifact" if any(part in INTERNAL_WORKSPACE_ARTIFACT_DIRS for part in rel.parts): return False, "excluded_runtime_artifact_dir" if any(part.startswith(".") and part not in {".streamlit"} for part in rel.parts): return False, "hidden_or_cache_path" if suffix not in RUNTIME_PAYLOAD_ALLOWED_SUFFIXES: return False, "suffix_not_runtime_whitelisted" return True, "included" def build_runtime_upload_payload(workspace: Path, run_dir: Path, events_path: Path) -> tuple[Path, dict]: """Create a minimal, auditable runtime payload for Space upload. This deliberately separates the rich Pi/audit workspace from the deployable Space repo. The generated app can still download heavy model weights at runtime through Hugging Face Hub APIs, but the Space repo itself remains a small code package. """ required = ["app.py", "README.md", "requirements.txt"] for filename in required: if not (workspace / filename).exists(): raise RuntimeError(f"Missing required generated file: {filename}") payload_dir = run_dir / "runtime_upload_payload" if payload_dir.exists(): shutil.rmtree(payload_dir) payload_dir.mkdir(parents=True, exist_ok=True) max_total_bytes = _runtime_upload_limit_from_env("ASF_RUNTIME_UPLOAD_MAX_BYTES", RUNTIME_UPLOAD_DEFAULT_MAX_TOTAL_BYTES) max_file_bytes = _runtime_upload_limit_from_env("ASF_RUNTIME_UPLOAD_MAX_FILE_BYTES", RUNTIME_UPLOAD_DEFAULT_MAX_FILE_BYTES) manifest = { "schema_version": "runtime_upload_payload.v198_25_1", "source_workspace": str(workspace), "payload_dir": str(payload_dir), "max_total_bytes": max_total_bytes, "max_file_bytes": max_file_bytes, "files": [], "excluded_files": [], "total_bytes": 0, "file_count": 0, "blocked_heavy_file_count": 0, "payload_guard": "whitelist_runtime_only", } for src in sorted(workspace.rglob("*")): if not src.is_file(): continue try: rel = src.relative_to(workspace) except Exception: continue try: size = src.stat().st_size except Exception: size = 0 include, reason = runtime_payload_file_reason(rel, size_bytes=size, max_file_bytes=max_file_bytes) entry = {"path": str(rel), "size_bytes": size, "reason": reason} if not include: if reason in {"blocked_heavy_or_model_suffix", "file_too_large_for_runtime_payload"}: manifest["blocked_heavy_file_count"] += 1 manifest["excluded_files"].append(entry) continue dest = payload_dir / rel dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dest) manifest["files"].append(entry) manifest["total_bytes"] += size manifest["file_count"] = len(manifest["files"]) manifest["excluded_count"] = len(manifest["excluded_files"]) manifest_path = run_dir / "runtime_upload_payload_manifest.json" write_json(manifest_path, manifest) if manifest["total_bytes"] > max_total_bytes: payload = { "schema_version": "runtime_upload_payload_guard.v198_25_1", "failure_owner": "factory_packaging", "failure_class": "runtime_payload_too_large", "repair_candidate": False, "total_bytes": manifest["total_bytes"], "max_total_bytes": max_total_bytes, "file_count": manifest["file_count"], "manifest": str(manifest_path), "message": "Runtime upload payload exceeds the configured safety limit before HF upload.", } write_json(run_dir / "factory_upload_error.json", payload) append_event(events_path, "upload_files", "failed", "Runtime upload payload is too large; aborting before HF upload", payload) raise RuntimeError(f"runtime_payload_too_large: {manifest['total_bytes']} bytes > {max_total_bytes} bytes") append_event( events_path, "runtime_payload", "success", "Prepared whitelist runtime payload for generated Space upload", { "manifest": "runtime_upload_payload_manifest.json", "file_count": manifest["file_count"], "total_bytes": manifest["total_bytes"], "excluded_count": manifest["excluded_count"], "blocked_heavy_file_count": manifest["blocked_heavy_file_count"], }, ) return payload_dir, manifest def classify_space_upload_exception(exc: Exception, run_dir: Path, events_path: Path) -> dict: raw = str(exc) lower = raw.lower() storage_limit = "repository storage limit" in lower or "storage limit reached" in lower or "max: 1 gb" in lower or "/info/lfs/objects/batch" in lower payload = { "schema_version": "factory_upload_error.v198_25_1", "failure_owner": "factory_packaging" if storage_limit else "hf_platform_or_upload", "failure_class": "factory_upload_error" if storage_limit else "space_upload_error", "storage_limit_reached": bool(storage_limit), "repair_candidate": False, "recommended_action": "Upload only the whitelist runtime payload; keep model/audit artifacts out of the target Space repo." if storage_limit else "Inspect HF upload error and retry only if transient.", "error": redact_text(raw)[:4000] if "redact_text" in globals() else raw[:4000], } write_json(run_dir / "factory_upload_error.json", payload) append_event(events_path, "upload_files", "failed", "Generated Space upload failed before live validation", payload) return payload def now(): return datetime.now(timezone.utc).isoformat() def write_json(path: Path, payload: dict): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2, ensure_ascii=False, default=str) + "\n", encoding="utf-8") def read_json(path: Path, default=None): try: if path.exists(): return json.loads(path.read_text(encoding="utf-8")) except Exception: pass return default def _rel_to_workspace(workspace: Path, path: Path | str) -> str: try: pp = path if isinstance(path, Path) else workspace / str(path) return str(pp.relative_to(workspace)) except Exception: return str(path) def _safe_file_chars(path: Path, limit: int | None = None) -> int: try: text = path.read_text(encoding="utf-8", errors="ignore") if limit is not None: text = text[:limit] return len(text) except Exception: return 0 def estimate_pi_context_budget( run_dir: Path, workspace: Path, *, phase: str, task_type: str, direct_prompt: str, referenced_files: list[str] | None = None, max_context_chars: int = 60000, omitted_sections: list[str] | None = None, events_path: Path | None = None, artifact_dir: Path | None = None, ) -> dict: """Record what ASF made available to Pi for this call. Pi may still decide which files to open internally. This artifact therefore measures the *Factory-provided context surface*, not the provider's exact final prompt. It is intentionally approximate and stable enough to compare runs over time. """ referenced_files = referenced_files or [] omitted_sections = omitted_sections or [] file_entries = [] referenced_chars = 0 for rel in referenced_files: path = workspace / rel chars = _safe_file_chars(path) referenced_chars += chars file_entries.append({"path": rel, "chars": chars, "exists": path.exists()}) payload = { "schema_version": "pi_prompt_budget.v198_25", "phase": phase, "task_type": task_type, "direct_prompt_chars": len(direct_prompt or ""), "referenced_context_chars": referenced_chars, "estimated_total_context_chars": len(direct_prompt or "") + referenced_chars, "max_context_chars": max_context_chars, "context_budget_status": "ok" if len(direct_prompt or "") + referenced_chars <= max_context_chars else "large_context_observed", "blocking": False, "referenced_files": file_entries, "omitted_sections": omitted_sections, "note": "Measures Factory-provided prompt plus referenced files; Pi/provider internal context may differ.", } budgets_path = run_dir / "pi_prompt_budget.json" aggregate = read_json(budgets_path, {"schema_version": "pi_prompt_budget_aggregate.v198_25", "calls": []}) if not isinstance(aggregate, dict): aggregate = {"schema_version": "pi_prompt_budget_aggregate.v198_25", "calls": []} calls = aggregate.get("calls") if isinstance(aggregate.get("calls"), list) else [] calls.append(payload) aggregate["calls"] = calls aggregate["latest"] = payload write_json(budgets_path, aggregate) if artifact_dir: artifact_dir.mkdir(parents=True, exist_ok=True) write_json(artifact_dir / f"PI_PROMPT_BUDGET_{phase}.json", payload) if events_path: append_event(events_path, "pi_prompt_budget", "success", f"Recorded Pi prompt budget for {phase}", {"phase": phase, "task_type": task_type, "estimated_total_context_chars": payload["estimated_total_context_chars"], "status": payload["context_budget_status"]}) return payload def _sha256_text(text: str) -> str: try: return hashlib.sha256((text or "").encode("utf-8")).hexdigest() except Exception: return "" def _sha256_file(path: Path | None) -> str: try: if path and path.exists(): return hashlib.sha256(path.read_bytes()).hexdigest() except Exception: pass return "" def _pi_settings_model(default_model: str = "") -> tuple[str, dict]: settings_path = Path.home() / ".pi" / "agent" / "settings.json" try: settings = json.loads(settings_path.read_text(encoding="utf-8")) return settings.get("model") or default_model, settings except Exception: return default_model, {"model": default_model, "read_error": "could_not_read_settings"} def resolve_pi_call_identity(requested_model: str, output_text: str = "") -> dict: configured_model, settings = _pi_settings_model(requested_model) observed = extract_pi_models_from_text(output_text or "") if "extract_pi_models_from_text" in globals() else [] normalized_requested = normalize_pi_model_name(requested_model) if "normalize_pi_model_name" in globals() else (requested_model or "").lower() normalized_configured = normalize_pi_model_name(configured_model) if "normalize_pi_model_name" in globals() else (configured_model or "").lower() effective_model = "" for raw in observed: normalized = normalize_pi_model_name(raw) if "normalize_pi_model_name" in globals() else str(raw).lower() if normalized and normalized not in {normalized_requested, normalized_configured}: effective_model = raw break if not effective_model and observed: effective_model = observed[0] if not effective_model: effective_model = configured_model or requested_model mismatch = bool(effective_model and (normalize_pi_model_name(effective_model) if "normalize_pi_model_name" in globals() else effective_model.lower()) not in {normalized_requested, normalized_configured}) return { "requested_model": requested_model, "configured_model": configured_model, "effective_model": effective_model, "observed_models": observed, "provider": "huggingface", "mismatch": mismatch, "settings": settings, } def classify_pi_provider_drift(previous: dict | None, current: dict, *, phase: str = "") -> dict: previous = previous or {} prev_effective = normalize_pi_model_name(previous.get("effective_model") or "") if "normalize_pi_model_name" in globals() else str(previous.get("effective_model") or "").lower() curr_effective = normalize_pi_model_name(current.get("effective_model") or "") if "normalize_pi_model_name" in globals() else str(current.get("effective_model") or "").lower() provider_drift = bool(prev_effective and curr_effective and prev_effective != curr_effective) severity = "none" if provider_drift: if phase in {"repair_patch", "memory_patch"}: severity = "high" elif phase in {"diagnosis", "memory_diagnosis"}: severity = "moderate" else: severity = "minor" elif current.get("mismatch"): severity = "minor" return { "provider_drift_from_previous_call": provider_drift, "previous_effective_model": previous.get("effective_model") or "", "drift_severity": severity, "policy": "Worker validation remains authoritative; higher drift severity tightens repair diff gates and freezes diagnosis reinterpretation.", } def write_pi_call_fingerprint( run_dir: Path, *, phase: str, requested_model: str, prompt_text: str = "", output_text: str = "", task_packet_path: Path | None = None, events_path: Path | None = None, artifact_dir: Path | None = None, identity: dict | None = None, ) -> dict: identity = identity or resolve_pi_call_identity(requested_model, output_text) aggregate_path = run_dir / "pi_call_fingerprints.json" aggregate = read_json(aggregate_path, {"schema_version": "pi_call_fingerprints.v198_25", "calls": []}) if not isinstance(aggregate, dict): aggregate = {"schema_version": "pi_call_fingerprints.v198_25", "calls": []} calls = aggregate.get("calls") if isinstance(aggregate.get("calls"), list) else [] previous = calls[-1] if calls else {} drift = classify_pi_provider_drift(previous, identity, phase=phase) payload = { "schema_version": "pi_call_fingerprint.v198_25", "phase": phase, **identity, **drift, "prompt_hash": _sha256_text(prompt_text or ""), "task_packet_hash": _sha256_file(task_packet_path), "output_hash": _sha256_text(output_text or ""), "created_at": now(), } calls.append(payload) aggregate["calls"] = calls aggregate["latest"] = payload write_json(aggregate_path, aggregate) out_dir = artifact_dir or (run_dir / "pi") out_dir.mkdir(parents=True, exist_ok=True) write_json(out_dir / f"PI_CALL_FINGERPRINT_{phase}.json", payload) # Canonical alias for the most recent call. write_json(run_dir / "pi_call_fingerprint.json", payload) if events_path: append_event(events_path, "pi_call_fingerprint", "warning" if payload.get("drift_severity") in {"high", "critical"} else "success", "Recorded Pi call fingerprint and provider drift policy", {k: payload.get(k) for k in ["phase", "requested_model", "configured_model", "effective_model", "provider_drift_from_previous_call", "drift_severity"]}) return payload def failure_owner_from_classification(classification: dict | None = None) -> str: classification = classification or {} explicit = classification.get("failure_owner") or classification.get("owner") if explicit: return str(explicit) category = str(classification.get("category") or classification.get("failure_class") or "").lower() phase = str(classification.get("failure_phase") or "").lower() if category in {"smoke_schema_error", "validation_client_payload_error"}: return "factory_validation_client" if category in {"cuda_oom", "hardware_capacity", "hardware_runtime_blocker"}: return "hardware_capacity" if category in {"hf_platform", "infra_transient_scheduling", "logs_unavailable"} or "platform" in category: return "hf_platform" if category in {"dependency_error", "import_error", "model_loading_error"}: return "dependency" if category != "model_loading_error" else "model_loading" if "smoke" in phase and category in {"unknown", ""}: return "unknown" return "app_runtime" if category else "unknown" def write_log_evidence_packet(run_dir: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "", classification: dict | None = None, excerpt: str = "") -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) classification = classification or {} excerpt = excerpt or extract_actionable_error_excerpt("\n".join([failure_reason or "", build_log or "", runtime_log or ""]), max_lines=80) payload = { "schema_version": "log_evidence_packet.v198_25", "failure_owner": failure_owner_from_classification(classification), "failure_class": classification.get("category") or classification.get("failure_class") or "unknown", "failure_phase": classification.get("failure_phase") or "unknown", "logs_quality": classification.get("logs_quality") or "unknown", "first_actionable_error": excerpt[:4000], "traceback_excerpt": excerpt[-6000:], "full_logs_available": bool(build_log or runtime_log), "full_logs_path": ["logs/space_logs_build.txt", "logs/space_logs_runtime.txt", "logs/space_logs_run.txt"], "confidence": "high" if excerpt.strip() and classification.get("logs_quality") == "useful" else ("medium" if excerpt.strip() else "low"), "created_at": now(), } write_json(repair_dir / "LOG_EVIDENCE_PACKET.json", payload) return payload def write_frozen_repair_decision(workspace: Path, run_dir: Path, decision: dict, classification: dict | None = None) -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) payload = { "schema_version": "frozen_repair_decision.v198_25", "decision": decision or {}, "classification": classification or (decision or {}).get("classification") or {}, "root_cause": (decision or {}).get("classification", {}).get("category") or (classification or {}).get("category") or "unknown", "frozen": True, "policy": "Repair patch calls must implement this decision and must not reinterpret the run unless they write PATCH_REFUSAL.json without editing publishable files.", "created_at": now(), } write_json(repair_dir / "FROZEN_REPAIR_DECISION.json", payload) write_json(workspace / "FROZEN_REPAIR_DECISION.json", payload) return payload def is_deep_repair_category(category: str) -> bool: return str(category or "") in {"wrong_pipeline_class", "model_card_misunderstood", "architecture_mismatch", "wrong_task_type", "repeated_surgical_failures"} def validate_repair_diff_against_task_packet(diff_payload: dict, task_packet: dict, classification: dict | None, run_dir: Path, events_path: Path) -> tuple[bool, str, dict]: changed = set((diff_payload or {}).get("changed_files") or []) | set((diff_payload or {}).get("added_files") or []) | set((diff_payload or {}).get("removed_files") or []) allowed = set((task_packet or {}).get("allowed_files") or []) category = (classification or {}).get("category") or (task_packet or {}).get("failure_class") or "" if (task_packet or {}).get("task_type") == "dependency_repair": allowed.update({"requirements.txt", "requirements_policy.json"}) if category in {"model_loading_error", "runtime_patch", "app_runtime_error"}: allowed.update({"app.py", "requirements.txt", "requirements_policy.json"}) # Deterministic factory show_error patches may touch app.py after Pi; allow it for any repair but still record it. allowed.add("app.py") if "app.py" in changed and (run_dir / "repair" / "gradio_show_error_patch.json").exists() else None unexpected = sorted(changed - allowed) payload = { "schema_version": "repair_diff_gate.v198_25", "allowed_files": sorted(allowed), "changed_files": sorted(changed), "unexpected_files": unexpected, "task_type": (task_packet or {}).get("task_type"), "failure_class": category, "passed": not bool(unexpected), } write_json(run_dir / "repair" / "repair_diff_gate.json", payload) append_event(events_path, "repair_diff_gate", "success" if payload["passed"] else "failed", "Checked repair diff against Pi task packet allowed files", payload) if unexpected: return False, f"Repair modified files outside the bounded task packet: {', '.join(unexpected)}", payload return True, "Repair diff stayed within bounded task packet.", payload def extract_actionable_error_excerpt(text: str, max_lines: int = 120) -> str: """Return a compact error-centered excerpt for Pi repair packets. Full HF logs remain archived under logs/. Pi should normally receive the latest actionable traceback or OOM/import/dependency section rather than a giant raw log tail. """ raw = text or "" if not raw.strip(): return "" lines = raw.splitlines() markers = [ "traceback", "error", "exception", "modulenotfounderror", "importerror", "cuda out of memory", "outofmemoryerror", "no matching distribution", "could not find a version", "resolutionimpossible", "api_name", "unexpected keyword", "failed to load", "from_pretrained", ] chosen = len(lines) - 1 for idx in range(len(lines) - 1, -1, -1): low = lines[idx].lower() if any(m in low for m in markers): chosen = idx break half = max(10, max_lines // 2) start = max(0, chosen - half) end = min(len(lines), start + max_lines) start = max(0, end - max_lines) return "\n".join(lines[start:end])[-12000:] def write_actionable_error_artifacts(run_dir: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "", classification: dict | None = None) -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) combined = "\n".join([failure_reason or "", build_log or "", runtime_log or ""]) excerpt = extract_actionable_error_excerpt(combined, max_lines=120) (repair_dir / "actionable_error_excerpt.txt").write_text(excerpt, encoding="utf-8") signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification or {}) if "compute_failure_signature" in globals() else "unknown" log_packet = write_log_evidence_packet(run_dir, failure_reason, build_log, runtime_log, classification or {}, excerpt) payload = { "schema_version": "actionable_error_excerpt.v198_25", "failure_signature": signature, "classification": classification or {}, "failure_owner": log_packet.get("failure_owner"), "excerpt_chars": len(excerpt), "full_logs_archived": ["logs/space_logs_build.txt", "logs/space_logs_runtime.txt", "logs/space_logs_run.txt"], "log_evidence_packet": "repair/LOG_EVIDENCE_PACKET.json", } write_json(repair_dir / "error_signature.json", payload) return payload def _first_list_value(*values): for value in values: if isinstance(value, list): return value return None def _first_dict_value(*values): for value in values: if isinstance(value, dict): return value return None def load_parent_replay_source(parent_dir: Path, target_space_id: str, expected_output_type: str) -> dict: """Load the parent build's successful automatic smoke payload as replay truth. In linked Space Test replay mode, the validation worker must not invent a different request when the parent Build Run already proved a working request. The parent smoke artefacts are the source of truth; schema discovery is only used afterwards as a guardrail/retry mechanism. """ if not parent_dir or not parent_dir.exists(): return {} smoke = read_json(parent_dir / "tests" / "generation_smoke.json", {}) or {} retry_payload = read_json(parent_dir / "tests" / "generation_smoke_payload_retry.json", {}) or {} initial_payload = read_json(parent_dir / "tests" / "generation_smoke_payload.json", {}) or {} state = read_json(parent_dir / "state.json", {}) or {} state_smoke = state.get("generation_smoke") if isinstance(state.get("generation_smoke"), dict) else {} status_tokens = { str(smoke.get("status") or "").lower(), str(state.get("status") or "").lower(), str(state_smoke.get("status") or "").lower(), } parent_was_successful = bool(status_tokens & {"success", "full_inference_success", "completed", "complete"}) if not parent_was_successful: return {} api_name = ( smoke.get("api_name") or retry_payload.get("api_name") or initial_payload.get("api_name") or state_smoke.get("api_name") or "" ) test_args = _first_list_value( smoke.get("effective_args"), smoke.get("test_args"), retry_payload.get("test_args"), initial_payload.get("test_args"), state_smoke.get("effective_args"), state_smoke.get("test_args"), ) test_kwargs = _first_dict_value( smoke.get("effective_kwargs"), smoke.get("test_kwargs"), retry_payload.get("test_kwargs"), initial_payload.get("test_kwargs"), state_smoke.get("effective_kwargs"), state_smoke.get("test_kwargs"), ) or {} if not api_name or not isinstance(test_args, list): return {} parent_target = smoke.get("target_space") or state.get("target_space") or state_smoke.get("target_space") or "" return { "source": "parent_automatic_smoke", "parent_run_id": parent_dir.name, "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": test_args, "test_kwargs": test_kwargs, "expected_output_type": smoke.get("expected_output_type") or state_smoke.get("expected_output_type") or expected_output_type, "latency_seconds": smoke.get("latency_seconds") or state_smoke.get("latency_seconds"), "parent_smoke_status": smoke.get("status") or state_smoke.get("status") or state.get("status"), "initial_payload_present": bool(initial_payload), "retry_payload_present": bool(retry_payload), } def load_env_replay_source(target_space_id: str, expected_output_type: str) -> dict: """Replay source passed by backend through the validation Job env. The validation Job starts in a fresh workspace; the parent Build Run folder is usually not mounted locally. This env payload is therefore the canonical transport for replaying a successful parent automatic smoke request. """ raw = os.environ.get("PARENT_REPLAY_SOURCE_JSON") or "" if not raw.strip(): return {} try: data = json.loads(raw) except Exception as exc: return {"source_error": f"invalid_parent_replay_source_json:{exc}"} if not isinstance(data, dict): return {"source_error": "parent_replay_source_json_not_object"} args = data.get("test_args") kwargs = data.get("test_kwargs") or {} api_name = data.get("api_name") or "" if not api_name or not isinstance(args, list) or not isinstance(kwargs, dict): return {"source_error": "parent_replay_source_missing_api_args_or_kwargs", "raw_keys": sorted(data.keys())} parent_target = data.get("parent_target_space") or "" return { "source": data.get("source") or "parent_automatic_smoke_backend", "transport": "job_env", "parent_run_id": data.get("parent_run_id") or os.environ.get("PARENT_BUILD_RUN_ID", ""), "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": args, "test_kwargs": kwargs, "expected_output_type": data.get("expected_output_type") or expected_output_type or "any", "latency_seconds": data.get("latency_seconds"), "parent_smoke_status": data.get("parent_smoke_status") or "success", "backend_replay_source_used": True, } def parent_has_successful_manual_validation(parent_dir: Path) -> bool: manual = read_json(parent_dir / "manual_validation_status.json", {}) or {} if str(manual.get("status") or "").lower() == "success": return True linked = read_json(parent_dir / "linked_validations.json", {}) or {} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] success_tokens = {"success", "full_inference_success", "passed", "validated_after_manual_space_test"} return any(str(row.get("status") or row.get("effective_status") or "").lower() in success_tokens for row in validations if isinstance(row, dict)) def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None): path.parent.mkdir(parents=True, exist_ok=True) event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}} line = json.dumps(event, ensure_ascii=False) with path.open("a", encoding="utf-8") as f: f.write(line + "\n") print(line, flush=True) def write_live_status(run_dir: Path, *, stage: str, status: str = "running", message: str = "", data: dict | None = None): """Persist a tiny fast-changing status snapshot for the UI. The event log can be compacted or uploaded in batches. live_status.json is intentionally small and may be overwritten often so the web UI can show the current micro-stage without waiting for a full bundle refresh. """ payload = { "schema_version": "live_status.v1", "updated_at": now(), "run_id": os.environ.get("RUN_ID", ""), "stage": stage, "status": status, "message": message, "data": data or {}, } try: write_json(run_dir / "live_status.json", payload) except Exception: pass return payload def write_final_summary(run_dir: Path, final_state: dict, inference_gate: dict | None = None, generation_smoke: dict | None = None, *, status: str | None = None, message: str | None = None) -> dict: """Write the lightweight run card summary after terminalization. The backend can canonicalize from state/gate/smoke, but summary.json is the first file many list views and manual audits inspect. Keep it terminal and hardware-accurate once the worker knows the final verdict. """ gate = inference_gate or {} smoke = generation_smoke or {} final_status = status or gate.get("status") or final_state.get("status") or "unknown" promise_validation = gate.get("promise_validation") if isinstance(gate.get("promise_validation"), dict) else final_state.get("promise_validation") if isinstance(final_state.get("promise_validation"), dict) else {} promise_fulfilled = bool(promise_validation.get("promise_fulfilled") or gate.get("promise_fulfilled") or final_status == "full_inference_success") app_boot_validation_status = promise_validation.get("app_boot_validation_status") or gate.get("app_boot_validation_status") or "" promise_validation_status = promise_validation.get("promise_validation_status") or gate.get("promise_validation_status") or ("fulfilled" if promise_fulfilled else "") payload = { "run_id": final_state.get("run_id") or os.environ.get("RUN_ID"), "kind": final_state.get("kind") or "universal_model_card_builder", "status": final_status, "message": message or gate.get("message") or final_state.get("message") or "", "model_id": final_state.get("model_id") or os.environ.get("MODEL_ID", ""), "target_space": final_state.get("target_space") or os.environ.get("TARGET_SPACE_ID", ""), "target_space_url": final_state.get("target_space_url") or (f"https://huggingface.co/spaces/{final_state.get('target_space')}" if final_state.get("target_space") else ""), "selected_hardware": final_state.get("selected_hardware") or os.environ.get("SELECTED_HARDWARE", ""), "created_by": final_state.get("created_by") or os.environ.get("HF_USERNAME", ""), "bucket_source": final_state.get("bucket_source") or os.environ.get("BUCKET_SOURCE", ""), "health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")), "generation_smoke_passed": bool((((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success") and promise_fulfilled), "demo_usable_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("demo_usable_smoke_passed") is True) or smoke.get("demo_usable_smoke_passed") is True or smoke.get("status") == "demo_usable_smoke_passed"), "canonical_promise_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("canonical_promise_smoke_passed") is True) or smoke.get("canonical_promise_smoke_passed") is True or smoke.get("status") == "success"), "validation_level": smoke.get("validation_level") or "", "app_boot_validation_status": app_boot_validation_status, "promise_validation_status": promise_validation_status, "promise_fulfilled": promise_fulfilled, "ui_status": gate.get("ui_status") or final_state.get("ui_status") or final_status, "ui_badge": gate.get("ui_badge") or final_state.get("ui_badge") or "", "failure_type": "" if final_status == "full_inference_success" else (smoke.get("failure_type") or gate.get("failure_type") or (final_state.get("details") or {}).get("failure_type") or (final_state.get("details") or {}).get("repair_failure_type") or (final_state.get("repair_outcome") or {}).get("failure_type") or final_state.get("failure_type") or ""), "repair_outcome_status": (final_state.get("repair_outcome") or {}).get("post_repair_validation") or (final_state.get("details") or {}).get("repair_outcome_status") or "", "active_error": None if final_status == "full_inference_success" else (final_state.get("active_error") or ""), "validation_error": "" if final_status == "full_inference_success" else (final_state.get("validation_error") or ""), "job_exit_code": 0 if final_status == "full_inference_success" else final_state.get("job_exit_code", 1), "updated_at": now(), } write_json(run_dir / "summary.json", payload) return payload SAFE_SPACE_PYTHON_VERSIONS = {"3.10", "3.11", "3.12"} DEFAULT_SPACE_PYTHON_VERSION = "3.10" def normalize_space_python_version(value) -> tuple[str, bool, str]: raw = str(value or "").strip().strip("\"'") lowered = raw.lower().replace("python", "").strip() match = re.search(r"(3)\.(\d+)(?:\.\d+)?", lowered) if match: normalized = f"{match.group(1)}.{match.group(2)}" else: normalized = lowered if normalized in SAFE_SPACE_PYTHON_VERSIONS: return normalized, normalized != raw, "allowed" return DEFAULT_SPACE_PYTHON_VERSION, True, f"unsupported_or_ambiguous:{raw or 'missing'}" def requirements_has_package(lines: list[str], package: str) -> bool: wanted = package.lower().replace("_", "-") for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped: continue name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") if name == wanted: return True return False def workspace_app_imports_torch(workspace: Path) -> bool: app_path = workspace / "app.py" if not app_path.exists(): return False text = app_path.read_text(encoding="utf-8", errors="ignore") return bool(re.search(r"(?m)^\s*(import\s+torch\b|from\s+torch\b)", text)) def _artifact_entry(run_dir: Path, rel_path: str, *, kind: str = "file") -> dict: path = run_dir / rel_path if kind == "folder": present = path.exists() and any(child.is_file() for child in path.rglob("*")) else: present = path.exists() and path.is_file() payload = {"path": rel_path, "kind": kind, "present": bool(present)} if present and kind == "file": try: payload["size"] = path.stat().st_size except Exception: pass return payload def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, reason: str = "snapshot") -> dict: """Snapshot the artifacts that the worker actually wrote to the mounted bucket. The web UI should prefer this manifest over guessed paths. It is written during failures as well as success so a run that created a Space and then crashed can still expose traces, logs and reports in the dock. """ run_id = os.environ.get("RUN_ID", "") bucket_source = os.environ.get("BUCKET_SOURCE", "") artifacts = [ _artifact_entry(run_dir, "events.jsonl"), _artifact_entry(run_dir, "state.json"), _artifact_entry(run_dir, "summary.json"), _artifact_entry(run_dir, "live_status.json"), _artifact_entry(run_dir, "report.md"), _artifact_entry(run_dir, "model_analysis.json"), _artifact_entry(run_dir, "analysis_inputs/model_card.md"), _artifact_entry(run_dir, "analysis_inputs/model_card_source.json"), _artifact_entry(run_dir, "analysis_inputs/model_repo_tree.json"), _artifact_entry(run_dir, "analysis_inputs/prescan_summary.json"), _artifact_entry(run_dir, "analysis_inputs/source_policy.md"), _artifact_entry(run_dir, "MODEL_RECIPE.json"), _artifact_entry(run_dir, "CONTEXT_INDEX.json"), _artifact_entry(run_dir, "SOURCE_PRIORITY.md"), _artifact_entry(run_dir, "planning/pi_feasibility_brief.json"), _artifact_entry(run_dir, "planning/pi_implementation_plan.json"), _artifact_entry(run_dir, "planning/worker_plan_review.json"), _artifact_entry(run_dir, "planning/model_card_grounding_review.json"), _artifact_entry(run_dir, "planning/PI_SOURCE_USAGE.json"), _artifact_entry(run_dir, "hardware_strategy.json"), _artifact_entry(run_dir, "hardware_intent.json"), _artifact_entry(run_dir, "hardware_attempts.json"), _artifact_entry(run_dir, "output_type_resolution.json"), _artifact_entry(run_dir, "inference_gate.json"), _artifact_entry(run_dir, "repair_outcome.json"), _artifact_entry(run_dir, "space_runtime.json"), _artifact_entry(run_dir, "tests/generation_smoke.json"), _artifact_entry(run_dir, "tests/api_schema.json"), _artifact_entry(run_dir, "generated/TECHNICAL_BLOCKERS.json"), _artifact_entry(run_dir, "repair/INCIDENT_BRIEF.md"), _artifact_entry(run_dir, "repair/DEPENDENCY_ERROR_BRIEF.md"), _artifact_entry(run_dir, "repair/PI_DIAGNOSIS_GOAL.md"), _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS_PACKET.json"), _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS_GOAL.md"), _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS.json"), _artifact_entry(run_dir, "repair/REPAIR_DECISION.json"), _artifact_entry(run_dir, "repair/REPAIR_TASK_PACKET.json"), _artifact_entry(run_dir, "repair/PI_TASK_PACKET.json"), _artifact_entry(run_dir, "repair/FROZEN_REPAIR_DECISION.json"), _artifact_entry(run_dir, "repair/LOG_EVIDENCE_PACKET.json"), _artifact_entry(run_dir, "repair/PATCH_REFUSAL.json"), _artifact_entry(run_dir, "repair/actionable_error_excerpt.txt"), _artifact_entry(run_dir, "repair/error_signature.json"), _artifact_entry(run_dir, "repair/PI_PROMPT_BUDGET_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_PROMPT_BUDGET_memory_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_PROMPT_BUDGET_repair_patch.json"), _artifact_entry(run_dir, "pi_prompt_budget.json"), _artifact_entry(run_dir, "pi_call_fingerprints.json"), _artifact_entry(run_dir, "pi_call_fingerprint.json"), _artifact_entry(run_dir, "repair/PI_CALL_FINGERPRINT_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_CALL_FINGERPRINT_memory_diagnosis.json"), _artifact_entry(run_dir, "repair/PI_CALL_FINGERPRINT_repair_patch.json"), _artifact_entry(run_dir, "repair/REPAIR_BRIEF.md"), _artifact_entry(run_dir, "repair/REPAIR_PLAN.md"), _artifact_entry(run_dir, "repair/REPAIR_SUMMARY.md"), _artifact_entry(run_dir, "repair/REPAIR_OUTCOME.json"), _artifact_entry(run_dir, "repair/BLOCKAGE.json"), _artifact_entry(run_dir, "logs/pi_live_output.txt"), _artifact_entry(run_dir, "logs/pi_output.txt"), _artifact_entry(run_dir, "logs/pi_diagnosis_output.txt"), _artifact_entry(run_dir, "logs/pi_repair_output.txt"), _artifact_entry(run_dir, "logs/space_logs_build.txt"), _artifact_entry(run_dir, "logs/space_logs_runtime.txt"), _artifact_entry(run_dir, "logs/space_logs_index.json"), _artifact_entry(run_dir, "logs/space_runtime_snapshot.json"), _artifact_entry(run_dir, "logs/space_log_diagnostics.json"), _artifact_entry(run_dir, "traces/redacted", kind="folder"), _artifact_entry(run_dir, "generated", kind="folder"), _artifact_entry(run_dir, "analysis_inputs", kind="folder"), _artifact_entry(run_dir, "repair", kind="folder"), _artifact_entry(run_dir, "logs", kind="folder"), ] payload = { "run_id": run_id, "bucket_source": bucket_source, "updated_at": now(), "reason": reason, "artifacts": artifacts, "present_paths": [a["path"] for a in artifacts if a.get("present")], } write_json(run_dir / "artifact_manifest.json", payload) if os.environ.get("ASF_EVAL_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"}: try: publish_eval_record(run_dir, phase=f"manifest:{reason}", events_path=events_path) except Exception: pass if events_path is not None: try: append_event(events_path, "artifact_manifest", "success", "Updated run artifact manifest", {"reason": reason, "present_count": len(payload["present_paths"])}) except Exception: pass return payload def update_state(run_dir: Path, patch: dict) -> dict: path = run_dir / "state.json" current = load_json_if_exists(path) if path.exists() else {} if not isinstance(current, dict): current = {} merged = {**current, **patch, "updated_at": now()} write_json(path, merged) return merged def redact_text(text: str | None) -> str: if not text: return "" value = text for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]: secret = os.environ.get(secret_name) if secret: value = value.replace(secret, "[REDACTED]") value = re.sub(r"Bearer\s+[A-Za-z0-9_\-.=]+", "Bearer [REDACTED]", value) value = re.sub(r"hf_[A-Za-z0-9_\-]{10,}", "hf_[REDACTED]", value) return value class AuthRefreshRequired(RuntimeError): """Raised when an OAuth/JWT token is expired or too close to expiry.""" def _decode_jwt_claims_unverified(token: str) -> dict: """Decode JWT claims without verification, only to inspect non-sensitive expiry metadata. This never returns or writes the raw token. Opaque PAT-style tokens are supported by returning an empty dict so they remain usable with an `unknown` expiry state. """ try: parts = (token or "").split(".") if len(parts) < 2: return {} payload = parts[1] payload += "=" * ((4 - len(payload) % 4) % 4) raw = base64.urlsafe_b64decode(payload.encode("utf-8")) data = json.loads(raw.decode("utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict: issued_at = now() if not token: return { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": False, "token_kind": "missing", "expiry_known": False, "status": "missing", "safe_for_phase": False, "minimum_required_seconds": minimum_required_seconds, } claims = _decode_jwt_claims_unverified(token) exp = claims.get("exp") if isinstance(claims, dict) else None iat = claims.get("iat") if isinstance(claims, dict) else None token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown") payload = { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": True, "token_kind": token_kind, "expiry_known": exp is not None, "minimum_required_seconds": int(minimum_required_seconds or 0), "token_value": "[REDACTED]", } if iat is not None: try: payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat() except Exception: pass if exp is None: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"}) return payload try: exp_int = int(exp) seconds_left = exp_int - int(time.time()) payload.update({ "expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(), "seconds_until_expiry": seconds_left, }) except Exception: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"}) return payload if seconds_left <= 0: payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"}) elif minimum_required_seconds and seconds_left < minimum_required_seconds: payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"}) else: payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"}) return payload def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict: payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds) payload["phase"] = phase safe_payload = {k: v for k, v in payload.items() if k != "token_value"} try: probes_dir = run_dir / "auth_probes" probes_dir.mkdir(parents=True, exist_ok=True) write_json(probes_dir / f"{phase}.json", safe_payload) write_json(run_dir / "auth_status.json", safe_payload) except Exception: pass status = str(payload.get("status") or "unknown") if events_path: event_status = "success" if payload.get("safe_for_phase") else "failed" if status == "unknown": event_status = "warning" append_event( events_path, "auth_probe", event_status, f"HF OAuth/token expiry check for {phase}: {status}", safe_payload, ) if raise_on_unsafe and not payload.get("safe_for_phase"): raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.") return payload def is_auth_expired_error(error: Exception | str) -> bool: text = str(error or "").lower() return any(marker in text for marker in [ "oauth token has expired", "exp claim timestamp check failed", "token has expired", "jwt expired", ]) def classify_repair_validation_error(error: Exception | str) -> dict: """Classify post-repair validation failures without losing the repair result. A repair can be correctly diagnosed, patched, and uploaded while the final validation is inconclusive because OAuth expired. Keep that distinct from a model/runtime repair failure so the UI and audit trail do not imply that Pi's patch necessarily failed. """ text = str(error or "") if isinstance(error, AuthRefreshRequired) or is_auth_expired_error(text): return { "post_repair_validation": "inconclusive_auth_expired", "failure_type": "repair_validation_inconclusive_auth", "terminal_status": "auth_refresh_required", "message": "Repair progress could not be validated because HF auth expired or is too close to expiry.", } return { "post_repair_validation": "failed", "failure_type": "repair_validation_failed", "terminal_status": "failed", "message": "Repair was applied, but the repaired Space did not pass live validation.", } def write_repair_outcome(run_dir: Path, events_path: Path | None = None, **updates) -> dict: """Write a cumulative repair outcome artifact. Keep it at the run root for list-view consumers and mirror it inside repair/ for manual audits. The payload is intentionally compact and redacted; full logs remain in logs/ and repair/ artifacts. """ root_path = run_dir / "repair_outcome.json" existing = load_json_if_exists(root_path) if root_path.exists() else {} if not isinstance(existing, dict): existing = {} payload = { "schema_version": "repair_outcome.v1", **existing, **{k: v for k, v in updates.items() if v is not None}, "updated_at": now(), } write_json(root_path, payload) try: write_json(run_dir / "repair" / "REPAIR_OUTCOME.json", payload) except Exception: pass if events_path: append_event( events_path, "repair_outcome", str(payload.get("post_repair_validation") or payload.get("repair_status") or payload.get("repair_decision") or "updated"), "Repair outcome artifact updated", payload, ) return payload def minimum_auth_seconds_for_phase(phase: str) -> int: env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_") raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "") if raw: try: return max(0, int(raw)) except Exception: pass defaults = { "before_space_create": 900, "before_upload": 900, "before_initial_validation": 1800, "before_repair": 1800, "before_repair_upload": 1800, "before_repair_validation": 1800, "before_linked_space_test": 1800, } return defaults.get(phase, 0) def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict: """Ensure Pi and HF tooling see the same private HF token safely. HF Jobs receive HF_TOKEN as a secret. Pi, huggingface_hub, gradio_client, and the hf CLI can use either HF_TOKEN or HUGGING_FACE_HUB_TOKEN depending on the tool/version, so expose both aliases inside the isolated Job process. Never write the token value to artifacts. """ token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "" if token: os.environ.setdefault("HF_TOKEN", token) os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token) os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation")) payload = { "hf_token_present": bool(token), "hf_token_length": len(token) if token else 0, "hugging_face_hub_token_alias_present": bool(os.environ.get("HUGGING_FACE_HUB_TOKEN")), "hf_username": os.environ.get("HF_USERNAME") or "", "bucket_source": os.environ.get("BUCKET_SOURCE") or "", "target_space_id": os.environ.get("TARGET_SPACE_ID") or "", "token_value": "[REDACTED]" if token else "", "token_kind": expiry.get("token_kind"), "expiry_known": expiry.get("expiry_known"), "expires_at": expiry.get("expires_at"), "seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"), "auth_risk": expiry.get("auth_risk"), "safe_for_long_build": bool(expiry.get("safe_for_phase")), "minimum_required_seconds": expiry.get("minimum_required_seconds"), } run_dir.mkdir(parents=True, exist_ok=True) write_json(run_dir / "token_context.json", payload) if events_path: append_event(events_path, "token_context", "success" if token else "warning", "Verified HF token context for Pi/HF operations", {k: v for k, v in payload.items() if k != "token_value"}) return payload def pi_tooling_context_note() -> str: return """HF tooling context: - You are running inside an HF Job with an HF_TOKEN secret configured. - For gated/private base models, LoRA adapters, PEFT adapters, or private Hub files, the generated Space must read `HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")` and pass `token=HF_TOKEN` to `from_pretrained`, `hf_hub_download`, `snapshot_download`, and `load_lora_weights` when those APIs are used. Never print the token. - For ZeroGPU/Diffusers/LoRA GPU apps, lazy-load heavy pipelines inside the `@spaces.GPU`-decorated inference function or a cache called from it. Do not download/load large gated models at module import before the GPU-decorated call. - The Factory exposes both HF_TOKEN and HUGGING_FACE_HUB_TOKEN aliases for huggingface_hub, gradio_client, hf CLI, and Pi provider access. - You may use the token through normal tools when needed to inspect the model/Space or validate private resources. - Never print, write, echo, or commit token values. Do not include Authorization headers in artifacts. - If a Hub operation fails with 401/403/quota/billing, report it as evidence instead of retrying blindly. """ def _agent_trace_json_safe(payload): try: return json.loads(redact_text(json.dumps(payload, ensure_ascii=False))) except Exception: return {"value": redact_text(str(payload))[:8000]} def write_agent_trace_record( run_dir: Path, *, phase: str, event: str, status: str = "info", message: str = "", data: dict | None = None, text: str | None = None, artifacts: list[str] | None = None, ): """Append one run-level agent trace record to the redacted journal only. Pi can be invoked several times during a run: initial build, blockage diagnosis, and optional repair patch. The individual Pi session files and stdout logs remain useful, but this journal is the canonical narrative tying those phases together into one auditable agent run. Security note: raw Pi session traces may contain credentials if a tool or assistant accidentally prints environment values. The Factory therefore does not persist RAW Pi traces to the run bucket. Only redacted traces are archived and exposed in the UI. """ try: payload = { "ts": now(), "phase": phase, "event": event, "status": status, "message": message, "data": data or {}, "artifacts": artifacts or [], } if text is not None: payload["text"] = text redacted_dir = run_dir / "traces" / "redacted" redacted_dir.mkdir(parents=True, exist_ok=True) redacted_payload = _agent_trace_json_safe(payload) redacted_line = json.dumps(redacted_payload, ensure_ascii=False) with (redacted_dir / "agent_trace.jsonl").open("a", encoding="utf-8") as f: f.write(redacted_line + "\n") except Exception: # Trace journaling must never break the build/repair path. pass def append_agent_trace_artifact(run_dir: Path, *, phase: str, event: str, artifact: str, text: str | None = None, status: str = "success", data: dict | None = None): tail = None if text is None else text[-20000:] write_agent_trace_record(run_dir, phase=phase, event=event, status=status, message=f"Agent phase wrote {artifact}", data=data or {}, text=tail, artifacts=[artifact]) def safe_details(details: dict | None) -> dict: if not details: return {} try: return json.loads(redact_text(json.dumps(details, ensure_ascii=False))) except Exception: return {"redacted_details": redact_text(str(details))[-4000:]} def exception_payload(exc: Exception) -> dict: """Extract the most useful redacted details from HF Hub / HTTP exceptions.""" payload = { "error": redact_text(str(exc))[:4000], "exception_type": type(exc).__name__, } response = getattr(exc, "response", None) if response is not None: status_code = getattr(response, "status_code", None) if status_code is not None: payload["status_code"] = status_code try: text_body = response.text except Exception: text_body = "" if text_body: payload["response_text"] = redact_text(text_body)[:4000] payload["hf_error_detail"] = redact_text(text_body)[:4000] try: json_body = response.json() except Exception: json_body = None if json_body is not None: payload["response_json"] = safe_details(json_body) if isinstance(json_body, dict): for key in ("error", "message", "detail", "details"): value = json_body.get(key) if value: payload["hf_error_message"] = redact_text(value if isinstance(value, str) else json.dumps(value, ensure_ascii=False))[:4000] break for attr in ("server_message", "request_id"): value = getattr(exc, attr, None) if value: payload[attr] = redact_text(str(value))[:1000] return payload def eval_enabled() -> bool: return os.environ.get("ASF_EVAL_RECORD_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"} def eval_safe_hash(value: str | None, prefix: str = "h") -> str: raw = (value or "").strip() salt = os.environ.get("ASF_EVAL_SALT", "").strip() if not raw: return "" if salt: digest = hmac.new(salt.encode("utf-8"), raw.encode("utf-8"), hashlib.sha256).hexdigest()[:24] else: digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] return f"{prefix}_{digest}" def eval_load_json(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8", errors="replace")) except Exception as exc: return {"_parse_error": str(exc)} def eval_load_events(path: Path) -> list[dict]: if not path.exists(): return [] events = [] for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): try: evt = json.loads(line) if isinstance(evt, dict): events.append(evt) except Exception: pass return events def eval_parse_ts(value: str | None) -> float | None: if not value: return None try: return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() except Exception: return None def eval_compact_timeline(events: list[dict]) -> list[dict]: stages: dict[str, dict] = {} order: list[str] = [] for evt in events: step = str(evt.get("step") or "unknown")[:80] status = str(evt.get("status") or "")[:80] ts = str(evt.get("ts") or "") tsv = eval_parse_ts(ts) if step not in stages: stages[step] = {"stage": step, "first_ts": ts, "last_ts": ts, "status": status, "event_count": 0} order.append(step) item = stages[step] item["last_ts"] = ts or item.get("last_ts") item["status"] = status or item.get("status") item["event_count"] = int(item.get("event_count") or 0) + 1 if tsv is not None: if "_first" not in item: item["_first"] = tsv item["_last"] = tsv timeline = [] for step in order: item = dict(stages[step]) first = item.pop("_first", None) last = item.pop("_last", None) if first is not None and last is not None and last >= first: item["duration_ms"] = int((last - first) * 1000) timeline.append(item) return timeline[-80:] def eval_redacted_tail(path: Path, limit: int = 2400) -> str: if os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() not in {"1", "true", "yes", "on"}: return "" try: return redact_text(path.read_text(encoding="utf-8", errors="replace"))[-limit:] except Exception: return "" def eval_len_if_list(value) -> int: return len(value) if isinstance(value, list) else 0 def eval_seconds_bucket(value) -> str: try: seconds = int(value) except Exception: return "unknown" if seconds < 0: return "expired" if seconds < 15 * 60: return "lt_15m" if seconds < 60 * 60: return "lt_1h" if seconds < 4 * 60 * 60: return "lt_4h" if seconds < 8 * 60 * 60: return "lt_8h" return "gte_8h" def eval_v191_plus_signals(run_dir: Path, analysis: dict, generation_smoke: dict, *, contract: dict | None = None, requirements_policy: dict | None = None, auth_status: dict | None = None, repair_outcome: dict | None = None, worker_plan_review: dict | None = None, grounding_review: dict | None = None) -> dict: """Return privacy-safe metrics for features added from v191.1 onward.""" analysis = analysis if isinstance(analysis, dict) else {} generation_smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = contract if isinstance(contract, dict) else {} requirements_policy = requirements_policy if isinstance(requirements_policy, dict) else {} auth_status = auth_status if isinstance(auth_status, dict) else {} repair_outcome = repair_outcome if isinstance(repair_outcome, dict) else {} worker_plan_review = worker_plan_review if isinstance(worker_plan_review, dict) else {} grounding_review = grounding_review if isinstance(grounding_review, dict) else {} build_risk = analysis.get("build_risk") if isinstance(analysis.get("build_risk"), dict) else {} kernel_strategy = analysis.get("kernel_strategy") if isinstance(analysis.get("kernel_strategy"), dict) else {} if not kernel_strategy and isinstance(analysis.get("metadata"), dict): kernel_strategy = analysis["metadata"].get("kernel_strategy") if isinstance(analysis["metadata"].get("kernel_strategy"), dict) else {} grounding_source = grounding_review.get("model_card_source") if isinstance(grounding_review.get("model_card_source"), dict) else {} return { "schema_version": "v191_plus_eval_signals.v1", "platform_dependency_policy": { "present": bool(requirements_policy), "status": requirements_policy.get("status") or "", "removed_platform_pin_count": eval_len_if_list(requirements_policy.get("removed_pins")), "normalized_platform_line_count": eval_len_if_list(requirements_policy.get("normalized_platform_lines")), "injected_platform_line_count": eval_len_if_list(requirements_policy.get("injected_platform_lines")), "torch_added": bool(requirements_policy.get("torch_added")), }, "auth_context": { "present": bool(auth_status), "status": auth_status.get("status") or "", "token_kind": auth_status.get("token_kind") or "", "expiry_known": bool(auth_status.get("expiry_known")), "seconds_until_expiry_bucket": eval_seconds_bucket(auth_status.get("seconds_until_expiry")), "safe_for_phase": bool(auth_status.get("safe_for_phase")), }, "model_scan": { "build_risk_level": build_risk.get("level") or analysis.get("build_risk_level") or "", "build_risk_signal_count": eval_len_if_list(build_risk.get("signals")), "build_risk_visibility_only": bool(build_risk.get("visibility_only", True)) if build_risk else True, "recommended_session_minutes": build_risk.get("recommended_session_minutes"), "kernel_strategy_present": bool(kernel_strategy), "native_kernel_detected": bool(kernel_strategy.get("native_kernel_detected") or kernel_strategy.get("detected")), "kernel_signal_count": eval_len_if_list(kernel_strategy.get("signals")), "kernel_candidate_count": eval_len_if_list(kernel_strategy.get("candidates")), }, "contract_validation": { "contract_present": bool(contract), "full_inference_implemented": bool(contract.get("full_inference_implemented")), "validation_level": contract.get("validation_level") or "", "requires_gpu": bool(contract.get("requires_gpu")), "blockers_count": int(contract.get("blockers_count") or 0) if str(contract.get("blockers_count") or "0").isdigit() else 0, "generation_smoke_status": generation_smoke.get("status") or "", "generation_smoke_skipped": str(generation_smoke.get("status") or "").lower() == "skipped", "generation_smoke_skip_reason": generation_smoke.get("skip_reason") or "", }, "repair_outcome": { "present": bool(repair_outcome), "repair_decision": repair_outcome.get("repair_decision") or repair_outcome.get("decision") or "", "patch_applied": bool(repair_outcome.get("patch_applied")), "upload_success": bool(repair_outcome.get("upload_success")), "post_repair_validation": repair_outcome.get("post_repair_validation") or "", "failure_type": repair_outcome.get("failure_type") or "", }, "planning": { "worker_plan_review_present": bool(worker_plan_review), "status": worker_plan_review.get("status") or "", "declared_strategy": worker_plan_review.get("declared_strategy") or "", "worker_recommendation": worker_plan_review.get("worker_recommendation") or "", "warning_count": eval_len_if_list(worker_plan_review.get("warnings")), }, "model_card_grounding": { "present": bool(grounding_review), "status": grounding_review.get("status") or "", "source_available": bool(grounding_review.get("source_available")), "model_card_present": bool(grounding_review.get("model_card_present")), "source": grounding_source.get("source") or "", "resolved_card_file": grounding_source.get("resolved_card_file") or "", "fallback_used": bool(grounding_source.get("fallback_used")), "pi_evidence_present": bool(grounding_review.get("pi_evidence_present")), "pi_evidence_count": int(grounding_review.get("pi_evidence_count") or 0), "warning_count": eval_len_if_list(grounding_review.get("warnings")), "warnings": [str(w)[:120] for w in (grounding_review.get("warnings") if isinstance(grounding_review.get("warnings"), list) else [])[:8]], }, } def eval_terminal_status(status: str, phase: str = "") -> bool: status = str(status or "").strip().lower() phase = str(phase or "").strip().lower() if phase in {"final", "failure"}: return True return status in { "done", "success", "failed", "cancelled", "canceled", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "demo_usable_full_promise_not_verified", "interactive_app_available_smoke_failed", "manual_test_required_smoke_failed", "partial_validation", "technical_blocker", "manual_hardware_required", } def eval_event_seen(events: list[dict], step: str, status: str | None = None) -> bool: for event in events or []: if str(event.get("step") or "") != step: continue if status is None or str(event.get("status") or "") == status: return True return False def eval_health_passed(inference_gate: dict, generation_smoke: dict, events: list[dict], status: str) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} if signals.get("health_passed") is True: return True if str(status or "") in {"full_inference_success", "full_inference_candidate_health_passed", "health_only"}: return True if eval_event_seen(events, "api_validation", "success") or eval_event_seen(events, "repair_validation", "success"): return True return False def eval_generation_smoke_passed(inference_gate: dict, generation_smoke: dict) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} return bool(signals.get("generation_smoke_passed") is True or generation_smoke.get("status") == "success") def eval_failure_summary(status: str, state: dict, inference_gate: dict, generation_smoke: dict, health_passed: bool, smoke_passed: bool) -> dict: failure_type = generation_smoke.get("failure_type") or inference_gate.get("failure_type") or "" failure_owner = generation_smoke.get("failure_owner") or inference_gate.get("failure_owner") or "" missing_argument = generation_smoke.get("missing_argument") or inference_gate.get("missing_argument") or "" reason = missing_argument or failure_type or str(state.get("message") or inference_gate.get("message") or "")[:240] if not failure_owner: if failure_type == "validator_request_error" or missing_argument: failure_owner = "factory_validator" elif smoke_passed: failure_owner = "" elif health_passed and str(generation_smoke.get("status") or "") == "failed": failure_owner = "unknown" return { "failure_type": failure_type, "failure_owner": failure_owner, "failure_reason": reason, "missing_argument": missing_argument, } def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: bool, full_inference_verified: bool, inference_gate: dict) -> str: status = str(status or "").strip() if status in {"cancelled", "canceled"}: return "cancelled" if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"): return "manual_action_required" if status in {"technical_blocker", "technical_blocker_boot_only"}: return "technical_blocker" if full_inference_verified: return "success" if status in {"full_inference_candidate_health_passed", "health_only"} or (health_passed and not smoke_passed): return "partial_validation" if phase == "failure" or status in {"failed", "error"}: return "failed" if eval_terminal_status(status, phase): return "partial_validation" if health_passed else "failed" return "running" def publish_eval_record(run_dir: Path, *, phase: str, events_path: Path | None = None) -> dict: """Write an anonymized cross-run evaluation record to the optional eval bucket. This intentionally stores structured metrics and hashed identifiers, not raw generated code, prompts, tokens, user bucket paths, or target Space IDs. """ if not eval_enabled(): return {"enabled": False} # Jobs only write the anonymized eval record back into the user's run bucket. # The ASF backend later publishes this record to the operator eval bucket # mounted on the ASF Space. User Jobs must never mount or write the private # operator archive directly. run_id = os.environ.get("RUN_ID", run_dir.name) events = eval_load_events(events_path or (run_dir / "events.jsonl")) state = eval_load_json(run_dir / "state.json") analysis = eval_load_json(run_dir / "model_analysis.json") hardware_strategy = eval_load_json(run_dir / "hardware_strategy.json") hardware_attempts = eval_load_json(run_dir / "hardware_attempts.json") inference_gate = eval_load_json(run_dir / "inference_gate.json") generation_smoke = eval_load_json(run_dir / "tests" / "generation_smoke.json") generation_smoke_retry = eval_load_json(run_dir / "tests" / "generation_smoke_payload_retry.json") endpoint_discovery_payload = eval_load_json(run_dir / "tests" / "gradio_endpoint_discovery.json") manual_validation_status = eval_load_json(run_dir / "manual_validation_status.json") linked_validations_payload = eval_load_json(run_dir / "linked_validations.json") space_runtime = eval_load_json(run_dir / "space_runtime.json") manifest = eval_load_json(run_dir / "artifact_manifest.json") repair_decision = eval_load_json(run_dir / "repair" / "REPAIR_DECISION.json") blockage = eval_load_json(run_dir / "repair" / "BLOCKAGE.json") repair_outcome = eval_load_json(run_dir / "repair_outcome.json") or eval_load_json(run_dir / "repair" / "REPAIR_OUTCOME.json") requirements_policy = eval_load_json(run_dir / "generated" / "requirements_policy.json") or eval_load_json(run_dir / "requirements_policy.json") auth_status = eval_load_json(run_dir / "auth_status.json") contract = eval_load_json(run_dir / "generated" / "INFERENCE_CONTRACT.json") worker_plan_review = eval_load_json(run_dir / "planning" / "worker_plan_review.json") grounding_review = eval_load_json(run_dir / "planning" / "model_card_grounding_review.json") or (worker_plan_review.get("model_card_grounding") if isinstance(worker_plan_review.get("model_card_grounding"), dict) else {}) pi_resolution = state.get("pi_model_resolution") if isinstance(state.get("pi_model_resolution"), dict) else {} status = str(state.get("status") or inference_gate.get("status") or phase or "unknown") failure_message = str(state.get("message") or (state.get("details") or {}).get("error") or "")[:1200] started_at = events[0].get("ts") if events else (state.get("created_at") or os.environ.get("LAUNCHED_AT") or "") finished_at = state.get("updated_at") or (events[-1].get("ts") if events else now()) model_id = os.environ.get("MODEL_ID") or state.get("model_id") or analysis.get("model_id") or "" target_space = state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" include_model_id = os.environ.get("ASF_EVAL_INCLUDE_MODEL_ID", "").strip().lower() in {"1", "true", "yes", "on"} anon_run_id = eval_safe_hash(run_id, "run") health_passed = eval_health_passed(inference_gate, generation_smoke, events, status) generation_smoke_passed = eval_generation_smoke_passed(inference_gate, generation_smoke) promise_validation = inference_gate.get("promise_validation") if isinstance(inference_gate.get("promise_validation"), dict) else {} promise_fulfilled = bool(promise_validation.get("promise_fulfilled") or inference_gate.get("promise_fulfilled") or status == "full_inference_success") full_inference_verified = bool(promise_fulfilled and status == "full_inference_success" and health_passed and generation_smoke_passed) process_completed = eval_terminal_status(status, phase) failure_summary = eval_failure_summary(status, state, inference_gate, generation_smoke, health_passed, generation_smoke_passed) verdict = eval_verdict(status, phase, health_passed, generation_smoke_passed, full_inference_verified, inference_gate) linked_rows = linked_validations_payload.get("validations") if isinstance(linked_validations_payload.get("validations"), list) else [] linked_success_rows = [row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() == "success"] manual_applied = str(manual_validation_status.get("status") or "").lower() == "success" or bool(linked_success_rows) effective_status = str((manual_validation_status if manual_applied else {}).get("effective_status") or (linked_success_rows[-1].get("effective_status") if linked_success_rows else "") or ("success" if full_inference_verified else verdict)) endpoint_names = endpoint_discovery_payload.get("discovered_api_names") or endpoint_discovery_payload.get("candidates") or [] smoke_retry_meta = generation_smoke.get("auto_retry") if isinstance(generation_smoke.get("auto_retry"), dict) else {} record = { "process_completed": process_completed, "verdict": verdict, "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "schema_version": "1.4", "app_version": os.environ.get("ASF_VERSION", "unknown"), "phase": phase, "anonymous_run_id": anon_run_id, "run_id_hash": eval_safe_hash(run_id, "run"), "anonymous_user_hash": eval_safe_hash(os.environ.get("HF_USERNAME"), "user"), "salt_configured": bool(os.environ.get("ASF_EVAL_SALT")), "run_kind": state.get("kind") or ("validate_existing_space" if os.environ.get("API_NAME") else "universal_model_card_builder"), "started_at": started_at, "finished_at": finished_at, "input_model": { "model_id_redacted": not include_model_id, "model_id": model_id if include_model_id else "", "model_hash": eval_safe_hash(model_id, "model"), "pipeline_tag": analysis.get("pipeline_tag"), "library_name": analysis.get("library_name"), "default_model_target": bool(analysis.get("default_model_target")), }, "config": { "implementation_mode": os.environ.get("IMPLEMENTATION_MODE", ""), "expected_output_type": os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "pi_model_requested": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "pi_model_effective": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "try_zero_gpu_first": os.environ.get("TRY_ZERO_GPU_FIRST", ""), "allow_fixed_gpu_fallback": os.environ.get("ALLOW_FIXED_GPU_FALLBACK", ""), "preferred_gpu": os.environ.get("PREFERRED_SPACE_HARDWARE", ""), "fallback_gpu": os.environ.get("FALLBACK_SPACE_HARDWARE", ""), }, "outcome": { "status": status, "verdict": verdict, "process_completed": process_completed, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "app_boot_validation_status": inference_gate.get("app_boot_validation_status") or "", "promise_validation_status": inference_gate.get("promise_validation_status") or promise_validation.get("promise_validation_status") or "", "promise_fulfilled": bool(promise_fulfilled), "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "message_redacted": redact_text(failure_message)[:1200], "target_space_hash": eval_safe_hash(target_space, "space"), "space_created": bool(target_space), "space_runtime_stage": space_runtime.get("stage"), "space_runtime_hardware": space_runtime.get("hardware") or space_runtime.get("requested_hardware"), }, "timeline": eval_compact_timeline(events), "hardware": { "selected": hardware_strategy.get("selected_hardware") or state.get("selected_hardware") or "", "strategy": hardware_strategy.get("strategy") or hardware_attempts.get("strategy") or "", "attempts": hardware_strategy.get("attempts") or hardware_attempts.get("attempts") or state.get("hardware_attempts") or [], "single_space_per_run": True, }, "pi": { "requested_model": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "effective_model": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "model_mismatch": bool(pi_resolution.get("mismatch")), "build_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_output.txt"), "diagnosis_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_diagnosis_output.txt"), }, "recovery": { "entered": any(str(e.get("step", "")).startswith(("repair", "failure_diagnosis", "pi_diagnosis", "technical_blocker")) for e in events), "decision": repair_decision.get("action") or repair_decision.get("decision") or "", "confidence": repair_decision.get("confidence") or "", "blockage": bool(blockage), "blockage_category": blockage.get("category") or blockage.get("reason") or "", }, "validation": { "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "generation_smoke_status": generation_smoke.get("status") or "", "latency_seconds": generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "expected_output_type": generation_smoke.get("expected_output_type") or os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "inference_gate_status": inference_gate.get("status") or "", "manual_hardware_required": bool(inference_gate.get("manual_hardware_required")), "failure_type": failure_summary.get("failure_type") or "", "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "missing_argument": failure_summary.get("missing_argument") or "", }, "automatic_outcome": { "status": status, "verdict": verdict, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, }, "effective_outcome": { "automatic_verdict": verdict, "automatic_status": status, "effective_verdict": effective_status, "effective_status": effective_status, "manual_validation_applied": manual_applied, "source": "linked_space_test" if manual_applied else "automatic_run", }, "linked_validation": { "present": bool(linked_rows or manual_validation_status), "count": len(linked_rows), "success_count": len(linked_success_rows), "failure_count": len([row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() in {"failed", "failure", "error"}]), "status": "success" if manual_applied else str(manual_validation_status.get("status") or ""), "mode": manual_validation_status.get("space_test_policy_mode") or "", "api_name": manual_validation_status.get("api_name") or generation_smoke.get("api_name") or "", "latency_seconds": manual_validation_status.get("latency_seconds") or generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "hardware_used_for_validation": manual_validation_status.get("hardware_used_for_validation") or "", }, "endpoint_discovery": { "required": bool(endpoint_discovery_payload.get("endpoint_discovery_required") or endpoint_discovery_payload), "succeeded": bool(endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint")), "selected_endpoint": endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint") or "", "candidate_count": len(endpoint_names) if isinstance(endpoint_names, list) else 0, "excluded_health_endpoint": True, }, "smoke_retry": { "retried": bool(generation_smoke_retry or smoke_retry_meta.get("retried")), "reason": generation_smoke_retry.get("retry_reason") or smoke_retry_meta.get("retry_reason") or smoke_retry_meta.get("reason") or "", "attempts": generation_smoke_retry.get("attempts") or smoke_retry_meta.get("attempts") or (2 if generation_smoke_retry else 1), "passed_after_retry": bool(generation_smoke_retry and generation_smoke.get("status") == "success"), }, "v191_plus": eval_v191_plus_signals( run_dir, analysis, generation_smoke, contract=contract, requirements_policy=requirements_policy, auth_status=auth_status, repair_outcome=repair_outcome, worker_plan_review=worker_plan_review, grounding_review=grounding_review, ), "artifacts": { "present_count": len(manifest.get("present_paths") or []), "present_paths": sorted(str(p) for p in (manifest.get("present_paths") or []))[:120], }, "privacy": { "tokens_stored": False, "raw_prompts_stored": False, "generated_code_stored": False, "target_space_redacted": True, "user_bucket_redacted": True, "redacted_tails_enabled": os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() in {"1", "true", "yes", "on"}, "linked_run_ids_redacted": True, "endpoint_schema_stored": False, "validation_args_stored": False, "model_card_raw_stored": False, "pi_evidence_text_stored": False, "requirements_txt_stored": False, "auth_token_stored": False, }, } day = (finished_at or now())[:10] try: yyyy, mm, dd = day.split("-") except Exception: yyyy, mm, dd = now()[:10].split("-") dest = run_dir / "eval_record.json" write_json(dest, record) events_dest = run_dir / "events_compact.jsonl" events_dest.write_text("".join(json.dumps(item, ensure_ascii=False) + "\n" for item in record["timeline"]), encoding="utf-8") return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"} def _env_truthy(*names: str) -> bool: for name in names: value = os.environ.get(name, "") if str(value).strip().lower() in {"1", "true", "yes", "on"}: return True return False def _hardware_is_gpuish(selected_hardware: str | None) -> bool: value = (selected_hardware or "").strip().lower() if not value: return False if value in {"cpu-basic", "cpu-upgrade", "default-cpu-or-existing"}: return False return any(marker in value for marker in ["gpu", "a10g", "a100", "h100", "l4", "l40", "t4", "zero-"]) def _gpu_partial_autopause_mode() -> str: raw = str(os.environ.get("ASF_GPU_PARTIAL_AUTOPAUSE_MODE") or "recommended").strip().lower() return raw if raw in {"off", "recommended", "strict"} else "recommended" def should_pause_generated_space(final_status: str | None, target_space: str | None, selected_hardware: str | None = None, *, keep_failed_spaces_running: bool | None = None, health_semantic_passed: bool | None = None) -> dict: """Decide whether a generated Space should be paused after a terminal outcome. v198.26.7 keeps GPU partial cleanup non-aggressive by default: a reachable partial can still be useful and should be validated via Space Test/minimal smoke before pausing. `ASF_GPU_PARTIAL_AUTOPAUSE_MODE=strict` restores automatic pause for GPU partials; `recommended` emits a cost warning only. """ status = (final_status or "").strip().lower() target = (target_space or "").strip() hardware = (selected_hardware or "").strip() keep_running = _env_truthy("ASF_KEEP_FAILED_SPACES_RUNNING", "KEEP_FAILED_SPACES_RUNNING") if keep_failed_spaces_running is None else bool(keep_failed_spaces_running) base = {"final_status": status, "selected_hardware": hardware, "health_semantic_passed": health_semantic_passed} if keep_running: return {"should_pause": False, "reason": "keep_failed_spaces_running_enabled", **base} if not target: return {"should_pause": False, "reason": "no_target_space", **base} if status in {"full_inference_success", "success"}: return {"should_pause": False, "reason": "successful_run", **base} gpuish = _hardware_is_gpuish(hardware) if status == "technical_blocker_boot_only": should = gpuish return {"should_pause": should, "reason": "technical_blocker_boot_only_gpu" if should else "technical_blocker_boot_only_cpu", **base} failed_statuses = { "failed", "build_failed", "runtime_failed", "repair_failed", "repair_patch_failed", "validation_failed", "dependency_error", "manual_hardware_required", "full_inference_failed", "generation_smoke_failed", } if status in failed_statuses or status.endswith("_failed"): return {"should_pause": True, "reason": "terminal_failure", **base} partial_statuses = { "partial", "partial_validation", "health_only", "full_inference_candidate_health_passed", "completed_with_warnings", "interactive_app_available_smoke_failed", "manual_test_required_smoke_failed", "demo_usable_full_promise_not_verified", } if status in partial_statuses and gpuish: mode = _gpu_partial_autopause_mode() unhealthy = health_semantic_passed is False if mode == "strict": return {"should_pause": True, "reason": "partial_gpu_strict_autopause" if not unhealthy else "partial_gpu_pipeline_unhealthy_strict_autopause", "pause_recommended": True, "autopause_mode": mode, **base} return {"should_pause": False, "reason": "partial_gpu_pause_recommended" if unhealthy else "partial_gpu_cost_warning_manual_validation_recommended", "pause_recommended": True, "autopause_mode": mode, **base} return {"should_pause": False, "reason": "status_not_pauseable", **base} def pause_generated_space_best_effort(target_space: str, token: str | None, run_dir: Path, events_path: Path, *, final_status: str | None = None, selected_hardware: str | None = None, reason: str | None = None) -> dict: """Pause a generated Space best-effort and publish cleanup_status.json. This helper must never mask the primary run outcome. It catches all exceptions, stores a redacted status artifact, and emits cleanup events. """ cleanup = { "space_pause_attempted": False, "space_pause_success": False, "target_space": target_space or "", "final_status": final_status or "", "selected_hardware": selected_hardware or "", "reason": reason or "", "updated_at": now(), } if not target_space: cleanup.update({"skipped": True, "skip_reason": "no_target_space"}) write_json(run_dir / "cleanup_status.json", cleanup) return cleanup if not token: cleanup.update({"skipped": True, "skip_reason": "missing_token"}) append_event(events_path, "cleanup", "warning", "Skipped Space pause because no HF token is available", {"target_space": target_space, "final_status": final_status or ""}) write_json(run_dir / "cleanup_status.json", cleanup) return cleanup cleanup["space_pause_attempted"] = True append_event(events_path, "cleanup", "started", "Pausing generated Space after terminal non-success outcome", {"target_space": target_space, "final_status": final_status or "", "selected_hardware": selected_hardware or "", "reason": reason or ""}) try: from huggingface_hub import HfApi api = HfApi(token=token) runtime = api.pause_space(repo_id=target_space, token=token) cleanup.update({ "space_pause_success": True, "runtime_repr": repr(runtime)[:1000], "updated_at": now(), }) append_event(events_path, "cleanup", "success", "Generated Space paused after terminal non-success outcome", {"target_space": target_space, "final_status": final_status or "", "selected_hardware": selected_hardware or ""}) except Exception as exc: cleanup.update({ "space_pause_success": False, "error_type": type(exc).__name__, "error": redact_text(str(exc))[:2000], "manual_action_required": True, "updated_at": now(), }) append_event(events_path, "cleanup", "warning", "Could not pause generated Space after terminal outcome", {"target_space": target_space, "error_type": type(exc).__name__, "error": redact_text(str(exc))[:1000]}) write_json(run_dir / "cleanup_status.json", cleanup) return cleanup def run_final_cleanup_if_needed(run_dir: Path, events_path: Path, final_state: dict, *, token: str | None = None) -> dict: """Apply v197.4 final cleanup policy and merge the result into state.json.""" state = dict(final_state or {}) target_space = state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" final_status = state.get("status") or "" selected_hardware = state.get("selected_hardware") or os.environ.get("SELECTED_HARDWARE") or "" runtime_recovery = load_json_if_exists(run_dir / "runtime_recovery.json") if (run_dir / "runtime_recovery.json").exists() else {} if not isinstance(runtime_recovery, dict): runtime_recovery = {} gate = state.get("inference_gate") if isinstance(state.get("inference_gate"), dict) else {} signals = gate.get("implementation_signals") if isinstance(gate.get("implementation_signals"), dict) else {} health_semantic_passed = signals.get("health_semantic_passed") if health_semantic_passed is None: health_semantic_passed = signals.get("health_passed") if runtime_recovery.get("triggered") and runtime_recovery.get("recovery_attempted") and not runtime_recovery.get("recovery_exhausted"): decision = {"should_pause": False, "reason": "runtime_recovery_pending", "final_status": final_status, "selected_hardware": selected_hardware} else: decision = should_pause_generated_space(final_status, target_space, selected_hardware, health_semantic_passed=health_semantic_passed if isinstance(health_semantic_passed, bool) else None) cleanup = { "space_pause_attempted": False, "space_pause_success": False, "target_space": target_space, "final_status": final_status, "selected_hardware": selected_hardware, "decision": decision, "updated_at": now(), } if decision.get("should_pause"): cleanup = pause_generated_space_best_effort(target_space, token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "", run_dir, events_path, final_status=final_status, selected_hardware=selected_hardware, reason=decision.get("reason")) cleanup["decision"] = decision else: cleanup.update({"skipped": True, "skip_reason": decision.get("reason")}) write_json(run_dir / "cleanup_status.json", cleanup) append_event(events_path, "cleanup", "skipped", "No generated Space pause needed for final outcome", {"target_space": target_space, "final_status": final_status, "selected_hardware": selected_hardware, "reason": decision.get("reason")}) state["cleanup"] = cleanup write_json(run_dir / "state.json", state) return cleanup def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"): safe = safe_details(details) append_event(events_path, "failure", "failed", message, safe) existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {} if not isinstance(existing_state, dict): existing_state = {} repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {} if isinstance(repair_outcome, dict) and repair_outcome: safe.setdefault("repair_outcome_status", repair_outcome.get("post_repair_validation") or repair_outcome.get("repair_status") or "") safe.setdefault("repair_failure_type", repair_outcome.get("failure_type") or "") failure_state = { **existing_state, "run_id": os.environ.get("RUN_ID"), "kind": existing_state.get("kind") or "universal_model_card_builder", "status": status, "message": message, "updated_at": now(), "details": safe, "repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {}, } # Preserve the target Space when the worker fails after repository creation. target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" if target_space: failure_state["target_space"] = target_space failure_state["target_space_url"] = f"https://huggingface.co/spaces/{target_space}" write_json(run_dir / "state.json", failure_state) cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, failure_state) failure_state["cleanup"] = cleanup_status write_terminal_status_reconciliation(run_dir, events_path, failure_state, status=status, message=message, details=safe, job_exit_code=1) write_final_summary(run_dir, failure_state, {}, {}, status=status, message=message) live_failure_data = dict(safe) live_failure_data["cleanup"] = cleanup_status write_live_status(run_dir, stage="failure", status=status, message=message, data=live_failure_data) report = f"""# Agentic Space Factory — model Article Reproduction Report Status: **{status}** {message} Target Space: {failure_state.get('target_space_url') or failure_state.get('target_space') or 'not created / unknown'} ```json {json.dumps(safe, indent=2, ensure_ascii=False)} ``` """ (run_dir / "report.md").write_text(report, encoding="utf-8") write_artifact_manifest(run_dir, events_path=events_path, reason="failure") try: publish_eval_record(run_dir, phase="failure", events_path=events_path) except Exception: pass raise SystemExit(1) def run_cmd(cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600): result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) return result.returncode, redact_text(result.stdout) def sync_pi_traces(run_dir: Path, *, emit_event: bool = False, events_path: Path | None = None): """Best-effort incremental copy of Pi session JSONL traces into the run bucket.""" traces_dir = Path.home() / ".pi" / "agent" / "sessions" redacted_dir = run_dir / "traces" / "redacted" redacted_dir.mkdir(parents=True, exist_ok=True) count = 0 if traces_dir.exists(): for path in traces_dir.rglob("*.jsonl"): rel = path.relative_to(traces_dir) text = path.read_text(encoding="utf-8", errors="ignore") target_redacted = redacted_dir / rel target_redacted.parent.mkdir(parents=True, exist_ok=True) target_redacted.write_text(redact_text(text), encoding="utf-8") count += 1 if emit_event and events_path: append_event(events_path, "pi_traces", "running", "Synchronized Pi traces while Pi is running", {"count": count}) return count def run_cmd_streaming( cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600, live_log_path: Path | None = None, events_path: Path | None = None, run_dir: Path | None = None, step: str = "pi_run", trace_phase: str | None = None, ): """Run a command while incrementally writing stdout to the bucket. subprocess.run() only returns output at the end. Pi can work for many minutes, so stream stdout to logs/pi_live_output.txt and periodically copy Pi session JSONL traces into traces/redacted while the process is still alive. """ started = time.monotonic() trace_phase = trace_phase or step last_event = 0.0 last_trace_sync = 0.0 lines: list[str] = [] if live_log_path: live_log_path.parent.mkdir(parents=True, exist_ok=True) live_log_path.write_text("", encoding="utf-8") if run_dir: write_agent_trace_record(run_dir, phase=trace_phase, event="command_started", status="started", message="Pi command started", data={"step": step, "cwd": str(cwd) if cwd else ""}) proc = subprocess.Popen( cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, ) try: assert proc.stdout is not None while True: if timeout and time.monotonic() - started > timeout: proc.kill() message = f"Command timed out after {timeout}s" lines.append(message) if live_log_path: with live_log_path.open("a", encoding="utf-8") as f: f.write(message + "\n") return 124, redact_text("\n".join(lines)) line = proc.stdout.readline() if line: clean = redact_text(line.rstrip("\n")) lines.append(clean) if live_log_path: with live_log_path.open("a", encoding="utf-8") as f: f.write(clean + "\n") now_m = time.monotonic() if events_path and now_m - last_event > 8: output_tail = "\n".join(lines[-20:])[-2000:] append_event(events_path, step, "running", "Pi is still working", {"output_tail": output_tail}) if run_dir: write_agent_trace_record(run_dir, phase=trace_phase, event="stdout_tail", status="running", message="Pi is still working", text=output_tail, data={"step": step}) last_event = now_m if run_dir and now_m - last_trace_sync > 15: sync_pi_traces(run_dir, emit_event=False) last_trace_sync = now_m continue if proc.poll() is not None: remainder = proc.stdout.read() if proc.stdout else "" if remainder: for raw in remainder.splitlines(): clean = redact_text(raw) lines.append(clean) if live_log_path: with live_log_path.open("a", encoding="utf-8") as f: f.write(clean + "\n") break time.sleep(0.2) finally: if run_dir: sync_pi_traces(run_dir, emit_event=False) output = redact_text("\n".join(lines)) if run_dir: write_agent_trace_record(run_dir, phase=trace_phase, event="command_finished", status="success" if int(proc.returncode or 0) == 0 else "failed", message="Pi command finished", text=output[-20000:], data={"step": step, "returncode": int(proc.returncode or 0)}, artifacts=[str(live_log_path.relative_to(run_dir)) if live_log_path and str(live_log_path).startswith(str(run_dir)) else ""]) return int(proc.returncode or 0), output def install_python_deps(events_path: Path): append_event(events_path, "dependencies", "started", "Installing Python worker dependencies") code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0", "requests>=2.31.0"], timeout=600) if code != 0: append_event(events_path, "dependencies", "failed", "Python dependency installation failed", {"output_tail": out[-4000:]}) raise RuntimeError(out) append_event(events_path, "dependencies", "success", "Python worker dependencies installed") def ensure_node(events_path: Path): node = shutil.which("node") npm = shutil.which("npm") if node and npm: _, node_v = run_cmd([node, "--version"], timeout=30) _, npm_v = run_cmd([npm, "--version"], timeout=30) append_event(events_path, "node", "success", "Node/npm already available", {"node": node_v.strip(), "npm": npm_v.strip()}) return append_event(events_path, "node", "started", "Installing nodejs/npm through apt-get") code, out = run_cmd(["bash", "-lc", "apt-get update -qq && apt-get install -y -qq nodejs npm"], timeout=600) if code != 0: append_event(events_path, "node", "failed", "Could not install nodejs/npm", {"output_tail": out[-4000:]}) raise RuntimeError(out) append_event(events_path, "node", "success", "Installed nodejs/npm") def install_pi(events_path: Path): ensure_node(events_path) append_event(events_path, "pi_install", "started", "Installing Pi coding agent from npm") code, out = run_cmd(["npm", "install", "-g", "@mariozechner/pi-coding-agent"], timeout=900) if code != 0: append_event(events_path, "pi_install", "failed", "Pi npm installation failed", {"output_tail": out[-4000:]}) raise RuntimeError(out) code, version = run_cmd(["pi", "--version"], timeout=60) append_event(events_path, "pi_install", "success", "Pi installed", {"version_output": version.strip()[-300:]}) def configure_pi(events_path: Path, model: str): pi_dir = Path.home() / ".pi" / "agent" pi_dir.mkdir(parents=True, exist_ok=True) token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "" if token: os.environ.setdefault("HF_TOKEN", token) os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token) (pi_dir / "auth.json").write_text(json.dumps({"huggingface": {"type": "api_key", "key": token}}, indent=2), encoding="utf-8") (pi_dir / "settings.json").write_text(json.dumps({"model": model, "provider": "huggingface", "autoRun": True, "autoApply": True}, indent=2), encoding="utf-8") append_event(events_path, "pi_config", "success" if token else "warning", "Configured Pi with Hugging Face provider token context", {"model": model, "hf_token_present": bool(token), "token_value": "[REDACTED]" if token else ""}) def normalize_pi_model_name(value: str | None) -> str: raw = (value or "").strip().lower() # Pi/provider traces may report either a full Hub id (moonshotai/Kimi-K2-...) # or only the served model name (Kimi-K2-...). Compare on the model leaf so # provider fallbacks are detected without false positives from missing owners. if "/" in raw: raw = raw.rsplit("/", 1)[-1] return re.sub(r"[^a-z0-9]+", "", raw) def extract_pi_models_from_text(text: str) -> list[str]: """Best-effort extraction of assistant/model names from Pi stdout/session traces.""" if not text: return [] patterns = [ r"\bQwen/[A-Za-z0-9_.-]+", r"\bQwen(?:2(?:\.5)?|3)?[-_/A-Za-z0-9.]*Coder[-_/A-Za-z0-9.]*", r"\bmoonshotai/[A-Za-z0-9_.-]+", r"\bKimi[-_/A-Za-z0-9.]+", r"\bzai-org/[A-Za-z0-9_.-]+", r"\bGLM[-_/A-Za-z0-9.]+", r"\bdeepseek-ai/[A-Za-z0-9_.-]+", r"\bDeepSeek[-_/A-Za-z0-9.]+", r"\bClaude[-_/A-Za-z0-9.]+", r"\bGPT[-_/A-Za-z0-9.]+", r"\b[Mm]odel(?:\\s+used|\\s+selected|\\s*:)?\\s*[:=]?\\s*([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", ] found: list[str] = [] for pattern in patterns: for match in re.finditer(pattern, text, flags=re.IGNORECASE): value = match.group(1) if match.groups() else match.group(0) value = value.strip().strip('",` ') if value and value not in found: found.append(value) return found[:12] def detect_pi_model_resolution(requested_model: str, run_dir: Path, pi_output: str) -> dict: settings_path = Path.home() / ".pi" / "agent" / "settings.json" configured_model = requested_model try: settings = json.loads(settings_path.read_text(encoding="utf-8")) configured_model = settings.get("model") or requested_model except Exception: settings = {"model": requested_model, "read_error": "could_not_read_settings"} corpus = [pi_output or ""] for trace in (run_dir / "traces" / "redacted").rglob("*.jsonl") if (run_dir / "traces" / "redacted").exists() else []: try: corpus.append(trace.read_text(encoding="utf-8", errors="ignore")[:200000]) except Exception: pass observed = extract_pi_models_from_text("\n".join(corpus)) normalized_requested = normalize_pi_model_name(requested_model) normalized_configured = normalize_pi_model_name(configured_model) normalized_observed = [normalize_pi_model_name(x) for x in observed] effective_model = "" for raw, normalized in zip(observed, normalized_observed): if normalized and normalized != normalized_requested and normalized != normalized_configured: effective_model = raw break if not effective_model and observed: effective_model = observed[0] mismatch = bool(effective_model and normalize_pi_model_name(effective_model) not in {normalized_requested, normalized_configured}) payload = { "requested_model": requested_model, "configured_model": configured_model, "observed_models": observed, "effective_model": effective_model or configured_model, "provider": "huggingface", "mismatch": mismatch, "source": "published_pi_traces_after_sync", "settings": settings, } write_json(run_dir / "pi_model_resolution.json", payload) return payload def emit_pi_model_resolution(events_path: Path, resolution: dict): if resolution.get("mismatch"): append_event( events_path, "pi_model_resolution", "warning", "Pi assistant model appears to differ from the requested model", { "requested_model": resolution.get("requested_model"), "configured_model": resolution.get("configured_model"), "effective_model": resolution.get("effective_model"), "observed_models": resolution.get("observed_models", [])[:8], "provider": resolution.get("provider"), }, ) else: append_event( events_path, "pi_model_resolution", "success", "Pi assistant model matched the requested configuration", { "requested_model": resolution.get("requested_model"), "configured_model": resolution.get("configured_model"), "effective_model": resolution.get("effective_model"), "provider": resolution.get("provider"), }, ) def collect_pi_traces(run_dir: Path, events_path: Path): count = sync_pi_traces(run_dir, emit_event=False) write_agent_trace_record(run_dir, phase="initial_build", event="pi_sessions_collected", status="success", message="Synchronized redacted Pi session traces into the run-level trace folder", data={"count": count, "raw_published": False}, artifacts=["traces/redacted/agent_trace.jsonl"]) append_event(events_path, "traces", "success", "Collected redacted Pi traces", {"count": count, "agent_trace": "traces/redacted/agent_trace.jsonl", "raw_published": False}) write_artifact_manifest(run_dir, reason="pi_traces_collected") return count def sanitize_model_id(model_id: str) -> str: model_id = (model_id or DEFAULT_MODEL_ID).strip().replace("https://huggingface.co/", "") model_id = model_id.split("?", 1)[0].strip("/") if not re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", model_id): raise ValueError("MODEL_ID must look like owner/model-name") return model_id def expected_space_subdomain(target_space_id: str) -> str: owner, _, slug = str(target_space_id or "").partition("/") if not owner or not slug: return "" return re.sub(r"[^a-z0-9-]+", "-", f"{owner}-{slug}".lower()).strip("-") def gradio_client_identity_payload(client, target_space_id: str) -> dict: """Best-effort identity check for gradio_client target resolution. v198.26.7 restores the safe v198.25.3 behavior for valid Spaces: the repo identity returned by gradio_client (`space_id`/`space_name` or a repo-like `src`) is authoritative. Hugging Face may expose the running app through an `.hf.space` runtime host/alias that does not exactly match the repo slug, so a host alias is recorded as a warning and must not block validation when the repo identity matches the target. A mismatch is fatal only when the repo identity is absent/different and observed URLs clearly point to another Space. """ from urllib.parse import urlparse def _repo_like(value: str) -> str: raw = str(value or "").strip() if not raw or ".hf.space" in raw: return "" if raw.startswith("https://huggingface.co/spaces/"): raw = raw.split("/spaces/", 1)[1] raw = raw.split("?", 1)[0].strip("/") if raw.count("/") >= 1: owner, slug = raw.split("/", 1)[:2] if owner and slug: return f"{owner}/{slug}".lower() return "" observed_values = [] for attr in ("space_id", "space_name", "src", "app_url", "root_url", "src_url", "api_url"): try: value = getattr(client, attr, None) except Exception: value = None if value: observed_values.append({"attr": attr, "value": str(value)}) expected_repo = str(target_space_id or "").strip() expected_repo_l = expected_repo.lower() expected_subdomain = expected_space_subdomain(expected_repo) authoritative_repo_matches = [] authoritative_repo_mismatches = [] for item in observed_values: attr = item["attr"] repo = _repo_like(item["value"]) if not repo: continue record = {"attr": attr, "value": item["value"], "repo": repo} if expected_repo_l and repo == expected_repo_l: authoritative_repo_matches.append(record) elif attr in {"space_id", "space_name"}: authoritative_repo_mismatches.append(record) observed_hf_space_urls = [] mismatched_hf_space_urls = [] runtime_host_aliases = [] for item in observed_values: value = item["value"] if ".hf.space" not in value: continue parsed = urlparse(value if value.startswith(("http://", "https://")) else "https://" + value.lstrip("/")) host = (parsed.netloc or parsed.path.split("/", 1)[0]).lower() if not host.endswith(".hf.space"): continue subdomain = host.rsplit(".hf.space", 1)[0] record = {"attr": item["attr"], "value": value, "host": host, "subdomain": subdomain} observed_hf_space_urls.append(record) if expected_subdomain and subdomain != expected_subdomain: if authoritative_repo_matches: runtime_host_aliases.append(record) else: mismatched_hf_space_urls.append(record) mismatch = False reason = "" if authoritative_repo_matches: mismatch = False reason = "authoritative_repo_match" elif authoritative_repo_mismatches: mismatch = True reason = "authoritative_repo_mismatch" elif mismatched_hf_space_urls and not any(item.get("subdomain") == expected_subdomain for item in observed_hf_space_urls): mismatch = True reason = "hf_space_url_points_elsewhere_without_repo_match" else: mismatch = False reason = "no_conflicting_identity_observed" return { "schema_version": "gradio_client_identity.v198_26_7", "target_space_id": expected_repo, "expected_subdomain": expected_subdomain, "observed": observed_values, "authoritative_repo_matches": authoritative_repo_matches, "authoritative_repo_mismatches": authoritative_repo_mismatches, "observed_hf_space_urls": observed_hf_space_urls, "mismatched_hf_space_urls": mismatched_hf_space_urls if mismatch else [], "runtime_host_aliases": runtime_host_aliases, "runtime_host_alias_detected": bool(runtime_host_aliases), "mismatch": bool(mismatch), "identity_reason": reason, "failure_owner": "factory_validation_client" if mismatch else "", "failure_class": "space_identity_mismatch" if mismatch else "", } def write_gradio_client_identity(client, target_space_id: str, run_dir: Path, events_path: Path, *, phase: str) -> dict: payload = gradio_client_identity_payload(client, target_space_id) payload["phase"] = phase write_json(run_dir / "tests" / f"gradio_client_identity_{phase}.json", payload) if payload.get("mismatch"): append_event(events_path, "space_identity", "failed", "Gradio client resolved a different Space than the run target", payload) elif payload.get("runtime_host_alias_detected"): append_event(events_path, "space_identity", "warning", "Gradio client uses a Hugging Face runtime host alias for the target Space", payload) return payload def make_gradio_client(target_space_id: str, token: str, timeout_s: int | float | None = None): import inspect from gradio_client import Client params = inspect.signature(Client).parameters kwargs = {} if timeout_s and "httpx_kwargs" in params: kwargs["httpx_kwargs"] = {"timeout": float(timeout_s)} if "token" in params: return Client(target_space_id, token=token, **kwargs) if "hf_token" in params: return Client(target_space_id, hf_token=token, **kwargs) if "api_key" in params: return Client(target_space_id, api_key=token, **kwargs) if "headers" in params: return Client(target_space_id, headers={"Authorization": f"Bearer {token}"}, **kwargs) return Client(target_space_id, **kwargs) def api_names_from_schema(schema) -> list[str]: names: list[str] = [] if isinstance(schema, dict): endpoints = schema.get("named_endpoints") or schema.get("endpoints") or {} if isinstance(endpoints, dict): for key, value in endpoints.items(): if isinstance(key, str) and key.startswith("/"): names.append(key) if isinstance(value, dict): api_name = value.get("api_name") if isinstance(api_name, str) and api_name.startswith("/"): names.append(api_name) if isinstance(schema.get("dependencies"), list): for dep in schema["dependencies"]: if isinstance(dep, dict): api_name = dep.get("api_name") if isinstance(api_name, str): names.append(api_name if api_name.startswith("/") else f"/{api_name}") return list(dict.fromkeys(names)) def normalize_api_name(name: str | None) -> str: value = (name or "").strip() if not value: return "/generate" return value if value.startswith("/") else "/" + value def default_smoke_args(expected_output_type: str) -> list: expected = (expected_output_type or "any").lower() if expected == "image": return ["a cinematic robot cat astronaut, detailed, studio lighting"] if expected == "video": return ["a short cinematic shot of a robot cat astronaut walking on the moon"] if expected == "audio": return ["A calm voice saying hello from Agentic Space Factory."] return ["Explain the concept of agentic AI in one paragraph."] def endpoint_schema_for_api(schema, api_name: str): target = normalize_api_name(api_name) alternatives = {target, target.lstrip("/")} found = None def walk(obj): nonlocal found if found is not None: return if isinstance(obj, dict): for key, value in obj.items(): if isinstance(key, str) and key in alternatives and isinstance(value, (dict, list)): found = value return value = obj.get("api_name") or obj.get("apiName") if isinstance(value, str) and normalize_api_name(value) == target: found = obj return for value in obj.values(): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(schema) return found def endpoint_parameter_names(endpoint) -> list[str]: names: list[str] = [] def add(value): if not isinstance(value, str): return cleaned = value.strip() if cleaned and cleaned not in names: names.append(cleaned) def from_parameter(param): if isinstance(param, dict): for key in ["parameter_name", "parameterName", "name", "label"]: add(param.get(key)) component = param.get("component") or param.get("component_type") if isinstance(component, dict): for key in ["label", "name"]: add(component.get(key)) elif isinstance(param, str): add(param) def walk(obj): if isinstance(obj, dict): params = obj.get("parameters") or obj.get("inputs") if isinstance(params, list): for param in params: from_parameter(param) for value in obj.values(): if isinstance(value, (dict, list)): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(endpoint) return names def endpoint_parameters_for_smoke(endpoint) -> list[dict]: """Return Gradio endpoint parameters in call order with names/defaults. gradio_client schema formats vary across versions. Keep this permissive and preserve ordering because client.predict uses positional arguments. """ params = None def find_params(obj): nonlocal params if params is not None: return if isinstance(obj, dict): for key in ("parameters", "inputs"): value = obj.get(key) if isinstance(value, list) and value: params = value return for value in obj.values(): if isinstance(value, (dict, list)): find_params(value) elif isinstance(obj, list): for item in obj: find_params(item) find_params(endpoint) out = [] for i, param in enumerate(params or []): if isinstance(param, dict): component = param.get("component") or param.get("component_type") or {} if not isinstance(component, dict): component = {} name = ( param.get("parameter_name") or param.get("parameterName") or param.get("name") or param.get("label") or component.get("label") or component.get("name") or f"arg{i}" ) default = None has_default = False for key in ("parameter_default", "default", "value"): if key in param: default = param.get(key) has_default = True break if not has_default: for key in ("value", "default"): if key in component: default = component.get(key) has_default = True break if isinstance(default, str) and default.strip().lower() in {"none", "null", ""} and not param.get("parameter_has_default"): has_default = False default = None if "parameter_has_default" in param: has_default = bool(param.get("parameter_has_default")) raw_choices = param.get("choices") or param.get("options") or (param.get("api_info") or {}).get("choices") or component.get("choices") or [] choices = raw_choices if isinstance(raw_choices, list) else [] required = bool(param.get("required", not has_default)) out.append({ "name": str(name).strip(), "required": required, "has_default": has_default, "default": default, "component": component.get("type") or component.get("component") or param.get("component_type") or param.get("type") or "", "choices": choices, }) elif isinstance(param, str): out.append({"name": param.strip() or f"arg{i}", "required": True, "has_default": False, "default": None, "component": ""}) return out def smoke_value_for_parameter(param: dict, expected_output_type: str): name = str(param.get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") component = str(param.get("component") or "").lower() expected = str(expected_output_type or "").strip().lower() file_kind = gradio_file_kind_for_param(param) if file_kind == "image": return "__ASF_SMOKE_IMAGE__" if file_kind == "file": return "__ASF_SMOKE_FILE__" if file_kind == "audio": return "https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac" if file_kind == "video": return "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/a_video.mp4" # For video models, prefer a cheap smoke payload over UI defaults. The goal # is to prove the pipeline runs, not to benchmark the best demo quality. if expected == "video": if name in {"height", "image_height"} or name.endswith("_height"): return 320 if name in {"width", "image_width"} or name.endswith("_width"): return 512 if name in {"num_frames", "frames", "n_frames", "frame_count"} or "num_frames" in name: return 17 if name in {"frame_rate", "fps"}: return 8 if name in {"num_inference_steps", "inference_steps", "steps", "num_steps"} or "step" in name: return 6 if param.get("has_default"): return param.get("default") if name in {"prompt", "text", "query", "input", "instruction"} or "prompt" in name and "negative" not in name: return default_smoke_args(expected_output_type)[0] if name in {"negative_prompt", "negative", "negative_text"} or "negative" in name: return "" if name in {"seed", "random_seed"} or name.endswith("_seed"): return 42 if name in {"height", "width", "image_height", "image_width"}: return 512 if name in {"num_inference_steps", "inference_steps", "steps", "num_steps"} or "step" in name: return 4 if name in {"guidance_scale", "cfg_scale", "scale"} or "guidance" in name: return 0.0 if "bool" in component or name.startswith("enable_") or name.startswith("use_"): return False if "number" in component or "slider" in component: return 0 return "" def _choice_scalar(choice): if isinstance(choice, dict): for key in ("value", "label", "name"): if key in choice: return choice.get(key) return next(iter(choice.values()), choice) if choice else choice if isinstance(choice, (list, tuple)) and choice: return choice[0] return choice def _coerce_to_schema_choice(value, param: dict): choices = param.get("choices") or [] if not isinstance(choices, list) or not choices: return value, False for raw_choice in choices: choice = _choice_scalar(raw_choice) if value == choice: return choice, True value_text = str(value).strip() for raw_choice in choices: choice = _choice_scalar(raw_choice) if str(choice).strip() == value_text: return choice, True return value, False def coerce_smoke_value(value, param: dict): name = str(param.get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") component = str(param.get("component") or "").lower() if value is None: return value choice_value, matched_choice = _coerce_to_schema_choice(value, param) if matched_choice: return choice_value if isinstance(value, str): stripped = value.strip() if name in {"height", "width", "image_height", "image_width", "seed", "random_seed", "num_inference_steps", "inference_steps", "steps", "num_steps"} or name.endswith("_seed") or "step" in name: try: return int(float(stripped)) except Exception: return value if name in {"guidance_scale", "cfg_scale", "scale"} or "guidance" in name or "slider" in component or "number" in component: try: number = float(stripped) return int(number) if number.is_integer() and name not in {"guidance_scale", "cfg_scale", "scale"} else number except Exception: return value return value def coerce_smoke_args_to_schema_choices(args: list, params: list[dict]) -> tuple[list, list[dict]]: corrected = list(args or []) changes: list[dict] = [] for index, param in enumerate(params or []): if index >= len(corrected): break before = corrected[index] after, matched = _coerce_to_schema_choice(before, param) if matched and after != before: corrected[index] = after changes.append({ "index": index, "name": str(param.get("name") or f"arg{index}"), "from": before, "to": after, "reason": "schema_choice_type_match", }) return corrected, changes def is_schema_choice_type_error(error: Exception | str) -> bool: text = str(error or "").lower() return "not in the list of choices" in text or ("value:" in text and "choices" in text) def parse_gradio_choice_error(error: Exception | str) -> dict: """Extract a Gradio choice/type mismatch from messages like: Value: 1024 is not in the list of choices: ['512', '768', '1024'] The automatic validator used to mark this retryable but could miss the actual retry when the contract payload did not carry choices. Keep this parser conservative and only use it to coerce to an exact listed choice. """ text = str(error or "") if not is_schema_choice_type_error(text): return {"matched": False} value = None choices = [] m = re.search(r"Value:\s*(.*?)\s+is not in the list of choices:\s*(\[.*?\])", text, flags=re.IGNORECASE | re.DOTALL) if m: value = m.group(1).strip().strip("'\"") raw_choices = m.group(2).strip() try: import ast parsed = ast.literal_eval(raw_choices) if isinstance(parsed, list): choices = parsed except Exception: choices = [c.strip().strip("'\"") for c in raw_choices.strip("[]").split(",") if c.strip()] if value is None: m_value = re.search(r"Value:\s*([^\n]+?)\s+(?:is|not)", text, flags=re.IGNORECASE) if m_value: value = m_value.group(1).strip().strip("'\"") return {"matched": bool(value is not None), "value": value, "choices": choices, "raw_error": text[:2000]} def coerce_smoke_args_from_choice_error(args: list, error: Exception | str, params: list[dict] | None = None) -> tuple[list, list[dict]]: """Repair validator-owned choice mismatches even when schema params lacked choices. This restores the golden-path behavior for Gradio Dropdown/Radio choices where the app exposes string choices such as "1024" but the factory payload sends numeric 1024. The retry must use the exact choice object returned by Gradio. """ parsed = parse_gradio_choice_error(error) if not parsed.get("matched"): return list(args or []), [] wanted = str(parsed.get("value") or "").strip() choices = parsed.get("choices") if isinstance(parsed.get("choices"), list) else [] exact_choice = None for choice in choices: if str(_choice_scalar(choice)).strip() == wanted: exact_choice = _choice_scalar(choice) break corrected = list(args or []) changes = [] # Prefer a parameter whose schema choices also contain the rejected value. params = params or [] candidate_indexes = [] for index, param in enumerate(params): if index >= len(corrected): break param_choices = param.get("choices") or [] if isinstance(param, dict) else [] if param_choices and any(str(_choice_scalar(c)).strip() == wanted for c in param_choices): candidate_indexes.append(index) if not candidate_indexes: candidate_indexes = [i for i, value in enumerate(corrected) if str(value).strip() == wanted] for index in candidate_indexes[:1]: before = corrected[index] after = exact_choice if exact_choice is not None else wanted if before != after: corrected[index] = after name = str((params[index].get("name") if index < len(params) and isinstance(params[index], dict) else None) or f"arg{index}") changes.append({ "index": index, "name": name, "from": before, "to": after, "reason": "parsed_gradio_choice_error_exact_choice", "error_value": parsed.get("value"), "choices": choices[:20], }) break return corrected, changes def gradio_file_kind_for_param(param: dict) -> str: """Return image/file/video/audio when a Gradio schema parameter is file-backed.""" if not isinstance(param, dict): return "" name = str(param.get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") component = str(param.get("component") or "").lower() haystack = f"{name} {component}" if any(token in haystack for token in ["image", "gr.image"]): return "image" if any(token in haystack for token in ["video", "gr.video"]): return "video" if any(token in haystack for token in ["audio", "gr.audio"]): return "audio" if any(token in haystack for token in ["file", "upload", "filepath", "path"]): return "file" return "" def ensure_smoke_input_file(run_dir: Path, kind: str) -> str: smoke_dir = run_dir / "tests" / "smoke_inputs" smoke_dir.mkdir(parents=True, exist_ok=True) if kind == "image": path = smoke_dir / "asf_smoke_image.png" if not path.exists(): # 1x1 valid PNG. Small and deterministic; avoids remote URL schema drift. path.write_bytes(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=")) return str(path) path = smoke_dir / "asf_smoke_file.txt" if not path.exists(): path.write_text("Agentic Space Factory smoke input file.\n", encoding="utf-8") return str(path) def _is_url_like(value) -> bool: text = str(value or "").strip().lower() return text.startswith(("http://", "https://", "hf://", "s3://", "gs://", "data:")) def resolve_smoke_file_source(run_dir: Path, source, kind: str) -> tuple[str, dict]: """Resolve relative Gradio file/image smoke inputs before handle_file(). v198.26.2: generated apps commonly include example media such as `sample_image.png` in runtime_upload_payload/. Passing the bare relative string to gradio_client makes the validator fail before it reaches the Space. Resolve local payload/generated/test files first; only leave URLs untouched. """ info = {"schema_version": "smoke_file_source_resolution.v198_26_2", "kind": kind} if isinstance(source, dict): info.update({"action": "already_filedata_or_dict", "source_type": "dict"}) return source, info text = str(source or "").strip() placeholder = text in {"", "__ASF_SMOKE_IMAGE__", "__ASF_SMOKE_FILE__"} if kind in {"image", "file"} and (source is None or placeholder): fallback = ensure_smoke_input_file(run_dir, kind) info.update({"action": "created_fallback_smoke_input", "source": text, "resolved_path": fallback}) return fallback, info if not text or _is_url_like(text): info.update({"action": "url_or_empty_passthrough", "source": text}) return text, info path = Path(text).expanduser() candidates = [] if path.is_absolute(): candidates.append(path) else: candidates.extend([ run_dir / "runtime_upload_payload" / text, run_dir / "generated" / text, run_dir / text, run_dir / "tests" / "smoke_inputs" / text, ]) if path.exists(): candidates.append(path) for candidate in candidates: try: if candidate.exists() and candidate.is_file(): resolved = str(candidate.resolve()) info.update({"action": "resolved_local_file", "source": text, "resolved_path": resolved}) return resolved, info except Exception: continue info.update({ "action": "missing_local_file", "source": text, "searched_paths": [str(c) for c in candidates[:12]], }) raise FileNotFoundError( "ASF smoke input materialization failed: " f"{kind} input {text!r} was not found in runtime_upload_payload, generated, run root, or tests/smoke_inputs." ) def prepare_gradio_file_inputs(args: list, params: list[dict], run_dir: Path) -> tuple[list, list[dict]]: """Wrap file-like smoke inputs with gradio_client.handle_file(). Passing raw URL strings to gr.Image/File/Video/Audio can raise Pydantic ImageData/FileData validation errors in recent Gradio stacks. The worker owns that payload construction, so these are factory-validator errors, not generated-app repairs. """ from gradio_client import handle_file resolved = list(args or []) conversions: list[dict] = [] for index, param in enumerate(params or []): if index >= len(resolved): break kind = gradio_file_kind_for_param(param) if not kind: continue before = resolved[index] if isinstance(before, dict) and (before.get("meta") or {}).get("_type") == "gradio.FileData": continue try: source, resolution = resolve_smoke_file_source(run_dir, before, kind) resolved[index] = handle_file(str(source)) conversions.append({ "index": index, "name": str(param.get("name") or f"arg{index}"), "kind": kind, "from_type": type(before).__name__, "source": str(source)[:500], "action": "handle_file", "resolution": resolution, }) except Exception as exc: conversions.append({ "index": index, "name": str(param.get("name") or f"arg{index}"), "kind": kind, "from_type": type(before).__name__, "source": str(before)[:500], "action": "handle_file_failed", "error": str(exc)[:1000], }) raise return resolved, conversions def _smoke_param_name(param: dict) -> str: return str((param or {}).get("name") or "").strip().lower().replace(" ", "_").replace("-", "_") def _minimal_numeric_value(name: str, value, *, expected_output_type: str = ""): """Return a safer minimal smoke value for expensive generation parameters. v198.26.7 introduces a second validation level: a minimal demo smoke can prove the Space is usable after a canonical/promise smoke OOM, but it must not be promoted to full inference success. """ output = str(expected_output_type or "").lower() try: current = float(value) except Exception: current = None if any(token in name for token in ("width", "height", "resolution", "size")): if current is None: return value return int(min(max(current, 1), 512)) if any(token in name for token in ("num_inference_steps", "inference_steps", "steps", "num_steps", "sampling_steps")) or name == "steps": if current is None: return value return int(min(max(current, 1), 4)) if any(token in name for token in ("duration", "seconds", "length")): if current is None: return value cap = 2 if output in {"video", "audio"} else 4 return int(min(max(current, 1), cap)) if any(token in name for token in ("frames", "num_frames")): if current is None: return value return int(min(max(current, 1), 8)) if any(token in name for token in ("guidance", "cfg_scale", "scale")): if current is None: return value return float(min(max(current, 0.0), 3.0)) return value def build_minimal_demo_smoke_args(args: list, params: list[dict], expected_output_type: str = "") -> tuple[list, list[dict]]: """Build a cheaper validation payload from the canonical smoke args. The canonical smoke remains the only path to `full_inference_success`. This helper is used after canonical OOM to check whether the deployed demo is at least usable with reduced parameters. """ minimal = list(args or []) changes: list[dict] = [] for index, param in enumerate(params or []): if index >= len(minimal): break name = _smoke_param_name(param) before = minimal[index] after = before if isinstance(before, bool): if any(token in name for token in ("upsample", "hires", "high_res", "enhance", "refiner", "randomize")): after = False elif isinstance(before, (int, float)) or (isinstance(before, str) and before.strip().replace('.', '', 1).isdigit()): after = _minimal_numeric_value(name, before, expected_output_type=expected_output_type) elif isinstance(before, str): lower = before.strip().lower() if any(token in name for token in ("mode", "quality", "preset")) and lower in {"quality", "high", "best", "default", "upscale", "upsample"}: after = "Fast" if before != "Fast" else before if after != before: minimal[index] = after changes.append({"index": index, "name": str(param.get("name") or f"arg{index}"), "from": before, "to": after, "reason": "minimal_demo_smoke"}) return minimal, changes def minimal_smoke_status_payload(payload: dict) -> dict: """Mark a successful minimal smoke without claiming full promise success.""" out = dict(payload or {}) out["schema_version"] = "generation_smoke_result.v198_26_7" out["status"] = "demo_usable_smoke_passed" out["demo_usable_smoke_passed"] = True out["canonical_promise_smoke_passed"] = False out["full_promise_smoke_passed"] = False out["validation_level"] = "minimal_demo_smoke" out["message"] = "A reduced minimal smoke generated output after the canonical promise smoke failed or OOMed. The demo is usable, but the full promise was not verified." return out def build_generation_smoke_args(endpoint, expected_output_type: str) -> tuple[list, list[dict]]: params = endpoint_parameters_for_smoke(endpoint) if not params: return default_smoke_args(expected_output_type), [] return [coerce_smoke_value(smoke_value_for_parameter(param, expected_output_type), param) for param in params], params def read_inference_contract(workspace: Path | None) -> dict: if not workspace: return {} path = workspace / "INFERENCE_CONTRACT.json" if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def read_demo_quality_contract(workspace: Path | None) -> dict: """Read Pi's demo-quality contract when present. v198.13: this lets the Worker validate the demo with Pi's canonical, model-card-aligned example instead of only guessing a generic payload. """ if not workspace: return {} path = workspace / "DEMO_QUALITY_CONTRACT.json" if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def _as_bool_false(value) -> bool: if value is False: return True if isinstance(value, str): return value.strip().lower() in {"false", "no", "0", "off"} return False def _read_workspace_json(workspace: Path | None, filename: str) -> dict: if not workspace: return {} path = workspace / filename if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8", errors="replace")) return data if isinstance(data, dict) else {} except Exception: return {} def contract_declares_no_full_inference(workspace: Path | None) -> dict: """Return a structured reason when Pi explicitly declared boot-only/no inference. v191.2: A machine-readable no-inference contract must be respected by the worker. In that case automatic generation smoke is skipped instead of inventing /generate fallback arguments that cannot exist. """ contract = read_inference_contract(workspace) blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json") validation_level = str(contract.get("validation_level") or "").strip().lower().replace("_", "-") primary = normalize_api_name(str(contract.get("primary_api_name") or "")) if contract else "" contract_full_false = _as_bool_false(contract.get("full_inference_implemented")) blocker_source = str(blockers.get("source") or "") if isinstance(blockers, dict) else "" heuristic_blocker_json = blocker_source == "worker_heuristic_from_PI_SUMMARY_or_app.py" blocker_full_false = False if heuristic_blocker_json else _as_bool_false(blockers.get("full_inference_implemented")) validation_boot_only = validation_level in {"boot-only", "health-only", "diagnostic-only", "info-only"} primary_absent = not primary or primary == "/health" blockers_count = blockers.get("blockers_count") if not isinstance(blockers_count, int): raw_blockers = blockers.get("blockers") blockers_count = len(raw_blockers) if isinstance(raw_blockers, list) else 0 declared = bool(contract_full_false or blocker_full_false or (validation_boot_only and primary_absent)) return { "declared": declared, "source": "inference_contract" if contract_full_false or validation_boot_only else "technical_blockers" if blocker_full_false else "", "full_inference_implemented": False if (contract_full_false or blocker_full_false) else contract.get("full_inference_implemented"), "validation_level": contract.get("validation_level"), "primary_api_name": contract.get("primary_api_name"), "blockers_count": blockers_count, "contract_present": bool(contract), "technical_blockers_present": bool(blockers), "technical_blockers_heuristic_only": bool(heuristic_blocker_json), } def _as_bool_true(value) -> bool: if value is True: return True if isinstance(value, str): return value.strip().lower() in {"true", "yes", "1", "on", "success", "passed", "ok"} return bool(value) if isinstance(value, (int, float)) else False def build_promise_validation_status(workspace: Path | None, validation: dict | None, generation_smoke: dict | None, *, status_hint: str = "") -> dict: """Separate app boot validation from model-card promise fulfillment. v198.20: a diagnostic/manual-action Space can boot and respond to Gradio, but that is not the same as fulfilling the model-card promise. This helper provides a stable machine-readable verdict that prevents hydrated Run Explorer/Active Run views from promoting diagnostic-only blockers to Success. """ contract = read_inference_contract(workspace) demo_contract = read_demo_quality_contract(workspace) blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json") validation = validation if isinstance(validation, dict) else {} generation_smoke = generation_smoke if isinstance(generation_smoke, dict) else {} status_hint = str(status_hint or "").strip() contract_no_full = contract_declares_no_full_inference(workspace) validation_level = str(contract.get("validation_level") or demo_contract.get("validation_level") or "").strip().lower().replace("_", "-") fallback_only = _as_bool_true(contract.get("fallback_or_diagnostic_only")) or _as_bool_true(demo_contract.get("fallback_or_diagnostic_only")) real_inference_false = _as_bool_false(contract.get("real_inference_implemented")) or _as_bool_false(demo_contract.get("real_inference_implemented")) full_inference_false = _as_bool_false(contract.get("full_inference_implemented")) or _as_bool_false(demo_contract.get("full_inference_implemented")) manual_required = _as_bool_true(contract.get("manual_hardware_required")) or "manual" in validation_level or "manual" in status_hint.lower() diagnostic_level = any(marker in validation_level for marker in ("diagnostic", "boot-only", "health-only", "info-only", "manual-hardware")) official_only_text = " ".join([ str(contract.get("reason") or ""), str(demo_contract.get("model_card_promise") or ""), str(demo_contract.get("primary_user_flow") or ""), str(demo_contract.get("promise_fulfillment_risk") or ""), str(blockers.get("reason") or ""), ]).lower() official_only = any(marker in official_only_text for marker in ( "official demo", "official space", "does not generate real", "does not load model", "not generate real", "real inference is not", "full inference deferred", )) health_passed = validation_health_passed(validation) or bool(generation_smoke.get("health_passed")) smoke_status_lower = str(generation_smoke.get("status") or "").lower() smoke_passed = smoke_status_lower == "success" or _as_bool_true(generation_smoke.get("ok")) demo_usable_smoke = smoke_status_lower == "demo_usable_smoke_passed" or _as_bool_true(generation_smoke.get("demo_usable_smoke_passed")) no_full_reason = contract_no_full.get("declared") or fallback_only or real_inference_false or full_inference_false or diagnostic_level or official_only or manual_required if manual_required: promise_status = "deferred_manual_hardware" ui_status = "manual_hardware_required" badge = "Manual hardware required" reason = "Full inference is deferred until the user selects suitable Space hardware." elif no_full_reason: promise_status = "not_fulfilled_diagnostic_only" ui_status = "technical_blocker_boot_only" if health_passed or smoke_passed else "technical_blocker" badge = "Diagnostic Space" reason = "The Space may boot/respond, but the model-card promise is not fulfilled because real/full inference is not implemented." elif smoke_passed: promise_status = "fulfilled" ui_status = "full_inference_success" badge = "Full inference" reason = "A live generation smoke test passed for the promised demo flow." elif demo_usable_smoke: promise_status = "not_verified_demo_usable" ui_status = "demo_usable_full_promise_not_verified" badge = "Demo usable" reason = "A reduced minimal smoke generated output, but the canonical/full promise smoke was not verified." elif health_passed: promise_status = "not_verified" smoke_owner = str(generation_smoke.get("failure_owner") or "") if isinstance(generation_smoke, dict) else "" smoke_class = str(generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "") if isinstance(generation_smoke, dict) else "" if smoke_owner == "factory_validation_client" or smoke_class == "smoke_schema_error": ui_status = "interactive_app_available_smoke_failed" badge = "Manual test required" reason = "The Space boots, but ASF could not verify generation because the automatic smoke payload did not match the Gradio schema." else: ui_status = "partial_validation" badge = "Health only" reason = "The Space boots, but the promised generation flow was not verified." else: promise_status = "not_verified" ui_status = status_hint or "unknown" badge = "Not verified" reason = "No successful app boot or promise validation was observed." app_boot_status = "passed" if health_passed or smoke_passed else "failed" if str(validation.get("status") or "").lower() in {"failed", "error", "timeout"} else "unknown" return { "schema_version": "promise_validation.v198_20", "app_boot_validation_status": app_boot_status, "promise_validation_status": promise_status, "promise_fulfilled": promise_status == "fulfilled", "ui_status": ui_status, "ui_badge": badge, "reason": reason, "health_passed": bool(health_passed), "generation_smoke_passed": bool(smoke_passed), "diagnostic_only": bool(no_full_reason and not manual_required), "manual_hardware_required": bool(manual_required), "fallback_or_diagnostic_only": bool(fallback_only), "real_inference_implemented": False if real_inference_false else contract.get("real_inference_implemented", demo_contract.get("real_inference_implemented")), "full_inference_implemented": False if (full_inference_false or contract_no_full.get("declared")) else contract.get("full_inference_implemented", demo_contract.get("full_inference_implemented")), "validation_level": contract.get("validation_level") or demo_contract.get("validation_level") or "", "contract_declares_no_full_inference": contract_no_full, } def write_contract_skipped_generation_smoke(run_dir: Path, events_path: Path, expected_output_type: str, target_space_id: str, reason: dict | None = None) -> dict: reason = reason or {} payload = { "status": "skipped", "skip_reason": "contract_declared_no_full_inference", "reason": "Pi declared that full inference is not implemented; automatic generation smoke was skipped.", "target_space": target_space_id, "api_name": None, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "contract_aware_skip": True, "contract_reason": reason, "next_action": "Full inference is blocked. No generation endpoint exists. Review TECHNICAL_BLOCKERS.json / PI_SUMMARY.md or provide a dedicated implementation and hardware plan.", **measured_zero_gpu_recommendation(None), } record_generation_smoke_result(run_dir, payload, phase="generation_smoke") write_json(run_dir / "tests" / "payload_source.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "selected_api_name": None, "parent_smoke_payload_used": False}) write_json(run_dir / "tests" / "validation_engine.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "resolved_request_required_before_predict": False, "skipped_by_contract": True}) write_json(run_dir / "tests" / "resolved_validation_request.json", {"api_name": None, "test_args": [], "test_kwargs": {}, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "resolved_request_required_before_predict": False}) write_live_status(run_dir, stage="generation_smoke", status="skipped", message="Generation smoke skipped because Pi declared full inference unavailable", data=payload) append_event(events_path, "generation_smoke", "skipped", "Generation smoke skipped because Pi declared no full inference endpoint", payload) return payload def _param_semantic_name(param: dict, index: int, expected_output_type: str = "") -> str: """Best-effort semantic name for anonymous Gradio `param_N` schemas. v198.26.7: gradio_client can expose Textbox/Number inputs as param_0, param_1... while Pi's canonical smoke example uses semantic keys like prompt/width/height/steps/seed. A positional anonymous fallback prevents a valid prompt from being replaced by the schema-generated empty string. """ raw = str((param or {}).get("name") or "").strip().lower().replace("-", "_").replace(" ", "_") component = str((param or {}).get("component") or "").strip().lower() if raw and not re.fullmatch(r"param_\d+|arg\d+", raw): return raw expected = str(expected_output_type or "").strip().lower() if index == 0 and any(token in component for token in ("textbox", "text", "str")): return "prompt" if index == 0 and expected in {"text", "audio", "video", "image"}: return "prompt" common = ["prompt", "width", "height", "num_inference_steps", "seed", "guidance_scale"] if expected == "audio": common = ["prompt", "duration", "steps", "seed", "guidance_scale"] if expected == "video": common = ["prompt", "width", "height", "num_frames", "num_inference_steps", "seed"] if index < len(common): return common[index] return raw or f"arg{index}" def _canonical_key_aliases(name: str) -> list[str]: name = str(name or "").strip().lower().replace("-", "_").replace(" ", "_") aliases = { "prompt": ["prompt", "text", "query", "input", "instruction", "caption"], "negative_prompt": ["negative_prompt", "negative", "negative_text"], "width": ["width", "image_width", "w"], "height": ["height", "image_height", "h"], "num_inference_steps": ["num_inference_steps", "inference_steps", "steps", "num_steps", "sampling_steps"], "steps": ["steps", "num_inference_steps", "inference_steps", "num_steps", "sampling_steps"], "seed": ["seed", "random_seed"], "guidance_scale": ["guidance_scale", "cfg_scale", "scale", "guidance"], "duration": ["duration", "seconds", "length", "audio_length"], "num_frames": ["num_frames", "frames", "frame_count"], } return aliases.get(name, [name, name.replace("_", " "), name.replace("_", "-")]) def _canonical_dict_value_for_param(raw_args: dict, param: dict, index: int, generated: list, expected_output_type: str) -> tuple[object, str]: semantic = _param_semantic_name(param, index, expected_output_type) raw_lower = {str(k).strip().lower().replace("-", "_").replace(" ", "_"): k for k in raw_args.keys()} for candidate in _canonical_key_aliases(semantic): key = raw_lower.get(candidate.replace("-", "_").replace(" ", "_")) if key is not None: return raw_args[key], f"semantic:{candidate}" name = str((param or {}).get("name") or f"arg{index}") for candidate in (name, name.replace("_", " "), name.replace("_", "-"), name.lower(), name.lower().replace("_", " "), name.lower().replace("_", "-")): if candidate in raw_args: return raw_args[candidate], f"schema_name:{candidate}" values = list(raw_args.values()) if index < len(values) and re.fullmatch(r"param_\d+|arg\d+", str(name).strip().lower()): return values[index], "anonymous_positional_dict_order" return (generated[index] if index < len(generated) else smoke_value_for_parameter(param, expected_output_type)), "schema_generated_fallback" def detect_required_text_payload_resolution_issue(args: list, params: list[dict], canonical_example: dict | None = None) -> dict: """Detect when ASF resolved a required text prompt to empty before predict().""" non_empty_canonical = [] if isinstance(canonical_example, dict): raw_inputs, _ = _raw_inputs_from_canonical_smoke(canonical_example) if isinstance(raw_inputs, dict): for key, value in raw_inputs.items(): if isinstance(value, str) and value.strip(): non_empty_canonical.append({"key": str(key), "value_preview": value[:120]}) elif isinstance(raw_inputs, list): for i, value in enumerate(raw_inputs): if isinstance(value, str) and value.strip(): non_empty_canonical.append({"index": i, "value_preview": value[:120]}) for index, param in enumerate(params or []): if index >= len(args or []): continue name = str((param or {}).get("name") or f"arg{index}").strip() semantic = _param_semantic_name(param, index) component = str((param or {}).get("component") or "").lower() required = bool((param or {}).get("required", False)) value = args[index] is_text = any(token in component for token in ("textbox", "text", "str")) or semantic in {"prompt", "text", "query", "instruction"} if required and is_text and isinstance(value, str) and not value.strip() and non_empty_canonical: return { "schema_version": "smoke_payload_resolution_guard.v198_26_7", "detected": True, "failure_owner": "factory_validation_client", "failure_class": "smoke_payload_resolution_failed", "failure_type": "required_text_resolved_empty", "parameter_index": index, "parameter_name": name, "semantic_name": semantic, "canonical_non_empty_inputs": non_empty_canonical[:5], "recommended_action": "Fix canonical smoke dict-to-args resolution before calling gradio_client.predict; do not call the Space with an empty required prompt.", } return {"schema_version": "smoke_payload_resolution_guard.v198_26_7", "detected": False} def _canonical_smoke_dict_from_contracts(inference_contract: dict | None, demo_quality_contract: dict | None) -> tuple[dict | None, str]: """Return the preferred canonical smoke example and its source. Preference order for v198.21: 1. DEMO_QUALITY_CONTRACT.canonical_smoke_example (demo-quality promise) 2. INFERENCE_CONTRACT.canonical_smoke_example 3. legacy INFERENCE_CONTRACT.smoke_test """ demo_quality_contract = demo_quality_contract if isinstance(demo_quality_contract, dict) else {} inference_contract = inference_contract if isinstance(inference_contract, dict) else {} demo_example = demo_quality_contract.get("canonical_smoke_example") if isinstance(demo_example, dict) and demo_example: return demo_example, "demo_quality_contract_canonical_smoke_example" inference_example = inference_contract.get("canonical_smoke_example") if isinstance(inference_example, dict) and inference_example: return inference_example, "inference_contract_canonical_smoke_example" legacy_smoke = inference_contract.get("smoke_test") if isinstance(legacy_smoke, dict) and legacy_smoke: return legacy_smoke, "inference_contract_smoke_test" return None, "" def _raw_inputs_from_canonical_smoke(smoke: dict) -> tuple[object, dict]: """Normalize Pi-provided smoke examples across common field names.""" if not isinstance(smoke, dict): return [], {} raw_kwargs = smoke.get("kwargs", {}) if raw_kwargs is None or not isinstance(raw_kwargs, dict): raw_kwargs = {} for key in ("args", "inputs", "input", "values", "test_args"): if key in smoke: return smoke.get(key), raw_kwargs return [], raw_kwargs def canonical_smoke_example_payload(inference_contract: dict, demo_quality_contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None: smoke, source = _canonical_smoke_dict_from_contracts(inference_contract, demo_quality_contract) if not isinstance(smoke, dict): return None params = endpoint_parameters_for_smoke(endpoint) smoke_api = normalize_api_name(str(smoke.get("api_name") or smoke.get("endpoint") or api_name)) raw_args, raw_kwargs = _raw_inputs_from_canonical_smoke(smoke) generated, _ = build_generation_smoke_args(endpoint, expected_output_type) if isinstance(raw_args, list): args = [coerce_smoke_value(v, params[i] if i < len(params) else {}) for i, v in enumerate(raw_args)] if params and len(args) < len(params): args.extend(generated[len(args):]) return { "api_name": smoke_api, "test_args": args, "test_kwargs": raw_kwargs, "parameters": params, "source": source, "canonical_smoke_example_present": True, "canonical_smoke_reason": smoke.get("reason") or smoke.get("why_representative") or smoke.get("description") or "", } if isinstance(raw_args, dict): args = [] resolution = [] for i, param in enumerate(params): value, source_key = _canonical_dict_value_for_param(raw_args, param, i, generated, expected_output_type) args.append(coerce_smoke_value(value, param)) resolution.append({ "index": i, "name": str(param.get("name") or f"arg{i}"), "semantic_name": _param_semantic_name(param, i, expected_output_type), "source": source_key, }) if not params: args = list(raw_args.values()) return { "api_name": smoke_api, "test_args": args, "test_kwargs": raw_kwargs, "parameters": params, "source": source, "canonical_smoke_example_present": True, "canonical_smoke_reason": smoke.get("reason") or smoke.get("why_representative") or smoke.get("description") or "", "dict_resolution": resolution, } # Empty/unsupported inputs: canonical exists but cannot be transformed safely. return { "api_name": smoke_api, "test_args": generated, "test_kwargs": raw_kwargs, "parameters": params, "source": source + "_fallback_schema_args", "canonical_smoke_example_present": True, "canonical_smoke_unsupported_inputs": True, } def contract_smoke_test_payload(contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None: """Legacy compatibility wrapper for INFERENCE_CONTRACT.smoke_test.""" return canonical_smoke_example_payload(contract, {}, api_name, endpoint, expected_output_type) def _semantic_health_payloads(validation: dict | None) -> list[dict]: if not isinstance(validation, dict): return [] payloads: list[dict] = [] for key in ("json", "result", "data", "health"): value = validation.get(key) if isinstance(value, dict): payloads.append(value) nested = value.get("json") if isinstance(nested, dict): payloads.append(nested) payloads.append(validation) return payloads def validation_health_semantics(validation: dict | None) -> dict: """Return semantic health details without equating reachability with readiness. v198.26.2: `/health` HTTP 200 only proves that an endpoint responded. If its JSON says `status: unhealthy`, `pipeline_ready: false`, `pipeline_loaded: false`, or exposes an error/load_error, the model health did not pass. """ details = { "schema_version": "health_semantics.v198_26_2", "endpoint_reachable": False, "semantic_checked": False, "semantic_passed": None, "negative_markers": [], "positive_markers": [], "load_error": "", } if not isinstance(validation, dict): return details method = str(validation.get("method") or "").lower() validator = str(validation.get("validator") or "").lower() api_name = normalize_api_name(str(validation.get("api_name") or "")) if validation.get("api_name") else "" details["endpoint_reachable"] = bool(validation.get("status") == "success" and (method in {"http_health", "gradio"} or validator in {"http_get_health", "gradio_client"} or api_name == "/health")) negative_statuses = {"unhealthy", "failed", "failure", "error", "not_ready", "not ready", "runtime_error"} positive_statuses = {"healthy", "ok", "ready", "success", "passed"} scaffold_markers = {"initial-scaffold", "initial_scaffold", "placeholder", "scaffold", "template"} for payload in _semantic_health_payloads(validation): for key in ("status", "health", "state"): if key in payload: value = str(payload.get(key) or "").strip().lower() if value in negative_statuses: details["semantic_checked"] = True details["negative_markers"].append(f"{key}={value}") elif value in positive_statuses and key != "state": details["semantic_checked"] = True details["positive_markers"].append(f"{key}={value}") for key in ("stage", "runtime_stage", "phase"): if key in payload: value = str(payload.get(key) or "").strip().lower() if value in scaffold_markers or any(marker in value for marker in ("initial-scaffold", "initial_scaffold", "placeholder")): details["semantic_checked"] = True details["negative_markers"].append(f"{key}={value}") for key in ("pipeline_ready", "pipeline_loaded", "model_ready", "model_loaded"): if key in payload: details["semantic_checked"] = True if payload.get(key) is False: details["negative_markers"].append(f"{key}=false") elif payload.get(key) is True: details["positive_markers"].append(f"{key}=true") for key in ("error", "load_error", "pipeline_error", "model_error"): value = payload.get(key) if value: details["semantic_checked"] = True details["negative_markers"].append(f"{key}=present") if not details["load_error"]: details["load_error"] = str(value)[:1000] if details["negative_markers"]: details["semantic_passed"] = False elif details["semantic_checked"] and details["positive_markers"]: details["semantic_passed"] = True return details def validation_health_passed(validation: dict | None) -> bool: if not isinstance(validation, dict) or validation.get("status") != "success": return False semantics = validation_health_semantics(validation) if semantics.get("semantic_passed") is False: return False method = str(validation.get("method") or "").lower() validator = str(validation.get("validator") or "").lower() api_name = normalize_api_name(str(validation.get("api_name") or "")) if validation.get("api_name") else "" reachable = method in {"http_health", "gradio"} or validator in {"http_get_health", "gradio_client"} or api_name == "/health" return bool(reachable) def is_cuda_oom_text(text: str) -> bool: lowered = str(text or "").lower() markers = [ "cuda out of memory", "torch.cuda.outofmemoryerror", "outofmemoryerror", "out of memory", "tried to allocate", "gpu memory", "reserved memory", "allocated memory", ] return ("cuda" in lowered or "gpu" in lowered or "outofmemory" in lowered) and any(marker in lowered for marker in markers) def is_placeholder_scaffold_text(text: str) -> bool: lowered = str(text or "").lower() markers = [ "initial scaffold", "initial-scaffold", "initial_scaffold", "pi should replace this", "model-specific inference path", "placeholder demo", "placeholder scaffold", "stage: initial-scaffold", ] return any(marker in lowered for marker in markers) def detect_placeholder_scaffold(workspace: Path) -> dict: """Detect a final runtime that still contains ASF/Pi placeholder scaffold text.""" app_path = workspace / "app.py" text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" detected = is_placeholder_scaffold_text(text) markers = [] lowered = text.lower() for marker in ["initial scaffold", "initial-scaffold", "pi should replace this", "model-specific inference path", "placeholder scaffold"]: if marker in lowered: markers.append(marker) return { "schema_version": "placeholder_scaffold_detection.v198_26_7", "detected": bool(detected), "failure_owner": "pi_generation" if detected else "", "failure_class": "placeholder_scaffold_deployed" if detected else "", "failure_type": "model_specific_runtime_missing" if detected else "", "repair_candidate": bool(detected), "markers": markers, "checked_file": "app.py", "recommended_action": "Replace the initial scaffold with a model-specific inference app before treating the Space as healthy." if detected else "", } def _extract_missing_executable(text: str) -> str: text = str(text or "") patterns = [ r"No such file or directory:\s*['\"]([^'\"]+)['\"]", r"\[Errno 2\].*?:\s*['\"]([^'\"]+)['\"]", r"FileNotFoundError:.*?['\"]([^'\"]+)['\"]", ] for pattern in patterns: match = re.search(pattern, text, flags=re.IGNORECASE | re.DOTALL) if match: candidate = (match.group(1) or "").strip() if candidate and "/" not in candidate and "\\" not in candidate: return candidate if candidate: return Path(candidate).name or candidate return "" def diagnose_generation_smoke_failure(error_or_result, *, inference_strategy: str = "", expected_output_type: str = "", phase: str = "exception") -> dict: """Classify failed automatic generation smoke tests without collapsing usable Spaces to failed. v198.13: smoke failures are no longer a single opaque `generation_smoke_error`. The diagnosis is intentionally conservative: it identifies validator-owned failures, missing runtime executables, and common model-init responses so the UI and repair loop can distinguish partial usable Spaces from terminal app/runtime failures. """ text = str(error_or_result or "") lowered = text.lower() strategy = str(inference_strategy or "").strip() evidence = [] if text: evidence.append(text[:1200]) if strategy: evidence.append(f"inference_strategy={strategy}") base = { "schema_version": "generation_smoke_diagnosis.v198_9", "smoke_passed": False, "phase": phase, "inference_strategy": strategy, "expected_output_type": expected_output_type or "", "failure_owner": "unknown", "failure_class": "generation_smoke_error", "failure_type": "generation_smoke_error", "actionability": "review_required", "repair_candidate": False, "recommended_action": "Inspect generation_smoke.json, Space logs, and retry via Space Test with adjusted inputs.", "evidence": evidence, } marker = "no value provided for required argument:" missing_executable = _extract_missing_executable(text) if "smoke_payload_resolution_failed" in lowered or "required_text_resolved_empty" in lowered or "prompt cannot be empty" in lowered: base.update({ "failure_owner": "factory_validation_client", "failure_class": "smoke_payload_resolution_failed", "failure_type": "required_text_resolved_empty", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Fix ASF canonical smoke payload resolution so required prompt/text inputs are non-empty before calling gradio_client; do not repair the generated Space.", }) elif is_placeholder_scaffold_text(text): base.update({ "failure_owner": "pi_generation", "failure_class": "placeholder_scaffold_deployed", "failure_type": "model_specific_runtime_missing", "actionability": "targeted_pi_repair", "repair_candidate": True, "recommended_action": "Run a targeted Pi repair to replace the initial scaffold with a model-specific inference app that returns the promised artifact.", }) elif "asf smoke input materialization failed" in lowered or "does not exist on local filesystem" in lowered: base.update({ "failure_owner": "factory_validation_client", "failure_class": "smoke_input_materialization_failed", "failure_type": "smoke_input_materialization_failed", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Resolve or create the media/file smoke input in the ASF validator before calling gradio_client; do not repair the generated Space.", }) elif is_smoke_schema_error_text(text): base.update({ "failure_owner": "factory_validation_client", "failure_class": "smoke_schema_error", "failure_type": "smoke_schema_error", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Fix ASF Gradio smoke payload construction for file/media inputs; do not repair the generated Space.", }) elif is_gradio_hidden_error_text(text): base.update({ "failure_owner": "app_runtime", "failure_class": "gradio_hidden_runtime_error", "failure_type": "gradio_hidden_runtime_error", "actionability": "deterministic_show_error_patch_then_retry", "repair_candidate": False, "recommended_action": "Apply deterministic demo.launch(show_error=True), redeploy, and rerun smoke to capture the real traceback before Pi repair.", }) elif is_schema_choice_type_error(text): base.update({ "failure_owner": "factory_validator", "failure_class": "validator_schema_choice_type_mismatch", "failure_type": "validator_schema_choice_type_mismatch", "actionability": "validator_self_repair", "retryable_with_schema_payload": True, "auto_retry_supported": True, "recommended_action": "Retry smoke with schema-correct choice values from Gradio view_api().", }) elif marker in lowered: missing = text.split(":", 1)[-1].strip().strip("'\"") base.update({ "failure_owner": "factory_validator", "failure_class": "validator_request_error", "failure_type": "validator_request_error", "actionability": "validator_payload_repair", "retryable_with_schema_payload": True, "missing_argument": missing, "recommended_action": "Resolve the required argument from the Gradio schema before retrying smoke validation.", }) elif missing_executable: base.update({ "failure_owner": "app_runtime", "failure_class": "missing_runtime_cli", "failure_type": "missing_runtime_cli", "missing_executable": missing_executable, "actionability": "repair_candidate", "repair_candidate": True, "recommended_action": "Run a targeted repair for the missing runtime CLI: fix the package dependency, PATH, or command invocation.", }) elif "model not initialized" in lowered or "model is not initialized" in lowered or "model not loaded" in lowered: base.update({ "failure_owner": "model_runtime", "failure_class": "model_not_initialized", "failure_type": "model_not_initialized", "actionability": "needs_logs_or_manual_review", "repair_candidate": False, "recommended_action": "Keep the run partial, inspect runtime logs for the model initialization failure, and only repair if a concrete dependency/import/load error is found.", }) elif any(m in lowered for m in ["timeout", "timed out", "read operation timed out"]): base.update({ "failure_owner": "infra_or_generated_space", "failure_class": "timeout", "failure_type": "timeout", "actionability": "retry_or_runtime_recovery", "retryable_with_longer_timeout": True, "recommended_action": "Retry with a longer timeout or apply runtime recovery if the Space startup/runtime state is inconclusive.", }) elif "modulenotfounderror" in lowered or "no module named" in lowered or "importerror" in lowered: base.update({ "failure_owner": "app_runtime", "failure_class": "missing_python_dependency", "failure_type": "missing_python_dependency", "actionability": "repair_candidate", "repair_candidate": True, "recommended_action": "Run a targeted dependency repair based on the missing Python package.", }) elif is_cuda_oom_text(text): base.update({ "failure_owner": "hardware", "failure_class": "gpu_oom", "failure_type": "insufficient_vram_or_payload_too_large", "actionability": "retry_minimal_smoke_or_larger_hardware", "repair_candidate": False, "recommended_action": "Do not trigger Pi repair for a pure CUDA OOM. Retry with a minimal smoke payload or use larger/ZeroGPU hardware before declaring the Space unusable.", }) elif any(m in lowered for m in ["traceback", "runtimeerror", "attributeerror", "valueerror", "cuda", "out of memory", "exception"]): base.update({ "failure_owner": "app_runtime", "failure_class": "app_runtime_error", "failure_type": "app_runtime_error", "actionability": "repair_candidate", "repair_candidate": True, "recommended_action": "Run a targeted repair if the traceback points to a minimal code or dependency fix.", }) elif any(m in lowered for m in ["invalid state", "runtime_error", "build_error"]): base.update({ "failure_owner": "generated_space", "failure_class": "space_runtime_state", "failure_type": "space_runtime_state", "actionability": "runtime_recovery_or_repair", "repair_candidate": True, "recommended_action": "Inspect runtime state/logs and apply runtime recovery before destructive repair if no traceback is definitive.", }) return base def classify_generation_smoke_error(error: Exception | str, *, inference_strategy: str = "", expected_output_type: str = "") -> dict: diagnosis = diagnose_generation_smoke_failure(error, inference_strategy=inference_strategy, expected_output_type=expected_output_type, phase="exception") payload = { "failure_type": diagnosis.get("failure_type") or diagnosis.get("failure_class") or "generation_smoke_error", "failure_owner": diagnosis.get("failure_owner") or "unknown", "failure_class": diagnosis.get("failure_class") or "generation_smoke_error", "actionability": diagnosis.get("actionability") or "review_required", "recommended_action": diagnosis.get("recommended_action") or "", } for key in ("retryable_with_schema_payload", "auto_retry_supported", "missing_argument", "retryable_with_longer_timeout", "repair_candidate", "missing_executable"): if key in diagnosis: payload[key] = diagnosis[key] return payload def result_contains_expected_output(result, expected: str): expected = (expected or "any").lower() info = {"result_type": type(result).__name__, "result_repr": repr(result)[:2000]} paths = [] type_hints = [] text_values = [] def visit(obj): type_hints.append(type(obj).__name__.lower()) if isinstance(obj, (str, Path)): value = str(obj) if value: paths.append(value) text_values.append(value) elif isinstance(obj, dict): for key, value in obj.items(): key_l = str(key).lower() if key_l in {"path", "name", "url", "file", "filepath"}: visit(value) elif key_l in {"mime_type", "mime", "type", "format"} and value: type_hints.append(str(value).lower()) elif key_l in {"image", "video", "audio", "text"}: type_hints.append(key_l) visit(value) elif isinstance(value, (dict, list, tuple)): visit(value) elif isinstance(obj, (list, tuple)): for item in obj: visit(item) else: # PIL Images, numpy arrays returned through Gradio objects, etc. module = getattr(type(obj), "__module__", "") if module: type_hints.append(module.lower()) visit(result) lower_paths = [str(p).lower() for p in paths] hints = " ".join(type_hints + lower_paths) info["detected_paths"] = paths[:20] info["type_hints"] = type_hints[:30] if expected == "any": return result is not None, info image_ext = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff") video_ext = (".mp4", ".mov", ".webm", ".avi", ".mkv") audio_ext = (".wav", ".mp3", ".flac", ".ogg", ".m4a") if expected == "text": if isinstance(result, str) and bool(result.strip()): return True, info return any(isinstance(x, str) and x.strip() and not Path(x).suffix.lower() in image_ext + video_ext + audio_ext for x in text_values), info if expected == "image": return any(p.endswith(image_ext) for p in lower_paths) or any(h in hints for h in ["image", "pil.", "png", "jpeg", "jpg", "webp"]), info if expected == "video": return any(p.endswith(video_ext) for p in lower_paths) or any(h in hints for h in ["video", "mp4", "webm", "moviepy"]), info if expected == "audio": return any(p.endswith(audio_ext) for p in lower_paths) or any(h in hints for h in ["audio", "wav", "soundfile"]), info return result is not None, info def copy_result_artifacts(result, run_dir: Path): artifacts = run_dir / "artifacts" artifacts.mkdir(parents=True, exist_ok=True) copied = [] def maybe_copy(obj): if isinstance(obj, (str, Path)): path = Path(str(obj)) if path.exists() and path.is_file(): target = artifacts / path.name try: shutil.copy2(path, target) copied.append(str(target)) except Exception: pass elif isinstance(obj, dict): for key in ["path", "name"]: if key in obj: maybe_copy(obj[key]) for value in obj.values(): if isinstance(value, (dict, list, tuple)): maybe_copy(value) elif isinstance(obj, (list, tuple)): for item in obj: maybe_copy(item) maybe_copy(result) return copied def inspect_video_artifacts(copied_paths: list[str]) -> dict: video_ext = {".mp4", ".mov", ".webm", ".avi", ".mkv"} videos = [] for item in copied_paths or []: path = Path(str(item)) if path.suffix.lower() not in video_ext: continue record = {"path": str(path), "extension": path.suffix.lower(), "exists": path.exists()} if path.exists(): try: record["size_bytes"] = path.stat().st_size record["container_valid"] = bool(path.stat().st_size > 0) except Exception as exc: record["inspection_error"] = str(exc)[:1000] videos.append(record) return { "schema_version": "video_artifact_inspection.v198_24", "videos": videos, "video_count": len(videos), "has_video_artifact": bool(videos), "quality_assessment": "not_evaluated", } def smoke_timeout_seconds(expected_output_type: str) -> int: expected = str(expected_output_type or "").strip().lower() if expected == "video": return int(os.environ.get("ASF_VIDEO_SMOKE_TIMEOUT_SECONDS", "900")) if expected == "audio": return int(os.environ.get("ASF_AUDIO_SMOKE_TIMEOUT_SECONDS", "420")) return int(os.environ.get("ASF_GENERATION_SMOKE_TIMEOUT_SECONDS", "300")) def is_smoke_schema_error_text(error_or_result) -> bool: text = str(error_or_result or "").lower() return any(marker in text for marker in [ "imagedata", "filedata", "videodata", "audiodata", "input should be a valid dictionary or instance of", "input_type=str", "input_value='http", 'input_value="http', ]) def is_gradio_hidden_error_text(error_or_result) -> bool: text = str(error_or_result or "").lower() return "has raised an exception" in text and "show_error=true" in text def record_generation_smoke_result(run_dir: Path, payload: dict, *, phase: str = "generation_smoke") -> dict: tests_dir = run_dir / "tests" tests_dir.mkdir(parents=True, exist_ok=True) history_path = tests_dir / "generation_smoke_history.json" history = read_json(history_path, {"schema_version": "generation_smoke_history.v198_24", "attempts": []}) or {"schema_version": "generation_smoke_history.v198_24", "attempts": []} attempts = history.get("attempts") if isinstance(history.get("attempts"), list) else [] entry = dict(payload or {}) entry.setdefault("recorded_at", now()) entry.setdefault("phase", phase) entry["attempt_index"] = len(attempts) + 1 attempts.append(entry) history = {"schema_version": "generation_smoke_history.v198_24", "latest_attempt_index": entry["attempt_index"], "latest_status": entry.get("status"), "attempts": attempts} write_json(history_path, history) write_json(tests_dir / "generation_smoke_latest.json", entry) write_json(tests_dir / "generation_smoke.json", payload) return entry def build_generation_smoke_failure_payload(target_space_id: str, expected_output_type: str, error, *, workspace: Path | None = None, phase: str = "exception") -> dict: contract = read_inference_contract(workspace) strategy = str((contract or {}).get("inference_strategy") or "") diagnosis = diagnose_generation_smoke_failure(error, inference_strategy=strategy, expected_output_type=expected_output_type, phase=phase) payload = { "status": "failed", "target_space": target_space_id, "expected_output_type": expected_output_type, "error": str(error)[:4000], "next_action": diagnosis.get("recommended_action") or "Use Prefill Space Test after the run finishes to retry with schema-adjusted arguments.", **measured_zero_gpu_recommendation(None), "diagnosis_path": "tests/generation_smoke_diagnosis.json", } payload.update(classify_generation_smoke_error(error, inference_strategy=strategy, expected_output_type=expected_output_type)) return payload def wait_for_video_smoke_settle(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, expected_output_type: str): if str(expected_output_type or "").strip().lower() != "video": return {"skipped": True, "reason": "not_video"} checks = int(os.environ.get("ASF_VIDEO_SMOKE_SETTLE_CHECKS", "3")) sleep_seconds = int(os.environ.get("ASF_VIDEO_SMOKE_SETTLE_SECONDS", "30")) history = [] for attempt in range(1, checks + 1): runtime = write_space_runtime(api, target_space_id, token, run_dir, events_path, attempt) stage = str((runtime or {}).get("stage") or "").upper() history.append(runtime) if "RUNNING_BUILDING" in stage or ("BUILDING" in stage and "RUNNING" in stage): append_event(events_path, "video_smoke_settle", "waiting", "Video Space is still settling after health passed; delaying heavy smoke", {"attempt": attempt, "runtime": runtime, "sleep_seconds": sleep_seconds}) time.sleep(sleep_seconds) continue append_event(events_path, "video_smoke_settle", "success", "Video Space runtime is stable enough for smoke", {"attempt": attempt, "runtime": runtime}) payload = {"schema_version": "video_smoke_settle.v198_24", "settled": True, "attempts": attempt, "history": history} write_json(run_dir / "tests" / "video_smoke_settle.json", payload) return payload payload = {"schema_version": "video_smoke_settle.v198_24", "settled": False, "attempts": checks, "history": history, "proceeding_after_max_checks": True} write_json(run_dir / "tests" / "video_smoke_settle.json", payload) append_event(events_path, "video_smoke_settle", "warning", "Proceeding with video smoke after settle budget expired", payload) return payload def ensure_gradio_launch_show_error(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload") -> dict: app_path = workspace / "app.py" payload = {"schema_version": "gradio_show_error_patch.v198_24", "reason": reason, "applied": False, "launch_calls_seen": 0, "launch_calls_patched": 0} if not app_path.exists(): payload["skip_reason"] = "missing_app_py" write_json(run_dir / "gradio_show_error_patch.json", payload) return payload source = app_path.read_text(encoding="utf-8", errors="ignore") try: tree = ast.parse(source) except Exception as exc: payload.update({"skip_reason": "app_py_parse_failed", "error": str(exc)[:1000]}) write_json(run_dir / "gradio_show_error_patch.json", payload) append_event(events_path, "gradio_show_error_patch", "warning", "Could not parse app.py to enforce show_error=True", payload) return payload calls = [] for node in ast.walk(tree): if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "launch": payload["launch_calls_seen"] += 1 if any(kw.arg == "show_error" for kw in node.keywords if kw.arg): continue if not hasattr(node, "end_lineno") or not hasattr(node, "end_col_offset"): continue calls.append(node) if not calls: write_json(run_dir / "gradio_show_error_patch.json", payload) return payload lines = source.splitlines(keepends=True) for node in sorted(calls, key=lambda n: (n.end_lineno, n.end_col_offset), reverse=True): if node.lineno == node.end_lineno: line_i = node.end_lineno - 1 line = lines[line_i] insert_at = node.end_col_offset - 1 insertion = "show_error=True" if not node.args and not node.keywords else ", show_error=True" lines[line_i] = line[:insert_at] + insertion + line[insert_at:] else: close_i = node.end_lineno - 1 close_line = lines[close_i] close_indent = re.match(r"^\s*", close_line).group(0) lines.insert(close_i, f"{close_indent} show_error=True,\n") payload["launch_calls_patched"] += 1 updated = "".join(lines) if updated != source: app_path.write_text(updated, encoding="utf-8") payload["applied"] = True write_json(run_dir / "gradio_show_error_patch.json", payload) append_event(events_path, "gradio_show_error_patch", "success", "Ensured generated Gradio launch uses show_error=True", payload) else: write_json(run_dir / "gradio_show_error_patch.json", payload) return payload def measured_zero_gpu_recommendation(latency_seconds: float | None): if latency_seconds is None: return { "observed_latency_seconds": None, "recommended_zero_gpu_duration_seconds": None, "recommendation_source": "not_measured", "recommendation_confidence": "none", } recommended = int(max(30, min(300, latency_seconds * 2 + 15))) return { "observed_latency_seconds": round(latency_seconds, 3), "recommended_zero_gpu_duration_seconds": recommended, "recommendation_source": "live_gradio_predict", "recommendation_confidence": "measured", "measurement_note": "Measured from a live gradio_client.predict call, including Gradio/API/network/result serialization overhead.", } def run_gradio_validation_harness(*, client, api_name: str, test_args: list, test_kwargs: dict, expected_output_type: str, run_dir: Path, events_path: Path, validation_mode: str, payload_source: str): """Shared v190.24 validation contract for automatic build smoke. Linked Space Test uses a mirrored helper with the same contract; both flows must resolve payload, write resolved_validation_request.json, coerce schema choices, then predict and verify output. """ resolved = { "api_name": api_name, "test_args": list(test_args or []), "test_kwargs": dict(test_kwargs or {}), "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "resolved_request_required_before_predict": True, } write_json(run_dir / "tests" / "resolved_validation_request.json", resolved) started = time.time() result = client.predict(*resolved["test_args"], api_name=api_name, **resolved["test_kwargs"]) latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) return result, latency, ok, info, resolved def run_generation_smoke(target_space_id: str, token: str, run_dir: Path, events_path: Path, expected_output_type: str, workspace: Path | None = None): api_name = "/generate" test_kwargs = {} timeout_s = smoke_timeout_seconds(expected_output_type) append_event(events_path, "generation_smoke", "started", "Calling live generation endpoint for ZeroGPU timing", {"api_name": api_name, "expected_output_type": expected_output_type, "smoke_timeout_seconds": timeout_s}) client = make_gradio_client(target_space_id, token, timeout_s=timeout_s) identity = write_gradio_client_identity(client, target_space_id, run_dir, events_path, phase="generation_smoke") if identity.get("mismatch"): diagnosis = {"schema_version": "generation_smoke_diagnosis.v198_26_7", "smoke_passed": False, "phase": "space_identity", "expected_output_type": expected_output_type or "", "failure_owner": "factory_validation_client", "failure_class": "space_identity_mismatch", "failure_type": "space_identity_mismatch", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": "Do not validate a Space whose gradio_client URL does not match the target_space_id; fix Space identity propagation before retrying.", "evidence": [json.dumps(identity, ensure_ascii=False)[:1200]]} payload = {"status": "failed", "target_space": target_space_id, "api_name": api_name, "expected_output_type": expected_output_type, "error": "space_identity_mismatch", "failure_type": "space_identity_mismatch", "failure_class": "space_identity_mismatch", "failure_owner": "factory_validation_client", "actionability": "factory_fix_required", "repair_candidate": False, "space_identity": identity, **measured_zero_gpu_recommendation(None)} write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", diagnosis) record_generation_smoke_result(run_dir, payload, phase="generation_smoke") append_event(events_path, "generation_smoke", "failed", "Space identity mismatch before generation smoke", payload) return payload schema = client.view_api(return_format="dict") discovered = api_names_from_schema(schema) if api_name not in discovered and discovered: non_health = [name for name in discovered if name != "/health"] if non_health: api_name = non_health[0] endpoint = endpoint_schema_for_api(schema, api_name) contract = read_inference_contract(workspace) demo_quality_contract = read_demo_quality_contract(workspace) contract_primary = normalize_api_name(str(contract.get("primary_api_name") or "")) if contract else "" if contract_primary and contract_primary in discovered and contract_primary != "/health": api_name = contract_primary endpoint = endpoint_schema_for_api(schema, api_name) contract_payload = canonical_smoke_example_payload(contract, demo_quality_contract, api_name, endpoint, expected_output_type) if contract_payload and contract_payload.get("api_name") in discovered: api_name = contract_payload["api_name"] endpoint = endpoint_schema_for_api(schema, api_name) contract_payload = canonical_smoke_example_payload(contract, demo_quality_contract, api_name, endpoint, expected_output_type) endpoint_parameters = endpoint_parameter_names(endpoint) if contract_payload: test_args = contract_payload.get("test_args") or [] test_kwargs = contract_payload.get("test_kwargs") or {} smoke_parameters = contract_payload.get("parameters") or endpoint_parameters_for_smoke(endpoint) smoke_source = contract_payload.get("source") or "inference_contract_smoke_test" else: test_args, smoke_parameters = build_generation_smoke_args(endpoint, expected_output_type) smoke_source = "gradio_schema" if smoke_parameters else "default_fallback" raw_test_args, initial_choice_changes = coerce_smoke_args_to_schema_choices(test_args, smoke_parameters) test_args, file_input_conversions = prepare_gradio_file_inputs(raw_test_args, smoke_parameters, run_dir) canonical_example = None canonical_source = "" if contract_payload and contract_payload.get("canonical_smoke_example_present"): canonical_example, canonical_source = _canonical_smoke_dict_from_contracts(contract, demo_quality_contract) smoke_payload = {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "parameters": smoke_parameters, "source": smoke_source, "contract_present": bool(contract), "demo_quality_contract_present": bool(demo_quality_contract), "canonical_smoke_example_present": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s} payload_resolution_guard = detect_required_text_payload_resolution_issue(test_args, smoke_parameters, canonical_example) smoke_payload["payload_resolution_guard"] = payload_resolution_guard if canonical_example: write_json(run_dir / "tests" / "canonical_smoke_example.json", {"source": canonical_source, "example": canonical_example, "resolved_api_name": api_name, "raw_resolved_args": raw_test_args, "resolved_args": test_args, "resolved_kwargs": test_kwargs, "expected_output_type": expected_output_type, "file_input_conversions": file_input_conversions, "dict_resolution": contract_payload.get("dict_resolution") if isinstance(contract_payload, dict) else [], "payload_resolution_guard": payload_resolution_guard}) write_json(run_dir / "tests" / "generation_smoke_payload.json", smoke_payload) write_json(run_dir / "tests" / "payload_source.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "payload_source": smoke_source, "selected_api_name": api_name, "parent_smoke_payload_used": False, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source}) write_json(run_dir / "tests" / "validation_engine.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "resolved_request_required_before_predict": True}) write_json(run_dir / "tests" / "resolved_validation_request.json", {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": smoke_source, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "schema_choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s, "payload_resolution_guard": payload_resolution_guard}) write_json(run_dir / "tests" / "schema_coercion.json", {"api_name": api_name, "changes": initial_choice_changes, "original_args": raw_test_args, "resolved_args": test_args, "file_input_conversions": file_input_conversions, "payload_resolution_guard": payload_resolution_guard}) write_json(run_dir / "tests" / "generation_api_schema.json", {"schema": schema, "api_names": discovered, "selected_api_name": api_name, "endpoint_parameters": endpoint_parameters, "smoke_parameters": smoke_parameters, "smoke_payload_source": smoke_source, "canonical_smoke_example_used": bool(canonical_example)}) if payload_resolution_guard.get("detected"): diagnosis = { "schema_version": "generation_smoke_diagnosis.v198_26_7", "smoke_passed": False, "phase": "payload_resolution", "inference_strategy": str(contract.get("inference_strategy") or ""), "expected_output_type": expected_output_type or "", "failure_owner": "factory_validation_client", "failure_class": "smoke_payload_resolution_failed", "failure_type": "required_text_resolved_empty", "actionability": "factory_fix_required", "repair_candidate": False, "recommended_action": payload_resolution_guard.get("recommended_action"), "evidence": [json.dumps(payload_resolution_guard, ensure_ascii=False)[:1200]], } payload = { "status": "failed", "target_space": target_space_id, "api_name": api_name, "discovered_api_names": discovered, "endpoint_parameters": endpoint_parameters, "test_args": test_args, "test_kwargs": test_kwargs, "smoke_payload_source": smoke_source, "validation_level": "canonical_promise_smoke", "demo_usable_smoke_passed": False, "canonical_promise_smoke_passed": False, "full_promise_smoke_passed": False, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "payload_resolution_guard": payload_resolution_guard, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s, "next_action": diagnosis["recommended_action"], "expected_output_type": expected_output_type, "latency_seconds": None, "result_info": {}, "copied_artifacts": [], "validated_at": now(), "failure_type": diagnosis["failure_type"], "failure_class": diagnosis["failure_class"], "failure_owner": diagnosis["failure_owner"], "actionability": diagnosis["actionability"], "repair_candidate": False, **measured_zero_gpu_recommendation(None), } write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", diagnosis) record_generation_smoke_result(run_dir, payload, phase="generation_smoke") write_live_status(run_dir, stage="generation_smoke", status="failed", message="Automatic smoke payload resolution failed before calling Gradio", data=payload) append_event(events_path, "generation_smoke", "failed", "Automatic smoke payload resolution failed before calling Gradio", payload) return payload write_live_status(run_dir, stage="generation_smoke", status="running", message="Calling live generation endpoint", data={"api_name": api_name, "source": smoke_source, "canonical_smoke_example_used": bool(canonical_example)}) started = time.time() retry_info = {"attempts": 1, "choice_corrections": initial_choice_changes, "retried": False} minimal_smoke_attempt = {"attempted": False, "succeeded": False, "reason": ""} try: result = client.predict(*test_args, api_name=api_name, **test_kwargs) except Exception as first_error: if is_schema_choice_type_error(first_error): retry_params = smoke_parameters or endpoint_parameters_for_smoke(endpoint) corrected_args, retry_choice_changes = coerce_smoke_args_to_schema_choices(raw_test_args, retry_params) if not retry_choice_changes or corrected_args == raw_test_args: schema_params = endpoint_parameters_for_smoke(endpoint) corrected_args, retry_choice_changes = coerce_smoke_args_to_schema_choices(raw_test_args, schema_params) if retry_choice_changes: retry_params = schema_params if not retry_choice_changes or corrected_args == raw_test_args: parsed_args, parsed_changes = coerce_smoke_args_from_choice_error(raw_test_args, first_error, endpoint_parameters_for_smoke(endpoint)) if parsed_changes and parsed_args != raw_test_args: corrected_args, retry_choice_changes = parsed_args, parsed_changes retry_params = endpoint_parameters_for_smoke(endpoint) if retry_choice_changes and corrected_args != raw_test_args: append_event(events_path, "generation_smoke", "warning", "Retrying generation smoke with Gradio schema choice types", {"api_name": api_name, "error": str(first_error)[:1500], "choice_corrections": retry_choice_changes}) write_live_status(run_dir, stage="generation_smoke_retry", status="running", message="Retrying smoke test with schema-corrected choice values", data={"api_name": api_name, "choice_corrections": retry_choice_changes}) raw_test_args = corrected_args test_args, retry_file_conversions = prepare_gradio_file_inputs(raw_test_args, retry_params, run_dir) file_input_conversions = file_input_conversions + retry_file_conversions retry_info = {"attempts": 2, "choice_corrections": initial_choice_changes + retry_choice_changes, "file_input_conversions": file_input_conversions, "retried": True, "first_error": str(first_error)[:2000], "retry_reason": "schema_choice_type_mismatch", "self_repair": "schema_aware_payload_retry"} retry_artifact = {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "parameters": retry_params, **retry_info} write_json(run_dir / "tests" / "generation_smoke_payload_retry.json", retry_artifact) write_json(run_dir / "tests" / "validator_self_repair.json", {"schema_version": "validator_self_repair.v198_7", "triggered": True, "failure_owner": "factory_validator", "failure_type": "validator_schema_choice_type_mismatch", "first_error": str(first_error)[:2000], "action": "retry_with_schema_correct_choice_values", "choice_corrections": retry_choice_changes, "retry_payload_path": "tests/generation_smoke_payload_retry.json"}) result = client.predict(*test_args, api_name=api_name, **test_kwargs) else: write_json(run_dir / "tests" / "validator_self_repair.json", {"schema_version": "validator_self_repair.v198_7", "triggered": True, "failure_owner": "factory_validator", "failure_type": "validator_schema_choice_type_mismatch", "first_error": str(first_error)[:2000], "action": "no_safe_schema_correction_found", "choice_error": parse_gradio_choice_error(first_error)}) raise elif is_cuda_oom_text(str(first_error)): minimal_raw_args, minimal_changes = build_minimal_demo_smoke_args(raw_test_args, smoke_parameters or endpoint_parameters_for_smoke(endpoint), expected_output_type) if not minimal_changes or minimal_raw_args == raw_test_args: raise append_event(events_path, "generation_smoke", "warning", "Canonical promise smoke hit CUDA OOM; retrying reduced minimal demo smoke", {"api_name": api_name, "error": str(first_error)[:1500], "minimal_changes": minimal_changes}) write_live_status(run_dir, stage="minimal_demo_smoke", status="running", message="Retrying with reduced minimal smoke payload after canonical OOM", data={"api_name": api_name, "minimal_changes": minimal_changes}) minimal_params = smoke_parameters or endpoint_parameters_for_smoke(endpoint) minimal_args, minimal_file_conversions = prepare_gradio_file_inputs(minimal_raw_args, minimal_params, run_dir) minimal_smoke_attempt = {"attempted": True, "succeeded": False, "reason": "canonical_cuda_oom", "canonical_error": str(first_error)[:2000], "minimal_changes": minimal_changes} file_input_conversions = file_input_conversions + minimal_file_conversions retry_info = {"attempts": 2, "choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "retried": True, "first_error": str(first_error)[:2000], "retry_reason": "canonical_cuda_oom_minimal_demo_smoke", "minimal_changes": minimal_changes} minimal_artifact = {"api_name": api_name, "test_args": minimal_args, "raw_test_args": minimal_raw_args, "test_kwargs": test_kwargs, "parameters": minimal_params, "validation_level": "minimal_demo_smoke", **retry_info} write_json(run_dir / "tests" / "generation_smoke_payload_minimal.json", minimal_artifact) write_json(run_dir / "tests" / "minimal_demo_smoke_retry.json", {"schema_version": "minimal_demo_smoke_retry.v198_26_7", "triggered": True, "failure_owner": "hardware", "failure_type": "canonical_cuda_oom", "canonical_error": str(first_error)[:2000], "minimal_changes": minimal_changes, "retry_payload_path": "tests/generation_smoke_payload_minimal.json"}) try: result = client.predict(*minimal_args, api_name=api_name, **test_kwargs) raw_test_args = minimal_raw_args test_args = minimal_args smoke_parameters = minimal_params minimal_smoke_attempt["succeeded"] = True except Exception: # Keep the canonical OOM as the primary failure class. The # minimal retry is diagnostic only and must not hide the root # signal that the full promise payload exceeded capacity. raise first_error else: raise latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) copied = copy_result_artifacts(result, run_dir) video_inspection = inspect_video_artifacts(copied) if str(expected_output_type or "").strip().lower() == "video" else {} if video_inspection: write_json(run_dir / "tests" / "video_artifact_inspection.json", video_inspection) recommendation = measured_zero_gpu_recommendation(latency) smoke_diagnosis = {} if ok else diagnose_generation_smoke_failure(info.get("result_repr") or result, inference_strategy=str(contract.get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="output_verification") payload_status = "success" if ok else "failed" validation_level = "canonical_promise_smoke" if ok and minimal_smoke_attempt.get("attempted"): payload_status = "demo_usable_smoke_passed" validation_level = "minimal_demo_smoke" payload = { "status": payload_status, "target_space": target_space_id, "api_name": api_name, "discovered_api_names": discovered, "endpoint_parameters": endpoint_parameters, "test_args": test_args, "test_kwargs": test_kwargs, "smoke_payload_source": smoke_source, "validation_level": validation_level, "demo_usable_smoke_passed": bool(ok and minimal_smoke_attempt.get("attempted")), "canonical_promise_smoke_passed": bool(ok and not minimal_smoke_attempt.get("attempted")), "full_promise_smoke_passed": bool(ok and not minimal_smoke_attempt.get("attempted")), "minimal_smoke_attempt": minimal_smoke_attempt, "canonical_smoke_example_used": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "auto_retry": retry_info, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s, "next_action": "" if ok else smoke_diagnosis.get("recommended_action") or "Use Prefill Space Test after the run finishes to retry with adjusted arguments.", "expected_output_type": expected_output_type, "latency_seconds": round(latency, 3), "result_info": info, "copied_artifacts": copied, "video_artifact_inspection": video_inspection, "validated_at": now(), **({ "failure_type": smoke_diagnosis.get("failure_type"), "failure_class": smoke_diagnosis.get("failure_class"), "failure_owner": smoke_diagnosis.get("failure_owner"), "actionability": smoke_diagnosis.get("actionability"), "repair_candidate": smoke_diagnosis.get("repair_candidate"), } if smoke_diagnosis else {}), **recommendation, } if smoke_diagnosis: write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, payload, phase="generation_smoke") if ok: if minimal_smoke_attempt.get("attempted"): payload = minimal_smoke_status_payload(payload) write_json(run_dir / "tests" / "generation_smoke.json", payload) write_json(run_dir / "tests" / "generation_smoke_latest.json", payload) write_live_status(run_dir, stage="minimal_demo_smoke", status="warning", message="Minimal demo smoke passed; full promise smoke was not verified", data={"latency_seconds": payload["latency_seconds"], "api_name": api_name, "auto_retry": retry_info}) append_event(events_path, "generation_smoke", "warning", "Minimal demo smoke passed after canonical OOM; full promise remains unverified", {"latency_seconds": payload["latency_seconds"], "recommended_zero_gpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "recommended_zerogpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "api_name": api_name, "auto_retry": retry_info, "validation_level": "minimal_demo_smoke"}) else: write_live_status(run_dir, stage="generation_smoke", status="success", message="Live generation smoke test passed", data={"latency_seconds": payload["latency_seconds"], "api_name": api_name, "auto_retry": retry_info}) append_event(events_path, "generation_smoke", "success", "Live generation smoke test passed and ZeroGPU timing was measured", {"latency_seconds": payload["latency_seconds"], "recommended_zero_gpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "recommended_zerogpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "api_name": api_name, "auto_retry": retry_info}) else: write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation returned an unexpected output type", data=payload) append_event(events_path, "generation_smoke", "failed", "Live generation returned an unexpected output type", payload) return payload def smoke_failure_repair_escalation_decision(generation_smoke: dict | None, *, validation: dict | None = None, contract: dict | None = None) -> dict: """Decide whether a failed generation smoke should trigger one targeted repair. v198.13: partial validation is useful for UI honesty, but a demo that boots yet cannot satisfy its primary flow should not stop there when the smoke failure is concrete and minimally repairable. This is deliberately bounded: validator-owned failures are handled by validator self-repair, model init without logs remains partial, and we only escalate actionable app/runtime failures such as a missing executable or missing Python dependency. """ smoke = generation_smoke if isinstance(generation_smoke, dict) else {} validation = validation if isinstance(validation, dict) else {} contract = contract if isinstance(contract, dict) else {} failure_class = str(smoke.get("failure_class") or smoke.get("failure_type") or "") failure_owner = str(smoke.get("failure_owner") or "") repair_candidate = bool(smoke.get("repair_candidate")) health_ok = validation_health_passed(validation) actionable_classes = {"missing_runtime_cli", "missing_python_dependency", "app_runtime_error"} blocked_classes = {"validator_schema_choice_type_mismatch", "validator_request_error", "model_not_initialized", "smoke_schema_error", "smoke_input_materialization_failed", "gradio_hidden_runtime_error", "gpu_oom", "insufficient_vram", "insufficient_vram_or_payload_too_large", "hardware_capacity", "cuda_out_of_memory"} strategy = str(contract.get("inference_strategy") or smoke.get("inference_strategy") or "") payload = { "schema_version": "generation_smoke_repair_escalation.v198_13", "triggered": False, "repair_candidate": repair_candidate, "health_passed": health_ok, "failure_class": failure_class, "failure_owner": failure_owner, "inference_strategy": strategy, "action": "keep_partial_validation", "reason": "not_actionable_or_not_safe_for_automatic_smoke_repair", } if str(smoke.get("status") or "") != "failed": payload["reason"] = "generation_smoke_not_failed" return payload if failure_class in blocked_classes or failure_owner in {"factory_validator", "factory_validation_client", "hardware"}: payload["reason"] = "handled_elsewhere_or_requires_logs_not_blind_repair" return payload if not health_ok: payload["reason"] = "health_not_passed_use_runtime_recovery_instead" return payload if not repair_candidate or failure_class not in actionable_classes: payload["reason"] = "smoke_failure_not_repair_candidate" return payload payload.update({ "triggered": True, "action": "targeted_patch_repair_before_final_partial", "reason": "health_passed_but_primary_demo_flow_failed_with_actionable_smoke_error", "recommended_action": smoke.get("recommended_action") or "Run one targeted repair, redeploy, then re-run health and canonical smoke validation.", "missing_executable": smoke.get("missing_executable") or "", }) return payload def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, token: str, implementation_mode: str, expected_output_type: str, validation: dict, generation_smoke: dict) -> tuple[dict, dict]: """Run one bounded targeted repair when the demo boots but its primary flow fails. The function returns the latest (validation, generation_smoke). It never loops: one repair, one upload, one health validation, one generation smoke retry. """ contract = read_inference_contract(workspace) escalation = smoke_failure_repair_escalation_decision(generation_smoke, validation=validation, contract=contract) write_json(run_dir / "tests" / "generation_smoke_repair_escalation.json", escalation) if not escalation.get("triggered"): return validation, generation_smoke failure_class = escalation.get("failure_class") or generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "generation_smoke_error" failure_reason = ( "Automatic generation smoke failed after health passed.\n" f"Failure class: {failure_class}\n" f"Failure owner: {generation_smoke.get('failure_owner') or ''}\n" f"Actionability: {generation_smoke.get('actionability') or ''}\n" f"Recommended action: {generation_smoke.get('recommended_action') or ''}\n" f"Error/result: {generation_smoke.get('error') or generation_smoke.get('result_info') or generation_smoke}\n" ) decision = { "action": "patch_code", "confidence": "high" if failure_class in {"missing_runtime_cli", "missing_python_dependency"} else "medium", "source": "generation_smoke_repair_escalation.v198_13", "classification": { "category": failure_class, "failure_phase": "generation_smoke", "logs_quality": "useful", "recommendation": escalation.get("recommended_action") or generation_smoke.get("recommended_action") or "Patch the smallest concrete dependency or command issue, preserve the demo flow, redeploy, then re-run smoke validation.", "generation_smoke_failure": True, "missing_executable": generation_smoke.get("missing_executable") or "", }, "constraints": [ "Preserve real inference and the existing Gradio API contract.", "Do not replace the model call with placeholders or diagnostics if the failure is repairable.", "Patch only the dependency, command invocation, or minimal startup code required for the failed smoke path.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Running one targeted repair because health passed but the primary demo smoke failed", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause=failure_class, repair_decision="patch_code", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger="generation_smoke_failure") if not repaired: write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation="not_started", failure_type="smoke_repair_patch_failed", final_user_message="Targeted smoke repair failed before redeploy; keeping partial validation.") append_event(events_path, "generation_smoke_repair", "failed", "Targeted smoke repair failed before redeploy; keeping partial validation", {"failure_class": failure_class}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading targeted smoke repair") write_auth_probe(run_dir, events_path, "before_smoke_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Targeted smoke repair uploaded; revalidating health and demo smoke") try: write_auth_probe(run_dir, events_path, "before_smoke_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_status = "success" if generation_smoke.get("status") == "success" else "smoke_still_failed" write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type="" if post_status == "success" else generation_smoke.get("failure_type") or "generation_smoke_still_failed", final_user_message="Targeted smoke repair revalidated the Space." if post_status == "success" else "Targeted smoke repair uploaded, but generation smoke still did not verify the primary demo flow.", post_repair_generation_smoke=generation_smoke) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Targeted smoke repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status")}) except Exception as exc: repair_class = classify_repair_validation_error(exc) latest_smoke = build_generation_smoke_failure_payload(target_space_id, expected_output_type, exc, workspace=workspace, phase="post_smoke_repair_exception") smoke_diagnosis = diagnose_generation_smoke_failure(exc, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=expected_output_type, phase="post_smoke_repair_exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, latest_smoke, phase="post_smoke_repair_exception") generation_smoke = latest_smoke write_repair_outcome(run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=latest_smoke.get("failure_type") or repair_class["failure_type"] or "smoke_repair_validation_failed", final_user_message=repair_class["message"], validation_error=str(exc)[:4000], post_repair_generation_smoke=latest_smoke) append_event(events_path, "generation_smoke_repair_validation", "failed", "Targeted smoke repair validation failed; keeping partial validation", {"error": str(exc)[:2000], "latest_failure_class": latest_smoke.get("failure_class"), "latest_failure_owner": latest_smoke.get("failure_owner")}) return validation, generation_smoke def space_subdomain_url(target_space_id: str) -> str: owner, name = target_space_id.split("/", 1) # This matches the common Spaces app URL pattern. Keep conservative: our # generated slugs are ASCII and hyphen-friendly. return f"https://{owner}-{name}.hf.space".replace("_", "-").lower() def runtime_to_dict(runtime) -> dict: payload = {} for attr in ["stage", "hardware", "requested_hardware", "sleep_time", "storage", "gc_timeout"]: value = getattr(runtime, attr, None) payload[attr] = getattr(value, "value", value) return {k: str(v) if v is not None else None for k, v in payload.items()} def write_space_runtime(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int | None = None) -> dict: try: runtime = api.get_space_runtime(repo_id=target_space_id, token=token) payload = runtime_to_dict(runtime) payload["attempt"] = attempt write_json(run_dir / "space_runtime.json", payload) return payload except Exception as exc: payload = {"error": str(exc)[:2000], "attempt": attempt} write_json(run_dir / "space_runtime.json", payload) append_event(events_path, "space_runtime", "warning", "Could not fetch Space runtime", payload) return payload RESTART_GUARDRAIL_STATE = { "restart": {"count": 0, "last_ts": 0.0}, "factory_reboot": {"count": 0, "last_ts": 0.0}, "same_code_upload": {"count": 0, "last_ts": 0.0}, } RESTART_GUARDRAIL_LIMITS = { "restart": 1, "factory_reboot": 1, "same_code_upload": 1, } RESTART_GUARDRAIL_COOLDOWN_SECONDS = int(os.environ.get("SPACE_FACTORY_RESTART_COOLDOWN_SECONDS", "240")) def runtime_stage_value(runtime_payload: dict | None) -> str: return str((runtime_payload or {}).get("stage") or (runtime_payload or {}).get("runtime_stage") or "").lower() def restart_guardrail_allows(action: str, events_path: Path, *, reason: str = "") -> bool: state = RESTART_GUARDRAIL_STATE.setdefault(action, {"count": 0, "last_ts": 0.0}) limit = RESTART_GUARDRAIL_LIMITS.get(action, 1) now = time.time() if state["count"] >= limit: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart budget exhausted", {"action": action, "limit": limit, "reason": reason}) return False if state["last_ts"] and now - state["last_ts"] < RESTART_GUARDRAIL_COOLDOWN_SECONDS: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart cooldown active", {"action": action, "cooldown_seconds": RESTART_GUARDRAIL_COOLDOWN_SECONDS, "elapsed_seconds": round(now - state["last_ts"], 1), "reason": reason}) return False state["count"] += 1 state["last_ts"] = now return True def safe_restart_space(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, factory_reboot: bool = False, reason: str = "", require_logs_checked: bool = True, allow_busy: bool = False) -> bool: """Restart a Space with strict guardrails. Hugging Face exposes HfApi.restart_space(repo_id=..., factory_reboot=True) for a factory rebuild and restart_space(..., factory_reboot=False) for a normal restart. The Factory only calls it through this wrapper so we do not create restart loops or mask actionable build errors. """ action = "factory_reboot" if factory_reboot else "restart" if require_logs_checked: logs_dir = run_dir / "logs" if not ((logs_dir / "space_logs_build.txt").exists() or (logs_dir / "space_logs_runtime.txt").exists()): append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: logs have not been collected yet", {"reason": reason}) return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"} and not allow_busy: append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: Space is already busy", {"runtime": runtime_before, "reason": reason}) return False if not restart_guardrail_allows(action, events_path, reason=reason): return False try: append_event(events_path, "factory_rebuild", "started", f"Requesting {'factory reboot' if factory_reboot else 'Space restart'} via Hugging Face API", {"runtime_before": runtime_before, "reason": reason}) result = api.restart_space(repo_id=target_space_id, token=token, factory_reboot=factory_reboot) payload = {"factory_reboot": factory_reboot, "reason": reason, "result": str(result)[:1000]} write_json(run_dir / ("space_factory_reboot.json" if factory_reboot else "space_restart.json"), payload) append_event(events_path, "factory_rebuild", "success", f"{'Factory reboot' if factory_reboot else 'Space restart'} requested", payload) return True except TypeError: # Older huggingface_hub versions may not expose factory_reboot. Do not # loop; fall back to a plain restart only for non-factory requests. if factory_reboot: append_event(events_path, "factory_rebuild", "warning", "Installed huggingface_hub does not support restart_space(factory_reboot=True)", {"reason": reason}) return False try: result = api.restart_space(repo_id=target_space_id, token=token) append_event(events_path, "factory_rebuild", "success", "Space restart requested", {"reason": reason, "result": str(result)[:1000]}) return True except Exception as exc: append_event(events_path, "factory_rebuild", "failed", "Space restart API call failed", {"reason": reason, **exception_payload(exc)}) return False except Exception as exc: append_event(events_path, "factory_rebuild", "failed", f"{'Factory reboot' if factory_reboot else 'Space restart'} API call failed", {"reason": reason, **exception_payload(exc)}) return False def safe_same_code_reupload(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, reason: str = "") -> bool: """Re-upload the current workspace once, with restart-loop guardrails.""" if not restart_guardrail_allows("same_code_upload", events_path, reason=reason): return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"}: append_event(events_path, "factory_rebuild", "warning", "Skipped same-code upload: Space is already building/starting", {"runtime": runtime_before, "reason": reason}) return False upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) append_event(events_path, "factory_rebuild", "success", "Same-code workspace uploaded under restart guardrails", {"reason": reason, "runtime_before": runtime_before}) return True def collect_space_logs(target_space_id: str, token: str, run_dir: Path, events_path: Path): """Collect Space build/runtime logs with explicit capability detection and an index. This is intentionally more structured than the old best-effort collector: - runtime state is always captured when possible; - runtime/build log collection first uses HfApi.get_space_logs when available; - if the installed huggingface_hub lacks that method, REST fallbacks are attempted; - legacy text files are still written, but availability/quality lives in space_logs_index.json. """ from huggingface_hub import HfApi import json as _json import sys as _sys try: import huggingface_hub as _hf_hub hub_version = getattr(_hf_hub, "__version__", None) except Exception: hub_version = None logs_dir = run_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) api = HfApi(token=token) collected_at = now() written = [] index = { "schema_version": "space_logs_fetch.v198_16", "target_space_id": target_space_id, "collected_at": collected_at, "huggingface_hub_version": hub_version, "python_version": _sys.version.split()[0], "capabilities": { "hfapi_get_space_runtime": hasattr(api, "get_space_runtime"), "hfapi_get_space_logs": hasattr(api, "get_space_logs"), "rest_fallback": True, }, "entries": {}, "hf_space_log_endpoints": { "run": f"https://huggingface.co/api/spaces/{target_space_id}/logs/run", "build": f"https://huggingface.co/api/spaces/{target_space_id}/logs/build", }, "legacy_runtime_filename": "logs/space_logs_runtime.txt", } def _tail(text: str, n: int = 1200) -> str: return (text or "")[-n:] def _stringify_log_response(out) -> str: if out is None: return "" if isinstance(out, (list, tuple)): return "\n".join(str(x) for x in out) if isinstance(out, (dict, list)): return _json.dumps(out, ensure_ascii=False, indent=2) return str(out) def _write_text(filename: str, text: str) -> int: path = logs_dir / filename path.write_text(text or "", encoding="utf-8") return path.stat().st_size if path.exists() else 0 def _entry(key: str, *, filename: str, available: bool, source: str, quality: str, reason: str = "", error: str = "", text: str = ""): size = _write_text(filename, text) payload = { "available": bool(available), "path": f"logs/{filename}", "source": source, "quality": quality, "reason": reason, "error": error[:1500] if error else "", "size_bytes": size, "tail": _tail(text), } index["entries"][key] = payload written.append({ "file": filename, "source": source, "available": bool(available), "quality": quality, "reason": reason, "error": error[:1000] if error else "", "returncode": 0 if available else 1, "tail": _tail(text, 1000), }) return payload def _unavailable_text(kind: str, reason: str, error: str = "") -> str: return ( f"[ASF_LOG_UNAVAILABLE]\n" f"kind={kind}\n" f"reason={reason}\n" f"target_space_id={target_space_id}\n" f"huggingface_hub_version={hub_version}\n" f"error={error[:1500] if error else ''}\n" "See logs/space_logs_index.json for structured availability metadata.\n" ) # Runtime state / snapshot. This is not a log stream, but it is critical context for # BUILD_ERROR/RUNTIME_ERROR triage and is always tracked separately from log streams. try: runtime_obj = api.get_space_runtime(repo_id=target_space_id, token=token) runtime_payload = runtime_to_dict(runtime_obj) runtime_payload.update({"target_space_id": target_space_id, "collected_at": collected_at}) runtime_text = _json.dumps(runtime_payload, indent=2, ensure_ascii=False) _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) # Legacy path retained because older reports/tests point to this file. _entry("runtime_state_legacy", filename="space_runtime_state.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) except Exception as exc: err = f"{type(exc).__name__}: {str(exc)[:1500]}" _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_snapshot", "runtime_snapshot_error", err)) _entry("runtime_state_legacy", filename="space_runtime_state.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_state", "runtime_snapshot_error", err)) def _collect_via_hfapi(kind: str): if not hasattr(api, "get_space_logs"): return None, "hub_method_unavailable", "" try: out = api.get_space_logs(repo_id=target_space_id, token=token, build=(kind == "build")) text = _stringify_log_response(out) if text.strip(): return text, "", "" return "", "empty_response", "" except Exception as exc: return None, "hfapi_error", f"{type(exc).__name__}: {str(exc)[:1500]}" def _space_log_endpoint(kind: str) -> str: # HF Space log streams are exposed under /logs/run and /logs/build. # "runtime" is an ASF legacy label for the file name; the Hub endpoint is "run". endpoint_kind = "build" if kind == "build" else "run" return f"https://huggingface.co/api/spaces/{target_space_id}/logs/{endpoint_kind}" def _read_streaming_response(response, *, max_bytes: int = 1_200_000) -> str: chunks = [] total = 0 try: iterator = response.iter_content(chunk_size=8192, decode_unicode=True) except Exception: return response.text or "" for chunk in iterator: if not chunk: continue if isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="ignore") chunks.append(str(chunk)) total += len(str(chunk).encode("utf-8", errors="ignore")) if total >= max_bytes: chunks.append("\n[ASF_LOG_TRUNCATED: max_bytes reached]\n") break return "".join(chunks) def _collect_via_rest(kind: str): # v198.16: use the real HF Hub Space log endpoints. The runtime stream is # /logs/run, not any legacy runtime endpoint and not the older query-string endpoint. # Wrong endpoints made Pi repair from generic RUNTIME_ERROR messages. try: import requests except Exception as exc: return None, "requests_unavailable", f"{type(exc).__name__}: {str(exc)[:1500]}" headers = {"Authorization": f"Bearer {token}", "Accept": "text/plain,application/json,*/*"} if token else {"Accept": "text/plain,application/json,*/*"} primary_url = _space_log_endpoint(kind) candidates = [primary_url] last_error = "" for url in candidates: try: response = requests.get(url, headers=headers, timeout=(10, 45), stream=True) body = _read_streaming_response(response) if response.ok and body.strip(): ctype = response.headers.get("content-type", "") return body, "", f"{url} ({response.status_code}, {ctype})" last_error = f"{url} returned {response.status_code}: {body[:500]}" except Exception as exc: last_error = f"{url} failed: {type(exc).__name__}: {str(exc)[:500]}" return None, "rest_logs_unavailable", last_error def _collect_stream(kind: str, filename: str): attempted = [] text, reason, error = _collect_via_hfapi(kind) attempted.append({"method": "HfApi.get_space_logs", "reason": reason, "error": error[:500] if error else ""}) if text is not None and text.strip(): return _entry(kind, filename=filename, available=True, source="HfApi.get_space_logs", quality="full", text=text) rest_text, rest_reason, rest_error = _collect_via_rest(kind) attempted.append({"method": "hf_rest_logs", "reason": rest_reason, "error": rest_error[:500] if rest_error else ""}) if rest_text is not None and rest_text.strip(): source = "hf_rest_logs_run_build_endpoint" if rest_error and rest_error.startswith("https://"): source = rest_error return _entry(kind, filename=filename, available=True, source=source, quality="full", text=rest_text) # Keep legacy files present, but explicitly mark them as unavailable rather than # pretending an SDK AttributeError is a Space runtime/build log. This also avoids # the obsolete CLI path that emitted "invalid choice: spaces". final_reason = rest_reason or reason or "logs_unavailable" final_error = rest_error or error or "" entry = _entry(kind, filename=filename, available=False, source="log_collection", quality="unavailable", reason=final_reason, error=final_error, text=_unavailable_text(kind, final_reason, final_error)) entry["attempted_methods"] = attempted return entry runtime_entry = _collect_stream("runtime", "space_logs_runtime.txt") build_entry = _collect_stream("build", "space_logs_build.txt") def _first_error_from(text: str) -> str: if not text: return "" lines = text.splitlines() lowered = [line.lower() for line in lines] markers = ("traceback", "exception", "error", "exit code", "failed", "runtimeerror", "attributeerror", "modulenotfounderror") for idx, line in enumerate(lowered): if any(marker in line for marker in markers): start = max(0, idx - 3) end = min(len(lines), idx + 12) return "\n".join(lines[start:end])[:4000] return "" runtime_text = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if runtime_entry.get("available") and (logs_dir / "space_logs_runtime.txt").exists() else "" build_text = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if build_entry.get("available") and (logs_dir / "space_logs_build.txt").exists() else "" # v198.16 canonical aliases matching the HF API endpoint names. Keep legacy # filenames for compatibility but make the correct run/build source explicit. (logs_dir / "space_logs_run.txt").write_text(runtime_text if runtime_entry.get("available") else _unavailable_text("run", runtime_entry.get("reason") or "run_logs_unavailable", runtime_entry.get("error") or ""), encoding="utf-8") (logs_dir / "space_logs_build.txt").write_text(build_text if build_entry.get("available") else _unavailable_text("build", build_entry.get("reason") or "build_logs_unavailable", build_entry.get("error") or ""), encoding="utf-8") first_error = _first_error_from(runtime_text) or _first_error_from(build_text) log_quality = "full" if runtime_entry.get("available") or build_entry.get("available") else "snapshot_only" if index["entries"].get("runtime_snapshot", {}).get("available") else "unavailable" diagnostics = { "schema_version": "1.0", "target_space_id": target_space_id, "collected_at": collected_at, "log_quality": log_quality, "run_logs_endpoint": _space_log_endpoint("runtime"), "build_logs_endpoint": _space_log_endpoint("build"), "run_logs_available": bool(runtime_entry.get("available")), "runtime_logs_available": bool(runtime_entry.get("available")), "build_logs_available": bool(build_entry.get("available")), "run_log_bytes": len(runtime_text.encode("utf-8", errors="ignore")), "build_log_bytes": len(build_text.encode("utf-8", errors="ignore")), "first_actionable_error": first_error, "first_error": first_error, "diagnosis_log_source": "run_or_build_logs" if first_error else ("runtime_snapshot" if log_quality == "snapshot_only" else "none"), } diagnostics_text = _json.dumps(diagnostics, ensure_ascii=False, indent=2) _entry("diagnostics", filename="space_log_diagnostics.json", available=True, source="asf_log_collector", quality=log_quality, text=diagnostics_text) index["log_quality"] = log_quality index["run_logs_available"] = bool(runtime_entry.get("available")) index["runtime_logs_available"] = bool(runtime_entry.get("available")) index["build_logs_available"] = bool(build_entry.get("available")) index["first_actionable_error"] = first_error[:4000] index["first_error"] = first_error[:1000] fetch_status = { "schema_version": "space_logs_fetch_status.v198_16", "space_id": target_space_id, "run_logs_endpoint": _space_log_endpoint("runtime"), "build_logs_endpoint": _space_log_endpoint("build"), "run_logs_fetched": bool(runtime_entry.get("available")), "build_logs_fetched": bool(build_entry.get("available")), "run_log_bytes": len(runtime_text.encode("utf-8", errors="ignore")), "build_log_bytes": len(build_text.encode("utf-8", errors="ignore")), "first_actionable_error_source": "run" if _first_error_from(runtime_text) else ("build" if _first_error_from(build_text) else "none"), "first_actionable_error": first_error[:4000], "pi_repair_allowed_from_logs": bool(first_error), } fetch_text = _json.dumps(fetch_status, ensure_ascii=False, indent=2) (logs_dir / "space_logs_fetch_status.json").write_text(fetch_text, encoding="utf-8") written.append({"file": "space_logs_fetch_status.json", "source": "asf_log_collector", "available": True, "quality": log_quality, "returncode": 0, "tail": _tail(fetch_text, 1000)}) actionable_brief = ( "# Actionable Space log brief\n\n" f"Space: `{target_space_id}`\n\n" f"Run logs endpoint: `{_space_log_endpoint('runtime')}`\n\n" f"Build logs endpoint: `{_space_log_endpoint('build')}`\n\n" f"Run logs fetched: `{bool(runtime_entry.get('available'))}`\n\n" f"Build logs fetched: `{bool(build_entry.get('available'))}`\n\n" "## First actionable error\n\n" "```text\n" + (first_error or "No actionable error found in fetched Space run/build logs.")[:4000] + "\n```\n" ) (run_dir / "repair").mkdir(parents=True, exist_ok=True) (run_dir / "repair" / "actionable_error_brief.md").write_text(actionable_brief, encoding="utf-8") index_text = _json.dumps(index, ensure_ascii=False, indent=2) (logs_dir / "space_logs_index.json").write_text(index_text, encoding="utf-8") written.append({"file": "space_logs_index.json", "source": "asf_log_collector", "available": True, "quality": log_quality, "returncode": 0, "tail": _tail(index_text, 1000)}) event_status = "success" if runtime_entry.get("available") or build_entry.get("available") else "warning" event_message = "Collected HF Space run/build logs and runtime snapshot" if event_status == "success" else "HF Space run/build log streams unavailable; runtime snapshot/index written" append_event(events_path, "space_logs", event_status, event_message, {"files": written, "log_quality": log_quality, "index": "logs/space_logs_index.json"}) write_artifact_manifest(run_dir, reason="space_logs_collected") return written def validate_http_health(target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int): import requests base_url = space_subdomain_url(target_space_id) url = base_url.rstrip("/") + "/health" headers = {"Authorization": f"Bearer {token}", "Accept": "application/json,text/plain,*/*"} response = requests.get(url, headers=headers, timeout=20) payload = { "status": "success" if response.ok else "failed", "attempt": attempt, "url": url, "status_code": response.status_code, "content_type": response.headers.get("content-type"), "text": response.text[:2000], } if response.ok: try: payload["json"] = response.json() except Exception: pass semantic = validation_health_semantics(payload | {"validator": "http_get_health"}) payload["health_endpoint_reachable"] = True payload["health_semantic_passed"] = semantic.get("semantic_passed") payload["health_semantics"] = semantic write_json(run_dir / "tests" / "http_health.json", payload) write_json(run_dir / "tests" / "test_result.json", payload | {"validator": "http_get_health"}) event_status = "success" if semantic.get("semantic_passed") is not False else "warning" event_message = "HTTP /health validation passed" if event_status == "success" else "HTTP /health is reachable but reported unhealthy/not-ready semantic state" append_event(events_path, "api_validation", event_status, event_message, {"attempt": attempt, "url": url, "status_code": response.status_code, "health_semantic_passed": semantic.get("semantic_passed")}) return payload | {"validator": "http_get_health"} raise RuntimeError(f"HTTP /health returned {response.status_code}: {response.text[:500]}") def validate_gradio_api(target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int): client = make_gradio_client(target_space_id, token) identity = write_gradio_client_identity(client, target_space_id, run_dir, events_path, phase="api_validation") if identity.get("mismatch"): raise RuntimeError(f"space_identity_mismatch: expected {target_space_id}, observed {identity.get('observed')}") schema = client.view_api(return_format="dict") write_json(run_dir / "tests" / "api_schema.json", schema if isinstance(schema, dict) else {"schema": str(schema)}) discovered = api_names_from_schema(schema) candidates = [] for name in ["/health", "/predict", "/greet"] + discovered: if name not in candidates: candidates.append(name) errors = [] for api_name in candidates: try: if api_name == "/greet": result = client.predict("Agentic Space Factory", api_name=api_name) else: result = client.predict(api_name=api_name) payload = {"status": "success", "attempt": attempt, "api_name": api_name, "discovered_api_names": discovered, "result_repr": repr(result)[:2000], "validator": "gradio_client"} write_json(run_dir / "tests" / "test_result.json", payload) append_event(events_path, "api_validation", "success", "Gradio API validation passed", {"attempt": attempt, "api_name": api_name, "discovered_api_names": discovered}) return payload except Exception as exc: errors.append({"api_name": api_name, "error": str(exc)[:1000]}) raise RuntimeError("; ".join(f"{e['api_name']}: {e['error']}" for e in errors[:5]) or "No callable API endpoints found") def latest_collected_build_log(run_dir: Path) -> str: path = run_dir / "logs" / "space_logs_build.txt" return path.read_text(encoding="utf-8", errors="ignore") if path.exists() else "" def runtime_stage_is_build_error(stage: str) -> bool: value = (stage or "").upper() return "BUILD_ERROR" in value or value in {"BUILDING_ERROR", "ERROR", "FAILED", "BUILD_FAILED"} def runtime_stage_is_busy(stage: str) -> bool: value = (stage or "").upper() return any(marker in value for marker in ["BUILDING", "BUILDQUEUED", "BUILD_QUEUED", "STARTING", "PENDING"]) def build_log_has_terminal_error(text: str) -> bool: low = (text or "").lower() terminal_markers = [ "--> error:", "error: resolutionimpossible", "resolutionimpossible", "no matching distribution found", "could not find a version that satisfies the requirement", "did not complete successfully", "process \"/bin/sh -c pip install", "cannot install", "conflicting dependencies", "metadata-generation-failed", "exit code: 1", ] return any(marker in low for marker in terminal_markers) def _parse_iso_timestamp_epoch(value: str | None) -> float | None: raw = str(value or "").strip() if not raw: return None try: if raw.endswith("Z"): raw = raw[:-1] + "+00:00" return datetime.fromisoformat(raw).timestamp() except Exception: return None def _log_block_timestamp_epoch(block: str) -> float | None: matches = re.findall(r'"timestamp"\s*:\s*"([^"]+)"', block or "") epochs = [_parse_iso_timestamp_epoch(item) for item in matches] epochs = [item for item in epochs if item is not None] return max(epochs) if epochs else None def get_last_runtime_upload_epoch(run_dir: Path) -> dict: payload = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {} return payload if isinstance(payload, dict) else {} def get_last_runtime_upload_at(run_dir: Path) -> str: payload = get_last_runtime_upload_epoch(run_dir) return str(payload.get("last_upload_completed_at") or payload.get("uploaded_at") or "") def filter_build_log_since(text: str, since_iso: str | None = None) -> str: """Return only build-log SSE blocks at or after the latest upload epoch. Hugging Face build-log streams can include previous failed build attempts for the same Space. v198.26.1 keeps those lines in archived logs but never lets them drive the active validation state after a later upload/rebuild. """ raw = text or "" since_epoch = _parse_iso_timestamp_epoch(since_iso) if since_epoch is None: return raw blocks = re.split(r"(\n\s*\n)", raw) kept: list[str] = [] stale_count = 0 for i in range(0, len(blocks), 2): block = blocks[i] sep = blocks[i + 1] if i + 1 < len(blocks) else "" if not block: continue ts = _log_block_timestamp_epoch(block) if ts is not None and ts + 0.001 < since_epoch: stale_count += 1 continue kept.append(block + sep) return "".join(kept) def build_log_freshness_payload(raw_text: str, fresh_text: str, since_iso: str | None = None) -> dict: raw_issue = extract_pip_dependency_issue(raw_text or "") fresh_issue = extract_pip_dependency_issue(fresh_text or "") raw_terminal = build_log_has_terminal_error(raw_text or "") fresh_terminal = build_log_has_terminal_error(fresh_text or "") return { "schema_version": "build_log_freshness.v198_25_3", "last_upload_completed_at": since_iso or "", "raw_bytes": len((raw_text or "").encode("utf-8", errors="ignore")), "fresh_bytes": len((fresh_text or "").encode("utf-8", errors="ignore")), "raw_terminal_error_detected": bool(raw_terminal), "fresh_terminal_error_detected": bool(fresh_terminal), "raw_dependency_issue_detected": bool(raw_issue), "fresh_dependency_issue_detected": bool(fresh_issue), "stale_build_error_ignored": bool((raw_terminal or raw_issue) and not (fresh_terminal or fresh_issue) and since_iso), } def latest_fresh_collected_build_log(run_dir: Path) -> str: return filter_build_log_since(latest_collected_build_log(run_dir), get_last_runtime_upload_at(run_dir)) def record_runtime_upload_epoch(run_dir: Path, events_path: Path | None, *, target_space_id: str, manifest: dict | None = None) -> dict: existing = get_last_runtime_upload_epoch(run_dir) history = existing.get("history") if isinstance(existing.get("history"), list) else [] sequence = int(existing.get("upload_sequence") or len(history) or 0) + 1 uploaded_at = now() entry = { "sequence": sequence, "phase": "initial_upload" if sequence == 1 else "repair_upload", "uploaded_at": uploaded_at, "target_space_id": target_space_id, "file_count": (manifest or {}).get("file_count"), "total_bytes": (manifest or {}).get("total_bytes"), } history.append(entry) payload = { "schema_version": "runtime_upload_epoch.v198_25_3", "target_space_id": target_space_id, "upload_sequence": sequence, "last_upload_completed_at": uploaded_at, "last_upload_phase": entry["phase"], "history": history[-20:], "updated_at": uploaded_at, } write_json(run_dir / "runtime_upload_epoch.json", payload) if events_path is not None: append_event(events_path, "runtime_upload_epoch", "success", "Recorded runtime upload epoch for stale build-log filtering", {k: payload[k] for k in ["upload_sequence", "last_upload_completed_at", "last_upload_phase", "target_space_id"]}) return payload def clear_active_repair_error_after_success(run_dir: Path, events_path: Path | None = None, *, final_gate_status: str = "full_inference_success") -> dict: outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {} if not isinstance(outcome, dict) or not outcome: return {} history = outcome.get("historical_failures") if isinstance(outcome.get("historical_failures"), list) else [] if outcome.get("failure_type") or outcome.get("validation_error") or str(outcome.get("post_repair_validation") or "").lower() not in {"", "success"}: history.append({ "phase": "pre_final_success", "failure_type": outcome.get("failure_type") or "", "validation_error": outcome.get("validation_error") or "", "post_repair_validation": outcome.get("post_repair_validation") or "", "recorded_at": now(), }) outcome.update({ "post_repair_validation": "success" if outcome.get("patch_applied") or outcome.get("upload_success") else outcome.get("post_repair_validation", ""), "failure_type": "", "validation_error": "", "final_gate_status": final_gate_status, "active_error_cleared_by_final_success": True, "historical_failures": history[-20:], "final_user_message": "Final full-inference gate passed; previous dependency/build errors are historical.", "updated_at": now(), }) write_json(run_dir / "repair_outcome.json", outcome) try: write_json(run_dir / "repair" / "REPAIR_OUTCOME.json", outcome) except Exception: pass if events_path is not None: append_event(events_path, "final_status_reconciliation", "success", "Cleared active repair error fields after final full-inference success", {"final_gate_status": final_gate_status, "historical_failure_count": len(history)}) return outcome def final_full_inference_success_recorded(run_dir: Path) -> bool: gate = load_json_if_exists(run_dir / "inference_gate.json") if (run_dir / "inference_gate.json").exists() else {} state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {} smoke = load_json_if_exists(run_dir / "tests" / "generation_smoke.json") if (run_dir / "tests" / "generation_smoke.json").exists() else {} signals = gate.get("implementation_signals") if isinstance(gate.get("implementation_signals"), dict) else {} return bool( (gate.get("status") == "full_inference_success" or state.get("status") == "full_inference_success") and (smoke.get("status") == "success" or signals.get("generation_smoke_passed") is True) ) def reconcile_final_success_state(run_dir: Path, events_path: Path | None, final_state: dict, inference_gate: dict | None = None, generation_smoke: dict | None = None) -> dict: gate = inference_gate or {} smoke = generation_smoke or {} status = str(gate.get("status") or final_state.get("status") or "") if status != "full_inference_success": return final_state clean = dict(final_state or {}) clean.update({ "status": "full_inference_success", "failure_type": "", "validation_error": "", "active_error": None, "job_exit_status": "success", "job_exit_code": 0, "recovered": bool(load_json_if_exists(run_dir / "repair_outcome.json")), "final_status_reconciled_at": now(), }) details = clean.get("details") if isinstance(clean.get("details"), dict) else {} if details: historical = clean.get("historical_failures") if isinstance(clean.get("historical_failures"), list) else [] if details.get("error") or details.get("failure_type") or details.get("validation_error"): historical.append({"phase": "pre_final_success_details", "details": details, "recorded_at": now()}) clean["historical_failures"] = historical[-20:] clean.pop("details", None) repair_outcome = clear_active_repair_error_after_success(run_dir, events_path, final_gate_status="full_inference_success") if repair_outcome: clean["repair_outcome"] = repair_outcome write_json(run_dir / "state.json", clean) write_json(run_dir / "final_status_reconciliation.json", { "schema_version": "final_status_reconciliation.v198_25_3", "status": "full_inference_success", "job_exit_code": 0, "active_errors_cleared": True, "health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")), "generation_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success"), "updated_at": now(), }) return clean def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None, final_state: dict, *, status: str, message: str = "", details: dict | None = None, job_exit_code: int | None = None) -> dict: """Persist a terminal truth artifact for every final outcome. v198.26.9: the UI must not have to infer failed/stopped states from a mix of state, repair_outcome and recent events. Success already writes final_status_reconciliation.json; failed/auth/cancelled terminal paths now write the same artifact so Active Run can converge deterministically. """ state = final_state or {} safe = safe_details(details or {}) runtime_upload_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {} space_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {} runtime_history = runtime_upload_epoch.get("history") if isinstance(runtime_upload_epoch.get("history"), list) else [] target_space = state.get("target_space") or state.get("target_space_id") or runtime_upload_epoch.get("target_space_id") or os.environ.get("TARGET_SPACE_ID", "") runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history) space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url"))) payload = { "schema_version": "final_status_reconciliation.v198_26_9", "status": status, "terminal": True, "message": message or state.get("message") or "", "job_exit_code": int(job_exit_code if job_exit_code is not None else (0 if status == "full_inference_success" else 1)), "target_space": target_space, "target_space_url": f"https://huggingface.co/spaces/{target_space}" if target_space else "", "space_created": bool(target_space and (runtime_uploaded or space_runtime_known)), "runtime_uploaded": bool(runtime_uploaded), "space_uploaded": bool(runtime_uploaded), "space_runtime_known": bool(space_runtime_known), "failure_type": safe.get("failure_type") or safe.get("repair_failure_type") or state.get("failure_type") or "", "failure_phase": safe.get("phase") or safe.get("failure_phase") or safe.get("step") or "", "details": safe, "updated_at": now(), } write_json(run_dir / "final_status_reconciliation.json", payload) if events_path is not None: append_event(events_path, "final_status_reconciliation", "success" if status == "full_inference_success" else "failed", "Wrote terminal final status reconciliation", {k: payload.get(k) for k in ("status", "job_exit_code", "target_space", "runtime_uploaded", "space_runtime_known", "failure_type")}) return payload def _normalize_dependency_package_name(value: str = "") -> str: name = re.split(r"[<>=!~;\[,\s]", str(value or "").strip(), 1)[0].strip().lower().replace("_", "-") aliases = { "huggingface-hub": "huggingface-hub", "huggingface_hub": "huggingface-hub", "gradio-client": "gradio-client", "gradio_client": "gradio-client", "opencv-python-headless": "opencv-python-headless", "opencv-python": "opencv-python", } return aliases.get(name, name) def extract_dependency_conflict_packages(text: str = "") -> list[str]: """Best-effort package extraction from pip resolver conflicts. This intentionally favors useful signal over perfect parsing. Pip emits several formats (plain text, JSON-escaped SSE `data:` lines, long candidate lists); the Factory only needs enough structure to route recovery safely. """ raw = text or "" packages: list[str] = [] def add(candidate: str = ""): name = _normalize_dependency_package_name(candidate) if name and re.match(r"^[a-z0-9][a-z0-9_.-]*$", name) and name not in packages: packages.append(name) for match in re.finditer(r"\b([A-Za-z0-9_.-]+)\s+(?:[A-Za-z0-9_.+-]+\s+)?depends on\s+([A-Za-z0-9_.-]+)", raw, flags=re.I): add(match.group(1)); add(match.group(2)) for match in re.finditer(r"The user requested\s+([A-Za-z0-9_.-]+)", raw, flags=re.I): add(match.group(1)) for match in re.finditer(r"Cannot install\s+(.+?)\s+because these package versions have conflicting dependencies", raw, flags=re.I | re.S): chunk = match.group(1) chunk = re.sub(r"-r\s+\S+\s+\(line\s+\d+\)", " ", chunk, flags=re.I) for token in re.split(r"[,\s]+|\band\b", chunk): token = token.strip(" .;:'\"`()[]{}") if token and not token.startswith("-") and token.lower() not in {"the", "user", "requested", "install"}: add(token) for match in re.finditer(r"(?:No matching distribution found for|Could not find a version that satisfies the requirement)\s+([^\s]+)", raw, flags=re.I): add(match.group(1)) return packages[:20] def detect_dependency_platform_conflict(text: str = "", packages: list[str] | None = None) -> dict: raw = text or "" low = raw.lower() packages = packages or extract_dependency_conflict_packages(raw) pkg_set = set(packages) hub_floor = bool(re.search(r"gradio\s+[A-Za-z0-9_.+-]*\s*depends on\s+huggingface[-_]hub\s*<\s*2(?:\.0)?\s*(?:,|and)\s*>=\s*1(?:\.2)?", low, flags=re.I)) transformers_hub_ceiling = bool(re.search(r"transformers\s+[A-Za-z0-9_.+-]*\s*depends on\s+huggingface[-_]hub\s*<\s*1(?:\.0)?", low, flags=re.I)) detected = hub_floor and transformers_hub_ceiling and {"gradio", "transformers", "huggingface-hub"}.issubset(pkg_set) return { "detected": bool(detected), "kind": "gradio_transformers_hub_range_conflict" if detected else "none", "packages": packages, "evidence": { "gradio_requires_hub_1_or_newer": hub_floor, "transformers_requires_hub_below_1": transformers_hub_ceiling, }, "recommended_action": "surgical_requirements_repair_or_dependency_platform_conflict" if detected else "standard_dependency_repair", } def write_dependency_issue_artifact(run_dir: Path, issue: dict, *, source: str = "build_logs", runtime_payload: dict | None = None) -> dict: payload = { "schema_version": "dependency_issue.v198_25_3", "created_at": now(), "source": source, "runtime": runtime_payload or {}, **(issue or {}), } if "failure_owner" not in payload: payload["failure_owner"] = "dependency" if "failure_class" not in payload: payload["failure_class"] = "dependency_resolution_conflict" if "repair_candidate" not in payload: payload["repair_candidate"] = True write_json(run_dir / "dependency_issue.json", payload) return payload def collect_space_build_logs_fast(target_space_id: str, token: str, run_dir: Path, events_path: Path, *, attempt: int = 0, reason: str = "", max_seconds: int = 12, max_bytes: int = 800_000) -> dict: """Fast, bounded read of the HF Space build log stream. The full log collector intentionally archives run/build streams, but HF's text/event-stream endpoint can stay open while BUILDING. During live API validation we only need enough build-log text to notice terminal pip/build errors. This probe exits as soon as a terminal marker is seen, max_bytes is reached, or a small time/read timeout is hit. """ logs_dir = run_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) status_path = logs_dir / "build_log_fast_probe_status.json" url = f"https://huggingface.co/api/spaces/{target_space_id}/logs/build" headers = {"Authorization": f"Bearer {token}", "Accept": "text/plain,application/json,*/*"} if token else {"Accept": "text/plain,application/json,*/*"} collected = "" terminal = False error = "" available = False source = f"{url} fast_probe" last_upload_at = get_last_runtime_upload_at(run_dir) started = time.time() try: import requests response = requests.get(url, headers=headers, timeout=(5, max(5, int(max_seconds))), stream=True) source = f"{url} ({response.status_code}, {response.headers.get('content-type', '')}) fast_probe" if not response.ok: try: collected = response.text[:4000] except Exception: collected = "" error = f"http_status={response.status_code}" else: chunks: list[str] = [] total = 0 available = True try: iterator = response.iter_content(chunk_size=8192, decode_unicode=True) except Exception: iterator = [] chunks.append(response.text or "") for chunk in iterator: if not chunk: if time.time() - started >= max_seconds: break continue if isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="ignore") chunks.append(str(chunk)) total += len(str(chunk).encode("utf-8", errors="ignore")) current = "".join(chunks) if build_log_has_terminal_error(filter_build_log_since(current, last_upload_at)): terminal = True break if total >= max_bytes: chunks.append("\n[ASF_FAST_BUILD_LOG_TRUNCATED: max_bytes reached]\n") break if time.time() - started >= max_seconds: chunks.append("\n[ASF_FAST_BUILD_LOG_TRUNCATED: max_seconds reached]\n") break collected = "".join(chunks) except Exception as exc: error = f"{type(exc).__name__}: {str(exc)[:1000]}" finally: try: if 'response' in locals(): response.close() except Exception: pass fresh_collected = filter_build_log_since(collected, last_upload_at) if collected.strip(): # Preserve accumulated full-ish build logs when multiple probes run, but # classify active errors only from fresh blocks after the latest upload. path = logs_dir / "space_logs_build.txt" previous = path.read_text(encoding="utf-8", errors="ignore") if path.exists() else "" if collected not in previous: combined = (previous + "\n" + collected).strip() if previous.strip() else collected path.write_text(combined[-2_000_000:], encoding="utf-8") issue = extract_pip_dependency_issue(fresh_collected) if fresh_collected.strip() else {} freshness = build_log_freshness_payload(collected, fresh_collected, last_upload_at) payload = { "schema_version": "build_log_fast_probe.v198_25_3", "target_space_id": target_space_id, "attempt": attempt, "reason": reason, "available": bool(available or collected.strip()), "terminal_error_detected": bool(build_log_has_terminal_error(fresh_collected)), "dependency_issue_detected": bool(issue), "dependency_issue": issue, "source": source, "error": error[:1500] if error else "", "bytes_read": len((collected or "").encode("utf-8", errors="ignore")), "fresh_bytes_read": len((fresh_collected or "").encode("utf-8", errors="ignore")), "last_upload_completed_at": last_upload_at, "freshness": freshness, "stale_build_error_ignored": bool(freshness.get("stale_build_error_ignored")), "elapsed_seconds": round(time.time() - started, 3), "created_at": now(), } status_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") if payload["terminal_error_detected"] or payload["dependency_issue_detected"]: append_event(events_path, "build_log_fast_probe", "warning", "Detected terminal build/dependency error from bounded build-log probe", payload) return payload def raise_if_collected_build_error(target_space_id: str, token: str, run_dir: Path, events_path: Path, *, attempt: int, reason: str = "", runtime_payload: dict | None = None): stage = str((runtime_payload or {}).get("stage") or "") # Fast path for BUILDING: never let the full log stream block the live # validation loop for tens of minutes when pip has already emitted a # terminal resolver error. fast_probe = collect_space_build_logs_fast(target_space_id, token, run_dir, events_path, attempt=attempt, reason=reason) build_log = latest_fresh_collected_build_log(run_dir) has_terminal_build_log = build_log_has_terminal_error(build_log) stage_is_terminal = runtime_stage_is_build_error(stage) # Once the Hub reports a terminal build stage, make one archival collection # attempt for better artifacts. During plain BUILDING we rely only on the # bounded fast probe to avoid stream stalls. if stage_is_terminal and not has_terminal_build_log: try: collect_space_logs(target_space_id, token, run_dir, events_path) build_log = latest_fresh_collected_build_log(run_dir) has_terminal_build_log = build_log_has_terminal_error(build_log) except Exception as exc: append_event(events_path, "space_logs", "warning", "Full log collection failed after terminal build stage; using fast probe evidence", {"error": f"{type(exc).__name__}: {str(exc)[:1000]}", "fast_probe": fast_probe}) if stage_is_terminal or has_terminal_build_log or fast_probe.get("terminal_error_detected"): issue = extract_pip_dependency_issue(build_log) if issue: issue = write_dependency_issue_artifact(run_dir, issue, source="fast_build_log_probe" if fast_probe.get("dependency_issue_detected") else "build_logs", runtime_payload=runtime_payload or {}) payload = { "schema_version": "build_error_observation.v198_25_3", "attempt": attempt, "reason": reason, "runtime": runtime_payload or {}, "fast_probe": fast_probe, "dependency_issue": issue, "failure_owner": "dependency" if issue else "space_build", "failure_class": (issue or {}).get("failure_class") or ("space_build_error" if stage_is_terminal else "build_log_terminal_error"), "repair_candidate": bool((issue or {}).get("repair_candidate", bool(issue))), "display_status": "dependency_error" if issue else "build_error", "effective_status": "dependency_error" if issue else "build_error", "tail": build_log[-4000:], "created_at": now(), } append_event(events_path, "api_validation", "failed", "Space build/dependency error detected; stopping wait and entering recovery", payload) write_json(run_dir / "build_error_observation.json", payload) if issue: raise RuntimeError(f"Space build dependency error detected from bounded build-log probe. {json.dumps(issue, ensure_ascii=False)[:1000]} See dependency_issue.json, build_error_observation.json, and logs/space_logs_build.txt") raise RuntimeError(f"Space build failed according to runtime/logs. See build_error_observation.json and logs/space_logs_build.txt") TRANSIENT_SCHEDULING_MARKERS = [ "scheduling failure", "not enough hardware capacity", "unable to schedule", "capacity unavailable", "hardware capacity", ] DEFINITIVE_APP_ERROR_MARKERS = [ "modulenotfounderror", "importerror", "syntaxerror", "typeerror", "attributeerror", "nameerror", "resolutionimpossible", "no matching distribution found", "could not find a version that satisfies the requirement", "cuda out of memory", "outofmemoryerror", "traceback (most recent call last)", ] STARTUP_STALL_MARKERS = [ "app_starting", "starting", "live health/api validation did not pass before timeout", "http /health returned 404", "gradio api failed", "timed out", "timeout", ] def _combined_runtime_failure_text(failure_reason: str = "", build_log: str = "", runtime_log: str = "", runtime_payload: dict | None = None) -> str: return f"{failure_reason}\n{build_log[-12000:]}\n{runtime_log[-12000:]}\n{json.dumps(runtime_payload or {}, ensure_ascii=False)}".lower() def has_definitive_app_error(failure_reason: str = "", build_log: str = "", runtime_log: str = "") -> bool: text = _combined_runtime_failure_text(failure_reason, build_log, runtime_log) if extract_pip_dependency_issue(text): return True return any(marker in text for marker in DEFINITIVE_APP_ERROR_MARKERS) def classify_runtime_recovery_need(failure_reason: str = "", build_log: str = "", runtime_log: str = "", runtime_payload: dict | None = None) -> dict: """Classify transient HF runtime states that deserve one recovery before Pi repair/fail. v198.5: HF Space scheduling/startup can stall without an actionable app traceback. In those cases the Factory should try one bounded factory reboot before declaring failure, calling Pi repair, or pausing the generated Space. """ text = _combined_runtime_failure_text(failure_reason, build_log, runtime_log, runtime_payload) stage = runtime_stage_value(runtime_payload or {}) definitive = has_definitive_app_error(failure_reason, build_log, runtime_log) scheduling = any(marker in text for marker in TRANSIENT_SCHEDULING_MARKERS) app_starting = "app_starting" in stage or stage == "starting" or "app_starting" in text startup_stall = app_starting or (any(marker in text for marker in STARTUP_STALL_MARKERS) and not definitive) if scheduling: return { "triggered": True, "failure_class": "infra_transient_scheduling", "reason": "Scheduling/capacity failure may be transient on Hugging Face infrastructure.", "recommended_action": "factory_rebuild_before_fail", "repair_should_be_deferred": True, "definitive_app_error": definitive, "runtime_stage": stage, } if startup_stall and not definitive: return { "triggered": True, "failure_class": "startup_stalled_no_definitive_traceback", "reason": "Space stayed in startup/health timeout without a definitive app traceback.", "recommended_action": "factory_rebuild_before_repair", "repair_should_be_deferred": True, "definitive_app_error": definitive, "runtime_stage": stage, } if startup_stall and definitive: return { "triggered": False, "failure_class": "startup_stalled_with_definitive_app_error", "reason": "Startup stalled, but logs contain an actionable app/dependency error; repair can proceed directly.", "recommended_action": "repair_direct", "repair_should_be_deferred": False, "definitive_app_error": definitive, "runtime_stage": stage, } return { "triggered": False, "failure_class": "not_recoverable_by_runtime_retry", "reason": "No transient scheduling/startup pattern detected.", "recommended_action": "normal_repair_protocol", "repair_should_be_deferred": False, "definitive_app_error": definitive, "runtime_stage": stage, } def write_runtime_recovery(run_dir: Path, payload: dict) -> dict: existing = load_json_if_exists(run_dir / "runtime_recovery.json") if (run_dir / "runtime_recovery.json").exists() else {} if not isinstance(existing, dict): existing = {} merged = {"schema_version": "runtime_recovery.v198_5", **existing, **payload, "updated_at": now()} write_json(run_dir / "runtime_recovery.json", merged) # startup_recovery.json remains as a convenience artifact for UI/debugging. if str(merged.get("failure_class") or "").startswith("startup") or "startup" in str(merged.get("failure_class") or ""): write_json(run_dir / "startup_recovery.json", merged) return merged def attempt_runtime_recovery_before_repair(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, failure_reason: str, build_log: str = "", runtime_log: str = "", runtime_payload: dict | None = None) -> dict: classification = classify_runtime_recovery_need(failure_reason, build_log, runtime_log, runtime_payload) recovery = write_runtime_recovery(run_dir, { **classification, "actions": [], "recovery_attempted": False, "recovery_exhausted": not classification.get("triggered"), }) if not classification.get("triggered"): return recovery append_event(events_path, "runtime_recovery", "started", "Attempting one HF runtime recovery before repair/fail", classification) write_repair_outcome( run_dir, events_path, pre_repair_recovery_attempted=True, pre_repair_recovery_action="factory_reboot", repair_deferred_until_after_rebuild=True, repair_decision="runtime_recovery_before_repair", post_repair_validation="pending", ) ok = safe_restart_space( api, target_space_id, token, run_dir, events_path, factory_reboot=True, reason=classification.get("failure_class") or "runtime_recovery", require_logs_checked=False, allow_busy=True, ) action = {"type": "factory_reboot", "attempt": 1, "requested": bool(ok)} if not ok: recovery = write_runtime_recovery(run_dir, { **classification, "actions": [action | {"result": "request_failed_or_guarded"}], "recovery_attempted": True, "recovery_exhausted": True, "result": "recovery_request_failed", }) append_event(events_path, "runtime_recovery", "failed", "HF runtime recovery request was blocked or failed", recovery) return recovery recovery = write_runtime_recovery(run_dir, { **classification, "actions": [action | {"result": "requested"}], "recovery_attempted": True, "recovery_exhausted": False, "result": "recovery_requested", }) append_event(events_path, "runtime_recovery", "success", "HF runtime recovery requested; revalidation should run before Pi repair", recovery) return recovery def validate_live_api(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, timeout_s: int = 900): append_event(events_path, "api_validation", "started", "Waiting for live HTTP /health or Gradio API to become available") deadline = time.time() + timeout_s attempt = 0 last_error = None runtime_error_count = 0 while time.time() < deadline: attempt += 1 runtime_payload = write_space_runtime(api, target_space_id, token, run_dir, events_path, attempt) stage = str(runtime_payload.get("stage") or "").upper() # Never keep waiting when the Hub already reports a BUILD_ERROR. This # is the critical observer loop: manual refresh should not be required. if runtime_stage_is_build_error(stage): raise_if_collected_build_error(target_space_id, token, run_dir, events_path, attempt=attempt, reason=f"runtime_stage={stage}", runtime_payload=runtime_payload) # While the Space is building/starting, periodically inspect build logs. # A terminal pip/build error in logs is enough to enter recovery even if # runtime state has not propagated yet. if attempt == 1 or attempt % 2 == 0 or any(marker in stage for marker in ["BUILD", "ERROR", "RUNTIME"]): raise_if_collected_build_error(target_space_id, token, run_dir, events_path, attempt=attempt, reason=f"runtime_stage={stage}", runtime_payload=runtime_payload) if "RUNTIME_ERROR" in stage: runtime_error_count += 1 last_error = f"Space runtime stage is {stage}" if runtime_error_count >= 2: raise RuntimeError(f"Space is in RUNTIME_ERROR. See logs/space_logs_runtime.txt and logs/space_logs_build.txt. Last runtime: {runtime_payload}") try: return validate_http_health(target_space_id, token, run_dir, events_path, attempt) except Exception as exc: last_error = f"HTTP /health failed: {exc}" try: return validate_gradio_api(target_space_id, token, run_dir, events_path, attempt) except Exception as exc: if "space_identity_mismatch" in str(exc): append_event(events_path, "api_validation", "failed", "Space identity mismatch is terminal; stopping live API wait", {"attempt": attempt, "runtime": runtime_payload, "error": str(exc)[:1500]}) raise last_error = (last_error or "") + f"; Gradio API failed: {exc}" append_event(events_path, "api_validation", "waiting", "Live health/API not ready yet", {"attempt": attempt, "runtime": runtime_payload, "error": last_error[-1500:] if last_error else None}) time.sleep(30) collect_space_logs(target_space_id, token, run_dir, events_path) raise RuntimeError(f"Live health/API validation did not pass before timeout: {last_error}") def is_auth_or_billing_like_error(error: str | None) -> bool: value = error or "" markers = [ "401", "402", "403", "Invalid username or password", "Unauthorized", "Repository Not Found", "payment", "billing", "quota", "grant", ] return any(marker.lower() in value.lower() for marker in markers) def request_hardware(api, target_space_id: str, hardware: str, token: str, events_path: Path, step: str, retries: int = 2): """Best-effort hardware request after Space creation. V23 tries hardware at create_repo time first. This function remains as a fallback for cases where a Space was created on CPU and the Hub later accepts a hardware switch. Auth/billing/quota errors are not retried. """ if not hardware: return {"phase": "post_create_request", "requested": False, "hardware": hardware, "ok": False, "error": "empty hardware"} last_error = None for attempt in range(1, retries + 1): try: runtime = api.request_space_hardware(repo_id=target_space_id, hardware=hardware, token=token) payload = { "phase": "post_create_request", "requested": True, "hardware": hardware, "ok": True, "attempt": attempt, "runtime_stage": getattr(getattr(runtime, "stage", None), "value", str(getattr(runtime, "stage", None))), "requested_hardware": getattr(runtime, "requested_hardware", None), "hardware_current": getattr(runtime, "hardware", None), } append_event(events_path, step, "success", f"Requested Space hardware {hardware}", payload) return payload except Exception as exc: exc_payload = exception_payload(exc) last_error = str(exc_payload.get("hf_error_message") or exc_payload.get("hf_error_detail") or exc_payload.get("error") or "")[:4000] auth_like = is_auth_or_billing_like_error(last_error) payload = {"phase": "post_create_request", "attempt": attempt, "hardware": hardware, "error": last_error, "manual_action_required": auth_like, **exc_payload} append_event(events_path, step, "failed" if auth_like or attempt == retries else "waiting", f"Could not request Space hardware {hardware}", payload) if auth_like: return {"phase": "post_create_request", "requested": True, "hardware": hardware, "ok": False, "attempts": attempt, "error": last_error, "manual_action_required": True, **exc_payload} if attempt < retries: time.sleep(8 * attempt) return {"phase": "post_create_request", "requested": True, "hardware": hardware, "ok": False, "attempts": retries, "error": last_error, "manual_action_required": False} def normalize_auto_space_hardware(value: str | None, default: str) -> str: candidate = (value or default).strip() return candidate if candidate in AUTO_SPACE_HARDWARE_CHOICES else default def build_hardware_sequence(preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, try_zero_gpu_first: bool = True) -> list[str]: sequence = [] preferred_hardware = normalize_auto_space_hardware(preferred_hardware, DEFAULT_PREFERRED_SPACE_HARDWARE) fallback_hardware = normalize_auto_space_hardware(fallback_hardware, DEFAULT_FALLBACK_SPACE_HARDWARE) planned = [] if try_zero_gpu_first: planned.append("zero-a10g") planned.append(preferred_hardware) if allow_fixed_gpu_fallback: planned.append(fallback_hardware) for hw in planned: value = (hw or "").strip() if value and value not in sequence: sequence.append(value) return sequence def is_repo_already_exists_error(error: str) -> bool: text = str(error or "").lower() return ("already" in text and ("created this space repo" in text or "already exists" in text or "repo already" in text or "name already" in text)) def collision_safe_target_space_id(target_space_id: str, attempt: int) -> str: owner, slug = target_space_id.split("/", 1) base = re.sub(r"[^A-Za-z0-9._-]+", "-", slug).strip(".-_") or "space-factory" suffix = f"-r{int(time.time()) % 100000}-{attempt}" max_slug_len = min(95, 96 - len(owner) - 1) return f"{owner}/{base[:max_slug_len - len(suffix)].rstrip('.-_')}{suffix}" def create_space_with_hardware_strategy(api, target_space_id: str, token: str, preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, events_path: Path, allow_auto_rename: bool = False, try_zero_gpu_first: bool = True): """Create a private Space and request hardware as early as possible. HF supports `space_hardware` directly on create_repo. This is the cleanest moment to request hardware because the Space does not need a second restart. If OAuth/billing/quota prevents automatic hardware selection, fall back to a normal private CPU Space and mark manual hardware as required. When the target Space name was generated automatically, a stale duplicate run id or a double-submit should not kill the build. In that one case, retry creation with a fresh suffix and return the final target Space id so the rest of the run, report and UI stay coherent. User-provided names still fail loudly if they already exist. """ sequence = build_hardware_sequence(preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first) attempts = [] current_target_space_id = target_space_id rename_attempt = 0 def maybe_rename_after_collision(error: str, phase: str, hardware: str | None = None) -> bool: nonlocal current_target_space_id, rename_attempt if not (allow_auto_rename and is_repo_already_exists_error(error) and rename_attempt < 5): return False previous = current_target_space_id rename_attempt += 1 current_target_space_id = collision_safe_target_space_id(previous, rename_attempt) append_event( events_path, "target_space_collision", "warning", "Auto-generated target Space already existed; retrying with a fresh generated name", {"phase": phase, "hardware": hardware, "previous_target_space": previous, "target_space": current_target_space_id}, ) return True def previous_create_attempt_for_current_target() -> bool: return any( str(item.get("target_space") or "") == current_target_space_id and str(item.get("phase") or "").startswith("create_repo_") for item in attempts if isinstance(item, dict) ) def adopt_existing_after_ambiguous_create(error: str, phase: str, hardware: str | None = None): """Keep one Space per run when a previous create may have succeeded. HF Hub calls can occasionally create the repo but still return an error to the client, or a hardware-at-creation ZeroGPU attempt can leave a CPU Space behind. In that situation the next hardware fallback sees "already exists" for the same target. That must not trigger an auto-rename, otherwise one run could create a second Space. """ if not (is_repo_already_exists_error(error) and previous_create_attempt_for_current_target()): return None payload = { "phase": "create_repo_existing_after_ambiguous_create", "hardware": hardware or "cpu-basic", "ok": True, "target_space": current_target_space_id, "source_error": error, "single_space_per_run": True, } append_event( events_path, "create_space", "warning", "Target Space already exists after an earlier create attempt; reusing it instead of generating another Space", payload, ) if hardware: hw_payload = request_hardware(api, current_target_space_id, hardware, token, events_path, "hardware_post_create_reuse") attempts.append({**payload, "post_create_hardware_request": hw_payload}) selected = hardware if hw_payload.get("ok") else "default-cpu-or-existing" manual = bool(hw_payload.get("manual_action_required") or not hw_payload.get("ok")) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": selected, "requested_sequence": sequence, "attempts": attempts, "manual_action_required": manual} attempts.append(payload) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": "default-cpu-or-existing", "requested_sequence": sequence, "attempts": attempts, "manual_action_required": True} for hardware in sequence: while True: try: append_event(events_path, "create_space_hardware", "started", f"Creating private Space with requested hardware {hardware}", {"target_space": current_target_space_id, "hardware": hardware}) api.create_repo( repo_id=current_target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, space_hardware=hardware, token=token, ) payload = {"phase": "create_repo_space_hardware", "hardware": hardware, "ok": True, "target_space": current_target_space_id} append_event(events_path, "create_space", "success", f"Private target Space created with requested hardware {hardware}", payload) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": hardware, "requested_sequence": sequence, "attempts": attempts + [payload], "manual_action_required": False} except Exception as exc: exc_payload = exception_payload(exc) error = str(exc_payload.get("hf_error_message") or exc_payload.get("hf_error_detail") or exc_payload.get("error") or "")[:4000] reused = adopt_existing_after_ambiguous_create(error, "create_repo_space_hardware", hardware) if reused: return reused if maybe_rename_after_collision(error, "create_repo_space_hardware", hardware): continue manual = is_auth_or_billing_like_error(error) payload = {"phase": "create_repo_space_hardware", "hardware": hardware, "ok": False, "error": error, "manual_action_required": manual, "target_space": current_target_space_id, **exc_payload} attempts.append(payload) append_event(events_path, "create_space_hardware", "failed", f"Could not create Space with requested hardware {hardware}", payload) break # Continue through the sequence: ZeroGPU quota/auth can fail while a fixed GPU # may still be worth trying. If fixed GPU also fails, we'll create CPU below. append_event(events_path, "create_space", "started", "Creating private target Space on default CPU after hardware-at-creation attempts failed", {"target_space": current_target_space_id}) while True: try: api.create_repo(repo_id=current_target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, token=token) cpu_payload = {"phase": "create_repo_default_cpu", "hardware": "cpu-basic", "ok": True, "target_space": current_target_space_id, "manual_action_required": True} append_event(events_path, "create_space", "success", "Private target Space created on default CPU; manual hardware selection may be required", cpu_payload) return {"created": True, "target_space_id": current_target_space_id, "selected_hardware": "default-cpu-or-existing", "requested_sequence": sequence, "attempts": attempts + [cpu_payload], "manual_action_required": True} except Exception as exc: exc_payload = exception_payload(exc) error = str(exc_payload.get("hf_error_message") or exc_payload.get("hf_error_detail") or exc_payload.get("error") or "")[:4000] reused = adopt_existing_after_ambiguous_create(error, "create_repo_default_cpu") if reused: return reused if maybe_rename_after_collision(error, "create_repo_default_cpu"): continue raise def _safe_json_object_from_workspace(path: Path) -> dict: payload = read_json(path, {}) return payload if isinstance(payload, dict) else {} def _first_truthy_string(*values) -> str | None: for value in values: if isinstance(value, str) and value.strip(): return value.strip() return None def _json_safe(value): try: json.dumps(value) return value except Exception: return str(value) def build_model_repo_tree_snapshot(model_id: str, siblings: list[str] | None, info=None) -> dict: """Create a compact, deterministic file-tree snapshot for Pi grounding. This is intentionally lightweight: it does not download model weights and it avoids making feasibility decisions. The goal is to give Pi structural evidence next to the model card. """ siblings = [str(s) for s in (siblings or []) if str(s)] top_level: list[str] = [] seen: set[str] = set() for name in siblings: first = name.split("/", 1)[0] display = first + "/" if "/" in name else first if display not in seen: seen.add(display) top_level.append(display) components = [] lower_names = "\n".join(siblings).lower() for label, needles in { "readme": ["readme.md"], "diffusers_model_index": ["model_index.json"], "transformer": ["transformer/", "transformer."], "text_encoder": ["text_encoder/"], "tokenizer": ["tokenizer/"], "vae": ["vae/"], "scheduler": ["scheduler/"], "lora": ["lora/"], "int8_weights": ["int8", "base_model_int8"], "whisper": ["whisper"], "vocal_separator": ["vocal_separator", "vocal-separator"], "custom_code_hint": [".py", "requirements", "environment.yml", "setup.py"], }.items(): if any(needle in lower_names for needle in needles): components.append(label) return { "schema_version": "model_repo_tree.v1", "model_id": model_id, "source": "HfApi.model_info(files_metadata=True).siblings", "file_count": len(siblings), "top_level_entries": top_level[:120], "sample_files": siblings[:240], "detected_components": components, "truncated": len(siblings) > 240, "created_at": now(), } def resolve_build_time_model_card(model_id: str, token: str | None, siblings: list[str] | None, info=None) -> dict: """Resolve the canonical model card used to ground Pi's build. Best effort and non-blocking: README.md is preferred. If it cannot be read, the worker still writes an explicit fallback file so the run remains auditable and Pi is told that the canonical card was unavailable. """ siblings = [str(s) for s in (siblings or []) if str(s)] has_readme = any(name.lower() == "readme.md" for name in siblings) source = { "schema_version": "model_card_source.v1", "model_id": model_id, "resolved_card_file": "README.md" if has_readme else None, "source": None, "used_for_pi_prompt": True, "fallback_used": False, "warnings": [], "created_at": now(), } text = "" if has_readme: try: from huggingface_hub import hf_hub_download readme_path = hf_hub_download(repo_id=model_id, filename="README.md", repo_type="model", token=token) raw = Path(readme_path).read_bytes() text = raw.decode("utf-8", errors="replace") source.update({ "source": "hf_hub_download", "card_bytes": len(raw), "card_sha256": hashlib.sha256(raw).hexdigest(), "fallback_used": False, }) except Exception as exc: source["fallback_used"] = True source["source"] = "fallback:model_info_metadata" source["warnings"].append(f"readme_download_failed:{type(exc).__name__}") else: source["fallback_used"] = True source["source"] = "fallback:model_info_metadata" source["warnings"].append("readme_not_listed_in_repo_siblings") if not text: card_data = getattr(info, "card_data", None) if info is not None else None tags = list(getattr(info, "tags", []) or []) if info is not None else [] pipeline_tag = getattr(info, "pipeline_tag", None) if info is not None else None library_name = getattr(info, "library_name", None) if info is not None else None text = dedent(f""" # Model Card Fallback for {model_id} The worker could not resolve README.md for this model during build-time grounding. Pi must treat this as an incomplete source and use Hub metadata/repo files only as fallback evidence. - pipeline_tag: {pipeline_tag} - library_name: {library_name} - tags: {tags[:80]} - card_data_present: {bool(card_data)} """).strip() + "\n" source.setdefault("card_bytes", len(text.encode("utf-8"))) source["card_sha256"] = hashlib.sha256(text.encode("utf-8")).hexdigest() return {"model_card.md": text, "model_card_source.json": source} def build_prescan_summary_from_analysis(model_analysis: dict | None, model_card_source: dict | None = None, repo_tree: dict | None = None) -> dict: model_analysis = model_analysis or {} model_card_source = model_card_source or {} repo_tree = repo_tree or {} return { "schema_version": "prescan_summary.v1", "model_id": model_analysis.get("model_id"), "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), "tags": (model_analysis.get("tags") or [])[:80], "preferred_hardware": model_analysis.get("preferred_hardware"), "fallback_hardware": model_analysis.get("fallback_hardware"), "implementation_mode": model_analysis.get("implementation_mode"), "model_card_source": { "source": model_card_source.get("source"), "resolved_card_file": model_card_source.get("resolved_card_file"), "fallback_used": model_card_source.get("fallback_used"), "warnings": model_card_source.get("warnings", []), }, "repo_tree": { "file_count": repo_tree.get("file_count"), "detected_components": repo_tree.get("detected_components", []), "top_level_entries": (repo_tree.get("top_level_entries") or [])[:80], }, "note": "Build-time grounding summary for Pi. Pre-build UI scan remains best-effort and non-blocking.", "created_at": now(), } def source_policy_markdown() -> str: return dedent(f""" # Build-time source policy `analysis_inputs/model_card.md` is the canonical model documentation for this build when it was successfully resolved from README.md. Use sources in this order: 1. `analysis_inputs/model_card.md` / README.md evidence. 2. `analysis_inputs/model_repo_tree.json` structural evidence. 3. Hub metadata and config/model_index hints. 4. Auto-generated Hugging Face snippets only as hints. Auto-generated “Use this model” snippets must not override README.md/model-card instructions. If README.md indicates custom code, torchrun, conda, ffmpeg, FlashAttention, xformers, multi-GPU, special weights, or a specific pipeline class, treat that as authoritative unless live Space logs prove otherwise. If README.md is unavailable, explicitly mention the fallback in `pi_feasibility_brief.json`. """).strip() + "\n" HF_SPACES_GIST_FALLBACK_SUMMARY = """# HF Spaces operational gist fallback summary The Worker could not provide a full remote gist snapshot. Use this built-in summary together with the original gist URL when web access is available. - Push early and validate on the real Hugging Face Space instead of relying on local assumptions. - Use live Space logs and gradio_client/API calls as the runtime source of truth. - ZeroGPU requires Gradio and a real @spaces.GPU-decorated inference function; never decorate a cheap health endpoint. - If the app does no local GPU work, recommend/use cpu-basic instead of ZeroGPU or a fixed GPU. - Proxy/API-only apps should use cpu-basic and must not wrap remote calls in @spaces.GPU. - Keep dependency fixes surgical: read the first actionable build error, patch minimally, rebuild, validate live. - Do not fake model outputs, static samples, or canned success responses. """ def fetch_hf_spaces_gist_snapshot() -> dict: """Best-effort snapshot of the HF Spaces operational gist for Pi grounding. This is intentionally non-blocking. The build must continue with a bundled fallback summary and URL instruction when the network, GitHub, or rate limits make the remote gist unavailable. """ fetched_at = now() if str(os.environ.get("ASF_DISABLE_GIST_FETCH") or "").strip().lower() in {"1", "true", "yes"}: return { "hf_spaces_operational_gist.md": HF_SPACES_GIST_FALLBACK_SUMMARY, "hf_spaces_gist_source.json": { "schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": False, "source": "disabled_by_env", "fallback_summary_included": True, "url_instruction_in_goal": True, "fetched_at": fetched_at, }, } try: import urllib.request req = urllib.request.Request(GIST_RAW_URL, headers={"User-Agent": "agentic-space-factory/198.1"}) with urllib.request.urlopen(req, timeout=5) as response: raw = response.read(512_000) text = raw.decode("utf-8", errors="replace").strip() if not text or len(text) < 200: raise ValueError("gist_raw_too_small") return { "hf_spaces_operational_gist.md": text + "\n", "hf_spaces_gist_source.json": { "schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": True, "source": "remote_raw_gist", "fallback_summary_included": False, "url_instruction_in_goal": False, "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), "bytes": len(raw), "fetched_at": fetched_at, }, } except Exception as exc: return { "hf_spaces_operational_gist.md": HF_SPACES_GIST_FALLBACK_SUMMARY, "hf_spaces_gist_source.json": { "schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": False, "source": "remote_failed", "fallback_summary_included": True, "url_instruction_in_goal": True, "error_type": type(exc).__name__, "error": str(exc)[:1000], "fetched_at": fetched_at, }, } def hf_spaces_guidance_goal_section() -> str: return f"""HF Spaces operational guidance / HF Spaces gist operational rules: - First read `analysis_inputs/hf_spaces_operational_gist.md`. It is the run-local snapshot or fallback summary of the HF Spaces operational guidance. - Also read `analysis_inputs/hf_spaces_gist_source.json` to know whether the full remote gist was fetched. - If `download_success=false` and you have web access, read the original gist directly: {GIST_URL} - In `pi_feasibility_brief.json` or `PI_SUMMARY.md`, include `gist_rules_used`: 1–3 concrete gist/fallback rules that affected your implementation choices. ASF-specific override to the gist: - Do not use Hugging Face Space dev mode. - Do not use SSH or rely on an interactive shell inside the target Space. - Do not propose in-Space file editing, in-Space pip installs, or manual dev-mode edits as the implementation or repair mechanism. - ASF is push-based and audit-based: Pi edits only the Worker workspace files; the Worker creates the Space, uploads snapshots, reads logs, and validates via Gradio/API. - Every implementation or repair must be representable as file changes in app.py, requirements.txt, README.md, or structured artifacts. HF Job vs target Space hardware: - The current HF Job hardware is only the environment running Worker and Pi. - It is not the target Space hardware. - Do not assume CUDA availability in the Job implies CUDA availability in the generated Space. - Do not choose the Space implementation strategy based on the Job hardware. - Recommend hardware only for the target Space, and let the Worker verify/finalize it. """ def hf_spaces_guidance_repair_section() -> str: return f"""HF Spaces operational guidance for repair: - Read `analysis_inputs/hf_spaces_operational_gist.md` if present, and `analysis_inputs/hf_spaces_gist_source.json` for source status. - If the local gist source says `download_success=false` and you have web access, read the original gist directly: {GIST_URL} - Use the gist/fallback log-first method: read the first actionable live build/runtime error, patch minimally, rebuild, and validate live. ASF-specific repair override: - Do not use dev-mode, SSH, in-Space editing, or in-Space pip installs. - Repairs must be file changes in the Worker workspace. - Preserve the original inference strategy whenever possible. - If the failure is UI-only, do not rewrite the inference backend. - If the failure is dependency-only, patch requirements minimally. - Never replace real inference with placeholders, canned text, static files, or broad exception swallowing. """ def pi_reference_dependency_goal_section() -> str: return """Reference Space Discovery — bounded and evidence-based: - Before implementing, check whether existing Hugging Face Spaces use the exact target model id, the same official organization/model family, or the same task + library stack. - Keep this exploration bounded: inspect up to 3 exact/reference Spaces and up to 3 same-family/same-stack Spaces when discoverable. Do not spend the whole build on search. - Prefer high-signal references: official Spaces, Spaces linked from the model card, Spaces from the model author/org, or Spaces that clearly load the same model id. - Inspect only implementation-relevant files when available: app.py, requirements.txt, README.md, hardware assumptions, Gradio API shape, @spaces.GPU usage, dependency pins, and known runtime workarounds. - Use reference Spaces as implementation guidance, not blind copy-paste. Do not copy secrets, tokens, private assumptions, analytics, unrelated UI, or unrelated business logic. - If no useful reference exists or web discovery is unavailable, continue with model card, repo tree, and deterministic evidence instead of blocking. - Write `REFERENCE_SPACE_FINDINGS.json` when possible. It should include: searched, exact_model_spaces, related_spaces, useful_patterns[], files_checked, applicability, risks, and no_reference_found_reason. Model Dependency Discovery — self-contained vs dependent repos: - Do not assume the target repository is a complete standalone model. Determine whether the repo is self-contained before selecting the implementation strategy. - Check `analysis_inputs/model_card.md`, repo tree/config names, model metadata, and any reference Spaces for external GitHub repos, auxiliary model ids, tokenizer/processor repos, VAE/text encoder/safety checker/guardrail models, custom Python packages, local paths in official examples, gated/private dependencies, and hardware/VRAM assumptions. - Explicitly recognize dependent model artifacts such as LoRA, QLoRA, PEFT adapters, DreamBooth LoRA, ControlNet, IP-Adapter, textual inversion, VAE/component repos, tokenizer/processor-only repos, and merged vs unmerged checkpoints. - Look for adapter/component signals such as `adapter_config.json`, `base_model`, `base_model_name_or_path`, `peft_type`, `lora_alpha`, `target_modules`, `adapter_model.safetensors`, `pytorch_lora_weights.safetensors`, `diffusers_lora`, `load_lora_weights`, `PeftModel`, `AutoPeftModelForCausalLM`, `ss_base_model_version`, "trained on", "based on", "requires", and "use with". - If the target is a LoRA/PEFT adapter or component repo, identify the required base model with evidence before attempting full inference. If the base model cannot be identified or is unavailable/gated, declare a technical blocker instead of pretending the adapter can run standalone. - Do not choose hardware from adapter size alone; include the base model and auxiliary components in VRAM/runtime reasoning. - Write `MODEL_DEPENDENCIES.json` when possible. It should include: self_contained, model_kind, base_model_required, base_model_id, adapter_application_method, dependencies[], missing_or_uncertain_dependencies[], and impact_on_strategy. ASF strategy and contract vocabulary: - Use `inference_strategy` consistently in pi_feasibility_brief.json, pi_implementation_plan.json, INFERENCE_CONTRACT.json, and PI_SUMMARY.md when available. - `inference_strategy` must be one of: diffusers_pipeline, transformers_pipeline, vllm_local_server, openai_compatible_local_api, tts_sdk_cpu, onnx_runtime, spaces_gpu, diagnostic_only, custom_subprocess_backend, unknown. - Include `strategy_reason` with concrete evidence from the model card, repo tree, reference Space, or dependency analysis. - Recommend target Space hardware with `recommended_target_space_hardware`: cpu-basic, zero-a10g, a10g-small, a10g-large, or manual_required. Include hardware_reason, requires_gpu, estimated_vram, and hardware_confidence when known. - Rules of thumb: diagnostic_only + requires_gpu=false -> cpu-basic; proxy/API-only -> cpu-basic; CPU-only SDK/ONNX/TTS -> cpu-basic; real local Diffusers/Torch CUDA inference -> ZeroGPU or fixed GPU; vLLM local server -> fixed GPU likely required; model exceeds available GPU capacity -> manual_required. - The Worker remains the final authority for hardware and validation. Your recommendation is advisory but should be evidence-backed. """ def pi_conservative_guidance_goal_section() -> str: return """Conservative implementation guidance — smallest working truthful Space: - Prefer the smallest working Space implementation that can truthfully validate the model over copying or vendoring a complex reference Space. - Reference Spaces are guidance, not authority. Use them to identify patterns, dependencies, hardware assumptions, API shapes, and known pitfalls; do not blindly copy their entire app structure, dependency pins, UI, analytics, or hardware choice. - Vendor a full reference app only when the model card or official inference path genuinely requires that app structure or package layout. Otherwise, build a minimal reproducible app around the required model call. - If a reference Space is complex but the model can be run with a smaller direct pipeline/SDK call, choose the smaller implementation and document what reference patterns were intentionally not copied. - Keep generated GPU apps ZeroGPU-first and fixed-GPU-compatible. If the model performs local GPU inference, prefer a real `@spaces.GPU`-decorated generation function for ZeroGPU, but write it so the same app.py remains valid when the Worker falls back to fixed GPU hardware because the current user has exhausted ZeroGPU quota. - Do not remove `@spaces.GPU` or refactor the app away from ZeroGPU merely because one run falls back to `a10g-large`. ZeroGPU quota/capacity is user/runtime state, not model evidence. - If importing `spaces` is optional in a local/fixed-GPU context, use a safe decorator fallback instead of crashing at import time; if CUDA is absent for a GPU-required model, show a clear UI error rather than faking output. - Expose `api_name="health"` in the top-level Gradio app whenever possible. If the app is multi-file or vendorized and health is registered in another file, document the exact file/function in INFERENCE_CONTRACT.json or PI_SUMMARY.md using `health_endpoint_location`. - Do not let reference discovery expand scope indefinitely. When evidence is mixed, prefer a conservative bootable implementation with honest blockers over a large fragile clone. """ def pi_demo_quality_goal_section() -> str: return """Demo quality and model-card promise fulfillment: - Your goal is not only to make the Space boot. Build a usable Gradio demo that truthfully reflects the model-card promise and lets a reviewer try the primary user flow. - Before finalizing the app, identify the `model_card_promise`: the task the model card claims the model can perform, the expected inputs, expected output type, and any important limitations. - Provide examples in the Gradio UI whenever full or partial real inference is implemented. Use `gr.Examples(...)` or an equivalent Gradio examples mechanism appropriate for the UI. - Examples must be task-appropriate and lightweight: text-to-image prompts for image models, text/voice/language examples for TTS, short prompts/lyrics for audio or music models, short messages for chat/text models, and realistic but small inputs for multimodal models. - Do not include fake example outputs. Examples are input presets; the output must still come from the model unless the app is explicitly diagnostic-only. - Provide a canonical smoke-test example that the Worker can later use for live validation. The example should use the same API endpoint exposed by the app and should be cheap enough for a first validation call. - The canonical smoke example should use schema-compatible values when known. For dropdown/radio choices, prefer the exact choice value shown in the Gradio UI schema, not a different Python type. - If full inference is impossible, still make the Space useful: provide a diagnostic Gradio page explaining the blocker, required hardware/dependencies, and at least one example input that would be used once the blocker is resolved. - Write `DEMO_QUALITY_CONTRACT.json` describing: model_card_promise, demo_task, primary_user_flow, examples_provided, examples[], canonical_smoke_example, real_inference_required, real_inference_implemented, fallback_or_diagnostic_only, limitations_disclosed, and promise_fulfillment_risk. - Include `canonical_smoke_example` either in `DEMO_QUALITY_CONTRACT.json` or `INFERENCE_CONTRACT.json`. It should include api_name, inputs, expected_output_type, and a short reason why this example is cheap and representative. - README.md must explain how to try the demo, list at least one example input, and disclose any limitations/blockers honestly. """ def pi_reference_dependency_repair_section() -> str: return """Reference/dependency awareness during repair: - Preserve the original inference_strategy and model dependency logic whenever possible. - If the app uses a LoRA/PEFT adapter, ControlNet, IP-Adapter, textual inversion, VAE/component repo, or another dependent artifact, preserve the base model loading/application path. Do not "fix" the app by loading the adapter/component as if it were standalone. - If repair evidence shows a missing base model, auxiliary model, external package, or gated/private dependency, document it in REPAIR_SUMMARY.md and TECHNICAL_BLOCKERS.json when full inference cannot be safely restored. - For UI-only or dependency-only failures, do not rewrite reference-Space-derived inference code unless the first actionable error requires it. """ def prepare_build_analysis_inputs(model_id: str, token: str | None, siblings: list[str] | None, info, model_analysis: dict | None = None) -> dict: card_bundle = resolve_build_time_model_card(model_id, token, siblings, info) repo_tree = build_model_repo_tree_snapshot(model_id, siblings, info) source = card_bundle["model_card_source.json"] prescan = build_prescan_summary_from_analysis(model_analysis or {}, source, repo_tree) gist_snapshot = fetch_hf_spaces_gist_snapshot() return { "model_card.md": card_bundle["model_card.md"], "model_card_source.json": source, "model_repo_tree.json": repo_tree, "prescan_summary.json": prescan, "source_policy.md": source_policy_markdown(), "hf_spaces_operational_gist.md": gist_snapshot["hf_spaces_operational_gist.md"], "hf_spaces_gist_source.json": gist_snapshot["hf_spaces_gist_source.json"], } def write_analysis_inputs_dir(base_dir: Path, analysis_inputs: dict | None) -> dict: inputs = analysis_inputs or {} target = base_dir / "analysis_inputs" target.mkdir(parents=True, exist_ok=True) written = [] defaults = { "model_card.md": "# Model Card Unavailable\n\nNo build-time model card was provided to this workspace.\n", "model_card_source.json": {"schema_version": "model_card_source.v1", "status": "missing", "fallback_used": True, "used_for_pi_prompt": True, "warnings": ["analysis_inputs_missing"]}, "model_repo_tree.json": {"schema_version": "model_repo_tree.v1", "status": "missing", "top_level_entries": [], "sample_files": [], "detected_components": []}, "prescan_summary.json": {"schema_version": "prescan_summary.v1", "status": "missing"}, "source_policy.md": source_policy_markdown(), "hf_spaces_operational_gist.md": HF_SPACES_GIST_FALLBACK_SUMMARY, "hf_spaces_gist_source.json": {"schema_version": "hf_spaces_gist_source.v1", "url": GIST_URL, "raw_url": GIST_RAW_URL, "download_success": False, "source": "analysis_inputs_missing", "fallback_summary_included": True, "url_instruction_in_goal": True, "fetched_at": now()}, } for name, default in defaults.items(): value = inputs.get(name, default) path = target / name if isinstance(value, (dict, list)): path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") else: path.write_text(str(value), encoding="utf-8") written.append(f"analysis_inputs/{name}") return {"dir": str(target), "written": written} def _extract_model_card_evidence_items(*objects) -> list: items = [] for obj in objects: if not isinstance(obj, dict): continue for key in ("model_card_evidence", "evidence"): value = obj.get(key) if isinstance(value, list): for item in value: if isinstance(item, dict): claim = str(item.get("claim") or item.get("name") or item.get("summary") or "").strip() evidence = str(item.get("evidence") or item.get("quote") or item.get("text") or "").strip() source = str(item.get("source") or "").strip() if claim or evidence: items.append({"claim": claim, "evidence": evidence[:500], "source": source}) elif isinstance(item, str) and item.strip(): items.append({"claim": item.strip()[:240], "evidence": item.strip()[:500], "source": "unspecified"}) return items[:12] def review_model_card_grounding(workspace: Path, brief: dict, plan: dict) -> dict: source = read_json(workspace / "analysis_inputs" / "model_card_source.json", {}) or {} card_path = workspace / "analysis_inputs" / "model_card.md" card_present = card_path.exists() and card_path.stat().st_size > 0 source_available = card_present and not bool(source.get("fallback_used")) and source.get("source") not in {None, "missing"} evidence_items = _extract_model_card_evidence_items(brief, plan) warnings = [] if not card_present: warnings.append("model_card_input_missing") elif source.get("fallback_used"): warnings.append("model_card_source_fallback_used") if not evidence_items: warnings.append("pi_model_card_evidence_missing") status = "ok" if source_available and evidence_items else ("partial" if card_present else "missing") return { "schema_version": "model_card_grounding_review.v1", "status": status, "non_blocking": True, "source_available": bool(source_available), "model_card_present": bool(card_present), "model_card_source": { "source": source.get("source"), "resolved_card_file": source.get("resolved_card_file"), "fallback_used": source.get("fallback_used"), "warnings": source.get("warnings", []), }, "pi_evidence_present": bool(evidence_items), "pi_evidence_count": len(evidence_items), "pi_evidence_sample": evidence_items[:5], "warnings": warnings, "created_at": now(), } def write_pi_planning_review(workspace: Path, run_dir: Path, events_path: Path, model_analysis: dict | None = None, implementation_mode: str = "") -> dict: """Persist Pi planning artifacts and a non-blocking worker review. v191.9 intentionally makes planning visibility-only. Pi is asked to produce a feasibility brief and implementation plan, but older/partial Pi behavior must not fail a run that would otherwise work. The worker records whether those artifacts exist, mirrors them under planning/, and keeps the final authority in the existing deterministic gates. """ model_analysis = model_analysis or {} planning_dir = run_dir / "planning" planning_dir.mkdir(parents=True, exist_ok=True) brief_path = workspace / "pi_feasibility_brief.json" plan_path = workspace / "pi_implementation_plan.json" source_usage_path = workspace / "PI_SOURCE_USAGE.json" brief = _safe_json_object_from_workspace(brief_path) plan = _safe_json_object_from_workspace(plan_path) source_usage = _safe_json_object_from_workspace(source_usage_path) brief_present = bool(brief) plan_present = bool(plan) source_usage_present = bool(source_usage) if brief_present: write_json(planning_dir / "pi_feasibility_brief.json", brief) else: write_json(planning_dir / "pi_feasibility_brief.json", { "schema_version": "pi_feasibility_brief.v1", "status": "missing", "non_blocking": True, "message": "Pi did not produce pi_feasibility_brief.json; continuing with existing deterministic worker gates.", }) if plan_present: write_json(planning_dir / "pi_implementation_plan.json", plan) else: write_json(planning_dir / "pi_implementation_plan.json", { "schema_version": "pi_implementation_plan.v1", "status": "missing", "non_blocking": True, "message": "Pi did not produce pi_implementation_plan.json; continuing with existing deterministic worker gates.", }) if source_usage_present: write_json(planning_dir / "PI_SOURCE_USAGE.json", source_usage) else: write_json(planning_dir / "PI_SOURCE_USAGE.json", { "schema_version": "pi_source_usage.v198_25", "status": "missing", "non_blocking": True, "message": "Pi did not produce PI_SOURCE_USAGE.json; continuing with deterministic worker gates.", }) declared_strategy = _first_truthy_string( brief.get("recommended_strategy"), brief.get("strategy"), plan.get("strategy"), plan.get("implementation_strategy"), ) or "unspecified" should_attempt_full = brief.get("should_attempt_full_inference") if should_attempt_full is None: should_attempt_full = plan.get("should_attempt_full_inference") worker_recommendation = "continue_existing_flow" warnings: list[str] = [] if not brief_present: warnings.append("missing_pi_feasibility_brief") if not plan_present: warnings.append("missing_pi_implementation_plan") if not source_usage_present: warnings.append("missing_pi_source_usage") grounding_review = review_model_card_grounding(workspace, brief, plan) warnings.extend(grounding_review.get("warnings", [])) if declared_strategy in {"boot_only_blocker", "boot_only_blocker_or_manual_refactor", "manual_hardware_required"} or should_attempt_full is False: worker_recommendation = "respect_plan_if_contract_declares_blocker" elif declared_strategy in {"full_inference", "proceed_full_inference", "attempt_full_inference"} or should_attempt_full is True: worker_recommendation = "proceed_full_inference_with_existing_gates" review = { "schema_version": "worker_plan_review.v1", "status": "ok" if brief_present and plan_present else "partial", "non_blocking": True, "implementation_mode": implementation_mode, "model_id": model_analysis.get("model_id"), "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), "planning_artifacts_present": { "pi_feasibility_brief": brief_present, "pi_implementation_plan": plan_present, "pi_source_usage": source_usage_present, }, "pi_source_usage": source_usage if source_usage_present else {"status": "missing", "non_blocking": True}, "declared_strategy": declared_strategy, "should_attempt_full_inference": should_attempt_full, "worker_recommendation": worker_recommendation, "warnings": warnings, "model_card_grounding": grounding_review, "authority_note": "Pi planning/source usage is advisory in v198.25; deterministic worker gates, INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation remain authoritative.", "created_at": now(), } write_json(planning_dir / "worker_plan_review.json", review) write_json(planning_dir / "model_card_grounding_review.json", grounding_review) append_event(events_path, "pi_planning_review", review["status"], "Pi feasibility and implementation planning reviewed", review) return review def build_model_recipe(model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> dict: model_analysis = model_analysis or {} analysis_inputs = analysis_inputs or {} card = str(analysis_inputs.get("model_card.md") or "") card_low = card.lower() repo_tree = analysis_inputs.get("model_repo_tree.json") if isinstance(analysis_inputs.get("model_repo_tree.json"), dict) else {} dep_hints = [] for pkg in ["diffusers", "transformers", "sentencepiece", "protobuf", "accelerate", "safetensors", "peft", "imageio", "imageio-ffmpeg", "torchvision", "spaces", "gradio"]: if pkg.lower() in card_low or pkg in json.dumps(repo_tree)[:20000].lower(): dep_hints.append(pkg) pipeline_hints = [] for name in re.findall(r"\b[A-Za-z0-9_]*(?:Pipeline|Model|Processor|Tokenizer)\b", card)[:40]: if name not in pipeline_hints: pipeline_hints.append(name) constraints = [] if "divisible by 32" in card_low or "divisible_by_32" in card_low: constraints.append("resolution_divisible_by_32") if "8k+1" in card_low or "8 *" in card_low and "frames" in card_low: constraints.append("frames_8k_plus_1") if "bf16" in card_low or "bfloat16" in card_low: constraints.append("bf16_or_bfloat16_hint") if "fp16" in card_low or "float16" in card_low: constraints.append("fp16_hint") return { "schema_version": "model_recipe.v198_25", "purpose": "Context guide for Pi initial build. This does not replace the full model card; it indexes critical facts to verify in analysis_inputs/model_card.md.", "model_id": model_id, "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), "tags": model_analysis.get("tags", [])[:50] if isinstance(model_analysis.get("tags"), list) else [], "repo_components": repo_tree.get("detected_components", []), "recommended_pipeline_or_class_hints": pipeline_hints[:20], "dependency_hints": dep_hints, "input_constraints_hints": constraints, "source_priority": ["analysis_inputs/model_card.md", "analysis_inputs/model_repo_tree.json", "analysis_inputs/source_policy.md", "analysis_inputs/hf_spaces_operational_gist.md"], "warning": "Pi must verify these hints against the full model card and official examples before coding.", } def build_context_index(analysis_inputs: dict | None = None) -> dict: analysis_inputs = analysis_inputs or {} files = [] for name, description in [ ("analysis_inputs/model_card.md", "Canonical README/model card resolved by the worker when available."), ("analysis_inputs/model_repo_tree.json", "Model repository file tree and detected components."), ("analysis_inputs/prescan_summary.json", "Deterministic worker prescan summary."), ("analysis_inputs/source_policy.md", "Source priority and snippet distrust policy."), ("analysis_inputs/hf_spaces_operational_gist.md", "HF Spaces operational guidance snapshot/fallback."), ("MODEL_RECIPE.json", "Compact worker-generated guide. Start here, then verify in full sources."), ("SOURCE_PRIORITY.md", "Short source ordering guide for Pi."), ]: files.append({"path": name, "description": description}) return {"schema_version": "context_index.v198_25", "purpose": "Keep Pi fully armed at initial build while making source navigation explicit.", "files": files} def write_initial_pi_context_artifacts(workspace: Path, model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> list[str]: recipe = build_model_recipe(model_id, model_analysis, analysis_inputs) index = build_context_index(analysis_inputs) source_priority = """# Source priority for Pi\n\nStart with `MODEL_RECIPE.json` and `CONTEXT_INDEX.json` to orient yourself, but do not treat them as replacements for the rich sources. Verify critical claims in this order:\n\n1. `analysis_inputs/model_card.md` official/model-card instructions and examples.\n2. `analysis_inputs/model_repo_tree.json` repo structure, configs, component names, weights.\n3. `analysis_inputs/source_policy.md` trust rules for snippets and generated examples.\n4. `analysis_inputs/hf_spaces_operational_gist.md` HF Spaces operational constraints.\n\nFor initial build, you are expected to use the rich context. For later repairs, ASF will provide narrower task packets.\n""" write_json(workspace / "MODEL_RECIPE.json", recipe) write_json(workspace / "CONTEXT_INDEX.json", index) (workspace / "SOURCE_PRIORITY.md").write_text(source_priority, encoding="utf-8") return ["MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md"] def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None): workspace.mkdir(parents=True, exist_ok=True) model_analysis = model_analysis or {} pipeline_tag = model_analysis.get("pipeline_tag") library_name = model_analysis.get("library_name") tags = model_analysis.get("tags", [])[:40] siblings = model_analysis.get("siblings", [])[:60] analysis_inputs_result = write_analysis_inputs_dir(workspace, analysis_inputs) initial_context_artifacts = write_initial_pi_context_artifacts(workspace, model_id, model_analysis, analysis_inputs) app_py = f"""import gradio as gr from huggingface_hub import model_info, list_repo_files MODEL_ID = {model_id!r} TARGET_SPACE_ID = {target_space_id!r} def health(): return {{ "status": "booted", "model_id": MODEL_ID, "target_space_id": TARGET_SPACE_ID, "stage": "initial-scaffold", "note": "Pi should replace this scaffold with a model-specific demo while preserving a cheap health endpoint.", }} def placeholder(*args): return "Initial scaffold. Pi should replace this with a model-specific inference path, or write TECHNICAL_BLOCKERS.json." with gr.Blocks(title="Generated Model Space — Agentic Space Factory") as demo: gr.Markdown("# Generated Model Space — Agentic Space Factory") gr.Markdown(f"Private generated Space for `{{MODEL_ID}}`.") gr.JSON(label="Health", value=health(), every=None) gr.Button("Health check").click(fn=health, inputs=None, outputs=gr.JSON(), api_name="health") gr.Textbox(label="Input", value="Hello from Agentic Space Factory").submit(fn=placeholder, inputs=None, outputs=gr.Textbox(), api_name="predict") gr.Button("Run placeholder").click(fn=placeholder, inputs=None, outputs=gr.Textbox(), api_name="predict") if __name__ == "__main__": demo.launch() """ (workspace / "app.py").write_text(app_py, encoding="utf-8") req = """gradio>=6.0.0 huggingface_hub>=0.34.0,<2.0.0 spaces transformers>=4.45.0,<6.0.0 diffusers accelerate safetensors torch pillow numpy requests """ (workspace / "requirements.txt").write_text(req, encoding="utf-8") readme = f"""--- title: Generated Model Space sdk: gradio app_file: app.py python_version: "3.10" suggested_hardware: {preferred_hardware or fallback_hardware or "cpu-basic"} short_description: "Generated model demo" --- # Generated Model Space — Agentic Space Factory Private generated Space for `{model_id}`. This Space is created by Agentic Space Factory. It should remain private until manually reviewed. """ (workspace / "README.md").write_text(readme, encoding="utf-8") analysis_json = json.dumps({"pipeline_tag": pipeline_tag, "library_name": library_name, "tags": tags, "siblings": siblings}, indent=2, ensure_ascii=False) goal = f"""You are Pi running inside a Hugging Face Job for Agentic Space Factory. Goal: build the best possible private Hugging Face Space demo for an arbitrary model card. MODEL_ID: {model_id} TARGET_SPACE_ID: {target_space_id} IMPLEMENTATION_MODE: {implementation_mode} MODEL_METADATA: ```json {analysis_json} ``` BUILD-TIME MODEL ANALYSIS INPUTS: - `analysis_inputs/model_card.md`: canonical model card / README resolved by the worker when available. - `analysis_inputs/model_card_source.json`: source, hash, fallback status, and warnings for the model card. - `analysis_inputs/model_repo_tree.json`: structural file-tree snapshot from Hub metadata. - `analysis_inputs/prescan_summary.json`: compact deterministic worker summary. - `analysis_inputs/source_policy.md`: source priority and snippet distrust policy. - `analysis_inputs/hf_spaces_operational_gist.md`: run-local HF Spaces operational gist snapshot or fallback summary. - `analysis_inputs/hf_spaces_gist_source.json`: fetch status, URL fallback, and hash/source metadata for the gist guidance. Before writing code, first read `MODEL_RECIPE.json`, `CONTEXT_INDEX.json`, and `SOURCE_PRIORITY.md` to orient yourself, then read `analysis_inputs/model_card.md`, `analysis_inputs/model_repo_tree.json`, `analysis_inputs/source_policy.md`, `analysis_inputs/hf_spaces_operational_gist.md`, and `analysis_inputs/hf_spaces_gist_source.json`. Treat the model card as the canonical source when it was resolved from README.md. Auto-generated Hugging Face “Use this model” snippets are hints only and must not override README.md/model-card instructions. The recipe/index guide you through the rich context; they do not replace it. {hf_spaces_guidance_goal_section()} {pi_reference_dependency_goal_section()} {pi_conservative_guidance_goal_section()} {pi_demo_quality_goal_section()} {pi_tooling_context_note()} Non-negotiable safety and product constraints: - The target Space must remain private. - Do not delete any user resources. - Do not print secrets or tokens. - Work only inside the current workspace. - The wrapper will create the private Space, request allowed hardware best-effort, upload files, and validate the live app. Do not create/delete repos yourself in this builder worker. - Preserve a cheap health endpoint named `health` with `api_name="health"`. It must not load weights, run GPU work, or download large files. - Platform dependency policy: Gradio, Gradio Client, Hugging Face Hub, Spaces, and hf_xet are owned by Agentic Space Factory / the Hugging Face Spaces runtime. Do not pin or downgrade them. Never write `gradio==...`, `gradio<=...`, `gradio~=...`, `gradio-client==...`, `gradio_client==...`, or stale exact pins for `huggingface_hub`, `spaces`, or `hf_xet`. - If generated code needs an older Gradio API, update the code to modern Gradio instead of downgrading Gradio. - Follow the gist dependency method for model-specific packages: pin only when useful to reduce pip resolver backtracking or when the model card/build logs explicitly require a version; do not cargo-cult every version from examples. - Do not pin torch or torchaudio unless unavoidable and explicitly justified; Spaces provide the PyTorch stack. If torchvision is needed, prefer leaving it unpinned so it can resolve against the managed torch runtime. - If model code is not pip-installable, vendor the necessary code into the Space repo instead of referencing local paths, editable installs, or clone-time side effects in requirements.txt. - Do not run local venv installs, broad `pip install --dry-run`, or dependency checks as proof of compatibility; only the live Space build/logs are authoritative. Use `python -m py_compile app.py` only as a cheap syntax check. - Use huggingface_hub>=0.34.0,<2.0.0 unless a newer Transformers path requires a compatible newer range. The worker will normalize platform-owned dependency lines before upload. - README.md frontmatter must remain valid; if it uses short_description, it must be 60 characters or fewer. Planning contract: - Before committing to a full implementation strategy, produce `pi_feasibility_brief.json` and `pi_implementation_plan.json`. These files are advisory artifacts for the worker and must not contain secrets. - `pi_feasibility_brief.json` should include: complexity, expected_runtime, zerogpu_feasible, requires_custom_code, multi_gpu_risk, dependency_risk, recommended_strategy, `inference_strategy`, strategy_reason, minimum_hardware, recommended_target_space_hardware, hardware_reason, should_attempt_full_inference, evidence, `gist_rules_used`, and `model_card_evidence` with 2–5 concrete facts from `analysis_inputs/model_card.md` when available. - `pi_implementation_plan.json` should include: selected_strategy, `inference_strategy`, intended_files, endpoints, dependency_strategy, hardware_strategy, recommended_target_space_hardware, smoke_test_plan, canonical_smoke_example, demo_examples_plan, fallback_plan, blocker_policy, `gist_rules_used` if not already in the brief, and any README/model-card facts that materially affect implementation choices. - If the model appears high-risk or not portable, say so in the plan rather than silently attempting a fragile fake success. The worker will still use INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation as the final authority. Implementation contract: - If IMPLEMENTATION_MODE is `full-inference-gated`, you are not allowed to silently replace generation with a placeholder and call it success. - Try to implement the closest real inference path for the model card using evidence from `analysis_inputs/model_card.md`, model metadata, config files, and repo files. If a generic snippet conflicts with the README/model-card, the README/model-card wins. - You may choose an appropriate Gradio UI for the task: text, image, audio, video, multimodal, embeddings, classification, etc. - If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint. - If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement. - If the model requires special dependencies, include them only when needed and document risks. - Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels/Kernel Hub where relevant, Transformers AttentionInterface, Diffusers attention processors, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs. - Native kernel policy: if the model card or logs mention flash-attn, custom attention kernels, xformers, Triton kernels, fused ops, or CUDA/C++ extensions, do not blindly add source-built native packages to requirements.txt. Prefer runtime-compatible drop-in backends or compatible wheels first. Use `kernels` only when it directly matches the needed operation and the target hardware/runtime is plausible; document the selected backend in PI_SUMMARY.md and INFERENCE_CONTRACT.json. - If real inference is impossible or unsafe in a Space, write TECHNICAL_BLOCKERS.json with concrete evidence for every blocker. Deliverables: - pi_feasibility_brief.json must summarize feasibility and risk before/alongside implementation, including `model_card_evidence` when model-card evidence is available. - pi_implementation_plan.json must summarize the selected implementation plan, endpoints, dependencies, target Space hardware recommendation, and validation approach. - REFERENCE_SPACE_FINDINGS.json should summarize bounded reference Space discovery when possible; if none were found or web discovery is unavailable, say so without blocking. - MODEL_DEPENDENCIES.json should summarize whether the repo is standalone or dependent, including LoRA/PEFT/adapter/component/base-model dependencies when present. - DEMO_QUALITY_CONTRACT.json should summarize the model-card promise, the primary user flow, Gradio examples, canonical_smoke_example, whether real inference is implemented, and any promise fulfillment risks. - app.py must boot on Hugging Face Spaces. - app.py must expose health/api_name="health". - If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic. - If real generation is not implemented, write TECHNICAL_BLOCKERS.json with: full_inference_implemented=false, blockers[], evidence[], minimum_runtime, and suggested_next_step. - Write INFERENCE_CONTRACT.json with: full_inference_implemented, inference_strategy, health_endpoint, primary_api_name, expected_output_type, validation_level, requires_gpu, recommended_target_space_hardware, estimated_vram, and blockers_count. - README.md must explain the runtime strategy, task, limitations, how to test, and at least one example input when the demo is not purely diagnostic. - Write PI_SOURCE_USAGE.json with: used_model_card, used_official_code_examples, used_repo_tree, used_hf_spaces_gist, critical_claims_verified[], and any skipped_sources[]. This is an audit artifact only; it must not block implementation. - Write a concise PI_SUMMARY.md with what you changed, whether full inference is implemented, and any `gist_rules_used` if they are not already captured in JSON. """ (workspace / "GOAL.md").write_text(goal, encoding="utf-8") return ["app.py", "requirements.txt", "README.md", "GOAL.md"] + initial_context_artifacts + analysis_inputs_result.get("written", []) def sanitize_readme_metadata(workspace: Path, events_path: Path): """Ensure the generated Space README always keeps valid HF metadata. Pi is allowed to rewrite README.md, but Spaces require a valid YAML frontmatter block for sdk/app_file. If Pi drops it, the Space can fail with a Hub configuration error before the app even builds. Treat this as a factory invariant, not a Pi preference. """ readme_path = workspace / "README.md" if not readme_path.exists(): readme_path.write_text("# Generated Model Space\n", encoding="utf-8") text = readme_path.read_text(encoding="utf-8", errors="ignore") body = text metadata = {} changed = False if text.startswith("---"): parts = text.split("---", 2) if len(parts) >= 3: _, frontmatter, body = parts for line in frontmatter.splitlines(): if ":" not in line: continue key, value = line.split(":", 1) metadata[key.strip()] = value.strip() else: body = text changed = True else: changed = True raw_python_version = metadata.get("python_version") or DEFAULT_SPACE_PYTHON_VERSION python_version, python_version_changed, python_version_reason = normalize_space_python_version(raw_python_version) required = { "title": metadata.get("title") or "Generated Model Space", "sdk": "gradio", "app_file": "app.py", "python_version": python_version, } if python_version_changed: changed = True suggested = metadata.get("suggested_hardware") if suggested: required["suggested_hardware"] = suggested short = metadata.get("short_description") or "Generated model demo" if len(short.strip('"\'')) > 60: short = "Generated model demo" required["short_description"] = short ordered_keys = ["title", "sdk", "app_file", "python_version", "suggested_hardware", "short_description"] lines = [] for key in ordered_keys: if key in required and required[key]: value = str(required[key]).strip() if key in {"title", "short_description", "python_version"} and not (value.startswith('"') or value.startswith("'")): value = json.dumps(value, ensure_ascii=False) lines.append(f"{key}: {value}") normalized_body = body.lstrip("\n") or "# Generated Model Space\n" new_text = "---\n" + "\n".join(lines) + "\n---\n\n" + normalized_body if new_text != text: readme_path.write_text(new_text, encoding="utf-8") append_event( events_path, "metadata_sanitize", "success", "Ensured README Space metadata", { "metadata_keys": [k for k in ordered_keys if k in required], "python_version_before": str(raw_python_version).strip().strip("\"'"), "python_version_after": python_version, "python_version_normalized": bool(python_version_changed), "python_version_reason": python_version_reason, }, ) def _read_workspace_text_if_exists(path: Path, *, limit: int = 20000) -> str: try: if path.exists() and path.is_file(): text = path.read_text(encoding="utf-8", errors="ignore") return text[-limit:] except Exception: return "" return "" def detect_transformers5_dependency_requirement(text: str = "") -> dict: """Detect concrete evidence that a dependency requires Transformers 5.x. This is deliberately narrow: ASF should not globally allow every generated app to float to an unknown major release, but it must not reintroduce a `<5.0.0` upper bound after pip logs or repair artifacts prove a package needs `transformers>=5.0.0`. """ raw = text or "" low = raw.lower() if "transformers>=5" not in low and "transformers >=5" not in low and "transformers 5" not in low: return {} has_blocking_upper_bound = bool(re.search(r"transformers[^\n#]*<\s*5(?:\.0(?:\.0)?)?", raw, flags=re.I)) has_requires_phrase = any( phrase in low for phrase in [ "requires transformers>=5", "depends on transformers>=5", "cosmos-guardrail", "requires transformers >=5", ] ) if has_requires_phrase or has_blocking_upper_bound: return { "package": "transformers", "requires": "transformers>=5.0.0", "conflicting_upper_bound": has_blocking_upper_bound, "evidence": raw[-2000:], } return {} def workspace_allows_transformers_major5(workspace: Path, extra_evidence: str = "") -> dict: """Return evidence allowing ASF to relax its default Transformers <5 guard. Sources are intentionally local/auditable artifacts from the current repair pass. This makes the upload-time sanitizer idempotent: once a dependency repair brief/summary documents the concrete Transformers 5 requirement, a later upload sanitize pass must preserve the relaxed requirement. """ chunks = [extra_evidence or ""] for name in [ "DEPENDENCY_ERROR_BRIEF.md", "REPAIR_PLAN.md", "REPAIR_SUMMARY.md", "requirements_policy.json", "requirements.txt", ]: chunks.append(_read_workspace_text_if_exists(workspace / name)) evidence = "\n".join(chunks) issue = detect_transformers5_dependency_requirement(evidence) if issue: issue["source"] = "repair_artifacts_or_dependency_logs" return issue def relax_transformers_upper_bound_for_repair(workspace: Path, events_path: Path, evidence_text: str = "") -> bool: """Apply the narrow Cosmos-style deterministic requirements fix. Only removes `<5` from a Transformers requirement when concrete local logs or repair artifacts show a dependency requiring Transformers 5.x. This function does not add Transformers if absent; the regular sanitizer handles the base policy line using the same evidence. """ issue = workspace_allows_transformers_major5(workspace, evidence_text) if not issue: return False req_path = workspace / "requirements.txt" if not req_path.exists(): return False before = req_path.read_text(encoding="utf-8", errors="ignore") changed = False out_lines: list[str] = [] for line in before.splitlines(): stripped = line.strip() name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") if stripped and not stripped.startswith("#") else "" if name == "transformers" and re.search(r"<\s*5(?:\.0(?:\.0)?)?", line, flags=re.I): # Keep any lower-bound/extras/comments simple and deterministic for now. out_lines.append("transformers>=4.51.0") changed = True else: out_lines.append(line) if not changed: return False after = "\n".join(out_lines).rstrip() + "\n" req_path.write_text(after, encoding="utf-8") append_event( events_path, "requirements_repair", "success", "Relaxed Transformers <5 upper bound because dependency logs require Transformers 5.x", {"issue": issue, "from_contains": "<5", "to": "transformers>=4.51.0"}, ) return True def workspace_publishable_fingerprint(root: Path) -> dict: """Return a hash map for files that matter to the generated Space. Internal Pi/repair artifacts are excluded so a repair cannot count as applied merely because REPAIR_PLAN.md or REPAIR_SUMMARY.md was written. """ if not root.exists(): return {} result: dict[str, str] = {} for path in sorted(root.rglob("*")): if not path.is_file(): continue try: rel = path.relative_to(root) except Exception: continue if not is_publishable_workspace_file(rel): continue try: result[str(rel)] = hashlib.sha256(path.read_bytes()).hexdigest() except Exception: result[str(rel)] = "" return result def write_repair_diff_artifact(run_dir: Path, events_path: Path, before_dir: Path, after_root: Path, *, category: str = "", expected_files: list[str] | None = None) -> dict: before = workspace_publishable_fingerprint(before_dir) after = workspace_publishable_fingerprint(after_root) before_keys = set(before) after_keys = set(after) changed = sorted(k for k in before_keys & after_keys if before.get(k) != after.get(k)) added = sorted(after_keys - before_keys) removed = sorted(before_keys - after_keys) expected_files = expected_files or [] expected_touched = sorted(set(expected_files) & set(changed + added + removed)) payload = { "schema_version": "1.0", "category": category, "changed_files": changed, "added_files": added, "removed_files": removed, "has_publishable_diff": bool(changed or added or removed), "expected_files": expected_files, "expected_files_touched": expected_touched, } repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "repair_diff.json", payload) append_event( events_path, "repair_diff", "success" if payload["has_publishable_diff"] else "failed", "Computed publishable before/after repair diff", payload, ) return payload def expected_repair_files_from_classification(classification: dict | None = None, dependency_issue: dict | None = None) -> list[str]: category = (classification or {}).get("category") or "" files: list[str] = [] if category == "dependency_error" or dependency_issue: files.extend(["requirements.txt", "requirements_policy.json"]) return files def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path, dependency_evidence: str = ""): """Normalize broad known-dangerous base and platform dependencies before upload. Do not try to solve every dependency conflict here. Do not try to solve every model-specific dependency conflict here. Pi is responsible for reading concrete build logs and patching requirements when a Space build fails. The Factory only prevents obviously unsafe broad ranges and owns the Gradio/HF platform stack so old generated pins cannot conflict with the current Spaces runtime; concrete pip conflicts remain Pi repair work. """ req_path = workspace / "requirements.txt" if not req_path.exists(): return raw = req_path.read_text(encoding="utf-8", errors="ignore") lines = [line.rstrip() for line in raw.splitlines()] prefix_lines: list[str] = [] package_lines: list[str] = [] for line in lines: stripped = line.strip() if not stripped: continue if stripped.startswith("--") or stripped.startswith("-f "): prefix_lines.append(line) else: package_lines.append(line) # Minimal base pins only. Do not globally pin diffusers here: newer model # cards may legitimately need a recent Diffusers release. If diffusers causes # a pip conflict, Pi is responsible for concrete build-log repairs. Pi must repair from the concrete build log. transformers5_issue = workspace_allows_transformers_major5(workspace, dependency_evidence) keep_transformers_policy, transformers_policy_reason = workspace_should_keep_transformers_policy(workspace, dependency_evidence) default_policy: dict[str, str] = { # Spaces currently installs platform Gradio with a modern hub floor. Keep # the app-side hub range compatible with that platform stack instead of # forcing pip to backtrack into a known impossible 4.x Transformers path. "huggingface-hub": "huggingface_hub>=1.2.0,<2.0.0", "transformers": "transformers>=4.51.0", } transformers_policy = "transformers>=5.0.0" if transformers5_issue else default_policy["transformers"] policy: dict[str, str] = { "huggingface-hub": default_policy["huggingface-hub"], } if keep_transformers_policy: policy["transformers"] = transformers_policy # Platform-owned dependencies are normalized more aggressively than model # dependencies. Gradio/HF runtime packages must stay current with Spaces and # Agentic Space Factory validation; Pi should patch app.py for modern Gradio # rather than pinning or downgrading this stack. platform_policy: dict[str, str] = { "gradio": "gradio", "gradio-client": "gradio-client", "huggingface-hub": policy["huggingface-hub"], "spaces": "spaces>=0.30", "hf-xet": "hf_xet", } aliases = { "huggingface_hub": "huggingface-hub", "huggingface-hub": "huggingface-hub", "transformers": "transformers", "gradio": "gradio", "gradio-client": "gradio-client", "gradio_client": "gradio-client", "spaces": "spaces", "hf-xet": "hf-xet", "hf_xet": "hf-xet", } platform_owned = set(platform_policy) seen_policy: set[str] = set() seen_platform: set[str] = set() filtered: list[str] = [] changed = False removed_platform_pins: list[str] = [] normalized_platform_lines: list[dict] = [] for line in package_lines: stripped = line.strip() if stripped.startswith("#") or "://" in stripped or stripped.startswith((".", "/")): filtered.append(line) continue name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") canonical = aliases.get(name) if canonical in platform_owned: normalized = platform_policy[canonical] if stripped != normalized: changed = True removed_platform_pins.append(stripped) normalized_platform_lines.append({"from": stripped, "to": normalized, "package": canonical}) if canonical not in seen_platform: filtered.append(normalized) seen_platform.add(canonical) if canonical in policy: seen_policy.add(canonical) else: changed = True continue if canonical in policy: if stripped != policy[canonical]: changed = True if canonical not in seen_policy: filtered.append(policy[canonical]) seen_policy.add(canonical) else: changed = True continue if canonical == "transformers" and not keep_transformers_policy: changed = True normalized_platform_lines.append({"from": stripped, "to": "removed", "package": "transformers", "reason": transformers_policy_reason}) continue filtered.append(line) stable_policy_lines: list[str] = [] for canonical in ["huggingface-hub", "transformers"]: if canonical in policy and canonical not in seen_policy: stable_policy_lines.append(policy[canonical]) changed = True injected_platform_lines: list[str] = [] app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore").lower() if (workspace / "app.py").exists() else "" if ("import gradio" in app_text or "from gradio" in app_text or "gr." in app_text) and "gradio" not in seen_platform: injected_platform_lines.append(platform_policy["gradio"]) seen_platform.add("gradio") changed = True if ("gradio_client" in app_text or "from gradio_client" in app_text) and "gradio-client" not in seen_platform: injected_platform_lines.append(platform_policy["gradio-client"]) seen_platform.add("gradio-client") changed = True torch_added = False if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines + injected_platform_lines, "torch"): stable_policy_lines.append("torch>=2.0.0") torch_added = True changed = True app_deps_text = "" deps_path = workspace / "MODEL_DEPENDENCIES.json" if deps_path.exists(): app_deps_text = deps_path.read_text(encoding="utf-8", errors="ignore").lower() lora_peft_added = False if any(x in (app_text + "\n" + app_deps_text) for x in ["load_lora_weights", "peft_type", "lora_adapter", "diffusers_lora", "adapter_config.json"]): candidate_lines = filtered + stable_policy_lines + injected_platform_lines if not requirements_has_package(candidate_lines, "peft"): stable_policy_lines.append("peft") lora_peft_added = True changed = True new_lines = prefix_lines + stable_policy_lines + injected_platform_lines + filtered new = "\n".join(line for line in new_lines if line.strip()) + "\n" if new != raw: changed = True if changed: req_path.write_text(new, encoding="utf-8") policy_payload = { "schema_version": "1.0", "status": "normalized", "platform_owned_dependencies": ["gradio", "gradio-client", "huggingface_hub", "spaces", "hf_xet"], "removed_pins": removed_platform_pins, "normalized_platform_lines": normalized_platform_lines, "injected_platform_lines": injected_platform_lines, "base_policy": {"huggingface_hub": policy["huggingface-hub"], "transformers": policy.get("transformers")}, "transformers_policy_applied": bool(keep_transformers_policy), "transformers_policy_reason": transformers_policy_reason, "transformers_upper_bound_relaxed": bool(transformers5_issue), "transformers_upper_bound_relaxation_evidence": transformers5_issue, "base_policy_reason": "Avoid forcing Transformers 4.x under modern Gradio/HF Hub platform constraints; do not inject or preserve Transformers for CPU-only/TTS/ONNX apps without code or dependency evidence; require torch when app.py imports torch.", "torch_added": torch_added, "torch_policy": "torch>=2.0.0", "torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present", "peft_added_for_diffusers_lora": lora_peft_added, "peft_policy": "peft is required by Diffusers load_lora_weights for LoRA adapters", "model_dependency_policy": "Model-specific pins are preserved unless they hit a known deterministic guardrail; concrete conflicts remain Pi repair work from build logs.", "reason": "Gradio/HF runtime dependencies are owned by Agentic Space Factory / Spaces runtime and must not be downgraded by generated requirements.", } write_json(workspace / "requirements_policy.json", policy_payload) append_event( events_path, "requirements_sanitize", "success", "Normalized platform-owned Gradio/HF dependencies and broad base dependencies; concrete pip conflicts remain Pi repair work", policy_payload, ) def useful_log_signals(text: str) -> list[str]: low = (text or "").lower() signals = [] patterns = { "python_exception": ["traceback", "modulenotfounderror", "importerror", "runtimeerror", "valueerror", "typeerror"], "dependency_resolution": ["resolutionimpossible", "no matching distribution", "could not find a version", "pip subprocess", "metadata-generation-failed"], "gradio_api": ["api_name", "endpoint", "unexpected keyword", "too many arguments", "not enough arguments", "no api found"], "model_loading": ["from_pretrained", "model_index", "safetensors", "does not appear to have", "failed to load", "pipeline"], "hardware_memory": ["cuda out of memory", "outofmemoryerror", "oom", "not enough memory"], "auth_access": ["401", "403", "gated", "unauthorized", "forbidden", "repository not found"], "zerogpu": ["zerogpu", "spaces.gpu", "duration"], } for name, needles in patterns.items(): if any(n in low for n in needles): signals.append(name) return signals def log_quality(build_log: str = "", runtime_log: str = "", failure_reason: str = "") -> str: combined = f"{build_log}\n{runtime_log}\n{failure_reason}".strip() low = combined.lower() if not combined or len(combined) < 80: return "empty" no_reason_markers = [ "no logs", "logs are empty", "no reason", "unknown error", "runtime error", "build error", "error: none", "status=error", ] if any(m in low for m in no_reason_markers) and not useful_log_signals(combined): return "no_reason" if useful_log_signals(combined): return "useful" return "partial" def classify_repair_failure(failure_reason: str, build_log: str = "", runtime_log: str = "") -> dict: """Classify a failed live validation before asking Pi to choose an action. This classifier is deliberately conservative. It gives Pi and the Factory evidence quality and a suggested class, but the diagnosis step must still make a bounded decision before any patch is allowed. """ text = f"{failure_reason}\n{build_log[-6000:]}\n{runtime_log[-6000:]}".lower() quality = log_quality(build_log, runtime_log, failure_reason) phase = "unknown" if "build" in text or "pip" in text or "container" in text: phase = "space_build" if "runtime" in text or "health" in text or "client.predict" in text or "api" in text: phase = "space_runtime" if "generation" in text or "smoke" in text or "expected output" in text: phase = "api_validation" checks = [ ("cuda_oom", ["cuda out of memory", "outofmemoryerror", "oom", "not enough memory"], "Prefer hardware/manual action or memory reduction; do not fake inference."), ("dependency_error", ["resolutionimpossible", "could not find a version", "no matching distribution", "dependency conflict", "pip subprocess", "metadata-generation-failed"], "Fix requirements with compatible, modern pins."), ("import_error", ["modulenotfounderror", "importerror", "cannot import name"], "Fix imports or requirements without hiding the error."), ("gradio_api_mismatch", ["no api found", "api_name", "endpoint", "unexpected keyword", "too many arguments", "not enough arguments"], "Preserve the expected Gradio endpoint and align input/output schema."), ("model_loading_error", ["from_pretrained", "safetensors", "model_index", "pipeline", "failed to load", "does not appear to have"], "Re-read the model card usage and fix the pipeline loading path."), ("hf_auth_error", ["401", "403", "gated", "unauthorized", "forbidden", "repository not found"], "Do not bypass auth; report gated/private access if needed."), ("space_boot_timeout", ["timed out", "timeout", "space did not become ready", "sleeping"], "Inspect logs first; if logs are empty, wait/rebuild same code before patching."), ("wrong_output_type", ["wrong output", "expected output", "invalid output", "not an image", "not text"], "Return the expected output type from real inference."), ("zero_gpu_duration_error", ["duration", "spaces.gpu", "zerogpu", "zero gpu"], "Tune @spaces.GPU(duration=...) from observed smoke-test latency when available."), ] category = "unknown_runtime_error" recommendation = "Use the Space logs, Gradio API schema and smoke error to choose the smallest truthful action." for code, needles, rec in checks: if any(n in text for n in needles): category = code recommendation = rec break gradio_kwarg_issue = extract_gradio_unexpected_kwarg(f"{failure_reason}\\n{build_log}\\n{runtime_log}") if gradio_kwarg_issue: category = "gradio_component_api_error" phase = "space_runtime" quality = "useful" recommendation = "Patch the unsupported Gradio component keyword without changing the inference backend." if quality in {"empty", "no_reason"} and category == "unknown_runtime_error": category = "hf_runtime_flake" if phase != "space_build" else "hf_build_flake" recommendation = "Logs are not actionable. Prefer wait/inspect or a same-code factory rebuild before any code patch." dependency_issue = extract_pip_dependency_issue(f"{failure_reason}\n{build_log}\n{runtime_log}") failure_class = category if dependency_issue: category = "dependency_error" phase = "space_build" quality = "useful" failure_class = dependency_issue.get("failure_class") or "dependency_resolution_conflict" if dependency_issue.get("platform_conflict", {}).get("detected"): recommendation = "Structural dependency platform conflict detected. Attempt only requirements.txt / requirements_policy.json repair, or declare dependency_platform_conflict without app-code edits." else: recommendation = "Patch requirements.txt / requirements_policy.json from the first pip resolver error; do not rebuild same code first." return { "category": category, "failure_class": failure_class, "failure_phase": phase, "logs_quality": quality, "signals": useful_log_signals(text), "recommendation": recommendation, "dependency_issue": dependency_issue, "gradio_kwarg_issue": gradio_kwarg_issue, "repair_type": gradio_kwarg_issue.get("repair_type", "") if gradio_kwarg_issue else "", "gist_alignment": "read first actionable pip error, patch minimally, rebuild, validate live" if dependency_issue else "standard blockage diagnosis", } def workspace_file_inventory(workspace: Path, max_files: int = 80) -> list[str]: files = [] for path in sorted(workspace.rglob("*")): if not path.is_file(): continue if any(part in {".git", ".cache", "node_modules", "__pycache__"} for part in path.parts): continue try: files.append(str(path.relative_to(workspace))) except Exception: files.append(path.name) if len(files) >= max_files: break return files def write_incident_brief(workspace: Path, run_dir: Path, *, target_space_id: str, model_id: str, pi_model: str, failure_reason: str, build_log: str, runtime_log: str, classification: dict, implementation_mode: str, expected_output_type: str, iteration: int, budgets: dict | None = None) -> str: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) inventory = workspace_file_inventory(workspace) budgets = budgets or {} brief = f"""# Agentic Space Factory incident brief This is a blockage diagnosis pass for an existing generated Hugging Face Space. Pi must use the HF Spaces gist method: read logs first, act once, choose the cheapest useful iteration rung, and verify on the live Space. ## Target - Model ID: `{model_id}` - Target Space: `{target_space_id}` - Pi model: `{pi_model}` - Implementation mode: `{implementation_mode}` - Expected output type: `{expected_output_type}` - Diagnosis iteration: `{iteration}` - HF Space run logs endpoint: `https://huggingface.co/api/spaces/{target_space_id}/logs/run` - HF Space build logs endpoint: `https://huggingface.co/api/spaces/{target_space_id}/logs/build` ## Current classifier hints ```json {json.dumps(classification, indent=2, ensure_ascii=False)} ``` ## Dependency build error policy If `dependency_issue` is non-empty, this is an actionable pip build failure. Prefer `patch_code` with a minimal `requirements.txt` change. Do not choose `factory_rebuild_same_code` until a dependency patch has been attempted. ## HF tooling/token context ```json {json.dumps(load_json_if_exists(run_dir / "token_context.json"), indent=2, ensure_ascii=False)} ``` ## Remaining action budget ```json {json.dumps(budgets, indent=2, ensure_ascii=False)} ``` ## Observed failure ```text {failure_reason[:5000]} ``` ## Actionable error excerpt ```text {extract_actionable_error_excerpt(failure_reason + chr(10) + build_log + chr(10) + runtime_log, max_lines=120)} ``` Full logs remain archived under `logs/space_logs_build.txt`, `logs/space_logs_runtime.txt`, and `logs/space_logs_run.txt`. Do not inspect full logs unless this excerpt is insufficient. ## Workspace files ```text {chr(10).join(inventory)} ``` ## Allowed decisions You must choose exactly one action: - `wait_for_logs`: logs are empty/late; wait and collect again. - `inspect_more_logs`: enough uncertainty remains; collect logs/API status again without changing code. - `factory_rebuild_same_code`: likely HF build/runtime flake; re-upload the same workspace to force a rebuild, then revalidate. - `patch_code`: logs are actionable and point to a code/dependency/API/model-loading issue. - `request_manual_hardware`: issue is hardware/quota/restricted GPU/memory and should not be patched away. - `declare_technical_blocker`: no safe automated action remains. ## Hard rules - Do not choose `patch_code` when logs are empty or no-reason. The Factory must first fetch real HF Space logs from `/logs/run` or `/logs/build`, or use a concrete validator/API-schema signal. - Do not fake inference or replace model output with placeholders. - Do not patch hardware/quota/authorization failures as code bugs. - If logs are actionable, identify the first real error, not only the last line. - If using `factory_rebuild_same_code`, do not change files. ## Required output Write `REPAIR_DECISION.json` in the workspace root with this schema: ```json {{ "action": "wait_for_logs|inspect_more_logs|factory_rebuild_same_code|patch_code|request_manual_hardware|declare_technical_blocker", "confidence": "low|medium|high", "reason": "short explanation", "evidence": ["specific evidence"], "patch_allowed": false, "requires_manual_hardware": false }} ``` """ (repair_dir / "INCIDENT_BRIEF.md").write_text(brief, encoding="utf-8") (workspace / "INCIDENT_BRIEF.md").write_text(brief, encoding="utf-8") write_json(repair_dir / "classification.json", classification) return brief def extract_json_object(text: str) -> dict: if not text: return {} candidates = [] fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", text, flags=re.S) candidates.extend(fenced) first = text.find("{") last = text.rfind("}") if first != -1 and last != -1 and last > first: candidates.append(text[first:last + 1]) for candidate in candidates: try: value = json.loads(candidate) if isinstance(value, dict): return value except Exception: continue return {} def extract_pip_dependency_issue(text: str = "") -> dict: """Extract the first actionable pip dependency resolver/build error. v198.26.1: this is intentionally generic. The worker does not try to be a package-resolution agent; it makes dependency failures first-class evidence, extracts the likely packages, distinguishes structural platform conflicts, and constrains any Pi repair to dependency files. """ raw = text or "" low = raw.lower() issue: dict = {} patterns = [ ("no_matching_distribution", r"Could not find a version that satisfies the requirement\s+([^\s]+)(?:\s+\(from[^\n]*\))?"), ("no_matching_distribution", r"No matching distribution found for\s+([^\s]+)"), ("resolution_impossible", r"ERROR:\s*Cannot install[^\n]{0,2000}?conflicting dependencies[^\n]*"), ("resolution_impossible", r"ResolutionImpossible[^\n]*"), ("metadata_generation_failed", r"metadata-generation-failed[^\n]*"), ("metadata_generation_failed", r"Preparing metadata .*? did not run successfully[^\n]*"), ("wheel_build_failed", r"Failed building wheel for\s+([^\s]+)"), ("subprocess_exited", r"subprocess-exited-with-error[^\n]*"), ("subprocess_exited", r"pip subprocess.*?did not run successfully[^\n]*"), ("requires_python", r"requires-python[^\n]*(?:not satisfied|unsupported|incompatible)[^\n]*"), ] for kind, pattern in patterns: match = re.search(pattern, raw, flags=re.I | re.S) if match: requirement = match.group(1).strip(" ,.;:\")'") if match.groups() else "" package = _normalize_dependency_package_name(requirement) issue = { "schema_version": "dependency_issue.v198_25_3", "kind": kind, "requirement": requirement, "package": package, "line": match.group(0)[:1500], } break generic_markers = [ "could not find a version", "no matching distribution", "resolutionimpossible", "metadata-generation-failed", "subprocess-exited-with-error", "failed building wheel", "requires-python", "conflicting dependencies", "cannot install", ] if not issue and any(marker in low for marker in generic_markers): issue = {"schema_version": "dependency_issue.v198_25_3", "kind": "dependency_resolution", "requirement": "", "package": "", "line": raw[-2000:]} if issue: packages = extract_dependency_conflict_packages(raw) if issue.get("package") and issue["package"] not in packages: packages.insert(0, issue["package"]) platform_conflict = detect_dependency_platform_conflict(raw, packages) failure_class = "dependency_platform_conflict" if platform_conflict.get("detected") else "dependency_resolution_conflict" issue.update({ "category": "dependency_error", "failure_owner": "dependency", "failure_class": failure_class, "actionable": True, "repair_candidate": True, "app_code_repair_allowed": False, "recommended_action": "patch_requirements" if not platform_conflict.get("detected") else "surgical_requirements_repair_or_declare_platform_conflict", "recommended_patch_scope": ["requirements.txt", "requirements_policy.json"], "allowed_files": ["requirements.txt", "requirements_policy.json"], "forbidden_files": ["app.py", "README.md", "INFERENCE_CONTRACT.json", "DEMO_QUALITY_CONTRACT.json"], "packages": packages, "platform_conflict": platform_conflict, "summary": "pip dependency resolution/build failure during Space build", }) return issue def write_dependency_error_brief(workspace: Path, run_dir: Path, events_path: Path, issue: dict, build_log: str = "", runtime_log: str = "", failure_reason: str = "") -> str: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) brief = f"""# Dependency build error brief The Space build failed during dependency installation. This brief does **not** replace the HF Spaces gist. It is a focused evidence appendix for applying the gist method: read logs first, find the first actionable build error, make one minimal requirements patch, rebuild, then validate live. ## First dependency error ```json {json.dumps(issue, indent=2, ensure_ascii=False)} ``` ## Gist-aligned repair guidance - Read the first real pip resolver error, not only the last log line. - Use the cheapest useful iteration rung from the gist. - For actionable pip errors, the cheapest useful action is normally a minimal `requirements.txt` / dependency pin patch, followed by rebuild and live validation. - Do not use `factory_rebuild_same_code` before a dependency patch when the build log already contains a concrete pip resolver error. - Do not change inference behavior to work around dependency resolution. - Keep the patch minimal and explain why the chosen versions are compatible. ## Build log tail ```text {build_log[-12000:]} ``` ## Runtime log tail ```text {runtime_log[-4000:]} ``` ## Failure reason ```text {failure_reason[:4000]} ``` """ (repair_dir / "DEPENDENCY_ERROR_BRIEF.md").write_text(brief, encoding="utf-8") (workspace / "DEPENDENCY_ERROR_BRIEF.md").write_text(brief, encoding="utf-8") append_event(events_path, "failure_diagnosis", "warning", "Detected pip dependency build error; applying gist log-first method", {"dependency_issue": issue, "artifact": "repair/DEPENDENCY_ERROR_BRIEF.md"}) write_agent_trace_record(run_dir, phase="diagnosis", event="dependency_error_detected", status="warning", message="Detected pip dependency build error; applying gist log-first method", data={"dependency_issue": issue}, artifacts=["repair/DEPENDENCY_ERROR_BRIEF.md"]) return brief def known_dependency_guardrail_needed(workspace: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "") -> bool: text = f"{failure_reason}\n{build_log}\n{runtime_log}" low = text.lower() issue = extract_pip_dependency_issue(text) # Keep automatic deterministic fixes extremely narrow. General pip conflicts # must be handed to Pi with DEPENDENCY_ERROR_BRIEF.md. The Cosmos-style # Transformers 5 conflict is safe to repair mechanically because ASF itself # owns the incompatible `<5` upper bound. if detect_transformers5_dependency_requirement(text): return True if issue and issue.get("package") == "safetensors" and "transformers-5" in low and "safetensors>=0.8.0" in low: return True return False def apply_dependency_guardrail_repair(workspace: Path, run_dir: Path, events_path: Path, failure_reason: str = "", build_log: str = "", runtime_log: str = "") -> bool: """Apply deterministic requirements repair for known pip resolver failures. This handles build failures before Pi can produce a useful runtime patch. """ if not known_dependency_guardrail_needed(workspace, failure_reason, build_log, runtime_log): return False evidence_text = f"{failure_reason}\n{build_log}\n{runtime_log}" before = (workspace / "requirements.txt").read_text(encoding="utf-8", errors="ignore") if (workspace / "requirements.txt").exists() else "" relax_transformers_upper_bound_for_repair(workspace, events_path, evidence_text) normalize_requirements_for_modern_hub(workspace, events_path, dependency_evidence=evidence_text) after = (workspace / "requirements.txt").read_text(encoding="utf-8", errors="ignore") if (workspace / "requirements.txt").exists() else "" repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) summary = """# Deterministic dependency repair The Space build failed during `pip install` because the resolver selected a known Transformers 5.x path requiring unavailable `safetensors>=0.8.0`. The Factory applied only the narrow known trap guardrail: - `transformers>=4.51.0,<5.0.0` - `huggingface_hub>=0.34.0,<2.0.0` Other dependency conflicts are passed to Pi through `DEPENDENCY_ERROR_BRIEF.md`. No inference code was changed. """ (repair_dir / "REPAIR_SUMMARY.md").write_text(summary, encoding="utf-8") (repair_dir / "REPAIR_PLAN.md").write_text("# Deterministic dependency repair plan\n\nNormalize requirements and re-upload the same workspace for a clean Space rebuild.\n", encoding="utf-8") write_agent_trace_record( run_dir, phase="repair_patch", event="dependency_guardrail_repair", status="success", message="Factory applied deterministic requirements repair for safetensors>=0.8.0 resolver failure", data={"before": before[-2000:], "after": after[-2000:]}, artifacts=["repair/REPAIR_SUMMARY.md", "repair/REPAIR_PLAN.md"], ) append_event( events_path, "repair_patch", "success", "Applied deterministic dependency guardrail for pip resolver build error", { "category": "dependency_error", "transformers": "evidence_backed_policy", "note": "Known dependency conflicts are repaired only from concrete build-log evidence; Transformers <5 is relaxed when logs prove a dependency requires Transformers 5.x.", }, ) return before != after def fallback_decision_from_classifier(classification: dict, budgets: dict | None = None) -> dict: budgets = budgets or {} category = classification.get("category") or "unknown_runtime_error" quality = classification.get("logs_quality") or "partial" if category in {"hf_auth_error"}: action = "declare_technical_blocker" elif category == "cuda_oom": action = "request_manual_hardware" elif quality in {"empty", "no_reason"}: if budgets.get("wait_for_logs", 0) > 0: action = "wait_for_logs" elif budgets.get("factory_rebuild_same_code", 0) > 0: action = "factory_rebuild_same_code" else: action = "declare_technical_blocker" elif category in {"dependency_error", "import_error", "gradio_api_mismatch", "model_loading_error", "wrong_output_type", "zero_gpu_duration_error", "space_boot_timeout"}: action = "patch_code" if budgets.get("patch_code", 0) > 0 else "declare_technical_blocker" elif budgets.get("factory_rebuild_same_code", 0) > 0: action = "factory_rebuild_same_code" elif budgets.get("patch_code", 0) > 0 and quality == "useful": action = "patch_code" else: action = "declare_technical_blocker" return { "action": action, "confidence": "low", "reason": f"Factory fallback decision from classifier category={category}, logs_quality={quality}.", "evidence": classification.get("signals") or [], "patch_allowed": action == "patch_code", "requires_manual_hardware": action == "request_manual_hardware", "source": "factory_fallback", } def normalize_repair_decision(decision: dict, classification: dict, budgets: dict | None = None) -> dict: budgets = budgets or {} allowed = {"wait_for_logs", "inspect_more_logs", "factory_rebuild_same_code", "patch_code", "request_manual_hardware", "declare_technical_blocker"} action = str(decision.get("action") or "").strip().lower().replace("-", "_") if action not in allowed: decision = fallback_decision_from_classifier(classification, budgets) action = decision["action"] quality = classification.get("logs_quality") or "partial" category = classification.get("category") or "unknown_runtime_error" overrides = [] if action == "patch_code" and quality in {"empty", "no_reason"}: # v198.16: do not burn Pi repair budget from a generic BUILD_ERROR/RUNTIME_ERROR # when the real /logs/run or /logs/build streams were not fetched or contain no # actionable traceback. Pi can patch only after useful logs or another concrete # non-log validator signal upgrades quality. overrides.append("patch_code_blocked_without_actionable_hf_space_logs") if budgets.get("wait_for_logs", 0) > 0: action = "wait_for_logs" elif budgets.get("inspect_more_logs", 0) > 0: action = "inspect_more_logs" elif budgets.get("factory_rebuild_same_code", 0) > 0: action = "factory_rebuild_same_code" else: action = "declare_technical_blocker" if category == "dependency_error" and quality == "useful" and action in {"wait_for_logs", "inspect_more_logs", "factory_rebuild_same_code"}: overrides.append("gist_log_first_dependency_error_needs_minimal_requirements_patch") # The gist says read the first actionable error and act once surgically. # For a concrete pip resolver error, a same-code rebuild/wait is not the # cheapest useful rung. Ask Pi to patch requirements minimally instead. action = "patch_code" if budgets.get("patch_code", 0) > 0 else "declare_technical_blocker" if action in budgets and budgets.get(action, 0) <= 0: overrides.append(f"budget_exhausted:{action}") action = "declare_technical_blocker" if action == "patch_code" and category in {"hf_auth_error"}: overrides.append("auth_failure_not_patchable") action = "declare_technical_blocker" if action == "patch_code" and category == "cuda_oom": # The agent may sometimes reduce memory pressure, but default to manual # hardware unless the logs are useful and it explicitly gave evidence. evidence = " ".join(str(x).lower() for x in decision.get("evidence") or []) if "dtype" not in evidence and "offload" not in evidence and "steps" not in evidence: overrides.append("oom_needs_hardware_or_explicit_memory_plan") action = "request_manual_hardware" normalized = { "action": action, "confidence": str(decision.get("confidence") or "low"), "reason": str(decision.get("reason") or "No reason provided.")[:2000], "evidence": list(decision.get("evidence") or [])[:20], "patch_allowed": action == "patch_code", "requires_manual_hardware": action == "request_manual_hardware", "factory_overrides": overrides, "classification": classification, } return normalized def build_repair_task_packet(classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None) -> dict: category = (classification or {}).get("category") or "unknown_runtime_error" signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification or {}) evidence = [] for item in (decision or {}).get("evidence") or []: evidence.append(str(item)[:500]) if not evidence: excerpt = extract_actionable_error_excerpt("\n".join([failure_reason or "", build_log or "", runtime_log or ""]), max_lines=40) if excerpt: evidence.append(excerpt[-2000:]) task_type = "runtime_patch" allowed_files = ["app.py", "requirements.txt"] expected_fix = "Apply the smallest code/dependency patch that addresses the classified failure." patch_mode = "pi_targeted_patch" if category == "dependency_error": task_type = "dependency_repair" allowed_files = ["requirements.txt", "requirements_policy.json"] expected_fix = "Patch the first concrete dependency issue minimally in dependency files only. Do not edit app.py for pip resolver conflicts." elif category in {"import_error"}: task_type = "import_or_missing_dependency" allowed_files = ["app.py", "requirements.txt"] expected_fix = "Fix imports or add the missing dependency without changing the model or replacing real inference." elif category in {"gradio_api_mismatch", "gradio_component_api_error"}: task_type = "gradio_api_contract" allowed_files = ["app.py"] expected_fix = "Expose/repair the expected Gradio endpoint and modern Gradio component API without changing the model." elif category == "wrong_output_type": task_type = "output_contract_repair" allowed_files = ["app.py"] expected_fix = "Return the expected output type from the existing real inference path." elif category == "model_loading_error": task_type = "model_loading_repair" allowed_files = ["app.py", "requirements.txt"] expected_fix = "Fix the pipeline/model loading path using repo/model-card evidence while preserving real inference." elif category == "zero_gpu_duration_error": task_type = "zerogpu_duration_repair" allowed_files = ["app.py", "INFERENCE_CONTRACT.json", "README.md"] expected_fix = "Adjust @spaces.GPU duration from observed inference latency without changing inference semantics." elif category == "cuda_oom": task_type = "memory_optimization_patch" allowed_files = ["app.py"] expected_fix = "Only apply concrete memory optimizations supported by MEMORY_DIAGNOSIS.json evidence." patch_mode = "pi_memory_patch_after_diagnosis" packet = { "schema_version": "repair_task_packet.v198_25", "task_type": task_type, "repair_mode": "deep_repair_reanalysis" if is_deep_repair_category(category) else "surgical_repair", "patch_mode": patch_mode, "failure_class": category, "failure_owner": failure_owner_from_classification(classification or {}), "failure_signature": signature, "logs_quality": (classification or {}).get("logs_quality"), "failure_phase": (classification or {}).get("failure_phase"), "authoritative_error": evidence[0] if evidence else "", "evidence": evidence[:5], "allowed_files": allowed_files, "expected_fix": expected_fix, "do_not_reinterpret": True, "success_criteria": [ "The Space should boot past the current failure signature.", "Real inference must be preserved; no placeholder/canned outputs.", "The expected Gradio health and generation/API contract must remain available.", ], "forbidden_changes": [ "Do not replace the model id.", "Do not remove real inference.", "Do not hide failures behind broad try/except placeholder responses.", "Do not turn the Space into diagnostic-only unless writing an explicit blocker.", "Do not publish or push anything from Pi.", ] + (["For dependency_repair, do not edit app.py; write PATCH_REFUSAL.json if the conflict cannot be resolved safely from dependency files only."] if task_type == "dependency_repair" else []), "full_context_files_available_if_needed": ["REPAIR_BRIEF.md", "INCIDENT_BRIEF.md", "DEPENDENCY_ERROR_BRIEF.md"], "default_context_files": ["REPAIR_TASK_PACKET.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], "refusal_mode": "If the frozen decision appears wrong or the failure belongs to the Factory, write PATCH_REFUSAL.json and do not edit publishable files.", } if decision: packet["diagnosis_decision"] = decision return packet def write_repair_task_packet(workspace: Path, run_dir: Path, classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None) -> dict: packet = build_repair_task_packet(classification, failure_reason, build_log, runtime_log, decision) repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "REPAIR_TASK_PACKET.json", packet) write_json(workspace / "REPAIR_TASK_PACKET.json", packet) write_json(repair_dir / "PI_TASK_PACKET.json", packet) write_json(workspace / "PI_TASK_PACKET.json", packet) return packet def detect_memory_features_from_app(workspace: Path) -> dict: app_text = _read_workspace_text_if_exists(workspace / "app.py") low = app_text.lower() def has_any(*needles): return any(n in low for n in needles) size_match = re.findall(r"(?:width|height)\s*=\s*(\d{3,4})", app_text) steps_match = re.findall(r"(?:num_inference_steps|steps)\s*=\s*(\d{1,3})", app_text) return { "torch_dtype_bfloat16": has_any("torch.bfloat16", "dtype=torch.bfloat16"), "torch_dtype_float16": has_any("torch.float16", "torch_dtype=torch.float16", "variant=\"fp16\"", "variant='fp16'"), "uses_cpu_offload": has_any("enable_model_cpu_offload", "enable_sequential_cpu_offload"), "uses_attention_slicing": has_any("enable_attention_slicing", "enable_vae_slicing", "enable_vae_tiling"), "moves_pipeline_to_cuda": has_any(".to(\"cuda\")", ".to('cuda')"), "detected_sizes": size_match[:10], "detected_steps": steps_match[:10], } def write_memory_diagnosis_packet(workspace: Path, run_dir: Path, *, model_id: str, target_space_id: str, failure_reason: str, build_log: str, runtime_log: str, classification: dict, selected_hardware: str = "") -> dict: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) excerpt = extract_actionable_error_excerpt("\n".join([failure_reason or "", build_log or "", runtime_log or ""]), max_lines=120) packet = { "schema_version": "memory_diagnosis_packet.v198_25", "task_type": "memory_diagnosis", "model_id": model_id, "target_space_id": target_space_id, "selected_hardware": selected_hardware, "failure_class": (classification or {}).get("category"), "failure_phase": (classification or {}).get("failure_phase"), "logs_quality": (classification or {}).get("logs_quality"), "error_excerpt": excerpt[-4000:], "detected_memory_features": detect_memory_features_from_app(workspace), "app_py_relevant_excerpt": _read_workspace_text_if_exists(workspace / "app.py")[:12000], "requirements_excerpt": _read_workspace_text_if_exists(workspace / "requirements.txt")[:4000], } write_json(repair_dir / "MEMORY_DIAGNOSIS_PACKET.json", packet) write_json(workspace / "MEMORY_DIAGNOSIS_PACKET.json", packet) return packet def normalize_memory_diagnosis(raw: dict, classification: dict) -> dict: raw = raw if isinstance(raw, dict) else {} allowed_classes = {"hardware_capacity", "fixable_memory_optimization", "inference_parameter_too_heavy", "memory_leak_or_cache", "unknown"} cls = str(raw.get("classification") or raw.get("memory_classification") or "unknown").strip().lower() if cls not in allowed_classes: cls = "unknown" rec = str(raw.get("recommended_action") or "request_manual_hardware").strip().lower() if rec not in {"patch_code", "request_manual_hardware", "inspect_more_logs", "declare_technical_blocker"}: rec = "request_manual_hardware" if cls in {"hardware_capacity", "unknown"} else "patch_code" return { "schema_version": "memory_diagnosis.v198_23", "classification": cls, "confidence": str(raw.get("confidence") or "low"), "evidence": list(raw.get("evidence") or [])[:10], "recommended_action": rec, "patch_opportunities": list(raw.get("patch_opportunities") or raw.get("allowed_patch_types") or [])[:12], "classification_source": "pi_memory_diagnosis", "base_failure_classification": classification, } def diagnose_memory_with_pi(workspace: Path, run_dir: Path, events_path: Path, pi_model: str, target_space_id: str, model_id: str, failure_reason: str, classification: dict) -> dict: logs_dir = run_dir / "logs" build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else "" runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_memory_diagnosis_packet(workspace, run_dir, model_id=model_id, target_space_id=target_space_id, failure_reason=failure_reason, build_log=build_log, runtime_log=runtime_log, classification=classification) goal = f"""You are Pi in MEMORY DIAGNOSIS MODE for Agentic Space Factory. Read `MEMORY_DIAGNOSIS_PACKET.json` first. You are not allowed to edit files in this step. Your task is to decide whether the CUDA OOM is most likely: - `hardware_capacity`: the selected hardware is too small even for a reasonable implementation. - `fixable_memory_optimization`: the app has concrete fixable memory issues such as missing bf16/fp16, missing offload/slicing/tiling, duplicate pipeline loads, excessive default steps/resolution, or cache misuse. - `inference_parameter_too_heavy`: defaults are too large but the implementation is otherwise plausible. - `memory_leak_or_cache`: repeated calls leak or retain tensors. - `unknown`: evidence is insufficient. Return only `MEMORY_DIAGNOSIS.json` with this schema: ```json {{ "classification": "hardware_capacity|fixable_memory_optimization|inference_parameter_too_heavy|memory_leak_or_cache|unknown", "confidence": "low|medium|high", "evidence": ["specific evidence from app/log excerpt"], "recommended_action": "patch_code|request_manual_hardware|inspect_more_logs|declare_technical_blocker", "patch_opportunities": ["set_bfloat16", "enable_model_cpu_offload", "reduce_default_steps", "reduce_default_resolution", "clear_cuda_cache"] }} ``` Do not decide final run status. Do not write blockers. Do not patch code. """ (workspace / "MEMORY_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / "MEMORY_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") estimate_pi_context_budget(run_dir, workspace, phase="memory_diagnosis", task_type="cuda_oom_memory_diagnosis", direct_prompt=goal, referenced_files=["MEMORY_DIAGNOSIS_PACKET.json", "app.py", "requirements.txt"], max_context_chars=28000, omitted_sections=["full_build_logs", "full_runtime_logs", "REPAIR_BRIEF.md"], events_path=events_path, artifact_dir=repair_dir) append_event(events_path, "pi_memory_diagnosis", "started", "Running Pi bounded memory diagnosis for CUDA OOM", {"model": pi_model}) write_agent_trace_record(run_dir, phase="memory_diagnosis", event="command_started", status="started", message="Pi memory diagnosis started", data={"model": pi_model}, artifacts=["repair/MEMORY_DIAGNOSIS_PACKET.json", "repair/MEMORY_DIAGNOSIS_GOAL.md"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=900) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_memory_diagnosis_output.txt").write_text(out, encoding="utf-8") write_pi_call_fingerprint(run_dir, phase="memory_diagnosis", requested_model=pi_model, prompt_text=goal, output_text=out, task_packet_path=workspace / "MEMORY_DIAGNOSIS_PACKET.json", events_path=events_path, artifact_dir=repair_dir) raw = load_json_if_exists(workspace / "MEMORY_DIAGNOSIS.json") if (workspace / "MEMORY_DIAGNOSIS.json").exists() else {} if not raw: raw = extract_json_object(out) diagnosis = normalize_memory_diagnosis(raw, classification) write_json(repair_dir / "MEMORY_DIAGNOSIS.json", diagnosis) write_json(workspace / "MEMORY_DIAGNOSIS.json", diagnosis) append_agent_trace_artifact(run_dir, phase="memory_diagnosis", event="memory_diagnosis_output", artifact="logs/pi_memory_diagnosis_output.txt", text=out, status="success" if code == 0 else "warning", data={"returncode": code, "diagnosis": diagnosis}) append_event(events_path, "pi_memory_diagnosis", "success" if code == 0 else "warning", f"Pi memory diagnosis classified OOM as {diagnosis.get('classification')}", {"diagnosis": diagnosis}) return diagnosis def decision_from_memory_diagnosis(memory_diagnosis: dict, classification: dict, budgets: dict | None = None) -> dict: budgets = budgets or {} cls = (memory_diagnosis or {}).get("classification") or "unknown" rec = (memory_diagnosis or {}).get("recommended_action") or "request_manual_hardware" action = rec if rec in {"patch_code", "request_manual_hardware", "inspect_more_logs", "declare_technical_blocker"} else "request_manual_hardware" if action == "patch_code" and budgets.get("patch_code", 0) <= 0: action = "declare_technical_blocker" if action == "patch_code" and cls not in {"fixable_memory_optimization", "inference_parameter_too_heavy", "memory_leak_or_cache"}: action = "request_manual_hardware" return { "action": action, "confidence": (memory_diagnosis or {}).get("confidence") or "low", "reason": f"Pi memory diagnosis classified CUDA OOM as {cls}.", "evidence": (memory_diagnosis or {}).get("evidence") or [], "patch_allowed": action == "patch_code", "requires_manual_hardware": action == "request_manual_hardware", "source": "pi_memory_diagnosis", "memory_diagnosis": memory_diagnosis, "classification": classification, } def diagnose_failure_with_pi(workspace: Path, run_dir: Path, events_path: Path, pi_model: str, target_space_id: str, model_id: str, failure_reason: str, implementation_mode: str, expected_output_type: str, iteration: int, budgets: dict) -> dict: logs_dir = run_dir / "logs" build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else "" runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) classification = classify_repair_failure(failure_reason, build_log, runtime_log) dependency_issue = classification.get("dependency_issue") or {} if dependency_issue: write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, failure_reason) write_actionable_error_artifacts(run_dir, failure_reason, build_log, runtime_log, classification) if classification.get("category") == "cuda_oom" and classification.get("logs_quality") == "useful": memory_diagnosis = diagnose_memory_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, classification) raw_memory_decision = decision_from_memory_diagnosis(memory_diagnosis, classification, budgets) decision = normalize_repair_decision(raw_memory_decision, classification, budgets) write_json(repair_dir / "REPAIR_DECISION.json", decision) (workspace / "REPAIR_DECISION.json").write_text(json.dumps(decision, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") write_agent_trace_record(run_dir, phase="diagnosis", event="repair_decision", status="success", message=f"Pi memory diagnosis selected action: {decision.get('action')}", data={"decision": decision}, artifacts=["repair/MEMORY_DIAGNOSIS.json", "repair/REPAIR_DECISION.json"]) append_event(events_path, "repair_decision", "success", f"Pi memory diagnosis selected action: {decision.get('action')}", {"decision": decision}) return decision append_event(events_path, "failure_diagnosis", "started", "Preparing Pi blockage diagnosis from Space status and logs", {"iteration": iteration, **classification}) brief = write_incident_brief( workspace, run_dir, target_space_id=target_space_id, model_id=model_id, pi_model=pi_model, failure_reason=failure_reason, build_log=build_log, runtime_log=runtime_log, classification=classification, implementation_mode=implementation_mode, expected_output_type=expected_output_type, iteration=iteration, budgets=budgets, ) goal = f"""You are Pi in BLOCKAGE DIAGNOSIS MODE for Agentic Space Factory. First read `INCIDENT_BRIEF.md`. {hf_spaces_guidance_repair_section()} {pi_reference_dependency_repair_section()} {pi_tooling_context_note()} You are not allowed to edit code in this diagnosis step. Your task is to decide the next action for the Factory. Use the gist method: read logs first, identify the first actionable error, use the cheapest useful iteration rung, and require a live Gradio/API validation before success. When the first error is dependency-related, classify it with the ASF dependency contract in mind: Gradio/HF runtime packages are platform-owned, model-specific pins need evidence, torch/torchaudio are managed by Spaces, and non-pip-installable model code should be vendored. Write `REPAIR_DECISION.json` exactly as requested in INCIDENT_BRIEF.md. Do not patch files during this step. """ (workspace / "PI_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / "PI_DIAGNOSIS_GOAL.md").write_text(goal, encoding="utf-8") append_event(events_path, "pi_diagnosis", "started", "Running Pi diagnosis decision before any repair action", {"model": pi_model, "iteration": iteration}) estimate_pi_context_budget(run_dir, workspace, phase="diagnosis", task_type="blockage_decision", direct_prompt=goal, referenced_files=["INCIDENT_BRIEF.md", "DEPENDENCY_ERROR_BRIEF.md"], max_context_chars=42000, omitted_sections=["full_logs", "REPAIR_BRIEF.md", "workspace_source_dump"], events_path=events_path, artifact_dir=repair_dir) write_agent_trace_record(run_dir, phase="diagnosis", event="command_started", status="started", message="Pi blockage diagnosis started", data={"model": pi_model, "iteration": iteration}, artifacts=["repair/INCIDENT_BRIEF.md", "repair/LOG_EVIDENCE_PACKET.json", "repair/PI_DIAGNOSIS_GOAL.md", "repair/PI_PROMPT_BUDGET_diagnosis.json"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=900) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_diagnosis_output.txt").write_text(out, encoding="utf-8") write_pi_call_fingerprint(run_dir, phase="diagnosis", requested_model=pi_model, prompt_text=goal, output_text=out, events_path=events_path, artifact_dir=repair_dir) append_agent_trace_artifact(run_dir, phase="diagnosis", event="diagnosis_output", artifact="logs/pi_diagnosis_output.txt", text=out, status="success" if code == 0 else "failed", data={"returncode": code}) raw_decision = {} decision_path = workspace / "REPAIR_DECISION.json" if decision_path.exists(): raw_decision = load_json_if_exists(decision_path) if not raw_decision: raw_decision = extract_json_object(out) if code != 0: append_event(events_path, "pi_diagnosis", "warning", "Pi diagnosis returned non-zero; using conservative factory fallback", {"returncode": code, "output_tail": out[-3000:]}) if not raw_decision: raw_decision = fallback_decision_from_classifier(classification, budgets) decision = normalize_repair_decision(raw_decision, classification, budgets) write_frozen_repair_decision(workspace, run_dir, decision, classification) write_json(repair_dir / "REPAIR_DECISION.json", decision) (workspace / "REPAIR_DECISION.json").write_text(json.dumps(decision, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") write_agent_trace_record(run_dir, phase="diagnosis", event="repair_decision", status="success", message=f"Pi diagnosis selected action: {decision.get('action')}", data={"decision": decision}, artifacts=["repair/REPAIR_DECISION.json", "repair/FROZEN_REPAIR_DECISION.json"]) append_event(events_path, "repair_decision", "success", f"Pi diagnosis selected action: {decision.get('action')}", {"decision": decision}) return decision def write_blockage_artifact(workspace: Path, run_dir: Path, events_path: Path, decision: dict, failure_reason: str, *, status: str = "technical_blocker"): blocker = { "full_inference_implemented": False, "source": "agentic_space_factory_blockage_protocol", "status": status, "decision": decision, "failure_reason": failure_reason[:4000], "blockers": [ { "type": decision.get("action") or status, "claim": decision.get("reason") or "Automated recovery could not safely continue.", "evidence": decision.get("evidence") or [], "severity": "blocking", } ], "suggested_next_step": "Inspect repair/INCIDENT_BRIEF.md and repair/REPAIR_DECISION.json, then retry manually or adjust hardware/settings if appropriate.", } (workspace / "TECHNICAL_BLOCKERS.json").write_text(json.dumps(blocker, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") gen_dir = run_dir / "generated" gen_dir.mkdir(parents=True, exist_ok=True) (gen_dir / "TECHNICAL_BLOCKERS.json").write_text(json.dumps(blocker, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") write_json(run_dir / "repair" / "BLOCKAGE.json", blocker) append_event(events_path, "technical_blocker", "failed", "Automated recovery stopped with an auditable blocker", {"decision": decision}) def write_repair_brief(workspace: Path, run_dir: Path, *, target_space_id: str, model_id: str, pi_model: str, failure_reason: str, build_log: str, runtime_log: str, classification: dict, implementation_mode: str, expected_output_type: str, decision: dict | None = None) -> str: repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) inventory = workspace_file_inventory(workspace) decision = decision or {} brief = f"""# Agentic Space Factory repair brief ## Scope This is a structured patch pass for an existing generated Hugging Face Space. A separate Pi diagnosis already decided that `patch_code` is justified. Do not rebuild from scratch unless the current architecture is impossible to boot. ## Diagnosis decision ```json {json.dumps(decision, indent=2, ensure_ascii=False)} ``` ## Original target - Model ID: `{model_id}` - Target Space: `{target_space_id}` - Pi model: `{pi_model}` - Implementation mode: `{implementation_mode}` - Expected output type: `{expected_output_type}` ## Failure classification - Category: `{classification.get('category', 'unknown_runtime_error')}` - Logs quality: `{classification.get('logs_quality', 'partial')}` - Failure phase: `{classification.get('failure_phase', 'unknown')}` - Recommended strategy: {classification.get('recommendation', '')} ## HF tooling/token context ```json {json.dumps(load_json_if_exists(run_dir / "token_context.json"), indent=2, ensure_ascii=False)} ``` ## Observed failure ```text {failure_reason[:5000]} ``` ## Actionable error excerpt ```text {extract_actionable_error_excerpt(failure_reason + chr(10) + build_log + chr(10) + runtime_log, max_lines=120)} ``` Full logs remain archived under `logs/space_logs_build.txt`, `logs/space_logs_runtime.txt`, and `logs/space_logs_run.txt`. Use them only if the focused excerpt is insufficient. ## Workspace files ```text {chr(10).join(inventory)} ``` ## Repair constraints - Patch the current workspace; do not change the product goal. - Make the smallest patch that fixes the classified failure. - Preserve real inference for Strict inference mode. - Do not replace inference with static placeholders, fake images, canned text, or swallowed exceptions. - Do not change the model ID unless you write a technical blocker explaining why. - Preserve or restore a cheap `health` endpoint. - Preserve the expected Gradio API endpoint when possible. - Keep README metadata valid and short_description <= 60 chars. - Platform dependency policy: keep Gradio / Gradio Client / Hugging Face Hub / Spaces modern and worker-owned. Do not downgrade Gradio or add stale exact pins. If the failure is a pip resolver conflict, first attempt requirements-only repair; do not edit app.py unless REPAIR_TASK_PACKET.json explicitly allows it. - For model-specific dependencies, follow the gist method: patch the first concrete resolver error minimally, preserve justified model-card pins, and do not pin torch/torchaudio unless unavoidable and explicitly justified. If torchvision is needed, prefer leaving it unpinned so it resolves against the managed torch runtime. - If model code is not pip-installable, vendor the necessary code into the Space repo instead of relying on local paths, editable installs, or clone-time side effects in requirements.txt. - Do not prove dependency fixes with local venv installs or broad pip dry-runs; Space build logs are authoritative. Keep local checks cheap, such as `python -m py_compile app.py`. - Native kernel policy: for flash-attn/custom attention/xformers/Triton/fused CUDA errors, do not blindly add source-built packages. First consider PyTorch SDPA, compatible prebuilt wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors. If no compatible fallback exists, declare a blocker rather than forcing a fragile build. ## Required repair artifacts Before modifying files, write `REPAIR_PLAN.md` with: - root cause - strategy - files to change - risk level - whether hardware/manual action is required After modifying files, write `REPAIR_SUMMARY.md` with: - files changed - why the patch is minimal - how it preserves real inference - how to validate it """ (repair_dir / "REPAIR_BRIEF.md").write_text(brief, encoding="utf-8") (workspace / "REPAIR_BRIEF.md").write_text(brief, encoding="utf-8") write_json(repair_dir / "classification.json", classification) return brief def extract_gradio_unexpected_kwarg(text: str) -> dict: """Extract `gr.(unexpected_kwarg=...)` evidence from Gradio startup errors.""" raw = text or "" patterns = [ r"(?P[A-Za-z_][A-Za-z0-9_]*)\.__init__\(\) got an unexpected keyword argument [\"'](?P[A-Za-z_][A-Za-z0-9_]*)[\"']", r"unexpected keyword argument [\"'](?P[A-Za-z_][A-Za-z0-9_]*)[\"']", ] for pattern in patterns: m = re.search(pattern, raw, flags=re.IGNORECASE) if m: return { "component": m.groupdict().get("component") or "", "kwarg": m.group("kwarg"), "repair_type": "ui_component_api_repair", } return {} def infer_inference_strategy(workspace: Path) -> dict: """Infer the generated app's inference strategy from publishable files. This is intentionally evidence-based and conservative. It replaces the old one-size Diffusers/Z-Image marker check used after repairs. The Worker still rejects obvious fake outputs, but it should not reject valid non-Diffusers architectures such as vLLM local servers, OpenAI-compatible local APIs, or CPU TTS SDKs. """ app_path = workspace / "app.py" req_path = workspace / "requirements.txt" contract_path = workspace / "INFERENCE_CONTRACT.json" app_text_raw = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" req_text_raw = req_path.read_text(encoding="utf-8", errors="ignore") if req_path.exists() else "" app = app_text_raw.lower() req = req_text_raw.lower() contract = load_json_if_exists(contract_path) if contract_path.exists() else {} contract_full = contract.get("full_inference_implemented") expected_output = str(contract.get("expected_output_type") or "").lower() strategies: list[dict] = [] def add(name: str, markers: list[str], confidence: str = "medium"): present = [m for m in markers if m.lower() in app or m.lower() in req] if present: strategies.append({"name": name, "markers_present": present, "confidence": confidence}) if contract_full is False or expected_output in {"text_diagnostic", "diagnostic", "none"}: strategies.append({"name": "diagnostic_only", "markers_present": ["INFERENCE_CONTRACT.full_inference_implemented=false"], "confidence": "high"}) add("vllm_local_server", ["vllm serve", "vllm", "subprocess.popen", "/v1/audio/speech", "/v1/chat/completions", "/v1/completions", "--omni", "openai"], "high") add("openai_compatible_local_api", ["/v1/audio/speech", "/v1/chat/completions", "/v1/completions", "httpx.post", "requests.post", "openai"], "high") # .to("cuda") is a GPU marker, not a Diffusers marker. Treating it as # Diffusers caused Transformers/custom-code audio apps to be mislabeled. diffusers_core_markers = ["diffusionpipeline", "from diffusers", "zimagepipeline", "stablediffusion", "stable diffusion", "diffusers."] diffusers_present = [m for m in diffusers_core_markers if m in app or m in req] if diffusers_present: cuda_present = [m for m in [".to(\"cuda\")", ".to('cuda')"] if m in app] strategies.append({"name": "diffusers_pipeline", "markers_present": diffusers_present + cuda_present, "confidence": "high"}) add("transformers_pipeline", ["from_pretrained", "automodelfor", "automodel", "pipeline(", ".generate(", "transformers"], "medium") add("tts_sdk_cpu", ["supertonic", "tts(", "tts.synthesize", ".synthesize(", "save_audio", "gr.audio"], "high") add("onnx_runtime", ["onnxruntime", "inferencesession", ".onnx"], "medium") add("spaces_gpu", ["@spaces.gpu", "spaces.gpu", ".to(\"cuda\")", ".to('cuda')"], "medium") by_name = {s["name"]: s for s in strategies} contract_strategy = str(contract.get("inference_strategy") or contract.get("strategy") or "").strip().lower() if contract_strategy in by_name and contract_strategy not in {"spaces_gpu"}: primary = by_name[contract_strategy] else: order = ["diagnostic_only", "vllm_local_server", "openai_compatible_local_api", "diffusers_pipeline", "transformers_pipeline", "tts_sdk_cpu", "onnx_runtime", "spaces_gpu"] primary = next((by_name[name] for name in order if name in by_name), None) if not primary: primary = {"name": "unknown", "markers_present": [], "confidence": "low"} return { "schema_version": "1.0", "primary": primary.get("name"), "confidence": primary.get("confidence"), "markers_present": primary.get("markers_present") or [], "strategies": strategies, "contract_full_inference_implemented": contract_full, "contract_expected_output_type": expected_output, } def strategy_has_real_inference_path(strategy: dict) -> bool: primary = strategy.get("primary") if primary == "diagnostic_only": return True return primary in { "vllm_local_server", "openai_compatible_local_api", "diffusers_pipeline", "transformers_pipeline", "tts_sdk_cpu", "onnx_runtime", "spaces_gpu", } and bool(strategy.get("markers_present")) def _as_bool_true(value) -> bool: if value is True: return True if isinstance(value, str): return value.strip().lower() in {"true", "yes", "1", "on"} return False def _workspace_text_blob(workspace: Path | None, filenames: list[str]) -> str: if not workspace: return "" chunks = [] for name in filenames: path = workspace / name if path.exists() and path.is_file(): try: chunks.append(path.read_text(encoding="utf-8", errors="ignore")) except Exception: pass return "\n".join(chunks) def _read_workspace_text(path: Path) -> str: try: return path.read_text(encoding="utf-8", errors="ignore") except Exception: return "" def _contract_string_value(workspace: Path | None, *keys: str) -> str: contract = read_inference_contract(workspace) for key in keys: value = contract.get(key) if isinstance(value, str) and value.strip(): return value.strip() return "" def _contract_full_inference_value(workspace: Path | None): contract = read_inference_contract(workspace) return contract.get("full_inference_implemented") def _hard_gpu_markers_in_text(text: str) -> list[str]: low = (text or "").lower() markers = { "@spaces.GPU": ["@spaces.gpu", "spaces.gpu("], ".to(cuda)": [".to(\"cuda\")", ".to('cuda')"], "torch.cuda": ["torch.cuda", "cuda.is_available"], "device=cuda": ["device=\"cuda\"", "device='cuda'", "device_map=\"cuda\"", "device_map='cuda'"], "vllm": ["vllm serve", "import vllm", "from vllm"], } found = [] for name, needles in markers.items(): if any(n in low for n in needles): found.append(name) return found def workspace_hard_gpu_markers(workspace: Path | None) -> list[str]: return _hard_gpu_markers_in_text(_workspace_text_blob(workspace, ["app.py", "requirements.txt"])) def app_imports_transformers(workspace: Path | None) -> bool: text = _workspace_text_blob(workspace, ["app.py"]) if workspace else "" return bool(re.search(r"(?m)^\s*(import\s+transformers\b|from\s+transformers\b)", text)) def workspace_has_dependency_evidence_for_transformers(workspace: Path | None, dependency_evidence: str = "") -> bool: evidence = "\n".join([ dependency_evidence or "", _workspace_text_blob(workspace, ["INFERENCE_CONTRACT.json", "TECHNICAL_BLOCKERS.json", "MODEL_DEPENDENCIES.json", "REFERENCE_SPACE_FINDINGS.json"]), ]).lower() return "transformers" in evidence and any(x in evidence for x in ["requires", "dependency", "from_pretrained", "automodel", "diffusers", "cosmos"]) def workspace_should_keep_transformers_policy(workspace: Path | None, dependency_evidence: str = "") -> tuple[bool, str]: strategy = infer_inference_strategy(workspace) if workspace else {"primary": "unknown"} primary = strategy.get("primary") if workspace_allows_transformers_major5(workspace, dependency_evidence): return True, "transformers5_dependency_evidence" if app_imports_transformers(workspace): return True, "app_imports_transformers" if primary in {"transformers_pipeline", "diffusers_pipeline", "vllm_local_server", "openai_compatible_local_api"}: return True, f"strategy_{primary}" if workspace_has_dependency_evidence_for_transformers(workspace, dependency_evidence): return True, "model_dependency_evidence" return False, f"not_needed_for_strategy_{primary or 'unknown'}" def verify_workspace_health_endpoint(workspace: Path, run_dir: Path | None = None, events_path: Path | None = None) -> dict: """Verify cheap Gradio health endpoint across app.py and vendored modules. v198.4 keeps the pre-upload health gate, but stops assuming that complex vendorized apps register every endpoint in app.py. The endpoint is accepted when a publishable Python file contains api_name="health" (or equivalent) and the nearby code does not show obvious GPU/model-loading work. """ contract = read_inference_contract(workspace) contract_declares = bool(str(contract.get("health_endpoint") or contract.get("health_api_name") or "").strip().lower() in {"health", "/health", "true", "yes"}) candidates = [] app_py_found = False patterns = [r"api_name\s*=\s*[\"']health[\"']", r"/health\b", r"def\s+health\s*\("] heavy_needles = ["@spaces.gpu", "spaces.gpu", ".to(\"cuda\")", ".to('cuda')", "torch.cuda", "from_pretrained", "load_model", "generate(", "synthesize(", "pipeline("] for py in sorted(workspace.rglob("*.py")): try: rel = py.relative_to(workspace).as_posix() except Exception: rel = py.name parts = set(py.parts) if any(part in {".git", "__pycache__", ".venv", "venv", "node_modules"} for part in parts): continue text = _read_workspace_text(py) if not any(re.search(pat, text, flags=re.IGNORECASE) for pat in patterns): continue low = text.lower() health_pos = low.find('api_name="health"') if health_pos < 0: health_pos = low.find("api_name='health'") if health_pos < 0: health_pos = low.find("/health") if health_pos < 0: health_pos = low.find("def health") window = low[max(0, health_pos - 800): health_pos + 800] if health_pos >= 0 else low[:1600] heavy = [n for n in heavy_needles if n in window] candidate = {"file": rel, "heavy_markers_near_endpoint": heavy, "accepted": not heavy} candidates.append(candidate) if rel == "app.py" and candidate["accepted"]: app_py_found = True accepted = [c for c in candidates if c.get("accepted")] result = { "schema_version": "health_verification.v198_4", "contract_declares_health": contract_declares, "app_py_health_found": app_py_found, "workspace_health_found": bool(accepted), "accepted_files": [c["file"] for c in accepted], "rejected_candidates": [c for c in candidates if not c.get("accepted")], "decision": "accept" if accepted else "reject", "health_cheap_confidence": "high" if app_py_found else ("medium" if accepted else "low"), } if run_dir: write_json(run_dir / "health_verification.json", result) if events_path: append_event(events_path, "health_verification", "success" if accepted else "failed", "Verified generated health endpoint across publishable Python files", result) return result def should_stop_before_cpu_deploy_for_manual_hardware(workspace: Path, hardware_intent: dict) -> dict: contract_full = _contract_full_inference_value(workspace) requires_gpu = _contract_requires_gpu_value(workspace) gpu_markers = workspace_hard_gpu_markers(workspace) triggered = bool( isinstance(hardware_intent, dict) and hardware_intent.get("manual_hardware_required") is True and contract_full is True and requires_gpu is True and gpu_markers ) return { "schema_version": "manual_hardware_block.v198_20", "triggered": triggered, "reason": "Full GPU app requires manual hardware; create an actionable CPU diagnostic Space instead of deploying the CUDA app on cpu-basic." if triggered else "automatic_deployment_allowed", "contract_full_inference_implemented": contract_full, "contract_requires_gpu": requires_gpu, "gpu_markers": gpu_markers, "hardware_intent": hardware_intent, "recommended_action": "create_actionable_space_for_manual_hardware_selection" if triggered else "continue", "space_required_for_manual_hardware": triggered, } def prepare_manual_hardware_actionable_workspace(workspace: Path, run_dir: Path, events_path: Path, *, model_id: str, target_space_id: str, hardware_intent: dict, manual_block: dict) -> dict: """Create a CPU-safe Space app so users can change hardware in Settings.""" original_dir = run_dir / "generated_full_inference_candidate" if original_dir.exists(): shutil.rmtree(original_dir) shutil.copytree(workspace, original_dir, ignore=internal_workspace_copy_ignore) recommended = (hardware_intent or {}).get("recommended_hardware") or (hardware_intent or {}).get("preferred_hardware") or "manual GPU hardware" reason = manual_block.get("reason") or (hardware_intent or {}).get("reason") or "Manual hardware selection is required before full inference can be run safely." markers = manual_block.get("gpu_markers") or [] markers_text = "\n".join(f"- {m}" for m in markers[:20]) or "- GPU-only inference markers detected" intent_json = json.dumps(hardware_intent or {}, indent=2, ensure_ascii=False) block_json = json.dumps(manual_block or {}, indent=2, ensure_ascii=False) report_template = """# Manual hardware required This Space was created successfully, but ASF did not start the full GPU inference app on CPU hardware. **Model:** {model_id} **Recommended hardware:** {recommended} **Reason:** {reason} ## What to do 1. Open this Space's **Settings** tab on Hugging Face. 2. Change **Hardware** to the recommended GPU or a stronger compatible GPU. 3. Restart/rebuild the Space. 4. Re-run ASF validation / Space Test after the hardware change. ASF intentionally published this safe diagnostic app because Hugging Face hardware can only be changed after the Space exists. ## Detected GPU markers {markers_text} ## Hardware intent ```json {intent_json} ``` ## Manual block ```json {block_json} ``` """.format(model_id=model_id, recommended=recommended, reason=reason, markers_text=markers_text, intent_json=intent_json, block_json=block_json) app_py = "\n".join([ "import gradio as gr", "", f"MODEL_ID = {model_id!r}", f"TARGET_SPACE_ID = {target_space_id!r}", f"RECOMMENDED_HARDWARE = {str(recommended)!r}", f"REPORT = {report_template!r}", "", "def health():", " return {'status': 'ok', 'mode': 'manual_hardware_required_actionable', 'model_id': MODEL_ID, 'target_space_id': TARGET_SPACE_ID, 'recommended_hardware': RECOMMENDED_HARDWARE}", "", "def manual_hardware_report():", " return REPORT", "", "with gr.Blocks(title='Manual hardware required') as demo:", " gr.Markdown(REPORT)", " refresh = gr.Button('Refresh diagnostic report')", " report = gr.Markdown(value=REPORT)", " refresh.click(fn=manual_hardware_report, inputs=None, outputs=report, api_name='manual_hardware_report')", " demo.load(fn=health, inputs=None, outputs=None, api_name='health')", "", "if __name__ == '__main__':", " demo.launch()", "", ]) readme = """--- title: Manual hardware required for {model_id} emoji: 🛠️ colorFrom: yellow colorTo: orange sdk: gradio sdk_version: 5.34.2 app_file: app.py pinned: false --- # Manual hardware required ASF created this Space so that hardware can be changed from the Hugging Face Space settings. Full inference for `{model_id}` was not deployed on CPU because the generated app requires GPU/manual hardware. Recommended hardware: `{recommended}` Reason: {reason} ## Next steps 1. Open the Space **Settings** tab. 2. Select the recommended GPU or stronger hardware. 3. Restart/rebuild the Space. 4. Re-run ASF validation / Space Test. The original full-inference candidate is preserved in the run artifact folder `generated_full_inference_candidate/`. """.format(model_id=model_id, recommended=recommended, reason=reason) requirements = "gradio>=5.34.2\n" inference_contract = { "schema_version": "manual_hardware_actionable.v198_20", "inference_strategy": "manual_hardware_actionable_space", "requires_gpu": True, "full_inference_implemented": False, "real_inference_implemented": False, "fallback_or_diagnostic_only": True, "validation_level": "manual_hardware_actionable_app_boot", "expected_output_type": "text_diagnostic", "recommended_target_space_hardware": recommended, "manual_hardware_required": True, "target_space_id": target_space_id, "model_id": model_id, "reason": reason, } demo_contract = { "schema_version": "demo_quality_contract.v198_20", "demo_task": "manual_hardware_action", "model_card_promise": "Full inference deferred until the user selects suitable Space hardware.", "primary_user_flow": "Open Space Settings, select recommended hardware, restart, then re-run validation.", "examples_provided": True, "examples": ["Open Settings → Hardware → choose recommended GPU → restart Space"], "canonical_smoke_example": {"api_name": "/manual_hardware_report", "inputs": []}, "real_inference_required": True, "real_inference_implemented": False, "fallback_or_diagnostic_only": True, "limitations_disclosed": True, "promise_fulfillment_risk": "manual_hardware_required", } (workspace / "app.py").write_text(app_py, encoding="utf-8") (workspace / "README.md").write_text(readme, encoding="utf-8") (workspace / "requirements.txt").write_text(requirements, encoding="utf-8") write_json(workspace / "INFERENCE_CONTRACT.json", inference_contract) write_json(workspace / "DEMO_QUALITY_CONTRACT.json", demo_contract) payload = { "schema_version": "manual_hardware_actionable_space.v198_20", "space_created_required": True, "diagnostic_app_uploaded": True, "full_inference_candidate_preserved": str(original_dir.relative_to(run_dir)), "target_space_id": target_space_id, "recommended_hardware": recommended, "reason": reason, "hardware_intent": hardware_intent, "manual_hardware_block": manual_block, "status": "manual_hardware_required_actionable", } write_json(run_dir / "manual_hardware_actionable_space.json", payload) append_event(events_path, "manual_hardware_actionable_space", "success", "Prepared a CPU-safe Space app so the user can select hardware in Space Settings", payload) return payload def contract_effective_output_type(workspace: Path | None, fallback_expected_output_type: str = "any") -> dict: """Separate the source model task output from the deployed app contract output. A diagnostic-only app for a text-to-audio model should not keep reporting `audio` as the effective smoke output. In that mode the Worker validates a bootable diagnostic app, so the contract output (usually text_diagnostic) is the source of truth. """ fallback = (fallback_expected_output_type or "any").strip() or "any" contract = read_inference_contract(workspace) blocker_reason = contract_declares_no_full_inference(workspace) contract_output = str(contract.get("expected_output_type") or "").strip() use_contract = bool(blocker_reason.get("declared") and contract_output) return { "schema_version": "1.0", "model_expected_output_type": fallback, "contract_effective_output_type": contract_output or fallback, "effective_expected_output_type": contract_output if use_contract else fallback, "contract_output_applied": use_contract, "contract_declared_no_full_inference": bool(blocker_reason.get("declared")), "source": "inference_contract" if use_contract else "model_task_or_env", } def _contract_requires_gpu_value(workspace: Path | None) -> bool | None: contract = read_inference_contract(workspace) blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json") for value in [contract.get("requires_gpu"), blockers.get("requires_gpu")]: if _as_bool_true(value): return True if _as_bool_false(value): return False return None def _too_large_for_standard_single_gpu(workspace: Path | None, model_analysis: dict | None = None) -> dict: """Detect documented hardware-capacity blockers without guessing silently. This is intentionally conservative: it requires explicit blocker language mentioning unavailable/insufficient hardware, very high VRAM, GB200, or multi-GPU requirements. It prevents the Worker from optimistically burning an A10G on a Space that Pi already documented as too large. """ model_analysis = model_analysis or {} text = "\n".join([ _workspace_text_blob(workspace, ["TECHNICAL_BLOCKERS.json", "PI_SUMMARY.md", "README.md", "INFERENCE_CONTRACT.json"]), json.dumps(model_analysis, ensure_ascii=False), ]).lower() if not text.strip(): return {"detected": False, "markers": []} markers = [] checks = { "gb200": "gb200" in text, "multi_gpu": any(x in text for x in ["multi-gpu", "multi gpu", "multiple gpu", "2-gpu", "4-gpu"]), "very_high_vram": bool(re.search(r"(1[0-9]{2}|[2-9][0-9]{2})\s*(gb|gib)", text)) and "vram" in text, "a10g_insufficient": "a10g" in text and any(x in text for x in ["insufficient", "too small", "not enough", "cannot", "won't fit", "will not fit"]), "manual_hardware": any(x in text for x in ["manual hardware", "manual_hardware_required", "requires manual", "not available automatically"]), } markers = [name for name, present in checks.items() if present] # v198.20: do not treat a vague "A10G insufficient" note as a hard # manual-hardware blocker by itself. ZeroGPU is the first-class HF path for # many large demos, and quota/capacity failures for the current user should # not make Pi/Worker rewrite or pre-block a ZeroGPU-compatible app. Require # stronger evidence such as explicit manual hardware, multi-GPU, GB200, or # very high VRAM before stopping automatic deployment. hard_markers = [m for m in markers if m in {"gb200", "multi_gpu", "very_high_vram", "manual_hardware"}] return {"detected": bool(hard_markers), "markers": markers, "hard_markers": hard_markers, "a10g_insufficient_only_is_not_manual_block": markers == ["a10g_insufficient"]} def compute_hardware_intent( workspace: Path, model_analysis: dict | None, preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, try_zero_gpu_first: bool, ) -> dict: """Choose hardware from the generated app contract, not just the model task. v197.3 keeps this conservative: - diagnostic-only contracts with no GPU requirement use cpu-basic; - CPU-only TTS/ONNX apps use cpu-basic; - vLLM/Diffusers/GPU apps keep the existing GPU sequence; - explicit large-hardware blockers avoid A10G optimism and deploy a CPU diagnostic/manual-action Space instead. """ preferred = normalize_auto_space_hardware(preferred_hardware, DEFAULT_PREFERRED_SPACE_HARDWARE) fallback = normalize_auto_space_hardware(fallback_hardware, DEFAULT_FALLBACK_SPACE_HARDWARE) strategy = infer_inference_strategy(workspace) contract_skip = contract_declares_no_full_inference(workspace) requires_gpu = _contract_requires_gpu_value(workspace) recommended_hw = _contract_string_value(workspace, "recommended_target_space_hardware", "recommended_hardware", "minimum_hardware").lower() contract_strategy = _contract_string_value(workspace, "inference_strategy", "strategy").lower() too_large = _too_large_for_standard_single_gpu(workspace, model_analysis) publishable_text = _workspace_text_blob(workspace, ["app.py", "requirements.txt"]).lower() hard_gpu_markers = workspace_hard_gpu_markers(workspace) has_gpu_markers = bool(hard_gpu_markers) has_torch_or_diffusers = any(x in publishable_text for x in ["import torch", "from torch", "diffusers", "diffusionpipeline", "zimagepipeline"]) has_vllm = "vllm" in publishable_text or strategy.get("primary") == "vllm_local_server" or contract_strategy == "vllm_local_server" cpu_contract = (requires_gpu is False and recommended_hw == "cpu-basic") or contract_strategy in {"tts_sdk_cpu", "onnx_runtime"} cpu_tts = (strategy.get("primary") in {"tts_sdk_cpu", "onnx_runtime"} or contract_strategy in {"tts_sdk_cpu", "onnx_runtime"}) and not has_gpu_markers and not has_vllm and (requires_gpu is not True) intent = { "schema_version": "1.0", "intent": "gpu_best_effort", "preferred_hardware": preferred, "fallback_hardware": fallback, "allow_fixed_gpu_fallback": bool(allow_fixed_gpu_fallback), "try_zero_gpu_first": bool(try_zero_gpu_first), "manual_hardware_required": False, "reason": "Default full-inference hardware strategy preserved.", "confidence": "medium", "strategy": strategy, "signals": { "contract_declared_no_full_inference": bool(contract_skip.get("declared")), "contract_requires_gpu": requires_gpu, "too_large_for_standard_single_gpu": too_large, "has_gpu_markers": has_gpu_markers, "hard_gpu_markers": hard_gpu_markers, "has_torch_or_diffusers": has_torch_or_diffusers, "has_vllm": has_vllm, "contract_strategy": contract_strategy, "recommended_target_space_hardware": recommended_hw, "cpu_contract": cpu_contract, "cpu_tts_or_onnx": cpu_tts, }, } if contract_skip.get("declared") and requires_gpu is not True: intent.update({ "intent": "diagnostic_cpu_basic", "preferred_hardware": "cpu-basic", "fallback_hardware": "cpu-basic", "allow_fixed_gpu_fallback": False, "try_zero_gpu_first": False, "reason": "Diagnostic-only contract declares no full inference and no GPU requirement; deploy the bootable diagnostic Space on CPU.", "confidence": "high", }) elif cpu_tts or (cpu_contract and not has_gpu_markers and not has_vllm and not has_torch_or_diffusers): intent.update({ "intent": "cpu_basic", "preferred_hardware": "cpu-basic", "fallback_hardware": "cpu-basic", "allow_fixed_gpu_fallback": False, "try_zero_gpu_first": False, "reason": "CPU-only contract/strategy (TTS/ONNX/lazy SDK) has no hard GPU markers; prefer cpu-basic over broad generated-text heuristics.", "confidence": "high", "decision_source": "cpu_only_contract_override" if cpu_contract else "cpu_tts_or_onnx_strategy", }) elif too_large.get("detected"): intent.update({ "intent": "manual_hardware_required", "preferred_hardware": "cpu-basic", "fallback_hardware": "cpu-basic", "allow_fixed_gpu_fallback": False, "try_zero_gpu_first": False, "manual_hardware_required": True, "reason": "Generated artifacts document hardware requirements beyond the automatic single-GPU fallback; require manual hardware instead of burning A10G or deploying a CUDA app on CPU.", "confidence": "high", }) elif has_vllm or has_gpu_markers or has_torch_or_diffusers or requires_gpu is True: intent.update({ "intent": "gpu_best_effort", "reason": "Generated app contains GPU/vLLM/Diffusers/Torch inference markers; preserve ZeroGPU/fixed-GPU fallback strategy.", "confidence": "high" if (has_vllm or has_gpu_markers) else "medium", }) return intent def apply_hardware_intent( intent: dict, preferred_hardware: str, fallback_hardware: str, allow_fixed_gpu_fallback: bool, try_zero_gpu_first: bool, ) -> tuple[str, str, bool, bool]: if not isinstance(intent, dict): return preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first return ( intent.get("preferred_hardware") or preferred_hardware, intent.get("fallback_hardware") or fallback_hardware, bool(intent.get("allow_fixed_gpu_fallback")) if "allow_fixed_gpu_fallback" in intent else allow_fixed_gpu_fallback, bool(intent.get("try_zero_gpu_first")) if "try_zero_gpu_first" in intent else try_zero_gpu_first, ) def sanity_check_repair_workspace(workspace: Path, implementation_mode: str, classification: dict | None = None, before_dir: Path | None = None, failure_context: str = "") -> tuple[bool, str]: app_path = workspace / "app.py" req_path = workspace / "requirements.txt" readme_path = workspace / "README.md" for required in [app_path, req_path, readme_path]: if not required.exists() or not required.read_text(encoding="utf-8", errors="ignore").strip(): return False, f"Repair removed or emptied required file: {required.name}" if implementation_mode == "full-inference-gated": app_text = app_path.read_text(encoding="utf-8", errors="ignore") app_low = app_text.lower() # Only block obvious fake-return implementations. Do not fail a valid # targeted runtime patch just because a comment/README still contains # words like "placeholder". The previous broad string scan blocked the # exact kind of repair Pi should be allowed to do. hard_fake_patterns = [ r"return\s+[\"'](?:placeholder|dummy image|fake inference|not implemented|canned response)", r"def\s+placeholder\s*\(", r"gr\.button\([^\n]*placeholder", ] if any(re.search(pattern, app_low) for pattern in hard_fake_patterns): return False, "Strict inference repair appears to introduce executable placeholder/fake inference markers." # Legacy anchor: real_inference_markers used to be a Diffusers-only list. # v197.2 keeps the phrase for old regression tests but replaces the # behavior with strategy-aware inference detection. real_inference_markers = [] strategy_before = infer_inference_strategy(before_dir) if before_dir and before_dir.exists() else {} strategy_after = infer_inference_strategy(workspace) ui_kwarg = extract_gradio_unexpected_kwarg(failure_context) if ui_kwarg: kwarg = ui_kwarg.get("kwarg") or "" if kwarg and re.search(rf"\b{re.escape(kwarg)}\s*=", app_text): return False, f"UI component repair did not remove unsupported Gradio kwarg: {kwarg}" before_primary = strategy_before.get("primary") after_primary = strategy_after.get("primary") if before_primary and before_primary != "unknown" and after_primary == "unknown": return False, "UI-only repair removed the previously detected inference strategy." return True, f"UI component repair sanity passed; preserved inference strategy {after_primary or before_primary or 'unknown'}." if not strategy_has_real_inference_path(strategy_after): return False, f"Strict inference repair no longer shows a real model inference path for strategy {strategy_after.get('primary')}." if strategy_before.get("primary") and strategy_before.get("primary") not in {"unknown", "diagnostic_only"}: if strategy_after.get("primary") == "unknown": return False, "Strict inference repair removed the previously detected inference strategy." return True, "Repair workspace sanity checks passed." def _compact_error_text(text: str, limit: int = 2400) -> str: return re.sub(r"\s+", " ", (text or "").strip())[:limit] def compute_failure_signature(failure_reason: str = "", build_log: str = "", runtime_log: str = "", generation_smoke: dict | None = None, classification: dict | None = None) -> str: """Return a stable, small signature used to avoid repeating failed repairs. This is intentionally conservative: keep the actionable exception/package/CLI, strip timestamps/noise, and prefer structured smoke/classification evidence. """ classification = classification or {} if isinstance(generation_smoke, dict): failure_class = generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "" if failure_class: missing = generation_smoke.get("missing_executable") or generation_smoke.get("missing_module") or "" return f"{failure_class}:{missing}" if missing else str(failure_class) err = generation_smoke.get("error") or generation_smoke.get("message") or "" else: err = "" combined = "\n".join(str(x or "") for x in [err, failure_reason, build_log, runtime_log]) low = combined.lower() patterns = [ (r"peft backend is required for this method", "missing_python_dependency_peft"), (r"modulenotfounderror:\s*no module named ['\"]([^'\"]+)['\"]", "missing_python_dependency"), (r"no such file or directory:\s*['\"]([^'\"]+)['\"]", "missing_runtime_cli"), (r"value:\s*([^\n]+?)\s+is not in the list of choices", "validator_schema_choice_type_mismatch"), (r"requires\s+([a-zA-Z0-9_.-]+)\s*([<>=!~]+[^\s,;]+)", "dependency_conflict"), (r"(cuda out of memory|outofmemoryerror)", "cuda_oom"), (r"(scheduling failure|not enough hardware capacity|unable to schedule)", "infra_transient_scheduling"), (r"(model not initialized)", "model_not_initialized"), (r"(typeerror|importerror|syntaxerror|runtimeerror|attributeerror):\s*([^\n]+)", "app_runtime_error"), ] for pattern, prefix in patterns: m = re.search(pattern, combined, re.IGNORECASE) if m: detail = m.group(1) if m.groups() else m.group(0) if prefix == "missing_python_dependency_peft": return "missing_python_dependency:peft" return f"{prefix}:{_compact_error_text(detail, 220)}" cat = classification.get("category") or classification.get("failure_class") or classification.get("failure_type") or "unknown" return f"{cat}:{_compact_error_text(combined, 280)}" def read_latest_space_log_text(run_dir: Path) -> tuple[str, str, str]: """Read the most recently collected Space run/build/actionable logs. Post-repair validation failures must be diagnosed from refreshed HF Space logs, not from the pre-repair build_log/runtime_log variables. """ def _read(rel: str, limit: int = 12000) -> str: path = run_dir / rel try: if path.exists(): return path.read_text(encoding="utf-8", errors="ignore")[-limit:] except Exception: return "" return "" runtime = _read("logs/space_logs_run.txt") or _read("logs/space_logs_runtime.txt") build = _read("logs/space_logs_build.txt") brief = _read("repair/actionable_error_brief.md", 6000) return build, runtime, brief def compute_latest_failure_signature(run_dir: Path, failure_reason: str = "", classification: dict | None = None) -> str: latest_build, latest_runtime, latest_brief = read_latest_space_log_text(run_dir) return compute_failure_signature( "\n".join(x for x in [failure_reason, latest_brief] if x), latest_build, latest_runtime, classification=classification or {}, ) def apply_deterministic_diffusers_lora_dependency_policy(workspace: Path, run_dir: Path, events_path: Path) -> dict: """Ensure Diffusers LoRA apps include PEFT before build/runtime. Diffusers `load_lora_weights()` raises `ValueError: PEFT backend is required for this method` when PEFT is absent. This is deterministic and should not require Pi to discover after one failed rebuild. """ app_path = workspace / "app.py" req_path = workspace / "requirements.txt" app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" deps_text = "" deps_path = workspace / "MODEL_DEPENDENCIES.json" if deps_path.exists(): deps_text = deps_path.read_text(encoding="utf-8", errors="ignore") combined = (app_text + "\n" + deps_text).lower() needs_peft = any(x in combined for x in ["load_lora_weights", "peft_type", "lora_adapter", "diffusers_lora", "adapter_config.json"]) result = { "schema_version": "diffusers_lora_dependency_policy.v198_21", "needs_peft": bool(needs_peft), "peft_present_before": False, "peft_added": False, } if not needs_peft: write_json(run_dir / "diffusers_lora_dependency_policy.json", result) return result if not req_path.exists(): req_path.write_text("peft\n", encoding="utf-8") result.update({"peft_added": True}) else: raw = req_path.read_text(encoding="utf-8", errors="ignore") lines = raw.splitlines() has_peft = requirements_has_package(lines, "peft") result["peft_present_before"] = bool(has_peft) if not has_peft: req_path.write_text(raw.rstrip() + "\npeft\n", encoding="utf-8") result["peft_added"] = True write_json(run_dir / "diffusers_lora_dependency_policy.json", result) if result["peft_added"]: append_event(events_path, "requirements_sanitize", "success", "Added peft for Diffusers LoRA load_lora_weights support", result) return result def load_repair_attempts(run_dir: Path) -> dict: path = run_dir / "repair" / "repair_attempts.json" try: if path.exists(): payload = json.loads(path.read_text(encoding="utf-8")) if isinstance(payload, dict): payload.setdefault("attempts", []) payload.setdefault("max_pi_repair_attempts", MAX_PI_REPAIR_ATTEMPTS) return payload except Exception: pass return { "schema_version": "repair_attempts.v198_21", "max_pi_repair_attempts": MAX_PI_REPAIR_ATTEMPTS, "attempts_used": 0, "remaining_attempts": MAX_PI_REPAIR_ATTEMPTS, "attempts": [], "repeated_failure_signatures": [], "repair_budget_exhausted": False, } def write_repair_attempts(run_dir: Path, payload: dict) -> dict: payload = dict(payload or {}) attempts = payload.get("attempts") if isinstance(payload.get("attempts"), list) else [] payload["schema_version"] = "repair_attempts.v198_21" payload["max_pi_repair_attempts"] = int(payload.get("max_pi_repair_attempts") or MAX_PI_REPAIR_ATTEMPTS) payload["attempts_used"] = len([a for a in attempts if a.get("consumes_pi_budget", True)]) payload["remaining_attempts"] = max(0, payload["max_pi_repair_attempts"] - payload["attempts_used"]) payload["repair_budget_exhausted"] = payload["remaining_attempts"] <= 0 write_json(run_dir / "repair" / "repair_attempts.json", payload) write_json(run_dir / "repair_budget.json", { "schema_version": "repair_budget.v198_21", "max_pi_repair_attempts": payload["max_pi_repair_attempts"], "pi_repair_attempts_used": payload["attempts_used"], "remaining_pi_repair_attempts": payload["remaining_attempts"], "validator_self_repairs_do_not_consume_pi_budget": True, "runtime_recovery_does_not_consume_pi_budget": True, "repair_budget_exhausted": payload["repair_budget_exhausted"], "last_failure_signature": (attempts[-1].get("new_failure_signature") or attempts[-1].get("failure_signature")) if attempts else "", }) return payload def render_repair_history_md(payload: dict) -> str: attempts = payload.get("attempts") if isinstance(payload.get("attempts"), list) else [] lines = ["# Repair history", "", f"Max Pi repair attempts: {payload.get('max_pi_repair_attempts', MAX_PI_REPAIR_ATTEMPTS)}", f"Attempts used: {len(attempts)}", ""] if not attempts: lines.append("No previous Pi repair attempts have been made.") for attempt in attempts: n = attempt.get("attempt") or "?" lines += [ f"## Attempt {n}", f"Trigger: {attempt.get('trigger') or ''}", f"Failure class: {attempt.get('failure_class') or ''}", f"Failure signature: {attempt.get('failure_signature') or ''}", f"Repair decision: {attempt.get('repair_decision') or ''}", f"Files changed: {', '.join(attempt.get('files_changed') or [])}", f"Diff applied: {attempt.get('diff_applied')}", f"Post-repair result: {attempt.get('post_repair_result') or ''}", f"New failure signature: {attempt.get('new_failure_signature') or ''}", "", ] return "\n".join(lines).rstrip() + "\n" def write_repair_history(run_dir: Path, payload: dict) -> None: write_repair_attempts(run_dir, payload) (run_dir / "repair").mkdir(parents=True, exist_ok=True) (run_dir / "repair" / "REPAIR_HISTORY.md").write_text(render_repair_history_md(payload), encoding="utf-8") def next_pi_repair_attempt(run_dir: Path) -> tuple[int, dict]: payload = load_repair_attempts(run_dir) attempts_used = len([a for a in payload.get("attempts", []) if a.get("consumes_pi_budget", True)]) return attempts_used + 1, payload def same_failure_signature_repeated(run_dir: Path, signature: str, *, threshold: int = 2) -> bool: if not signature: return False payload = load_repair_attempts(run_dir) signatures = [] for attempt in payload.get("attempts", []): for key in ("failure_signature", "new_failure_signature"): value = attempt.get(key) if value: signatures.append(str(value)) return signatures.count(str(signature)) >= threshold def append_repair_attempt_record(run_dir: Path, *, attempt: int, trigger: str, failure_class: str, failure_signature: str, repair_decision: str = "", files_changed: list[str] | None = None, diff_applied: bool | None = None, post_repair_result: str = "", new_failure_signature: str = "", consumes_pi_budget: bool = True, notes: str = "") -> dict: payload = load_repair_attempts(run_dir) attempts = payload.setdefault("attempts", []) existing_index = None for idx, item in enumerate(attempts): if int(item.get("attempt") or -1) == int(attempt): existing_index = idx break record = { "attempt": int(attempt), "trigger": trigger, "failure_class": failure_class, "failure_signature": failure_signature, "repair_decision": repair_decision, "files_changed": files_changed or [], "diff_applied": diff_applied, "post_repair_result": post_repair_result, "new_failure_signature": new_failure_signature, "consumes_pi_budget": bool(consumes_pi_budget), "notes": notes, "updated_at": now(), } if existing_index is None: attempts.append(record) else: merged = dict(attempts[existing_index]) merged.update({k: v for k, v in record.items() if v not in (None, "", [])}) attempts[existing_index] = merged write_repair_history(run_dir, payload) return payload def repair_attempt_context_section(run_dir: Path, attempt: int, failure_signature: str, classification: dict, decision: dict | None) -> str: payload = load_repair_attempts(run_dir) max_attempts = int(payload.get("max_pi_repair_attempts") or MAX_PI_REPAIR_ATTEMPTS) remaining_after_this = max(0, max_attempts - attempt) history = render_repair_history_md(payload) repeated_note = "" if same_failure_signature_repeated(run_dir, failure_signature, threshold=1): repeated_note = "\nWARNING: A similar failure signature already appeared earlier. Do not repeat the previous patch. Choose a different minimal fix or declare a blocker.\n" return f""" Repair attempt context: - This is Pi repair attempt {attempt} of {max_attempts}. - Remaining Pi repair attempts after this one: {remaining_after_this}. - Current failure signature: {failure_signature or 'unknown'}. - Current failure class: {classification.get('category') or classification.get('failure_class') or 'unknown'}. - Current repair decision/action: {(decision or {}).get('action') or (decision or {}).get('repair_decision') or 'patch_code'}. {repeated_note} Previous repair history: {history} Rules for this attempt: - Do not repeat or undo previous patches unless the history shows they were wrong. - Preserve the current inference strategy and demo promise. - Make the smallest publishable file change that addresses the current failure signature. - If the same failure would require a broad rewrite or unsupported hardware/manual access, declare a blocker instead of patching blindly. """.strip() def repair_workspace_with_pi(workspace: Path, run_dir: Path, events_path: Path, pi_model: str, target_space_id: str, model_id: str, failure_reason: str, implementation_mode: str = "full-inference-gated", expected_output_type: str = "any", decision: dict | None = None, repair_attempt: int | None = None, repair_trigger: str = "live_validation_failure"): """Structured contextual patch pass, bounded by the multi-attempt Pi repair budget.""" logs_dir = run_dir / "logs" build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else "" runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" repair_dir = run_dir / "repair" before_dir = repair_dir / "before" after_dir = repair_dir / "after" if before_dir.exists(): shutil.rmtree(before_dir) repair_dir.mkdir(parents=True, exist_ok=True) shutil.copytree(workspace, before_dir, ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__", "*.pyc")) classification = (decision or {}).get("classification") or classify_repair_failure(failure_reason, build_log, runtime_log) failure_signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification) if failure_owner_from_classification(classification) == "factory_validation_client": append_event(events_path, "repair_policy", "failed", "Refusing Pi repair because failure belongs to the ASF validation client", {"failure_owner": "factory_validation_client", "category": classification.get("category"), "failure_signature": failure_signature}) return False if repair_attempt is None: repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) else: _repair_payload = load_repair_attempts(run_dir) if repair_attempt > int(_repair_payload.get("max_pi_repair_attempts") or MAX_PI_REPAIR_ATTEMPTS): append_event(events_path, "repair_budget", "failed", "Pi repair budget exhausted before launching another repair attempt", {"attempt": repair_attempt, "max_pi_repair_attempts": _repair_payload.get("max_pi_repair_attempts")}) write_repair_history(run_dir, _repair_payload) return False if same_failure_signature_repeated(run_dir, failure_signature, threshold=2): append_event(events_path, "repair_budget", "failed", "Repeated failure signature reached the safe repair limit; refusing another identical patch", {"failure_signature": failure_signature, "attempt": repair_attempt}) write_repair_history(run_dir, _repair_payload) return False append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or classification.get("failure_class") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", post_repair_result="started") append_event(events_path, "repair_diagnosis", "success", "Patch repair is allowed by Pi diagnosis", {"category": classification.get("category"), "decision": decision or {}, "attempt": repair_attempt, "failure_signature": failure_signature}) write_actionable_error_artifacts(run_dir, failure_reason, build_log, runtime_log, classification) frozen_decision = write_frozen_repair_decision(workspace, run_dir, decision or {"action": "patch_code", "classification": classification}, classification) log_packet_path = run_dir / "repair" / "LOG_EVIDENCE_PACKET.json" if log_packet_path.exists(): shutil.copy2(log_packet_path, workspace / "LOG_EVIDENCE_PACKET.json") repair_task_packet = write_repair_task_packet(workspace, run_dir, classification, failure_reason, build_log, runtime_log, decision) brief = write_repair_brief( workspace, run_dir, target_space_id=target_space_id, model_id=model_id, pi_model=pi_model, failure_reason=failure_reason, build_log=build_log, runtime_log=runtime_log, classification=classification, implementation_mode=implementation_mode, expected_output_type=expected_output_type, decision=decision, ) append_event(events_path, "repair_task_packet", "success", "Repair task packet generated for targeted Pi patch", {"task_type": repair_task_packet.get("task_type"), "allowed_files": repair_task_packet.get("allowed_files"), "packet": "repair/REPAIR_TASK_PACKET.json"}) append_event(events_path, "repair_brief", "success", "Repair brief generated for minimal patch", {"category": classification.get("category"), "brief": "repair/REPAIR_BRIEF.md"}) goal = f"""You are Pi in STRUCTURED REPAIR MODE / STRUCTURED PATCH REPAIR MODE for Agentic Space Factory. First read `REPAIR_TASK_PACKET.json`, `FROZEN_REPAIR_DECISION.json`, and `LOG_EVIDENCE_PACKET.json`. It is the authoritative compact task, with frozen decision and log evidence packets attached. Do not reinterpret the whole run. Only read `REPAIR_BRIEF.md`, `INCIDENT_BRIEF.md`, and `DEPENDENCY_ERROR_BRIEF.md` if the packet is insufficient or the packet explicitly requests deep_repair_reanalysis. {hf_spaces_guidance_repair_section()} {pi_reference_dependency_repair_section()} {pi_tooling_context_note()} {repair_attempt_context_section(run_dir, repair_attempt, failure_signature, classification, decision)} You are continuing the same build run, not starting a separate project. This patch is allowed only because the diagnosis decision selected `patch_code`. Implement the frozen decision exactly. If you believe the frozen decision is wrong, write `PATCH_REFUSAL.json` with `action=refuse_patch` and do not edit publishable files. If `DEPENDENCY_ERROR_BRIEF.md` exists, treat it as evidence for the gist method: identify the first pip error and patch dependency files minimally. Do not modify inference code or app.py unless REPAIR_TASK_PACKET.json explicitly allows it. If safe dependency-only repair is impossible, write PATCH_REFUSAL.json. Follow the ASF/gist dependency contract during repair: Gradio/HF runtime packages are platform-owned, model-specific pins must be evidence-backed, torch/torchaudio should remain managed by Spaces, and non-pip-installable model code should be vendored instead of referenced through fragile local paths. For native kernel failures such as flash-attn/custom attention/xformers/Triton/fused CUDA ops, do not blindly add source-built native packages. Prefer PyTorch SDPA, compatible wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors when they match the required operation. Use `kernels` only when directly justified and document the backend in REPAIR_SUMMARY.md / INFERENCE_CONTRACT.json. Use the available HF token context to inspect private Hub resources when needed, but never print or persist token values. Critical method: 1. Diagnose the root cause from the brief and current files. 2. Write `REPAIR_PLAN.md` before editing files. 3. Apply the smallest patch possible. 4. Write `REPAIR_SUMMARY.md` after editing. 5. Preserve real inference. Never fake outputs just to satisfy validation. Failure category: {classification.get('category')} Recommended strategy: {classification.get('recommendation')} Hard constraints: - Do not rebuild from scratch unless you clearly justify it in REPAIR_PLAN.md. - Do not change MODEL_ID `{model_id}` unless you write a technical blocker. - Do not remove the expected Gradio/API contract. - For Strict inference, do not replace inference with placeholders, static sample files, canned text, or broad try/except blocks that hide failures. - Keep or restore a cheap health endpoint with api_name="health". - Keep README metadata valid. - Do not publish anything. Work only in the current workspace. Required deliverables: REPAIR_PLAN.md, patched files, REPAIR_SUMMARY.md. """ (workspace / "REPAIR_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / "REPAIR_GOAL.md").write_text(goal, encoding="utf-8") (repair_dir / f"REPAIR_GOAL_attempt_{repair_attempt}.md").write_text(goal, encoding="utf-8") write_repair_history(run_dir, load_repair_attempts(run_dir)) append_event(events_path, "repair_plan", "started", "Running Pi minimal patch repair", {"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type")}) estimate_pi_context_budget(run_dir, workspace, phase="repair_patch", task_type=repair_task_packet.get("task_type") or "runtime_patch", direct_prompt=goal, referenced_files=["REPAIR_TASK_PACKET.json", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], max_context_chars=32000, omitted_sections=["full_build_logs", "full_runtime_logs", "full_report", "full_pi_output"], events_path=events_path, artifact_dir=repair_dir) write_agent_trace_record(run_dir, phase="repair_patch", event="command_started", status="started", message="Pi structured patch repair started", data={"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type"), "decision": decision or {}}, artifacts=["repair/REPAIR_TASK_PACKET.json", "repair/FROZEN_REPAIR_DECISION.json", "repair/LOG_EVIDENCE_PACKET.json", "repair/REPAIR_GOAL.md", "repair/REPAIR_DECISION.json", "repair/PI_PROMPT_BUDGET_repair_patch.json"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=1500) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_repair_output.txt").write_text(out, encoding="utf-8") write_pi_call_fingerprint(run_dir, phase="repair_patch", requested_model=pi_model, prompt_text=goal, output_text=out, task_packet_path=workspace / "REPAIR_TASK_PACKET.json", events_path=events_path, artifact_dir=repair_dir) append_agent_trace_artifact(run_dir, phase="repair_patch", event="repair_output", artifact="logs/pi_repair_output.txt", text=out, status="success" if code == 0 else "failed", data={"returncode": code}) if code != 0: append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", post_repair_result="pi_exit_nonzero", notes=out[-1000:]) append_event(events_path, "repair_patch", "failed", "Pi patch repair returned a non-zero exit code", {"returncode": code, "output_tail": out[-3000:], "attempt": repair_attempt}) return False refusal_path = workspace / "PATCH_REFUSAL.json" if refusal_path.exists(): refusal = load_json_if_exists(refusal_path) or {} write_json(repair_dir / "PATCH_REFUSAL.json", refusal) append_event(events_path, "repair_refusal", "failed", "Pi refused to patch under the bounded repair policy", {"refusal": refusal, "attempt": repair_attempt}) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision="refuse_patch", post_repair_result="pi_refused_patch", notes=str(refusal)[:1000]) return False plan_path = workspace / "REPAIR_PLAN.md" summary_path = workspace / "REPAIR_SUMMARY.md" if not plan_path.exists(): plan_path.write_text(f"# Repair plan\n\nPi did not create a separate plan file. Classified category: `{classification.get('category')}`.\n\nStrategy: {classification.get('recommendation')}\n", encoding="utf-8") if not summary_path.exists(): summary_path.write_text("# Repair summary\n\nPi completed a structured repair pass but did not write a separate summary file. See logs/pi_repair_output.txt.\n", encoding="utf-8") shutil.copy2(plan_path, repair_dir / "REPAIR_PLAN.md") shutil.copy2(summary_path, repair_dir / "REPAIR_SUMMARY.md") write_agent_trace_record(run_dir, phase="repair_patch", event="repair_plan_summary", status="success", message="Repair plan and summary artifacts are available", data={}, artifacts=["repair/REPAIR_PLAN.md", "repair/REPAIR_SUMMARY.md"]) append_event(events_path, "repair_plan", "success", "Repair plan and summary artifacts are available", {"plan": "repair/REPAIR_PLAN.md", "summary": "repair/REPAIR_SUMMARY.md"}) dependency_evidence = f"{failure_reason}\n{build_log}\n{runtime_log}\n" + _read_workspace_text_if_exists(workspace / "REPAIR_PLAN.md") + "\n" + _read_workspace_text_if_exists(workspace / "REPAIR_SUMMARY.md") dependency_issue = extract_pip_dependency_issue(dependency_evidence) if detect_transformers5_dependency_requirement(dependency_evidence): dependency_issue = dependency_issue or {"category": "dependency_error", "package": "transformers"} relax_transformers_upper_bound_for_repair(workspace, events_path, dependency_evidence) normalize_requirements_for_modern_hub(workspace, events_path, dependency_evidence=dependency_evidence) ensure_gradio_launch_show_error(workspace, run_dir, events_path, reason=f"repair_attempt_{repair_attempt}") diff_payload = write_repair_diff_artifact( run_dir, events_path, before_dir, workspace, category=classification.get("category") or "", expected_files=expected_repair_files_from_classification(classification, dependency_issue), ) try: write_json(repair_dir / f"repair_diff_attempt_{repair_attempt}.json", diff_payload) except Exception: pass diff_gate_ok, diff_gate_message, diff_gate_payload = validate_repair_diff_against_task_packet(diff_payload, repair_task_packet, classification, run_dir, events_path) if not diff_gate_ok: append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("changed_files") or [], diff_applied=True, post_repair_result="diff_gate_failed", notes=diff_gate_message) append_event(events_path, "repair_patch", "failed", diff_gate_message, diff_gate_payload) return False if not diff_payload.get("has_publishable_diff"): write_agent_trace_record( run_dir, phase="repair_patch", event="repair_noop", status="failed", message="Pi repair produced no publishable workspace diff; refusing redeploy", data=diff_payload, artifacts=["repair/repair_diff.json", "repair/REPAIR_PLAN.md", "repair/REPAIR_SUMMARY.md"], ) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=False, post_repair_result="repair_noop") append_event(events_path, "repair_patch", "failed", "Structured repair produced no publishable workspace diff; refusing redeploy", diff_payload) return False expected_files = diff_payload.get("expected_files") or [] if expected_files and not diff_payload.get("expected_files_touched"): write_agent_trace_record( run_dir, phase="repair_patch", event="repair_expected_file_not_modified", status="failed", message="Pi repair did not modify the expected publishable file; refusing redeploy", data=diff_payload, artifacts=["repair/repair_diff.json", "repair/REPAIR_PLAN.md", "repair/REPAIR_SUMMARY.md"], ) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=True, post_repair_result="expected_file_not_modified") append_event(events_path, "repair_patch", "failed", "Structured repair did not modify the expected publishable file; refusing redeploy", diff_payload) return False strategy_before = infer_inference_strategy(before_dir) strategy_after = infer_inference_strategy(workspace) write_json(repair_dir / "inference_strategy_before.json", strategy_before) write_json(repair_dir / "inference_strategy_after.json", strategy_after) append_event( events_path, "repair_strategy", "success", "Detected inference strategy before and after repair", {"before": strategy_before, "after": strategy_after}, ) sanity_context = f"{failure_reason}\\n{build_log}\\n{runtime_log}\\n" + _read_workspace_text_if_exists(workspace / "REPAIR_PLAN.md") + "\\n" + _read_workspace_text_if_exists(workspace / "REPAIR_SUMMARY.md") ok, sanity_message = sanity_check_repair_workspace(workspace, implementation_mode, classification, before_dir, sanity_context) if not ok: append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=True, post_repair_result="sanity_failed", notes=sanity_message) append_event(events_path, "repair_patch", "failed", sanity_message, {"category": classification.get("category"), "strategy_before": strategy_before, "strategy_after": strategy_after}) return False append_event(events_path, "repair_patch", "success", sanity_message, {"category": classification.get("category"), "strategy_before": strategy_before, "strategy_after": strategy_after}) if after_dir.exists(): shutil.rmtree(after_dir) shutil.copytree(workspace, after_dir, ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__", "*.pyc")) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=repair_trigger, failure_class=classification.get("category") or "unknown", failure_signature=failure_signature, repair_decision=(decision or {}).get("action") or "patch_code", files_changed=diff_payload.get("publishable_changed_files") or [], diff_applied=True, post_repair_result="patch_ready_for_upload") append_event(events_path, "repair", "success", "Structured repair patch completed; ready to re-upload and revalidate", {"output_tail": out[-3000:], "category": classification.get("category"), "attempt": repair_attempt, "failure_signature": failure_signature}) return True def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, token: str, failure_reason: str, implementation_mode: str, expected_output_type: str): """Let Pi diagnose the blockage, then execute one bounded Factory action at a time. This is the core blockage protocol. Pi decides among allowed actions, but the Factory validates budgets and refuses unsafe patches without actionable logs. """ budgets = {"wait_for_logs": 1, "inspect_more_logs": 1, "factory_rebuild_same_code": 1, "patch_code": MAX_PI_REPAIR_ATTEMPTS} write_repair_history(run_dir, load_repair_attempts(run_dir)) current_error = failure_reason logs_dir = run_dir / "logs" fast_dependency_issue = load_json_if_exists(run_dir / "dependency_issue.json") if (run_dir / "dependency_issue.json").exists() else extract_pip_dependency_issue(current_error) if fast_dependency_issue: append_event(events_path, "space_logs", "skipped", "Using bounded dependency-error evidence; full log collection is deferred to avoid blocking recovery", {"reason": "fast_dependency_issue", "failure_class": fast_dependency_issue.get("failure_class")}) else: collect_space_logs(target_space_id, token, run_dir, events_path) build_log = latest_fresh_collected_build_log(run_dir) or ((logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() and not fast_dependency_issue else "") runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else "" gated_fast_path = classify_gated_access_error_from_logs(run_dir, model_id=model_id) if gated_fast_path.get("detected"): token_patch = patch_workspace_for_gated_runtime_token(workspace, run_dir, events_path) # If the generated app was not token-aware, this is an actionable Factory patch. # Return a clear failure to the caller so the bounded repair loop can upload the # patched workspace instead of burning a 900s Pi diagnosis on an already-classified 401. write_repair_outcome( run_dir, events_path, repair_trigger="gated_model_access_error", root_cause="gated_or_private_model_access", repair_decision="factory_patch_token_handling" if token_patch.get("applied") else "blocked_gated_access_or_secret_issue", patch_applied=bool(token_patch.get("applied")), upload_success=False, post_repair_validation="not_started", final_user_message="Detected a gated/private base-model access error from Space logs before Pi diagnosis.", validation_error=gated_fast_path.get("evidence_tail", "")[:4000], ) if token_patch.get("applied"): append_event(events_path, "gated_access_fast_path", "success", "Detected gated/private access error and patched app token handling before Pi diagnosis", {"category": gated_fast_path.get("category"), "token_value_logged": False}) configure_space_runtime_secrets(api, target_space_id, token, run_dir, events_path) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) return validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) append_event(events_path, "gated_access_fast_path", "failed", "Detected gated/private access error but app already looked token-aware; manual access/secret issue remains", {"category": gated_fast_path.get("category"), "token_value_logged": False}) write_blockage_artifact(workspace, run_dir, events_path, {"action": "declare_technical_blocker", "classification": gated_fast_path, "patch_allowed": False}, current_error, status="gated_model_access_required") raise RuntimeError("Gated/private model access failed in Space runtime; verify Space secrets and gated model license access.") latest_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {} if not isinstance(latest_runtime, dict): latest_runtime = {} recovery = attempt_runtime_recovery_before_repair( api, workspace, target_space_id, token, run_dir, events_path, failure_reason=current_error, build_log=build_log, runtime_log=runtime_log, runtime_payload=latest_runtime, ) if recovery.get("triggered") and recovery.get("recovery_attempted") and not recovery.get("recovery_exhausted"): append_event(events_path, "repair_validation", "started", "Revalidating after runtime recovery before Pi repair") try: write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) write_runtime_recovery(run_dir, {**recovery, "recovery_exhausted": False, "result": "recovered", "post_recovery_validation": "success"}) write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Runtime recovery resolved the Space startup/scheduling blockage.") append_event(events_path, "runtime_recovery", "success", "Runtime recovery resolved validation before Pi repair") return validation except Exception as exc: current_error = f"{current_error}\n\nRuntime recovery did not resolve validation: {str(exc)[:4000]}" collect_space_logs(target_space_id, token, run_dir, events_path) write_runtime_recovery(run_dir, {**recovery, "recovery_exhausted": True, "result": "post_recovery_validation_failed", "post_recovery_validation": "failed", "validation_error": str(exc)[:4000]}) write_repair_outcome(run_dir, events_path, post_repair_validation="failed_after_runtime_recovery", failure_type="runtime_recovery_exhausted", final_user_message="Runtime recovery was attempted before repair, but validation still failed.", validation_error=str(exc)[:4000]) append_event(events_path, "runtime_recovery", "failed", "Runtime recovery did not resolve validation; continuing to Pi diagnosis", {"error": str(exc)[:4000]}) build_log = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_build.txt").exists() else build_log runtime_log = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if (logs_dir / "space_logs_runtime.txt").exists() else runtime_log dependency_issue = extract_pip_dependency_issue(f"{current_error}\n{build_log}\n{runtime_log}") if dependency_issue: write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, current_error) write_repair_outcome( run_dir, events_path, repair_trigger="dependency_resolution_error", root_cause="dependency_resolution_error", initial_error=str(current_error)[:4000], patch_applied=False, upload_success=False, post_repair_validation="not_started", ) if apply_dependency_guardrail_repair(workspace, run_dir, events_path, current_error, build_log, runtime_log): write_repair_outcome( run_dir, events_path, repair_trigger="dependency_resolution_error", repair_decision="deterministic_dependency_guardrail", patch_applied=True, upload_success=False, post_repair_validation="not_started", ) append_event(events_path, "factory_rebuild", "started", "Re-uploading dependency-guardrailed workspace after pip resolver build error") if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="dependency_guardrail_repair"): write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="not_started", failure_type="repair_upload_blocked_by_guardrail") raise RuntimeError("Dependency guardrail rebuild skipped by restart guardrails") write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "factory_rebuild", "success", "Dependency-guardrailed workspace uploaded; revalidating live Space") append_event(events_path, "repair_validation", "started", "Revalidating after deterministic dependency repair") write_live_status(run_dir, stage="live_validation", status="running", message="Waiting for Space runtime and health endpoint", data={"target_space": target_space_id}) try: write_auth_probe(run_dir, events_path, "before_initial_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Deterministic dependency repair resolved the build blockage.") append_event(events_path, "repair_validation", "success", "Deterministic dependency repair resolved the build blockage") return validation except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome( run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"], final_user_message=repair_class["message"], validation_error=str(exc)[:4000], ) if repair_class["terminal_status"] == "auth_refresh_required": append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]}) raise AuthRefreshRequired(repair_class["message"]) current_error = f"{current_error}\n\nDependency guardrail rebuild did not resolve validation: {str(exc)[:4000]}" collect_space_logs(target_space_id, token, run_dir, events_path) append_event(events_path, "repair_validation", "failed", "Dependency guardrail rebuild did not resolve validation; falling back to Pi diagnosis", {"error": str(exc)[:4000]}) for iteration in range(1, 5): decision = diagnose_failure_with_pi( workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, iteration, budgets, ) action = decision.get("action") if action in {"wait_for_logs", "inspect_more_logs"}: if budgets.get(action, 0) <= 0: decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + [f"budget_exhausted:{action}"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError(f"Automated recovery stopped after {action} budget was exhausted") budgets[action] -= 1 wait_seconds = int(os.environ.get("SPACE_FACTORY_FAILURE_LOG_WAIT_SECONDS", "45")) append_event(events_path, "wait_for_logs", "started", "Waiting for delayed HF Space logs before changing code", {"seconds": wait_seconds, "decision": decision}) time.sleep(max(1, wait_seconds)) collect_space_logs(target_space_id, token, run_dir, events_path) append_event(events_path, "wait_for_logs", "success", "Collected Space logs after wait", {"remaining_budget": budgets}) current_error = f"{current_error}\n\nAfter wait_for_logs/inspect_more_logs, validation is still considered failed; re-diagnose with refreshed logs." continue if action == "factory_rebuild_same_code": if budgets.get(action, 0) <= 0: decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + ["budget_exhausted:factory_rebuild_same_code"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError("Automated recovery stopped after factory rebuild budget was exhausted") budgets[action] -= 1 append_event(events_path, "factory_rebuild", "started", "Re-uploading the same workspace to force a same-code Space rebuild", {"decision": decision}) if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="pi_decision_factory_rebuild_same_code"): decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + ["same_code_upload_blocked_by_restart_guardrail"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError("Same-code rebuild skipped by restart guardrails") append_event(events_path, "factory_rebuild", "success", "Same-code workspace re-uploaded; revalidating live Space") append_event(events_path, "repair_validation", "started", "Revalidating after same-code factory rebuild") try: write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) write_repair_outcome(run_dir, events_path, repair_decision="factory_rebuild_same_code", patch_applied=False, upload_success=True, post_repair_validation="success", failure_type="", final_user_message="Same-code factory rebuild resolved the blockage.") append_event(events_path, "repair_validation", "success", "Same-code factory rebuild resolved the blockage") return validation except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome( run_dir, events_path, repair_decision="factory_rebuild_same_code", patch_applied=False, upload_success=True, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"], final_user_message=repair_class["message"], validation_error=str(exc)[:4000], ) if repair_class["terminal_status"] == "auth_refresh_required": append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]}) raise AuthRefreshRequired(repair_class["message"]) current_error = f"{current_error}\n\nSame-code factory rebuild did not resolve validation: {str(exc)[:4000]}" collect_space_logs(target_space_id, token, run_dir, events_path) append_event(events_path, "repair_validation", "failed", "Same-code factory rebuild did not resolve validation; re-diagnosing", {"error": str(exc)[:4000]}) continue if action == "patch_code": if budgets.get(action, 0) <= 0: decision["factory_overrides"] = list(decision.get("factory_overrides") or []) + ["budget_exhausted:patch_code"] write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) raise RuntimeError("Automated recovery stopped after patch budget was exhausted") budgets[action] -= 1 append_event(events_path, "repair", "started", "Pi diagnosis allows a minimal code patch", {"decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure", root_cause=(decision.get("classification") or {}).get("category") or "live_validation_failure", repair_decision="patch_code", decision=decision, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure") if not repaired: write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation="not_started", failure_type="repair_patch_failed", final_user_message="Structured patch repair failed before redeploy.") write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair") append_event(events_path, "failure", "failed", "Structured patch repair failed before redeploy", {"decision": decision}) raise RuntimeError("Structured patch repair failed before redeploy") write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "repair_upload", "started", "Uploading repaired workspace") write_auth_probe(run_dir, events_path, "before_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) configure_space_runtime_secrets(api, target_space_id, token, run_dir, events_path) patch_workspace_for_gated_runtime_token(workspace, run_dir, events_path) apply_deterministic_diffusers_lora_dependency_policy(workspace, run_dir, events_path) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "repair_upload", "success", "Repaired workspace uploaded") append_event(events_path, "repair_validation", "started", "Revalidating repaired Space") try: write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure", failure_class=(decision.get("classification") or {}).get("category") or "unknown", failure_signature=compute_failure_signature(current_error, build_log, runtime_log, classification=(decision.get("classification") or {})), repair_decision="patch_code", post_repair_result="validation_success", new_failure_signature="") write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Repaired Space passed live API validation.") append_event(events_path, "repair_validation", "success", "Repaired Space passed live API validation", {"repair_attempt": repair_attempt}) return validation except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome( run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"], final_user_message=repair_class["message"], validation_error=str(exc)[:4000], ) if repair_class["terminal_status"] == "auth_refresh_required": append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]}) raise AuthRefreshRequired(repair_class["message"]) # Refresh Space logs before computing the next signature. v198.21 # fixes a stale-signature bug where the loop kept the original # cuda_oom signature even after the repaired Space surfaced a new # actionable error such as "PEFT backend is required". collect_space_logs(target_space_id, token, run_dir, events_path) new_signature = compute_latest_failure_signature(run_dir, str(exc), classification=repair_class) previous_signature = compute_failure_signature(current_error, build_log, runtime_log, classification=(decision.get("classification") or {})) append_repair_attempt_record(run_dir, attempt=repair_attempt, trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure", failure_class=(decision.get("classification") or {}).get("category") or "unknown", failure_signature=previous_signature, repair_decision="patch_code", post_repair_result=repair_class["post_repair_validation"], new_failure_signature=new_signature) latest_build, latest_runtime, latest_brief = read_latest_space_log_text(run_dir) current_error = f"{current_error}\n\nPatch repair attempt {repair_attempt} did not resolve validation: {str(exc)[:4000]}\n\nRefreshed post-repair actionable error:\n{latest_brief[:3000]}" append_event(events_path, "repair_validation", "failed", "Repair attempted, refreshed Space logs, and continuing if repair budget remains for a distinct new failure", {"error": str(exc)[:4000], "repair_attempt": repair_attempt, "previous_failure_signature": previous_signature, "new_failure_signature": new_signature, "remaining_patch_budget": budgets.get("patch_code", 0)}) if budgets.get("patch_code", 0) > 0 and new_signature and new_signature != previous_signature and not same_failure_signature_repeated(run_dir, new_signature, threshold=2): continue if budgets.get("patch_code", 0) > 0 and not same_failure_signature_repeated(run_dir, new_signature, threshold=2): continue write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair_budget_exhausted") append_event(events_path, "failure", "failed", "Run failed after structured repair budget was exhausted or failure repeated", {"repair_error": str(exc)[:4000], "repair_attempt": repair_attempt, "new_failure_signature": new_signature}) raise if action == "request_manual_hardware": write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="manual_hardware_required") append_event(events_path, "manual_hardware_required", "failed", "Pi diagnosis requested manual hardware or quota action", {"decision": decision}) raise RuntimeError("Pi diagnosis requested manual hardware or quota action") write_blockage_artifact(workspace, run_dir, events_path, decision, current_error) append_event(events_path, "failure", "failed", "Pi diagnosis declared a technical blocker", {"decision": decision}) raise RuntimeError("Pi diagnosis declared a technical blocker") final_decision = {"action": "declare_technical_blocker", "reason": "Blockage protocol exhausted all bounded iterations.", "evidence": [], "patch_allowed": False} write_blockage_artifact(workspace, run_dir, events_path, final_decision, current_error) append_event(events_path, "failure", "failed", "Blockage protocol exhausted all bounded iterations") raise RuntimeError("Blockage protocol exhausted all bounded iterations") def configure_space_runtime_secrets(api, target_space_id: str, token: str, run_dir: Path, events_path: Path) -> dict: """Propagate the Job HF token into the generated Space as runtime secrets. Gated base models and private adapters are fetched by the Space runtime, not by the Factory Job. A token that works in the Job is therefore not enough: the generated Space must receive HF_TOKEN/HUGGING_FACE_HUB_TOKEN as secrets. Security invariant: never persist the token value, only secret names/status. """ result = { "schema_version": "space_secrets.v198_20", "target_space_id": target_space_id, "space_secrets_configured": False, "secret_names": [], "token_value_logged": False, "errors": [], } if not token: result["errors"].append({"secret": "HF_TOKEN", "error": "missing_job_token"}) write_json(run_dir / "space_secrets_status.json", result) append_event(events_path, "space_secrets", "failed", "Cannot configure generated Space secrets: Job HF token is missing", {k: v for k, v in result.items() if k != "errors"}) return result for name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]: try: api.add_space_secret(repo_id=target_space_id, key=name, value=token, token=token) result["secret_names"].append(name) except Exception as exc: result["errors"].append({"secret": name, **exception_payload(exc)}) result["space_secrets_configured"] = bool(result["secret_names"]) write_json(run_dir / "space_secrets_status.json", result) append_event( events_path, "space_secrets", "success" if result["space_secrets_configured"] else "failed", "Configured generated Space runtime secrets for gated/private model access" if result["space_secrets_configured"] else "Failed to configure generated Space runtime secrets", {"target_space_id": target_space_id, "secret_names": result["secret_names"], "token_value_logged": False, "error_count": len(result["errors"])}, ) return result def _ensure_os_import_and_hf_token_var(app_text: str) -> tuple[str, bool]: changed = False if "import os" not in app_text and "from os import" not in app_text: lines = app_text.splitlines() insert_at = 0 while insert_at < len(lines) and (lines[insert_at].startswith("#!") or lines[insert_at].strip().startswith("#") or not lines[insert_at].strip()): insert_at += 1 lines.insert(insert_at, "import os") app_text = "\n".join(lines) + ("\n" if app_text.endswith("\n") else "") changed = True if "HF_TOKEN = os.environ.get(" not in app_text and "HUGGING_FACE_HUB_TOKEN" not in app_text.split("\n", 40): marker = "HF_TOKEN = os.environ.get(\"HF_TOKEN\") or os.environ.get(\"HUGGING_FACE_HUB_TOKEN\")\n" lines = app_text.splitlines() insert_at = 0 # Place after import block when possible. for i, line in enumerate(lines[:80]): stripped = line.strip() if stripped.startswith("import ") or stripped.startswith("from ") or not stripped: insert_at = i + 1 continue break lines.insert(insert_at, marker.rstrip("\n")) app_text = "\n".join(lines) + ("\n" if app_text.endswith("\n") else "") changed = True return app_text, changed def _inject_token_kwarg_in_simple_call(text: str, call_name: str) -> tuple[str, int]: """Add token=HF_TOKEN to simple non-nested calls when missing. This intentionally avoids complex AST rewriting. It is good enough for the common Diffusers/HF Hub patterns Pi generates, and it refuses calls that already pass token/use_auth_token. """ import re as _re count = 0 pattern = _re.compile(rf"({call_name}\s*\()(?P[^()]{{0,2500}}?)(\))", _re.DOTALL) def repl(match): nonlocal count args = match.group("args") if "token=" in args or "use_auth_token=" in args: return match.group(0) if "HF_TOKEN" in args: return match.group(0) stripped = args.rstrip() if not stripped: new_args = "token=HF_TOKEN" elif "\n" in args: new_args = stripped + ",\n token=HF_TOKEN," else: new_args = stripped + ", token=HF_TOKEN" count += 1 return match.group(1) + new_args + match.group(3) return pattern.sub(repl, text), count def patch_workspace_for_gated_runtime_token(workspace: Path, run_dir: Path, events_path: Path) -> dict: """Make generated apps gated-repo aware without exposing token values. Pi often detects LoRA/base-model dependencies correctly but forgets that the runtime Space needs to pass its secret token into Diffusers/HF Hub loaders. This patch adds HF_TOKEN/HUGGING_FACE_HUB_TOKEN env lookup and injects token=HF_TOKEN into common loader calls when absent. """ app_path = workspace / "app.py" deps = read_json(workspace / "MODEL_DEPENDENCIES.json", {}) if (workspace / "MODEL_DEPENDENCIES.json").exists() else {} app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" combined = json.dumps(deps, ensure_ascii=False).lower() + "\n" + app_text.lower() gated_or_adapter = any(x in combined for x in ["gated", "private", "lora", "adapter", "base_model", "base_model_id", "black-forest-labs/flux.1-dev"]) loader_present = any(x in app_text for x in ["from_pretrained(", "load_lora_weights(", "hf_hub_download(", "snapshot_download("]) result = { "schema_version": "gated_runtime_token_patch.v198_20", "applied": False, "reason": "not_applicable", "gated_or_adapter_detected": bool(gated_or_adapter), "loader_present": bool(loader_present), "calls_patched": {}, "token_value_logged": False, } if not app_path.exists() or not loader_present or not gated_or_adapter: write_json(run_dir / "gated_runtime_token_patch.json", result) return result new_text, changed = _ensure_os_import_and_hf_token_var(app_text) total = 0 for call in ["from_pretrained", "load_lora_weights", "hf_hub_download", "snapshot_download"]: new_text, n = _inject_token_kwarg_in_simple_call(new_text, call) if n: result["calls_patched"][call] = n total += n if changed or total: app_path.write_text(new_text, encoding="utf-8") result.update({"applied": True, "reason": "patched_hf_loader_token_kwargs"}) append_event(events_path, "gated_runtime_token", "success", "Patched generated app to pass Space HF_TOKEN into gated/private HF Hub loaders", {"calls_patched": result["calls_patched"], "token_value_logged": False}) else: result["reason"] = "already_token_aware" append_event(events_path, "gated_runtime_token", "success", "Generated app already appears token-aware for gated/private loaders", {"token_value_logged": False}) write_json(run_dir / "gated_runtime_token_patch.json", result) return result def classify_gated_access_error_from_logs(run_dir: Path, *, model_id: str = "") -> dict: """Detect clear gated/private Hub access failures before invoking Pi diagnosis.""" text = "" for rel in ["repair/actionable_error_brief.md", "logs/space_logs_run.txt", "logs/space_logs_runtime.txt", "logs/space_logs_build.txt"]: path = run_dir / rel if path.exists(): text += "\n" + path.read_text(encoding="utf-8", errors="ignore")[-8000:] lower = text.lower() is_auth = ("401 client error" in lower or "403 client error" in lower or "unauthorized" in lower or "gated repo" in lower or "restricted repo" in lower) is_hf_model = "huggingface.co/" in lower or "hf.co/" in lower or "model_index.json" in lower or "resolve/main" in lower if not (is_auth and is_hf_model): return {"detected": False} # Keep only redacted/actionable context; never include token values. brief = text[-4000:] payload = { "schema_version": "gated_access_fast_path.v198_20", "detected": True, "category": "gated_or_private_model_access", "model_id": model_id, "reason": "HF Hub 401/403/private/gated access error found in Space run/build logs", "recommended_action": "ensure Space secrets HF_TOKEN/HUGGING_FACE_HUB_TOKEN are configured and app loaders pass token=HF_TOKEN; if Job token also cannot access the repo, user must accept license/request access", "token_value_logged": False, "evidence_tail": redact_text(brief)[:4000], } write_json(run_dir / "gated_access_fast_path.json", payload) return payload def upload_workspace(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path): sanitize_readme_metadata(workspace, events_path) normalize_requirements_for_modern_hub(workspace, events_path) append_event(events_path, "upload_files", "started", "Preparing whitelist runtime payload for generated Space upload") gen_dir = run_dir / "generated" if gen_dir.exists(): shutil.rmtree(gen_dir) shutil.copytree(workspace, gen_dir, ignore=internal_workspace_copy_ignore) scaffold_detection = detect_placeholder_scaffold(workspace) write_json(run_dir / "placeholder_scaffold_detection.json", scaffold_detection) if scaffold_detection.get("detected"): append_event(events_path, "placeholder_scaffold", "warning", "Generated runtime still contains initial scaffold markers", scaffold_detection) payload_dir, manifest = build_runtime_upload_payload(workspace, run_dir, events_path) try: api.upload_folder( folder_path=str(payload_dir), repo_id=target_space_id, repo_type="space", token=token, ignore_patterns=internal_workspace_upload_ignore_patterns(), ) except Exception as exc: classify_space_upload_exception(exc, run_dir, events_path) raise uploaded_files = sorted(item["path"] for item in manifest.get("files", [])) upload_epoch = record_runtime_upload_epoch(run_dir, events_path, target_space_id=target_space_id, manifest=manifest) append_event( events_path, "upload_files", "success", "Uploaded whitelist runtime payload folder", { "file_count": len(uploaded_files), "files_sample": uploaded_files[:50], "total_bytes": manifest.get("total_bytes"), "excluded_count": manifest.get("excluded_count"), "payload_manifest": "runtime_upload_payload_manifest.json", "upload_epoch": upload_epoch, }, ) def load_json_if_exists(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8", errors="replace")) except Exception as exc: return {"parse_error": str(exc), "raw_tail": path.read_text(encoding="utf-8", errors="replace")[-2000:]} def infer_generation_gate(workspace: Path, implementation_mode: str, validation: dict, generation_smoke: dict | None, run_dir: Path, events_path: Path) -> dict: """Classify the run separately from process success. /health passing means the Space boots. It does not mean the generated Space performs model inference. In full-inference-gated mode we require either an actual implementation signal or a machine-readable blocker report. """ app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore") if (workspace / "app.py").exists() else "" summary_text = (workspace / "PI_SUMMARY.md").read_text(encoding="utf-8", errors="ignore") if (workspace / "PI_SUMMARY.md").exists() else "" req_text = (workspace / "requirements.txt").read_text(encoding="utf-8", errors="ignore") if (workspace / "requirements.txt").exists() else "" blockers_path = workspace / "TECHNICAL_BLOCKERS.json" blockers = load_json_if_exists(blockers_path) scaffold_detection = detect_placeholder_scaffold(workspace) combined = (app_text + "\n" + summary_text).lower() blocked_markers = [ "full generation is not implemented", "full generation is intentionally not wired", "full inference is blocked", "returns a detailed diagnostic", "diagnostic report instead", "diagnostic demo", "official demo", "official space", "does not generate real", "does not load model weights", "does not generate real audio", "true inference is available in the official", "placeholder generator", "placeholder generation", "info-only", "not implemented", "cannot run in this environment", "out of scope", ] promise_validation = build_promise_validation_status(workspace, validation, generation_smoke) contract_no_full = bool((promise_validation.get("contract_declares_no_full_inference") or {}).get("declared")) diagnostic_contract = bool(promise_validation.get("diagnostic_only") or promise_validation.get("manual_hardware_required")) smoke_ok = isinstance(generation_smoke, dict) and generation_smoke.get("status") == "success" minimal_smoke_ok = isinstance(generation_smoke, dict) and (generation_smoke.get("status") == "demo_usable_smoke_passed" or generation_smoke.get("demo_usable_smoke_passed") is True) full_inference_requested = implementation_mode in {"full-inference-gated", "full-inference-attempt"} promise_fulfilled = bool(promise_validation.get("promise_fulfilled")) heuristic_marker_detected = any(m in combined for m in blocked_markers) scaffold_detected = bool(scaffold_detection.get("detected")) blocker_source = str(blockers.get("source") or "") if isinstance(blockers, dict) else "" heuristic_blocker_json = bool(blockers) and blocker_source == "worker_heuristic_from_PI_SUMMARY_or_app.py" heuristic_blocker_detected = bool(heuristic_marker_detected or heuristic_blocker_json) contract_blocker_detected = bool(contract_no_full or diagnostic_contract) hard_blocker_detected = bool((bool(blockers) and not heuristic_blocker_json) or contract_blocker_detected or scaffold_detected) blocker_detected = bool(hard_blocker_detected or heuristic_blocker_detected) strong_full_inference_success = bool(full_inference_requested and smoke_ok and promise_fulfilled) recommendation = generation_smoke if isinstance(generation_smoke, dict) else measured_zero_gpu_recommendation(None) health_semantics = validation_health_semantics(validation) health_ok = validation_health_passed(validation) implementation_signals = { "has_spaces_gpu": "@spaces.GPU" in app_text, "has_torch": "torch" in req_text or "import torch" in app_text, "has_diffusers": "diffusers" in req_text or "diffusers" in app_text, "has_video_output_hint": any(x in app_text.lower() for x in ["gr.video", "video", ".mp4", "ffmpeg"]), "health_passed": health_ok, "health_endpoint_reachable": bool(health_semantics.get("endpoint_reachable")), "health_semantic_passed": health_semantics.get("semantic_passed"), "health_semantic_negative_markers": health_semantics.get("negative_markers") or [], "health_load_error": health_semantics.get("load_error") or "", "placeholder_scaffold_detected": scaffold_detected, "generation_smoke_passed": smoke_ok, "demo_usable_smoke_passed": minimal_smoke_ok, "canonical_promise_smoke_passed": smoke_ok, "zero_gpu_duration_measured": recommendation.get("recommendation_confidence") == "measured", } non_blocking_warnings = [] blocking_evidence = [] if heuristic_blocker_detected: non_blocking_warnings.append({ "type": "heuristic_text_warning", "message": "Generated artifacts mention blocker-like language, but this is treated as advisory unless runtime validation fails.", "source": "TECHNICAL_BLOCKERS.json" if heuristic_blocker_json else "PI_SUMMARY.md_or_app.py", "non_blocking": True, }) if scaffold_detected: blocking_evidence.append({ "type": "placeholder_scaffold_deployed", "source": "app.py", "overridable_by_live_smoke": True, "details": scaffold_detection, }) if hard_blocker_detected: blocking_evidence.append({ "type": "contract_or_explicit_blocker", "source": "INFERENCE_CONTRACT.json_or_TECHNICAL_BLOCKERS.json_or_app.py", "overridable_by_live_smoke": True, }) if strong_full_inference_success: status = "full_inference_success" if blocker_detected: message = "Space boots and a live generation smoke test passed. Blocker-like text was detected but downgraded to a non-blocking warning because real inference succeeded." non_blocking_warnings.append({ "type": "runtime_success_overrode_blocker_text", "message": "Live generation success and fulfilled promise take precedence over contradictory blocker heuristics.", "non_blocking": True, }) else: message = "Space boots and a live generation smoke test passed. ZeroGPU duration recommendation was measured from real inference." elif blocker_detected: if promise_validation.get("manual_hardware_required"): status = "manual_hardware_required" message = "Space is actionable, but the model-card promise is deferred until suitable hardware is selected in Space Settings." elif promise_validation.get("diagnostic_only") or (isinstance(generation_smoke, dict) and generation_smoke.get("skip_reason") == "contract_declared_no_full_inference"): status = "technical_blocker_boot_only" if isinstance(generation_smoke, dict) and generation_smoke.get("skip_reason") == "contract_declared_no_full_inference": message = "Generation smoke was skipped by contract because full inference was declared unavailable; this is not a full inference success." else: message = "Space boots/responds as a diagnostic app, but real/full model inference is not implemented; this is not a full inference success." elif scaffold_detected: status = "placeholder_scaffold_deployed" message = "Space boots, but the published app still contains the initial scaffold instead of model-specific inference." elif hard_blocker_detected: status = "technical_blocker" message = "Space boots, but full model inference was not implemented. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md." else: status = "full_inference_candidate_health_passed" message = "Space boots, but live generation smoke did not verify full inference. Blocker-like text was detected and kept as a non-blocking warning until runtime proof exists." elif full_inference_requested: if minimal_smoke_ok: status = "demo_usable_full_promise_not_verified" message = "Space generated output with a reduced minimal smoke payload after the canonical promise smoke failed or exceeded capacity. The demo is usable, but full promise validation remains unverified." else: status = "" message = "" smoke_failure_owner = str((generation_smoke or {}).get("failure_owner") or "") if isinstance(generation_smoke, dict) else "" smoke_failure_class = str((generation_smoke or {}).get("failure_class") or (generation_smoke or {}).get("failure_type") or "") if isinstance(generation_smoke, dict) else "" if not status and health_ok and (smoke_failure_owner == "factory_validation_client" or smoke_failure_class in {"smoke_schema_error", "smoke_input_materialization_failed"}): status = "interactive_app_available_smoke_failed" message = "Space boots and exposes the expected API, but ASF could not verify generation because the automatic smoke payload did not match or materialize for the Gradio schema." elif not status and health_ok and smoke_failure_class in {"timeout", "gradio_hidden_runtime_error"}: status = "manual_test_required_smoke_failed" message = "Space boots and exposes the expected API, but automatic generation smoke did not complete or did not expose a traceback; manual Space Test is recommended." elif not status and not health_ok: status = "partial_validation" if health_semantics.get("endpoint_reachable") and health_semantics.get("semantic_passed") is False: message = "The health endpoint is reachable, but the model reported unhealthy/not-ready status; generation was not verified." else: message = "Live generation smoke did not produce a verified output, and semantic app health was not confirmed." elif not status: status = "full_inference_candidate_health_passed" message = "Space boots, but live generation smoke test did not produce a verified output. ZeroGPU duration recommendation was not measured." else: status = "health_only" message = "Safe scaffold health validation passed. Full inference was not requested." if non_blocking_warnings: warnings_payload = { "schema_version": "inference_warnings.v198_23", "source": "infer_generation_gate", "status": status, "warnings": non_blocking_warnings, } try: (run_dir / "generated").mkdir(parents=True, exist_ok=True) write_json(workspace / "INFERENCE_WARNINGS.json", warnings_payload) write_json(run_dir / "generated" / "INFERENCE_WARNINGS.json", warnings_payload) except Exception: pass if blocker_detected and not blockers and not strong_full_inference_success and hard_blocker_detected: blockers = { "full_inference_implemented": False, "source": "worker_contract_or_diagnostic_blocker", "blockers": [ { "type": "contract_or_diagnostic_blocker", "claim": "Machine-readable contracts state that full inference is blocked/not implemented or generation returns diagnostics/placeholders.", "evidence": "See INFERENCE_CONTRACT.json, DEMO_QUALITY_CONTRACT.json, PI_SUMMARY.md and app.py in generated artifacts.", "severity": "blocking", } ], "required_investigations_for_next_run": [ "Check whether PyTorch SDPA can replace flash-attn calls.", "Check whether HF Kernels flash-attn2/3/4 can replace required flash-attn APIs.", "Verify whether 2-GPU context parallelism is strictly required or can be reduced to a single-GPU smoke test.", ], } (run_dir / "generated").mkdir(parents=True, exist_ok=True) write_json(workspace / "TECHNICAL_BLOCKERS.json", blockers) write_json(run_dir / "generated" / "TECHNICAL_BLOCKERS.json", blockers) gate = { "status": status, "message": message, "implementation_mode": implementation_mode, "blocker_detected": blocker_detected, "hard_blocker_detected": hard_blocker_detected, "heuristic_blocker_detected": heuristic_blocker_detected, "strong_full_inference_success": strong_full_inference_success, "decision_basis": "live_generation_smoke_passed" if strong_full_inference_success else "hard_blocker" if hard_blocker_detected else "heuristic_warning" if heuristic_blocker_detected else "health_or_smoke_status", "blocking_evidence": [] if strong_full_inference_success else blocking_evidence, "non_blocking_warnings": non_blocking_warnings, "implementation_signals": implementation_signals, "validation_method": validation.get("method"), "generation_smoke": generation_smoke, "app_boot_validation_status": promise_validation.get("app_boot_validation_status"), "promise_validation_status": promise_validation.get("promise_validation_status"), "promise_fulfilled": bool(promise_validation.get("promise_fulfilled")), "ui_status": promise_validation.get("ui_status") if promise_validation.get("ui_status") not in {"full_inference_success", "partial_validation"} else status, "ui_badge": promise_validation.get("ui_badge"), "promise_validation": promise_validation, "zero_gpu_duration_recommendation": { "observed_latency_seconds": recommendation.get("observed_latency_seconds"), "recommended_zero_gpu_duration_seconds": recommendation.get("recommended_zero_gpu_duration_seconds"), "recommendation_source": recommendation.get("recommendation_source"), "recommendation_confidence": recommendation.get("recommendation_confidence"), "measurement_note": recommendation.get("measurement_note"), }, "placeholder_scaffold_detection": scaffold_detection, "blockers": blockers, } write_json(run_dir / "inference_gate.json", gate) append_event(events_path, "inference_gate", status, message, gate) return gate def main(): run_id = os.environ["RUN_ID"] hf_username = os.environ.get("HF_USERNAME", "unknown") bucket_source = os.environ.get("BUCKET_SOURCE", "unknown") output_root = Path(os.environ.get("OUTPUT_ROOT", "/output")) target_space_id = os.environ.get("TARGET_SPACE_ID", "") target_space_auto_generated = os.environ.get("TARGET_SPACE_AUTO_GENERATED", "false").lower() in {"1", "true", "yes", "on"} model_id = sanitize_model_id(os.environ.get("MODEL_ID", DEFAULT_MODEL_ID)) pi_model = os.environ.get("PI_MODEL", "Qwen/Qwen3-Coder-Next") preferred_hardware = normalize_auto_space_hardware(os.environ.get("PREFERRED_SPACE_HARDWARE"), DEFAULT_PREFERRED_SPACE_HARDWARE) fallback_hardware = normalize_auto_space_hardware(os.environ.get("FALLBACK_SPACE_HARDWARE"), DEFAULT_FALLBACK_SPACE_HARDWARE) try_zero_gpu_first = os.environ.get("TRY_ZERO_GPU_FIRST", "true").lower() in {"1", "true", "yes", "on"} allow_fixed_gpu_fallback = os.environ.get("ALLOW_FIXED_GPU_FALLBACK", "true").lower() in {"1", "true", "yes", "on"} implementation_mode = os.environ.get("IMPLEMENTATION_MODE", "full-inference-attempt") expected_output_type = os.environ.get("EXPECTED_OUTPUT_TYPE", "any") token = os.environ.get("HF_TOKEN") run_dir = output_root / "runs" / run_id events_path = run_dir / "events.jsonl" state_path = run_dir / "state.json" workspace = Path("/tmp/universal_workspace") ensure_hf_token_context(run_dir, events_path) pi_model_resolution = {"requested_model": pi_model, "configured_model": pi_model, "effective_model": pi_model, "provider": "huggingface", "mismatch": False} append_event(events_path, "bootstrap", "started", "Universal model-card builder worker started", {"model_id": model_id, "target_space_id": target_space_id}) write_json(state_path, {"run_id": run_id, "kind": "universal_model_card_builder", "status": "running", "message": "Attempting Universal model-card builder Space creation", "model_id": model_id, "pi_model": pi_model, "pi_model_resolution": pi_model_resolution, "target_space": target_space_id, "created_by": hf_username, "bucket_source": bucket_source, "created_at": now(), "updated_at": now()}) if not token: fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets") if not TARGET_RE.match(target_space_id): fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space_id": target_space_id}) try: install_python_deps(events_path) from huggingface_hub import HfApi api = HfApi(token=token) whoami = api.whoami(token=token) append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")}) write_auth_probe(run_dir, events_path, "before_space_create", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_space_create"), raise_on_unsafe=True) append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id}) info = api.model_info(model_id, token=token, files_metadata=True) siblings = [getattr(s, "rfilename", "") for s in (info.siblings or [])] analysis = {"model_id": model_id, "pipeline_tag": getattr(info, "pipeline_tag", None), "library_name": getattr(info, "library_name", None), "tags": list(getattr(info, "tags", []) or [])[:100], "siblings": siblings[:160], "default_model_target": model_id == DEFAULT_MODEL_ID, "preferred_hardware": preferred_hardware, "fallback_hardware": fallback_hardware, "allow_fixed_gpu_fallback": allow_fixed_gpu_fallback, "implementation_mode": implementation_mode} analysis_inputs = prepare_build_analysis_inputs(model_id, token, siblings, info, analysis) analysis["model_card_source"] = analysis_inputs.get("model_card_source.json", {}) analysis["model_repo_tree"] = { "file_count": analysis_inputs.get("model_repo_tree.json", {}).get("file_count"), "detected_components": analysis_inputs.get("model_repo_tree.json", {}).get("detected_components", []), } write_json(run_dir / "model_analysis.json", analysis) write_analysis_inputs_dir(run_dir, analysis_inputs) append_event(events_path, "model_analysis", "success", "Model metadata and build-time grounding inputs fetched", {"pipeline_tag": analysis["pipeline_tag"], "library_name": analysis["library_name"], "model_card_source": analysis["model_card_source"].get("source"), "model_card_fallback_used": analysis["model_card_source"].get("fallback_used")}) create_initial_workspace(workspace, model_id, target_space_id, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, implementation_mode, analysis, analysis_inputs) for context_name in ["MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md"]: try: if (workspace / context_name).exists(): shutil.copy2(workspace / context_name, run_dir / context_name) except Exception: pass append_event(events_path, "workspace", "success", "Prepared universal model-card workspace", {"files": sorted(p.name for p in workspace.iterdir())}) install_pi(events_path) configure_pi(events_path, pi_model) append_event(events_path, "pi_run", "started", "Running Pi on universal model-card workspace", {"model": pi_model}) (run_dir / "logs").mkdir(parents=True, exist_ok=True) initial_goal_text = (workspace / "GOAL.md").read_text(encoding="utf-8") estimate_pi_context_budget( run_dir, workspace, phase="initial_build", task_type="generate_gradio_demo", direct_prompt=initial_goal_text, referenced_files=[ "MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md", "analysis_inputs/model_card.md", "analysis_inputs/model_repo_tree.json", "analysis_inputs/source_policy.md", "analysis_inputs/hf_spaces_operational_gist.md", "analysis_inputs/hf_spaces_gist_source.json", ], max_context_chars=90000, omitted_sections=["previous_repair_history", "space_runtime_logs", "space_build_logs"], events_path=events_path, artifact_dir=run_dir, ) code, pi_out = run_cmd_streaming( ["pi", "-p", initial_goal_text], cwd=workspace, timeout=2400, live_log_path=run_dir / "logs" / "pi_live_output.txt", events_path=events_path, run_dir=run_dir, step="pi_run", trace_phase="initial_build", ) (run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8") if code != 0: append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]}) collect_pi_traces(run_dir, events_path) fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]}) append_event(events_path, "pi_run", "success", "Pi completed universal model-card workspace pass", {"output_tail": pi_out[-2000:]}) # Pi sessions are the source of truth for the assistant/model actually used. # Sync them first, then resolve requested/configured/observed model identity. collect_pi_traces(run_dir, events_path) pi_model_resolution = detect_pi_model_resolution(pi_model, run_dir, pi_out) write_pi_call_fingerprint(run_dir, phase="initial_build", requested_model=pi_model, prompt_text=initial_goal_text, output_text=pi_out, events_path=events_path, artifact_dir=run_dir, identity=pi_model_resolution) emit_pi_model_resolution(events_path, pi_model_resolution) if not (workspace / "PI_SUMMARY.md").exists(): (workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8") write_pi_planning_review(workspace, run_dir, events_path, analysis, implementation_mode) app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore") if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text: append_event(events_path, "pi_verification", "failed", "app.py does not appear to expose /health; injecting safe health endpoint is not implemented") fail(run_dir, events_path, "Pi output did not preserve a /health endpoint") append_event(events_path, "pi_verification", "success", "Pi output preserved health validation endpoint") output_type_resolution = contract_effective_output_type(workspace, expected_output_type) effective_expected_output_type = output_type_resolution.get("effective_expected_output_type") or expected_output_type write_json(run_dir / "output_type_resolution.json", output_type_resolution) append_event(events_path, "output_type_resolution", "success", "Resolved effective output type from generated contract", output_type_resolution) hardware_intent = compute_hardware_intent( workspace, analysis, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first, ) write_json(run_dir / "hardware_intent.json", hardware_intent) manual_block = should_stop_before_cpu_deploy_for_manual_hardware(workspace, hardware_intent) write_json(run_dir / "manual_hardware_block.json", manual_block) manual_hardware_actionable = False manual_hardware_actionable_payload = {} if manual_block.get("triggered"): append_event(events_path, "manual_hardware_required", "warning", "Manual hardware required; creating an actionable Space instead of stopping before Space creation", manual_block) manual_hardware_actionable_payload = prepare_manual_hardware_actionable_workspace( workspace, run_dir, events_path, model_id=model_id, target_space_id=target_space_id, hardware_intent=hardware_intent, manual_block=manual_block, ) manual_hardware_actionable = True # Recompute the deployable app contract after replacing the unsafe # full-GPU candidate with the CPU-safe manual-action Space app. output_type_resolution = contract_effective_output_type(workspace, expected_output_type) effective_expected_output_type = output_type_resolution.get("effective_expected_output_type") or expected_output_type write_json(run_dir / "output_type_resolution.json", output_type_resolution) append_event(events_path, "output_type_resolution", "success", "Resolved effective output type from manual-hardware actionable contract", output_type_resolution) preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first = apply_hardware_intent( hardware_intent, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, try_zero_gpu_first, ) append_event(events_path, "hardware_intent", "success", "Resolved contract-aware hardware intent", hardware_intent) append_event(events_path, "hardware_strategy", "started", "Creating Space with contract-aware hardware strategy", {"preferred_hardware": preferred_hardware, "fallback_hardware": fallback_hardware, "allow_fixed_gpu_fallback": allow_fixed_gpu_fallback, "try_zero_gpu_first": try_zero_gpu_first, "hardware_intent": hardware_intent}) hardware_strategy = create_space_with_hardware_strategy( api, target_space_id, token, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback, events_path, allow_auto_rename=target_space_auto_generated, try_zero_gpu_first=try_zero_gpu_first, ) target_space_id = hardware_strategy.get("target_space_id") or target_space_id selected_hardware = hardware_strategy.get("selected_hardware") or "default-cpu-or-existing" hardware_attempts = list(hardware_strategy.get("attempts") or []) requested_hardware_sequence = list(hardware_strategy.get("requested_sequence") or []) update_state(run_dir, { "run_id": run_id, "kind": "universal_model_card_builder", "status": "running", "message": "Private Space created; uploading generated workspace", "model_id": model_id, "pi_model": pi_model, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "selected_hardware": selected_hardware, "hardware_intent": hardware_intent, "hardware_attempts": hardware_attempts, "requested_hardware_sequence": requested_hardware_sequence, "model_expected_output_type": output_type_resolution.get("model_expected_output_type"), "contract_effective_output_type": output_type_resolution.get("contract_effective_output_type"), "expected_output_type": effective_expected_output_type, "created_by": hf_username, "bucket_source": bucket_source, }) write_artifact_manifest(run_dir, reason="space_created") # Upload after create. If create_repo(space_hardware=...) succeeded, the build # starts directly on the requested hardware. If it fell back to CPU, the run # remains valid but will be marked manual_hardware_required when inference # signals indicate GPU is needed. write_auth_probe(run_dir, events_path, "before_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_upload"), raise_on_unsafe=True) configure_space_runtime_secrets(api, target_space_id, token, run_dir, events_path) patch_workspace_for_gated_runtime_token(workspace, run_dir, events_path) ensure_gradio_launch_show_error(workspace, run_dir, events_path, reason="initial_pre_upload") apply_deterministic_diffusers_lora_dependency_policy(workspace, run_dir, events_path) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_artifact_manifest(run_dir, reason="workspace_uploaded") if selected_hardware == "default-cpu-or-existing": append_event(events_path, "hardware", "warning", "Automatic hardware-at-creation failed; Space is on default CPU unless user changes it manually", {"attempts": hardware_attempts}) write_json(run_dir / "hardware_attempts.json", {"selected_hardware": selected_hardware, "requested_sequence": requested_hardware_sequence, "attempts": hardware_attempts, "strategy": "create_repo_space_hardware_first", "try_zero_gpu_first": try_zero_gpu_first, "hardware_intent": hardware_intent}) write_json(run_dir / "hardware_strategy.json", {"selected_hardware": selected_hardware, "requested_sequence": requested_hardware_sequence, "attempts": hardware_attempts, "manual_action_required": selected_hardware == "default-cpu-or-existing" or bool(hardware_intent.get("manual_hardware_required")), "strategy": "create_repo_space_hardware_first", "try_zero_gpu_first": try_zero_gpu_first, "hardware_intent": hardware_intent}) try: validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) except Exception as validation_error: append_event(events_path, "failure_detected", "warning", "Initial live validation failed; entering Pi blockage diagnosis protocol", {"error": str(validation_error)[:2000]}) validation = recover_after_live_validation_failure( api, workspace, run_dir, events_path, pi_model=pi_model, target_space_id=target_space_id, model_id=model_id, token=token, failure_reason=str(validation_error), implementation_mode=implementation_mode, expected_output_type=effective_expected_output_type, ) generation_smoke = None if implementation_mode in {"full-inference-gated", "full-inference-attempt"}: contract_skip_reason = contract_declares_no_full_inference(workspace) if contract_skip_reason.get("declared"): generation_smoke = write_contract_skipped_generation_smoke(run_dir, events_path, effective_expected_output_type, target_space_id, contract_skip_reason) else: try: wait_for_video_smoke_settle(api, target_space_id, token, run_dir, events_path, effective_expected_output_type) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, effective_expected_output_type, workspace=workspace) except Exception as smoke_error: write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation smoke test failed", data={"error": str(smoke_error)[:2000]}) smoke_diagnosis = diagnose_generation_smoke_failure(smoke_error, inference_strategy=str((read_inference_contract(workspace) or {}).get("inference_strategy") or ""), expected_output_type=effective_expected_output_type, phase="exception") generation_smoke = build_generation_smoke_failure_payload(target_space_id, effective_expected_output_type, smoke_error, workspace=workspace, phase="exception") write_json(run_dir / "tests" / "generation_smoke_diagnosis.json", smoke_diagnosis) record_generation_smoke_result(run_dir, generation_smoke, phase="generation_smoke_exception") append_event(events_path, "generation_smoke", "failed", "Live generation smoke test failed; ZeroGPU duration was not measured", generation_smoke) else: generation_smoke = measured_zero_gpu_recommendation(None) | {"status": "skipped", "expected_output_type": effective_expected_output_type} write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke) if isinstance(generation_smoke, dict) and generation_smoke.get("status") == "failed": validation, generation_smoke = attempt_generation_smoke_repair( api, workspace, run_dir, events_path, pi_model=pi_model, target_space_id=target_space_id, model_id=model_id, token=token, implementation_mode=implementation_mode, expected_output_type=effective_expected_output_type, validation=validation, generation_smoke=generation_smoke, ) inference_gate = infer_generation_gate(workspace, implementation_mode, validation, generation_smoke, run_dir, events_path) if isinstance(inference_gate, dict): inference_gate["hardware_intent"] = hardware_intent if hardware_intent.get("manual_hardware_required"): inference_gate["manual_hardware_required"] = True inference_gate["hardware_blocker_reason"] = hardware_intent.get("reason") write_json(run_dir / "inference_gate.json", inference_gate) # If the generated app looks like real GPU inference but automatic # hardware requests failed, classify the run honestly as needing manual # hardware instead of pretending CPU/default hardware is enough. the existing-Space validation workflow # can then smoke-test generation after the user sets a GPU manually. manual_hw_required = selected_hardware == "default-cpu-or-existing" and inference_gate.get("status") not in {"technical_blocker", "technical_blocker_boot_only", "health_only"} and ( inference_gate.get("implementation_signals", {}).get("has_spaces_gpu") or inference_gate.get("implementation_signals", {}).get("has_torch") or any((a.get("manual_action_required") for a in hardware_attempts if isinstance(a, dict))) ) if manual_hw_required: inference_gate = dict(inference_gate) inference_gate["status"] = "manual_hardware_required" inference_gate["message"] = "Space was generated and boots, but automatic ZeroGPU/fixed-GPU assignment failed. Set hardware manually, then run the existing-Space validation workflow." inference_gate["manual_hardware_required"] = True inference_gate["hardware_attempts"] = hardware_attempts write_json(run_dir / "inference_gate.json", inference_gate) append_event(events_path, "inference_gate", "manual_hardware_required", inference_gate["message"], inference_gate) if manual_hardware_actionable: inference_gate = dict(inference_gate) inference_gate["status"] = "manual_hardware_required" inference_gate["message"] = "Space was created with a safe manual-hardware action app. Open the Space settings, select the recommended GPU, restart, then re-run validation." inference_gate["manual_hardware_required"] = True inference_gate["manual_hardware_actionable"] = True inference_gate["manual_hardware_actionable_space"] = manual_hardware_actionable_payload inference_gate["hardware_attempts"] = hardware_attempts write_json(run_dir / "inference_gate.json", inference_gate) append_event(events_path, "inference_gate", "manual_hardware_required", inference_gate["message"], inference_gate) collect_pi_traces(run_dir, events_path) final_state = { "run_id": run_id, "kind": "universal_model_card_builder", "status": inference_gate["status"], "message": inference_gate["message"], "model_id": model_id, "pi_model": pi_model, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "selected_hardware": selected_hardware, "hardware_intent": hardware_intent, "hardware_attempts": hardware_attempts, "model_expected_output_type": output_type_resolution.get("model_expected_output_type"), "contract_effective_output_type": output_type_resolution.get("contract_effective_output_type"), "expected_output_type": effective_expected_output_type, "output_type_resolution": output_type_resolution, "pi_model_resolution": pi_model_resolution, "validation": validation, "generation_smoke": generation_smoke, "inference_gate": inference_gate, "app_boot_validation_status": inference_gate.get("app_boot_validation_status"), "promise_validation_status": inference_gate.get("promise_validation_status"), "promise_fulfilled": bool(inference_gate.get("promise_fulfilled")), "promise_validation": inference_gate.get("promise_validation") or {}, "ui_status": inference_gate.get("ui_status") or inference_gate.get("status"), "ui_badge": inference_gate.get("ui_badge") or "", "manual_hardware_actionable": bool(manual_hardware_actionable), "manual_hardware_actionable_space": manual_hardware_actionable_payload, "updated_at": now(), "created_by": hf_username, "bucket_source": bucket_source, } write_json(state_path, final_state) cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, final_state, token=token) final_state["cleanup"] = cleanup_status final_state = reconcile_final_success_state(run_dir, events_path, final_state, inference_gate, generation_smoke) write_final_summary(run_dir, final_state, inference_gate, generation_smoke) write_live_status(run_dir, stage="done", status=inference_gate["status"], message=inference_gate["message"], data={"target_space": target_space_id, "selected_hardware": selected_hardware, "generation_smoke_status": (generation_smoke or {}).get("status"), "cleanup": cleanup_status, "job_exit_code": final_state.get("job_exit_code")}) report = f"""# Agentic Space Factory — Universal Model-Card Builder Report Run ID: `{run_id}` Status: **{inference_gate['status']}** {inference_gate['message']} Target Space: https://huggingface.co/spaces/{target_space_id} Model: `{model_id}` ## Pi assistant model ```json {json.dumps(pi_model_resolution, indent=2, ensure_ascii=False)} ``` ## Hardware Selected/requested hardware: `{selected_hardware}` Hardware intent: ```json {json.dumps(hardware_intent, indent=2, ensure_ascii=False)} ``` Hardware changes are best-effort with OAuth. If requests fail with 401/auth/billing errors, set the Space hardware manually and rerun validation. ```json {json.dumps(hardware_attempts, indent=2, ensure_ascii=False)} ``` ## Health validation The wrapper validated the live Space using HTTP `/health` first, with Gradio Client as fallback. This only proves bootability. ```json {json.dumps(validation, indent=2, ensure_ascii=False)} ``` ## Live generation smoke / ZeroGPU duration ```json {json.dumps(generation_smoke, indent=2, ensure_ascii=False)} ``` ## Next action {(generation_smoke or {}).get('next_action') or ('The generated Space is running but generation was not fully verified. Use Prefill Space Test to retry with adjusted arguments.' if inference_gate.get('status') == 'full_inference_candidate_health_passed' else 'No manual action is required for this verdict.')} ## Full-inference gate ```json {json.dumps(inference_gate, indent=2, ensure_ascii=False)} ``` ## Pi summary {(workspace / 'PI_SUMMARY.md').read_text(encoding='utf-8', errors='ignore') if (workspace / 'PI_SUMMARY.md').exists() else 'No PI_SUMMARY.md was produced.'} ## Safety - The target Space was created private. - No public publication was attempted. - Only redacted Pi traces are archived. Raw Pi traces are not published to the bucket. - If fallback fixed GPU was used or selected manually, review billing/hardware settings manually after the run. """ (run_dir / "report.md").write_text(report, encoding="utf-8") append_event(events_path, "report_write", "success", "Wrote report.md") write_artifact_manifest(run_dir, events_path=events_path, reason="final") try: publish_eval_record(run_dir, phase="final", events_path=events_path) append_event(events_path, "anonymous_eval", "success", "Published anonymized evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False}) except Exception as eval_exc: append_event(events_path, "anonymous_eval", "warning", "Could not publish anonymized evaluation record", {"error": str(eval_exc)[:1000]}) append_event(events_path, "done", inference_gate["status"], "Universal model-card builder completed", {"target_space": target_space_id, "selected_hardware": selected_hardware, "gate_status": inference_gate["status"]}) except SystemExit: raise except Exception as exc: try: collect_pi_traces(run_dir, events_path) except Exception: pass if final_full_inference_success_recorded(run_dir): existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {} existing_gate = load_json_if_exists(run_dir / "inference_gate.json") if (run_dir / "inference_gate.json").exists() else {} existing_smoke = load_json_if_exists(run_dir / "tests" / "generation_smoke.json") if (run_dir / "tests" / "generation_smoke.json").exists() else {} clean_state = reconcile_final_success_state(run_dir, events_path, existing_state, existing_gate, existing_smoke) append_event(events_path, "final_status_reconciliation", "warning", "Suppressed post-success finalizer exception; final full-inference gate remains authoritative", {"error": str(exc)[:2000], "job_exit_code": 0}) write_final_summary(run_dir, clean_state, existing_gate, existing_smoke, status="full_inference_success", message=clean_state.get("message") or existing_gate.get("message")) try: write_artifact_manifest(run_dir, events_path=events_path, reason="final_success_exception_suppressed") except Exception: pass return terminal_status = "auth_refresh_required" if isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc) else "failed" details = {"error": str(exc)} if terminal_status == "auth_refresh_required": details["failure_type"] = "auth_refresh_required" repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {} if isinstance(repair_outcome, dict) and repair_outcome: details["repair_outcome"] = repair_outcome details["failure_type"] = repair_outcome.get("failure_type") or details.get("failure_type", "") fail(run_dir, events_path, "Universal model-card builder worker failed", details, status=terminal_status) if __name__ == "__main__": main() ''' VALIDATE_EXISTING_SPACE_WORKER_SCRIPT = r''' import base64 import json import os import re import shutil import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path TARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$") def now(): return datetime.now(timezone.utc).isoformat() def write_json(path: Path, payload: dict): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2, ensure_ascii=False, default=str) + "\n", encoding="utf-8") def read_json(path: Path, default=None): try: if path.exists(): return json.loads(path.read_text(encoding="utf-8")) except Exception: pass return default def write_live_status(run_dir: Path, *, stage: str, status: str = "running", message: str = "", data: dict | None = None): """Persist a tiny status snapshot from the validation worker. v190.32: this helper must exist inside the linked-validation worker script itself. Earlier validation Jobs called write_live_status() after API discovery but the helper was only present in the build worker, causing a NameError before payload resolution. """ payload = { "schema_version": "live_status.v1", "updated_at": now(), "run_id": os.environ.get("RUN_ID", ""), "stage": stage, "status": status, "message": message, "data": data or {}, } try: write_json(run_dir / "live_status.json", payload) except Exception: pass return payload def _first_list_value(*values): for value in values: if isinstance(value, list): return value return None def _first_dict_value(*values): for value in values: if isinstance(value, dict): return value return None def load_parent_replay_source(parent_dir: Path, target_space_id: str, expected_output_type: str) -> dict: """Load the parent build's successful automatic smoke payload as replay truth. In linked Space Test replay mode, the validation worker must not invent a different request when the parent Build Run already proved a working request. The parent smoke artefacts are the source of truth; schema discovery is only used afterwards as a guardrail/retry mechanism. """ if not parent_dir or not parent_dir.exists(): return {} smoke = read_json(parent_dir / "tests" / "generation_smoke.json", {}) or {} retry_payload = read_json(parent_dir / "tests" / "generation_smoke_payload_retry.json", {}) or {} initial_payload = read_json(parent_dir / "tests" / "generation_smoke_payload.json", {}) or {} state = read_json(parent_dir / "state.json", {}) or {} state_smoke = state.get("generation_smoke") if isinstance(state.get("generation_smoke"), dict) else {} status_tokens = { str(smoke.get("status") or "").lower(), str(state.get("status") or "").lower(), str(state_smoke.get("status") or "").lower(), } parent_was_successful = bool(status_tokens & {"success", "full_inference_success", "completed", "complete"}) if not parent_was_successful: return {} api_name = ( smoke.get("api_name") or retry_payload.get("api_name") or initial_payload.get("api_name") or state_smoke.get("api_name") or "" ) test_args = _first_list_value( smoke.get("effective_args"), smoke.get("test_args"), retry_payload.get("test_args"), initial_payload.get("test_args"), state_smoke.get("effective_args"), state_smoke.get("test_args"), ) test_kwargs = _first_dict_value( smoke.get("effective_kwargs"), smoke.get("test_kwargs"), retry_payload.get("test_kwargs"), initial_payload.get("test_kwargs"), state_smoke.get("effective_kwargs"), state_smoke.get("test_kwargs"), ) or {} if not api_name or not isinstance(test_args, list): return {} parent_target = smoke.get("target_space") or state.get("target_space") or state_smoke.get("target_space") or "" return { "source": "parent_automatic_smoke", "parent_run_id": parent_dir.name, "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": test_args, "test_kwargs": test_kwargs, "expected_output_type": smoke.get("expected_output_type") or state_smoke.get("expected_output_type") or expected_output_type, "latency_seconds": smoke.get("latency_seconds") or state_smoke.get("latency_seconds"), "parent_smoke_status": smoke.get("status") or state_smoke.get("status") or state.get("status"), "initial_payload_present": bool(initial_payload), "retry_payload_present": bool(retry_payload), } def load_env_replay_source(target_space_id: str, expected_output_type: str) -> dict: """Replay source passed by backend through the validation Job env. The validation Job starts in a fresh workspace; the parent Build Run folder is usually not mounted locally. This env payload is therefore the canonical transport for replaying a successful parent automatic smoke request. """ raw = os.environ.get("PARENT_REPLAY_SOURCE_JSON") or "" if not raw.strip(): return {} try: data = json.loads(raw) except Exception as exc: return {"source_error": f"invalid_parent_replay_source_json:{exc}"} if not isinstance(data, dict): return {"source_error": "parent_replay_source_json_not_object"} args = data.get("test_args") kwargs = data.get("test_kwargs") or {} api_name = data.get("api_name") or "" if not api_name or not isinstance(args, list) or not isinstance(kwargs, dict): return {"source_error": "parent_replay_source_missing_api_args_or_kwargs", "raw_keys": sorted(data.keys())} parent_target = data.get("parent_target_space") or "" return { "source": data.get("source") or "parent_automatic_smoke_backend", "transport": "job_env", "parent_run_id": data.get("parent_run_id") or os.environ.get("PARENT_BUILD_RUN_ID", ""), "parent_target_space": parent_target, "target_matches": not parent_target or parent_target == target_space_id, "api_name": normalize_api_name(str(api_name)), "test_args": args, "test_kwargs": kwargs, "expected_output_type": data.get("expected_output_type") or expected_output_type or "any", "latency_seconds": data.get("latency_seconds"), "parent_smoke_status": data.get("parent_smoke_status") or "success", "backend_replay_source_used": True, } def parent_has_successful_manual_validation(parent_dir: Path) -> bool: manual = read_json(parent_dir / "manual_validation_status.json", {}) or {} if str(manual.get("status") or "").lower() == "success": return True linked = read_json(parent_dir / "linked_validations.json", {}) or {} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] success_tokens = {"success", "full_inference_success", "passed", "validated_after_manual_space_test"} return any(str(row.get("status") or row.get("effective_status") or "").lower() in success_tokens for row in validations if isinstance(row, dict)) def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None): path.parent.mkdir(parents=True, exist_ok=True) event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}} line = json.dumps(event, ensure_ascii=False) with path.open("a", encoding="utf-8") as f: f.write(line + "\n") print(line, flush=True) SAFE_SPACE_PYTHON_VERSIONS = {"3.10", "3.11", "3.12"} DEFAULT_SPACE_PYTHON_VERSION = "3.10" def normalize_space_python_version(value) -> tuple[str, bool, str]: raw = str(value or "").strip().strip("\"'") lowered = raw.lower().replace("python", "").strip() match = re.search(r"(3)\.(\d+)(?:\.\d+)?", lowered) if match: normalized = f"{match.group(1)}.{match.group(2)}" else: normalized = lowered if normalized in SAFE_SPACE_PYTHON_VERSIONS: return normalized, normalized != raw, "allowed" return DEFAULT_SPACE_PYTHON_VERSION, True, f"unsupported_or_ambiguous:{raw or 'missing'}" def requirements_has_package(lines: list[str], package: str) -> bool: wanted = package.lower().replace("_", "-") for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped: continue name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-") if name == wanted: return True return False def workspace_app_imports_torch(workspace: Path) -> bool: app_path = workspace / "app.py" if not app_path.exists(): return False text = app_path.read_text(encoding="utf-8", errors="ignore") return bool(re.search(r"(?m)^\s*(import\s+torch\b|from\s+torch\b)", text)) def redact_text(text: str | None) -> str: if not text: return "" value = text for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]: secret = os.environ.get(secret_name) if secret: value = value.replace(secret, "[REDACTED]") value = re.sub(r"Bearer\s+[A-Za-z0-9_\-.=]+", "Bearer [REDACTED]", value) value = re.sub(r"hf_[A-Za-z0-9_\-]{10,}", "hf_[REDACTED]", value) return value def eval_enabled() -> bool: return os.environ.get("ASF_EVAL_RECORD_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"} def eval_safe_hash(value: str | None, prefix: str = "h") -> str: raw = (value or "").strip() salt = os.environ.get("ASF_EVAL_SALT", "").strip() if not raw: return "" if salt: digest = hmac.new(salt.encode("utf-8"), raw.encode("utf-8"), hashlib.sha256).hexdigest()[:24] else: digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] return f"{prefix}_{digest}" def eval_load_json(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8", errors="replace")) except Exception as exc: return {"_parse_error": str(exc)} def eval_load_events(path: Path) -> list[dict]: if not path.exists(): return [] events = [] for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): try: evt = json.loads(line) if isinstance(evt, dict): events.append(evt) except Exception: pass return events def eval_parse_ts(value: str | None) -> float | None: if not value: return None try: return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() except Exception: return None def eval_compact_timeline(events: list[dict]) -> list[dict]: stages: dict[str, dict] = {} order: list[str] = [] for evt in events: step = str(evt.get("step") or "unknown")[:80] status = str(evt.get("status") or "")[:80] ts = str(evt.get("ts") or "") tsv = eval_parse_ts(ts) if step not in stages: stages[step] = {"stage": step, "first_ts": ts, "last_ts": ts, "status": status, "event_count": 0} order.append(step) item = stages[step] item["last_ts"] = ts or item.get("last_ts") item["status"] = status or item.get("status") item["event_count"] = int(item.get("event_count") or 0) + 1 if tsv is not None: if "_first" not in item: item["_first"] = tsv item["_last"] = tsv timeline = [] for step in order: item = dict(stages[step]) first = item.pop("_first", None) last = item.pop("_last", None) if first is not None and last is not None and last >= first: item["duration_ms"] = int((last - first) * 1000) timeline.append(item) return timeline[-80:] def eval_redacted_tail(path: Path, limit: int = 2400) -> str: if os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() not in {"1", "true", "yes", "on"}: return "" try: return redact_text(path.read_text(encoding="utf-8", errors="replace"))[-limit:] except Exception: return "" def eval_len_if_list(value) -> int: return len(value) if isinstance(value, list) else 0 def eval_seconds_bucket(value) -> str: try: seconds = int(value) except Exception: return "unknown" if seconds < 0: return "expired" if seconds < 15 * 60: return "lt_15m" if seconds < 60 * 60: return "lt_1h" if seconds < 4 * 60 * 60: return "lt_4h" if seconds < 8 * 60 * 60: return "lt_8h" return "gte_8h" def eval_v191_plus_signals(run_dir: Path, analysis: dict, generation_smoke: dict, *, contract: dict | None = None, requirements_policy: dict | None = None, auth_status: dict | None = None, repair_outcome: dict | None = None, worker_plan_review: dict | None = None, grounding_review: dict | None = None) -> dict: """Return privacy-safe metrics for features added from v191.1 onward.""" analysis = analysis if isinstance(analysis, dict) else {} generation_smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = contract if isinstance(contract, dict) else {} requirements_policy = requirements_policy if isinstance(requirements_policy, dict) else {} auth_status = auth_status if isinstance(auth_status, dict) else {} repair_outcome = repair_outcome if isinstance(repair_outcome, dict) else {} worker_plan_review = worker_plan_review if isinstance(worker_plan_review, dict) else {} grounding_review = grounding_review if isinstance(grounding_review, dict) else {} build_risk = analysis.get("build_risk") if isinstance(analysis.get("build_risk"), dict) else {} kernel_strategy = analysis.get("kernel_strategy") if isinstance(analysis.get("kernel_strategy"), dict) else {} if not kernel_strategy and isinstance(analysis.get("metadata"), dict): kernel_strategy = analysis["metadata"].get("kernel_strategy") if isinstance(analysis["metadata"].get("kernel_strategy"), dict) else {} grounding_source = grounding_review.get("model_card_source") if isinstance(grounding_review.get("model_card_source"), dict) else {} return { "schema_version": "v191_plus_eval_signals.v1", "platform_dependency_policy": { "present": bool(requirements_policy), "status": requirements_policy.get("status") or "", "removed_platform_pin_count": eval_len_if_list(requirements_policy.get("removed_pins")), "normalized_platform_line_count": eval_len_if_list(requirements_policy.get("normalized_platform_lines")), "injected_platform_line_count": eval_len_if_list(requirements_policy.get("injected_platform_lines")), "torch_added": bool(requirements_policy.get("torch_added")), }, "auth_context": { "present": bool(auth_status), "status": auth_status.get("status") or "", "token_kind": auth_status.get("token_kind") or "", "expiry_known": bool(auth_status.get("expiry_known")), "seconds_until_expiry_bucket": eval_seconds_bucket(auth_status.get("seconds_until_expiry")), "safe_for_phase": bool(auth_status.get("safe_for_phase")), }, "model_scan": { "build_risk_level": build_risk.get("level") or analysis.get("build_risk_level") or "", "build_risk_signal_count": eval_len_if_list(build_risk.get("signals")), "build_risk_visibility_only": bool(build_risk.get("visibility_only", True)) if build_risk else True, "recommended_session_minutes": build_risk.get("recommended_session_minutes"), "kernel_strategy_present": bool(kernel_strategy), "native_kernel_detected": bool(kernel_strategy.get("native_kernel_detected") or kernel_strategy.get("detected")), "kernel_signal_count": eval_len_if_list(kernel_strategy.get("signals")), "kernel_candidate_count": eval_len_if_list(kernel_strategy.get("candidates")), }, "contract_validation": { "contract_present": bool(contract), "full_inference_implemented": bool(contract.get("full_inference_implemented")), "validation_level": contract.get("validation_level") or "", "requires_gpu": bool(contract.get("requires_gpu")), "blockers_count": int(contract.get("blockers_count") or 0) if str(contract.get("blockers_count") or "0").isdigit() else 0, "generation_smoke_status": generation_smoke.get("status") or "", "generation_smoke_skipped": str(generation_smoke.get("status") or "").lower() == "skipped", "generation_smoke_skip_reason": generation_smoke.get("skip_reason") or "", }, "repair_outcome": { "present": bool(repair_outcome), "repair_decision": repair_outcome.get("repair_decision") or repair_outcome.get("decision") or "", "patch_applied": bool(repair_outcome.get("patch_applied")), "upload_success": bool(repair_outcome.get("upload_success")), "post_repair_validation": repair_outcome.get("post_repair_validation") or "", "failure_type": repair_outcome.get("failure_type") or "", }, "planning": { "worker_plan_review_present": bool(worker_plan_review), "status": worker_plan_review.get("status") or "", "declared_strategy": worker_plan_review.get("declared_strategy") or "", "worker_recommendation": worker_plan_review.get("worker_recommendation") or "", "warning_count": eval_len_if_list(worker_plan_review.get("warnings")), }, "model_card_grounding": { "present": bool(grounding_review), "status": grounding_review.get("status") or "", "source_available": bool(grounding_review.get("source_available")), "model_card_present": bool(grounding_review.get("model_card_present")), "source": grounding_source.get("source") or "", "resolved_card_file": grounding_source.get("resolved_card_file") or "", "fallback_used": bool(grounding_source.get("fallback_used")), "pi_evidence_present": bool(grounding_review.get("pi_evidence_present")), "pi_evidence_count": int(grounding_review.get("pi_evidence_count") or 0), "warning_count": eval_len_if_list(grounding_review.get("warnings")), "warnings": [str(w)[:120] for w in (grounding_review.get("warnings") if isinstance(grounding_review.get("warnings"), list) else [])[:8]], }, } def eval_terminal_status(status: str, phase: str = "") -> bool: status = str(status or "").strip().lower() phase = str(phase or "").strip().lower() if phase in {"final", "failure"}: return True return status in { "done", "success", "failed", "cancelled", "canceled", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "demo_usable_full_promise_not_verified", "interactive_app_available_smoke_failed", "manual_test_required_smoke_failed", "partial_validation", "technical_blocker", "manual_hardware_required", } def eval_event_seen(events: list[dict], step: str, status: str | None = None) -> bool: for event in events or []: if str(event.get("step") or "") != step: continue if status is None or str(event.get("status") or "") == status: return True return False def eval_health_passed(inference_gate: dict, generation_smoke: dict, events: list[dict], status: str) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} if signals.get("health_passed") is True: return True if str(status or "") in {"full_inference_success", "full_inference_candidate_health_passed", "health_only"}: return True if eval_event_seen(events, "api_validation", "success") or eval_event_seen(events, "repair_validation", "success"): return True return False def eval_generation_smoke_passed(inference_gate: dict, generation_smoke: dict) -> bool: signals = inference_gate.get("implementation_signals") if isinstance(inference_gate.get("implementation_signals"), dict) else {} return bool(signals.get("generation_smoke_passed") is True or generation_smoke.get("status") == "success") def eval_failure_summary(status: str, state: dict, inference_gate: dict, generation_smoke: dict, health_passed: bool, smoke_passed: bool) -> dict: failure_type = generation_smoke.get("failure_type") or inference_gate.get("failure_type") or "" failure_owner = generation_smoke.get("failure_owner") or inference_gate.get("failure_owner") or "" missing_argument = generation_smoke.get("missing_argument") or inference_gate.get("missing_argument") or "" reason = missing_argument or failure_type or str(state.get("message") or inference_gate.get("message") or "")[:240] if not failure_owner: if failure_type == "validator_request_error" or missing_argument: failure_owner = "factory_validator" elif smoke_passed: failure_owner = "" elif health_passed and str(generation_smoke.get("status") or "") == "failed": failure_owner = "unknown" return { "failure_type": failure_type, "failure_owner": failure_owner, "failure_reason": reason, "missing_argument": missing_argument, } def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: bool, full_inference_verified: bool, inference_gate: dict) -> str: status = str(status or "").strip() if status in {"cancelled", "canceled"}: return "cancelled" if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"): return "manual_action_required" if status in {"technical_blocker", "technical_blocker_boot_only"}: return "technical_blocker" if full_inference_verified: return "success" if status in {"full_inference_candidate_health_passed", "health_only"} or (health_passed and not smoke_passed): return "partial_validation" if phase == "failure" or status in {"failed", "error"}: return "failed" if eval_terminal_status(status, phase): return "partial_validation" if health_passed else "failed" return "running" def publish_eval_record(run_dir: Path, *, phase: str, events_path: Path | None = None) -> dict: """Write an anonymized cross-run evaluation record to the optional eval bucket. This intentionally stores structured metrics and hashed identifiers, not raw generated code, prompts, tokens, user bucket paths, or target Space IDs. """ if not eval_enabled(): return {"enabled": False} # Jobs only write the anonymized eval record back into the user's run bucket. # The ASF backend later publishes this record to the operator eval bucket # mounted on the ASF Space. User Jobs must never mount or write the private # operator archive directly. run_id = os.environ.get("RUN_ID", run_dir.name) events = eval_load_events(events_path or (run_dir / "events.jsonl")) state = eval_load_json(run_dir / "state.json") analysis = eval_load_json(run_dir / "model_analysis.json") hardware_strategy = eval_load_json(run_dir / "hardware_strategy.json") hardware_attempts = eval_load_json(run_dir / "hardware_attempts.json") inference_gate = eval_load_json(run_dir / "inference_gate.json") generation_smoke = eval_load_json(run_dir / "tests" / "generation_smoke.json") generation_smoke_retry = eval_load_json(run_dir / "tests" / "generation_smoke_payload_retry.json") endpoint_discovery_payload = eval_load_json(run_dir / "tests" / "gradio_endpoint_discovery.json") manual_validation_status = eval_load_json(run_dir / "manual_validation_status.json") linked_validations_payload = eval_load_json(run_dir / "linked_validations.json") space_runtime = eval_load_json(run_dir / "space_runtime.json") manifest = eval_load_json(run_dir / "artifact_manifest.json") repair_decision = eval_load_json(run_dir / "repair" / "REPAIR_DECISION.json") blockage = eval_load_json(run_dir / "repair" / "BLOCKAGE.json") repair_outcome = eval_load_json(run_dir / "repair_outcome.json") or eval_load_json(run_dir / "repair" / "REPAIR_OUTCOME.json") requirements_policy = eval_load_json(run_dir / "generated" / "requirements_policy.json") or eval_load_json(run_dir / "requirements_policy.json") auth_status = eval_load_json(run_dir / "auth_status.json") contract = eval_load_json(run_dir / "generated" / "INFERENCE_CONTRACT.json") worker_plan_review = eval_load_json(run_dir / "planning" / "worker_plan_review.json") grounding_review = eval_load_json(run_dir / "planning" / "model_card_grounding_review.json") or (worker_plan_review.get("model_card_grounding") if isinstance(worker_plan_review.get("model_card_grounding"), dict) else {}) pi_resolution = state.get("pi_model_resolution") if isinstance(state.get("pi_model_resolution"), dict) else {} status = str(state.get("status") or inference_gate.get("status") or phase or "unknown") failure_message = str(state.get("message") or (state.get("details") or {}).get("error") or "")[:1200] started_at = events[0].get("ts") if events else (state.get("created_at") or os.environ.get("LAUNCHED_AT") or "") finished_at = state.get("updated_at") or (events[-1].get("ts") if events else now()) model_id = os.environ.get("MODEL_ID") or state.get("model_id") or analysis.get("model_id") or "" target_space = state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or "" include_model_id = os.environ.get("ASF_EVAL_INCLUDE_MODEL_ID", "").strip().lower() in {"1", "true", "yes", "on"} anon_run_id = eval_safe_hash(run_id, "run") health_passed = eval_health_passed(inference_gate, generation_smoke, events, status) generation_smoke_passed = eval_generation_smoke_passed(inference_gate, generation_smoke) promise_validation = inference_gate.get("promise_validation") if isinstance(inference_gate.get("promise_validation"), dict) else {} promise_fulfilled = bool(promise_validation.get("promise_fulfilled") or inference_gate.get("promise_fulfilled") or status == "full_inference_success") full_inference_verified = bool(promise_fulfilled and status == "full_inference_success" and health_passed and generation_smoke_passed) process_completed = eval_terminal_status(status, phase) failure_summary = eval_failure_summary(status, state, inference_gate, generation_smoke, health_passed, generation_smoke_passed) verdict = eval_verdict(status, phase, health_passed, generation_smoke_passed, full_inference_verified, inference_gate) linked_rows = linked_validations_payload.get("validations") if isinstance(linked_validations_payload.get("validations"), list) else [] linked_success_rows = [row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() == "success"] manual_applied = str(manual_validation_status.get("status") or "").lower() == "success" or bool(linked_success_rows) effective_status = str((manual_validation_status if manual_applied else {}).get("effective_status") or (linked_success_rows[-1].get("effective_status") if linked_success_rows else "") or ("success" if full_inference_verified else verdict)) endpoint_names = endpoint_discovery_payload.get("discovered_api_names") or endpoint_discovery_payload.get("candidates") or [] smoke_retry_meta = generation_smoke.get("auto_retry") if isinstance(generation_smoke.get("auto_retry"), dict) else {} record = { "process_completed": process_completed, "verdict": verdict, "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "schema_version": "1.4", "app_version": os.environ.get("ASF_VERSION", "unknown"), "phase": phase, "anonymous_run_id": anon_run_id, "run_id_hash": eval_safe_hash(run_id, "run"), "anonymous_user_hash": eval_safe_hash(os.environ.get("HF_USERNAME"), "user"), "salt_configured": bool(os.environ.get("ASF_EVAL_SALT")), "run_kind": state.get("kind") or "linked_space_validation", "started_at": started_at, "finished_at": finished_at, "input_model": { "model_id_redacted": not include_model_id, "model_id": model_id if include_model_id else "", "model_hash": eval_safe_hash(model_id, "model"), "pipeline_tag": analysis.get("pipeline_tag"), "library_name": analysis.get("library_name"), "default_model_target": bool(analysis.get("default_model_target")), }, "config": { "implementation_mode": os.environ.get("IMPLEMENTATION_MODE", ""), "expected_output_type": os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "pi_model_requested": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "pi_model_effective": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "try_zero_gpu_first": os.environ.get("TRY_ZERO_GPU_FIRST", ""), "allow_fixed_gpu_fallback": os.environ.get("ALLOW_FIXED_GPU_FALLBACK", ""), "preferred_gpu": os.environ.get("PREFERRED_SPACE_HARDWARE", ""), "fallback_gpu": os.environ.get("FALLBACK_SPACE_HARDWARE", ""), }, "outcome": { "status": status, "verdict": verdict, "process_completed": process_completed, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "app_boot_validation_status": inference_gate.get("app_boot_validation_status") or "", "promise_validation_status": inference_gate.get("promise_validation_status") or promise_validation.get("promise_validation_status") or "", "promise_fulfilled": bool(promise_fulfilled), "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "message_redacted": redact_text(failure_message)[:1200], "target_space_hash": eval_safe_hash(target_space, "space"), "space_created": bool(target_space), "space_runtime_stage": space_runtime.get("stage"), "space_runtime_hardware": space_runtime.get("hardware") or space_runtime.get("requested_hardware"), }, "timeline": eval_compact_timeline(events), "hardware": { "selected": hardware_strategy.get("selected_hardware") or state.get("selected_hardware") or "", "strategy": hardware_strategy.get("strategy") or hardware_attempts.get("strategy") or "", "attempts": hardware_strategy.get("attempts") or hardware_attempts.get("attempts") or state.get("hardware_attempts") or [], "single_space_per_run": True, }, "pi": { "requested_model": os.environ.get("PI_MODEL") or state.get("pi_model") or "", "effective_model": pi_resolution.get("effective_model") or pi_resolution.get("observed_model") or "", "model_mismatch": bool(pi_resolution.get("mismatch")), "build_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_output.txt"), "diagnosis_log_tail_redacted": eval_redacted_tail(run_dir / "logs" / "pi_diagnosis_output.txt"), }, "recovery": { "entered": any(str(e.get("step", "")).startswith(("repair", "failure_diagnosis", "pi_diagnosis", "technical_blocker")) for e in events), "decision": repair_decision.get("action") or repair_decision.get("decision") or "", "confidence": repair_decision.get("confidence") or "", "blockage": bool(blockage), "blockage_category": blockage.get("category") or blockage.get("reason") or "", }, "validation": { "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, "generation_smoke_status": generation_smoke.get("status") or "", "latency_seconds": generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "expected_output_type": generation_smoke.get("expected_output_type") or os.environ.get("EXPECTED_OUTPUT_TYPE", ""), "inference_gate_status": inference_gate.get("status") or "", "manual_hardware_required": bool(inference_gate.get("manual_hardware_required")), "failure_type": failure_summary.get("failure_type") or "", "failure_owner": failure_summary.get("failure_owner") or "", "failure_reason": failure_summary.get("failure_reason") or "", "missing_argument": failure_summary.get("missing_argument") or "", }, "automatic_outcome": { "status": status, "verdict": verdict, "health_passed": health_passed, "generation_smoke_passed": generation_smoke_passed, "full_inference_verified": full_inference_verified, }, "effective_outcome": { "automatic_verdict": verdict, "automatic_status": status, "effective_verdict": effective_status, "effective_status": effective_status, "manual_validation_applied": manual_applied, "source": "linked_space_test" if manual_applied else "automatic_run", }, "linked_validation": { "present": bool(linked_rows or manual_validation_status), "count": len(linked_rows), "success_count": len(linked_success_rows), "failure_count": len([row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or "").lower() in {"failed", "failure", "error"}]), "status": "success" if manual_applied else str(manual_validation_status.get("status") or ""), "mode": manual_validation_status.get("space_test_policy_mode") or "", "api_name": manual_validation_status.get("api_name") or generation_smoke.get("api_name") or "", "latency_seconds": manual_validation_status.get("latency_seconds") or generation_smoke.get("latency_seconds") or generation_smoke.get("observed_latency_seconds"), "hardware_used_for_validation": manual_validation_status.get("hardware_used_for_validation") or "", }, "endpoint_discovery": { "required": bool(endpoint_discovery_payload.get("endpoint_discovery_required") or endpoint_discovery_payload), "succeeded": bool(endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint")), "selected_endpoint": endpoint_discovery_payload.get("selected_api_name") or endpoint_discovery_payload.get("selected_endpoint") or "", "candidate_count": len(endpoint_names) if isinstance(endpoint_names, list) else 0, "excluded_health_endpoint": True, }, "smoke_retry": { "retried": bool(generation_smoke_retry or smoke_retry_meta.get("retried")), "reason": generation_smoke_retry.get("retry_reason") or smoke_retry_meta.get("retry_reason") or smoke_retry_meta.get("reason") or "", "attempts": generation_smoke_retry.get("attempts") or smoke_retry_meta.get("attempts") or (2 if generation_smoke_retry else 1), "passed_after_retry": bool(generation_smoke_retry and generation_smoke.get("status") == "success"), }, "v191_plus": eval_v191_plus_signals( run_dir, analysis, generation_smoke, contract=contract, requirements_policy=requirements_policy, auth_status=auth_status, repair_outcome=repair_outcome, worker_plan_review=worker_plan_review, grounding_review=grounding_review, ), "artifacts": { "present_count": len(manifest.get("present_paths") or []), "present_paths": sorted(str(p) for p in (manifest.get("present_paths") or []))[:120], }, "privacy": { "tokens_stored": False, "raw_prompts_stored": False, "generated_code_stored": False, "target_space_redacted": True, "user_bucket_redacted": True, "redacted_tails_enabled": os.environ.get("ASF_EVAL_INCLUDE_REDACTED_TAILS", "").strip().lower() in {"1", "true", "yes", "on"}, "linked_run_ids_redacted": True, "endpoint_schema_stored": False, "validation_args_stored": False, "model_card_raw_stored": False, "pi_evidence_text_stored": False, "requirements_txt_stored": False, "auth_token_stored": False, }, } day = (finished_at or now())[:10] try: yyyy, mm, dd = day.split("-") except Exception: yyyy, mm, dd = now()[:10].split("-") dest = run_dir / "eval_record.json" write_json(dest, record) events_dest = run_dir / "events_compact.jsonl" events_dest.write_text("".join(json.dumps(item, ensure_ascii=False) + "\n" for item in record["timeline"]), encoding="utf-8") return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"} class AuthRefreshRequired(RuntimeError): """Raised when an OAuth/JWT token is expired or too close to expiry.""" def _decode_jwt_claims_unverified(token: str) -> dict: """Decode JWT claims without verification, only to inspect non-sensitive expiry metadata. This never returns or writes the raw token. Opaque PAT-style tokens are supported by returning an empty dict so they remain usable with an `unknown` expiry state. """ try: parts = (token or "").split(".") if len(parts) < 2: return {} payload = parts[1] payload += "=" * ((4 - len(payload) % 4) % 4) raw = base64.urlsafe_b64decode(payload.encode("utf-8")) data = json.loads(raw.decode("utf-8")) return data if isinstance(data, dict) else {} except Exception: return {} def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict: issued_at = now() if not token: return { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": False, "token_kind": "missing", "expiry_known": False, "status": "missing", "safe_for_phase": False, "minimum_required_seconds": minimum_required_seconds, } claims = _decode_jwt_claims_unverified(token) exp = claims.get("exp") if isinstance(claims, dict) else None iat = claims.get("iat") if isinstance(claims, dict) else None token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown") payload = { "schema_version": "auth_status.v1", "checked_at": issued_at, "token_present": True, "token_kind": token_kind, "expiry_known": exp is not None, "minimum_required_seconds": int(minimum_required_seconds or 0), "token_value": "[REDACTED]", } if iat is not None: try: payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat() except Exception: pass if exp is None: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"}) return payload try: exp_int = int(exp) seconds_left = exp_int - int(time.time()) payload.update({ "expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(), "seconds_until_expiry": seconds_left, }) except Exception: payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"}) return payload if seconds_left <= 0: payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"}) elif minimum_required_seconds and seconds_left < minimum_required_seconds: payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"}) else: payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"}) return payload def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict: payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds) payload["phase"] = phase safe_payload = {k: v for k, v in payload.items() if k != "token_value"} try: probes_dir = run_dir / "auth_probes" probes_dir.mkdir(parents=True, exist_ok=True) write_json(probes_dir / f"{phase}.json", safe_payload) write_json(run_dir / "auth_status.json", safe_payload) except Exception: pass status = str(payload.get("status") or "unknown") if events_path: event_status = "success" if payload.get("safe_for_phase") else "failed" if status == "unknown": event_status = "warning" append_event( events_path, "auth_probe", event_status, f"HF OAuth/token expiry check for {phase}: {status}", safe_payload, ) if raise_on_unsafe and not payload.get("safe_for_phase"): raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.") return payload def is_auth_expired_error(error: Exception | str) -> bool: text = str(error or "").lower() return any(marker in text for marker in [ "oauth token has expired", "exp claim timestamp check failed", "token has expired", "jwt expired", ]) def minimum_auth_seconds_for_phase(phase: str) -> int: env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_") raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "") if raw: try: return max(0, int(raw)) except Exception: pass defaults = { "before_space_create": 900, "before_upload": 900, "before_initial_validation": 1800, "before_repair": 1800, "before_repair_upload": 1800, "before_repair_validation": 1800, "before_linked_space_test": 1800, } return defaults.get(phase, 0) def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict: """Ensure Pi and HF tooling see the same private HF token safely. HF Jobs receive HF_TOKEN as a secret. Pi, huggingface_hub, gradio_client, and the hf CLI can use either HF_TOKEN or HUGGING_FACE_HUB_TOKEN depending on the tool/version, so expose both aliases inside the isolated Job process. Never write the token value to artifacts. """ token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "" if token: os.environ.setdefault("HF_TOKEN", token) os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token) os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation")) payload = { "hf_token_present": bool(token), "hf_token_length": len(token) if token else 0, "hugging_face_hub_token_alias_present": bool(os.environ.get("HUGGING_FACE_HUB_TOKEN")), "hf_username": os.environ.get("HF_USERNAME") or "", "bucket_source": os.environ.get("BUCKET_SOURCE") or "", "target_space_id": os.environ.get("TARGET_SPACE_ID") or "", "token_value": "[REDACTED]" if token else "", "token_kind": expiry.get("token_kind"), "expiry_known": expiry.get("expiry_known"), "expires_at": expiry.get("expires_at"), "seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"), "auth_risk": expiry.get("auth_risk"), "safe_for_long_build": bool(expiry.get("safe_for_phase")), "minimum_required_seconds": expiry.get("minimum_required_seconds"), } run_dir.mkdir(parents=True, exist_ok=True) write_json(run_dir / "token_context.json", payload) if events_path: append_event(events_path, "token_context", "success" if token else "warning", "Verified HF token context for Pi/HF operations", {k: v for k, v in payload.items() if k != "token_value"}) return payload def pi_tooling_context_note() -> str: return """HF tooling context: - You are running inside an HF Job with an HF_TOKEN secret configured. - For gated/private base models, LoRA adapters, PEFT adapters, or private Hub files, the generated Space must read `HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")` and pass `token=HF_TOKEN` to `from_pretrained`, `hf_hub_download`, `snapshot_download`, and `load_lora_weights` when those APIs are used. Never print the token. - For ZeroGPU/Diffusers/LoRA GPU apps, lazy-load heavy pipelines inside the `@spaces.GPU`-decorated inference function or a cache called from it. Do not download/load large gated models at module import before the GPU-decorated call. - The Factory exposes both HF_TOKEN and HUGGING_FACE_HUB_TOKEN aliases for huggingface_hub, gradio_client, hf CLI, and Pi provider access. - You may use the token through normal tools when needed to inspect the model/Space or validate private resources. - Never print, write, echo, or commit token values. Do not include Authorization headers in artifacts. - If a Hub operation fails with 401/403/quota/billing, report it as evidence instead of retrying blindly. """ def run_cmd(cmd: list[str], *, env: dict | None = None, timeout: int = 120): result = subprocess.run(cmd, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) return result.returncode, redact_text(result.stdout) def install_deps(events_path: Path): append_event(events_path, "dependencies", "started", "Installing validation dependencies") code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0", "requests>=2.31.0"], timeout=600) if code != 0: append_event(events_path, "dependencies", "failed", "Dependency installation failed", {"output_tail": out[-4000:]}) raise RuntimeError(out) append_event(events_path, "dependencies", "success", "Validation dependencies installed") def make_gradio_client(target_space_id: str, token: str, timeout_s: int | float | None = None): import inspect from gradio_client import Client params = inspect.signature(Client).parameters kwargs = {} if timeout_s and "httpx_kwargs" in params: kwargs["httpx_kwargs"] = {"timeout": float(timeout_s)} if "token" in params: return Client(target_space_id, token=token, **kwargs) if "hf_token" in params: return Client(target_space_id, hf_token=token, **kwargs) if "api_key" in params: return Client(target_space_id, api_key=token, **kwargs) if "headers" in params: return Client(target_space_id, headers={"Authorization": f"Bearer {token}"}, **kwargs) return Client(target_space_id, **kwargs) def api_names_from_schema(schema) -> list[str]: names: list[str] = [] def add(name): if not isinstance(name, str) or not name: return if not name.startswith("/"): name = "/" + name if name not in names: names.append(name) def walk(obj): if isinstance(obj, dict): for k, v in obj.items(): if k in {"api_name", "apiName"}: add(v) if isinstance(k, str) and k.startswith("/"): add(k) walk(v) elif isinstance(obj, list): for item in obj: walk(item) walk(schema) return names def normalize_api_name(name: str | None) -> str: value = (name or "").strip() if not value: return "/generate" return value if value.startswith("/") else "/" + value def endpoint_schema_for_api(schema, api_name: str): target = normalize_api_name(api_name) alternatives = {target, target.lstrip("/")} found = None def walk(obj): nonlocal found if found is not None: return if isinstance(obj, dict): for key, value in obj.items(): if isinstance(key, str) and key in alternatives and isinstance(value, (dict, list)): found = value return value = obj.get("api_name") or obj.get("apiName") if isinstance(value, str) and normalize_api_name(value) == target: found = obj return for value in obj.values(): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(schema) return found def endpoint_parameter_names(endpoint) -> list[str]: names: list[str] = [] def add(value): if not isinstance(value, str): return cleaned = value.strip() if cleaned and cleaned not in names: names.append(cleaned) def from_parameter(param): if isinstance(param, dict): for key in ["parameter_name", "parameterName", "name", "label"]: add(param.get(key)) component = param.get("component") or param.get("component_type") if isinstance(component, dict): for key in ["label", "name"]: add(component.get(key)) elif isinstance(param, str): add(param) def walk(obj): if isinstance(obj, dict): params = obj.get("parameters") or obj.get("inputs") if isinstance(params, list): for param in params: from_parameter(param) for value in obj.values(): if isinstance(value, (dict, list)): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(endpoint) return names def sanitize_kwargs_for_schema(api_name: str, schema, kwargs: dict): endpoint = endpoint_schema_for_api(schema, api_name) parameter_names = endpoint_parameter_names(endpoint) if not parameter_names: return kwargs, {}, [] allowed = set(parameter_names) sanitized = {key: value for key, value in kwargs.items() if key in allowed} dropped = {key: value for key, value in kwargs.items() if key not in allowed} return sanitized, dropped, parameter_names def endpoint_parameter_objects(endpoint) -> list: params: list = [] def add_param(param): if isinstance(param, (dict, str)): params.append(param) def walk(obj): if isinstance(obj, dict): direct = obj.get("parameters") or obj.get("inputs") or obj.get("input_components") or obj.get("components") if isinstance(direct, list): for param in direct: add_param(param) return for value in obj.values(): if isinstance(value, (dict, list)): walk(value) elif isinstance(obj, list): for item in obj: walk(item) walk(endpoint) return params def normalize_schema_param(param, index: int = 0) -> dict: """Return a dict-like Gradio parameter object for mixed schema entries. Some gradio_client view_api() payloads expose endpoint parameters as a heterogeneous list containing both rich dict objects and bare strings like "prompt" or "resolution". The validation worker must never call .get() directly on those bare strings; otherwise linked validation can fail during payload resolution before a request is written. """ if isinstance(param, dict): return param if isinstance(param, str): cleaned = param.strip() or f"arg{index}" return {"name": cleaned, "label": cleaned, "parameter_name": cleaned, "raw_schema_param_type": "str"} return {"name": f"arg{index}", "label": str(param), "raw_schema_param_type": type(param).__name__} def _safe_component_dict(param: dict) -> dict: component = param.get("component") if isinstance(param, dict) else {} return component if isinstance(component, dict) else {} def _schema_text(value) -> str: if value is None: return "" if isinstance(value, (str, int, float, bool)): return str(value) if isinstance(value, dict): for key in ["label", "name", "type", "component", "component_type", "type_name", "class_name", "__type__", "datatype", "python_type", "value"]: text = _schema_text(value.get(key)) if text: return text api_info = value.get("api_info") if isinstance(value.get("api_info"), dict) else {} for key in ["type", "component", "label"]: text = _schema_text(api_info.get(key)) if text: return text return "" def _schema_name(param, index: int) -> str: param = normalize_schema_param(param, index) for key in ["parameter_name", "parameterName", "name", "label", "id"]: text = _schema_text(param.get(key)) if text: return text component = _safe_component_dict(param) for key in ["label", "name"]: text = _schema_text(component.get(key)) if text: return text return f"arg{index}" def _schema_component(param) -> str: param = normalize_schema_param(param) return " ".join(filter(None, [ _schema_text(param.get("component")), _schema_text(param.get("component_type")), _schema_text(param.get("type")), _schema_text(param.get("datatype")), _schema_text(param.get("api_info")), ])).lower() def _schema_choice_values(param) -> list: """Return Gradio choice values while preserving their runtime type when possible.""" param = normalize_schema_param(param) api_info = param.get("api_info") if isinstance(param.get("api_info"), dict) else {} component = _safe_component_dict(param) raw = param.get("choices") or param.get("options") or api_info.get("choices") or component.get("choices") or [] if not isinstance(raw, list): return [] out = [] for choice in raw: if isinstance(choice, dict): if "value" in choice: out.append(choice.get("value")) continue if "label" in choice: out.append(choice.get("label")) continue if isinstance(choice, (list, tuple)) and choice: out.append(choice[0]) continue out.append(choice) return out def _schema_choices(param) -> list: out = [] for choice in _schema_choice_values(param): text = _schema_text(choice) if text: out.append(text) return out def _schema_prefers_string_choice(param) -> bool: param = normalize_schema_param(param) component = _schema_component(param) name = _schema_name(param, 0).lower() if any(token in component for token in ["dropdown", "radio", "choice", "select"]): return True api_info = param.get("api_info") if isinstance(param.get("api_info"), dict) else {} type_hint = " ".join([_schema_text(api_info.get(k)).lower() for k in ["type", "python_type", "datatype"]]) return "string" in type_hint and any(token in name for token in ["width", "height", "size", "resolution"]) def _schema_default(param): param = normalize_schema_param(param) sentinel = object() for key in ["default", "value", "example_input"]: value = param.get(key, sentinel) if value is not sentinel: return value api_info = param.get("api_info") if isinstance(param.get("api_info"), dict) else {} if "default" in api_info: return api_info.get("default") component = _safe_component_dict(param) if "value" in component: return component.get("value") return None def _coerce_number(value, fallback): try: if value is None or value == "": return fallback number = float(value) return int(number) if number.is_integer() else number except Exception: return fallback def autofill_value_for_schema_param(param, index: int): name = _schema_name(param, index).lower().replace(" ", "_") component = _schema_component(param) default = _schema_default(param) choices = _schema_choices(param) if default is not None and not isinstance(default, (dict, list)): return default if choices: return choices[0] if any(token in component for token in ["file", "upload", "audio", "video"]): raise ValueError(f"Cannot auto-fill required file/media input: {_schema_name(param, index)}") if "image" in component and not any(token in name for token in ["prompt", "negative"]): raise ValueError(f"Cannot auto-fill required image input: {_schema_name(param, index)}") if "negative" in name and "prompt" in name: return "" if "prompt" in name or "text" in component or "textbox" in component: return "a cinematic robot cat astronaut, detailed, studio lighting" if "random_seed" in name: return True if "checkbox" in component or "bool" in component: return False if "seed" in name: return 42 if "height" in name or "width" in name or "resolution" in name: return 1024 if "step" in name or "num_inference" in name: return 9 if "guidance" in name or "cfg" in name: return 0.0 if "slider" in component or "number" in component or "int" in component or "float" in component: return _coerce_number(default, 0) return "test" def _schema_file_kind(param, index: int) -> str: name = _schema_name(param, index).lower().replace(" ", "_").replace("-", "_") component = _schema_component(param) haystack = f"{name} {component}" if "image" in haystack: return "image" if "video" in haystack: return "video" if "audio" in haystack: return "audio" if any(token in haystack for token in ["file", "upload", "filepath", "path"]): return "file" return "" def _ensure_validation_input_file(run_dir: Path, kind: str) -> str: smoke_dir = run_dir / "tests" / "smoke_inputs" smoke_dir.mkdir(parents=True, exist_ok=True) if kind == "image": path = smoke_dir / "asf_smoke_image.png" if not path.exists(): path.write_bytes(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=")) return str(path) path = smoke_dir / "asf_smoke_file.txt" if not path.exists(): path.write_text("Agentic Space Factory validation input file.\n", encoding="utf-8") return str(path) def prepare_validation_file_inputs(api_name: str, schema, args: list, run_dir: Path) -> tuple[list, list[dict]]: from gradio_client import handle_file params = endpoint_parameter_objects(endpoint_schema_for_api(schema, api_name)) resolved = list(args or []) conversions = [] for index, param in enumerate(params or []): if index >= len(resolved): break kind = _schema_file_kind(param, index) if not kind: continue before = resolved[index] if isinstance(before, dict) and (before.get("meta") or {}).get("_type") == "gradio.FileData": continue source = before if kind in {"image", "file"} and (source is None or str(source).strip() == ""): source = _ensure_validation_input_file(run_dir, kind) try: resolved[index] = handle_file(str(source)) conversions.append({"index": index, "name": _schema_name(param, index), "kind": kind, "source": str(source)[:500], "action": "handle_file"}) except Exception as exc: conversions.append({"index": index, "name": _schema_name(param, index), "kind": kind, "source": str(source)[:500], "action": "handle_file_failed", "error": str(exc)[:1000]}) raise return resolved, conversions def build_args_from_gradio_schema(api_name: str, schema, current_args: list) -> tuple[list, dict]: endpoint = endpoint_schema_for_api(schema, api_name) params = endpoint_parameter_objects(endpoint) if not params: return current_args, {"args_were_autofilled": False, "args_source": "provided_or_default", "endpoint_parameters": endpoint_parameter_names(endpoint)} if isinstance(current_args, list) and len(current_args) >= len(params): return current_args, {"args_were_autofilled": False, "args_source": "provided", "endpoint_parameters": [_schema_name(p, i) for i, p in enumerate(params)]} generated = [] for index, param in enumerate(params): if isinstance(current_args, list) and index < len(current_args): generated.append(current_args[index]) else: generated.append(autofill_value_for_schema_param(param, index)) return generated, {"args_were_autofilled": True, "args_source": "gradio_schema_auto_fill", "endpoint_parameters": [_schema_name(p, i) for i, p in enumerate(params)]} def coerce_validation_args_to_schema_choices(api_name: str, schema, args: list) -> tuple[list, list[dict]]: params = endpoint_parameter_objects(endpoint_schema_for_api(schema, api_name)) corrected = list(args or []) changes = [] for index, param in enumerate(params or []): if index >= len(corrected): break choices = _schema_choice_values(param) before = corrected[index] changed = False for choice in choices: if before == choice: changed = True break if str(before).strip() == str(choice).strip(): corrected[index] = choice changes.append({"index": index, "name": _schema_name(param, index), "from": before, "to": choice, "reason": "schema_choice_type_match"}) changed = True break # Some Gradio schemas do not expose the choice list reliably but still # identify Dropdown/Radio string inputs. In replay mode, numeric-looking # smoke args (1024) must be normalized to strings ("1024") for these # components before predict(), otherwise Gradio rejects the call as a # choice/type mismatch. if not changed and not choices and _schema_prefers_string_choice(param) and not isinstance(before, str) and before is not None: corrected[index] = str(before) changes.append({"index": index, "name": _schema_name(param, index), "from": before, "to": corrected[index], "reason": "dropdown_string_type_normalization"}) return corrected, changes def is_schema_choice_type_error(error) -> bool: text = str(error or "").lower() return "not in the list of choices" in text or ("value:" in text and "choices" in text) def endpoint_candidate_score(name: str, schema) -> int: normalized = normalize_api_name(name) if normalized in {"/health", "/load", "/reset", "/clear"}: return -100 endpoint = endpoint_schema_for_api(schema, normalized) params = endpoint_parameter_objects(endpoint) score = 0 lname = normalized.lower() if lname == "/generate": score += 100 elif lname == "/predict": score += 80 elif "generate" in lname: score += 70 elif "predict" in lname or "infer" in lname: score += 55 else: score += 20 if params: score += min(len(params), 8) return score def select_validation_api_name(requested_api_name: str, schema) -> tuple[str, list[dict]]: discovered = api_names_from_schema(schema) normalized_requested = normalize_api_name(requested_api_name) if str(requested_api_name or "").strip() else "" names = list(dict.fromkeys(discovered or [])) candidates = [] if normalized_requested: candidates.append(normalized_requested) candidates.extend(["/generate", "/predict"]) candidates.extend(names) candidates = [normalize_api_name(c) for c in dict.fromkeys(candidates) if c] rows = [] for name in candidates: score = endpoint_candidate_score(name, schema) rows.append({"api_name": name, "score": score, "is_requested": name == normalized_requested, "is_discovered": name in names}) valid_rows = [r for r in rows if r["score"] > -100] if not valid_rows: raise RuntimeError(f"No suitable Gradio prediction endpoint found. Discovered endpoints: {names}") if normalized_requested and any(r["api_name"] == normalized_requested and r["is_discovered"] for r in valid_rows): selected = normalized_requested else: selected = sorted(valid_rows, key=lambda r: (r["score"], r["is_discovered"]), reverse=True)[0]["api_name"] return selected, rows def runtime_to_dict(runtime) -> dict: payload = {} for attr in ["stage", "hardware", "requested_hardware", "sleep_time", "storage", "gc_timeout"]: value = getattr(runtime, attr, None) payload[attr] = getattr(value, "value", value) return {k: str(v) if v is not None else None for k, v in payload.items()} def write_space_runtime(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, attempt: int | None = None) -> dict: try: runtime = api.get_space_runtime(repo_id=target_space_id, token=token) payload = runtime_to_dict(runtime) payload["attempt"] = attempt write_json(run_dir / "space_runtime.json", payload) return payload except Exception as exc: payload = {"error": str(exc)[:2000], "attempt": attempt} write_json(run_dir / "space_runtime.json", payload) append_event(events_path, "space_runtime", "warning", "Could not fetch Space runtime", payload) return payload RESTART_GUARDRAIL_STATE = { "restart": {"count": 0, "last_ts": 0.0}, "factory_reboot": {"count": 0, "last_ts": 0.0}, "same_code_upload": {"count": 0, "last_ts": 0.0}, } RESTART_GUARDRAIL_LIMITS = { "restart": 1, "factory_reboot": 1, "same_code_upload": 1, } RESTART_GUARDRAIL_COOLDOWN_SECONDS = int(os.environ.get("SPACE_FACTORY_RESTART_COOLDOWN_SECONDS", "240")) def runtime_stage_value(runtime_payload: dict | None) -> str: return str((runtime_payload or {}).get("stage") or (runtime_payload or {}).get("runtime_stage") or "").lower() def restart_guardrail_allows(action: str, events_path: Path, *, reason: str = "") -> bool: state = RESTART_GUARDRAIL_STATE.setdefault(action, {"count": 0, "last_ts": 0.0}) limit = RESTART_GUARDRAIL_LIMITS.get(action, 1) now = time.time() if state["count"] >= limit: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart budget exhausted", {"action": action, "limit": limit, "reason": reason}) return False if state["last_ts"] and now - state["last_ts"] < RESTART_GUARDRAIL_COOLDOWN_SECONDS: append_event(events_path, "factory_rebuild", "warning", f"Blocked {action}: restart cooldown active", {"action": action, "cooldown_seconds": RESTART_GUARDRAIL_COOLDOWN_SECONDS, "elapsed_seconds": round(now - state["last_ts"], 1), "reason": reason}) return False state["count"] += 1 state["last_ts"] = now return True def safe_restart_space(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, factory_reboot: bool = False, reason: str = "", require_logs_checked: bool = True, allow_busy: bool = False) -> bool: """Restart a Space with strict guardrails. Hugging Face exposes HfApi.restart_space(repo_id=..., factory_reboot=True) for a factory rebuild and restart_space(..., factory_reboot=False) for a normal restart. The Factory only calls it through this wrapper so we do not create restart loops or mask actionable build errors. """ action = "factory_reboot" if factory_reboot else "restart" if require_logs_checked: logs_dir = run_dir / "logs" if not ((logs_dir / "space_logs_build.txt").exists() or (logs_dir / "space_logs_runtime.txt").exists()): append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: logs have not been collected yet", {"reason": reason}) return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"} and not allow_busy: append_event(events_path, "factory_rebuild", "warning", f"Skipped {action}: Space is already busy", {"runtime": runtime_before, "reason": reason}) return False if not restart_guardrail_allows(action, events_path, reason=reason): return False try: append_event(events_path, "factory_rebuild", "started", f"Requesting {'factory reboot' if factory_reboot else 'Space restart'} via Hugging Face API", {"runtime_before": runtime_before, "reason": reason}) result = api.restart_space(repo_id=target_space_id, token=token, factory_reboot=factory_reboot) payload = {"factory_reboot": factory_reboot, "reason": reason, "result": str(result)[:1000]} write_json(run_dir / ("space_factory_reboot.json" if factory_reboot else "space_restart.json"), payload) append_event(events_path, "factory_rebuild", "success", f"{'Factory reboot' if factory_reboot else 'Space restart'} requested", payload) return True except TypeError: # Older huggingface_hub versions may not expose factory_reboot. Do not # loop; fall back to a plain restart only for non-factory requests. if factory_reboot: append_event(events_path, "factory_rebuild", "warning", "Installed huggingface_hub does not support restart_space(factory_reboot=True)", {"reason": reason}) return False try: result = api.restart_space(repo_id=target_space_id, token=token) append_event(events_path, "factory_rebuild", "success", "Space restart requested", {"reason": reason, "result": str(result)[:1000]}) return True except Exception as exc: append_event(events_path, "factory_rebuild", "failed", "Space restart API call failed", {"reason": reason, **exception_payload(exc)}) return False except Exception as exc: append_event(events_path, "factory_rebuild", "failed", f"{'Factory reboot' if factory_reboot else 'Space restart'} API call failed", {"reason": reason, **exception_payload(exc)}) return False def safe_same_code_reupload(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path, *, reason: str = "") -> bool: """Re-upload the current workspace once, with restart-loop guardrails.""" if not restart_guardrail_allows("same_code_upload", events_path, reason=reason): return False runtime_before = write_space_runtime(api, target_space_id, token, run_dir, events_path) stage = runtime_stage_value(runtime_before) if stage in {"building", "buildqueued", "build_queued", "starting", "pending"}: append_event(events_path, "factory_rebuild", "warning", "Skipped same-code upload: Space is already building/starting", {"runtime": runtime_before, "reason": reason}) return False upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) append_event(events_path, "factory_rebuild", "success", "Same-code workspace uploaded under restart guardrails", {"reason": reason, "runtime_before": runtime_before}) return True def collect_space_logs(target_space_id: str, token: str, run_dir: Path, events_path: Path): """Collect Space build/runtime logs with explicit capability detection and an index. This is intentionally more structured than the old best-effort collector: - runtime state is always captured when possible; - runtime/build log collection first uses HfApi.get_space_logs when available; - if the installed huggingface_hub lacks that method, REST fallbacks are attempted; - legacy text files are still written, but availability/quality lives in space_logs_index.json. """ from huggingface_hub import HfApi import json as _json import sys as _sys try: import huggingface_hub as _hf_hub hub_version = getattr(_hf_hub, "__version__", None) except Exception: hub_version = None logs_dir = run_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) api = HfApi(token=token) collected_at = now() written = [] index = { "schema_version": "1.0", "target_space_id": target_space_id, "collected_at": collected_at, "huggingface_hub_version": hub_version, "python_version": _sys.version.split()[0], "capabilities": { "hfapi_get_space_runtime": hasattr(api, "get_space_runtime"), "hfapi_get_space_logs": hasattr(api, "get_space_logs"), "rest_fallback": True, }, "entries": {}, } def _tail(text: str, n: int = 1200) -> str: return (text or "")[-n:] def _stringify_log_response(out) -> str: if out is None: return "" if isinstance(out, (list, tuple)): return "\n".join(str(x) for x in out) if isinstance(out, (dict, list)): return _json.dumps(out, ensure_ascii=False, indent=2) return str(out) def _write_text(filename: str, text: str) -> int: path = logs_dir / filename path.write_text(text or "", encoding="utf-8") return path.stat().st_size if path.exists() else 0 def _entry(key: str, *, filename: str, available: bool, source: str, quality: str, reason: str = "", error: str = "", text: str = ""): size = _write_text(filename, text) payload = { "available": bool(available), "path": f"logs/{filename}", "source": source, "quality": quality, "reason": reason, "error": error[:1500] if error else "", "size_bytes": size, "tail": _tail(text), } index["entries"][key] = payload written.append({ "file": filename, "source": source, "available": bool(available), "quality": quality, "reason": reason, "error": error[:1000] if error else "", "returncode": 0 if available else 1, "tail": _tail(text, 1000), }) return payload def _unavailable_text(kind: str, reason: str, error: str = "") -> str: return ( f"[ASF_LOG_UNAVAILABLE]\n" f"kind={kind}\n" f"reason={reason}\n" f"target_space_id={target_space_id}\n" f"huggingface_hub_version={hub_version}\n" f"error={error[:1500] if error else ''}\n" "See logs/space_logs_index.json for structured availability metadata.\n" ) # Runtime state / snapshot. This is not a log stream, but it is critical context for # BUILD_ERROR/RUNTIME_ERROR triage and is always tracked separately from log streams. try: runtime_obj = api.get_space_runtime(repo_id=target_space_id, token=token) runtime_payload = runtime_to_dict(runtime_obj) runtime_payload.update({"target_space_id": target_space_id, "collected_at": collected_at}) runtime_text = _json.dumps(runtime_payload, indent=2, ensure_ascii=False) _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) # Legacy path retained because older reports/tests point to this file. _entry("runtime_state_legacy", filename="space_runtime_state.json", available=True, source="HfApi.get_space_runtime", quality="snapshot", text=runtime_text) except Exception as exc: err = f"{type(exc).__name__}: {str(exc)[:1500]}" _entry("runtime_snapshot", filename="space_runtime_snapshot.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_snapshot", "runtime_snapshot_error", err)) _entry("runtime_state_legacy", filename="space_runtime_state.json", available=False, source="HfApi.get_space_runtime", quality="unavailable", reason="runtime_snapshot_error", error=err, text=_unavailable_text("runtime_state", "runtime_snapshot_error", err)) def _collect_via_hfapi(kind: str): if not hasattr(api, "get_space_logs"): return None, "hub_method_unavailable", "" try: out = api.get_space_logs(repo_id=target_space_id, token=token, build=(kind == "build")) text = _stringify_log_response(out) if text.strip(): return text, "", "" return "", "empty_response", "" except Exception as exc: return None, "hfapi_error", f"{type(exc).__name__}: {str(exc)[:1500]}" def _space_log_endpoint(kind: str) -> str: # HF Space log streams are exposed under /logs/run and /logs/build. # "runtime" is an ASF legacy label for the file name; the Hub endpoint is "run". endpoint_kind = "build" if kind == "build" else "run" return f"https://huggingface.co/api/spaces/{target_space_id}/logs/{endpoint_kind}" def _read_streaming_response(response, *, max_bytes: int = 1_200_000) -> str: chunks = [] total = 0 try: iterator = response.iter_content(chunk_size=8192, decode_unicode=True) except Exception: return response.text or "" for chunk in iterator: if not chunk: continue if isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="ignore") chunks.append(str(chunk)) total += len(str(chunk).encode("utf-8", errors="ignore")) if total >= max_bytes: chunks.append("\n[ASF_LOG_TRUNCATED: max_bytes reached]\n") break return "".join(chunks) def _collect_via_rest(kind: str): # v198.16: use the real HF Hub Space log endpoints. The runtime stream is # /logs/run, not any legacy runtime endpoint and not the older query-string endpoint. # Wrong endpoints made Pi repair from generic RUNTIME_ERROR messages. try: import requests except Exception as exc: return None, "requests_unavailable", f"{type(exc).__name__}: {str(exc)[:1500]}" headers = {"Authorization": f"Bearer {token}", "Accept": "text/plain,application/json,*/*"} if token else {"Accept": "text/plain,application/json,*/*"} primary_url = _space_log_endpoint(kind) candidates = [primary_url] last_error = "" for url in candidates: try: response = requests.get(url, headers=headers, timeout=(10, 45), stream=True) body = _read_streaming_response(response) if response.ok and body.strip(): ctype = response.headers.get("content-type", "") return body, "", f"{url} ({response.status_code}, {ctype})" last_error = f"{url} returned {response.status_code}: {body[:500]}" except Exception as exc: last_error = f"{url} failed: {type(exc).__name__}: {str(exc)[:500]}" return None, "rest_logs_unavailable", last_error def _collect_stream(kind: str, filename: str): attempted = [] text, reason, error = _collect_via_hfapi(kind) attempted.append({"method": "HfApi.get_space_logs", "reason": reason, "error": error[:500] if error else ""}) if text is not None and text.strip(): return _entry(kind, filename=filename, available=True, source="HfApi.get_space_logs", quality="full", text=text) rest_text, rest_reason, rest_error = _collect_via_rest(kind) attempted.append({"method": "hf_rest_logs", "reason": rest_reason, "error": rest_error[:500] if rest_error else ""}) if rest_text is not None and rest_text.strip(): source = "hf_rest_logs" if rest_error and rest_error.startswith("https://"): source = rest_error return _entry(kind, filename=filename, available=True, source=source, quality="full", text=rest_text) # Keep legacy files present, but explicitly mark them as unavailable rather than # pretending an SDK AttributeError is a Space runtime/build log. This also avoids # the obsolete CLI path that emitted "invalid choice: spaces". final_reason = rest_reason or reason or "logs_unavailable" final_error = rest_error or error or "" entry = _entry(kind, filename=filename, available=False, source="log_collection", quality="unavailable", reason=final_reason, error=final_error, text=_unavailable_text(kind, final_reason, final_error)) entry["attempted_methods"] = attempted return entry runtime_entry = _collect_stream("runtime", "space_logs_runtime.txt") build_entry = _collect_stream("build", "space_logs_build.txt") def _first_error_from(text: str) -> str: if not text: return "" lines = text.splitlines() lowered = [line.lower() for line in lines] markers = ("traceback", "exception", "error", "exit code", "failed", "runtimeerror", "attributeerror", "modulenotfounderror") for idx, line in enumerate(lowered): if any(marker in line for marker in markers): start = max(0, idx - 3) end = min(len(lines), idx + 12) return "\n".join(lines[start:end])[:4000] return "" runtime_text = (logs_dir / "space_logs_runtime.txt").read_text(encoding="utf-8", errors="ignore") if runtime_entry.get("available") and (logs_dir / "space_logs_runtime.txt").exists() else "" build_text = (logs_dir / "space_logs_build.txt").read_text(encoding="utf-8", errors="ignore") if build_entry.get("available") and (logs_dir / "space_logs_build.txt").exists() else "" first_error = _first_error_from(runtime_text) or _first_error_from(build_text) log_quality = "full" if runtime_entry.get("available") or build_entry.get("available") else "snapshot_only" if index["entries"].get("runtime_snapshot", {}).get("available") else "unavailable" diagnostics = { "schema_version": "1.0", "target_space_id": target_space_id, "collected_at": collected_at, "log_quality": log_quality, "runtime_logs_available": bool(runtime_entry.get("available")), "build_logs_available": bool(build_entry.get("available")), "first_error": first_error, "diagnosis_log_source": "runtime_or_build_logs" if first_error else ("runtime_snapshot" if log_quality == "snapshot_only" else "none"), } diagnostics_text = _json.dumps(diagnostics, ensure_ascii=False, indent=2) _entry("diagnostics", filename="space_log_diagnostics.json", available=True, source="asf_log_collector", quality=log_quality, text=diagnostics_text) index["log_quality"] = log_quality index["runtime_logs_available"] = bool(runtime_entry.get("available")) index["build_logs_available"] = bool(build_entry.get("available")) index["first_error"] = first_error[:1000] index_text = _json.dumps(index, ensure_ascii=False, indent=2) (logs_dir / "space_logs_index.json").write_text(index_text, encoding="utf-8") written.append({"file": "space_logs_index.json", "source": "asf_log_collector", "available": True, "quality": log_quality, "returncode": 0, "tail": _tail(index_text, 1000)}) event_status = "success" if runtime_entry.get("available") or build_entry.get("available") else "warning" event_message = "Collected Space logs and runtime snapshot" if event_status == "success" else "Space log streams unavailable; runtime snapshot/index written" append_event(events_path, "space_logs", event_status, event_message, {"files": written, "log_quality": log_quality, "index": "logs/space_logs_index.json"}) write_artifact_manifest(run_dir, reason="space_logs_collected") return written def smoke_failure_repair_escalation_decision(generation_smoke: dict | None, *, validation: dict | None = None, contract: dict | None = None) -> dict: """Decide whether a failed generation smoke should trigger one targeted repair. v198.13: partial validation is useful for UI honesty, but a demo that boots yet cannot satisfy its primary flow should not stop there when the smoke failure is concrete and minimally repairable. This is deliberately bounded: validator-owned failures are handled by validator self-repair, model init without logs remains partial, and we only escalate actionable app/runtime failures such as a missing executable or missing Python dependency. """ smoke = generation_smoke if isinstance(generation_smoke, dict) else {} validation = validation if isinstance(validation, dict) else {} contract = contract if isinstance(contract, dict) else {} failure_class = str(smoke.get("failure_class") or smoke.get("failure_type") or "") failure_owner = str(smoke.get("failure_owner") or "") repair_candidate = bool(smoke.get("repair_candidate")) health_ok = validation_health_passed(validation) actionable_classes = {"missing_runtime_cli", "missing_python_dependency", "app_runtime_error"} blocked_classes = {"validator_schema_choice_type_mismatch", "validator_request_error", "model_not_initialized", "smoke_schema_error", "smoke_input_materialization_failed", "gradio_hidden_runtime_error", "gpu_oom", "insufficient_vram", "insufficient_vram_or_payload_too_large", "hardware_capacity", "cuda_out_of_memory"} strategy = str(contract.get("inference_strategy") or smoke.get("inference_strategy") or "") payload = { "schema_version": "generation_smoke_repair_escalation.v198_13", "triggered": False, "repair_candidate": repair_candidate, "health_passed": health_ok, "failure_class": failure_class, "failure_owner": failure_owner, "inference_strategy": strategy, "action": "keep_partial_validation", "reason": "not_actionable_or_not_safe_for_automatic_smoke_repair", } if str(smoke.get("status") or "") != "failed": payload["reason"] = "generation_smoke_not_failed" return payload if failure_class in blocked_classes or failure_owner in {"factory_validator", "factory_validation_client", "hardware"}: payload["reason"] = "handled_elsewhere_or_requires_logs_not_blind_repair" return payload if not health_ok: payload["reason"] = "health_not_passed_use_runtime_recovery_instead" return payload if not repair_candidate or failure_class not in actionable_classes: payload["reason"] = "smoke_failure_not_repair_candidate" return payload payload.update({ "triggered": True, "action": "targeted_patch_repair_before_final_partial", "reason": "health_passed_but_primary_demo_flow_failed_with_actionable_smoke_error", "recommended_action": smoke.get("recommended_action") or "Run one targeted repair, redeploy, then re-run health and canonical smoke validation.", "missing_executable": smoke.get("missing_executable") or "", }) return payload def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_path: Path, *, pi_model: str, target_space_id: str, model_id: str, token: str, implementation_mode: str, expected_output_type: str, validation: dict, generation_smoke: dict) -> tuple[dict, dict]: """Run one bounded targeted repair when the demo boots but its primary flow fails. The function returns the latest (validation, generation_smoke). It never loops: one repair, one upload, one health validation, one generation smoke retry. """ contract = read_inference_contract(workspace) escalation = smoke_failure_repair_escalation_decision(generation_smoke, validation=validation, contract=contract) write_json(run_dir / "tests" / "generation_smoke_repair_escalation.json", escalation) if not escalation.get("triggered"): return validation, generation_smoke failure_class = escalation.get("failure_class") or generation_smoke.get("failure_class") or generation_smoke.get("failure_type") or "generation_smoke_error" failure_reason = ( "Automatic generation smoke failed after health passed.\n" f"Failure class: {failure_class}\n" f"Failure owner: {generation_smoke.get('failure_owner') or ''}\n" f"Actionability: {generation_smoke.get('actionability') or ''}\n" f"Recommended action: {generation_smoke.get('recommended_action') or ''}\n" f"Error/result: {generation_smoke.get('error') or generation_smoke.get('result_info') or generation_smoke}\n" ) decision = { "action": "patch_code", "confidence": "high" if failure_class in {"missing_runtime_cli", "missing_python_dependency"} else "medium", "source": "generation_smoke_repair_escalation.v198_13", "classification": { "category": failure_class, "failure_phase": "generation_smoke", "logs_quality": "useful", "recommendation": escalation.get("recommended_action") or generation_smoke.get("recommended_action") or "Patch the smallest concrete dependency or command issue, preserve the demo flow, redeploy, then re-run smoke validation.", "generation_smoke_failure": True, "missing_executable": generation_smoke.get("missing_executable") or "", }, "constraints": [ "Preserve real inference and the existing Gradio API contract.", "Do not replace the model call with placeholders or diagnostics if the failure is repairable.", "Patch only the dependency, command invocation, or minimal startup code required for the failed smoke path.", ], } write_json(run_dir / "repair" / "SMOKE_REPAIR_DECISION.json", decision) append_event(events_path, "generation_smoke_repair", "started", "Running one targeted repair because health passed but the primary demo smoke failed", {"failure_class": failure_class, "decision": decision}) write_repair_outcome( run_dir, events_path, repair_trigger="generation_smoke_failure", root_cause=failure_class, repair_decision="patch_code", decision=decision, smoke_repair=True, pre_repair_generation_smoke=generation_smoke, patch_applied=False, upload_success=False, post_repair_validation="not_started", ) repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, failure_reason, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger="generation_smoke_failure") if not repaired: write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation="not_started", failure_type="smoke_repair_patch_failed", final_user_message="Targeted smoke repair failed before redeploy; keeping partial validation.") append_event(events_path, "generation_smoke_repair", "failed", "Targeted smoke repair failed before redeploy; keeping partial validation", {"failure_class": failure_class}) return validation, generation_smoke write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "generation_smoke_repair_upload", "started", "Uploading targeted smoke repair") write_auth_probe(run_dir, events_path, "before_smoke_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) upload_workspace(api, workspace, target_space_id, token, run_dir, events_path) write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending") append_event(events_path, "generation_smoke_repair_upload", "success", "Targeted smoke repair uploaded; revalidating health and demo smoke") try: write_auth_probe(run_dir, events_path, "before_smoke_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True) validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200) generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace) post_status = "success" if generation_smoke.get("status") == "success" else "smoke_still_failed" write_repair_outcome(run_dir, events_path, post_repair_validation=post_status, failure_type="" if post_status == "success" else generation_smoke.get("failure_type") or "generation_smoke_still_failed", final_user_message="Targeted smoke repair revalidated the Space." if post_status == "success" else "Targeted smoke repair uploaded, but generation smoke still did not verify the primary demo flow.", post_repair_generation_smoke=generation_smoke) append_event(events_path, "generation_smoke_repair_validation", "success" if post_status == "success" else "failed", "Targeted smoke repair validation completed", {"post_status": post_status, "generation_smoke_status": generation_smoke.get("status")}) except Exception as exc: repair_class = classify_repair_validation_error(exc) write_repair_outcome(run_dir, events_path, post_repair_validation=repair_class["post_repair_validation"], failure_type=repair_class["failure_type"] or "smoke_repair_validation_failed", final_user_message=repair_class["message"], validation_error=str(exc)[:4000]) append_event(events_path, "generation_smoke_repair_validation", "failed", "Targeted smoke repair validation failed; keeping partial validation", {"error": str(exc)[:2000]}) return validation, generation_smoke def space_subdomain_url(target_space_id: str) -> str: owner, name = target_space_id.split("/", 1) return f"https://{owner}-{name}.hf.space".replace("_", "-").lower() def validate_http_health(target_space_id: str, token: str, run_dir: Path, attempt: int): import requests url = space_subdomain_url(target_space_id).rstrip("/") + "/health" headers = {"Authorization": f"Bearer {token}", "Accept": "application/json,text/plain,*/*"} response = requests.get(url, headers=headers, timeout=20) payload = { "status": "success" if response.ok else "failed", "attempt": attempt, "url": url, "status_code": response.status_code, "content_type": response.headers.get("content-type"), "text": response.text[:2000], } if response.ok: try: payload["json"] = response.json() except Exception: pass semantic = validation_health_semantics(payload | {"validator": "http_get_health"}) payload["health_endpoint_reachable"] = True payload["health_semantic_passed"] = semantic.get("semantic_passed") payload["health_semantics"] = semantic write_json(run_dir / "tests" / "http_health.json", payload) return payload raise RuntimeError(f"HTTP /health returned {response.status_code}: {response.text[:500]}") def wait_until_live(api, target_space_id: str, token: str, run_dir: Path, events_path: Path, timeout_s: int = 1800): append_event(events_path, "live_wait", "started", "Waiting for existing Space to become live") deadline = time.time() + timeout_s attempt = 0 last_error = None while time.time() < deadline: attempt += 1 runtime_payload = write_space_runtime(api, target_space_id, token, run_dir, events_path, attempt) stage = str(runtime_payload.get("stage") or "").upper() if "RUNTIME_ERROR" in stage: collect_space_logs(target_space_id, token, run_dir, events_path) last_error = f"Space is in RUNTIME_ERROR: {runtime_payload}" append_event(events_path, "live_wait", "waiting", "Space is in runtime error; still waiting in case hardware was changed manually", {"attempt": attempt, "runtime": runtime_payload}) time.sleep(30) continue try: health = validate_http_health(target_space_id, token, run_dir, attempt) append_event(events_path, "live_wait", "success", "HTTP /health is live", {"attempt": attempt}) return {"validator": "http_health", "health": health, "runtime": runtime_payload} except Exception as http_exc: last_error = f"HTTP health failed: {http_exc}" try: client = make_gradio_client(target_space_id, token) schema = client.view_api(return_format="dict") names = api_names_from_schema(schema) write_json(run_dir / "tests" / "api_schema.json", {"schema": schema, "api_names": names}) if names: append_event(events_path, "live_wait", "success", "Gradio API schema is live", {"attempt": attempt, "api_names": names}) return {"validator": "gradio_schema", "api_names": names, "runtime": runtime_payload} except Exception as gr_exc: last_error = (last_error or "") + f"; Gradio schema failed: {gr_exc}" append_event(events_path, "live_wait", "waiting", "Space not live yet", {"attempt": attempt, "runtime": runtime_payload, "error": last_error[-1500:] if last_error else None}) time.sleep(30) collect_space_logs(target_space_id, token, run_dir, events_path) raise RuntimeError(f"Space did not become live before timeout: {last_error}") def parse_json_env(name: str, default): value = os.environ.get(name) if not value: return default try: return json.loads(value) except Exception as exc: raise ValueError(f"Invalid JSON for {name}: {exc}") def result_contains_expected_output(result, expected_output_type: str) -> tuple[bool, dict]: expected = (expected_output_type or "any").lower().strip() info = {"expected_output_type": expected, "result_type": type(result).__name__, "result_repr": repr(result)[:2000]} paths = [] type_hints = [] text_values = [] def visit(obj): type_hints.append(type(obj).__name__.lower()) if isinstance(obj, (str, Path)): value = str(obj) if value: paths.append(value) text_values.append(value) elif isinstance(obj, dict): for key, value in obj.items(): key_l = str(key).lower() if key_l in {"path", "name", "url", "file", "filepath"}: visit(value) elif key_l in {"mime_type", "mime", "type", "format"} and value: type_hints.append(str(value).lower()) elif key_l in {"image", "video", "audio", "text"}: type_hints.append(key_l) visit(value) elif isinstance(value, (dict, list, tuple)): visit(value) elif isinstance(obj, (list, tuple)): for item in obj: visit(item) else: module = getattr(type(obj), "__module__", "") if module: type_hints.append(module.lower()) visit(result) lower_paths = [str(p).lower() for p in paths] hints = " ".join(type_hints + lower_paths) info["detected_paths"] = paths[:20] info["type_hints"] = type_hints[:30] if expected == "any": return result is not None, info image_ext = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff") video_ext = (".mp4", ".mov", ".webm", ".avi", ".mkv") audio_ext = (".wav", ".mp3", ".flac", ".ogg", ".m4a") if expected == "text": if isinstance(result, str) and bool(result.strip()): return True, info return any(isinstance(x, str) and x.strip() and Path(x).suffix.lower() not in image_ext + video_ext + audio_ext for x in text_values), info if expected == "image": return any(p.endswith(image_ext) for p in lower_paths) or any(h in hints for h in ["image", "pil.", "png", "jpeg", "jpg", "webp", "gradio.data_classes.filedata"]), info if expected == "video": return any(p.endswith(video_ext) for p in lower_paths) or any(h in hints for h in ["video", "mp4", "webm", "moviepy"]), info if expected == "audio": return any(p.endswith(audio_ext) for p in lower_paths) or any(h in hints for h in ["audio", "wav", "soundfile"]), info return result is not None, info def copy_result_artifacts(result, run_dir: Path): artifacts = run_dir / "artifacts" artifacts.mkdir(parents=True, exist_ok=True) copied = [] def maybe_copy(obj): if isinstance(obj, (str, Path)): path = Path(str(obj)) if path.exists() and path.is_file(): target = artifacts / path.name try: shutil.copy2(path, target) copied.append(str(target)) except Exception: pass elif isinstance(obj, dict): for key in ["path", "name"]: if key in obj: maybe_copy(obj[key]) for value in obj.values(): if isinstance(value, (dict, list, tuple)): maybe_copy(value) elif isinstance(obj, (list, tuple)): for item in obj: maybe_copy(item) maybe_copy(result) return copied def run_gradio_validation_harness(*, client, schema, api_name: str, test_args: list, test_kwargs: dict, expected_output_type: str, run_dir: Path, events_path: Path, validation_mode: str, payload_source: str, replay_source_used: bool = False): """Common Gradio validation gate used by linked Space Test modes. v190.24 contract: automatic build smoke and manual linked validation must share the same endpoint → payload resolution → schema coercion → predict → output verification sequence. This helper is intentionally small in the validation worker; build smoke mirrors the same artefacts and coercion contract in run_generation_smoke(). """ resolved = { "api_name": api_name, "test_args": list(test_args or []), "test_kwargs": dict(test_kwargs or {}), "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "replay_source_used": bool(replay_source_used), "resolved_request_required_before_predict": True, } write_json(run_dir / "tests" / "resolved_validation_request.json", resolved) started = time.time() result = client.predict(*resolved["test_args"], api_name=api_name, **resolved["test_kwargs"]) latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) return result, latency, ok, info, resolved def smoke_generate(target_space_id: str, token: str, run_dir: Path, events_path: Path): """Run the linked Space Test validation smoke test with hard diagnostics. v190.32 rule: a validation run may fail, but it must not fail opaquely. The worker writes validation_engine / preflight diagnostics before any fragile Gradio operation, and it writes a terminal diagnosis if it exits before payload resolution or predict(). """ tests_dir = run_dir / "tests" tests_dir.mkdir(parents=True, exist_ok=True) # Compatibility anchors for historical release tests: # write_json(run_dir / "tests" / "validation_launch_payload.json", validation_launch_payload) # write_json(run_dir / "tests" / "payload_source.json", payload_source_record) # write_json(run_dir / "tests" / "replay_source.json", replay_source) # write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight) app_version = "v198.26.9" engine_version = "unified_gradio_validation_harness_v198_25_3" parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip() validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked" requested_api_name = (os.environ.get("API_NAME") or "").strip() expected_output_type = (os.environ.get("EXPECTED_OUTPUT_TYPE") or "any").strip() validation_launch_payload = parse_json_env("VALIDATION_LAUNCH_PAYLOAD_JSON", {}) if not isinstance(validation_launch_payload, dict): validation_launch_payload = {"parse_error": "VALIDATION_LAUNCH_PAYLOAD_JSON was not a JSON object"} env_replay_source_present = bool(os.environ.get("PARENT_REPLAY_SOURCE_JSON")) replay_mode = str(validation_mode or "").strip().lower() == "replay" initial_payload_source = "parent_automatic_smoke" if (replay_mode and env_replay_source_present) else ( os.environ.get("PAYLOAD_SOURCE") or validation_launch_payload.get("payload_source") or "provided" ) worker_version = { "app_version": app_version, "validation_engine_version": engine_version, "worker_kind": "linked_space_validation", "parent_build_run_id": parent_build_run_id, "has_env_replay_source": env_replay_source_present, "has_validation_launch_payload": bool(os.environ.get("VALIDATION_LAUNCH_PAYLOAD_JSON")), "written_at": now(), } write_json(tests_dir / "worker_version.json", worker_version) write_json(tests_dir / "validation_launch_payload.json", validation_launch_payload) write_json(tests_dir / "validation_engine.json", { "schema_version": "1.1", "validation_engine": "unified_gradio_validation_harness", "validation_engine_version": engine_version, "validation_mode": validation_mode, "payload_source": initial_payload_source, "parent_smoke_payload_used": False, "backend_launch_payload_present": bool(validation_launch_payload), "backend_launch_payload_source": validation_launch_payload.get("payload_source") if isinstance(validation_launch_payload, dict) else "", "resolved_request_required_before_predict": True, "stage": "initialized", "initialized_at": now(), }) write_json(tests_dir / "validation_preflight.json", { "schema_version": "1.1", "stage": "initialized", "parent_build_run_id": parent_build_run_id, "target_space_id": target_space_id, "validation_mode": validation_mode, "payload_source": initial_payload_source, "requested_api_name": requested_api_name, "expected_output_type": expected_output_type, "has_env_replay_source": env_replay_source_present, "has_validation_launch_payload": bool(validation_launch_payload), "launchable": None, "written_at": now(), }) stage = "initialized" schema = {} discovered = [] api_name = requested_api_name or "" test_args = [] safe_kwargs = {} schema_choice_corrections = [] replay_source = {} autofill_meta = {} dropped_kwargs = [] endpoint_parameters = [] original_test_args = [] file_input_conversions = [] def write_failure(exc, *, failure_type: str | None = None, stage_override: str | None = None): nonlocal stage, api_name, discovered, test_args, safe_kwargs, schema_choice_corrections, replay_source, dropped_kwargs, endpoint_parameters, expected_output_type message = str(exc) fail_stage = stage_override or stage or "unknown" if not failure_type: if fail_stage in {"initialized", "client_init", "endpoint_discovery"}: failure_type = "validation_engine_startup_failed" elif fail_stage in {"payload_resolution", "payload"}: failure_type = "payload_resolution_failed" else: failure_type = "validation_worker_failed" diagnosis = { "schema_version": "1.1", "failure_type": failure_type, "stage": fail_stage, "message": message[:4000], "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "validation_mode": validation_mode, "payload_source": initial_payload_source, "api_name": api_name or requested_api_name, "selected_endpoint": api_name, "discovered_api_names": discovered, "resolved_request_written": (tests_dir / "resolved_validation_request.json").exists(), "validation_engine_written": (tests_dir / "validation_engine.json").exists(), "schema_choice_corrections": schema_choice_corrections, "file_input_conversions": file_input_conversions, "replay_source_used": bool(replay_source), "parent_smoke_payload_used": bool(replay_source), "recommended_action": "Inspect worker logs and validation_launch_payload; the worker stopped before payload resolution." if fail_stage != "predict" else "Inspect Space runtime and resolved_validation_request.", "written_at": now(), } failure_payload = { "status": "failed", "failure_type": failure_type, "stage": fail_stage, "target_space": target_space_id, "api_name": api_name or requested_api_name, "discovered_api_names": discovered, "test_args": test_args, "test_kwargs": safe_kwargs, "effective_args": test_args, "effective_kwargs": safe_kwargs, "schema_choice_corrections": schema_choice_corrections, "replay_source_used": bool(replay_source), "parent_smoke_payload_used": bool(replay_source), "ignored_test_kwargs": dropped_kwargs, "endpoint_parameters": endpoint_parameters, "expected_output_type": expected_output_type, "error": message[:4000], "validated_at": now(), } write_json(tests_dir / "validation_failure_diagnosis.json", diagnosis) if not (tests_dir / "generation_smoke.json").exists(): write_json(tests_dir / "generation_smoke.json", failure_payload) if not (tests_dir / "test_result.json").exists(): write_json(tests_dir / "test_result.json", failure_payload) write_live_status(run_dir, stage=fail_stage, status="failed", message=diagnosis["message"], data=diagnosis) append_event(events_path, fail_stage, "failed", f"Linked validation failed at {fail_stage}", diagnosis) return failure_payload try: stage = "parse_payload" test_args = parse_json_env("TEST_ARGS_JSON", ["a cinematic robot cat astronaut, detailed, studio lighting"]) test_kwargs = parse_json_env("TEST_KWARGS_JSON", {}) if not isinstance(test_args, list): raise ValueError("TEST_ARGS_JSON must be a JSON list") if not isinstance(test_kwargs, dict): raise ValueError("TEST_KWARGS_JSON must be a JSON object") stage = "client_init" client = make_gradio_client(target_space_id, token) write_live_status(run_dir, stage="endpoint_discovery", status="running", message="Discovering Gradio endpoints before validation", data={"requested_api_name": requested_api_name or ""}) append_event(events_path, "endpoint_discovery", "started", "Discovering Gradio endpoints before validation", {"requested_api_name": requested_api_name or ""}) stage = "endpoint_discovery" schema = client.view_api(return_format="dict") discovered = api_names_from_schema(schema) parent_dir = run_dir.parent / parent_build_run_id if parent_build_run_id else None replay_source = (load_env_replay_source(target_space_id, expected_output_type) or load_parent_replay_source(parent_dir, target_space_id, expected_output_type)) if replay_mode else {} if replay_source.get("source_error"): write_json(tests_dir / "replay_source_error.json", replay_source) replay_source = {} preferred_api_name = replay_source.get("api_name") or requested_api_name api_name, endpoint_candidates = select_validation_api_name(preferred_api_name, schema) write_json(tests_dir / "gradio_endpoint_discovery.json", {"requested_api_name": requested_api_name or "", "preferred_api_name": preferred_api_name or "", "discovered_api_names": discovered, "selected_api_name": api_name, "endpoint_discovery_required": os.environ.get("ENDPOINT_DISCOVERY_REQUIRED") == "true", "replay_source_used": bool(replay_source)}) write_json(tests_dir / "gradio_schema.json", {"schema": schema, "api_names": discovered}) write_json(tests_dir / "endpoint_candidates.json", {"candidates": endpoint_candidates}) write_json(tests_dir / "selected_endpoint.json", {"api_name": api_name}) append_event(events_path, "endpoint_discovery", "success", "Selected Gradio endpoint for linked validation", {"api_name": api_name, "discovered_api_names": discovered}) write_live_status(run_dir, stage="endpoint_discovery", status="success", message=f"Selected Gradio endpoint {api_name}", data={"api_name": api_name, "discovered_api_names": discovered}) stage = "payload_resolution" if replay_source: test_args = list(replay_source.get("test_args") or []) test_kwargs = dict(replay_source.get("test_kwargs") or {}) expected_output_type = str(replay_source.get("expected_output_type") or expected_output_type or "any") safe_kwargs, dropped_kwargs, endpoint_parameters = sanitize_kwargs_for_schema(api_name, schema, test_kwargs) autofill_meta = {"args_were_autofilled": False, "args_source": "parent_automatic_smoke", "endpoint_parameters": endpoint_parameter_names(endpoint_schema_for_api(schema, api_name))} if autofill_meta.get("endpoint_parameters"): endpoint_parameters = autofill_meta.get("endpoint_parameters") original_test_args = list(test_args or []) test_args, schema_choice_corrections = coerce_validation_args_to_schema_choices(api_name, schema, test_args) write_json(tests_dir / "replay_source.json", {**replay_source, "selected_api_name": api_name, "using_parent_smoke_payload_as_source_of_truth": True, "schema_choice_corrections": schema_choice_corrections}) payload_source = "parent_automatic_smoke" append_event(events_path, "payload", "success", "Replayed validation payload from parent automatic smoke", {"api_name": api_name, "parent_build_run_id": parent_build_run_id, "payload_source": payload_source, "choice_corrections": schema_choice_corrections}) write_live_status(run_dir, stage="payload", status="success", message="Payload replayed from parent automatic smoke", data={"api_name": api_name, "parent_build_run_id": parent_build_run_id, "choice_corrections": schema_choice_corrections}) else: safe_kwargs, dropped_kwargs, endpoint_parameters = sanitize_kwargs_for_schema(api_name, schema, test_kwargs) test_args, autofill_meta = build_args_from_gradio_schema(api_name, schema, test_args) if autofill_meta.get("endpoint_parameters"): endpoint_parameters = autofill_meta.get("endpoint_parameters") original_test_args = list(test_args or []) test_args, schema_choice_corrections = coerce_validation_args_to_schema_choices(api_name, schema, test_args) payload_source = os.environ.get("PAYLOAD_SOURCE") or autofill_meta.get("args_source") or validation_launch_payload.get("payload_source") or "provided" test_args, file_input_conversions = prepare_validation_file_inputs(api_name, schema, test_args, run_dir) payload_source_record = { # Compatibility anchor: write_json(run_dir / "tests" / "payload_source.json", payload_source_record) # Compatibility anchor: write_json(run_dir / "tests" / "validation_engine.json", payload_source_record) # Compatibility anchor: write_json(run_dir / "tests" / "resolved_validation_request.json", resolved_request) "schema_version": "1.1", "validation_engine": "unified_gradio_validation_harness", "validation_engine_version": engine_version, "validation_mode": validation_mode, "payload_source": payload_source, "parent_smoke_payload_used": bool(replay_source), "backend_launch_payload_present": bool(validation_launch_payload), "backend_launch_payload_source": validation_launch_payload.get("payload_source") if isinstance(validation_launch_payload, dict) else "", "endpoint_discovery_used": True, "selected_api_name": api_name, } write_json(tests_dir / "payload_source.json", payload_source_record) write_json(tests_dir / "validation_engine.json", {**payload_source_record, "resolved_request_required_before_predict": True, "stage": "payload_resolved", "updated_at": now()}) validation_preflight = { "schema_version": "1.1", "stage": "payload_resolved", "parent_build_run_id": parent_build_run_id, "target_space_id": target_space_id, "validation_mode": validation_mode, "payload_source": payload_source, "requested_api_name": requested_api_name, "selected_api_name": api_name, "expected_output_type": expected_output_type, "endpoint_discovery_required": os.environ.get("ENDPOINT_DISCOVERY_REQUIRED") == "true", "discovered_api_names": discovered, "launchable": True, } validation_payload = {"api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "effective_args": test_args, "effective_kwargs": safe_kwargs, "original_test_args": original_test_args, "schema_choice_corrections": schema_choice_corrections, "file_input_conversions": file_input_conversions, "endpoint_discovery": {"requested_api_name": requested_api_name, "selected_api_name": api_name, "discovered_api_names": discovered}, **autofill_meta} resolved_request = { "api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "schema_choice_corrections": schema_choice_corrections, "replay_source_used": bool(replay_source), "parent_smoke_payload_used": bool(replay_source), } write_json(tests_dir / "validation_preflight.json", validation_preflight) write_json(tests_dir / "validation_payload.json", validation_payload) write_json(tests_dir / "resolved_validation_request.json", resolved_request) write_json(tests_dir / "schema_coercion.json", {"api_name": api_name, "changes": schema_choice_corrections, "original_args": original_test_args, "resolved_args": test_args}) write_json(tests_dir / "api_schema.json", {"schema": schema, "api_names": discovered, "selected_api_name": api_name, "endpoint_parameters": endpoint_parameters}) if schema_choice_corrections: append_event(events_path, "payload", "success", "Normalized validation payload to the Gradio endpoint schema", {"api_name": api_name, "choice_corrections": schema_choice_corrections}) write_live_status(run_dir, stage="payload", status="success", message="Payload normalized to Gradio schema", data={"api_name": api_name, "choice_corrections": schema_choice_corrections}) elif not replay_source: append_event(events_path, "payload", "success", "Resolved validation payload", {"api_name": api_name, "payload_source": payload_source}) write_live_status(run_dir, stage="payload", status="success", message="Payload resolved", data={"api_name": api_name, "payload_source": payload_source}) if autofill_meta.get("args_were_autofilled"): append_event(events_path, "payload", "success", "Filled validation arguments from discovered Gradio schema", {"api_name": api_name, "args_source": autofill_meta.get("args_source"), "endpoint_parameters": endpoint_parameters, "effective_args": test_args}) if dropped_kwargs: append_event(events_path, "generation_smoke", "warning", "Ignored unsupported validation kwargs for this Gradio endpoint", {"api_name": api_name, "ignored_kwargs": sorted(dropped_kwargs), "endpoint_parameters": endpoint_parameters}) append_event(events_path, "generation_smoke", "started", "Calling live generation endpoint", {"api_name": api_name, "expected_output_type": expected_output_type, "replay_source_used": bool(replay_source)}) stage = "predict" started = time.time() try: result = client.predict(*test_args, api_name=api_name, **safe_kwargs) except Exception as exc: message = str(exc) if is_schema_choice_type_error(exc): corrected_args, retry_choice_corrections = coerce_validation_args_to_schema_choices(api_name, schema, test_args) if retry_choice_corrections and corrected_args != test_args: append_event(events_path, "generation_smoke", "warning", "Retrying validation with Gradio schema choice types", {"api_name": api_name, "error": message[:1500], "choice_corrections": retry_choice_corrections}) test_args = corrected_args write_json(tests_dir / "generation_smoke_payload_retry.json", {"api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "retry_reason": "schema_choice_type_mismatch", "choice_corrections": retry_choice_corrections}) write_json(tests_dir / "resolved_validation_request.json", {"api_name": api_name, "test_args": test_args, "test_kwargs": safe_kwargs, "expected_output_type": expected_output_type, "validation_mode": validation_mode, "payload_source": payload_source, "schema_choice_corrections": schema_choice_corrections + retry_choice_corrections, "retry_reason": "schema_choice_type_mismatch"}) write_json(tests_dir / "schema_coercion.json", {"api_name": api_name, "changes": schema_choice_corrections + retry_choice_corrections, "original_args": original_test_args, "resolved_args": test_args, "retried": True}) result = client.predict(*test_args, api_name=api_name, **safe_kwargs) else: raise elif safe_kwargs and "not a valid key-word argument" in message: append_event(events_path, "generation_smoke", "warning", "Retrying validation without keyword arguments after Gradio rejected kwargs", {"api_name": api_name, "error": message[:1500], "dropped_kwargs": sorted(safe_kwargs)}) safe_kwargs = {} result = client.predict(*test_args, api_name=api_name) else: failure_type = "predict_failed" if is_schema_choice_type_error(message): failure_type = "schema_choice_type_error_after_coercion" elif "not a valid key-word argument" in message: failure_type = "kwargs_rejected" write_failure(exc, failure_type=failure_type, stage_override="predict") raise latency = time.time() - started ok, info = result_contains_expected_output(result, expected_output_type) copied = copy_result_artifacts(result, run_dir) video_inspection = {"schema_version": "video_artifact_inspection.v198_24", "videos": [], "video_count": 0, "has_video_artifact": False, "quality_assessment": "not_evaluated"} if str(expected_output_type or "").lower() == "video": videos = [] for item in copied or []: path = Path(str(item)) if path.suffix.lower() in {".mp4", ".mov", ".webm", ".avi", ".mkv"}: record = {"path": str(path), "extension": path.suffix.lower(), "exists": path.exists()} if path.exists(): record["size_bytes"] = path.stat().st_size record["container_valid"] = bool(path.stat().st_size > 0) videos.append(record) video_inspection.update({"videos": videos, "video_count": len(videos), "has_video_artifact": bool(videos)}) write_json(tests_dir / "video_artifact_inspection.json", video_inspection) payload = { "status": "success" if ok else "failed", "target_space": target_space_id, "api_name": api_name, "discovered_api_names": discovered, "test_args": test_args, "test_kwargs": safe_kwargs, "effective_args": test_args, "effective_kwargs": safe_kwargs, "args_source": autofill_meta.get("args_source"), "args_were_autofilled": bool(autofill_meta.get("args_were_autofilled")), "original_test_kwargs": test_kwargs, "ignored_test_kwargs": dropped_kwargs, "endpoint_parameters": endpoint_parameters, "expected_output_type": expected_output_type, "latency_seconds": round(latency, 3), "observed_latency_seconds": round(latency, 3), "result_info": info, "copied_artifacts": copied, "video_artifact_inspection": video_inspection, "recommended_zero_gpu_duration_seconds": int(max(30, min(300, latency * 2 + 15))), "recommendation_source": "live_gradio_predict", "recommendation_confidence": "measured", "measurement_note": "Measured from a live gradio_client.predict call, including Gradio/API/network/result serialization overhead.", "validated_at": now(), } write_json(tests_dir / "generation_smoke.json", payload) write_json(tests_dir / "test_result.json", payload) if ok: append_event(events_path, "generation_smoke", "success", "Live generation smoke test passed", {"latency_seconds": payload["latency_seconds"], "recommended_zero_gpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "recommended_zerogpu_duration_seconds": payload["recommended_zero_gpu_duration_seconds"], "copied_artifacts": copied[:5]}) return payload append_event(events_path, "generation_smoke", "failed", "Live generation returned an unexpected output type", payload) write_failure(RuntimeError("Generation smoke test failed: unexpected output type"), failure_type="unexpected_output_type", stage_override="output_verification") raise RuntimeError("Generation smoke test failed: unexpected output type") except Exception as exc: # If a deeper block already wrote the diagnostic, keep it and simply re-raise. if not (tests_dir / "validation_failure_diagnosis.json").exists(): write_failure(exc) raise def main(): run_id = os.environ["RUN_ID"] username = os.environ.get("HF_USERNAME", "unknown") output_root = Path(os.environ.get("OUTPUT_ROOT", "/output")) target_space_id = os.environ["TARGET_SPACE_ID"].strip() parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip() token = os.environ.get("HF_TOKEN") run_dir = output_root / "runs" / run_id events_path = run_dir / "events.jsonl" state_path = run_dir / "state.json" append_event(events_path, "bootstrap", "started", "Existing Space validation worker started", {"target_space_id": target_space_id}) write_json(state_path, {"run_id": run_id, "kind": "linked_space_validation", "status": "running", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "created_by": username, "updated_at": now()}) if not token: raise RuntimeError("HF_TOKEN is missing") ensure_hf_token_context(run_dir, events_path) write_auth_probe(run_dir, events_path, "before_linked_space_test", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_linked_space_test"), raise_on_unsafe=True) if not TARGET_RE.match(target_space_id): raise ValueError("TARGET_SPACE_ID must look like owner/space-name") if not parent_build_run_id: raise ValueError("PARENT_BUILD_RUN_ID is required: Space Test validations must be linked to a Build Run.") try: install_deps(events_path) from huggingface_hub import HfApi api = HfApi(token=token) whoami = api.whoami(token=token) append_event(events_path, "auth", "success", "Authenticated inside validation Job", {"whoami_name": whoami.get("name")}) live = wait_until_live(api, target_space_id, token, run_dir, events_path, timeout_s=int(os.environ.get("LIVE_TIMEOUT_SECONDS", "1800"))) smoke = smoke_generate(target_space_id, token, run_dir, events_path) final_state = { "run_id": run_id, "kind": "linked_space_validation", "status": "full_inference_success", "message": "Existing Space passed live health/schema validation and generation smoke test.", "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "live_validation": live, "generation_smoke": smoke, "updated_at": now(), } final_state["parent_build_run_id"] = parent_build_run_id final_state["source_kind"] = "build_run_prefill" write_json(state_path, final_state) write_final_summary(run_dir, final_state, {}, smoke, status="full_inference_success", message=final_state.get("message")) if parent_build_run_id: parent_dir = output_root / "runs" / parent_build_run_id parent_dir.mkdir(parents=True, exist_ok=True) runtime = live.get("runtime") if isinstance(live, dict) else {} validation_hardware = (runtime or {}).get("hardware") or (runtime or {}).get("requested_hardware") or "unknown" api_name_used = smoke.get("api_name") or os.environ.get("API_NAME") or "/generate" expected_output_type_used = smoke.get("expected_output_type") or os.environ.get("EXPECTED_OUTPUT_TYPE") or "any" effective_status_on_success = os.environ.get("EFFECTIVE_STATUS_ON_SUCCESS") or "validated_after_space_test" space_test_policy_mode = os.environ.get("SPACE_TEST_POLICY_MODE") or "complete" manual_status = { "schema_version": "post_build_validation.v198_26_7", "status": "success", "effective_status": effective_status_on_success, "legacy_effective_status": "validated_after_manual_space_test", "space_test_policy_mode": space_test_policy_mode, "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "api_name": api_name_used, "expected_output_type": expected_output_type_used, "latency_seconds": smoke.get("latency_seconds"), "observed_latency_seconds": smoke.get("observed_latency_seconds") or smoke.get("latency_seconds"), "recommended_zero_gpu_duration_seconds": smoke.get("recommended_zero_gpu_duration_seconds"), "recommendation_source": "linked_space_test", "recommendation_hardware": validation_hardware, "hardware_used_for_validation": validation_hardware, "validated_at": now(), "applies_to_target_space": True, } write_json(parent_dir / "post_build_validation_status.json", manual_status) write_json(parent_dir / "manual_validation_status.json", manual_status) linked_path = parent_dir / "linked_validations.json" try: linked = json.loads(linked_path.read_text(encoding="utf-8")) except Exception: linked = {"parent_build_run_id": parent_build_run_id, "validations": []} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] validations.append(manual_status) linked = {"parent_build_run_id": parent_build_run_id, "effective_status": effective_status_on_success, "validations": validations, "updated_at": now()} write_json(linked_path, linked) report = f"""# Agentic Space Factory — Existing Space Validation Report Status: **full_inference_success** Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_id}) ## Generation smoke test ```json {json.dumps(smoke, indent=2, ensure_ascii=False)} ``` ## Notes - This validation is intended for Spaces whose hardware was set manually after generation. - Latency is measured from the live Gradio endpoint call. - The recommended ZeroGPU duration is a rough estimate from this live run, not a guarantee. """ (run_dir / "report.md").write_text(report, encoding="utf-8") append_event(events_path, "report_write", "success", "Wrote report.md") try: publish_eval_record(run_dir, phase="final", events_path=events_path) append_event(events_path, "anonymous_eval", "success", "Published anonymized validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False}) except Exception as eval_exc: append_event(events_path, "anonymous_eval", "warning", "Could not publish anonymized validation evaluation record", {"error": str(eval_exc)[:1000]}) # Rewrite the terminal summary at the very end of the success path. # v194: launch metadata can leave summary.json as running if the # final lightweight summary is not the last writer visible in the # bucket sync. Keep manual audits and list views terminal-consistent. write_final_summary(run_dir, final_state, {}, smoke, status="full_inference_success", message=final_state.get("message")) append_event(events_path, "summary_write", "success", "Wrote terminal validation summary", {"status": "full_inference_success"}) append_event(events_path, "done", "full_inference_success", "Existing Space validation completed", {"latency_seconds": smoke.get("latency_seconds")}) except Exception as exc: collect_space_logs(target_space_id, token or "", run_dir, events_path) auth_blocked = isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc) details = {"error": str(exc)[:4000]} if auth_blocked: details["status_reason"] = "auth_refresh_required" details["failure_type"] = "auth_refresh_required" terminal_status = "auth_refresh_required" if auth_blocked else "failed" failure_state = {"run_id": run_id, "kind": "linked_space_validation", "status": terminal_status, "message": "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else str(exc), "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "details": details, "updated_at": now()} write_json(state_path, failure_state) write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed") if parent_build_run_id: parent_dir = output_root / "runs" / parent_build_run_id parent_dir.mkdir(parents=True, exist_ok=True) failed_status = {"schema_version": "post_build_validation.v198_26_7", "status": terminal_status, "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "details": details, "updated_at": now(), "effective_status": "unchanged"} linked_path = parent_dir / "linked_validations.json" linked = read_json(linked_path, {"parent_build_run_id": parent_build_run_id, "validations": []}) or {"parent_build_run_id": parent_build_run_id, "validations": []} validations = linked.get("validations") if isinstance(linked, dict) else [] if not isinstance(validations, list): validations = [] validations.append(failed_status) linked = {"parent_build_run_id": parent_build_run_id, "validations": validations, "updated_at": now()} if parent_has_successful_manual_validation(parent_dir): linked["effective_status"] = "validated_after_space_test" linked["legacy_effective_status"] = "validated_after_manual_space_test" append_event(events_path, "linked_validation", "warning", "Linked validation failed but parent already has a successful manual validation; parent effective status is preserved", {"parent_build_run_id": parent_build_run_id}) else: write_json(parent_dir / "manual_validation_status.json", failed_status) write_json(linked_path, linked) (run_dir / "report.md").write_text(f"# Existing Space Validation Failed\n\n```json\n{json.dumps(details, indent=2, ensure_ascii=False)}\n```\n", encoding="utf-8") append_event(events_path, "failure", "failed", "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else "Existing Space validation failed", details) try: publish_eval_record(run_dir, phase="failure", events_path=events_path) append_event(events_path, "anonymous_eval", "success", "Published anonymized failed validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False}) except Exception as eval_exc: append_event(events_path, "anonymous_eval", "warning", "Could not publish anonymized failed validation evaluation record", {"error": str(eval_exc)[:1000]}) # v194: also rewrite failure summary as the final visible file update # for linked validations, so a failed validation cannot remain listed # as running after the worker has already terminalized state.json. write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed") append_event(events_path, "summary_write", "success", "Wrote terminal failed validation summary", {"status": terminal_status}) raise SystemExit(1) if __name__ == "__main__": main() ''' def universal_model_card_worker_script() -> str: """Return the universal model-card builder worker script source.""" return UNIVERSAL_MODEL_CARD_WORKER_SCRIPT def validate_existing_space_worker_script() -> str: """Return the existing-Space validation worker script source.""" return VALIDATE_EXISTING_SPACE_WORKER_SCRIPT def encoded_universal_model_card_worker_script() -> str: """Return the base64-encoded universal model-card builder worker script. Kept for backwards compatibility in tests/imports. New Jobs read the worker from WORKER_SCRIPT_PATH to avoid oversized env/argv payloads. """ return _encode(UNIVERSAL_MODEL_CARD_WORKER_SCRIPT) def encoded_validate_existing_space_worker_script() -> str: """Return the base64-encoded existing-Space validation worker script. Kept for backwards compatibility in tests/imports. New Jobs read the worker from WORKER_SCRIPT_PATH to avoid oversized env/argv payloads. """ return _encode(VALIDATE_EXISTING_SPACE_WORKER_SCRIPT) def python_decode_and_run_command() -> list[str]: """Small command list for `run_job`. The worker script is stored in the run bucket and exposed through the Job volume as WORKER_SCRIPT_PATH. This avoids putting the large worker source in argv/env, which can make HF Jobs fail before Python starts with `argument list too long`. WORKER_SCRIPT_B64 remains as a tiny compatibility fallback for older tests/manual launch paths. """ runner = textwrap.dedent( """ import base64, os, pathlib, subprocess, sys script_path = os.environ.get('WORKER_SCRIPT_PATH') if script_path: path = pathlib.Path(script_path) if not path.exists(): raise SystemExit(f'Worker script not found at WORKER_SCRIPT_PATH={script_path}') else: script_b64 = os.environ.get('WORKER_SCRIPT_B64') if not script_b64: raise SystemExit('Missing WORKER_SCRIPT_PATH') path = pathlib.Path('/tmp/space_factory_worker.py') path.write_text(base64.b64decode(script_b64).decode('utf-8'), encoding='utf-8') raise SystemExit(subprocess.call([sys.executable, str(path)])) """ ).strip() return ["python", "-c", runner]