Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Upload 14 files
Browse files- src/auth.py +59 -0
- src/bucket.py +2 -0
- src/eval_archive.py +1 -1
- src/model_scan.py +210 -0
- src/progress.py +4 -1
- src/timeline_model.py +17 -7
- src/version.py +2 -2
- src/view_models.py +66 -8
- src/worker_payload.py +767 -54
src/auth.py
CHANGED
|
@@ -46,6 +46,64 @@ class OAuthContext:
|
|
| 46 |
return self.expires_at <= datetime.now(timezone.utc)
|
| 47 |
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
def _parse_scope(scope: Any) -> set[str]:
|
| 50 |
if not scope:
|
| 51 |
return set()
|
|
@@ -167,6 +225,7 @@ def public_oauth_context(ctx: OAuthContext) -> dict[str, Any]:
|
|
| 167 |
"scopes": sorted(ctx.scopes),
|
| 168 |
"missing_scopes": ctx.missing_scopes,
|
| 169 |
"expires_at": ctx.expires_at.isoformat() if ctx.expires_at else None,
|
|
|
|
| 170 |
"authenticated": True,
|
| 171 |
}
|
| 172 |
|
|
|
|
| 46 |
return self.expires_at <= datetime.now(timezone.utc)
|
| 47 |
|
| 48 |
|
| 49 |
+
def oauth_lifetime_summary(ctx: OAuthContext, *, now: datetime | None = None) -> dict[str, Any]:
|
| 50 |
+
"""Return a safe, UI-ready OAuth lifetime summary without exposing tokens."""
|
| 51 |
+
current = now or datetime.now(timezone.utc)
|
| 52 |
+
if current.tzinfo is None:
|
| 53 |
+
current = current.replace(tzinfo=timezone.utc)
|
| 54 |
+
if ctx.expires_at is None:
|
| 55 |
+
return {
|
| 56 |
+
"status": "unknown",
|
| 57 |
+
"severity": "neutral",
|
| 58 |
+
"label": "HF Auth: expiry unknown",
|
| 59 |
+
"expires_at": None,
|
| 60 |
+
"seconds_until_expiry": None,
|
| 61 |
+
"recommendation": "OAuth expiry is not exposed by this runtime. Refresh sign-in before long builds if unsure.",
|
| 62 |
+
}
|
| 63 |
+
seconds = int((ctx.expires_at - current).total_seconds())
|
| 64 |
+
if seconds <= 0:
|
| 65 |
+
status = "expired"
|
| 66 |
+
severity = "error"
|
| 67 |
+
label = "HF Auth: expired"
|
| 68 |
+
recommendation = "Sign in again before launching or validating Jobs."
|
| 69 |
+
elif seconds < 30 * 60:
|
| 70 |
+
status = "critical"
|
| 71 |
+
severity = "error"
|
| 72 |
+
label = f"HF Auth: {format_duration_compact(seconds)} left"
|
| 73 |
+
recommendation = "Refresh sign-in before launching a build, repair, or linked Space Test."
|
| 74 |
+
elif seconds < 90 * 60:
|
| 75 |
+
status = "warning"
|
| 76 |
+
severity = "warn"
|
| 77 |
+
label = f"HF Auth: {format_duration_compact(seconds)} left"
|
| 78 |
+
recommendation = "Refresh sign-in before long or high-risk builds."
|
| 79 |
+
else:
|
| 80 |
+
status = "ok"
|
| 81 |
+
severity = "success"
|
| 82 |
+
label = f"HF Auth: {format_duration_compact(seconds)} left"
|
| 83 |
+
recommendation = "OAuth session looks safe for normal builds."
|
| 84 |
+
return {
|
| 85 |
+
"status": status,
|
| 86 |
+
"severity": severity,
|
| 87 |
+
"label": label,
|
| 88 |
+
"expires_at": ctx.expires_at.isoformat(),
|
| 89 |
+
"seconds_until_expiry": seconds,
|
| 90 |
+
"recommendation": recommendation,
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def format_duration_compact(seconds: int | float | None) -> str:
|
| 95 |
+
if seconds is None:
|
| 96 |
+
return "unknown"
|
| 97 |
+
total = max(0, int(seconds))
|
| 98 |
+
hours, rem = divmod(total, 3600)
|
| 99 |
+
minutes = rem // 60
|
| 100 |
+
if hours:
|
| 101 |
+
return f"{hours}h {minutes}m"
|
| 102 |
+
if minutes:
|
| 103 |
+
return f"{minutes}m"
|
| 104 |
+
return f"{total}s"
|
| 105 |
+
|
| 106 |
+
|
| 107 |
def _parse_scope(scope: Any) -> set[str]:
|
| 108 |
if not scope:
|
| 109 |
return set()
|
|
|
|
| 225 |
"scopes": sorted(ctx.scopes),
|
| 226 |
"missing_scopes": ctx.missing_scopes,
|
| 227 |
"expires_at": ctx.expires_at.isoformat() if ctx.expires_at else None,
|
| 228 |
+
"auth_lifetime": oauth_lifetime_summary(ctx),
|
| 229 |
"authenticated": True,
|
| 230 |
}
|
| 231 |
|
src/bucket.py
CHANGED
|
@@ -1073,6 +1073,8 @@ def read_run_bundle(run_id: str, *, bucket_source: str, token: str | None = None
|
|
| 1073 |
"hardware_strategy": _safe_read_json(f"{paths.root}/hardware_strategy.json", token=token),
|
| 1074 |
"hardware_attempts": _safe_read_json(f"{paths.root}/hardware_attempts.json", token=token),
|
| 1075 |
"technical_blockers": _safe_read_json(f"{paths.root}/generated/TECHNICAL_BLOCKERS.json", token=token) or _safe_read_json(f"{paths.root}/TECHNICAL_BLOCKERS.json", token=token),
|
|
|
|
|
|
|
| 1076 |
"model_analysis": _safe_read_json(f"{paths.root}/model_analysis.json", token=token),
|
| 1077 |
"pi_model_resolution": _safe_read_json(f"{paths.root}/pi_model_resolution.json", token=token) or (read_json(paths.state, token=token) or {}).get("pi_model_resolution") or {},
|
| 1078 |
"space_runtime": _safe_read_json(f"{paths.root}/space_runtime.json", token=token),
|
|
|
|
| 1073 |
"hardware_strategy": _safe_read_json(f"{paths.root}/hardware_strategy.json", token=token),
|
| 1074 |
"hardware_attempts": _safe_read_json(f"{paths.root}/hardware_attempts.json", token=token),
|
| 1075 |
"technical_blockers": _safe_read_json(f"{paths.root}/generated/TECHNICAL_BLOCKERS.json", token=token) or _safe_read_json(f"{paths.root}/TECHNICAL_BLOCKERS.json", token=token),
|
| 1076 |
+
"inference_contract": _safe_read_json(f"{paths.root}/generated/INFERENCE_CONTRACT.json", token=token) or _safe_read_json(f"{paths.root}/INFERENCE_CONTRACT.json", token=token),
|
| 1077 |
+
"repair_outcome": _safe_read_json(f"{paths.root}/repair_outcome.json", token=token) or _safe_read_json(f"{paths.root}/repair/REPAIR_OUTCOME.json", token=token),
|
| 1078 |
"model_analysis": _safe_read_json(f"{paths.root}/model_analysis.json", token=token),
|
| 1079 |
"pi_model_resolution": _safe_read_json(f"{paths.root}/pi_model_resolution.json", token=token) or (read_json(paths.state, token=token) or {}).get("pi_model_resolution") or {},
|
| 1080 |
"space_runtime": _safe_read_json(f"{paths.root}/space_runtime.json", token=token),
|
src/eval_archive.py
CHANGED
|
@@ -9,7 +9,7 @@ from .bucket import RunPaths, read_json, read_text, write_json
|
|
| 9 |
from .eval_config import effective_eval_config
|
| 10 |
from .security import redact
|
| 11 |
|
| 12 |
-
_TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "manual_hardware_required"}
|
| 13 |
|
| 14 |
|
| 15 |
def _now() -> str:
|
|
|
|
| 9 |
from .eval_config import effective_eval_config
|
| 10 |
from .security import redact
|
| 11 |
|
| 12 |
+
_TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"}
|
| 13 |
|
| 14 |
|
| 15 |
def _now() -> str:
|
src/model_scan.py
CHANGED
|
@@ -188,6 +188,192 @@ def _extract_model_card_signals(readme: str, model_index: dict[str, Any]) -> dic
|
|
| 188 |
}
|
| 189 |
|
| 190 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
def _classify(score: int, *, blocking: bool = False) -> str:
|
| 192 |
if blocking:
|
| 193 |
return "unsupported"
|
|
@@ -356,6 +542,24 @@ def analyze_model_metadata(
|
|
| 356 |
|
| 357 |
score = max(0, min(100, score))
|
| 358 |
verdict = _classify(score, blocking=unsupported)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
if not recommendations:
|
| 360 |
if verdict == "safe":
|
| 361 |
recommendations.append("Good candidate for Strict inference.")
|
|
@@ -381,6 +585,8 @@ def analyze_model_metadata(
|
|
| 381 |
"risk_signals": risk[:10],
|
| 382 |
"recommendations": recommendations[:6],
|
| 383 |
"expected_output_type": expected_output_type or "",
|
|
|
|
|
|
|
| 384 |
"metadata": {
|
| 385 |
"pipeline_tag": pipeline_tag or "",
|
| 386 |
"library_name": library_name or "",
|
|
@@ -397,6 +603,10 @@ def analyze_model_metadata(
|
|
| 397 |
"runtime_hints": runtime_hints,
|
| 398 |
"expected_output_type": expected_output_type or "",
|
| 399 |
"diffusers_standard": bool(is_diffusers and has_model_index and has_safetensors),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 400 |
},
|
| 401 |
}
|
| 402 |
|
|
|
|
| 188 |
}
|
| 189 |
|
| 190 |
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
NATIVE_KERNEL_PATTERNS: tuple[tuple[str, tuple[str, ...], str], ...] = (
|
| 196 |
+
("flash_attn", ("flash-attn", "flash_attn", "flash attention", "flashattention", "flash-attention", "enable_flashattn", "flash_attention_2"), "Flash Attention dependency or runtime flag"),
|
| 197 |
+
("xformers", ("xformers", "memory_efficient_attention"), "xFormers attention dependency"),
|
| 198 |
+
("triton", ("triton", "triton kernel", "@triton", "triton.jit"), "Triton/fused kernel dependency"),
|
| 199 |
+
("custom_cuda", ("cuda extension", "custom cuda", "cpp_extension", "setup.py build_ext", "fused kernel", "fused ops", "custom kernel"), "Custom native/CUDA extension"),
|
| 200 |
+
("attention_interface", ("attentioninterface", "attention interface", "attn_implementation"), "Transformers attention backend hook"),
|
| 201 |
+
("hf_kernels", ("hf kernels", "kernel hub", "hugging face kernels", "kernels-community", "from kernels import", "pip install kernels"), "HF Kernels / Kernel Hub mention"),
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def build_kernel_strategy(*, readme: str | None = None, files: list[str] | None = None) -> dict[str, Any]:
|
| 206 |
+
"""Return a visibility-only mitigation plan for native kernel dependencies.
|
| 207 |
+
|
| 208 |
+
This is metadata/readme based. It must not inject packages or mark a model
|
| 209 |
+
terminally blocked by itself; it gives Pi and the UI a safer strategy than
|
| 210 |
+
blindly adding source-built CUDA packages to requirements.txt.
|
| 211 |
+
"""
|
| 212 |
+
text = (readme or "").lower()
|
| 213 |
+
file_list = [str(f).lower() for f in (files or [])]
|
| 214 |
+
detected: list[str] = []
|
| 215 |
+
signals: list[str] = []
|
| 216 |
+
for key, patterns, label in NATIVE_KERNEL_PATTERNS:
|
| 217 |
+
if any(pattern in text for pattern in patterns):
|
| 218 |
+
detected.append(key)
|
| 219 |
+
signals.append(label)
|
| 220 |
+
if any(f.endswith((".cu", ".cuh")) or "/csrc/" in f or f.startswith("csrc/") for f in file_list):
|
| 221 |
+
if "custom_cuda" not in detected:
|
| 222 |
+
detected.append("custom_cuda")
|
| 223 |
+
signals.append("Native CUDA/C++ source files in repository")
|
| 224 |
+
if any(f.endswith((".cpp", ".cc")) and ("cuda" in f or "/csrc/" in f or f.startswith("csrc/")) for f in file_list):
|
| 225 |
+
if "custom_cuda" not in detected:
|
| 226 |
+
detected.append("custom_cuda")
|
| 227 |
+
signals.append("Native C++ extension source files in repository")
|
| 228 |
+
|
| 229 |
+
native_risk = any(k in detected for k in {"flash_attn", "xformers", "triton", "custom_cuda"})
|
| 230 |
+
candidate_backends: list[str] = []
|
| 231 |
+
if native_risk:
|
| 232 |
+
candidate_backends.extend(["torch_sdpa", "hf_kernels", "xformers_wheel"])
|
| 233 |
+
if "attention_interface" in detected:
|
| 234 |
+
candidate_backends.append("transformers_attention_interface")
|
| 235 |
+
if "hf_kernels" in detected and "hf_kernels" not in candidate_backends:
|
| 236 |
+
candidate_backends.append("hf_kernels")
|
| 237 |
+
|
| 238 |
+
rejected_actions = []
|
| 239 |
+
if native_risk:
|
| 240 |
+
rejected_actions.append({
|
| 241 |
+
"action": "blind_pip_install_native_cuda_package",
|
| 242 |
+
"reason": "Source-built CUDA/native packages are fragile in Spaces and can mismatch the managed PyTorch/CUDA runtime.",
|
| 243 |
+
})
|
| 244 |
+
if "flash_attn" in detected:
|
| 245 |
+
rejected_actions.append({
|
| 246 |
+
"action": "pip_install_flash_attn_without_fallback",
|
| 247 |
+
"reason": "Prefer PyTorch SDPA, Transformers AttentionInterface, HF Kernels/Kernel Hub, or a compatible wheel before forcing flash-attn source builds.",
|
| 248 |
+
})
|
| 249 |
+
|
| 250 |
+
selected = "none"
|
| 251 |
+
if native_risk:
|
| 252 |
+
selected = "prefer_runtime_backends_before_source_builds"
|
| 253 |
+
elif "hf_kernels" in detected:
|
| 254 |
+
selected = "hf_kernels_available_if_model_uses_supported_kernel"
|
| 255 |
+
|
| 256 |
+
return {
|
| 257 |
+
"schema_version": "kernel_strategy.v1",
|
| 258 |
+
"native_kernel_risk": bool(native_risk),
|
| 259 |
+
"detected_dependencies": detected,
|
| 260 |
+
"signals": signals[:10],
|
| 261 |
+
"selected_strategy": selected,
|
| 262 |
+
"candidate_backends": candidate_backends[:8],
|
| 263 |
+
"rejected_actions": rejected_actions,
|
| 264 |
+
"requires_manual_review": bool("custom_cuda" in detected),
|
| 265 |
+
"pi_instruction": (
|
| 266 |
+
"Do not compile native CUDA packages blindly. Prefer PyTorch SDPA, compatible wheels, HF Kernels/Kernel Hub, "
|
| 267 |
+
"Transformers AttentionInterface, or Diffusers attention processors when they match the required operation. "
|
| 268 |
+
"Declare a technical blocker if a strict native extension has no plausible fallback."
|
| 269 |
+
if native_risk else
|
| 270 |
+
"No native kernel dependency was detected by the metadata scan."
|
| 271 |
+
),
|
| 272 |
+
"visibility_only": True,
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _has_any(text: str, patterns: tuple[str, ...]) -> bool:
|
| 277 |
+
lower = text.lower()
|
| 278 |
+
return any(pattern in lower for pattern in patterns)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def _build_complexity_assessment(
|
| 282 |
+
*,
|
| 283 |
+
pipeline_tag: str | None,
|
| 284 |
+
library_name: str | None,
|
| 285 |
+
tags: list[str],
|
| 286 |
+
files: list[str],
|
| 287 |
+
readme: str,
|
| 288 |
+
expected_output_type: str | None,
|
| 289 |
+
custom_code: bool,
|
| 290 |
+
gated: Any,
|
| 291 |
+
) -> dict[str, Any]:
|
| 292 |
+
"""Estimate build/runtime risk without changing the existing pre-scan verdict.
|
| 293 |
+
|
| 294 |
+
This is deliberately a lightweight metadata/model-card heuristic. It helps
|
| 295 |
+
the UI warn users before long Jobs; it is not a hard launch gate.
|
| 296 |
+
"""
|
| 297 |
+
lower_readme = (readme or "").lower()
|
| 298 |
+
lower_files = [str(f).lower() for f in files]
|
| 299 |
+
tag_set = {str(t).lower() for t in tags}
|
| 300 |
+
task = (pipeline_tag or "").lower()
|
| 301 |
+
library = (library_name or "").lower()
|
| 302 |
+
|
| 303 |
+
points = 0
|
| 304 |
+
signals: list[str] = []
|
| 305 |
+
mitigations: list[str] = []
|
| 306 |
+
|
| 307 |
+
def add(points_delta: int, signal: str, mitigation: str | None = None) -> None:
|
| 308 |
+
nonlocal points
|
| 309 |
+
points += points_delta
|
| 310 |
+
if signal not in signals:
|
| 311 |
+
signals.append(signal)
|
| 312 |
+
if mitigation and mitigation not in mitigations:
|
| 313 |
+
mitigations.append(mitigation)
|
| 314 |
+
|
| 315 |
+
if expected_output_type == "video" or "video" in task or {"text-to-video", "image-to-video"} & tag_set:
|
| 316 |
+
add(3, "video or image-to-video output", "Expect a long build/repair cycle and prefer strong fallback hardware.")
|
| 317 |
+
if any(word in lower_readme for word in ("avatar", "audio-driven", "audio driven", "talking head", "lip sync", "lip-sync")):
|
| 318 |
+
add(2, "audio/video avatar workflow", "Refresh sign-in before launch and expect larger validation payloads.")
|
| 319 |
+
if custom_code or "trust_remote_code" in tag_set or "custom_code" in tag_set:
|
| 320 |
+
add(2, "custom code or trust_remote_code required", "Review generated code and blockers before paid hardware attempts.")
|
| 321 |
+
if gated in {True, "auto", "manual"} or str(gated).lower() in {"true", "auto", "manual"}:
|
| 322 |
+
add(1, "gated model access", "Confirm the same HF account has accepted access terms.")
|
| 323 |
+
|
| 324 |
+
if _has_any(lower_readme, ("torchrun", "nproc_per_node", "distributed", "init_process_group", "nccl", "context_parallel", "tensor parallel", "pipeline parallel", "multi-gpu", "multi gpu")):
|
| 325 |
+
add(4, "multi-GPU / distributed runtime hints", "Treat ZeroGPU as unlikely unless Pi can prove a single-GPU refactor.")
|
| 326 |
+
if _has_any(lower_readme, ("flash-attn", "flash_attn", "flash attention", "flashattention", "flash-attention")):
|
| 327 |
+
add(2, "flash-attn or custom attention dependency", "Prefer PyTorch SDPA, xformers wheels, or HF Kernels before source builds.")
|
| 328 |
+
if _has_any(lower_readme, ("xformers", "triton", "cuda extension", "fused kernel", "fused ops", "custom kernel")) or any(f.endswith((".cu", ".cpp")) or "/csrc/" in f for f in lower_files):
|
| 329 |
+
add(2, "native CUDA/kernel dependency risk", "Prefer runtime-compatible wheels or HF Kernels over compiling during Space build.")
|
| 330 |
+
if "ffmpeg" in lower_readme or any("ffmpeg" in f for f in lower_files):
|
| 331 |
+
add(1, "ffmpeg or system media dependency", "Ensure generated app declares media/system requirements clearly.")
|
| 332 |
+
if _has_any(lower_readme, ("conda ", "mamba ", "apt-get", "sudo apt", "pip install -e", "git clone")):
|
| 333 |
+
add(1, "non-standard install instructions", "Pi should vendor required code or simplify requirements for Spaces.")
|
| 334 |
+
|
| 335 |
+
weight_files = [f for f in lower_files if f.endswith(WEIGHT_EXTENSIONS)]
|
| 336 |
+
safetensor_shards = [f for f in lower_files if f.endswith(".safetensors")]
|
| 337 |
+
if len(weight_files) >= 20 or len(safetensor_shards) >= 12:
|
| 338 |
+
add(2, "many weight shards", "Expect longer cold start and validation windows.")
|
| 339 |
+
elif len(weight_files) >= 8 or len(safetensor_shards) >= 6:
|
| 340 |
+
add(1, "multiple weight shards", "Allow extra boot time before validation.")
|
| 341 |
+
|
| 342 |
+
vram_match = re.search(r"\b(?:vram|gpu memory|memory)\D{0,16}([3-9]\d|1\d{2})\s*(?:gb|gib)\b|\b([3-9]\d|1\d{2})\s*(?:gb|gib)\s*(?:vram|gpu)", lower_readme)
|
| 343 |
+
if vram_match:
|
| 344 |
+
add(3, "high VRAM mentioned in model card", "Prefer fixed GPU fallback and refresh sign-in before launch.")
|
| 345 |
+
|
| 346 |
+
# Keep the assessment independent from the existing pass/fail scan score.
|
| 347 |
+
if points >= 9:
|
| 348 |
+
level = "very_high"
|
| 349 |
+
label = "Very high risk"
|
| 350 |
+
recommended_seconds = 180 * 60
|
| 351 |
+
elif points >= 6:
|
| 352 |
+
level = "high"
|
| 353 |
+
label = "High risk"
|
| 354 |
+
recommended_seconds = 120 * 60
|
| 355 |
+
elif points >= 3:
|
| 356 |
+
level = "medium"
|
| 357 |
+
label = "Medium risk"
|
| 358 |
+
recommended_seconds = 60 * 60
|
| 359 |
+
else:
|
| 360 |
+
level = "low"
|
| 361 |
+
label = "Low risk"
|
| 362 |
+
recommended_seconds = 30 * 60
|
| 363 |
+
|
| 364 |
+
return {
|
| 365 |
+
"schema_version": "model_build_risk.v1",
|
| 366 |
+
"level": level,
|
| 367 |
+
"label": label,
|
| 368 |
+
"score": max(0, points),
|
| 369 |
+
"recommended_session_seconds": recommended_seconds,
|
| 370 |
+
"recommended_session_minutes": recommended_seconds // 60,
|
| 371 |
+
"signals": signals[:10],
|
| 372 |
+
"mitigations": mitigations[:6],
|
| 373 |
+
"summary": f"{label}; recommended HF session remaining: {recommended_seconds // 60}m+.",
|
| 374 |
+
"visibility_only": True,
|
| 375 |
+
}
|
| 376 |
+
|
| 377 |
def _classify(score: int, *, blocking: bool = False) -> str:
|
| 378 |
if blocking:
|
| 379 |
return "unsupported"
|
|
|
|
| 542 |
|
| 543 |
score = max(0, min(100, score))
|
| 544 |
verdict = _classify(score, blocking=unsupported)
|
| 545 |
+
build_risk = _build_complexity_assessment(
|
| 546 |
+
pipeline_tag=pipeline_tag,
|
| 547 |
+
library_name=library_name,
|
| 548 |
+
tags=tags,
|
| 549 |
+
files=files,
|
| 550 |
+
readme=readme,
|
| 551 |
+
expected_output_type=expected_output_type,
|
| 552 |
+
custom_code=custom_code,
|
| 553 |
+
gated=gated,
|
| 554 |
+
)
|
| 555 |
+
kernel_strategy = build_kernel_strategy(readme=readme, files=files)
|
| 556 |
+
if kernel_strategy.get("native_kernel_risk"):
|
| 557 |
+
recommendations.append("Native kernel risk detected; prefer PyTorch SDPA, HF Kernels/Kernel Hub, compatible wheels, or model-specific fallbacks before source builds.")
|
| 558 |
+
if build_risk["level"] in {"high", "very_high"}:
|
| 559 |
+
recommendations.append(
|
| 560 |
+
f"Refresh HF sign-in before launch; this scan recommends {build_risk['recommended_session_minutes']}m+ remaining for this model."
|
| 561 |
+
)
|
| 562 |
+
|
| 563 |
if not recommendations:
|
| 564 |
if verdict == "safe":
|
| 565 |
recommendations.append("Good candidate for Strict inference.")
|
|
|
|
| 585 |
"risk_signals": risk[:10],
|
| 586 |
"recommendations": recommendations[:6],
|
| 587 |
"expected_output_type": expected_output_type or "",
|
| 588 |
+
"build_risk": build_risk,
|
| 589 |
+
"kernel_strategy": kernel_strategy,
|
| 590 |
"metadata": {
|
| 591 |
"pipeline_tag": pipeline_tag or "",
|
| 592 |
"library_name": library_name or "",
|
|
|
|
| 603 |
"runtime_hints": runtime_hints,
|
| 604 |
"expected_output_type": expected_output_type or "",
|
| 605 |
"diffusers_standard": bool(is_diffusers and has_model_index and has_safetensors),
|
| 606 |
+
"build_risk_level": build_risk["level"],
|
| 607 |
+
"recommended_session_minutes": build_risk["recommended_session_minutes"],
|
| 608 |
+
"native_kernel_risk": bool(kernel_strategy.get("native_kernel_risk")),
|
| 609 |
+
"kernel_strategy": kernel_strategy,
|
| 610 |
},
|
| 611 |
}
|
| 612 |
|
src/progress.py
CHANGED
|
@@ -36,6 +36,7 @@ STEP_ORDER = [
|
|
| 36 |
"repair_upload",
|
| 37 |
"repair_validation",
|
| 38 |
"technical_blocker",
|
|
|
|
| 39 |
"manual_hardware_required",
|
| 40 |
"upload_files",
|
| 41 |
"space_runtime",
|
|
@@ -80,6 +81,7 @@ STEP_LABELS = {
|
|
| 80 |
"repair_upload": "Repair upload",
|
| 81 |
"repair_validation": "Repair validation",
|
| 82 |
"technical_blocker": "Technical blocker",
|
|
|
|
| 83 |
"manual_hardware_required": "Manual hardware",
|
| 84 |
"upload_files": "Upload files",
|
| 85 |
"space_runtime": "Space runtime",
|
|
@@ -116,6 +118,7 @@ DONE_STATUSES = {
|
|
| 116 |
"full_inference_success",
|
| 117 |
"manual_hardware_required",
|
| 118 |
"technical_blocker",
|
|
|
|
| 119 |
}
|
| 120 |
PARTIAL_STATUSES = {
|
| 121 |
"full_inference_candidate_health_passed",
|
|
@@ -341,7 +344,7 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
|
|
| 341 |
"current_step": current_step,
|
| 342 |
"current_step_label": STEP_LABELS.get(current_step, current_step),
|
| 343 |
"last_event": last_message or "No events yet",
|
| 344 |
-
"visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker"} else ("running" if overall_status == "running" else "neutral"))))),
|
| 345 |
"elapsed_seconds": max(0, elapsed),
|
| 346 |
"eta_seconds": None,
|
| 347 |
"timeline": timeline,
|
|
|
|
| 36 |
"repair_upload",
|
| 37 |
"repair_validation",
|
| 38 |
"technical_blocker",
|
| 39 |
+
"technical_blocker_boot_only",
|
| 40 |
"manual_hardware_required",
|
| 41 |
"upload_files",
|
| 42 |
"space_runtime",
|
|
|
|
| 81 |
"repair_upload": "Repair upload",
|
| 82 |
"repair_validation": "Repair validation",
|
| 83 |
"technical_blocker": "Technical blocker",
|
| 84 |
+
"technical_blocker_boot_only": "Boot-only blocker",
|
| 85 |
"manual_hardware_required": "Manual hardware",
|
| 86 |
"upload_files": "Upload files",
|
| 87 |
"space_runtime": "Space runtime",
|
|
|
|
| 118 |
"full_inference_success",
|
| 119 |
"manual_hardware_required",
|
| 120 |
"technical_blocker",
|
| 121 |
+
"technical_blocker_boot_only",
|
| 122 |
}
|
| 123 |
PARTIAL_STATUSES = {
|
| 124 |
"full_inference_candidate_health_passed",
|
|
|
|
| 344 |
"current_step": current_step,
|
| 345 |
"current_step_label": STEP_LABELS.get(current_step, current_step),
|
| 346 |
"last_event": last_message or "No events yet",
|
| 347 |
+
"visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))),
|
| 348 |
"elapsed_seconds": max(0, elapsed),
|
| 349 |
"eta_seconds": None,
|
| 350 |
"timeline": timeline,
|
src/timeline_model.py
CHANGED
|
@@ -51,7 +51,7 @@ PHASE_STEPS = {
|
|
| 51 |
"repair_validation",
|
| 52 |
},
|
| 53 |
"archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
|
| 54 |
-
"done": {"done", "failure", "technical_blocker", "manual_hardware_required"},
|
| 55 |
}
|
| 56 |
|
| 57 |
STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
|
|
@@ -62,7 +62,8 @@ FAILED_STATUSES = {"failed", "failure", "error"}
|
|
| 62 |
RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
|
| 63 |
CANCELLED_STATUSES = {"cancelled", "canceled"}
|
| 64 |
MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
|
| 65 |
-
BLOCKED_STATUSES = {"technical_blocker", "blocked"}
|
|
|
|
| 66 |
|
| 67 |
|
| 68 |
def _lower(value: Any) -> str:
|
|
@@ -316,7 +317,7 @@ def _is_record_not_ready(status: dict[str, Any]) -> bool:
|
|
| 316 |
|
| 317 |
|
| 318 |
def _has_terminal_event(bundle: dict[str, Any]) -> bool:
|
| 319 |
-
return any(_lower(event.get("step")) in {"done", "failure", "technical_blocker", "manual_hardware_required"} for event in _events(bundle))
|
| 320 |
|
| 321 |
|
| 322 |
def _latest_recovery_event(bundle: dict[str, Any]) -> dict[str, Any] | None:
|
|
@@ -351,9 +352,12 @@ def _verdict(bundle: dict[str, Any]) -> str:
|
|
| 351 |
return _lower(eval_record.get("verdict"))
|
| 352 |
gate = _gate(bundle)
|
| 353 |
state = bundle.get("state") or {}
|
| 354 |
-
|
|
|
|
| 355 |
normalized = _lower(value)
|
| 356 |
if normalized:
|
|
|
|
|
|
|
| 357 |
if normalized in SUCCESS_STATUSES:
|
| 358 |
return "full_inference_success" if normalized == "full_inference_success" else "success"
|
| 359 |
if normalized in PARTIAL_STATUSES:
|
|
@@ -361,7 +365,7 @@ def _verdict(bundle: dict[str, Any]) -> str:
|
|
| 361 |
if normalized in MANUAL_STATUSES:
|
| 362 |
return "manual_action_required"
|
| 363 |
if normalized in BLOCKED_STATUSES:
|
| 364 |
-
return "technical_blocker"
|
| 365 |
if normalized in FAILED_STATUSES:
|
| 366 |
return "failed"
|
| 367 |
if normalized in CANCELLED_STATUSES:
|
|
@@ -375,8 +379,10 @@ def _final_visual_status(verdict: str) -> str:
|
|
| 375 |
return "success"
|
| 376 |
if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
|
| 377 |
return "warn"
|
| 378 |
-
if verdict in {"failed", "failure", "technical_blocker"}:
|
| 379 |
return "error"
|
|
|
|
|
|
|
| 380 |
if verdict in {"manual_action_required", "cancelled"}:
|
| 381 |
return "stopped"
|
| 382 |
return "running"
|
|
@@ -387,8 +393,12 @@ def _status_label(verdict: str) -> tuple[str, str]:
|
|
| 387 |
return "Full inference success", "Space boots, generation passed, and latency was measured."
|
| 388 |
if verdict == "partial_validation":
|
| 389 |
return "Completed with partial validation", "The run finished, but full generation was not verified."
|
|
|
|
|
|
|
| 390 |
if verdict == "technical_blocker":
|
| 391 |
return "Technical blocker", "The run found a technical blocker."
|
|
|
|
|
|
|
| 392 |
if verdict == "manual_action_required":
|
| 393 |
return "Manual action required", "The run needs user action before validation can continue."
|
| 394 |
if verdict == "failed":
|
|
@@ -598,7 +608,7 @@ def _phase_status(bundle: dict[str, Any], phase: str, verdict: str) -> str:
|
|
| 598 |
return "complete"
|
| 599 |
if verdict in {"partial_validation", "partial", "completed_with_warnings", "health_only"}:
|
| 600 |
return "warning"
|
| 601 |
-
if verdict in {"failed", "technical_blocker"}:
|
| 602 |
return "failed"
|
| 603 |
if verdict in {"manual_action_required", "cancelled"}:
|
| 604 |
return "stopped"
|
|
|
|
| 51 |
"repair_validation",
|
| 52 |
},
|
| 53 |
"archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
|
| 54 |
+
"done": {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"},
|
| 55 |
}
|
| 56 |
|
| 57 |
STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
|
|
|
|
| 62 |
RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
|
| 63 |
CANCELLED_STATUSES = {"cancelled", "canceled"}
|
| 64 |
MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
|
| 65 |
+
BLOCKED_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
|
| 66 |
+
AUTH_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
|
| 67 |
|
| 68 |
|
| 69 |
def _lower(value: Any) -> str:
|
|
|
|
| 317 |
|
| 318 |
|
| 319 |
def _has_terminal_event(bundle: dict[str, Any]) -> bool:
|
| 320 |
+
return any(_lower(event.get("step")) in {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"} for event in _events(bundle))
|
| 321 |
|
| 322 |
|
| 323 |
def _latest_recovery_event(bundle: dict[str, Any]) -> dict[str, Any] | None:
|
|
|
|
| 352 |
return _lower(eval_record.get("verdict"))
|
| 353 |
gate = _gate(bundle)
|
| 354 |
state = bundle.get("state") or {}
|
| 355 |
+
repair = bundle.get("repair_outcome") or {}
|
| 356 |
+
for value in (gate.get("status"), state.get("gate_status"), state.get("status"), (bundle.get("summary") or {}).get("status"), repair.get("post_repair_validation"), repair.get("failure_type")):
|
| 357 |
normalized = _lower(value)
|
| 358 |
if normalized:
|
| 359 |
+
if normalized in AUTH_STATUSES:
|
| 360 |
+
return "auth_refresh_required"
|
| 361 |
if normalized in SUCCESS_STATUSES:
|
| 362 |
return "full_inference_success" if normalized == "full_inference_success" else "success"
|
| 363 |
if normalized in PARTIAL_STATUSES:
|
|
|
|
| 365 |
if normalized in MANUAL_STATUSES:
|
| 366 |
return "manual_action_required"
|
| 367 |
if normalized in BLOCKED_STATUSES:
|
| 368 |
+
return "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
|
| 369 |
if normalized in FAILED_STATUSES:
|
| 370 |
return "failed"
|
| 371 |
if normalized in CANCELLED_STATUSES:
|
|
|
|
| 379 |
return "success"
|
| 380 |
if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
|
| 381 |
return "warn"
|
| 382 |
+
if verdict in {"failed", "failure", "technical_blocker", "technical_blocker_boot_only"}:
|
| 383 |
return "error"
|
| 384 |
+
if verdict == "auth_refresh_required":
|
| 385 |
+
return "warn"
|
| 386 |
if verdict in {"manual_action_required", "cancelled"}:
|
| 387 |
return "stopped"
|
| 388 |
return "running"
|
|
|
|
| 393 |
return "Full inference success", "Space boots, generation passed, and latency was measured."
|
| 394 |
if verdict == "partial_validation":
|
| 395 |
return "Completed with partial validation", "The run finished, but full generation was not verified."
|
| 396 |
+
if verdict == "technical_blocker_boot_only":
|
| 397 |
+
return "Technical blocker", "Health may pass, but no generation endpoint exists for full inference."
|
| 398 |
if verdict == "technical_blocker":
|
| 399 |
return "Technical blocker", "The run found a technical blocker."
|
| 400 |
+
if verdict == "auth_refresh_required":
|
| 401 |
+
return "Auth refresh required", "HF OAuth expired or is too close to expiry; sign in again before retrying validation."
|
| 402 |
if verdict == "manual_action_required":
|
| 403 |
return "Manual action required", "The run needs user action before validation can continue."
|
| 404 |
if verdict == "failed":
|
|
|
|
| 608 |
return "complete"
|
| 609 |
if verdict in {"partial_validation", "partial", "completed_with_warnings", "health_only"}:
|
| 610 |
return "warning"
|
| 611 |
+
if verdict in {"failed", "technical_blocker", "technical_blocker_boot_only"}:
|
| 612 |
return "failed"
|
| 613 |
if verdict in {"manual_action_required", "cancelled"}:
|
| 614 |
return "stopped"
|
src/version.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
ASF_APP_VERSION = "v190.
|
| 4 |
-
ASF_RELEASE_NAME = "Agentic Space Factory v190.
|
| 5 |
|
| 6 |
|
| 7 |
def resolve_app_version(value: str | None = None) -> str:
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
ASF_APP_VERSION = "v190.33"
|
| 4 |
+
ASF_RELEASE_NAME = "Agentic Space Factory v190.33"
|
| 5 |
|
| 6 |
|
| 7 |
def resolve_app_version(value: str | None = None) -> str:
|
src/view_models.py
CHANGED
|
@@ -13,7 +13,7 @@ PRODUCT_STEPS = [{"id": step, "label": STEP_LABELS[step]} for step in STEP_ORDER
|
|
| 13 |
def _runs_prefix() -> str:
|
| 14 |
return settings.bucket_runs_prefix.strip().strip("/") or "runs"
|
| 15 |
|
| 16 |
-
TERMINAL_GLOBAL_STATUSES = {"succeeded", "partial", "failed", "cancelled", "blocked", "waiting_manual_action"}
|
| 17 |
SUCCESS_RAW_STATUSES = {
|
| 18 |
"success",
|
| 19 |
"done",
|
|
@@ -37,8 +37,9 @@ MANUAL_RAW_STATUSES = {
|
|
| 37 |
}
|
| 38 |
FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed"}
|
| 39 |
CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
|
| 40 |
-
BLOCKED_RAW_STATUSES = {"technical_blocker", "blocked", "health_only"}
|
| 41 |
RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
|
|
|
|
| 42 |
|
| 43 |
STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
|
| 44 |
|
|
@@ -133,6 +134,8 @@ def _raw_statuses(bundle: dict[str, Any]) -> set[str]:
|
|
| 133 |
_lower(smoke.get("status")),
|
| 134 |
_lower(hardware.get("status")),
|
| 135 |
_lower(blockers.get("status")),
|
|
|
|
|
|
|
| 136 |
}
|
| 137 |
# Event-level statuses describe individual steps and must not by themselves
|
| 138 |
# turn a whole run into success/failure. The product phase is derived from
|
|
@@ -174,12 +177,18 @@ def normalize_run_status(
|
|
| 174 |
manual = requires_manual_action(bundle)
|
| 175 |
blocker = has_technical_blocker(bundle)
|
| 176 |
|
| 177 |
-
if statuses.intersection(
|
|
|
|
|
|
|
|
|
|
| 178 |
global_status = "cancelled"
|
| 179 |
verdict = "cancelled"
|
| 180 |
elif manual:
|
| 181 |
global_status = "waiting_manual_action"
|
| 182 |
verdict = "manual_action_required"
|
|
|
|
|
|
|
|
|
|
| 183 |
elif statuses.intersection(SUCCESS_RAW_STATUSES):
|
| 184 |
global_status = "succeeded"
|
| 185 |
verdict = "passed"
|
|
@@ -189,9 +198,6 @@ def normalize_run_status(
|
|
| 189 |
elif statuses.intersection(FAILED_RAW_STATUSES):
|
| 190 |
global_status = "failed"
|
| 191 |
verdict = "failed"
|
| 192 |
-
elif blocker:
|
| 193 |
-
global_status = "blocked"
|
| 194 |
-
verdict = "technical_blocker"
|
| 195 |
elif statuses.intersection(RUNNING_RAW_STATUSES) or bundle.get("events") or launch:
|
| 196 |
global_status = "running"
|
| 197 |
verdict = "pending"
|
|
@@ -219,7 +225,7 @@ def normalize_run_status(
|
|
| 219 |
"has_target_space": bool(summary.get("target_space") or state.get("target_space") or launch.get("target_space")),
|
| 220 |
"has_job_url": bool(summary.get("job_url") or state.get("job_url") or launch.get("job_url") or summary.get("job_id") or state.get("job_id") or launch.get("job_id")),
|
| 221 |
"has_live_api_result": bool((bundle.get("generation_smoke") or {}).get("ok") or (bundle.get("generation_smoke") or {}).get("status") == "success"),
|
| 222 |
-
"is_terminal": global_status in {"succeeded", "partial", "failed", "blocked", "waiting_manual_action", "cancelled"},
|
| 223 |
"is_pollable": global_status in {"queued", "running", "validating", "unknown"},
|
| 224 |
"is_stale": is_stale,
|
| 225 |
"latest_activity_at": latest.isoformat() if latest else "",
|
|
@@ -422,6 +428,40 @@ def _known_gradio_endpoint_info(bundle: dict[str, Any]) -> dict[str, Any]:
|
|
| 422 |
return {"endpoint": endpoint or "/generate", "endpoint_count": endpoint_count, "endpoint_known": endpoint_known, "endpoints": endpoints[:20]}
|
| 423 |
|
| 424 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
|
| 426 |
"""Return the canonical linked Space Test policy for a Build Run.
|
| 427 |
|
|
@@ -467,6 +507,15 @@ def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]
|
|
| 467 |
|
| 468 |
if not target:
|
| 469 |
return policy("unavailable", False, "no_target_space", "Space Test unavailable", "No generated Space is available to validate.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 470 |
if global_status in {"running", "unknown"}:
|
| 471 |
return policy("blocked", False, "build_not_terminal", "Available after final Space state", "Space Test becomes available after the Build Run reaches a final Space state.")
|
| 472 |
if global_status == "cancelled" or raw_status in {"stopped", "cancelled", "canceled"}:
|
|
@@ -495,10 +544,19 @@ def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any])
|
|
| 495 |
smoke = bundle.get("generation_smoke") or {}
|
| 496 |
summary = bundle.get("summary") or {}
|
| 497 |
policy = build_space_test_policy(bundle, status_model)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 498 |
return {
|
| 499 |
"target_space": policy.get("target_space") or summary.get("target_space") or "",
|
| 500 |
"target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "",
|
| 501 |
-
"endpoint":
|
| 502 |
"status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending",
|
| 503 |
"latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"),
|
| 504 |
"expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
|
|
|
|
| 13 |
def _runs_prefix() -> str:
|
| 14 |
return settings.bucket_runs_prefix.strip().strip("/") or "runs"
|
| 15 |
|
| 16 |
+
TERMINAL_GLOBAL_STATUSES = {"succeeded", "partial", "failed", "cancelled", "blocked", "waiting_manual_action", "auth_refresh_required"}
|
| 17 |
SUCCESS_RAW_STATUSES = {
|
| 18 |
"success",
|
| 19 |
"done",
|
|
|
|
| 37 |
}
|
| 38 |
FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed"}
|
| 39 |
CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
|
| 40 |
+
BLOCKED_RAW_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked", "health_only"}
|
| 41 |
RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
|
| 42 |
+
AUTH_RAW_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
|
| 43 |
|
| 44 |
STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
|
| 45 |
|
|
|
|
| 134 |
_lower(smoke.get("status")),
|
| 135 |
_lower(hardware.get("status")),
|
| 136 |
_lower(blockers.get("status")),
|
| 137 |
+
_lower((bundle.get("repair_outcome") or {}).get("post_repair_validation")),
|
| 138 |
+
_lower((bundle.get("repair_outcome") or {}).get("failure_type")),
|
| 139 |
}
|
| 140 |
# Event-level statuses describe individual steps and must not by themselves
|
| 141 |
# turn a whole run into success/failure. The product phase is derived from
|
|
|
|
| 177 |
manual = requires_manual_action(bundle)
|
| 178 |
blocker = has_technical_blocker(bundle)
|
| 179 |
|
| 180 |
+
if statuses.intersection(AUTH_RAW_STATUSES):
|
| 181 |
+
global_status = "auth_refresh_required"
|
| 182 |
+
verdict = "auth_refresh_required"
|
| 183 |
+
elif statuses.intersection(CANCELLED_RAW_STATUSES):
|
| 184 |
global_status = "cancelled"
|
| 185 |
verdict = "cancelled"
|
| 186 |
elif manual:
|
| 187 |
global_status = "waiting_manual_action"
|
| 188 |
verdict = "manual_action_required"
|
| 189 |
+
elif blocker:
|
| 190 |
+
global_status = "blocked"
|
| 191 |
+
verdict = "technical_blocker_boot_only" if "technical_blocker_boot_only" in statuses else "technical_blocker"
|
| 192 |
elif statuses.intersection(SUCCESS_RAW_STATUSES):
|
| 193 |
global_status = "succeeded"
|
| 194 |
verdict = "passed"
|
|
|
|
| 198 |
elif statuses.intersection(FAILED_RAW_STATUSES):
|
| 199 |
global_status = "failed"
|
| 200 |
verdict = "failed"
|
|
|
|
|
|
|
|
|
|
| 201 |
elif statuses.intersection(RUNNING_RAW_STATUSES) or bundle.get("events") or launch:
|
| 202 |
global_status = "running"
|
| 203 |
verdict = "pending"
|
|
|
|
| 225 |
"has_target_space": bool(summary.get("target_space") or state.get("target_space") or launch.get("target_space")),
|
| 226 |
"has_job_url": bool(summary.get("job_url") or state.get("job_url") or launch.get("job_url") or summary.get("job_id") or state.get("job_id") or launch.get("job_id")),
|
| 227 |
"has_live_api_result": bool((bundle.get("generation_smoke") or {}).get("ok") or (bundle.get("generation_smoke") or {}).get("status") == "success"),
|
| 228 |
+
"is_terminal": global_status in {"succeeded", "partial", "failed", "blocked", "waiting_manual_action", "cancelled", "auth_refresh_required"},
|
| 229 |
"is_pollable": global_status in {"queued", "running", "validating", "unknown"},
|
| 230 |
"is_stale": is_stale,
|
| 231 |
"latest_activity_at": latest.isoformat() if latest else "",
|
|
|
|
| 428 |
return {"endpoint": endpoint or "/generate", "endpoint_count": endpoint_count, "endpoint_known": endpoint_known, "endpoints": endpoints[:20]}
|
| 429 |
|
| 430 |
|
| 431 |
+
def _contract_declares_no_full_inference(bundle: dict[str, Any]) -> bool:
|
| 432 |
+
contract = bundle.get("inference_contract") or bundle.get("INFERENCE_CONTRACT") or {}
|
| 433 |
+
if not isinstance(contract, dict):
|
| 434 |
+
contract = {}
|
| 435 |
+
smoke = bundle.get("generation_smoke") or {}
|
| 436 |
+
blockers = bundle.get("technical_blockers") or {}
|
| 437 |
+
statuses = _raw_statuses(bundle)
|
| 438 |
+
return bool(
|
| 439 |
+
"technical_blocker_boot_only" in statuses
|
| 440 |
+
or contract.get("full_inference_implemented") is False
|
| 441 |
+
or (contract.get("primary_api_name") in {None, "", False} and _lower(contract.get("validation_level")) in {"boot-only", "boot_only", "health-only", "health_only"})
|
| 442 |
+
or (_lower(smoke.get("status")) == "skipped" and _lower(smoke.get("skip_reason") or smoke.get("reason")) in {"contract_declared_no_full_inference", "full_inference_not_implemented"})
|
| 443 |
+
or (_lower(blockers.get("status")) in {"technical_blocker_boot_only", "technical_blocker"} and contract.get("full_inference_implemented") is False)
|
| 444 |
+
)
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def _repair_auth_context(bundle: dict[str, Any], status_model: dict[str, Any]) -> bool:
|
| 448 |
+
repair = bundle.get("repair_outcome") or {}
|
| 449 |
+
summary = bundle.get("summary") or {}
|
| 450 |
+
state = bundle.get("state") or {}
|
| 451 |
+
values = {
|
| 452 |
+
_lower(status_model.get("global_status")),
|
| 453 |
+
_lower(status_model.get("verdict")),
|
| 454 |
+
_lower(summary.get("status")),
|
| 455 |
+
_lower(state.get("status")),
|
| 456 |
+
_lower(summary.get("failure_type")),
|
| 457 |
+
_lower(state.get("failure_type")),
|
| 458 |
+
_lower(repair.get("post_repair_validation")),
|
| 459 |
+
_lower(repair.get("failure_type")),
|
| 460 |
+
_lower(repair.get("status")),
|
| 461 |
+
}
|
| 462 |
+
return bool(values.intersection(AUTH_RAW_STATUSES))
|
| 463 |
+
|
| 464 |
+
|
| 465 |
def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
|
| 466 |
"""Return the canonical linked Space Test policy for a Build Run.
|
| 467 |
|
|
|
|
| 507 |
|
| 508 |
if not target:
|
| 509 |
return policy("unavailable", False, "no_target_space", "Space Test unavailable", "No generated Space is available to validate.")
|
| 510 |
+
if _repair_auth_context(bundle, status_model):
|
| 511 |
+
return policy("recover", True, "auth_refresh_required", "Retry after sign-in refresh", "Repair patch may have been uploaded, but validation could not continue because HF OAuth expired. Sign in again, then retry linked validation.", on_success="recovered_by_manual_validation", allow_recovery=True)
|
| 512 |
+
if _contract_declares_no_full_inference(bundle):
|
| 513 |
+
blocked = policy("blocked", False, "no_generation_endpoint_by_contract", "Blocked — no generation endpoint", "Health may pass, but Pi declared full inference unavailable and no generation endpoint exists. Review TECHNICAL_BLOCKERS.json / PI_SUMMARY.md or provide a dedicated implementation and hardware plan.")
|
| 514 |
+
blocked["endpoint"] = ""
|
| 515 |
+
blocked["requires_endpoint_discovery"] = False
|
| 516 |
+
blocked["no_generation_endpoint"] = True
|
| 517 |
+
blocked["can_retry_schema"] = False
|
| 518 |
+
return blocked
|
| 519 |
if global_status in {"running", "unknown"}:
|
| 520 |
return policy("blocked", False, "build_not_terminal", "Available after final Space state", "Space Test becomes available after the Build Run reaches a final Space state.")
|
| 521 |
if global_status == "cancelled" or raw_status in {"stopped", "cancelled", "canceled"}:
|
|
|
|
| 544 |
smoke = bundle.get("generation_smoke") or {}
|
| 545 |
summary = bundle.get("summary") or {}
|
| 546 |
policy = build_space_test_policy(bundle, status_model)
|
| 547 |
+
# v191.12 regression guard: a contract-declared boot-only blocker must not
|
| 548 |
+
# resurrect the historical /generate fallback in the display model. Keep the
|
| 549 |
+
# endpoint intentionally blank when the parent policy says no generation
|
| 550 |
+
# endpoint exists by contract. Other states still use the legacy fallback so
|
| 551 |
+
# endpoint discovery/recovery behavior is unchanged.
|
| 552 |
+
if policy.get("no_generation_endpoint") is True or policy.get("reason") == "no_generation_endpoint_by_contract":
|
| 553 |
+
endpoint = ""
|
| 554 |
+
else:
|
| 555 |
+
endpoint = policy.get("endpoint") or smoke.get("api_name") or smoke.get("endpoint") or "/generate"
|
| 556 |
return {
|
| 557 |
"target_space": policy.get("target_space") or summary.get("target_space") or "",
|
| 558 |
"target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "",
|
| 559 |
+
"endpoint": endpoint,
|
| 560 |
"status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending",
|
| 561 |
"latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"),
|
| 562 |
"expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
|
src/worker_payload.py
CHANGED
|
@@ -10,6 +10,7 @@ def _encode(script: str) -> str:
|
|
| 10 |
|
| 11 |
UNIVERSAL_MODEL_CARD_WORKER_SCRIPT = r'''
|
| 12 |
|
|
|
|
| 13 |
import hashlib
|
| 14 |
import hmac
|
| 15 |
import hashlib
|
|
@@ -48,6 +49,8 @@ INTERNAL_WORKSPACE_ARTIFACT_NAMES = {
|
|
| 48 |
"REPAIR_SUMMARY.md",
|
| 49 |
"PI_SUMMARY.md",
|
| 50 |
"TECHNICAL_BLOCKERS.json",
|
|
|
|
|
|
|
| 51 |
}
|
| 52 |
|
| 53 |
|
|
@@ -278,7 +281,8 @@ def write_final_summary(run_dir: Path, final_state: dict, inference_gate: dict |
|
|
| 278 |
"bucket_source": final_state.get("bucket_source") or os.environ.get("BUCKET_SOURCE", ""),
|
| 279 |
"health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")),
|
| 280 |
"generation_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success"),
|
| 281 |
-
"failure_type": smoke.get("failure_type") or gate.get("failure_type") or "",
|
|
|
|
| 282 |
"updated_at": now(),
|
| 283 |
}
|
| 284 |
write_json(run_dir / "summary.json", payload)
|
|
@@ -308,7 +312,7 @@ def requirements_has_package(lines: list[str], package: str) -> bool:
|
|
| 308 |
stripped = line.strip()
|
| 309 |
if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
|
| 310 |
continue
|
| 311 |
-
name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
|
| 312 |
if name == wanted:
|
| 313 |
return True
|
| 314 |
return False
|
|
@@ -353,9 +357,13 @@ def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, r
|
|
| 353 |
_artifact_entry(run_dir, "live_status.json"),
|
| 354 |
_artifact_entry(run_dir, "report.md"),
|
| 355 |
_artifact_entry(run_dir, "model_analysis.json"),
|
|
|
|
|
|
|
|
|
|
| 356 |
_artifact_entry(run_dir, "hardware_strategy.json"),
|
| 357 |
_artifact_entry(run_dir, "hardware_attempts.json"),
|
| 358 |
_artifact_entry(run_dir, "inference_gate.json"),
|
|
|
|
| 359 |
_artifact_entry(run_dir, "space_runtime.json"),
|
| 360 |
_artifact_entry(run_dir, "tests/generation_smoke.json"),
|
| 361 |
_artifact_entry(run_dir, "tests/api_schema.json"),
|
|
@@ -367,6 +375,7 @@ def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, r
|
|
| 367 |
_artifact_entry(run_dir, "repair/REPAIR_BRIEF.md"),
|
| 368 |
_artifact_entry(run_dir, "repair/REPAIR_PLAN.md"),
|
| 369 |
_artifact_entry(run_dir, "repair/REPAIR_SUMMARY.md"),
|
|
|
|
| 370 |
_artifact_entry(run_dir, "repair/BLOCKAGE.json"),
|
| 371 |
_artifact_entry(run_dir, "logs/pi_live_output.txt"),
|
| 372 |
_artifact_entry(run_dir, "logs/pi_output.txt"),
|
|
@@ -429,6 +438,198 @@ def redact_text(text: str | None) -> str:
|
|
| 429 |
|
| 430 |
|
| 431 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 432 |
def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
|
| 433 |
"""Ensure Pi and HF tooling see the same private HF token safely.
|
| 434 |
|
|
@@ -442,6 +643,7 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
|
|
| 442 |
os.environ.setdefault("HF_TOKEN", token)
|
| 443 |
os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
|
| 444 |
os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
|
|
|
|
| 445 |
payload = {
|
| 446 |
"hf_token_present": bool(token),
|
| 447 |
"hf_token_length": len(token) if token else 0,
|
|
@@ -450,6 +652,13 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
|
|
| 450 |
"bucket_source": os.environ.get("BUCKET_SOURCE") or "",
|
| 451 |
"target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
|
| 452 |
"token_value": "[REDACTED]" if token else "",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 453 |
}
|
| 454 |
run_dir.mkdir(parents=True, exist_ok=True)
|
| 455 |
write_json(run_dir / "token_context.json", payload)
|
|
@@ -734,7 +943,7 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
|
|
| 734 |
return "cancelled"
|
| 735 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 736 |
return "manual_action_required"
|
| 737 |
-
if status
|
| 738 |
return "technical_blocker"
|
| 739 |
if full_inference_verified:
|
| 740 |
return "success"
|
|
@@ -951,6 +1160,10 @@ def fail(run_dir: Path, events_path: Path, message: str, details: dict | None =
|
|
| 951 |
existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
|
| 952 |
if not isinstance(existing_state, dict):
|
| 953 |
existing_state = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 954 |
failure_state = {
|
| 955 |
**existing_state,
|
| 956 |
"run_id": os.environ.get("RUN_ID"),
|
|
@@ -959,6 +1172,7 @@ def fail(run_dir: Path, events_path: Path, message: str, details: dict | None =
|
|
| 959 |
"message": message,
|
| 960 |
"updated_at": now(),
|
| 961 |
"details": safe,
|
|
|
|
| 962 |
}
|
| 963 |
# Preserve the target Space when the worker fails after repository creation.
|
| 964 |
target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
|
|
@@ -1594,6 +1808,85 @@ def read_inference_contract(workspace: Path | None) -> dict:
|
|
| 1594 |
return {}
|
| 1595 |
|
| 1596 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1597 |
def contract_smoke_test_payload(contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None:
|
| 1598 |
smoke = contract.get("smoke_test") if isinstance(contract, dict) else None
|
| 1599 |
if not isinstance(smoke, dict):
|
|
@@ -2583,6 +2876,100 @@ def create_space_with_hardware_strategy(api, target_space_id: str, token: str, p
|
|
| 2583 |
raise
|
| 2584 |
|
| 2585 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2586 |
def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None):
|
| 2587 |
workspace.mkdir(parents=True, exist_ok=True)
|
| 2588 |
model_analysis = model_analysis or {}
|
|
@@ -2630,7 +3017,6 @@ diffusers
|
|
| 2630 |
accelerate
|
| 2631 |
safetensors
|
| 2632 |
torch
|
| 2633 |
-
kernels
|
| 2634 |
pillow
|
| 2635 |
numpy
|
| 2636 |
requests
|
|
@@ -2677,9 +3063,21 @@ Non-negotiable safety and product constraints:
|
|
| 2677 |
- Work only inside the current workspace.
|
| 2678 |
- The wrapper will create the private Space, request allowed hardware best-effort, upload files, and validate the live app. Do not create/delete repos yourself in this builder worker.
|
| 2679 |
- Preserve a cheap health endpoint named `health` with `api_name="health"`. It must not load weights, run GPU work, or download large files.
|
| 2680 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2681 |
- README.md frontmatter must remain valid; if it uses short_description, it must be 60 characters or fewer.
|
| 2682 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2683 |
Implementation contract:
|
| 2684 |
- If IMPLEMENTATION_MODE is `full-inference-gated`, you are not allowed to silently replace generation with a placeholder and call it success.
|
| 2685 |
- Try to implement the closest real inference path for the model card using evidence from README, model metadata, config files, and repo files.
|
|
@@ -2687,10 +3085,13 @@ Implementation contract:
|
|
| 2687 |
- If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint.
|
| 2688 |
- If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement.
|
| 2689 |
- If the model requires special dependencies, include them only when needed and document risks.
|
| 2690 |
-
- Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels where relevant, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs.
|
|
|
|
| 2691 |
- If real inference is impossible or unsafe in a Space, write TECHNICAL_BLOCKERS.json with concrete evidence for every blocker.
|
| 2692 |
|
| 2693 |
Deliverables:
|
|
|
|
|
|
|
| 2694 |
- app.py must boot on Hugging Face Spaces.
|
| 2695 |
- app.py must expose health/api_name="health".
|
| 2696 |
- If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic.
|
|
@@ -2778,13 +3179,13 @@ def sanitize_readme_metadata(workspace: Path, events_path: Path):
|
|
| 2778 |
)
|
| 2779 |
|
| 2780 |
def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
|
| 2781 |
-
"""Normalize
|
| 2782 |
|
| 2783 |
-
Do not try to solve every dependency conflict here.
|
| 2784 |
-
reading concrete build logs and patching requirements when
|
| 2785 |
-
fails. The Factory only prevents obviously unsafe broad ranges
|
| 2786 |
-
|
| 2787 |
-
|
| 2788 |
"""
|
| 2789 |
req_path = workspace / "requirements.txt"
|
| 2790 |
if not req_path.exists():
|
|
@@ -2805,27 +3206,62 @@ def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
|
|
| 2805 |
|
| 2806 |
# Minimal base pins only. Do not globally pin diffusers here: newer model
|
| 2807 |
# cards may legitimately need a recent Diffusers release. If diffusers causes
|
| 2808 |
-
# a pip conflict, Pi must repair from the concrete build log.
|
| 2809 |
policy: dict[str, str] = {
|
| 2810 |
"huggingface-hub": "huggingface_hub>=0.34.0,<2.0.0",
|
| 2811 |
"transformers": "transformers>=4.51.0,<5.0.0",
|
| 2812 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2813 |
aliases = {
|
| 2814 |
"huggingface_hub": "huggingface-hub",
|
| 2815 |
"huggingface-hub": "huggingface-hub",
|
| 2816 |
"transformers": "transformers",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2817 |
}
|
|
|
|
| 2818 |
seen_policy: set[str] = set()
|
|
|
|
| 2819 |
filtered: list[str] = []
|
| 2820 |
changed = False
|
|
|
|
|
|
|
| 2821 |
|
| 2822 |
for line in package_lines:
|
| 2823 |
stripped = line.strip()
|
| 2824 |
if stripped.startswith("#") or "://" in stripped or stripped.startswith((".", "/")):
|
| 2825 |
filtered.append(line)
|
| 2826 |
continue
|
| 2827 |
-
name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
|
| 2828 |
canonical = aliases.get(name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2829 |
if canonical in policy:
|
| 2830 |
if stripped != policy[canonical]:
|
| 2831 |
changed = True
|
|
@@ -2843,34 +3279,53 @@ def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
|
|
| 2843 |
stable_policy_lines.append(policy[canonical])
|
| 2844 |
changed = True
|
| 2845 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2846 |
torch_added = False
|
| 2847 |
-
if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines, "torch"):
|
| 2848 |
stable_policy_lines.append("torch>=2.0.0")
|
| 2849 |
torch_added = True
|
| 2850 |
changed = True
|
| 2851 |
|
| 2852 |
-
new_lines = prefix_lines + stable_policy_lines + filtered
|
| 2853 |
new = "\n".join(line for line in new_lines if line.strip()) + "\n"
|
| 2854 |
if new != raw:
|
| 2855 |
changed = True
|
| 2856 |
if changed:
|
| 2857 |
req_path.write_text(new, encoding="utf-8")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2858 |
append_event(
|
| 2859 |
events_path,
|
| 2860 |
"requirements_sanitize",
|
| 2861 |
"success",
|
| 2862 |
-
"Normalized broad base dependencies; concrete pip conflicts remain Pi repair work",
|
| 2863 |
-
|
| 2864 |
-
"huggingface_hub": policy["huggingface-hub"],
|
| 2865 |
-
"transformers": policy["transformers"],
|
| 2866 |
-
"reason": "Avoid uncontrolled Transformers 5.x while preserving model-specific dependency choices for Pi to repair from build logs; require torch when app.py imports torch.",
|
| 2867 |
-
"torch_added": torch_added,
|
| 2868 |
-
"torch_policy": "torch>=2.0.0",
|
| 2869 |
-
"torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present",
|
| 2870 |
-
},
|
| 2871 |
)
|
| 2872 |
|
| 2873 |
-
|
| 2874 |
def useful_log_signals(text: str) -> list[str]:
|
| 2875 |
low = (text or "").lower()
|
| 2876 |
signals = []
|
|
@@ -3354,6 +3809,7 @@ First read `INCIDENT_BRIEF.md` and the HF Spaces gist operational rules: {GIST_U
|
|
| 3354 |
|
| 3355 |
You are not allowed to edit code in this diagnosis step. Your task is to decide the next action for the Factory.
|
| 3356 |
Use the gist method: read logs first, identify the first actionable error, use the cheapest useful iteration rung, and require a live Gradio/API validation before success.
|
|
|
|
| 3357 |
|
| 3358 |
Write `REPAIR_DECISION.json` exactly as requested in INCIDENT_BRIEF.md. Do not patch files during this step.
|
| 3359 |
"""
|
|
@@ -3471,7 +3927,11 @@ A separate Pi diagnosis already decided that `patch_code` is justified. Do not r
|
|
| 3471 |
- Preserve or restore a cheap `health` endpoint.
|
| 3472 |
- Preserve the expected Gradio API endpoint when possible.
|
| 3473 |
- Keep README metadata valid and short_description <= 60 chars.
|
| 3474 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3475 |
|
| 3476 |
## Required repair artifacts
|
| 3477 |
Before modifying files, write `REPAIR_PLAN.md` with:
|
|
@@ -3552,6 +4012,8 @@ First read `REPAIR_BRIEF.md`, `INCIDENT_BRIEF.md`, `DEPENDENCY_ERROR_BRIEF.md` i
|
|
| 3552 |
You are continuing the same build run, not starting a separate project.
|
| 3553 |
This patch is allowed only because the diagnosis decision selected `patch_code`.
|
| 3554 |
If `DEPENDENCY_ERROR_BRIEF.md` exists, treat it as evidence for the gist method: identify the first pip error, patch dependency pins minimally, and do not modify inference code unless the dependency fix alone cannot address that first error.
|
|
|
|
|
|
|
| 3555 |
Use the available HF token context to inspect private Hub resources when needed, but never print or persist token values.
|
| 3556 |
|
| 3557 |
Critical method:
|
|
@@ -3627,18 +4089,53 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e
|
|
| 3627 |
dependency_issue = extract_pip_dependency_issue(f"{current_error}\n{build_log}\n{runtime_log}")
|
| 3628 |
if dependency_issue:
|
| 3629 |
write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, current_error)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3630 |
if apply_dependency_guardrail_repair(workspace, run_dir, events_path, current_error, build_log, runtime_log):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3631 |
append_event(events_path, "factory_rebuild", "started", "Re-uploading dependency-guardrailed workspace after pip resolver build error")
|
| 3632 |
if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="dependency_guardrail_repair"):
|
|
|
|
| 3633 |
raise RuntimeError("Dependency guardrail rebuild skipped by restart guardrails")
|
|
|
|
| 3634 |
append_event(events_path, "factory_rebuild", "success", "Dependency-guardrailed workspace uploaded; revalidating live Space")
|
| 3635 |
append_event(events_path, "repair_validation", "started", "Revalidating after deterministic dependency repair")
|
| 3636 |
write_live_status(run_dir, stage="live_validation", status="running", message="Waiting for Space runtime and health endpoint", data={"target_space": target_space_id})
|
| 3637 |
try:
|
|
|
|
| 3638 |
validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
|
|
|
|
| 3639 |
append_event(events_path, "repair_validation", "success", "Deterministic dependency repair resolved the build blockage")
|
| 3640 |
return validation
|
| 3641 |
except Exception as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3642 |
current_error = f"{current_error}\n\nDependency guardrail rebuild did not resolve validation: {str(exc)[:4000]}"
|
| 3643 |
collect_space_logs(target_space_id, token, run_dir, events_path)
|
| 3644 |
append_event(events_path, "repair_validation", "failed", "Dependency guardrail rebuild did not resolve validation; falling back to Pi diagnosis", {"error": str(exc)[:4000]})
|
|
@@ -3676,10 +4173,27 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e
|
|
| 3676 |
append_event(events_path, "factory_rebuild", "success", "Same-code workspace re-uploaded; revalidating live Space")
|
| 3677 |
append_event(events_path, "repair_validation", "started", "Revalidating after same-code factory rebuild")
|
| 3678 |
try:
|
|
|
|
| 3679 |
validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
|
|
|
|
| 3680 |
append_event(events_path, "repair_validation", "success", "Same-code factory rebuild resolved the blockage")
|
| 3681 |
return validation
|
| 3682 |
except Exception as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3683 |
current_error = f"{current_error}\n\nSame-code factory rebuild did not resolve validation: {str(exc)[:4000]}"
|
| 3684 |
collect_space_logs(target_space_id, token, run_dir, events_path)
|
| 3685 |
append_event(events_path, "repair_validation", "failed", "Same-code factory rebuild did not resolve validation; re-diagnosing", {"error": str(exc)[:4000]})
|
|
@@ -3692,20 +4206,49 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e
|
|
| 3692 |
raise RuntimeError("Automated recovery stopped after patch budget was exhausted")
|
| 3693 |
budgets[action] -= 1
|
| 3694 |
append_event(events_path, "repair", "started", "Pi diagnosis allows a minimal code patch", {"decision": decision})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3695 |
repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision)
|
| 3696 |
if not repaired:
|
|
|
|
| 3697 |
write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair")
|
| 3698 |
append_event(events_path, "failure", "failed", "Structured patch repair failed before redeploy", {"decision": decision})
|
| 3699 |
raise RuntimeError("Structured patch repair failed before redeploy")
|
|
|
|
| 3700 |
append_event(events_path, "repair_upload", "started", "Uploading repaired workspace")
|
|
|
|
| 3701 |
upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
|
|
|
|
| 3702 |
append_event(events_path, "repair_upload", "success", "Repaired workspace uploaded")
|
| 3703 |
append_event(events_path, "repair_validation", "started", "Revalidating repaired Space")
|
| 3704 |
try:
|
|
|
|
| 3705 |
validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
|
|
|
|
| 3706 |
append_event(events_path, "repair_validation", "success", "Repaired Space passed live API validation")
|
| 3707 |
return validation
|
| 3708 |
except Exception as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3709 |
current_error = f"{current_error}\n\nPatch repair did not resolve validation: {str(exc)[:4000]}"
|
| 3710 |
collect_space_logs(target_space_id, token, run_dir, events_path)
|
| 3711 |
append_event(events_path, "repair_validation", "failed", "Repair attempted, but validation still failed", {"error": str(exc)[:4000]})
|
|
@@ -3799,8 +4342,12 @@ def infer_generation_gate(workspace: Path, implementation_mode: str, validation:
|
|
| 3799 |
}
|
| 3800 |
|
| 3801 |
if blocker_detected:
|
| 3802 |
-
|
| 3803 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3804 |
elif implementation_mode in {"full-inference-gated", "full-inference-attempt"} and smoke_ok:
|
| 3805 |
status = "full_inference_success"
|
| 3806 |
message = "Space boots and a live generation smoke test passed. ZeroGPU duration recommendation was measured from real inference."
|
|
@@ -3891,6 +4438,7 @@ def main():
|
|
| 3891 |
api = HfApi(token=token)
|
| 3892 |
whoami = api.whoami(token=token)
|
| 3893 |
append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})
|
|
|
|
| 3894 |
|
| 3895 |
append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})
|
| 3896 |
info = api.model_info(model_id, token=token, files_metadata=True)
|
|
@@ -3929,6 +4477,7 @@ def main():
|
|
| 3929 |
emit_pi_model_resolution(events_path, pi_model_resolution)
|
| 3930 |
if not (workspace / "PI_SUMMARY.md").exists():
|
| 3931 |
(workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8")
|
|
|
|
| 3932 |
|
| 3933 |
app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")
|
| 3934 |
if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text:
|
|
@@ -3973,6 +4522,7 @@ def main():
|
|
| 3973 |
# starts directly on the requested hardware. If it fell back to CPU, the run
|
| 3974 |
# remains valid but will be marked manual_hardware_required when inference
|
| 3975 |
# signals indicate GPU is needed.
|
|
|
|
| 3976 |
upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
|
| 3977 |
write_artifact_manifest(run_dir, reason="workspace_uploaded")
|
| 3978 |
|
|
@@ -3993,21 +4543,25 @@ def main():
|
|
| 3993 |
)
|
| 3994 |
generation_smoke = None
|
| 3995 |
if implementation_mode in {"full-inference-gated", "full-inference-attempt"}:
|
| 3996 |
-
|
| 3997 |
-
|
| 3998 |
-
|
| 3999 |
-
|
| 4000 |
-
|
| 4001 |
-
|
| 4002 |
-
|
| 4003 |
-
"
|
| 4004 |
-
|
| 4005 |
-
|
| 4006 |
-
|
| 4007 |
-
|
| 4008 |
-
|
| 4009 |
-
|
| 4010 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4011 |
else:
|
| 4012 |
generation_smoke = measured_zero_gpu_recommendation(None) | {"status": "skipped", "expected_output_type": expected_output_type}
|
| 4013 |
write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
|
|
@@ -4017,7 +4571,7 @@ def main():
|
|
| 4017 |
# hardware requests failed, classify the run honestly as needing manual
|
| 4018 |
# hardware instead of pretending CPU/default hardware is enough. the existing-Space validation workflow
|
| 4019 |
# can then smoke-test generation after the user sets a GPU manually.
|
| 4020 |
-
manual_hw_required = selected_hardware == "default-cpu-or-existing" and inference_gate.get("status") not in {"technical_blocker", "health_only"} and (
|
| 4021 |
inference_gate.get("implementation_signals", {}).get("has_spaces_gpu")
|
| 4022 |
or inference_gate.get("implementation_signals", {}).get("has_torch")
|
| 4023 |
or any((a.get("manual_action_required") for a in hardware_attempts if isinstance(a, dict)))
|
|
@@ -4134,7 +4688,15 @@ The wrapper validated the live Space using HTTP `/health` first, with Gradio Cli
|
|
| 4134 |
collect_pi_traces(run_dir, events_path)
|
| 4135 |
except Exception:
|
| 4136 |
pass
|
| 4137 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4138 |
|
| 4139 |
|
| 4140 |
if __name__ == "__main__":
|
|
@@ -4144,6 +4706,7 @@ if __name__ == "__main__":
|
|
| 4144 |
|
| 4145 |
|
| 4146 |
VALIDATE_EXISTING_SPACE_WORKER_SCRIPT = r'''
|
|
|
|
| 4147 |
import json
|
| 4148 |
import os
|
| 4149 |
import re
|
|
@@ -4361,7 +4924,7 @@ def requirements_has_package(lines: list[str], package: str) -> bool:
|
|
| 4361 |
stripped = line.strip()
|
| 4362 |
if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
|
| 4363 |
continue
|
| 4364 |
-
name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
|
| 4365 |
if name == wanted:
|
| 4366 |
return True
|
| 4367 |
return False
|
|
@@ -4547,7 +5110,7 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
|
|
| 4547 |
return "cancelled"
|
| 4548 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 4549 |
return "manual_action_required"
|
| 4550 |
-
if status
|
| 4551 |
return "technical_blocker"
|
| 4552 |
if full_inference_verified:
|
| 4553 |
return "success"
|
|
@@ -4758,6 +5321,141 @@ def publish_eval_record(run_dir: Path, *, phase: str, events_path: Path | None =
|
|
| 4758 |
return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"}
|
| 4759 |
|
| 4760 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4761 |
def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
|
| 4762 |
"""Ensure Pi and HF tooling see the same private HF token safely.
|
| 4763 |
|
|
@@ -4771,6 +5469,7 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
|
|
| 4771 |
os.environ.setdefault("HF_TOKEN", token)
|
| 4772 |
os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
|
| 4773 |
os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
|
|
|
|
| 4774 |
payload = {
|
| 4775 |
"hf_token_present": bool(token),
|
| 4776 |
"hf_token_length": len(token) if token else 0,
|
|
@@ -4779,6 +5478,13 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
|
|
| 4779 |
"bucket_source": os.environ.get("BUCKET_SOURCE") or "",
|
| 4780 |
"target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
|
| 4781 |
"token_value": "[REDACTED]" if token else "",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4782 |
}
|
| 4783 |
run_dir.mkdir(parents=True, exist_ok=True)
|
| 4784 |
write_json(run_dir / "token_context.json", payload)
|
|
@@ -6086,6 +6792,8 @@ def main():
|
|
| 6086 |
write_json(state_path, {"run_id": run_id, "kind": "linked_space_validation", "status": "running", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "created_by": username, "updated_at": now()})
|
| 6087 |
if not token:
|
| 6088 |
raise RuntimeError("HF_TOKEN is missing")
|
|
|
|
|
|
|
| 6089 |
if not TARGET_RE.match(target_space_id):
|
| 6090 |
raise ValueError("TARGET_SPACE_ID must look like owner/space-name")
|
| 6091 |
if not parent_build_run_id:
|
|
@@ -6186,14 +6894,19 @@ Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_
|
|
| 6186 |
append_event(events_path, "done", "full_inference_success", "Existing Space validation completed", {"latency_seconds": smoke.get("latency_seconds")})
|
| 6187 |
except Exception as exc:
|
| 6188 |
collect_space_logs(target_space_id, token or "", run_dir, events_path)
|
|
|
|
| 6189 |
details = {"error": str(exc)[:4000]}
|
| 6190 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6191 |
write_json(state_path, failure_state)
|
| 6192 |
-
write_final_summary(run_dir, failure_state, {}, {"status":
|
| 6193 |
if parent_build_run_id:
|
| 6194 |
parent_dir = output_root / "runs" / parent_build_run_id
|
| 6195 |
parent_dir.mkdir(parents=True, exist_ok=True)
|
| 6196 |
-
failed_status = {"status":
|
| 6197 |
linked_path = parent_dir / "linked_validations.json"
|
| 6198 |
linked = read_json(linked_path, {"parent_build_run_id": parent_build_run_id, "validations": []}) or {"parent_build_run_id": parent_build_run_id, "validations": []}
|
| 6199 |
validations = linked.get("validations") if isinstance(linked, dict) else []
|
|
@@ -6208,7 +6921,7 @@ Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_
|
|
| 6208 |
write_json(parent_dir / "manual_validation_status.json", failed_status)
|
| 6209 |
write_json(linked_path, linked)
|
| 6210 |
(run_dir / "report.md").write_text(f"# Existing Space Validation Failed\n\n```json\n{json.dumps(details, indent=2, ensure_ascii=False)}\n```\n", encoding="utf-8")
|
| 6211 |
-
append_event(events_path, "failure", "failed", "Existing Space validation failed", details)
|
| 6212 |
try:
|
| 6213 |
publish_eval_record(run_dir, phase="failure", events_path=events_path)
|
| 6214 |
append_event(events_path, "anonymous_eval", "success", "Published anonymized failed validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False})
|
|
@@ -6217,8 +6930,8 @@ Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_
|
|
| 6217 |
# v190.33: also rewrite failure summary as the final visible file update
|
| 6218 |
# for linked validations, so a failed validation cannot remain listed
|
| 6219 |
# as running after the worker has already terminalized state.json.
|
| 6220 |
-
write_final_summary(run_dir, failure_state, {}, {"status":
|
| 6221 |
-
append_event(events_path, "summary_write", "success", "Wrote terminal failed validation summary", {"status":
|
| 6222 |
raise SystemExit(1)
|
| 6223 |
|
| 6224 |
|
|
|
|
| 10 |
|
| 11 |
UNIVERSAL_MODEL_CARD_WORKER_SCRIPT = r'''
|
| 12 |
|
| 13 |
+
import base64
|
| 14 |
import hashlib
|
| 15 |
import hmac
|
| 16 |
import hashlib
|
|
|
|
| 49 |
"REPAIR_SUMMARY.md",
|
| 50 |
"PI_SUMMARY.md",
|
| 51 |
"TECHNICAL_BLOCKERS.json",
|
| 52 |
+
"pi_feasibility_brief.json",
|
| 53 |
+
"pi_implementation_plan.json",
|
| 54 |
}
|
| 55 |
|
| 56 |
|
|
|
|
| 281 |
"bucket_source": final_state.get("bucket_source") or os.environ.get("BUCKET_SOURCE", ""),
|
| 282 |
"health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")),
|
| 283 |
"generation_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success"),
|
| 284 |
+
"failure_type": smoke.get("failure_type") or gate.get("failure_type") or (final_state.get("details") or {}).get("failure_type") or (final_state.get("details") or {}).get("repair_failure_type") or (final_state.get("repair_outcome") or {}).get("failure_type") or final_state.get("failure_type") or "",
|
| 285 |
+
"repair_outcome_status": (final_state.get("repair_outcome") or {}).get("post_repair_validation") or (final_state.get("details") or {}).get("repair_outcome_status") or "",
|
| 286 |
"updated_at": now(),
|
| 287 |
}
|
| 288 |
write_json(run_dir / "summary.json", payload)
|
|
|
|
| 312 |
stripped = line.strip()
|
| 313 |
if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
|
| 314 |
continue
|
| 315 |
+
name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-")
|
| 316 |
if name == wanted:
|
| 317 |
return True
|
| 318 |
return False
|
|
|
|
| 357 |
_artifact_entry(run_dir, "live_status.json"),
|
| 358 |
_artifact_entry(run_dir, "report.md"),
|
| 359 |
_artifact_entry(run_dir, "model_analysis.json"),
|
| 360 |
+
_artifact_entry(run_dir, "planning/pi_feasibility_brief.json"),
|
| 361 |
+
_artifact_entry(run_dir, "planning/pi_implementation_plan.json"),
|
| 362 |
+
_artifact_entry(run_dir, "planning/worker_plan_review.json"),
|
| 363 |
_artifact_entry(run_dir, "hardware_strategy.json"),
|
| 364 |
_artifact_entry(run_dir, "hardware_attempts.json"),
|
| 365 |
_artifact_entry(run_dir, "inference_gate.json"),
|
| 366 |
+
_artifact_entry(run_dir, "repair_outcome.json"),
|
| 367 |
_artifact_entry(run_dir, "space_runtime.json"),
|
| 368 |
_artifact_entry(run_dir, "tests/generation_smoke.json"),
|
| 369 |
_artifact_entry(run_dir, "tests/api_schema.json"),
|
|
|
|
| 375 |
_artifact_entry(run_dir, "repair/REPAIR_BRIEF.md"),
|
| 376 |
_artifact_entry(run_dir, "repair/REPAIR_PLAN.md"),
|
| 377 |
_artifact_entry(run_dir, "repair/REPAIR_SUMMARY.md"),
|
| 378 |
+
_artifact_entry(run_dir, "repair/REPAIR_OUTCOME.json"),
|
| 379 |
_artifact_entry(run_dir, "repair/BLOCKAGE.json"),
|
| 380 |
_artifact_entry(run_dir, "logs/pi_live_output.txt"),
|
| 381 |
_artifact_entry(run_dir, "logs/pi_output.txt"),
|
|
|
|
| 438 |
|
| 439 |
|
| 440 |
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
class AuthRefreshRequired(RuntimeError):
|
| 444 |
+
"""Raised when an OAuth/JWT token is expired or too close to expiry."""
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def _decode_jwt_claims_unverified(token: str) -> dict:
|
| 448 |
+
"""Decode JWT claims without verification, only to inspect non-sensitive expiry metadata.
|
| 449 |
+
|
| 450 |
+
This never returns or writes the raw token. Opaque PAT-style tokens are supported by
|
| 451 |
+
returning an empty dict so they remain usable with an `unknown` expiry state.
|
| 452 |
+
"""
|
| 453 |
+
try:
|
| 454 |
+
parts = (token or "").split(".")
|
| 455 |
+
if len(parts) < 2:
|
| 456 |
+
return {}
|
| 457 |
+
payload = parts[1]
|
| 458 |
+
payload += "=" * ((4 - len(payload) % 4) % 4)
|
| 459 |
+
raw = base64.urlsafe_b64decode(payload.encode("utf-8"))
|
| 460 |
+
data = json.loads(raw.decode("utf-8"))
|
| 461 |
+
return data if isinstance(data, dict) else {}
|
| 462 |
+
except Exception:
|
| 463 |
+
return {}
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict:
|
| 467 |
+
issued_at = now()
|
| 468 |
+
if not token:
|
| 469 |
+
return {
|
| 470 |
+
"schema_version": "auth_status.v1",
|
| 471 |
+
"checked_at": issued_at,
|
| 472 |
+
"token_present": False,
|
| 473 |
+
"token_kind": "missing",
|
| 474 |
+
"expiry_known": False,
|
| 475 |
+
"status": "missing",
|
| 476 |
+
"safe_for_phase": False,
|
| 477 |
+
"minimum_required_seconds": minimum_required_seconds,
|
| 478 |
+
}
|
| 479 |
+
claims = _decode_jwt_claims_unverified(token)
|
| 480 |
+
exp = claims.get("exp") if isinstance(claims, dict) else None
|
| 481 |
+
iat = claims.get("iat") if isinstance(claims, dict) else None
|
| 482 |
+
token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown")
|
| 483 |
+
payload = {
|
| 484 |
+
"schema_version": "auth_status.v1",
|
| 485 |
+
"checked_at": issued_at,
|
| 486 |
+
"token_present": True,
|
| 487 |
+
"token_kind": token_kind,
|
| 488 |
+
"expiry_known": exp is not None,
|
| 489 |
+
"minimum_required_seconds": int(minimum_required_seconds or 0),
|
| 490 |
+
"token_value": "[REDACTED]",
|
| 491 |
+
}
|
| 492 |
+
if iat is not None:
|
| 493 |
+
try:
|
| 494 |
+
payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat()
|
| 495 |
+
except Exception:
|
| 496 |
+
pass
|
| 497 |
+
if exp is None:
|
| 498 |
+
payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"})
|
| 499 |
+
return payload
|
| 500 |
+
try:
|
| 501 |
+
exp_int = int(exp)
|
| 502 |
+
seconds_left = exp_int - int(time.time())
|
| 503 |
+
payload.update({
|
| 504 |
+
"expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(),
|
| 505 |
+
"seconds_until_expiry": seconds_left,
|
| 506 |
+
})
|
| 507 |
+
except Exception:
|
| 508 |
+
payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"})
|
| 509 |
+
return payload
|
| 510 |
+
if seconds_left <= 0:
|
| 511 |
+
payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"})
|
| 512 |
+
elif minimum_required_seconds and seconds_left < minimum_required_seconds:
|
| 513 |
+
payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"})
|
| 514 |
+
else:
|
| 515 |
+
payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"})
|
| 516 |
+
return payload
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict:
|
| 520 |
+
payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds)
|
| 521 |
+
payload["phase"] = phase
|
| 522 |
+
safe_payload = {k: v for k, v in payload.items() if k != "token_value"}
|
| 523 |
+
try:
|
| 524 |
+
probes_dir = run_dir / "auth_probes"
|
| 525 |
+
probes_dir.mkdir(parents=True, exist_ok=True)
|
| 526 |
+
write_json(probes_dir / f"{phase}.json", safe_payload)
|
| 527 |
+
write_json(run_dir / "auth_status.json", safe_payload)
|
| 528 |
+
except Exception:
|
| 529 |
+
pass
|
| 530 |
+
status = str(payload.get("status") or "unknown")
|
| 531 |
+
if events_path:
|
| 532 |
+
event_status = "success" if payload.get("safe_for_phase") else "failed"
|
| 533 |
+
if status == "unknown":
|
| 534 |
+
event_status = "warning"
|
| 535 |
+
append_event(
|
| 536 |
+
events_path,
|
| 537 |
+
"auth_probe",
|
| 538 |
+
event_status,
|
| 539 |
+
f"HF OAuth/token expiry check for {phase}: {status}",
|
| 540 |
+
safe_payload,
|
| 541 |
+
)
|
| 542 |
+
if raise_on_unsafe and not payload.get("safe_for_phase"):
|
| 543 |
+
raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.")
|
| 544 |
+
return payload
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
def is_auth_expired_error(error: Exception | str) -> bool:
|
| 548 |
+
text = str(error or "").lower()
|
| 549 |
+
return any(marker in text for marker in [
|
| 550 |
+
"oauth token has expired",
|
| 551 |
+
"exp claim timestamp check failed",
|
| 552 |
+
"token has expired",
|
| 553 |
+
"jwt expired",
|
| 554 |
+
])
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def classify_repair_validation_error(error: Exception | str) -> dict:
|
| 559 |
+
"""Classify post-repair validation failures without losing the repair result.
|
| 560 |
+
|
| 561 |
+
A repair can be correctly diagnosed, patched, and uploaded while the final
|
| 562 |
+
validation is inconclusive because OAuth expired. Keep that distinct from a
|
| 563 |
+
model/runtime repair failure so the UI and audit trail do not imply that Pi's
|
| 564 |
+
patch necessarily failed.
|
| 565 |
+
"""
|
| 566 |
+
text = str(error or "")
|
| 567 |
+
if isinstance(error, AuthRefreshRequired) or is_auth_expired_error(text):
|
| 568 |
+
return {
|
| 569 |
+
"post_repair_validation": "inconclusive_auth_expired",
|
| 570 |
+
"failure_type": "repair_validation_inconclusive_auth",
|
| 571 |
+
"terminal_status": "auth_refresh_required",
|
| 572 |
+
"message": "Repair progress could not be validated because HF auth expired or is too close to expiry.",
|
| 573 |
+
}
|
| 574 |
+
return {
|
| 575 |
+
"post_repair_validation": "failed",
|
| 576 |
+
"failure_type": "repair_validation_failed",
|
| 577 |
+
"terminal_status": "failed",
|
| 578 |
+
"message": "Repair was applied, but the repaired Space did not pass live validation.",
|
| 579 |
+
}
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
def write_repair_outcome(run_dir: Path, events_path: Path | None = None, **updates) -> dict:
|
| 583 |
+
"""Write a cumulative repair outcome artifact.
|
| 584 |
+
|
| 585 |
+
Keep it at the run root for list-view consumers and mirror it inside
|
| 586 |
+
repair/ for manual audits. The payload is intentionally compact and
|
| 587 |
+
redacted; full logs remain in logs/ and repair/ artifacts.
|
| 588 |
+
"""
|
| 589 |
+
root_path = run_dir / "repair_outcome.json"
|
| 590 |
+
existing = load_json_if_exists(root_path) if root_path.exists() else {}
|
| 591 |
+
if not isinstance(existing, dict):
|
| 592 |
+
existing = {}
|
| 593 |
+
payload = {
|
| 594 |
+
"schema_version": "repair_outcome.v1",
|
| 595 |
+
**existing,
|
| 596 |
+
**{k: v for k, v in updates.items() if v is not None},
|
| 597 |
+
"updated_at": now(),
|
| 598 |
+
}
|
| 599 |
+
write_json(root_path, payload)
|
| 600 |
+
try:
|
| 601 |
+
write_json(run_dir / "repair" / "REPAIR_OUTCOME.json", payload)
|
| 602 |
+
except Exception:
|
| 603 |
+
pass
|
| 604 |
+
if events_path:
|
| 605 |
+
append_event(
|
| 606 |
+
events_path,
|
| 607 |
+
"repair_outcome",
|
| 608 |
+
str(payload.get("post_repair_validation") or payload.get("repair_status") or payload.get("repair_decision") or "updated"),
|
| 609 |
+
"Repair outcome artifact updated",
|
| 610 |
+
payload,
|
| 611 |
+
)
|
| 612 |
+
return payload
|
| 613 |
+
|
| 614 |
+
def minimum_auth_seconds_for_phase(phase: str) -> int:
|
| 615 |
+
env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_")
|
| 616 |
+
raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "")
|
| 617 |
+
if raw:
|
| 618 |
+
try:
|
| 619 |
+
return max(0, int(raw))
|
| 620 |
+
except Exception:
|
| 621 |
+
pass
|
| 622 |
+
defaults = {
|
| 623 |
+
"before_space_create": 900,
|
| 624 |
+
"before_upload": 900,
|
| 625 |
+
"before_initial_validation": 1800,
|
| 626 |
+
"before_repair": 1800,
|
| 627 |
+
"before_repair_upload": 1800,
|
| 628 |
+
"before_repair_validation": 1800,
|
| 629 |
+
"before_linked_space_test": 1800,
|
| 630 |
+
}
|
| 631 |
+
return defaults.get(phase, 0)
|
| 632 |
+
|
| 633 |
def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
|
| 634 |
"""Ensure Pi and HF tooling see the same private HF token safely.
|
| 635 |
|
|
|
|
| 643 |
os.environ.setdefault("HF_TOKEN", token)
|
| 644 |
os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
|
| 645 |
os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
|
| 646 |
+
expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"))
|
| 647 |
payload = {
|
| 648 |
"hf_token_present": bool(token),
|
| 649 |
"hf_token_length": len(token) if token else 0,
|
|
|
|
| 652 |
"bucket_source": os.environ.get("BUCKET_SOURCE") or "",
|
| 653 |
"target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
|
| 654 |
"token_value": "[REDACTED]" if token else "",
|
| 655 |
+
"token_kind": expiry.get("token_kind"),
|
| 656 |
+
"expiry_known": expiry.get("expiry_known"),
|
| 657 |
+
"expires_at": expiry.get("expires_at"),
|
| 658 |
+
"seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"),
|
| 659 |
+
"auth_risk": expiry.get("auth_risk"),
|
| 660 |
+
"safe_for_long_build": bool(expiry.get("safe_for_phase")),
|
| 661 |
+
"minimum_required_seconds": expiry.get("minimum_required_seconds"),
|
| 662 |
}
|
| 663 |
run_dir.mkdir(parents=True, exist_ok=True)
|
| 664 |
write_json(run_dir / "token_context.json", payload)
|
|
|
|
| 943 |
return "cancelled"
|
| 944 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 945 |
return "manual_action_required"
|
| 946 |
+
if status in {"technical_blocker", "technical_blocker_boot_only"}:
|
| 947 |
return "technical_blocker"
|
| 948 |
if full_inference_verified:
|
| 949 |
return "success"
|
|
|
|
| 1160 |
existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
|
| 1161 |
if not isinstance(existing_state, dict):
|
| 1162 |
existing_state = {}
|
| 1163 |
+
repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {}
|
| 1164 |
+
if isinstance(repair_outcome, dict) and repair_outcome:
|
| 1165 |
+
safe.setdefault("repair_outcome_status", repair_outcome.get("post_repair_validation") or repair_outcome.get("repair_status") or "")
|
| 1166 |
+
safe.setdefault("repair_failure_type", repair_outcome.get("failure_type") or "")
|
| 1167 |
failure_state = {
|
| 1168 |
**existing_state,
|
| 1169 |
"run_id": os.environ.get("RUN_ID"),
|
|
|
|
| 1172 |
"message": message,
|
| 1173 |
"updated_at": now(),
|
| 1174 |
"details": safe,
|
| 1175 |
+
"repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {},
|
| 1176 |
}
|
| 1177 |
# Preserve the target Space when the worker fails after repository creation.
|
| 1178 |
target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
|
|
|
|
| 1808 |
return {}
|
| 1809 |
|
| 1810 |
|
| 1811 |
+
def _as_bool_false(value) -> bool:
|
| 1812 |
+
if value is False:
|
| 1813 |
+
return True
|
| 1814 |
+
if isinstance(value, str):
|
| 1815 |
+
return value.strip().lower() in {"false", "no", "0", "off"}
|
| 1816 |
+
return False
|
| 1817 |
+
|
| 1818 |
+
|
| 1819 |
+
def _read_workspace_json(workspace: Path | None, filename: str) -> dict:
|
| 1820 |
+
if not workspace:
|
| 1821 |
+
return {}
|
| 1822 |
+
path = workspace / filename
|
| 1823 |
+
if not path.exists():
|
| 1824 |
+
return {}
|
| 1825 |
+
try:
|
| 1826 |
+
data = json.loads(path.read_text(encoding="utf-8", errors="replace"))
|
| 1827 |
+
return data if isinstance(data, dict) else {}
|
| 1828 |
+
except Exception:
|
| 1829 |
+
return {}
|
| 1830 |
+
|
| 1831 |
+
|
| 1832 |
+
def contract_declares_no_full_inference(workspace: Path | None) -> dict:
|
| 1833 |
+
"""Return a structured reason when Pi explicitly declared boot-only/no inference.
|
| 1834 |
+
|
| 1835 |
+
v191.2: A machine-readable no-inference contract must be respected by the
|
| 1836 |
+
worker. In that case automatic generation smoke is skipped instead of
|
| 1837 |
+
inventing /generate fallback arguments that cannot exist.
|
| 1838 |
+
"""
|
| 1839 |
+
contract = read_inference_contract(workspace)
|
| 1840 |
+
blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json")
|
| 1841 |
+
validation_level = str(contract.get("validation_level") or "").strip().lower().replace("_", "-")
|
| 1842 |
+
primary = normalize_api_name(str(contract.get("primary_api_name") or "")) if contract else ""
|
| 1843 |
+
contract_full_false = _as_bool_false(contract.get("full_inference_implemented"))
|
| 1844 |
+
blocker_full_false = _as_bool_false(blockers.get("full_inference_implemented"))
|
| 1845 |
+
validation_boot_only = validation_level in {"boot-only", "health-only", "diagnostic-only", "info-only"}
|
| 1846 |
+
primary_absent = not primary or primary == "/health"
|
| 1847 |
+
blockers_count = blockers.get("blockers_count")
|
| 1848 |
+
if not isinstance(blockers_count, int):
|
| 1849 |
+
raw_blockers = blockers.get("blockers")
|
| 1850 |
+
blockers_count = len(raw_blockers) if isinstance(raw_blockers, list) else 0
|
| 1851 |
+
|
| 1852 |
+
declared = bool(contract_full_false or blocker_full_false or (validation_boot_only and primary_absent))
|
| 1853 |
+
return {
|
| 1854 |
+
"declared": declared,
|
| 1855 |
+
"source": "inference_contract" if contract_full_false or validation_boot_only else "technical_blockers" if blocker_full_false else "",
|
| 1856 |
+
"full_inference_implemented": False if (contract_full_false or blocker_full_false) else contract.get("full_inference_implemented"),
|
| 1857 |
+
"validation_level": contract.get("validation_level"),
|
| 1858 |
+
"primary_api_name": contract.get("primary_api_name"),
|
| 1859 |
+
"blockers_count": blockers_count,
|
| 1860 |
+
"contract_present": bool(contract),
|
| 1861 |
+
"technical_blockers_present": bool(blockers),
|
| 1862 |
+
}
|
| 1863 |
+
|
| 1864 |
+
|
| 1865 |
+
def write_contract_skipped_generation_smoke(run_dir: Path, events_path: Path, expected_output_type: str, target_space_id: str, reason: dict | None = None) -> dict:
|
| 1866 |
+
reason = reason or {}
|
| 1867 |
+
payload = {
|
| 1868 |
+
"status": "skipped",
|
| 1869 |
+
"skip_reason": "contract_declared_no_full_inference",
|
| 1870 |
+
"reason": "Pi declared that full inference is not implemented; automatic generation smoke was skipped.",
|
| 1871 |
+
"target_space": target_space_id,
|
| 1872 |
+
"api_name": None,
|
| 1873 |
+
"expected_output_type": expected_output_type,
|
| 1874 |
+
"validation_mode": "automatic_smoke",
|
| 1875 |
+
"payload_source": "contract_declared_no_full_inference",
|
| 1876 |
+
"contract_aware_skip": True,
|
| 1877 |
+
"contract_reason": reason,
|
| 1878 |
+
"next_action": "Full inference is blocked. No generation endpoint exists. Review TECHNICAL_BLOCKERS.json / PI_SUMMARY.md or provide a dedicated implementation and hardware plan.",
|
| 1879 |
+
**measured_zero_gpu_recommendation(None),
|
| 1880 |
+
}
|
| 1881 |
+
write_json(run_dir / "tests" / "generation_smoke.json", payload)
|
| 1882 |
+
write_json(run_dir / "tests" / "payload_source.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "selected_api_name": None, "parent_smoke_payload_used": False})
|
| 1883 |
+
write_json(run_dir / "tests" / "validation_engine.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "resolved_request_required_before_predict": False, "skipped_by_contract": True})
|
| 1884 |
+
write_json(run_dir / "tests" / "resolved_validation_request.json", {"api_name": None, "test_args": [], "test_kwargs": {}, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "resolved_request_required_before_predict": False})
|
| 1885 |
+
write_live_status(run_dir, stage="generation_smoke", status="skipped", message="Generation smoke skipped because Pi declared full inference unavailable", data=payload)
|
| 1886 |
+
append_event(events_path, "generation_smoke", "skipped", "Generation smoke skipped because Pi declared no full inference endpoint", payload)
|
| 1887 |
+
return payload
|
| 1888 |
+
|
| 1889 |
+
|
| 1890 |
def contract_smoke_test_payload(contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None:
|
| 1891 |
smoke = contract.get("smoke_test") if isinstance(contract, dict) else None
|
| 1892 |
if not isinstance(smoke, dict):
|
|
|
|
| 2876 |
raise
|
| 2877 |
|
| 2878 |
|
| 2879 |
+
|
| 2880 |
+
|
| 2881 |
+
def _safe_json_object_from_workspace(path: Path) -> dict:
|
| 2882 |
+
payload = read_json(path, {})
|
| 2883 |
+
return payload if isinstance(payload, dict) else {}
|
| 2884 |
+
|
| 2885 |
+
|
| 2886 |
+
def _first_truthy_string(*values) -> str | None:
|
| 2887 |
+
for value in values:
|
| 2888 |
+
if isinstance(value, str) and value.strip():
|
| 2889 |
+
return value.strip()
|
| 2890 |
+
return None
|
| 2891 |
+
|
| 2892 |
+
|
| 2893 |
+
def write_pi_planning_review(workspace: Path, run_dir: Path, events_path: Path, model_analysis: dict | None = None, implementation_mode: str = "") -> dict:
|
| 2894 |
+
"""Persist Pi planning artifacts and a non-blocking worker review.
|
| 2895 |
+
|
| 2896 |
+
v191.9 intentionally makes planning visibility-only. Pi is asked to produce
|
| 2897 |
+
a feasibility brief and implementation plan, but older/partial Pi behavior
|
| 2898 |
+
must not fail a run that would otherwise work. The worker records whether
|
| 2899 |
+
those artifacts exist, mirrors them under planning/, and keeps the final
|
| 2900 |
+
authority in the existing deterministic gates.
|
| 2901 |
+
"""
|
| 2902 |
+
model_analysis = model_analysis or {}
|
| 2903 |
+
planning_dir = run_dir / "planning"
|
| 2904 |
+
planning_dir.mkdir(parents=True, exist_ok=True)
|
| 2905 |
+
brief_path = workspace / "pi_feasibility_brief.json"
|
| 2906 |
+
plan_path = workspace / "pi_implementation_plan.json"
|
| 2907 |
+
brief = _safe_json_object_from_workspace(brief_path)
|
| 2908 |
+
plan = _safe_json_object_from_workspace(plan_path)
|
| 2909 |
+
brief_present = bool(brief)
|
| 2910 |
+
plan_present = bool(plan)
|
| 2911 |
+
if brief_present:
|
| 2912 |
+
write_json(planning_dir / "pi_feasibility_brief.json", brief)
|
| 2913 |
+
else:
|
| 2914 |
+
write_json(planning_dir / "pi_feasibility_brief.json", {
|
| 2915 |
+
"schema_version": "pi_feasibility_brief.v1",
|
| 2916 |
+
"status": "missing",
|
| 2917 |
+
"non_blocking": True,
|
| 2918 |
+
"message": "Pi did not produce pi_feasibility_brief.json; continuing with existing deterministic worker gates.",
|
| 2919 |
+
})
|
| 2920 |
+
if plan_present:
|
| 2921 |
+
write_json(planning_dir / "pi_implementation_plan.json", plan)
|
| 2922 |
+
else:
|
| 2923 |
+
write_json(planning_dir / "pi_implementation_plan.json", {
|
| 2924 |
+
"schema_version": "pi_implementation_plan.v1",
|
| 2925 |
+
"status": "missing",
|
| 2926 |
+
"non_blocking": True,
|
| 2927 |
+
"message": "Pi did not produce pi_implementation_plan.json; continuing with existing deterministic worker gates.",
|
| 2928 |
+
})
|
| 2929 |
+
|
| 2930 |
+
declared_strategy = _first_truthy_string(
|
| 2931 |
+
brief.get("recommended_strategy"),
|
| 2932 |
+
brief.get("strategy"),
|
| 2933 |
+
plan.get("strategy"),
|
| 2934 |
+
plan.get("implementation_strategy"),
|
| 2935 |
+
) or "unspecified"
|
| 2936 |
+
should_attempt_full = brief.get("should_attempt_full_inference")
|
| 2937 |
+
if should_attempt_full is None:
|
| 2938 |
+
should_attempt_full = plan.get("should_attempt_full_inference")
|
| 2939 |
+
worker_recommendation = "continue_existing_flow"
|
| 2940 |
+
warnings: list[str] = []
|
| 2941 |
+
if not brief_present:
|
| 2942 |
+
warnings.append("missing_pi_feasibility_brief")
|
| 2943 |
+
if not plan_present:
|
| 2944 |
+
warnings.append("missing_pi_implementation_plan")
|
| 2945 |
+
if declared_strategy in {"boot_only_blocker", "boot_only_blocker_or_manual_refactor", "manual_hardware_required"} or should_attempt_full is False:
|
| 2946 |
+
worker_recommendation = "respect_plan_if_contract_declares_blocker"
|
| 2947 |
+
elif declared_strategy in {"full_inference", "proceed_full_inference", "attempt_full_inference"} or should_attempt_full is True:
|
| 2948 |
+
worker_recommendation = "proceed_full_inference_with_existing_gates"
|
| 2949 |
+
|
| 2950 |
+
review = {
|
| 2951 |
+
"schema_version": "worker_plan_review.v1",
|
| 2952 |
+
"status": "ok" if brief_present and plan_present else "partial",
|
| 2953 |
+
"non_blocking": True,
|
| 2954 |
+
"implementation_mode": implementation_mode,
|
| 2955 |
+
"model_id": model_analysis.get("model_id"),
|
| 2956 |
+
"pipeline_tag": model_analysis.get("pipeline_tag"),
|
| 2957 |
+
"library_name": model_analysis.get("library_name"),
|
| 2958 |
+
"planning_artifacts_present": {
|
| 2959 |
+
"pi_feasibility_brief": brief_present,
|
| 2960 |
+
"pi_implementation_plan": plan_present,
|
| 2961 |
+
},
|
| 2962 |
+
"declared_strategy": declared_strategy,
|
| 2963 |
+
"should_attempt_full_inference": should_attempt_full,
|
| 2964 |
+
"worker_recommendation": worker_recommendation,
|
| 2965 |
+
"warnings": warnings,
|
| 2966 |
+
"authority_note": "Pi planning is advisory in v191.9; deterministic worker gates, INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation remain authoritative.",
|
| 2967 |
+
"created_at": now(),
|
| 2968 |
+
}
|
| 2969 |
+
write_json(planning_dir / "worker_plan_review.json", review)
|
| 2970 |
+
append_event(events_path, "pi_planning_review", review["status"], "Pi feasibility and implementation planning reviewed", review)
|
| 2971 |
+
return review
|
| 2972 |
+
|
| 2973 |
def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None):
|
| 2974 |
workspace.mkdir(parents=True, exist_ok=True)
|
| 2975 |
model_analysis = model_analysis or {}
|
|
|
|
| 3017 |
accelerate
|
| 3018 |
safetensors
|
| 3019 |
torch
|
|
|
|
| 3020 |
pillow
|
| 3021 |
numpy
|
| 3022 |
requests
|
|
|
|
| 3063 |
- Work only inside the current workspace.
|
| 3064 |
- The wrapper will create the private Space, request allowed hardware best-effort, upload files, and validate the live app. Do not create/delete repos yourself in this builder worker.
|
| 3065 |
- Preserve a cheap health endpoint named `health` with `api_name="health"`. It must not load weights, run GPU work, or download large files.
|
| 3066 |
+
- Platform dependency policy: Gradio, Gradio Client, Hugging Face Hub, Spaces, and hf_xet are owned by Agentic Space Factory / the Hugging Face Spaces runtime. Do not pin or downgrade them. Never write `gradio==...`, `gradio<=...`, `gradio~=...`, `gradio-client==...`, `gradio_client==...`, or stale exact pins for `huggingface_hub`, `spaces`, or `hf_xet`.
|
| 3067 |
+
- If generated code needs an older Gradio API, update the code to modern Gradio instead of downgrading Gradio.
|
| 3068 |
+
- Follow the gist dependency method for model-specific packages: pin only when useful to reduce pip resolver backtracking or when the model card/build logs explicitly require a version; do not cargo-cult every version from examples.
|
| 3069 |
+
- Do not pin torch or torchaudio unless unavoidable and explicitly justified; Spaces provide the PyTorch stack. If torchvision is needed, prefer leaving it unpinned so it can resolve against the managed torch runtime.
|
| 3070 |
+
- If model code is not pip-installable, vendor the necessary code into the Space repo instead of referencing local paths, editable installs, or clone-time side effects in requirements.txt.
|
| 3071 |
+
- Do not run local venv installs, broad `pip install --dry-run`, or dependency checks as proof of compatibility; only the live Space build/logs are authoritative. Use `python -m py_compile app.py` only as a cheap syntax check.
|
| 3072 |
+
- Use huggingface_hub>=0.34.0,<2.0.0 unless a newer Transformers path requires a compatible newer range. The worker will normalize platform-owned dependency lines before upload.
|
| 3073 |
- README.md frontmatter must remain valid; if it uses short_description, it must be 60 characters or fewer.
|
| 3074 |
|
| 3075 |
+
Planning contract:
|
| 3076 |
+
- Before committing to a full implementation strategy, produce `pi_feasibility_brief.json` and `pi_implementation_plan.json`. These files are advisory artifacts for the worker and must not contain secrets.
|
| 3077 |
+
- `pi_feasibility_brief.json` should include: complexity, expected_runtime, zerogpu_feasible, requires_custom_code, multi_gpu_risk, dependency_risk, recommended_strategy, minimum_hardware, should_attempt_full_inference, and evidence.
|
| 3078 |
+
- `pi_implementation_plan.json` should include: selected_strategy, intended_files, endpoints, dependency_strategy, hardware_strategy, smoke_test_plan, fallback_plan, and blocker_policy.
|
| 3079 |
+
- If the model appears high-risk or not portable, say so in the plan rather than silently attempting a fragile fake success. The worker will still use INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation as the final authority.
|
| 3080 |
+
|
| 3081 |
Implementation contract:
|
| 3082 |
- If IMPLEMENTATION_MODE is `full-inference-gated`, you are not allowed to silently replace generation with a placeholder and call it success.
|
| 3083 |
- Try to implement the closest real inference path for the model card using evidence from README, model metadata, config files, and repo files.
|
|
|
|
| 3085 |
- If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint.
|
| 3086 |
- If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement.
|
| 3087 |
- If the model requires special dependencies, include them only when needed and document risks.
|
| 3088 |
+
- Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels/Kernel Hub where relevant, Transformers AttentionInterface, Diffusers attention processors, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs.
|
| 3089 |
+
- Native kernel policy: if the model card or logs mention flash-attn, custom attention kernels, xformers, Triton kernels, fused ops, or CUDA/C++ extensions, do not blindly add source-built native packages to requirements.txt. Prefer runtime-compatible drop-in backends or compatible wheels first. Use `kernels` only when it directly matches the needed operation and the target hardware/runtime is plausible; document the selected backend in PI_SUMMARY.md and INFERENCE_CONTRACT.json.
|
| 3090 |
- If real inference is impossible or unsafe in a Space, write TECHNICAL_BLOCKERS.json with concrete evidence for every blocker.
|
| 3091 |
|
| 3092 |
Deliverables:
|
| 3093 |
+
- pi_feasibility_brief.json must summarize feasibility and risk before/alongside implementation.
|
| 3094 |
+
- pi_implementation_plan.json must summarize the selected implementation plan, endpoints, dependencies, and validation approach.
|
| 3095 |
- app.py must boot on Hugging Face Spaces.
|
| 3096 |
- app.py must expose health/api_name="health".
|
| 3097 |
- If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic.
|
|
|
|
| 3179 |
)
|
| 3180 |
|
| 3181 |
def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
|
| 3182 |
+
"""Normalize broad known-dangerous base and platform dependencies before upload.
|
| 3183 |
|
| 3184 |
+
Do not try to solve every dependency conflict here. Do not try to solve every model-specific dependency conflict here. Pi is
|
| 3185 |
+
responsible for reading concrete build logs and patching requirements when
|
| 3186 |
+
a Space build fails. The Factory only prevents obviously unsafe broad ranges
|
| 3187 |
+
and owns the Gradio/HF platform stack so old generated pins cannot conflict
|
| 3188 |
+
with the current Spaces runtime; concrete pip conflicts remain Pi repair work.
|
| 3189 |
"""
|
| 3190 |
req_path = workspace / "requirements.txt"
|
| 3191 |
if not req_path.exists():
|
|
|
|
| 3206 |
|
| 3207 |
# Minimal base pins only. Do not globally pin diffusers here: newer model
|
| 3208 |
# cards may legitimately need a recent Diffusers release. If diffusers causes
|
| 3209 |
+
# a pip conflict, Pi is responsible for concrete build-log repairs. Pi must repair from the concrete build log.
|
| 3210 |
policy: dict[str, str] = {
|
| 3211 |
"huggingface-hub": "huggingface_hub>=0.34.0,<2.0.0",
|
| 3212 |
"transformers": "transformers>=4.51.0,<5.0.0",
|
| 3213 |
}
|
| 3214 |
+
# Platform-owned dependencies are normalized more aggressively than model
|
| 3215 |
+
# dependencies. Gradio/HF runtime packages must stay current with Spaces and
|
| 3216 |
+
# Agentic Space Factory validation; Pi should patch app.py for modern Gradio
|
| 3217 |
+
# rather than pinning or downgrading this stack.
|
| 3218 |
+
platform_policy: dict[str, str] = {
|
| 3219 |
+
"gradio": "gradio",
|
| 3220 |
+
"gradio-client": "gradio-client",
|
| 3221 |
+
"huggingface-hub": policy["huggingface-hub"],
|
| 3222 |
+
"spaces": "spaces>=0.30",
|
| 3223 |
+
"hf-xet": "hf_xet",
|
| 3224 |
+
}
|
| 3225 |
aliases = {
|
| 3226 |
"huggingface_hub": "huggingface-hub",
|
| 3227 |
"huggingface-hub": "huggingface-hub",
|
| 3228 |
"transformers": "transformers",
|
| 3229 |
+
"gradio": "gradio",
|
| 3230 |
+
"gradio-client": "gradio-client",
|
| 3231 |
+
"gradio_client": "gradio-client",
|
| 3232 |
+
"spaces": "spaces",
|
| 3233 |
+
"hf-xet": "hf-xet",
|
| 3234 |
+
"hf_xet": "hf-xet",
|
| 3235 |
}
|
| 3236 |
+
platform_owned = set(platform_policy)
|
| 3237 |
seen_policy: set[str] = set()
|
| 3238 |
+
seen_platform: set[str] = set()
|
| 3239 |
filtered: list[str] = []
|
| 3240 |
changed = False
|
| 3241 |
+
removed_platform_pins: list[str] = []
|
| 3242 |
+
normalized_platform_lines: list[dict] = []
|
| 3243 |
|
| 3244 |
for line in package_lines:
|
| 3245 |
stripped = line.strip()
|
| 3246 |
if stripped.startswith("#") or "://" in stripped or stripped.startswith((".", "/")):
|
| 3247 |
filtered.append(line)
|
| 3248 |
continue
|
| 3249 |
+
name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-")
|
| 3250 |
canonical = aliases.get(name)
|
| 3251 |
+
if canonical in platform_owned:
|
| 3252 |
+
normalized = platform_policy[canonical]
|
| 3253 |
+
if stripped != normalized:
|
| 3254 |
+
changed = True
|
| 3255 |
+
removed_platform_pins.append(stripped)
|
| 3256 |
+
normalized_platform_lines.append({"from": stripped, "to": normalized, "package": canonical})
|
| 3257 |
+
if canonical not in seen_platform:
|
| 3258 |
+
filtered.append(normalized)
|
| 3259 |
+
seen_platform.add(canonical)
|
| 3260 |
+
if canonical in policy:
|
| 3261 |
+
seen_policy.add(canonical)
|
| 3262 |
+
else:
|
| 3263 |
+
changed = True
|
| 3264 |
+
continue
|
| 3265 |
if canonical in policy:
|
| 3266 |
if stripped != policy[canonical]:
|
| 3267 |
changed = True
|
|
|
|
| 3279 |
stable_policy_lines.append(policy[canonical])
|
| 3280 |
changed = True
|
| 3281 |
|
| 3282 |
+
injected_platform_lines: list[str] = []
|
| 3283 |
+
app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore").lower() if (workspace / "app.py").exists() else ""
|
| 3284 |
+
if ("import gradio" in app_text or "from gradio" in app_text or "gr." in app_text) and "gradio" not in seen_platform:
|
| 3285 |
+
injected_platform_lines.append(platform_policy["gradio"])
|
| 3286 |
+
seen_platform.add("gradio")
|
| 3287 |
+
changed = True
|
| 3288 |
+
if ("gradio_client" in app_text or "from gradio_client" in app_text) and "gradio-client" not in seen_platform:
|
| 3289 |
+
injected_platform_lines.append(platform_policy["gradio-client"])
|
| 3290 |
+
seen_platform.add("gradio-client")
|
| 3291 |
+
changed = True
|
| 3292 |
+
|
| 3293 |
torch_added = False
|
| 3294 |
+
if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines + injected_platform_lines, "torch"):
|
| 3295 |
stable_policy_lines.append("torch>=2.0.0")
|
| 3296 |
torch_added = True
|
| 3297 |
changed = True
|
| 3298 |
|
| 3299 |
+
new_lines = prefix_lines + stable_policy_lines + injected_platform_lines + filtered
|
| 3300 |
new = "\n".join(line for line in new_lines if line.strip()) + "\n"
|
| 3301 |
if new != raw:
|
| 3302 |
changed = True
|
| 3303 |
if changed:
|
| 3304 |
req_path.write_text(new, encoding="utf-8")
|
| 3305 |
+
policy_payload = {
|
| 3306 |
+
"schema_version": "1.0",
|
| 3307 |
+
"status": "normalized",
|
| 3308 |
+
"platform_owned_dependencies": ["gradio", "gradio-client", "huggingface_hub", "spaces", "hf_xet"],
|
| 3309 |
+
"removed_pins": removed_platform_pins,
|
| 3310 |
+
"normalized_platform_lines": normalized_platform_lines,
|
| 3311 |
+
"injected_platform_lines": injected_platform_lines,
|
| 3312 |
+
"base_policy": {"huggingface_hub": policy["huggingface-hub"], "transformers": policy["transformers"]},
|
| 3313 |
+
"base_policy_reason": "Avoid uncontrolled Transformers 5.x while preserving model-specific dependency choices for Pi to repair from build logs; require torch when app.py imports torch.",
|
| 3314 |
+
"torch_added": torch_added,
|
| 3315 |
+
"torch_policy": "torch>=2.0.0",
|
| 3316 |
+
"torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present",
|
| 3317 |
+
"model_dependency_policy": "Model-specific pins are preserved unless they hit a known deterministic guardrail; concrete conflicts remain Pi repair work from build logs.",
|
| 3318 |
+
"reason": "Gradio/HF runtime dependencies are owned by Agentic Space Factory / Spaces runtime and must not be downgraded by generated requirements.",
|
| 3319 |
+
}
|
| 3320 |
+
write_json(workspace / "requirements_policy.json", policy_payload)
|
| 3321 |
append_event(
|
| 3322 |
events_path,
|
| 3323 |
"requirements_sanitize",
|
| 3324 |
"success",
|
| 3325 |
+
"Normalized platform-owned Gradio/HF dependencies and broad base dependencies; concrete pip conflicts remain Pi repair work",
|
| 3326 |
+
policy_payload,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3327 |
)
|
| 3328 |
|
|
|
|
| 3329 |
def useful_log_signals(text: str) -> list[str]:
|
| 3330 |
low = (text or "").lower()
|
| 3331 |
signals = []
|
|
|
|
| 3809 |
|
| 3810 |
You are not allowed to edit code in this diagnosis step. Your task is to decide the next action for the Factory.
|
| 3811 |
Use the gist method: read logs first, identify the first actionable error, use the cheapest useful iteration rung, and require a live Gradio/API validation before success.
|
| 3812 |
+
When the first error is dependency-related, classify it with the ASF dependency contract in mind: Gradio/HF runtime packages are platform-owned, model-specific pins need evidence, torch/torchaudio are managed by Spaces, and non-pip-installable model code should be vendored.
|
| 3813 |
|
| 3814 |
Write `REPAIR_DECISION.json` exactly as requested in INCIDENT_BRIEF.md. Do not patch files during this step.
|
| 3815 |
"""
|
|
|
|
| 3927 |
- Preserve or restore a cheap `health` endpoint.
|
| 3928 |
- Preserve the expected Gradio API endpoint when possible.
|
| 3929 |
- Keep README metadata valid and short_description <= 60 chars.
|
| 3930 |
+
- Platform dependency policy: keep Gradio / Gradio Client / Hugging Face Hub / Spaces modern and worker-owned. Do not downgrade Gradio or add stale exact pins. If the failure involves a Gradio dependency conflict, remove the stale Gradio pin and patch app.py for modern Gradio instead of downgrading Gradio.
|
| 3931 |
+
- For model-specific dependencies, follow the gist method: patch the first concrete resolver error minimally, preserve justified model-card pins, and do not pin torch/torchaudio unless unavoidable and explicitly justified. If torchvision is needed, prefer leaving it unpinned so it resolves against the managed torch runtime.
|
| 3932 |
+
- If model code is not pip-installable, vendor the necessary code into the Space repo instead of relying on local paths, editable installs, or clone-time side effects in requirements.txt.
|
| 3933 |
+
- Do not prove dependency fixes with local venv installs or broad pip dry-runs; Space build logs are authoritative. Keep local checks cheap, such as `python -m py_compile app.py`.
|
| 3934 |
+
- Native kernel policy: for flash-attn/custom attention/xformers/Triton/fused CUDA errors, do not blindly add source-built packages. First consider PyTorch SDPA, compatible prebuilt wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors. If no compatible fallback exists, declare a blocker rather than forcing a fragile build.
|
| 3935 |
|
| 3936 |
## Required repair artifacts
|
| 3937 |
Before modifying files, write `REPAIR_PLAN.md` with:
|
|
|
|
| 4012 |
You are continuing the same build run, not starting a separate project.
|
| 4013 |
This patch is allowed only because the diagnosis decision selected `patch_code`.
|
| 4014 |
If `DEPENDENCY_ERROR_BRIEF.md` exists, treat it as evidence for the gist method: identify the first pip error, patch dependency pins minimally, and do not modify inference code unless the dependency fix alone cannot address that first error.
|
| 4015 |
+
Follow the ASF/gist dependency contract during repair: Gradio/HF runtime packages are platform-owned, model-specific pins must be evidence-backed, torch/torchaudio should remain managed by Spaces, and non-pip-installable model code should be vendored instead of referenced through fragile local paths.
|
| 4016 |
+
For native kernel failures such as flash-attn/custom attention/xformers/Triton/fused CUDA ops, do not blindly add source-built native packages. Prefer PyTorch SDPA, compatible wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors when they match the required operation. Use `kernels` only when directly justified and document the backend in REPAIR_SUMMARY.md / INFERENCE_CONTRACT.json.
|
| 4017 |
Use the available HF token context to inspect private Hub resources when needed, but never print or persist token values.
|
| 4018 |
|
| 4019 |
Critical method:
|
|
|
|
| 4089 |
dependency_issue = extract_pip_dependency_issue(f"{current_error}\n{build_log}\n{runtime_log}")
|
| 4090 |
if dependency_issue:
|
| 4091 |
write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, current_error)
|
| 4092 |
+
write_repair_outcome(
|
| 4093 |
+
run_dir,
|
| 4094 |
+
events_path,
|
| 4095 |
+
repair_trigger="dependency_resolution_error",
|
| 4096 |
+
root_cause="dependency_resolution_error",
|
| 4097 |
+
initial_error=str(current_error)[:4000],
|
| 4098 |
+
patch_applied=False,
|
| 4099 |
+
upload_success=False,
|
| 4100 |
+
post_repair_validation="not_started",
|
| 4101 |
+
)
|
| 4102 |
if apply_dependency_guardrail_repair(workspace, run_dir, events_path, current_error, build_log, runtime_log):
|
| 4103 |
+
write_repair_outcome(
|
| 4104 |
+
run_dir,
|
| 4105 |
+
events_path,
|
| 4106 |
+
repair_trigger="dependency_resolution_error",
|
| 4107 |
+
repair_decision="deterministic_dependency_guardrail",
|
| 4108 |
+
patch_applied=True,
|
| 4109 |
+
upload_success=False,
|
| 4110 |
+
post_repair_validation="not_started",
|
| 4111 |
+
)
|
| 4112 |
append_event(events_path, "factory_rebuild", "started", "Re-uploading dependency-guardrailed workspace after pip resolver build error")
|
| 4113 |
if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="dependency_guardrail_repair"):
|
| 4114 |
+
write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="not_started", failure_type="repair_upload_blocked_by_guardrail")
|
| 4115 |
raise RuntimeError("Dependency guardrail rebuild skipped by restart guardrails")
|
| 4116 |
+
write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending")
|
| 4117 |
append_event(events_path, "factory_rebuild", "success", "Dependency-guardrailed workspace uploaded; revalidating live Space")
|
| 4118 |
append_event(events_path, "repair_validation", "started", "Revalidating after deterministic dependency repair")
|
| 4119 |
write_live_status(run_dir, stage="live_validation", status="running", message="Waiting for Space runtime and health endpoint", data={"target_space": target_space_id})
|
| 4120 |
try:
|
| 4121 |
+
write_auth_probe(run_dir, events_path, "before_initial_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"), raise_on_unsafe=True)
|
| 4122 |
validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
|
| 4123 |
+
write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Deterministic dependency repair resolved the build blockage.")
|
| 4124 |
append_event(events_path, "repair_validation", "success", "Deterministic dependency repair resolved the build blockage")
|
| 4125 |
return validation
|
| 4126 |
except Exception as exc:
|
| 4127 |
+
repair_class = classify_repair_validation_error(exc)
|
| 4128 |
+
write_repair_outcome(
|
| 4129 |
+
run_dir,
|
| 4130 |
+
events_path,
|
| 4131 |
+
post_repair_validation=repair_class["post_repair_validation"],
|
| 4132 |
+
failure_type=repair_class["failure_type"],
|
| 4133 |
+
final_user_message=repair_class["message"],
|
| 4134 |
+
validation_error=str(exc)[:4000],
|
| 4135 |
+
)
|
| 4136 |
+
if repair_class["terminal_status"] == "auth_refresh_required":
|
| 4137 |
+
append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]})
|
| 4138 |
+
raise AuthRefreshRequired(repair_class["message"])
|
| 4139 |
current_error = f"{current_error}\n\nDependency guardrail rebuild did not resolve validation: {str(exc)[:4000]}"
|
| 4140 |
collect_space_logs(target_space_id, token, run_dir, events_path)
|
| 4141 |
append_event(events_path, "repair_validation", "failed", "Dependency guardrail rebuild did not resolve validation; falling back to Pi diagnosis", {"error": str(exc)[:4000]})
|
|
|
|
| 4173 |
append_event(events_path, "factory_rebuild", "success", "Same-code workspace re-uploaded; revalidating live Space")
|
| 4174 |
append_event(events_path, "repair_validation", "started", "Revalidating after same-code factory rebuild")
|
| 4175 |
try:
|
| 4176 |
+
write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True)
|
| 4177 |
validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
|
| 4178 |
+
write_repair_outcome(run_dir, events_path, repair_decision="factory_rebuild_same_code", patch_applied=False, upload_success=True, post_repair_validation="success", failure_type="", final_user_message="Same-code factory rebuild resolved the blockage.")
|
| 4179 |
append_event(events_path, "repair_validation", "success", "Same-code factory rebuild resolved the blockage")
|
| 4180 |
return validation
|
| 4181 |
except Exception as exc:
|
| 4182 |
+
repair_class = classify_repair_validation_error(exc)
|
| 4183 |
+
write_repair_outcome(
|
| 4184 |
+
run_dir,
|
| 4185 |
+
events_path,
|
| 4186 |
+
repair_decision="factory_rebuild_same_code",
|
| 4187 |
+
patch_applied=False,
|
| 4188 |
+
upload_success=True,
|
| 4189 |
+
post_repair_validation=repair_class["post_repair_validation"],
|
| 4190 |
+
failure_type=repair_class["failure_type"],
|
| 4191 |
+
final_user_message=repair_class["message"],
|
| 4192 |
+
validation_error=str(exc)[:4000],
|
| 4193 |
+
)
|
| 4194 |
+
if repair_class["terminal_status"] == "auth_refresh_required":
|
| 4195 |
+
append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]})
|
| 4196 |
+
raise AuthRefreshRequired(repair_class["message"])
|
| 4197 |
current_error = f"{current_error}\n\nSame-code factory rebuild did not resolve validation: {str(exc)[:4000]}"
|
| 4198 |
collect_space_logs(target_space_id, token, run_dir, events_path)
|
| 4199 |
append_event(events_path, "repair_validation", "failed", "Same-code factory rebuild did not resolve validation; re-diagnosing", {"error": str(exc)[:4000]})
|
|
|
|
| 4206 |
raise RuntimeError("Automated recovery stopped after patch budget was exhausted")
|
| 4207 |
budgets[action] -= 1
|
| 4208 |
append_event(events_path, "repair", "started", "Pi diagnosis allows a minimal code patch", {"decision": decision})
|
| 4209 |
+
write_repair_outcome(
|
| 4210 |
+
run_dir,
|
| 4211 |
+
events_path,
|
| 4212 |
+
repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure",
|
| 4213 |
+
root_cause=(decision.get("classification") or {}).get("category") or "live_validation_failure",
|
| 4214 |
+
repair_decision="patch_code",
|
| 4215 |
+
decision=decision,
|
| 4216 |
+
patch_applied=False,
|
| 4217 |
+
upload_success=False,
|
| 4218 |
+
post_repair_validation="not_started",
|
| 4219 |
+
)
|
| 4220 |
repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision)
|
| 4221 |
if not repaired:
|
| 4222 |
+
write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation="not_started", failure_type="repair_patch_failed", final_user_message="Structured patch repair failed before redeploy.")
|
| 4223 |
write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair")
|
| 4224 |
append_event(events_path, "failure", "failed", "Structured patch repair failed before redeploy", {"decision": decision})
|
| 4225 |
raise RuntimeError("Structured patch repair failed before redeploy")
|
| 4226 |
+
write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started")
|
| 4227 |
append_event(events_path, "repair_upload", "started", "Uploading repaired workspace")
|
| 4228 |
+
write_auth_probe(run_dir, events_path, "before_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True)
|
| 4229 |
upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
|
| 4230 |
+
write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending")
|
| 4231 |
append_event(events_path, "repair_upload", "success", "Repaired workspace uploaded")
|
| 4232 |
append_event(events_path, "repair_validation", "started", "Revalidating repaired Space")
|
| 4233 |
try:
|
| 4234 |
+
write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True)
|
| 4235 |
validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
|
| 4236 |
+
write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Repaired Space passed live API validation.")
|
| 4237 |
append_event(events_path, "repair_validation", "success", "Repaired Space passed live API validation")
|
| 4238 |
return validation
|
| 4239 |
except Exception as exc:
|
| 4240 |
+
repair_class = classify_repair_validation_error(exc)
|
| 4241 |
+
write_repair_outcome(
|
| 4242 |
+
run_dir,
|
| 4243 |
+
events_path,
|
| 4244 |
+
post_repair_validation=repair_class["post_repair_validation"],
|
| 4245 |
+
failure_type=repair_class["failure_type"],
|
| 4246 |
+
final_user_message=repair_class["message"],
|
| 4247 |
+
validation_error=str(exc)[:4000],
|
| 4248 |
+
)
|
| 4249 |
+
if repair_class["terminal_status"] == "auth_refresh_required":
|
| 4250 |
+
append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]})
|
| 4251 |
+
raise AuthRefreshRequired(repair_class["message"])
|
| 4252 |
current_error = f"{current_error}\n\nPatch repair did not resolve validation: {str(exc)[:4000]}"
|
| 4253 |
collect_space_logs(target_space_id, token, run_dir, events_path)
|
| 4254 |
append_event(events_path, "repair_validation", "failed", "Repair attempted, but validation still failed", {"error": str(exc)[:4000]})
|
|
|
|
| 4342 |
}
|
| 4343 |
|
| 4344 |
if blocker_detected:
|
| 4345 |
+
if isinstance(generation_smoke, dict) and generation_smoke.get("skip_reason") == "contract_declared_no_full_inference":
|
| 4346 |
+
status = "technical_blocker_boot_only"
|
| 4347 |
+
message = "Space boots, but Pi declared full inference unavailable; generation smoke was skipped by contract. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md."
|
| 4348 |
+
else:
|
| 4349 |
+
status = "technical_blocker"
|
| 4350 |
+
message = "Space boots, but full model inference was not implemented. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md."
|
| 4351 |
elif implementation_mode in {"full-inference-gated", "full-inference-attempt"} and smoke_ok:
|
| 4352 |
status = "full_inference_success"
|
| 4353 |
message = "Space boots and a live generation smoke test passed. ZeroGPU duration recommendation was measured from real inference."
|
|
|
|
| 4438 |
api = HfApi(token=token)
|
| 4439 |
whoami = api.whoami(token=token)
|
| 4440 |
append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})
|
| 4441 |
+
write_auth_probe(run_dir, events_path, "before_space_create", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_space_create"), raise_on_unsafe=True)
|
| 4442 |
|
| 4443 |
append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})
|
| 4444 |
info = api.model_info(model_id, token=token, files_metadata=True)
|
|
|
|
| 4477 |
emit_pi_model_resolution(events_path, pi_model_resolution)
|
| 4478 |
if not (workspace / "PI_SUMMARY.md").exists():
|
| 4479 |
(workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8")
|
| 4480 |
+
write_pi_planning_review(workspace, run_dir, events_path, analysis, implementation_mode)
|
| 4481 |
|
| 4482 |
app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")
|
| 4483 |
if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text:
|
|
|
|
| 4522 |
# starts directly on the requested hardware. If it fell back to CPU, the run
|
| 4523 |
# remains valid but will be marked manual_hardware_required when inference
|
| 4524 |
# signals indicate GPU is needed.
|
| 4525 |
+
write_auth_probe(run_dir, events_path, "before_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_upload"), raise_on_unsafe=True)
|
| 4526 |
upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
|
| 4527 |
write_artifact_manifest(run_dir, reason="workspace_uploaded")
|
| 4528 |
|
|
|
|
| 4543 |
)
|
| 4544 |
generation_smoke = None
|
| 4545 |
if implementation_mode in {"full-inference-gated", "full-inference-attempt"}:
|
| 4546 |
+
contract_skip_reason = contract_declares_no_full_inference(workspace)
|
| 4547 |
+
if contract_skip_reason.get("declared"):
|
| 4548 |
+
generation_smoke = write_contract_skipped_generation_smoke(run_dir, events_path, expected_output_type, target_space_id, contract_skip_reason)
|
| 4549 |
+
else:
|
| 4550 |
+
try:
|
| 4551 |
+
generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace)
|
| 4552 |
+
except Exception as smoke_error:
|
| 4553 |
+
write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation smoke test failed", data={"error": str(smoke_error)[:2000]})
|
| 4554 |
+
generation_smoke = {
|
| 4555 |
+
"status": "failed",
|
| 4556 |
+
"target_space": target_space_id,
|
| 4557 |
+
"expected_output_type": expected_output_type,
|
| 4558 |
+
"error": str(smoke_error)[:4000],
|
| 4559 |
+
"next_action": "Use Prefill Space Test after the run finishes to retry with schema-adjusted arguments.",
|
| 4560 |
+
**measured_zero_gpu_recommendation(None),
|
| 4561 |
+
}
|
| 4562 |
+
generation_smoke.update(classify_generation_smoke_error(smoke_error))
|
| 4563 |
+
write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
|
| 4564 |
+
append_event(events_path, "generation_smoke", "failed", "Live generation smoke test failed; ZeroGPU duration was not measured", generation_smoke)
|
| 4565 |
else:
|
| 4566 |
generation_smoke = measured_zero_gpu_recommendation(None) | {"status": "skipped", "expected_output_type": expected_output_type}
|
| 4567 |
write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
|
|
|
|
| 4571 |
# hardware requests failed, classify the run honestly as needing manual
|
| 4572 |
# hardware instead of pretending CPU/default hardware is enough. the existing-Space validation workflow
|
| 4573 |
# can then smoke-test generation after the user sets a GPU manually.
|
| 4574 |
+
manual_hw_required = selected_hardware == "default-cpu-or-existing" and inference_gate.get("status") not in {"technical_blocker", "technical_blocker_boot_only", "health_only"} and (
|
| 4575 |
inference_gate.get("implementation_signals", {}).get("has_spaces_gpu")
|
| 4576 |
or inference_gate.get("implementation_signals", {}).get("has_torch")
|
| 4577 |
or any((a.get("manual_action_required") for a in hardware_attempts if isinstance(a, dict)))
|
|
|
|
| 4688 |
collect_pi_traces(run_dir, events_path)
|
| 4689 |
except Exception:
|
| 4690 |
pass
|
| 4691 |
+
terminal_status = "auth_refresh_required" if isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc) else "failed"
|
| 4692 |
+
details = {"error": str(exc)}
|
| 4693 |
+
if terminal_status == "auth_refresh_required":
|
| 4694 |
+
details["failure_type"] = "auth_refresh_required"
|
| 4695 |
+
repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {}
|
| 4696 |
+
if isinstance(repair_outcome, dict) and repair_outcome:
|
| 4697 |
+
details["repair_outcome"] = repair_outcome
|
| 4698 |
+
details["failure_type"] = repair_outcome.get("failure_type") or details.get("failure_type", "")
|
| 4699 |
+
fail(run_dir, events_path, "Universal model-card builder worker failed", details, status=terminal_status)
|
| 4700 |
|
| 4701 |
|
| 4702 |
if __name__ == "__main__":
|
|
|
|
| 4706 |
|
| 4707 |
|
| 4708 |
VALIDATE_EXISTING_SPACE_WORKER_SCRIPT = r'''
|
| 4709 |
+
import base64
|
| 4710 |
import json
|
| 4711 |
import os
|
| 4712 |
import re
|
|
|
|
| 4924 |
stripped = line.strip()
|
| 4925 |
if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
|
| 4926 |
continue
|
| 4927 |
+
name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-")
|
| 4928 |
if name == wanted:
|
| 4929 |
return True
|
| 4930 |
return False
|
|
|
|
| 5110 |
return "cancelled"
|
| 5111 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 5112 |
return "manual_action_required"
|
| 5113 |
+
if status in {"technical_blocker", "technical_blocker_boot_only"}:
|
| 5114 |
return "technical_blocker"
|
| 5115 |
if full_inference_verified:
|
| 5116 |
return "success"
|
|
|
|
| 5321 |
return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"}
|
| 5322 |
|
| 5323 |
|
| 5324 |
+
|
| 5325 |
+
|
| 5326 |
+
class AuthRefreshRequired(RuntimeError):
|
| 5327 |
+
"""Raised when an OAuth/JWT token is expired or too close to expiry."""
|
| 5328 |
+
|
| 5329 |
+
|
| 5330 |
+
def _decode_jwt_claims_unverified(token: str) -> dict:
|
| 5331 |
+
"""Decode JWT claims without verification, only to inspect non-sensitive expiry metadata.
|
| 5332 |
+
|
| 5333 |
+
This never returns or writes the raw token. Opaque PAT-style tokens are supported by
|
| 5334 |
+
returning an empty dict so they remain usable with an `unknown` expiry state.
|
| 5335 |
+
"""
|
| 5336 |
+
try:
|
| 5337 |
+
parts = (token or "").split(".")
|
| 5338 |
+
if len(parts) < 2:
|
| 5339 |
+
return {}
|
| 5340 |
+
payload = parts[1]
|
| 5341 |
+
payload += "=" * ((4 - len(payload) % 4) % 4)
|
| 5342 |
+
raw = base64.urlsafe_b64decode(payload.encode("utf-8"))
|
| 5343 |
+
data = json.loads(raw.decode("utf-8"))
|
| 5344 |
+
return data if isinstance(data, dict) else {}
|
| 5345 |
+
except Exception:
|
| 5346 |
+
return {}
|
| 5347 |
+
|
| 5348 |
+
|
| 5349 |
+
def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict:
|
| 5350 |
+
issued_at = now()
|
| 5351 |
+
if not token:
|
| 5352 |
+
return {
|
| 5353 |
+
"schema_version": "auth_status.v1",
|
| 5354 |
+
"checked_at": issued_at,
|
| 5355 |
+
"token_present": False,
|
| 5356 |
+
"token_kind": "missing",
|
| 5357 |
+
"expiry_known": False,
|
| 5358 |
+
"status": "missing",
|
| 5359 |
+
"safe_for_phase": False,
|
| 5360 |
+
"minimum_required_seconds": minimum_required_seconds,
|
| 5361 |
+
}
|
| 5362 |
+
claims = _decode_jwt_claims_unverified(token)
|
| 5363 |
+
exp = claims.get("exp") if isinstance(claims, dict) else None
|
| 5364 |
+
iat = claims.get("iat") if isinstance(claims, dict) else None
|
| 5365 |
+
token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown")
|
| 5366 |
+
payload = {
|
| 5367 |
+
"schema_version": "auth_status.v1",
|
| 5368 |
+
"checked_at": issued_at,
|
| 5369 |
+
"token_present": True,
|
| 5370 |
+
"token_kind": token_kind,
|
| 5371 |
+
"expiry_known": exp is not None,
|
| 5372 |
+
"minimum_required_seconds": int(minimum_required_seconds or 0),
|
| 5373 |
+
"token_value": "[REDACTED]",
|
| 5374 |
+
}
|
| 5375 |
+
if iat is not None:
|
| 5376 |
+
try:
|
| 5377 |
+
payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat()
|
| 5378 |
+
except Exception:
|
| 5379 |
+
pass
|
| 5380 |
+
if exp is None:
|
| 5381 |
+
payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"})
|
| 5382 |
+
return payload
|
| 5383 |
+
try:
|
| 5384 |
+
exp_int = int(exp)
|
| 5385 |
+
seconds_left = exp_int - int(time.time())
|
| 5386 |
+
payload.update({
|
| 5387 |
+
"expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(),
|
| 5388 |
+
"seconds_until_expiry": seconds_left,
|
| 5389 |
+
})
|
| 5390 |
+
except Exception:
|
| 5391 |
+
payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"})
|
| 5392 |
+
return payload
|
| 5393 |
+
if seconds_left <= 0:
|
| 5394 |
+
payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"})
|
| 5395 |
+
elif minimum_required_seconds and seconds_left < minimum_required_seconds:
|
| 5396 |
+
payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"})
|
| 5397 |
+
else:
|
| 5398 |
+
payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"})
|
| 5399 |
+
return payload
|
| 5400 |
+
|
| 5401 |
+
|
| 5402 |
+
def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict:
|
| 5403 |
+
payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds)
|
| 5404 |
+
payload["phase"] = phase
|
| 5405 |
+
safe_payload = {k: v for k, v in payload.items() if k != "token_value"}
|
| 5406 |
+
try:
|
| 5407 |
+
probes_dir = run_dir / "auth_probes"
|
| 5408 |
+
probes_dir.mkdir(parents=True, exist_ok=True)
|
| 5409 |
+
write_json(probes_dir / f"{phase}.json", safe_payload)
|
| 5410 |
+
write_json(run_dir / "auth_status.json", safe_payload)
|
| 5411 |
+
except Exception:
|
| 5412 |
+
pass
|
| 5413 |
+
status = str(payload.get("status") or "unknown")
|
| 5414 |
+
if events_path:
|
| 5415 |
+
event_status = "success" if payload.get("safe_for_phase") else "failed"
|
| 5416 |
+
if status == "unknown":
|
| 5417 |
+
event_status = "warning"
|
| 5418 |
+
append_event(
|
| 5419 |
+
events_path,
|
| 5420 |
+
"auth_probe",
|
| 5421 |
+
event_status,
|
| 5422 |
+
f"HF OAuth/token expiry check for {phase}: {status}",
|
| 5423 |
+
safe_payload,
|
| 5424 |
+
)
|
| 5425 |
+
if raise_on_unsafe and not payload.get("safe_for_phase"):
|
| 5426 |
+
raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.")
|
| 5427 |
+
return payload
|
| 5428 |
+
|
| 5429 |
+
|
| 5430 |
+
def is_auth_expired_error(error: Exception | str) -> bool:
|
| 5431 |
+
text = str(error or "").lower()
|
| 5432 |
+
return any(marker in text for marker in [
|
| 5433 |
+
"oauth token has expired",
|
| 5434 |
+
"exp claim timestamp check failed",
|
| 5435 |
+
"token has expired",
|
| 5436 |
+
"jwt expired",
|
| 5437 |
+
])
|
| 5438 |
+
|
| 5439 |
+
|
| 5440 |
+
def minimum_auth_seconds_for_phase(phase: str) -> int:
|
| 5441 |
+
env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_")
|
| 5442 |
+
raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "")
|
| 5443 |
+
if raw:
|
| 5444 |
+
try:
|
| 5445 |
+
return max(0, int(raw))
|
| 5446 |
+
except Exception:
|
| 5447 |
+
pass
|
| 5448 |
+
defaults = {
|
| 5449 |
+
"before_space_create": 900,
|
| 5450 |
+
"before_upload": 900,
|
| 5451 |
+
"before_initial_validation": 1800,
|
| 5452 |
+
"before_repair": 1800,
|
| 5453 |
+
"before_repair_upload": 1800,
|
| 5454 |
+
"before_repair_validation": 1800,
|
| 5455 |
+
"before_linked_space_test": 1800,
|
| 5456 |
+
}
|
| 5457 |
+
return defaults.get(phase, 0)
|
| 5458 |
+
|
| 5459 |
def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
|
| 5460 |
"""Ensure Pi and HF tooling see the same private HF token safely.
|
| 5461 |
|
|
|
|
| 5469 |
os.environ.setdefault("HF_TOKEN", token)
|
| 5470 |
os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
|
| 5471 |
os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
|
| 5472 |
+
expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"))
|
| 5473 |
payload = {
|
| 5474 |
"hf_token_present": bool(token),
|
| 5475 |
"hf_token_length": len(token) if token else 0,
|
|
|
|
| 5478 |
"bucket_source": os.environ.get("BUCKET_SOURCE") or "",
|
| 5479 |
"target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
|
| 5480 |
"token_value": "[REDACTED]" if token else "",
|
| 5481 |
+
"token_kind": expiry.get("token_kind"),
|
| 5482 |
+
"expiry_known": expiry.get("expiry_known"),
|
| 5483 |
+
"expires_at": expiry.get("expires_at"),
|
| 5484 |
+
"seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"),
|
| 5485 |
+
"auth_risk": expiry.get("auth_risk"),
|
| 5486 |
+
"safe_for_long_build": bool(expiry.get("safe_for_phase")),
|
| 5487 |
+
"minimum_required_seconds": expiry.get("minimum_required_seconds"),
|
| 5488 |
}
|
| 5489 |
run_dir.mkdir(parents=True, exist_ok=True)
|
| 5490 |
write_json(run_dir / "token_context.json", payload)
|
|
|
|
| 6792 |
write_json(state_path, {"run_id": run_id, "kind": "linked_space_validation", "status": "running", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "created_by": username, "updated_at": now()})
|
| 6793 |
if not token:
|
| 6794 |
raise RuntimeError("HF_TOKEN is missing")
|
| 6795 |
+
ensure_hf_token_context(run_dir, events_path)
|
| 6796 |
+
write_auth_probe(run_dir, events_path, "before_linked_space_test", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_linked_space_test"), raise_on_unsafe=True)
|
| 6797 |
if not TARGET_RE.match(target_space_id):
|
| 6798 |
raise ValueError("TARGET_SPACE_ID must look like owner/space-name")
|
| 6799 |
if not parent_build_run_id:
|
|
|
|
| 6894 |
append_event(events_path, "done", "full_inference_success", "Existing Space validation completed", {"latency_seconds": smoke.get("latency_seconds")})
|
| 6895 |
except Exception as exc:
|
| 6896 |
collect_space_logs(target_space_id, token or "", run_dir, events_path)
|
| 6897 |
+
auth_blocked = isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc)
|
| 6898 |
details = {"error": str(exc)[:4000]}
|
| 6899 |
+
if auth_blocked:
|
| 6900 |
+
details["status_reason"] = "auth_refresh_required"
|
| 6901 |
+
details["failure_type"] = "auth_refresh_required"
|
| 6902 |
+
terminal_status = "auth_refresh_required" if auth_blocked else "failed"
|
| 6903 |
+
failure_state = {"run_id": run_id, "kind": "linked_space_validation", "status": terminal_status, "message": "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else str(exc), "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "details": details, "updated_at": now()}
|
| 6904 |
write_json(state_path, failure_state)
|
| 6905 |
+
write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed")
|
| 6906 |
if parent_build_run_id:
|
| 6907 |
parent_dir = output_root / "runs" / parent_build_run_id
|
| 6908 |
parent_dir.mkdir(parents=True, exist_ok=True)
|
| 6909 |
+
failed_status = {"status": terminal_status, "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "details": details, "updated_at": now(), "effective_status": "unchanged"}
|
| 6910 |
linked_path = parent_dir / "linked_validations.json"
|
| 6911 |
linked = read_json(linked_path, {"parent_build_run_id": parent_build_run_id, "validations": []}) or {"parent_build_run_id": parent_build_run_id, "validations": []}
|
| 6912 |
validations = linked.get("validations") if isinstance(linked, dict) else []
|
|
|
|
| 6921 |
write_json(parent_dir / "manual_validation_status.json", failed_status)
|
| 6922 |
write_json(linked_path, linked)
|
| 6923 |
(run_dir / "report.md").write_text(f"# Existing Space Validation Failed\n\n```json\n{json.dumps(details, indent=2, ensure_ascii=False)}\n```\n", encoding="utf-8")
|
| 6924 |
+
append_event(events_path, "failure", "failed", "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else "Existing Space validation failed", details)
|
| 6925 |
try:
|
| 6926 |
publish_eval_record(run_dir, phase="failure", events_path=events_path)
|
| 6927 |
append_event(events_path, "anonymous_eval", "success", "Published anonymized failed validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False})
|
|
|
|
| 6930 |
# v190.33: also rewrite failure summary as the final visible file update
|
| 6931 |
# for linked validations, so a failed validation cannot remain listed
|
| 6932 |
# as running after the worker has already terminalized state.json.
|
| 6933 |
+
write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed")
|
| 6934 |
+
append_event(events_path, "summary_write", "success", "Wrote terminal failed validation summary", {"status": terminal_status})
|
| 6935 |
raise SystemExit(1)
|
| 6936 |
|
| 6937 |
|