fffiloni commited on
Commit
30f60d0
·
verified ·
1 Parent(s): 39fbd00

Upload 14 files

Browse files
src/auth.py CHANGED
@@ -46,6 +46,64 @@ class OAuthContext:
46
  return self.expires_at <= datetime.now(timezone.utc)
47
 
48
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  def _parse_scope(scope: Any) -> set[str]:
50
  if not scope:
51
  return set()
@@ -167,6 +225,7 @@ def public_oauth_context(ctx: OAuthContext) -> dict[str, Any]:
167
  "scopes": sorted(ctx.scopes),
168
  "missing_scopes": ctx.missing_scopes,
169
  "expires_at": ctx.expires_at.isoformat() if ctx.expires_at else None,
 
170
  "authenticated": True,
171
  }
172
 
 
46
  return self.expires_at <= datetime.now(timezone.utc)
47
 
48
 
49
+ def oauth_lifetime_summary(ctx: OAuthContext, *, now: datetime | None = None) -> dict[str, Any]:
50
+ """Return a safe, UI-ready OAuth lifetime summary without exposing tokens."""
51
+ current = now or datetime.now(timezone.utc)
52
+ if current.tzinfo is None:
53
+ current = current.replace(tzinfo=timezone.utc)
54
+ if ctx.expires_at is None:
55
+ return {
56
+ "status": "unknown",
57
+ "severity": "neutral",
58
+ "label": "HF Auth: expiry unknown",
59
+ "expires_at": None,
60
+ "seconds_until_expiry": None,
61
+ "recommendation": "OAuth expiry is not exposed by this runtime. Refresh sign-in before long builds if unsure.",
62
+ }
63
+ seconds = int((ctx.expires_at - current).total_seconds())
64
+ if seconds <= 0:
65
+ status = "expired"
66
+ severity = "error"
67
+ label = "HF Auth: expired"
68
+ recommendation = "Sign in again before launching or validating Jobs."
69
+ elif seconds < 30 * 60:
70
+ status = "critical"
71
+ severity = "error"
72
+ label = f"HF Auth: {format_duration_compact(seconds)} left"
73
+ recommendation = "Refresh sign-in before launching a build, repair, or linked Space Test."
74
+ elif seconds < 90 * 60:
75
+ status = "warning"
76
+ severity = "warn"
77
+ label = f"HF Auth: {format_duration_compact(seconds)} left"
78
+ recommendation = "Refresh sign-in before long or high-risk builds."
79
+ else:
80
+ status = "ok"
81
+ severity = "success"
82
+ label = f"HF Auth: {format_duration_compact(seconds)} left"
83
+ recommendation = "OAuth session looks safe for normal builds."
84
+ return {
85
+ "status": status,
86
+ "severity": severity,
87
+ "label": label,
88
+ "expires_at": ctx.expires_at.isoformat(),
89
+ "seconds_until_expiry": seconds,
90
+ "recommendation": recommendation,
91
+ }
92
+
93
+
94
+ def format_duration_compact(seconds: int | float | None) -> str:
95
+ if seconds is None:
96
+ return "unknown"
97
+ total = max(0, int(seconds))
98
+ hours, rem = divmod(total, 3600)
99
+ minutes = rem // 60
100
+ if hours:
101
+ return f"{hours}h {minutes}m"
102
+ if minutes:
103
+ return f"{minutes}m"
104
+ return f"{total}s"
105
+
106
+
107
  def _parse_scope(scope: Any) -> set[str]:
108
  if not scope:
109
  return set()
 
225
  "scopes": sorted(ctx.scopes),
226
  "missing_scopes": ctx.missing_scopes,
227
  "expires_at": ctx.expires_at.isoformat() if ctx.expires_at else None,
228
+ "auth_lifetime": oauth_lifetime_summary(ctx),
229
  "authenticated": True,
230
  }
231
 
src/bucket.py CHANGED
@@ -1073,6 +1073,8 @@ def read_run_bundle(run_id: str, *, bucket_source: str, token: str | None = None
1073
  "hardware_strategy": _safe_read_json(f"{paths.root}/hardware_strategy.json", token=token),
1074
  "hardware_attempts": _safe_read_json(f"{paths.root}/hardware_attempts.json", token=token),
1075
  "technical_blockers": _safe_read_json(f"{paths.root}/generated/TECHNICAL_BLOCKERS.json", token=token) or _safe_read_json(f"{paths.root}/TECHNICAL_BLOCKERS.json", token=token),
 
 
1076
  "model_analysis": _safe_read_json(f"{paths.root}/model_analysis.json", token=token),
1077
  "pi_model_resolution": _safe_read_json(f"{paths.root}/pi_model_resolution.json", token=token) or (read_json(paths.state, token=token) or {}).get("pi_model_resolution") or {},
1078
  "space_runtime": _safe_read_json(f"{paths.root}/space_runtime.json", token=token),
 
1073
  "hardware_strategy": _safe_read_json(f"{paths.root}/hardware_strategy.json", token=token),
1074
  "hardware_attempts": _safe_read_json(f"{paths.root}/hardware_attempts.json", token=token),
1075
  "technical_blockers": _safe_read_json(f"{paths.root}/generated/TECHNICAL_BLOCKERS.json", token=token) or _safe_read_json(f"{paths.root}/TECHNICAL_BLOCKERS.json", token=token),
1076
+ "inference_contract": _safe_read_json(f"{paths.root}/generated/INFERENCE_CONTRACT.json", token=token) or _safe_read_json(f"{paths.root}/INFERENCE_CONTRACT.json", token=token),
1077
+ "repair_outcome": _safe_read_json(f"{paths.root}/repair_outcome.json", token=token) or _safe_read_json(f"{paths.root}/repair/REPAIR_OUTCOME.json", token=token),
1078
  "model_analysis": _safe_read_json(f"{paths.root}/model_analysis.json", token=token),
1079
  "pi_model_resolution": _safe_read_json(f"{paths.root}/pi_model_resolution.json", token=token) or (read_json(paths.state, token=token) or {}).get("pi_model_resolution") or {},
1080
  "space_runtime": _safe_read_json(f"{paths.root}/space_runtime.json", token=token),
src/eval_archive.py CHANGED
@@ -9,7 +9,7 @@ from .bucket import RunPaths, read_json, read_text, write_json
9
  from .eval_config import effective_eval_config
10
  from .security import redact
11
 
12
- _TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "manual_hardware_required"}
13
 
14
 
15
  def _now() -> str:
 
9
  from .eval_config import effective_eval_config
10
  from .security import redact
11
 
12
+ _TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"}
13
 
14
 
15
  def _now() -> str:
src/model_scan.py CHANGED
@@ -188,6 +188,192 @@ def _extract_model_card_signals(readme: str, model_index: dict[str, Any]) -> dic
188
  }
189
 
190
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  def _classify(score: int, *, blocking: bool = False) -> str:
192
  if blocking:
193
  return "unsupported"
@@ -356,6 +542,24 @@ def analyze_model_metadata(
356
 
357
  score = max(0, min(100, score))
358
  verdict = _classify(score, blocking=unsupported)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
359
  if not recommendations:
360
  if verdict == "safe":
361
  recommendations.append("Good candidate for Strict inference.")
@@ -381,6 +585,8 @@ def analyze_model_metadata(
381
  "risk_signals": risk[:10],
382
  "recommendations": recommendations[:6],
383
  "expected_output_type": expected_output_type or "",
 
 
384
  "metadata": {
385
  "pipeline_tag": pipeline_tag or "",
386
  "library_name": library_name or "",
@@ -397,6 +603,10 @@ def analyze_model_metadata(
397
  "runtime_hints": runtime_hints,
398
  "expected_output_type": expected_output_type or "",
399
  "diffusers_standard": bool(is_diffusers and has_model_index and has_safetensors),
 
 
 
 
400
  },
401
  }
402
 
 
188
  }
189
 
190
 
191
+
192
+
193
+
194
+
195
+ NATIVE_KERNEL_PATTERNS: tuple[tuple[str, tuple[str, ...], str], ...] = (
196
+ ("flash_attn", ("flash-attn", "flash_attn", "flash attention", "flashattention", "flash-attention", "enable_flashattn", "flash_attention_2"), "Flash Attention dependency or runtime flag"),
197
+ ("xformers", ("xformers", "memory_efficient_attention"), "xFormers attention dependency"),
198
+ ("triton", ("triton", "triton kernel", "@triton", "triton.jit"), "Triton/fused kernel dependency"),
199
+ ("custom_cuda", ("cuda extension", "custom cuda", "cpp_extension", "setup.py build_ext", "fused kernel", "fused ops", "custom kernel"), "Custom native/CUDA extension"),
200
+ ("attention_interface", ("attentioninterface", "attention interface", "attn_implementation"), "Transformers attention backend hook"),
201
+ ("hf_kernels", ("hf kernels", "kernel hub", "hugging face kernels", "kernels-community", "from kernels import", "pip install kernels"), "HF Kernels / Kernel Hub mention"),
202
+ )
203
+
204
+
205
+ def build_kernel_strategy(*, readme: str | None = None, files: list[str] | None = None) -> dict[str, Any]:
206
+ """Return a visibility-only mitigation plan for native kernel dependencies.
207
+
208
+ This is metadata/readme based. It must not inject packages or mark a model
209
+ terminally blocked by itself; it gives Pi and the UI a safer strategy than
210
+ blindly adding source-built CUDA packages to requirements.txt.
211
+ """
212
+ text = (readme or "").lower()
213
+ file_list = [str(f).lower() for f in (files or [])]
214
+ detected: list[str] = []
215
+ signals: list[str] = []
216
+ for key, patterns, label in NATIVE_KERNEL_PATTERNS:
217
+ if any(pattern in text for pattern in patterns):
218
+ detected.append(key)
219
+ signals.append(label)
220
+ if any(f.endswith((".cu", ".cuh")) or "/csrc/" in f or f.startswith("csrc/") for f in file_list):
221
+ if "custom_cuda" not in detected:
222
+ detected.append("custom_cuda")
223
+ signals.append("Native CUDA/C++ source files in repository")
224
+ if any(f.endswith((".cpp", ".cc")) and ("cuda" in f or "/csrc/" in f or f.startswith("csrc/")) for f in file_list):
225
+ if "custom_cuda" not in detected:
226
+ detected.append("custom_cuda")
227
+ signals.append("Native C++ extension source files in repository")
228
+
229
+ native_risk = any(k in detected for k in {"flash_attn", "xformers", "triton", "custom_cuda"})
230
+ candidate_backends: list[str] = []
231
+ if native_risk:
232
+ candidate_backends.extend(["torch_sdpa", "hf_kernels", "xformers_wheel"])
233
+ if "attention_interface" in detected:
234
+ candidate_backends.append("transformers_attention_interface")
235
+ if "hf_kernels" in detected and "hf_kernels" not in candidate_backends:
236
+ candidate_backends.append("hf_kernels")
237
+
238
+ rejected_actions = []
239
+ if native_risk:
240
+ rejected_actions.append({
241
+ "action": "blind_pip_install_native_cuda_package",
242
+ "reason": "Source-built CUDA/native packages are fragile in Spaces and can mismatch the managed PyTorch/CUDA runtime.",
243
+ })
244
+ if "flash_attn" in detected:
245
+ rejected_actions.append({
246
+ "action": "pip_install_flash_attn_without_fallback",
247
+ "reason": "Prefer PyTorch SDPA, Transformers AttentionInterface, HF Kernels/Kernel Hub, or a compatible wheel before forcing flash-attn source builds.",
248
+ })
249
+
250
+ selected = "none"
251
+ if native_risk:
252
+ selected = "prefer_runtime_backends_before_source_builds"
253
+ elif "hf_kernels" in detected:
254
+ selected = "hf_kernels_available_if_model_uses_supported_kernel"
255
+
256
+ return {
257
+ "schema_version": "kernel_strategy.v1",
258
+ "native_kernel_risk": bool(native_risk),
259
+ "detected_dependencies": detected,
260
+ "signals": signals[:10],
261
+ "selected_strategy": selected,
262
+ "candidate_backends": candidate_backends[:8],
263
+ "rejected_actions": rejected_actions,
264
+ "requires_manual_review": bool("custom_cuda" in detected),
265
+ "pi_instruction": (
266
+ "Do not compile native CUDA packages blindly. Prefer PyTorch SDPA, compatible wheels, HF Kernels/Kernel Hub, "
267
+ "Transformers AttentionInterface, or Diffusers attention processors when they match the required operation. "
268
+ "Declare a technical blocker if a strict native extension has no plausible fallback."
269
+ if native_risk else
270
+ "No native kernel dependency was detected by the metadata scan."
271
+ ),
272
+ "visibility_only": True,
273
+ }
274
+
275
+
276
+ def _has_any(text: str, patterns: tuple[str, ...]) -> bool:
277
+ lower = text.lower()
278
+ return any(pattern in lower for pattern in patterns)
279
+
280
+
281
+ def _build_complexity_assessment(
282
+ *,
283
+ pipeline_tag: str | None,
284
+ library_name: str | None,
285
+ tags: list[str],
286
+ files: list[str],
287
+ readme: str,
288
+ expected_output_type: str | None,
289
+ custom_code: bool,
290
+ gated: Any,
291
+ ) -> dict[str, Any]:
292
+ """Estimate build/runtime risk without changing the existing pre-scan verdict.
293
+
294
+ This is deliberately a lightweight metadata/model-card heuristic. It helps
295
+ the UI warn users before long Jobs; it is not a hard launch gate.
296
+ """
297
+ lower_readme = (readme or "").lower()
298
+ lower_files = [str(f).lower() for f in files]
299
+ tag_set = {str(t).lower() for t in tags}
300
+ task = (pipeline_tag or "").lower()
301
+ library = (library_name or "").lower()
302
+
303
+ points = 0
304
+ signals: list[str] = []
305
+ mitigations: list[str] = []
306
+
307
+ def add(points_delta: int, signal: str, mitigation: str | None = None) -> None:
308
+ nonlocal points
309
+ points += points_delta
310
+ if signal not in signals:
311
+ signals.append(signal)
312
+ if mitigation and mitigation not in mitigations:
313
+ mitigations.append(mitigation)
314
+
315
+ if expected_output_type == "video" or "video" in task or {"text-to-video", "image-to-video"} & tag_set:
316
+ add(3, "video or image-to-video output", "Expect a long build/repair cycle and prefer strong fallback hardware.")
317
+ if any(word in lower_readme for word in ("avatar", "audio-driven", "audio driven", "talking head", "lip sync", "lip-sync")):
318
+ add(2, "audio/video avatar workflow", "Refresh sign-in before launch and expect larger validation payloads.")
319
+ if custom_code or "trust_remote_code" in tag_set or "custom_code" in tag_set:
320
+ add(2, "custom code or trust_remote_code required", "Review generated code and blockers before paid hardware attempts.")
321
+ if gated in {True, "auto", "manual"} or str(gated).lower() in {"true", "auto", "manual"}:
322
+ add(1, "gated model access", "Confirm the same HF account has accepted access terms.")
323
+
324
+ if _has_any(lower_readme, ("torchrun", "nproc_per_node", "distributed", "init_process_group", "nccl", "context_parallel", "tensor parallel", "pipeline parallel", "multi-gpu", "multi gpu")):
325
+ add(4, "multi-GPU / distributed runtime hints", "Treat ZeroGPU as unlikely unless Pi can prove a single-GPU refactor.")
326
+ if _has_any(lower_readme, ("flash-attn", "flash_attn", "flash attention", "flashattention", "flash-attention")):
327
+ add(2, "flash-attn or custom attention dependency", "Prefer PyTorch SDPA, xformers wheels, or HF Kernels before source builds.")
328
+ if _has_any(lower_readme, ("xformers", "triton", "cuda extension", "fused kernel", "fused ops", "custom kernel")) or any(f.endswith((".cu", ".cpp")) or "/csrc/" in f for f in lower_files):
329
+ add(2, "native CUDA/kernel dependency risk", "Prefer runtime-compatible wheels or HF Kernels over compiling during Space build.")
330
+ if "ffmpeg" in lower_readme or any("ffmpeg" in f for f in lower_files):
331
+ add(1, "ffmpeg or system media dependency", "Ensure generated app declares media/system requirements clearly.")
332
+ if _has_any(lower_readme, ("conda ", "mamba ", "apt-get", "sudo apt", "pip install -e", "git clone")):
333
+ add(1, "non-standard install instructions", "Pi should vendor required code or simplify requirements for Spaces.")
334
+
335
+ weight_files = [f for f in lower_files if f.endswith(WEIGHT_EXTENSIONS)]
336
+ safetensor_shards = [f for f in lower_files if f.endswith(".safetensors")]
337
+ if len(weight_files) >= 20 or len(safetensor_shards) >= 12:
338
+ add(2, "many weight shards", "Expect longer cold start and validation windows.")
339
+ elif len(weight_files) >= 8 or len(safetensor_shards) >= 6:
340
+ add(1, "multiple weight shards", "Allow extra boot time before validation.")
341
+
342
+ vram_match = re.search(r"\b(?:vram|gpu memory|memory)\D{0,16}([3-9]\d|1\d{2})\s*(?:gb|gib)\b|\b([3-9]\d|1\d{2})\s*(?:gb|gib)\s*(?:vram|gpu)", lower_readme)
343
+ if vram_match:
344
+ add(3, "high VRAM mentioned in model card", "Prefer fixed GPU fallback and refresh sign-in before launch.")
345
+
346
+ # Keep the assessment independent from the existing pass/fail scan score.
347
+ if points >= 9:
348
+ level = "very_high"
349
+ label = "Very high risk"
350
+ recommended_seconds = 180 * 60
351
+ elif points >= 6:
352
+ level = "high"
353
+ label = "High risk"
354
+ recommended_seconds = 120 * 60
355
+ elif points >= 3:
356
+ level = "medium"
357
+ label = "Medium risk"
358
+ recommended_seconds = 60 * 60
359
+ else:
360
+ level = "low"
361
+ label = "Low risk"
362
+ recommended_seconds = 30 * 60
363
+
364
+ return {
365
+ "schema_version": "model_build_risk.v1",
366
+ "level": level,
367
+ "label": label,
368
+ "score": max(0, points),
369
+ "recommended_session_seconds": recommended_seconds,
370
+ "recommended_session_minutes": recommended_seconds // 60,
371
+ "signals": signals[:10],
372
+ "mitigations": mitigations[:6],
373
+ "summary": f"{label}; recommended HF session remaining: {recommended_seconds // 60}m+.",
374
+ "visibility_only": True,
375
+ }
376
+
377
  def _classify(score: int, *, blocking: bool = False) -> str:
378
  if blocking:
379
  return "unsupported"
 
542
 
543
  score = max(0, min(100, score))
544
  verdict = _classify(score, blocking=unsupported)
545
+ build_risk = _build_complexity_assessment(
546
+ pipeline_tag=pipeline_tag,
547
+ library_name=library_name,
548
+ tags=tags,
549
+ files=files,
550
+ readme=readme,
551
+ expected_output_type=expected_output_type,
552
+ custom_code=custom_code,
553
+ gated=gated,
554
+ )
555
+ kernel_strategy = build_kernel_strategy(readme=readme, files=files)
556
+ if kernel_strategy.get("native_kernel_risk"):
557
+ recommendations.append("Native kernel risk detected; prefer PyTorch SDPA, HF Kernels/Kernel Hub, compatible wheels, or model-specific fallbacks before source builds.")
558
+ if build_risk["level"] in {"high", "very_high"}:
559
+ recommendations.append(
560
+ f"Refresh HF sign-in before launch; this scan recommends {build_risk['recommended_session_minutes']}m+ remaining for this model."
561
+ )
562
+
563
  if not recommendations:
564
  if verdict == "safe":
565
  recommendations.append("Good candidate for Strict inference.")
 
585
  "risk_signals": risk[:10],
586
  "recommendations": recommendations[:6],
587
  "expected_output_type": expected_output_type or "",
588
+ "build_risk": build_risk,
589
+ "kernel_strategy": kernel_strategy,
590
  "metadata": {
591
  "pipeline_tag": pipeline_tag or "",
592
  "library_name": library_name or "",
 
603
  "runtime_hints": runtime_hints,
604
  "expected_output_type": expected_output_type or "",
605
  "diffusers_standard": bool(is_diffusers and has_model_index and has_safetensors),
606
+ "build_risk_level": build_risk["level"],
607
+ "recommended_session_minutes": build_risk["recommended_session_minutes"],
608
+ "native_kernel_risk": bool(kernel_strategy.get("native_kernel_risk")),
609
+ "kernel_strategy": kernel_strategy,
610
  },
611
  }
612
 
src/progress.py CHANGED
@@ -36,6 +36,7 @@ STEP_ORDER = [
36
  "repair_upload",
37
  "repair_validation",
38
  "technical_blocker",
 
39
  "manual_hardware_required",
40
  "upload_files",
41
  "space_runtime",
@@ -80,6 +81,7 @@ STEP_LABELS = {
80
  "repair_upload": "Repair upload",
81
  "repair_validation": "Repair validation",
82
  "technical_blocker": "Technical blocker",
 
83
  "manual_hardware_required": "Manual hardware",
84
  "upload_files": "Upload files",
85
  "space_runtime": "Space runtime",
@@ -116,6 +118,7 @@ DONE_STATUSES = {
116
  "full_inference_success",
117
  "manual_hardware_required",
118
  "technical_blocker",
 
119
  }
120
  PARTIAL_STATUSES = {
121
  "full_inference_candidate_health_passed",
@@ -341,7 +344,7 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
341
  "current_step": current_step,
342
  "current_step_label": STEP_LABELS.get(current_step, current_step),
343
  "last_event": last_message or "No events yet",
344
- "visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker"} else ("running" if overall_status == "running" else "neutral"))))),
345
  "elapsed_seconds": max(0, elapsed),
346
  "eta_seconds": None,
347
  "timeline": timeline,
 
36
  "repair_upload",
37
  "repair_validation",
38
  "technical_blocker",
39
+ "technical_blocker_boot_only",
40
  "manual_hardware_required",
41
  "upload_files",
42
  "space_runtime",
 
81
  "repair_upload": "Repair upload",
82
  "repair_validation": "Repair validation",
83
  "technical_blocker": "Technical blocker",
84
+ "technical_blocker_boot_only": "Boot-only blocker",
85
  "manual_hardware_required": "Manual hardware",
86
  "upload_files": "Upload files",
87
  "space_runtime": "Space runtime",
 
118
  "full_inference_success",
119
  "manual_hardware_required",
120
  "technical_blocker",
121
+ "technical_blocker_boot_only",
122
  }
123
  PARTIAL_STATUSES = {
124
  "full_inference_candidate_health_passed",
 
344
  "current_step": current_step,
345
  "current_step_label": STEP_LABELS.get(current_step, current_step),
346
  "last_event": last_message or "No events yet",
347
+ "visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))),
348
  "elapsed_seconds": max(0, elapsed),
349
  "eta_seconds": None,
350
  "timeline": timeline,
src/timeline_model.py CHANGED
@@ -51,7 +51,7 @@ PHASE_STEPS = {
51
  "repair_validation",
52
  },
53
  "archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
54
- "done": {"done", "failure", "technical_blocker", "manual_hardware_required"},
55
  }
56
 
57
  STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
@@ -62,7 +62,8 @@ FAILED_STATUSES = {"failed", "failure", "error"}
62
  RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
63
  CANCELLED_STATUSES = {"cancelled", "canceled"}
64
  MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
65
- BLOCKED_STATUSES = {"technical_blocker", "blocked"}
 
66
 
67
 
68
  def _lower(value: Any) -> str:
@@ -316,7 +317,7 @@ def _is_record_not_ready(status: dict[str, Any]) -> bool:
316
 
317
 
318
  def _has_terminal_event(bundle: dict[str, Any]) -> bool:
319
- return any(_lower(event.get("step")) in {"done", "failure", "technical_blocker", "manual_hardware_required"} for event in _events(bundle))
320
 
321
 
322
  def _latest_recovery_event(bundle: dict[str, Any]) -> dict[str, Any] | None:
@@ -351,9 +352,12 @@ def _verdict(bundle: dict[str, Any]) -> str:
351
  return _lower(eval_record.get("verdict"))
352
  gate = _gate(bundle)
353
  state = bundle.get("state") or {}
354
- for value in (gate.get("status"), state.get("gate_status"), state.get("status"), (bundle.get("summary") or {}).get("status")):
 
355
  normalized = _lower(value)
356
  if normalized:
 
 
357
  if normalized in SUCCESS_STATUSES:
358
  return "full_inference_success" if normalized == "full_inference_success" else "success"
359
  if normalized in PARTIAL_STATUSES:
@@ -361,7 +365,7 @@ def _verdict(bundle: dict[str, Any]) -> str:
361
  if normalized in MANUAL_STATUSES:
362
  return "manual_action_required"
363
  if normalized in BLOCKED_STATUSES:
364
- return "technical_blocker"
365
  if normalized in FAILED_STATUSES:
366
  return "failed"
367
  if normalized in CANCELLED_STATUSES:
@@ -375,8 +379,10 @@ def _final_visual_status(verdict: str) -> str:
375
  return "success"
376
  if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
377
  return "warn"
378
- if verdict in {"failed", "failure", "technical_blocker"}:
379
  return "error"
 
 
380
  if verdict in {"manual_action_required", "cancelled"}:
381
  return "stopped"
382
  return "running"
@@ -387,8 +393,12 @@ def _status_label(verdict: str) -> tuple[str, str]:
387
  return "Full inference success", "Space boots, generation passed, and latency was measured."
388
  if verdict == "partial_validation":
389
  return "Completed with partial validation", "The run finished, but full generation was not verified."
 
 
390
  if verdict == "technical_blocker":
391
  return "Technical blocker", "The run found a technical blocker."
 
 
392
  if verdict == "manual_action_required":
393
  return "Manual action required", "The run needs user action before validation can continue."
394
  if verdict == "failed":
@@ -598,7 +608,7 @@ def _phase_status(bundle: dict[str, Any], phase: str, verdict: str) -> str:
598
  return "complete"
599
  if verdict in {"partial_validation", "partial", "completed_with_warnings", "health_only"}:
600
  return "warning"
601
- if verdict in {"failed", "technical_blocker"}:
602
  return "failed"
603
  if verdict in {"manual_action_required", "cancelled"}:
604
  return "stopped"
 
51
  "repair_validation",
52
  },
53
  "archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
54
+ "done": {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"},
55
  }
56
 
57
  STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
 
62
  RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
63
  CANCELLED_STATUSES = {"cancelled", "canceled"}
64
  MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
65
+ BLOCKED_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
66
+ AUTH_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
67
 
68
 
69
  def _lower(value: Any) -> str:
 
317
 
318
 
319
  def _has_terminal_event(bundle: dict[str, Any]) -> bool:
320
+ return any(_lower(event.get("step")) in {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"} for event in _events(bundle))
321
 
322
 
323
  def _latest_recovery_event(bundle: dict[str, Any]) -> dict[str, Any] | None:
 
352
  return _lower(eval_record.get("verdict"))
353
  gate = _gate(bundle)
354
  state = bundle.get("state") or {}
355
+ repair = bundle.get("repair_outcome") or {}
356
+ for value in (gate.get("status"), state.get("gate_status"), state.get("status"), (bundle.get("summary") or {}).get("status"), repair.get("post_repair_validation"), repair.get("failure_type")):
357
  normalized = _lower(value)
358
  if normalized:
359
+ if normalized in AUTH_STATUSES:
360
+ return "auth_refresh_required"
361
  if normalized in SUCCESS_STATUSES:
362
  return "full_inference_success" if normalized == "full_inference_success" else "success"
363
  if normalized in PARTIAL_STATUSES:
 
365
  if normalized in MANUAL_STATUSES:
366
  return "manual_action_required"
367
  if normalized in BLOCKED_STATUSES:
368
+ return "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
369
  if normalized in FAILED_STATUSES:
370
  return "failed"
371
  if normalized in CANCELLED_STATUSES:
 
379
  return "success"
380
  if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
381
  return "warn"
382
+ if verdict in {"failed", "failure", "technical_blocker", "technical_blocker_boot_only"}:
383
  return "error"
384
+ if verdict == "auth_refresh_required":
385
+ return "warn"
386
  if verdict in {"manual_action_required", "cancelled"}:
387
  return "stopped"
388
  return "running"
 
393
  return "Full inference success", "Space boots, generation passed, and latency was measured."
394
  if verdict == "partial_validation":
395
  return "Completed with partial validation", "The run finished, but full generation was not verified."
396
+ if verdict == "technical_blocker_boot_only":
397
+ return "Technical blocker", "Health may pass, but no generation endpoint exists for full inference."
398
  if verdict == "technical_blocker":
399
  return "Technical blocker", "The run found a technical blocker."
400
+ if verdict == "auth_refresh_required":
401
+ return "Auth refresh required", "HF OAuth expired or is too close to expiry; sign in again before retrying validation."
402
  if verdict == "manual_action_required":
403
  return "Manual action required", "The run needs user action before validation can continue."
404
  if verdict == "failed":
 
608
  return "complete"
609
  if verdict in {"partial_validation", "partial", "completed_with_warnings", "health_only"}:
610
  return "warning"
611
+ if verdict in {"failed", "technical_blocker", "technical_blocker_boot_only"}:
612
  return "failed"
613
  if verdict in {"manual_action_required", "cancelled"}:
614
  return "stopped"
src/version.py CHANGED
@@ -1,7 +1,7 @@
1
  from __future__ import annotations
2
 
3
- ASF_APP_VERSION = "v190.32"
4
- ASF_RELEASE_NAME = "Agentic Space Factory v190.32"
5
 
6
 
7
  def resolve_app_version(value: str | None = None) -> str:
 
1
  from __future__ import annotations
2
 
3
+ ASF_APP_VERSION = "v190.33"
4
+ ASF_RELEASE_NAME = "Agentic Space Factory v190.33"
5
 
6
 
7
  def resolve_app_version(value: str | None = None) -> str:
src/view_models.py CHANGED
@@ -13,7 +13,7 @@ PRODUCT_STEPS = [{"id": step, "label": STEP_LABELS[step]} for step in STEP_ORDER
13
  def _runs_prefix() -> str:
14
  return settings.bucket_runs_prefix.strip().strip("/") or "runs"
15
 
16
- TERMINAL_GLOBAL_STATUSES = {"succeeded", "partial", "failed", "cancelled", "blocked", "waiting_manual_action"}
17
  SUCCESS_RAW_STATUSES = {
18
  "success",
19
  "done",
@@ -37,8 +37,9 @@ MANUAL_RAW_STATUSES = {
37
  }
38
  FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed"}
39
  CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
40
- BLOCKED_RAW_STATUSES = {"technical_blocker", "blocked", "health_only"}
41
  RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
 
42
 
43
  STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
44
 
@@ -133,6 +134,8 @@ def _raw_statuses(bundle: dict[str, Any]) -> set[str]:
133
  _lower(smoke.get("status")),
134
  _lower(hardware.get("status")),
135
  _lower(blockers.get("status")),
 
 
136
  }
137
  # Event-level statuses describe individual steps and must not by themselves
138
  # turn a whole run into success/failure. The product phase is derived from
@@ -174,12 +177,18 @@ def normalize_run_status(
174
  manual = requires_manual_action(bundle)
175
  blocker = has_technical_blocker(bundle)
176
 
177
- if statuses.intersection(CANCELLED_RAW_STATUSES):
 
 
 
178
  global_status = "cancelled"
179
  verdict = "cancelled"
180
  elif manual:
181
  global_status = "waiting_manual_action"
182
  verdict = "manual_action_required"
 
 
 
183
  elif statuses.intersection(SUCCESS_RAW_STATUSES):
184
  global_status = "succeeded"
185
  verdict = "passed"
@@ -189,9 +198,6 @@ def normalize_run_status(
189
  elif statuses.intersection(FAILED_RAW_STATUSES):
190
  global_status = "failed"
191
  verdict = "failed"
192
- elif blocker:
193
- global_status = "blocked"
194
- verdict = "technical_blocker"
195
  elif statuses.intersection(RUNNING_RAW_STATUSES) or bundle.get("events") or launch:
196
  global_status = "running"
197
  verdict = "pending"
@@ -219,7 +225,7 @@ def normalize_run_status(
219
  "has_target_space": bool(summary.get("target_space") or state.get("target_space") or launch.get("target_space")),
220
  "has_job_url": bool(summary.get("job_url") or state.get("job_url") or launch.get("job_url") or summary.get("job_id") or state.get("job_id") or launch.get("job_id")),
221
  "has_live_api_result": bool((bundle.get("generation_smoke") or {}).get("ok") or (bundle.get("generation_smoke") or {}).get("status") == "success"),
222
- "is_terminal": global_status in {"succeeded", "partial", "failed", "blocked", "waiting_manual_action", "cancelled"},
223
  "is_pollable": global_status in {"queued", "running", "validating", "unknown"},
224
  "is_stale": is_stale,
225
  "latest_activity_at": latest.isoformat() if latest else "",
@@ -422,6 +428,40 @@ def _known_gradio_endpoint_info(bundle: dict[str, Any]) -> dict[str, Any]:
422
  return {"endpoint": endpoint or "/generate", "endpoint_count": endpoint_count, "endpoint_known": endpoint_known, "endpoints": endpoints[:20]}
423
 
424
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
425
  def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
426
  """Return the canonical linked Space Test policy for a Build Run.
427
 
@@ -467,6 +507,15 @@ def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]
467
 
468
  if not target:
469
  return policy("unavailable", False, "no_target_space", "Space Test unavailable", "No generated Space is available to validate.")
 
 
 
 
 
 
 
 
 
470
  if global_status in {"running", "unknown"}:
471
  return policy("blocked", False, "build_not_terminal", "Available after final Space state", "Space Test becomes available after the Build Run reaches a final Space state.")
472
  if global_status == "cancelled" or raw_status in {"stopped", "cancelled", "canceled"}:
@@ -495,10 +544,19 @@ def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any])
495
  smoke = bundle.get("generation_smoke") or {}
496
  summary = bundle.get("summary") or {}
497
  policy = build_space_test_policy(bundle, status_model)
 
 
 
 
 
 
 
 
 
498
  return {
499
  "target_space": policy.get("target_space") or summary.get("target_space") or "",
500
  "target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "",
501
- "endpoint": policy.get("endpoint") or smoke.get("api_name") or smoke.get("endpoint") or "/generate",
502
  "status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending",
503
  "latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"),
504
  "expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
 
13
  def _runs_prefix() -> str:
14
  return settings.bucket_runs_prefix.strip().strip("/") or "runs"
15
 
16
+ TERMINAL_GLOBAL_STATUSES = {"succeeded", "partial", "failed", "cancelled", "blocked", "waiting_manual_action", "auth_refresh_required"}
17
  SUCCESS_RAW_STATUSES = {
18
  "success",
19
  "done",
 
37
  }
38
  FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed"}
39
  CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
40
+ BLOCKED_RAW_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked", "health_only"}
41
  RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
42
+ AUTH_RAW_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
43
 
44
  STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
45
 
 
134
  _lower(smoke.get("status")),
135
  _lower(hardware.get("status")),
136
  _lower(blockers.get("status")),
137
+ _lower((bundle.get("repair_outcome") or {}).get("post_repair_validation")),
138
+ _lower((bundle.get("repair_outcome") or {}).get("failure_type")),
139
  }
140
  # Event-level statuses describe individual steps and must not by themselves
141
  # turn a whole run into success/failure. The product phase is derived from
 
177
  manual = requires_manual_action(bundle)
178
  blocker = has_technical_blocker(bundle)
179
 
180
+ if statuses.intersection(AUTH_RAW_STATUSES):
181
+ global_status = "auth_refresh_required"
182
+ verdict = "auth_refresh_required"
183
+ elif statuses.intersection(CANCELLED_RAW_STATUSES):
184
  global_status = "cancelled"
185
  verdict = "cancelled"
186
  elif manual:
187
  global_status = "waiting_manual_action"
188
  verdict = "manual_action_required"
189
+ elif blocker:
190
+ global_status = "blocked"
191
+ verdict = "technical_blocker_boot_only" if "technical_blocker_boot_only" in statuses else "technical_blocker"
192
  elif statuses.intersection(SUCCESS_RAW_STATUSES):
193
  global_status = "succeeded"
194
  verdict = "passed"
 
198
  elif statuses.intersection(FAILED_RAW_STATUSES):
199
  global_status = "failed"
200
  verdict = "failed"
 
 
 
201
  elif statuses.intersection(RUNNING_RAW_STATUSES) or bundle.get("events") or launch:
202
  global_status = "running"
203
  verdict = "pending"
 
225
  "has_target_space": bool(summary.get("target_space") or state.get("target_space") or launch.get("target_space")),
226
  "has_job_url": bool(summary.get("job_url") or state.get("job_url") or launch.get("job_url") or summary.get("job_id") or state.get("job_id") or launch.get("job_id")),
227
  "has_live_api_result": bool((bundle.get("generation_smoke") or {}).get("ok") or (bundle.get("generation_smoke") or {}).get("status") == "success"),
228
+ "is_terminal": global_status in {"succeeded", "partial", "failed", "blocked", "waiting_manual_action", "cancelled", "auth_refresh_required"},
229
  "is_pollable": global_status in {"queued", "running", "validating", "unknown"},
230
  "is_stale": is_stale,
231
  "latest_activity_at": latest.isoformat() if latest else "",
 
428
  return {"endpoint": endpoint or "/generate", "endpoint_count": endpoint_count, "endpoint_known": endpoint_known, "endpoints": endpoints[:20]}
429
 
430
 
431
+ def _contract_declares_no_full_inference(bundle: dict[str, Any]) -> bool:
432
+ contract = bundle.get("inference_contract") or bundle.get("INFERENCE_CONTRACT") or {}
433
+ if not isinstance(contract, dict):
434
+ contract = {}
435
+ smoke = bundle.get("generation_smoke") or {}
436
+ blockers = bundle.get("technical_blockers") or {}
437
+ statuses = _raw_statuses(bundle)
438
+ return bool(
439
+ "technical_blocker_boot_only" in statuses
440
+ or contract.get("full_inference_implemented") is False
441
+ or (contract.get("primary_api_name") in {None, "", False} and _lower(contract.get("validation_level")) in {"boot-only", "boot_only", "health-only", "health_only"})
442
+ or (_lower(smoke.get("status")) == "skipped" and _lower(smoke.get("skip_reason") or smoke.get("reason")) in {"contract_declared_no_full_inference", "full_inference_not_implemented"})
443
+ or (_lower(blockers.get("status")) in {"technical_blocker_boot_only", "technical_blocker"} and contract.get("full_inference_implemented") is False)
444
+ )
445
+
446
+
447
+ def _repair_auth_context(bundle: dict[str, Any], status_model: dict[str, Any]) -> bool:
448
+ repair = bundle.get("repair_outcome") or {}
449
+ summary = bundle.get("summary") or {}
450
+ state = bundle.get("state") or {}
451
+ values = {
452
+ _lower(status_model.get("global_status")),
453
+ _lower(status_model.get("verdict")),
454
+ _lower(summary.get("status")),
455
+ _lower(state.get("status")),
456
+ _lower(summary.get("failure_type")),
457
+ _lower(state.get("failure_type")),
458
+ _lower(repair.get("post_repair_validation")),
459
+ _lower(repair.get("failure_type")),
460
+ _lower(repair.get("status")),
461
+ }
462
+ return bool(values.intersection(AUTH_RAW_STATUSES))
463
+
464
+
465
  def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
466
  """Return the canonical linked Space Test policy for a Build Run.
467
 
 
507
 
508
  if not target:
509
  return policy("unavailable", False, "no_target_space", "Space Test unavailable", "No generated Space is available to validate.")
510
+ if _repair_auth_context(bundle, status_model):
511
+ return policy("recover", True, "auth_refresh_required", "Retry after sign-in refresh", "Repair patch may have been uploaded, but validation could not continue because HF OAuth expired. Sign in again, then retry linked validation.", on_success="recovered_by_manual_validation", allow_recovery=True)
512
+ if _contract_declares_no_full_inference(bundle):
513
+ blocked = policy("blocked", False, "no_generation_endpoint_by_contract", "Blocked — no generation endpoint", "Health may pass, but Pi declared full inference unavailable and no generation endpoint exists. Review TECHNICAL_BLOCKERS.json / PI_SUMMARY.md or provide a dedicated implementation and hardware plan.")
514
+ blocked["endpoint"] = ""
515
+ blocked["requires_endpoint_discovery"] = False
516
+ blocked["no_generation_endpoint"] = True
517
+ blocked["can_retry_schema"] = False
518
+ return blocked
519
  if global_status in {"running", "unknown"}:
520
  return policy("blocked", False, "build_not_terminal", "Available after final Space state", "Space Test becomes available after the Build Run reaches a final Space state.")
521
  if global_status == "cancelled" or raw_status in {"stopped", "cancelled", "canceled"}:
 
544
  smoke = bundle.get("generation_smoke") or {}
545
  summary = bundle.get("summary") or {}
546
  policy = build_space_test_policy(bundle, status_model)
547
+ # v191.12 regression guard: a contract-declared boot-only blocker must not
548
+ # resurrect the historical /generate fallback in the display model. Keep the
549
+ # endpoint intentionally blank when the parent policy says no generation
550
+ # endpoint exists by contract. Other states still use the legacy fallback so
551
+ # endpoint discovery/recovery behavior is unchanged.
552
+ if policy.get("no_generation_endpoint") is True or policy.get("reason") == "no_generation_endpoint_by_contract":
553
+ endpoint = ""
554
+ else:
555
+ endpoint = policy.get("endpoint") or smoke.get("api_name") or smoke.get("endpoint") or "/generate"
556
  return {
557
  "target_space": policy.get("target_space") or summary.get("target_space") or "",
558
  "target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "",
559
+ "endpoint": endpoint,
560
  "status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending",
561
  "latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"),
562
  "expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
src/worker_payload.py CHANGED
@@ -10,6 +10,7 @@ def _encode(script: str) -> str:
10
 
11
  UNIVERSAL_MODEL_CARD_WORKER_SCRIPT = r'''
12
 
 
13
  import hashlib
14
  import hmac
15
  import hashlib
@@ -48,6 +49,8 @@ INTERNAL_WORKSPACE_ARTIFACT_NAMES = {
48
  "REPAIR_SUMMARY.md",
49
  "PI_SUMMARY.md",
50
  "TECHNICAL_BLOCKERS.json",
 
 
51
  }
52
 
53
 
@@ -278,7 +281,8 @@ def write_final_summary(run_dir: Path, final_state: dict, inference_gate: dict |
278
  "bucket_source": final_state.get("bucket_source") or os.environ.get("BUCKET_SOURCE", ""),
279
  "health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")),
280
  "generation_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success"),
281
- "failure_type": smoke.get("failure_type") or gate.get("failure_type") or "",
 
282
  "updated_at": now(),
283
  }
284
  write_json(run_dir / "summary.json", payload)
@@ -308,7 +312,7 @@ def requirements_has_package(lines: list[str], package: str) -> bool:
308
  stripped = line.strip()
309
  if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
310
  continue
311
- name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
312
  if name == wanted:
313
  return True
314
  return False
@@ -353,9 +357,13 @@ def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, r
353
  _artifact_entry(run_dir, "live_status.json"),
354
  _artifact_entry(run_dir, "report.md"),
355
  _artifact_entry(run_dir, "model_analysis.json"),
 
 
 
356
  _artifact_entry(run_dir, "hardware_strategy.json"),
357
  _artifact_entry(run_dir, "hardware_attempts.json"),
358
  _artifact_entry(run_dir, "inference_gate.json"),
 
359
  _artifact_entry(run_dir, "space_runtime.json"),
360
  _artifact_entry(run_dir, "tests/generation_smoke.json"),
361
  _artifact_entry(run_dir, "tests/api_schema.json"),
@@ -367,6 +375,7 @@ def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, r
367
  _artifact_entry(run_dir, "repair/REPAIR_BRIEF.md"),
368
  _artifact_entry(run_dir, "repair/REPAIR_PLAN.md"),
369
  _artifact_entry(run_dir, "repair/REPAIR_SUMMARY.md"),
 
370
  _artifact_entry(run_dir, "repair/BLOCKAGE.json"),
371
  _artifact_entry(run_dir, "logs/pi_live_output.txt"),
372
  _artifact_entry(run_dir, "logs/pi_output.txt"),
@@ -429,6 +438,198 @@ def redact_text(text: str | None) -> str:
429
 
430
 
431
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
432
  def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
433
  """Ensure Pi and HF tooling see the same private HF token safely.
434
 
@@ -442,6 +643,7 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
442
  os.environ.setdefault("HF_TOKEN", token)
443
  os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
444
  os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
 
445
  payload = {
446
  "hf_token_present": bool(token),
447
  "hf_token_length": len(token) if token else 0,
@@ -450,6 +652,13 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
450
  "bucket_source": os.environ.get("BUCKET_SOURCE") or "",
451
  "target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
452
  "token_value": "[REDACTED]" if token else "",
 
 
 
 
 
 
 
453
  }
454
  run_dir.mkdir(parents=True, exist_ok=True)
455
  write_json(run_dir / "token_context.json", payload)
@@ -734,7 +943,7 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
734
  return "cancelled"
735
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
736
  return "manual_action_required"
737
- if status == "technical_blocker":
738
  return "technical_blocker"
739
  if full_inference_verified:
740
  return "success"
@@ -951,6 +1160,10 @@ def fail(run_dir: Path, events_path: Path, message: str, details: dict | None =
951
  existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
952
  if not isinstance(existing_state, dict):
953
  existing_state = {}
 
 
 
 
954
  failure_state = {
955
  **existing_state,
956
  "run_id": os.environ.get("RUN_ID"),
@@ -959,6 +1172,7 @@ def fail(run_dir: Path, events_path: Path, message: str, details: dict | None =
959
  "message": message,
960
  "updated_at": now(),
961
  "details": safe,
 
962
  }
963
  # Preserve the target Space when the worker fails after repository creation.
964
  target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
@@ -1594,6 +1808,85 @@ def read_inference_contract(workspace: Path | None) -> dict:
1594
  return {}
1595
 
1596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1597
  def contract_smoke_test_payload(contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None:
1598
  smoke = contract.get("smoke_test") if isinstance(contract, dict) else None
1599
  if not isinstance(smoke, dict):
@@ -2583,6 +2876,100 @@ def create_space_with_hardware_strategy(api, target_space_id: str, token: str, p
2583
  raise
2584
 
2585
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2586
  def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None):
2587
  workspace.mkdir(parents=True, exist_ok=True)
2588
  model_analysis = model_analysis or {}
@@ -2630,7 +3017,6 @@ diffusers
2630
  accelerate
2631
  safetensors
2632
  torch
2633
- kernels
2634
  pillow
2635
  numpy
2636
  requests
@@ -2677,9 +3063,21 @@ Non-negotiable safety and product constraints:
2677
  - Work only inside the current workspace.
2678
  - The wrapper will create the private Space, request allowed hardware best-effort, upload files, and validate the live app. Do not create/delete repos yourself in this builder worker.
2679
  - Preserve a cheap health endpoint named `health` with `api_name="health"`. It must not load weights, run GPU work, or download large files.
2680
- - Do not pin huggingface_hub below 1.0. Use huggingface_hub>=0.34.0,<2.0.0 unless the model card requires a narrower compatible range. If transformers>=5 is used, keep huggingface_hub compatible with it, for example huggingface_hub>=1.5.0,<2.0.0.
 
 
 
 
 
 
2681
  - README.md frontmatter must remain valid; if it uses short_description, it must be 60 characters or fewer.
2682
 
 
 
 
 
 
 
2683
  Implementation contract:
2684
  - If IMPLEMENTATION_MODE is `full-inference-gated`, you are not allowed to silently replace generation with a placeholder and call it success.
2685
  - Try to implement the closest real inference path for the model card using evidence from README, model metadata, config files, and repo files.
@@ -2687,10 +3085,13 @@ Implementation contract:
2687
  - If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint.
2688
  - If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement.
2689
  - If the model requires special dependencies, include them only when needed and document risks.
2690
- - Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels where relevant, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs.
 
2691
  - If real inference is impossible or unsafe in a Space, write TECHNICAL_BLOCKERS.json with concrete evidence for every blocker.
2692
 
2693
  Deliverables:
 
 
2694
  - app.py must boot on Hugging Face Spaces.
2695
  - app.py must expose health/api_name="health".
2696
  - If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic.
@@ -2778,13 +3179,13 @@ def sanitize_readme_metadata(workspace: Path, events_path: Path):
2778
  )
2779
 
2780
  def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
2781
- """Normalize only broad known-dangerous base dependencies before upload.
2782
 
2783
- Do not try to solve every dependency conflict here. Pi is responsible for
2784
- reading concrete build logs and patching requirements when a Space build
2785
- fails. The Factory only prevents obviously unsafe broad ranges that would
2786
- make the initial build non-deterministic, then observes the Space status and
2787
- hands real build errors back to Pi with evidence.
2788
  """
2789
  req_path = workspace / "requirements.txt"
2790
  if not req_path.exists():
@@ -2805,27 +3206,62 @@ def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
2805
 
2806
  # Minimal base pins only. Do not globally pin diffusers here: newer model
2807
  # cards may legitimately need a recent Diffusers release. If diffusers causes
2808
- # a pip conflict, Pi must repair from the concrete build log.
2809
  policy: dict[str, str] = {
2810
  "huggingface-hub": "huggingface_hub>=0.34.0,<2.0.0",
2811
  "transformers": "transformers>=4.51.0,<5.0.0",
2812
  }
 
 
 
 
 
 
 
 
 
 
 
2813
  aliases = {
2814
  "huggingface_hub": "huggingface-hub",
2815
  "huggingface-hub": "huggingface-hub",
2816
  "transformers": "transformers",
 
 
 
 
 
 
2817
  }
 
2818
  seen_policy: set[str] = set()
 
2819
  filtered: list[str] = []
2820
  changed = False
 
 
2821
 
2822
  for line in package_lines:
2823
  stripped = line.strip()
2824
  if stripped.startswith("#") or "://" in stripped or stripped.startswith((".", "/")):
2825
  filtered.append(line)
2826
  continue
2827
- name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
2828
  canonical = aliases.get(name)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2829
  if canonical in policy:
2830
  if stripped != policy[canonical]:
2831
  changed = True
@@ -2843,34 +3279,53 @@ def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
2843
  stable_policy_lines.append(policy[canonical])
2844
  changed = True
2845
 
 
 
 
 
 
 
 
 
 
 
 
2846
  torch_added = False
2847
- if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines, "torch"):
2848
  stable_policy_lines.append("torch>=2.0.0")
2849
  torch_added = True
2850
  changed = True
2851
 
2852
- new_lines = prefix_lines + stable_policy_lines + filtered
2853
  new = "\n".join(line for line in new_lines if line.strip()) + "\n"
2854
  if new != raw:
2855
  changed = True
2856
  if changed:
2857
  req_path.write_text(new, encoding="utf-8")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2858
  append_event(
2859
  events_path,
2860
  "requirements_sanitize",
2861
  "success",
2862
- "Normalized broad base dependencies; concrete pip conflicts remain Pi repair work",
2863
- {
2864
- "huggingface_hub": policy["huggingface-hub"],
2865
- "transformers": policy["transformers"],
2866
- "reason": "Avoid uncontrolled Transformers 5.x while preserving model-specific dependency choices for Pi to repair from build logs; require torch when app.py imports torch.",
2867
- "torch_added": torch_added,
2868
- "torch_policy": "torch>=2.0.0",
2869
- "torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present",
2870
- },
2871
  )
2872
 
2873
-
2874
  def useful_log_signals(text: str) -> list[str]:
2875
  low = (text or "").lower()
2876
  signals = []
@@ -3354,6 +3809,7 @@ First read `INCIDENT_BRIEF.md` and the HF Spaces gist operational rules: {GIST_U
3354
 
3355
  You are not allowed to edit code in this diagnosis step. Your task is to decide the next action for the Factory.
3356
  Use the gist method: read logs first, identify the first actionable error, use the cheapest useful iteration rung, and require a live Gradio/API validation before success.
 
3357
 
3358
  Write `REPAIR_DECISION.json` exactly as requested in INCIDENT_BRIEF.md. Do not patch files during this step.
3359
  """
@@ -3471,7 +3927,11 @@ A separate Pi diagnosis already decided that `patch_code` is justified. Do not r
3471
  - Preserve or restore a cheap `health` endpoint.
3472
  - Preserve the expected Gradio API endpoint when possible.
3473
  - Keep README metadata valid and short_description <= 60 chars.
3474
- - Keep Hugging Face Hub requirements modern; do not pin below 1.0 unless unavoidable and explained.
 
 
 
 
3475
 
3476
  ## Required repair artifacts
3477
  Before modifying files, write `REPAIR_PLAN.md` with:
@@ -3552,6 +4012,8 @@ First read `REPAIR_BRIEF.md`, `INCIDENT_BRIEF.md`, `DEPENDENCY_ERROR_BRIEF.md` i
3552
  You are continuing the same build run, not starting a separate project.
3553
  This patch is allowed only because the diagnosis decision selected `patch_code`.
3554
  If `DEPENDENCY_ERROR_BRIEF.md` exists, treat it as evidence for the gist method: identify the first pip error, patch dependency pins minimally, and do not modify inference code unless the dependency fix alone cannot address that first error.
 
 
3555
  Use the available HF token context to inspect private Hub resources when needed, but never print or persist token values.
3556
 
3557
  Critical method:
@@ -3627,18 +4089,53 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e
3627
  dependency_issue = extract_pip_dependency_issue(f"{current_error}\n{build_log}\n{runtime_log}")
3628
  if dependency_issue:
3629
  write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, current_error)
 
 
 
 
 
 
 
 
 
 
3630
  if apply_dependency_guardrail_repair(workspace, run_dir, events_path, current_error, build_log, runtime_log):
 
 
 
 
 
 
 
 
 
3631
  append_event(events_path, "factory_rebuild", "started", "Re-uploading dependency-guardrailed workspace after pip resolver build error")
3632
  if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="dependency_guardrail_repair"):
 
3633
  raise RuntimeError("Dependency guardrail rebuild skipped by restart guardrails")
 
3634
  append_event(events_path, "factory_rebuild", "success", "Dependency-guardrailed workspace uploaded; revalidating live Space")
3635
  append_event(events_path, "repair_validation", "started", "Revalidating after deterministic dependency repair")
3636
  write_live_status(run_dir, stage="live_validation", status="running", message="Waiting for Space runtime and health endpoint", data={"target_space": target_space_id})
3637
  try:
 
3638
  validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
 
3639
  append_event(events_path, "repair_validation", "success", "Deterministic dependency repair resolved the build blockage")
3640
  return validation
3641
  except Exception as exc:
 
 
 
 
 
 
 
 
 
 
 
 
3642
  current_error = f"{current_error}\n\nDependency guardrail rebuild did not resolve validation: {str(exc)[:4000]}"
3643
  collect_space_logs(target_space_id, token, run_dir, events_path)
3644
  append_event(events_path, "repair_validation", "failed", "Dependency guardrail rebuild did not resolve validation; falling back to Pi diagnosis", {"error": str(exc)[:4000]})
@@ -3676,10 +4173,27 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e
3676
  append_event(events_path, "factory_rebuild", "success", "Same-code workspace re-uploaded; revalidating live Space")
3677
  append_event(events_path, "repair_validation", "started", "Revalidating after same-code factory rebuild")
3678
  try:
 
3679
  validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
 
3680
  append_event(events_path, "repair_validation", "success", "Same-code factory rebuild resolved the blockage")
3681
  return validation
3682
  except Exception as exc:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3683
  current_error = f"{current_error}\n\nSame-code factory rebuild did not resolve validation: {str(exc)[:4000]}"
3684
  collect_space_logs(target_space_id, token, run_dir, events_path)
3685
  append_event(events_path, "repair_validation", "failed", "Same-code factory rebuild did not resolve validation; re-diagnosing", {"error": str(exc)[:4000]})
@@ -3692,20 +4206,49 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e
3692
  raise RuntimeError("Automated recovery stopped after patch budget was exhausted")
3693
  budgets[action] -= 1
3694
  append_event(events_path, "repair", "started", "Pi diagnosis allows a minimal code patch", {"decision": decision})
 
 
 
 
 
 
 
 
 
 
 
3695
  repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision)
3696
  if not repaired:
 
3697
  write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair")
3698
  append_event(events_path, "failure", "failed", "Structured patch repair failed before redeploy", {"decision": decision})
3699
  raise RuntimeError("Structured patch repair failed before redeploy")
 
3700
  append_event(events_path, "repair_upload", "started", "Uploading repaired workspace")
 
3701
  upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
 
3702
  append_event(events_path, "repair_upload", "success", "Repaired workspace uploaded")
3703
  append_event(events_path, "repair_validation", "started", "Revalidating repaired Space")
3704
  try:
 
3705
  validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
 
3706
  append_event(events_path, "repair_validation", "success", "Repaired Space passed live API validation")
3707
  return validation
3708
  except Exception as exc:
 
 
 
 
 
 
 
 
 
 
 
 
3709
  current_error = f"{current_error}\n\nPatch repair did not resolve validation: {str(exc)[:4000]}"
3710
  collect_space_logs(target_space_id, token, run_dir, events_path)
3711
  append_event(events_path, "repair_validation", "failed", "Repair attempted, but validation still failed", {"error": str(exc)[:4000]})
@@ -3799,8 +4342,12 @@ def infer_generation_gate(workspace: Path, implementation_mode: str, validation:
3799
  }
3800
 
3801
  if blocker_detected:
3802
- status = "technical_blocker"
3803
- message = "Space boots, but full model inference was not implemented. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md."
 
 
 
 
3804
  elif implementation_mode in {"full-inference-gated", "full-inference-attempt"} and smoke_ok:
3805
  status = "full_inference_success"
3806
  message = "Space boots and a live generation smoke test passed. ZeroGPU duration recommendation was measured from real inference."
@@ -3891,6 +4438,7 @@ def main():
3891
  api = HfApi(token=token)
3892
  whoami = api.whoami(token=token)
3893
  append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})
 
3894
 
3895
  append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})
3896
  info = api.model_info(model_id, token=token, files_metadata=True)
@@ -3929,6 +4477,7 @@ def main():
3929
  emit_pi_model_resolution(events_path, pi_model_resolution)
3930
  if not (workspace / "PI_SUMMARY.md").exists():
3931
  (workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8")
 
3932
 
3933
  app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")
3934
  if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text:
@@ -3973,6 +4522,7 @@ def main():
3973
  # starts directly on the requested hardware. If it fell back to CPU, the run
3974
  # remains valid but will be marked manual_hardware_required when inference
3975
  # signals indicate GPU is needed.
 
3976
  upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
3977
  write_artifact_manifest(run_dir, reason="workspace_uploaded")
3978
 
@@ -3993,21 +4543,25 @@ def main():
3993
  )
3994
  generation_smoke = None
3995
  if implementation_mode in {"full-inference-gated", "full-inference-attempt"}:
3996
- try:
3997
- generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace)
3998
- except Exception as smoke_error:
3999
- write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation smoke test failed", data={"error": str(smoke_error)[:2000]})
4000
- generation_smoke = {
4001
- "status": "failed",
4002
- "target_space": target_space_id,
4003
- "expected_output_type": expected_output_type,
4004
- "error": str(smoke_error)[:4000],
4005
- "next_action": "Use Prefill Space Test after the run finishes to retry with schema-adjusted arguments.",
4006
- **measured_zero_gpu_recommendation(None),
4007
- }
4008
- generation_smoke.update(classify_generation_smoke_error(smoke_error))
4009
- write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
4010
- append_event(events_path, "generation_smoke", "failed", "Live generation smoke test failed; ZeroGPU duration was not measured", generation_smoke)
 
 
 
 
4011
  else:
4012
  generation_smoke = measured_zero_gpu_recommendation(None) | {"status": "skipped", "expected_output_type": expected_output_type}
4013
  write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
@@ -4017,7 +4571,7 @@ def main():
4017
  # hardware requests failed, classify the run honestly as needing manual
4018
  # hardware instead of pretending CPU/default hardware is enough. the existing-Space validation workflow
4019
  # can then smoke-test generation after the user sets a GPU manually.
4020
- manual_hw_required = selected_hardware == "default-cpu-or-existing" and inference_gate.get("status") not in {"technical_blocker", "health_only"} and (
4021
  inference_gate.get("implementation_signals", {}).get("has_spaces_gpu")
4022
  or inference_gate.get("implementation_signals", {}).get("has_torch")
4023
  or any((a.get("manual_action_required") for a in hardware_attempts if isinstance(a, dict)))
@@ -4134,7 +4688,15 @@ The wrapper validated the live Space using HTTP `/health` first, with Gradio Cli
4134
  collect_pi_traces(run_dir, events_path)
4135
  except Exception:
4136
  pass
4137
- fail(run_dir, events_path, "Universal model-card builder worker failed", {"error": str(exc)})
 
 
 
 
 
 
 
 
4138
 
4139
 
4140
  if __name__ == "__main__":
@@ -4144,6 +4706,7 @@ if __name__ == "__main__":
4144
 
4145
 
4146
  VALIDATE_EXISTING_SPACE_WORKER_SCRIPT = r'''
 
4147
  import json
4148
  import os
4149
  import re
@@ -4361,7 +4924,7 @@ def requirements_has_package(lines: list[str], package: str) -> bool:
4361
  stripped = line.strip()
4362
  if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
4363
  continue
4364
- name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
4365
  if name == wanted:
4366
  return True
4367
  return False
@@ -4547,7 +5110,7 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
4547
  return "cancelled"
4548
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
4549
  return "manual_action_required"
4550
- if status == "technical_blocker":
4551
  return "technical_blocker"
4552
  if full_inference_verified:
4553
  return "success"
@@ -4758,6 +5321,141 @@ def publish_eval_record(run_dir: Path, *, phase: str, events_path: Path | None =
4758
  return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"}
4759
 
4760
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4761
  def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
4762
  """Ensure Pi and HF tooling see the same private HF token safely.
4763
 
@@ -4771,6 +5469,7 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
4771
  os.environ.setdefault("HF_TOKEN", token)
4772
  os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
4773
  os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
 
4774
  payload = {
4775
  "hf_token_present": bool(token),
4776
  "hf_token_length": len(token) if token else 0,
@@ -4779,6 +5478,13 @@ def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> d
4779
  "bucket_source": os.environ.get("BUCKET_SOURCE") or "",
4780
  "target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
4781
  "token_value": "[REDACTED]" if token else "",
 
 
 
 
 
 
 
4782
  }
4783
  run_dir.mkdir(parents=True, exist_ok=True)
4784
  write_json(run_dir / "token_context.json", payload)
@@ -6086,6 +6792,8 @@ def main():
6086
  write_json(state_path, {"run_id": run_id, "kind": "linked_space_validation", "status": "running", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "created_by": username, "updated_at": now()})
6087
  if not token:
6088
  raise RuntimeError("HF_TOKEN is missing")
 
 
6089
  if not TARGET_RE.match(target_space_id):
6090
  raise ValueError("TARGET_SPACE_ID must look like owner/space-name")
6091
  if not parent_build_run_id:
@@ -6186,14 +6894,19 @@ Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_
6186
  append_event(events_path, "done", "full_inference_success", "Existing Space validation completed", {"latency_seconds": smoke.get("latency_seconds")})
6187
  except Exception as exc:
6188
  collect_space_logs(target_space_id, token or "", run_dir, events_path)
 
6189
  details = {"error": str(exc)[:4000]}
6190
- failure_state = {"run_id": run_id, "kind": "linked_space_validation", "status": "failed", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "details": details, "updated_at": now()}
 
 
 
 
6191
  write_json(state_path, failure_state)
6192
- write_final_summary(run_dir, failure_state, {}, {"status": "failed", "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status="failed", message=str(details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed")
6193
  if parent_build_run_id:
6194
  parent_dir = output_root / "runs" / parent_build_run_id
6195
  parent_dir.mkdir(parents=True, exist_ok=True)
6196
- failed_status = {"status": "failed", "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "details": details, "updated_at": now(), "effective_status": "unchanged"}
6197
  linked_path = parent_dir / "linked_validations.json"
6198
  linked = read_json(linked_path, {"parent_build_run_id": parent_build_run_id, "validations": []}) or {"parent_build_run_id": parent_build_run_id, "validations": []}
6199
  validations = linked.get("validations") if isinstance(linked, dict) else []
@@ -6208,7 +6921,7 @@ Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_
6208
  write_json(parent_dir / "manual_validation_status.json", failed_status)
6209
  write_json(linked_path, linked)
6210
  (run_dir / "report.md").write_text(f"# Existing Space Validation Failed\n\n```json\n{json.dumps(details, indent=2, ensure_ascii=False)}\n```\n", encoding="utf-8")
6211
- append_event(events_path, "failure", "failed", "Existing Space validation failed", details)
6212
  try:
6213
  publish_eval_record(run_dir, phase="failure", events_path=events_path)
6214
  append_event(events_path, "anonymous_eval", "success", "Published anonymized failed validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False})
@@ -6217,8 +6930,8 @@ Target Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_
6217
  # v190.33: also rewrite failure summary as the final visible file update
6218
  # for linked validations, so a failed validation cannot remain listed
6219
  # as running after the worker has already terminalized state.json.
6220
- write_final_summary(run_dir, failure_state, {}, {"status": "failed", "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status="failed", message=str(details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed")
6221
- append_event(events_path, "summary_write", "success", "Wrote terminal failed validation summary", {"status": "failed"})
6222
  raise SystemExit(1)
6223
 
6224
 
 
10
 
11
  UNIVERSAL_MODEL_CARD_WORKER_SCRIPT = r'''
12
 
13
+ import base64
14
  import hashlib
15
  import hmac
16
  import hashlib
 
49
  "REPAIR_SUMMARY.md",
50
  "PI_SUMMARY.md",
51
  "TECHNICAL_BLOCKERS.json",
52
+ "pi_feasibility_brief.json",
53
+ "pi_implementation_plan.json",
54
  }
55
 
56
 
 
281
  "bucket_source": final_state.get("bucket_source") or os.environ.get("BUCKET_SOURCE", ""),
282
  "health_passed": bool(((gate.get("implementation_signals") or {}).get("health_passed") is True) or smoke.get("health_passed")),
283
  "generation_smoke_passed": bool(((gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) or smoke.get("status") == "success"),
284
+ "failure_type": smoke.get("failure_type") or gate.get("failure_type") or (final_state.get("details") or {}).get("failure_type") or (final_state.get("details") or {}).get("repair_failure_type") or (final_state.get("repair_outcome") or {}).get("failure_type") or final_state.get("failure_type") or "",
285
+ "repair_outcome_status": (final_state.get("repair_outcome") or {}).get("post_repair_validation") or (final_state.get("details") or {}).get("repair_outcome_status") or "",
286
  "updated_at": now(),
287
  }
288
  write_json(run_dir / "summary.json", payload)
 
312
  stripped = line.strip()
313
  if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
314
  continue
315
+ name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-")
316
  if name == wanted:
317
  return True
318
  return False
 
357
  _artifact_entry(run_dir, "live_status.json"),
358
  _artifact_entry(run_dir, "report.md"),
359
  _artifact_entry(run_dir, "model_analysis.json"),
360
+ _artifact_entry(run_dir, "planning/pi_feasibility_brief.json"),
361
+ _artifact_entry(run_dir, "planning/pi_implementation_plan.json"),
362
+ _artifact_entry(run_dir, "planning/worker_plan_review.json"),
363
  _artifact_entry(run_dir, "hardware_strategy.json"),
364
  _artifact_entry(run_dir, "hardware_attempts.json"),
365
  _artifact_entry(run_dir, "inference_gate.json"),
366
+ _artifact_entry(run_dir, "repair_outcome.json"),
367
  _artifact_entry(run_dir, "space_runtime.json"),
368
  _artifact_entry(run_dir, "tests/generation_smoke.json"),
369
  _artifact_entry(run_dir, "tests/api_schema.json"),
 
375
  _artifact_entry(run_dir, "repair/REPAIR_BRIEF.md"),
376
  _artifact_entry(run_dir, "repair/REPAIR_PLAN.md"),
377
  _artifact_entry(run_dir, "repair/REPAIR_SUMMARY.md"),
378
+ _artifact_entry(run_dir, "repair/REPAIR_OUTCOME.json"),
379
  _artifact_entry(run_dir, "repair/BLOCKAGE.json"),
380
  _artifact_entry(run_dir, "logs/pi_live_output.txt"),
381
  _artifact_entry(run_dir, "logs/pi_output.txt"),
 
438
 
439
 
440
 
441
+
442
+
443
+ class AuthRefreshRequired(RuntimeError):
444
+ """Raised when an OAuth/JWT token is expired or too close to expiry."""
445
+
446
+
447
+ def _decode_jwt_claims_unverified(token: str) -> dict:
448
+ """Decode JWT claims without verification, only to inspect non-sensitive expiry metadata.
449
+
450
+ This never returns or writes the raw token. Opaque PAT-style tokens are supported by
451
+ returning an empty dict so they remain usable with an `unknown` expiry state.
452
+ """
453
+ try:
454
+ parts = (token or "").split(".")
455
+ if len(parts) < 2:
456
+ return {}
457
+ payload = parts[1]
458
+ payload += "=" * ((4 - len(payload) % 4) % 4)
459
+ raw = base64.urlsafe_b64decode(payload.encode("utf-8"))
460
+ data = json.loads(raw.decode("utf-8"))
461
+ return data if isinstance(data, dict) else {}
462
+ except Exception:
463
+ return {}
464
+
465
+
466
+ def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict:
467
+ issued_at = now()
468
+ if not token:
469
+ return {
470
+ "schema_version": "auth_status.v1",
471
+ "checked_at": issued_at,
472
+ "token_present": False,
473
+ "token_kind": "missing",
474
+ "expiry_known": False,
475
+ "status": "missing",
476
+ "safe_for_phase": False,
477
+ "minimum_required_seconds": minimum_required_seconds,
478
+ }
479
+ claims = _decode_jwt_claims_unverified(token)
480
+ exp = claims.get("exp") if isinstance(claims, dict) else None
481
+ iat = claims.get("iat") if isinstance(claims, dict) else None
482
+ token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown")
483
+ payload = {
484
+ "schema_version": "auth_status.v1",
485
+ "checked_at": issued_at,
486
+ "token_present": True,
487
+ "token_kind": token_kind,
488
+ "expiry_known": exp is not None,
489
+ "minimum_required_seconds": int(minimum_required_seconds or 0),
490
+ "token_value": "[REDACTED]",
491
+ }
492
+ if iat is not None:
493
+ try:
494
+ payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat()
495
+ except Exception:
496
+ pass
497
+ if exp is None:
498
+ payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"})
499
+ return payload
500
+ try:
501
+ exp_int = int(exp)
502
+ seconds_left = exp_int - int(time.time())
503
+ payload.update({
504
+ "expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(),
505
+ "seconds_until_expiry": seconds_left,
506
+ })
507
+ except Exception:
508
+ payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"})
509
+ return payload
510
+ if seconds_left <= 0:
511
+ payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"})
512
+ elif minimum_required_seconds and seconds_left < minimum_required_seconds:
513
+ payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"})
514
+ else:
515
+ payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"})
516
+ return payload
517
+
518
+
519
+ def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict:
520
+ payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds)
521
+ payload["phase"] = phase
522
+ safe_payload = {k: v for k, v in payload.items() if k != "token_value"}
523
+ try:
524
+ probes_dir = run_dir / "auth_probes"
525
+ probes_dir.mkdir(parents=True, exist_ok=True)
526
+ write_json(probes_dir / f"{phase}.json", safe_payload)
527
+ write_json(run_dir / "auth_status.json", safe_payload)
528
+ except Exception:
529
+ pass
530
+ status = str(payload.get("status") or "unknown")
531
+ if events_path:
532
+ event_status = "success" if payload.get("safe_for_phase") else "failed"
533
+ if status == "unknown":
534
+ event_status = "warning"
535
+ append_event(
536
+ events_path,
537
+ "auth_probe",
538
+ event_status,
539
+ f"HF OAuth/token expiry check for {phase}: {status}",
540
+ safe_payload,
541
+ )
542
+ if raise_on_unsafe and not payload.get("safe_for_phase"):
543
+ raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.")
544
+ return payload
545
+
546
+
547
+ def is_auth_expired_error(error: Exception | str) -> bool:
548
+ text = str(error or "").lower()
549
+ return any(marker in text for marker in [
550
+ "oauth token has expired",
551
+ "exp claim timestamp check failed",
552
+ "token has expired",
553
+ "jwt expired",
554
+ ])
555
+
556
+
557
+
558
+ def classify_repair_validation_error(error: Exception | str) -> dict:
559
+ """Classify post-repair validation failures without losing the repair result.
560
+
561
+ A repair can be correctly diagnosed, patched, and uploaded while the final
562
+ validation is inconclusive because OAuth expired. Keep that distinct from a
563
+ model/runtime repair failure so the UI and audit trail do not imply that Pi's
564
+ patch necessarily failed.
565
+ """
566
+ text = str(error or "")
567
+ if isinstance(error, AuthRefreshRequired) or is_auth_expired_error(text):
568
+ return {
569
+ "post_repair_validation": "inconclusive_auth_expired",
570
+ "failure_type": "repair_validation_inconclusive_auth",
571
+ "terminal_status": "auth_refresh_required",
572
+ "message": "Repair progress could not be validated because HF auth expired or is too close to expiry.",
573
+ }
574
+ return {
575
+ "post_repair_validation": "failed",
576
+ "failure_type": "repair_validation_failed",
577
+ "terminal_status": "failed",
578
+ "message": "Repair was applied, but the repaired Space did not pass live validation.",
579
+ }
580
+
581
+
582
+ def write_repair_outcome(run_dir: Path, events_path: Path | None = None, **updates) -> dict:
583
+ """Write a cumulative repair outcome artifact.
584
+
585
+ Keep it at the run root for list-view consumers and mirror it inside
586
+ repair/ for manual audits. The payload is intentionally compact and
587
+ redacted; full logs remain in logs/ and repair/ artifacts.
588
+ """
589
+ root_path = run_dir / "repair_outcome.json"
590
+ existing = load_json_if_exists(root_path) if root_path.exists() else {}
591
+ if not isinstance(existing, dict):
592
+ existing = {}
593
+ payload = {
594
+ "schema_version": "repair_outcome.v1",
595
+ **existing,
596
+ **{k: v for k, v in updates.items() if v is not None},
597
+ "updated_at": now(),
598
+ }
599
+ write_json(root_path, payload)
600
+ try:
601
+ write_json(run_dir / "repair" / "REPAIR_OUTCOME.json", payload)
602
+ except Exception:
603
+ pass
604
+ if events_path:
605
+ append_event(
606
+ events_path,
607
+ "repair_outcome",
608
+ str(payload.get("post_repair_validation") or payload.get("repair_status") or payload.get("repair_decision") or "updated"),
609
+ "Repair outcome artifact updated",
610
+ payload,
611
+ )
612
+ return payload
613
+
614
+ def minimum_auth_seconds_for_phase(phase: str) -> int:
615
+ env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_")
616
+ raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "")
617
+ if raw:
618
+ try:
619
+ return max(0, int(raw))
620
+ except Exception:
621
+ pass
622
+ defaults = {
623
+ "before_space_create": 900,
624
+ "before_upload": 900,
625
+ "before_initial_validation": 1800,
626
+ "before_repair": 1800,
627
+ "before_repair_upload": 1800,
628
+ "before_repair_validation": 1800,
629
+ "before_linked_space_test": 1800,
630
+ }
631
+ return defaults.get(phase, 0)
632
+
633
  def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
634
  """Ensure Pi and HF tooling see the same private HF token safely.
635
 
 
643
  os.environ.setdefault("HF_TOKEN", token)
644
  os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
645
  os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
646
+ expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"))
647
  payload = {
648
  "hf_token_present": bool(token),
649
  "hf_token_length": len(token) if token else 0,
 
652
  "bucket_source": os.environ.get("BUCKET_SOURCE") or "",
653
  "target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
654
  "token_value": "[REDACTED]" if token else "",
655
+ "token_kind": expiry.get("token_kind"),
656
+ "expiry_known": expiry.get("expiry_known"),
657
+ "expires_at": expiry.get("expires_at"),
658
+ "seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"),
659
+ "auth_risk": expiry.get("auth_risk"),
660
+ "safe_for_long_build": bool(expiry.get("safe_for_phase")),
661
+ "minimum_required_seconds": expiry.get("minimum_required_seconds"),
662
  }
663
  run_dir.mkdir(parents=True, exist_ok=True)
664
  write_json(run_dir / "token_context.json", payload)
 
943
  return "cancelled"
944
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
945
  return "manual_action_required"
946
+ if status in {"technical_blocker", "technical_blocker_boot_only"}:
947
  return "technical_blocker"
948
  if full_inference_verified:
949
  return "success"
 
1160
  existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
1161
  if not isinstance(existing_state, dict):
1162
  existing_state = {}
1163
+ repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {}
1164
+ if isinstance(repair_outcome, dict) and repair_outcome:
1165
+ safe.setdefault("repair_outcome_status", repair_outcome.get("post_repair_validation") or repair_outcome.get("repair_status") or "")
1166
+ safe.setdefault("repair_failure_type", repair_outcome.get("failure_type") or "")
1167
  failure_state = {
1168
  **existing_state,
1169
  "run_id": os.environ.get("RUN_ID"),
 
1172
  "message": message,
1173
  "updated_at": now(),
1174
  "details": safe,
1175
+ "repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {},
1176
  }
1177
  # Preserve the target Space when the worker fails after repository creation.
1178
  target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
 
1808
  return {}
1809
 
1810
 
1811
+ def _as_bool_false(value) -> bool:
1812
+ if value is False:
1813
+ return True
1814
+ if isinstance(value, str):
1815
+ return value.strip().lower() in {"false", "no", "0", "off"}
1816
+ return False
1817
+
1818
+
1819
+ def _read_workspace_json(workspace: Path | None, filename: str) -> dict:
1820
+ if not workspace:
1821
+ return {}
1822
+ path = workspace / filename
1823
+ if not path.exists():
1824
+ return {}
1825
+ try:
1826
+ data = json.loads(path.read_text(encoding="utf-8", errors="replace"))
1827
+ return data if isinstance(data, dict) else {}
1828
+ except Exception:
1829
+ return {}
1830
+
1831
+
1832
+ def contract_declares_no_full_inference(workspace: Path | None) -> dict:
1833
+ """Return a structured reason when Pi explicitly declared boot-only/no inference.
1834
+
1835
+ v191.2: A machine-readable no-inference contract must be respected by the
1836
+ worker. In that case automatic generation smoke is skipped instead of
1837
+ inventing /generate fallback arguments that cannot exist.
1838
+ """
1839
+ contract = read_inference_contract(workspace)
1840
+ blockers = _read_workspace_json(workspace, "TECHNICAL_BLOCKERS.json")
1841
+ validation_level = str(contract.get("validation_level") or "").strip().lower().replace("_", "-")
1842
+ primary = normalize_api_name(str(contract.get("primary_api_name") or "")) if contract else ""
1843
+ contract_full_false = _as_bool_false(contract.get("full_inference_implemented"))
1844
+ blocker_full_false = _as_bool_false(blockers.get("full_inference_implemented"))
1845
+ validation_boot_only = validation_level in {"boot-only", "health-only", "diagnostic-only", "info-only"}
1846
+ primary_absent = not primary or primary == "/health"
1847
+ blockers_count = blockers.get("blockers_count")
1848
+ if not isinstance(blockers_count, int):
1849
+ raw_blockers = blockers.get("blockers")
1850
+ blockers_count = len(raw_blockers) if isinstance(raw_blockers, list) else 0
1851
+
1852
+ declared = bool(contract_full_false or blocker_full_false or (validation_boot_only and primary_absent))
1853
+ return {
1854
+ "declared": declared,
1855
+ "source": "inference_contract" if contract_full_false or validation_boot_only else "technical_blockers" if blocker_full_false else "",
1856
+ "full_inference_implemented": False if (contract_full_false or blocker_full_false) else contract.get("full_inference_implemented"),
1857
+ "validation_level": contract.get("validation_level"),
1858
+ "primary_api_name": contract.get("primary_api_name"),
1859
+ "blockers_count": blockers_count,
1860
+ "contract_present": bool(contract),
1861
+ "technical_blockers_present": bool(blockers),
1862
+ }
1863
+
1864
+
1865
+ def write_contract_skipped_generation_smoke(run_dir: Path, events_path: Path, expected_output_type: str, target_space_id: str, reason: dict | None = None) -> dict:
1866
+ reason = reason or {}
1867
+ payload = {
1868
+ "status": "skipped",
1869
+ "skip_reason": "contract_declared_no_full_inference",
1870
+ "reason": "Pi declared that full inference is not implemented; automatic generation smoke was skipped.",
1871
+ "target_space": target_space_id,
1872
+ "api_name": None,
1873
+ "expected_output_type": expected_output_type,
1874
+ "validation_mode": "automatic_smoke",
1875
+ "payload_source": "contract_declared_no_full_inference",
1876
+ "contract_aware_skip": True,
1877
+ "contract_reason": reason,
1878
+ "next_action": "Full inference is blocked. No generation endpoint exists. Review TECHNICAL_BLOCKERS.json / PI_SUMMARY.md or provide a dedicated implementation and hardware plan.",
1879
+ **measured_zero_gpu_recommendation(None),
1880
+ }
1881
+ write_json(run_dir / "tests" / "generation_smoke.json", payload)
1882
+ write_json(run_dir / "tests" / "payload_source.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "selected_api_name": None, "parent_smoke_payload_used": False})
1883
+ write_json(run_dir / "tests" / "validation_engine.json", {"schema_version": "1.0", "validation_engine": "unified_gradio_validation_harness", "validation_mode": "automatic_smoke", "resolved_request_required_before_predict": False, "skipped_by_contract": True})
1884
+ write_json(run_dir / "tests" / "resolved_validation_request.json", {"api_name": None, "test_args": [], "test_kwargs": {}, "expected_output_type": expected_output_type, "validation_mode": "automatic_smoke", "payload_source": "contract_declared_no_full_inference", "resolved_request_required_before_predict": False})
1885
+ write_live_status(run_dir, stage="generation_smoke", status="skipped", message="Generation smoke skipped because Pi declared full inference unavailable", data=payload)
1886
+ append_event(events_path, "generation_smoke", "skipped", "Generation smoke skipped because Pi declared no full inference endpoint", payload)
1887
+ return payload
1888
+
1889
+
1890
  def contract_smoke_test_payload(contract: dict, api_name: str, endpoint, expected_output_type: str) -> dict | None:
1891
  smoke = contract.get("smoke_test") if isinstance(contract, dict) else None
1892
  if not isinstance(smoke, dict):
 
2876
  raise
2877
 
2878
 
2879
+
2880
+
2881
+ def _safe_json_object_from_workspace(path: Path) -> dict:
2882
+ payload = read_json(path, {})
2883
+ return payload if isinstance(payload, dict) else {}
2884
+
2885
+
2886
+ def _first_truthy_string(*values) -> str | None:
2887
+ for value in values:
2888
+ if isinstance(value, str) and value.strip():
2889
+ return value.strip()
2890
+ return None
2891
+
2892
+
2893
+ def write_pi_planning_review(workspace: Path, run_dir: Path, events_path: Path, model_analysis: dict | None = None, implementation_mode: str = "") -> dict:
2894
+ """Persist Pi planning artifacts and a non-blocking worker review.
2895
+
2896
+ v191.9 intentionally makes planning visibility-only. Pi is asked to produce
2897
+ a feasibility brief and implementation plan, but older/partial Pi behavior
2898
+ must not fail a run that would otherwise work. The worker records whether
2899
+ those artifacts exist, mirrors them under planning/, and keeps the final
2900
+ authority in the existing deterministic gates.
2901
+ """
2902
+ model_analysis = model_analysis or {}
2903
+ planning_dir = run_dir / "planning"
2904
+ planning_dir.mkdir(parents=True, exist_ok=True)
2905
+ brief_path = workspace / "pi_feasibility_brief.json"
2906
+ plan_path = workspace / "pi_implementation_plan.json"
2907
+ brief = _safe_json_object_from_workspace(brief_path)
2908
+ plan = _safe_json_object_from_workspace(plan_path)
2909
+ brief_present = bool(brief)
2910
+ plan_present = bool(plan)
2911
+ if brief_present:
2912
+ write_json(planning_dir / "pi_feasibility_brief.json", brief)
2913
+ else:
2914
+ write_json(planning_dir / "pi_feasibility_brief.json", {
2915
+ "schema_version": "pi_feasibility_brief.v1",
2916
+ "status": "missing",
2917
+ "non_blocking": True,
2918
+ "message": "Pi did not produce pi_feasibility_brief.json; continuing with existing deterministic worker gates.",
2919
+ })
2920
+ if plan_present:
2921
+ write_json(planning_dir / "pi_implementation_plan.json", plan)
2922
+ else:
2923
+ write_json(planning_dir / "pi_implementation_plan.json", {
2924
+ "schema_version": "pi_implementation_plan.v1",
2925
+ "status": "missing",
2926
+ "non_blocking": True,
2927
+ "message": "Pi did not produce pi_implementation_plan.json; continuing with existing deterministic worker gates.",
2928
+ })
2929
+
2930
+ declared_strategy = _first_truthy_string(
2931
+ brief.get("recommended_strategy"),
2932
+ brief.get("strategy"),
2933
+ plan.get("strategy"),
2934
+ plan.get("implementation_strategy"),
2935
+ ) or "unspecified"
2936
+ should_attempt_full = brief.get("should_attempt_full_inference")
2937
+ if should_attempt_full is None:
2938
+ should_attempt_full = plan.get("should_attempt_full_inference")
2939
+ worker_recommendation = "continue_existing_flow"
2940
+ warnings: list[str] = []
2941
+ if not brief_present:
2942
+ warnings.append("missing_pi_feasibility_brief")
2943
+ if not plan_present:
2944
+ warnings.append("missing_pi_implementation_plan")
2945
+ if declared_strategy in {"boot_only_blocker", "boot_only_blocker_or_manual_refactor", "manual_hardware_required"} or should_attempt_full is False:
2946
+ worker_recommendation = "respect_plan_if_contract_declares_blocker"
2947
+ elif declared_strategy in {"full_inference", "proceed_full_inference", "attempt_full_inference"} or should_attempt_full is True:
2948
+ worker_recommendation = "proceed_full_inference_with_existing_gates"
2949
+
2950
+ review = {
2951
+ "schema_version": "worker_plan_review.v1",
2952
+ "status": "ok" if brief_present and plan_present else "partial",
2953
+ "non_blocking": True,
2954
+ "implementation_mode": implementation_mode,
2955
+ "model_id": model_analysis.get("model_id"),
2956
+ "pipeline_tag": model_analysis.get("pipeline_tag"),
2957
+ "library_name": model_analysis.get("library_name"),
2958
+ "planning_artifacts_present": {
2959
+ "pi_feasibility_brief": brief_present,
2960
+ "pi_implementation_plan": plan_present,
2961
+ },
2962
+ "declared_strategy": declared_strategy,
2963
+ "should_attempt_full_inference": should_attempt_full,
2964
+ "worker_recommendation": worker_recommendation,
2965
+ "warnings": warnings,
2966
+ "authority_note": "Pi planning is advisory in v191.9; deterministic worker gates, INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation remain authoritative.",
2967
+ "created_at": now(),
2968
+ }
2969
+ write_json(planning_dir / "worker_plan_review.json", review)
2970
+ append_event(events_path, "pi_planning_review", review["status"], "Pi feasibility and implementation planning reviewed", review)
2971
+ return review
2972
+
2973
  def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None):
2974
  workspace.mkdir(parents=True, exist_ok=True)
2975
  model_analysis = model_analysis or {}
 
3017
  accelerate
3018
  safetensors
3019
  torch
 
3020
  pillow
3021
  numpy
3022
  requests
 
3063
  - Work only inside the current workspace.
3064
  - The wrapper will create the private Space, request allowed hardware best-effort, upload files, and validate the live app. Do not create/delete repos yourself in this builder worker.
3065
  - Preserve a cheap health endpoint named `health` with `api_name="health"`. It must not load weights, run GPU work, or download large files.
3066
+ - Platform dependency policy: Gradio, Gradio Client, Hugging Face Hub, Spaces, and hf_xet are owned by Agentic Space Factory / the Hugging Face Spaces runtime. Do not pin or downgrade them. Never write `gradio==...`, `gradio<=...`, `gradio~=...`, `gradio-client==...`, `gradio_client==...`, or stale exact pins for `huggingface_hub`, `spaces`, or `hf_xet`.
3067
+ - If generated code needs an older Gradio API, update the code to modern Gradio instead of downgrading Gradio.
3068
+ - Follow the gist dependency method for model-specific packages: pin only when useful to reduce pip resolver backtracking or when the model card/build logs explicitly require a version; do not cargo-cult every version from examples.
3069
+ - Do not pin torch or torchaudio unless unavoidable and explicitly justified; Spaces provide the PyTorch stack. If torchvision is needed, prefer leaving it unpinned so it can resolve against the managed torch runtime.
3070
+ - If model code is not pip-installable, vendor the necessary code into the Space repo instead of referencing local paths, editable installs, or clone-time side effects in requirements.txt.
3071
+ - Do not run local venv installs, broad `pip install --dry-run`, or dependency checks as proof of compatibility; only the live Space build/logs are authoritative. Use `python -m py_compile app.py` only as a cheap syntax check.
3072
+ - Use huggingface_hub>=0.34.0,<2.0.0 unless a newer Transformers path requires a compatible newer range. The worker will normalize platform-owned dependency lines before upload.
3073
  - README.md frontmatter must remain valid; if it uses short_description, it must be 60 characters or fewer.
3074
 
3075
+ Planning contract:
3076
+ - Before committing to a full implementation strategy, produce `pi_feasibility_brief.json` and `pi_implementation_plan.json`. These files are advisory artifacts for the worker and must not contain secrets.
3077
+ - `pi_feasibility_brief.json` should include: complexity, expected_runtime, zerogpu_feasible, requires_custom_code, multi_gpu_risk, dependency_risk, recommended_strategy, minimum_hardware, should_attempt_full_inference, and evidence.
3078
+ - `pi_implementation_plan.json` should include: selected_strategy, intended_files, endpoints, dependency_strategy, hardware_strategy, smoke_test_plan, fallback_plan, and blocker_policy.
3079
+ - If the model appears high-risk or not portable, say so in the plan rather than silently attempting a fragile fake success. The worker will still use INFERENCE_CONTRACT.json, TECHNICAL_BLOCKERS.json, and live validation as the final authority.
3080
+
3081
  Implementation contract:
3082
  - If IMPLEMENTATION_MODE is `full-inference-gated`, you are not allowed to silently replace generation with a placeholder and call it success.
3083
  - Try to implement the closest real inference path for the model card using evidence from README, model metadata, config files, and repo files.
 
3085
  - If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint.
3086
  - If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement.
3087
  - If the model requires special dependencies, include them only when needed and document risks.
3088
+ - Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels/Kernel Hub where relevant, Transformers AttentionInterface, Diffusers attention processors, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs.
3089
+ - Native kernel policy: if the model card or logs mention flash-attn, custom attention kernels, xformers, Triton kernels, fused ops, or CUDA/C++ extensions, do not blindly add source-built native packages to requirements.txt. Prefer runtime-compatible drop-in backends or compatible wheels first. Use `kernels` only when it directly matches the needed operation and the target hardware/runtime is plausible; document the selected backend in PI_SUMMARY.md and INFERENCE_CONTRACT.json.
3090
  - If real inference is impossible or unsafe in a Space, write TECHNICAL_BLOCKERS.json with concrete evidence for every blocker.
3091
 
3092
  Deliverables:
3093
+ - pi_feasibility_brief.json must summarize feasibility and risk before/alongside implementation.
3094
+ - pi_implementation_plan.json must summarize the selected implementation plan, endpoints, dependencies, and validation approach.
3095
  - app.py must boot on Hugging Face Spaces.
3096
  - app.py must expose health/api_name="health".
3097
  - If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic.
 
3179
  )
3180
 
3181
  def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
3182
+ """Normalize broad known-dangerous base and platform dependencies before upload.
3183
 
3184
+ Do not try to solve every dependency conflict here. Do not try to solve every model-specific dependency conflict here. Pi is
3185
+ responsible for reading concrete build logs and patching requirements when
3186
+ a Space build fails. The Factory only prevents obviously unsafe broad ranges
3187
+ and owns the Gradio/HF platform stack so old generated pins cannot conflict
3188
+ with the current Spaces runtime; concrete pip conflicts remain Pi repair work.
3189
  """
3190
  req_path = workspace / "requirements.txt"
3191
  if not req_path.exists():
 
3206
 
3207
  # Minimal base pins only. Do not globally pin diffusers here: newer model
3208
  # cards may legitimately need a recent Diffusers release. If diffusers causes
3209
+ # a pip conflict, Pi is responsible for concrete build-log repairs. Pi must repair from the concrete build log.
3210
  policy: dict[str, str] = {
3211
  "huggingface-hub": "huggingface_hub>=0.34.0,<2.0.0",
3212
  "transformers": "transformers>=4.51.0,<5.0.0",
3213
  }
3214
+ # Platform-owned dependencies are normalized more aggressively than model
3215
+ # dependencies. Gradio/HF runtime packages must stay current with Spaces and
3216
+ # Agentic Space Factory validation; Pi should patch app.py for modern Gradio
3217
+ # rather than pinning or downgrading this stack.
3218
+ platform_policy: dict[str, str] = {
3219
+ "gradio": "gradio",
3220
+ "gradio-client": "gradio-client",
3221
+ "huggingface-hub": policy["huggingface-hub"],
3222
+ "spaces": "spaces>=0.30",
3223
+ "hf-xet": "hf_xet",
3224
+ }
3225
  aliases = {
3226
  "huggingface_hub": "huggingface-hub",
3227
  "huggingface-hub": "huggingface-hub",
3228
  "transformers": "transformers",
3229
+ "gradio": "gradio",
3230
+ "gradio-client": "gradio-client",
3231
+ "gradio_client": "gradio-client",
3232
+ "spaces": "spaces",
3233
+ "hf-xet": "hf-xet",
3234
+ "hf_xet": "hf-xet",
3235
  }
3236
+ platform_owned = set(platform_policy)
3237
  seen_policy: set[str] = set()
3238
+ seen_platform: set[str] = set()
3239
  filtered: list[str] = []
3240
  changed = False
3241
+ removed_platform_pins: list[str] = []
3242
+ normalized_platform_lines: list[dict] = []
3243
 
3244
  for line in package_lines:
3245
  stripped = line.strip()
3246
  if stripped.startswith("#") or "://" in stripped or stripped.startswith((".", "/")):
3247
  filtered.append(line)
3248
  continue
3249
+ name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-")
3250
  canonical = aliases.get(name)
3251
+ if canonical in platform_owned:
3252
+ normalized = platform_policy[canonical]
3253
+ if stripped != normalized:
3254
+ changed = True
3255
+ removed_platform_pins.append(stripped)
3256
+ normalized_platform_lines.append({"from": stripped, "to": normalized, "package": canonical})
3257
+ if canonical not in seen_platform:
3258
+ filtered.append(normalized)
3259
+ seen_platform.add(canonical)
3260
+ if canonical in policy:
3261
+ seen_policy.add(canonical)
3262
+ else:
3263
+ changed = True
3264
+ continue
3265
  if canonical in policy:
3266
  if stripped != policy[canonical]:
3267
  changed = True
 
3279
  stable_policy_lines.append(policy[canonical])
3280
  changed = True
3281
 
3282
+ injected_platform_lines: list[str] = []
3283
+ app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore").lower() if (workspace / "app.py").exists() else ""
3284
+ if ("import gradio" in app_text or "from gradio" in app_text or "gr." in app_text) and "gradio" not in seen_platform:
3285
+ injected_platform_lines.append(platform_policy["gradio"])
3286
+ seen_platform.add("gradio")
3287
+ changed = True
3288
+ if ("gradio_client" in app_text or "from gradio_client" in app_text) and "gradio-client" not in seen_platform:
3289
+ injected_platform_lines.append(platform_policy["gradio-client"])
3290
+ seen_platform.add("gradio-client")
3291
+ changed = True
3292
+
3293
  torch_added = False
3294
+ if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines + injected_platform_lines, "torch"):
3295
  stable_policy_lines.append("torch>=2.0.0")
3296
  torch_added = True
3297
  changed = True
3298
 
3299
+ new_lines = prefix_lines + stable_policy_lines + injected_platform_lines + filtered
3300
  new = "\n".join(line for line in new_lines if line.strip()) + "\n"
3301
  if new != raw:
3302
  changed = True
3303
  if changed:
3304
  req_path.write_text(new, encoding="utf-8")
3305
+ policy_payload = {
3306
+ "schema_version": "1.0",
3307
+ "status": "normalized",
3308
+ "platform_owned_dependencies": ["gradio", "gradio-client", "huggingface_hub", "spaces", "hf_xet"],
3309
+ "removed_pins": removed_platform_pins,
3310
+ "normalized_platform_lines": normalized_platform_lines,
3311
+ "injected_platform_lines": injected_platform_lines,
3312
+ "base_policy": {"huggingface_hub": policy["huggingface-hub"], "transformers": policy["transformers"]},
3313
+ "base_policy_reason": "Avoid uncontrolled Transformers 5.x while preserving model-specific dependency choices for Pi to repair from build logs; require torch when app.py imports torch.",
3314
+ "torch_added": torch_added,
3315
+ "torch_policy": "torch>=2.0.0",
3316
+ "torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present",
3317
+ "model_dependency_policy": "Model-specific pins are preserved unless they hit a known deterministic guardrail; concrete conflicts remain Pi repair work from build logs.",
3318
+ "reason": "Gradio/HF runtime dependencies are owned by Agentic Space Factory / Spaces runtime and must not be downgraded by generated requirements.",
3319
+ }
3320
+ write_json(workspace / "requirements_policy.json", policy_payload)
3321
  append_event(
3322
  events_path,
3323
  "requirements_sanitize",
3324
  "success",
3325
+ "Normalized platform-owned Gradio/HF dependencies and broad base dependencies; concrete pip conflicts remain Pi repair work",
3326
+ policy_payload,
 
 
 
 
 
 
 
3327
  )
3328
 
 
3329
  def useful_log_signals(text: str) -> list[str]:
3330
  low = (text or "").lower()
3331
  signals = []
 
3809
 
3810
  You are not allowed to edit code in this diagnosis step. Your task is to decide the next action for the Factory.
3811
  Use the gist method: read logs first, identify the first actionable error, use the cheapest useful iteration rung, and require a live Gradio/API validation before success.
3812
+ When the first error is dependency-related, classify it with the ASF dependency contract in mind: Gradio/HF runtime packages are platform-owned, model-specific pins need evidence, torch/torchaudio are managed by Spaces, and non-pip-installable model code should be vendored.
3813
 
3814
  Write `REPAIR_DECISION.json` exactly as requested in INCIDENT_BRIEF.md. Do not patch files during this step.
3815
  """
 
3927
  - Preserve or restore a cheap `health` endpoint.
3928
  - Preserve the expected Gradio API endpoint when possible.
3929
  - Keep README metadata valid and short_description <= 60 chars.
3930
+ - Platform dependency policy: keep Gradio / Gradio Client / Hugging Face Hub / Spaces modern and worker-owned. Do not downgrade Gradio or add stale exact pins. If the failure involves a Gradio dependency conflict, remove the stale Gradio pin and patch app.py for modern Gradio instead of downgrading Gradio.
3931
+ - For model-specific dependencies, follow the gist method: patch the first concrete resolver error minimally, preserve justified model-card pins, and do not pin torch/torchaudio unless unavoidable and explicitly justified. If torchvision is needed, prefer leaving it unpinned so it resolves against the managed torch runtime.
3932
+ - If model code is not pip-installable, vendor the necessary code into the Space repo instead of relying on local paths, editable installs, or clone-time side effects in requirements.txt.
3933
+ - Do not prove dependency fixes with local venv installs or broad pip dry-runs; Space build logs are authoritative. Keep local checks cheap, such as `python -m py_compile app.py`.
3934
+ - Native kernel policy: for flash-attn/custom attention/xformers/Triton/fused CUDA errors, do not blindly add source-built packages. First consider PyTorch SDPA, compatible prebuilt wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors. If no compatible fallback exists, declare a blocker rather than forcing a fragile build.
3935
 
3936
  ## Required repair artifacts
3937
  Before modifying files, write `REPAIR_PLAN.md` with:
 
4012
  You are continuing the same build run, not starting a separate project.
4013
  This patch is allowed only because the diagnosis decision selected `patch_code`.
4014
  If `DEPENDENCY_ERROR_BRIEF.md` exists, treat it as evidence for the gist method: identify the first pip error, patch dependency pins minimally, and do not modify inference code unless the dependency fix alone cannot address that first error.
4015
+ Follow the ASF/gist dependency contract during repair: Gradio/HF runtime packages are platform-owned, model-specific pins must be evidence-backed, torch/torchaudio should remain managed by Spaces, and non-pip-installable model code should be vendored instead of referenced through fragile local paths.
4016
+ For native kernel failures such as flash-attn/custom attention/xformers/Triton/fused CUDA ops, do not blindly add source-built native packages. Prefer PyTorch SDPA, compatible wheels, HF Kernels/Kernel Hub, Transformers AttentionInterface, or Diffusers attention processors when they match the required operation. Use `kernels` only when directly justified and document the backend in REPAIR_SUMMARY.md / INFERENCE_CONTRACT.json.
4017
  Use the available HF token context to inspect private Hub resources when needed, but never print or persist token values.
4018
 
4019
  Critical method:
 
4089
  dependency_issue = extract_pip_dependency_issue(f"{current_error}\n{build_log}\n{runtime_log}")
4090
  if dependency_issue:
4091
  write_dependency_error_brief(workspace, run_dir, events_path, dependency_issue, build_log, runtime_log, current_error)
4092
+ write_repair_outcome(
4093
+ run_dir,
4094
+ events_path,
4095
+ repair_trigger="dependency_resolution_error",
4096
+ root_cause="dependency_resolution_error",
4097
+ initial_error=str(current_error)[:4000],
4098
+ patch_applied=False,
4099
+ upload_success=False,
4100
+ post_repair_validation="not_started",
4101
+ )
4102
  if apply_dependency_guardrail_repair(workspace, run_dir, events_path, current_error, build_log, runtime_log):
4103
+ write_repair_outcome(
4104
+ run_dir,
4105
+ events_path,
4106
+ repair_trigger="dependency_resolution_error",
4107
+ repair_decision="deterministic_dependency_guardrail",
4108
+ patch_applied=True,
4109
+ upload_success=False,
4110
+ post_repair_validation="not_started",
4111
+ )
4112
  append_event(events_path, "factory_rebuild", "started", "Re-uploading dependency-guardrailed workspace after pip resolver build error")
4113
  if not safe_same_code_reupload(api, workspace, target_space_id, token, run_dir, events_path, reason="dependency_guardrail_repair"):
4114
+ write_repair_outcome(run_dir, events_path, upload_success=False, post_repair_validation="not_started", failure_type="repair_upload_blocked_by_guardrail")
4115
  raise RuntimeError("Dependency guardrail rebuild skipped by restart guardrails")
4116
+ write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending")
4117
  append_event(events_path, "factory_rebuild", "success", "Dependency-guardrailed workspace uploaded; revalidating live Space")
4118
  append_event(events_path, "repair_validation", "started", "Revalidating after deterministic dependency repair")
4119
  write_live_status(run_dir, stage="live_validation", status="running", message="Waiting for Space runtime and health endpoint", data={"target_space": target_space_id})
4120
  try:
4121
+ write_auth_probe(run_dir, events_path, "before_initial_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"), raise_on_unsafe=True)
4122
  validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
4123
+ write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Deterministic dependency repair resolved the build blockage.")
4124
  append_event(events_path, "repair_validation", "success", "Deterministic dependency repair resolved the build blockage")
4125
  return validation
4126
  except Exception as exc:
4127
+ repair_class = classify_repair_validation_error(exc)
4128
+ write_repair_outcome(
4129
+ run_dir,
4130
+ events_path,
4131
+ post_repair_validation=repair_class["post_repair_validation"],
4132
+ failure_type=repair_class["failure_type"],
4133
+ final_user_message=repair_class["message"],
4134
+ validation_error=str(exc)[:4000],
4135
+ )
4136
+ if repair_class["terminal_status"] == "auth_refresh_required":
4137
+ append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]})
4138
+ raise AuthRefreshRequired(repair_class["message"])
4139
  current_error = f"{current_error}\n\nDependency guardrail rebuild did not resolve validation: {str(exc)[:4000]}"
4140
  collect_space_logs(target_space_id, token, run_dir, events_path)
4141
  append_event(events_path, "repair_validation", "failed", "Dependency guardrail rebuild did not resolve validation; falling back to Pi diagnosis", {"error": str(exc)[:4000]})
 
4173
  append_event(events_path, "factory_rebuild", "success", "Same-code workspace re-uploaded; revalidating live Space")
4174
  append_event(events_path, "repair_validation", "started", "Revalidating after same-code factory rebuild")
4175
  try:
4176
+ write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True)
4177
  validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
4178
+ write_repair_outcome(run_dir, events_path, repair_decision="factory_rebuild_same_code", patch_applied=False, upload_success=True, post_repair_validation="success", failure_type="", final_user_message="Same-code factory rebuild resolved the blockage.")
4179
  append_event(events_path, "repair_validation", "success", "Same-code factory rebuild resolved the blockage")
4180
  return validation
4181
  except Exception as exc:
4182
+ repair_class = classify_repair_validation_error(exc)
4183
+ write_repair_outcome(
4184
+ run_dir,
4185
+ events_path,
4186
+ repair_decision="factory_rebuild_same_code",
4187
+ patch_applied=False,
4188
+ upload_success=True,
4189
+ post_repair_validation=repair_class["post_repair_validation"],
4190
+ failure_type=repair_class["failure_type"],
4191
+ final_user_message=repair_class["message"],
4192
+ validation_error=str(exc)[:4000],
4193
+ )
4194
+ if repair_class["terminal_status"] == "auth_refresh_required":
4195
+ append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]})
4196
+ raise AuthRefreshRequired(repair_class["message"])
4197
  current_error = f"{current_error}\n\nSame-code factory rebuild did not resolve validation: {str(exc)[:4000]}"
4198
  collect_space_logs(target_space_id, token, run_dir, events_path)
4199
  append_event(events_path, "repair_validation", "failed", "Same-code factory rebuild did not resolve validation; re-diagnosing", {"error": str(exc)[:4000]})
 
4206
  raise RuntimeError("Automated recovery stopped after patch budget was exhausted")
4207
  budgets[action] -= 1
4208
  append_event(events_path, "repair", "started", "Pi diagnosis allows a minimal code patch", {"decision": decision})
4209
+ write_repair_outcome(
4210
+ run_dir,
4211
+ events_path,
4212
+ repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure",
4213
+ root_cause=(decision.get("classification") or {}).get("category") or "live_validation_failure",
4214
+ repair_decision="patch_code",
4215
+ decision=decision,
4216
+ patch_applied=False,
4217
+ upload_success=False,
4218
+ post_repair_validation="not_started",
4219
+ )
4220
  repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision)
4221
  if not repaired:
4222
+ write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation="not_started", failure_type="repair_patch_failed", final_user_message="Structured patch repair failed before redeploy.")
4223
  write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair")
4224
  append_event(events_path, "failure", "failed", "Structured patch repair failed before redeploy", {"decision": decision})
4225
  raise RuntimeError("Structured patch repair failed before redeploy")
4226
+ write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started")
4227
  append_event(events_path, "repair_upload", "started", "Uploading repaired workspace")
4228
+ write_auth_probe(run_dir, events_path, "before_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True)
4229
  upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
4230
+ write_repair_outcome(run_dir, events_path, upload_success=True, post_repair_validation="pending")
4231
  append_event(events_path, "repair_upload", "success", "Repaired workspace uploaded")
4232
  append_event(events_path, "repair_validation", "started", "Revalidating repaired Space")
4233
  try:
4234
+ write_auth_probe(run_dir, events_path, "before_repair_validation", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_validation"), raise_on_unsafe=True)
4235
  validation = validate_live_api(api, target_space_id, token, run_dir, events_path, timeout_s=1200)
4236
+ write_repair_outcome(run_dir, events_path, post_repair_validation="success", failure_type="", final_user_message="Repaired Space passed live API validation.")
4237
  append_event(events_path, "repair_validation", "success", "Repaired Space passed live API validation")
4238
  return validation
4239
  except Exception as exc:
4240
+ repair_class = classify_repair_validation_error(exc)
4241
+ write_repair_outcome(
4242
+ run_dir,
4243
+ events_path,
4244
+ post_repair_validation=repair_class["post_repair_validation"],
4245
+ failure_type=repair_class["failure_type"],
4246
+ final_user_message=repair_class["message"],
4247
+ validation_error=str(exc)[:4000],
4248
+ )
4249
+ if repair_class["terminal_status"] == "auth_refresh_required":
4250
+ append_event(events_path, "repair_validation", "auth_refresh_required", repair_class["message"], {"error": str(exc)[:4000]})
4251
+ raise AuthRefreshRequired(repair_class["message"])
4252
  current_error = f"{current_error}\n\nPatch repair did not resolve validation: {str(exc)[:4000]}"
4253
  collect_space_logs(target_space_id, token, run_dir, events_path)
4254
  append_event(events_path, "repair_validation", "failed", "Repair attempted, but validation still failed", {"error": str(exc)[:4000]})
 
4342
  }
4343
 
4344
  if blocker_detected:
4345
+ if isinstance(generation_smoke, dict) and generation_smoke.get("skip_reason") == "contract_declared_no_full_inference":
4346
+ status = "technical_blocker_boot_only"
4347
+ message = "Space boots, but Pi declared full inference unavailable; generation smoke was skipped by contract. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md."
4348
+ else:
4349
+ status = "technical_blocker"
4350
+ message = "Space boots, but full model inference was not implemented. See TECHNICAL_BLOCKERS.json / PI_SUMMARY.md."
4351
  elif implementation_mode in {"full-inference-gated", "full-inference-attempt"} and smoke_ok:
4352
  status = "full_inference_success"
4353
  message = "Space boots and a live generation smoke test passed. ZeroGPU duration recommendation was measured from real inference."
 
4438
  api = HfApi(token=token)
4439
  whoami = api.whoami(token=token)
4440
  append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})
4441
+ write_auth_probe(run_dir, events_path, "before_space_create", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_space_create"), raise_on_unsafe=True)
4442
 
4443
  append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})
4444
  info = api.model_info(model_id, token=token, files_metadata=True)
 
4477
  emit_pi_model_resolution(events_path, pi_model_resolution)
4478
  if not (workspace / "PI_SUMMARY.md").exists():
4479
  (workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8")
4480
+ write_pi_planning_review(workspace, run_dir, events_path, analysis, implementation_mode)
4481
 
4482
  app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")
4483
  if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text:
 
4522
  # starts directly on the requested hardware. If it fell back to CPU, the run
4523
  # remains valid but will be marked manual_hardware_required when inference
4524
  # signals indicate GPU is needed.
4525
+ write_auth_probe(run_dir, events_path, "before_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_upload"), raise_on_unsafe=True)
4526
  upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
4527
  write_artifact_manifest(run_dir, reason="workspace_uploaded")
4528
 
 
4543
  )
4544
  generation_smoke = None
4545
  if implementation_mode in {"full-inference-gated", "full-inference-attempt"}:
4546
+ contract_skip_reason = contract_declares_no_full_inference(workspace)
4547
+ if contract_skip_reason.get("declared"):
4548
+ generation_smoke = write_contract_skipped_generation_smoke(run_dir, events_path, expected_output_type, target_space_id, contract_skip_reason)
4549
+ else:
4550
+ try:
4551
+ generation_smoke = run_generation_smoke(target_space_id, token, run_dir, events_path, expected_output_type, workspace=workspace)
4552
+ except Exception as smoke_error:
4553
+ write_live_status(run_dir, stage="generation_smoke", status="failed", message="Live generation smoke test failed", data={"error": str(smoke_error)[:2000]})
4554
+ generation_smoke = {
4555
+ "status": "failed",
4556
+ "target_space": target_space_id,
4557
+ "expected_output_type": expected_output_type,
4558
+ "error": str(smoke_error)[:4000],
4559
+ "next_action": "Use Prefill Space Test after the run finishes to retry with schema-adjusted arguments.",
4560
+ **measured_zero_gpu_recommendation(None),
4561
+ }
4562
+ generation_smoke.update(classify_generation_smoke_error(smoke_error))
4563
+ write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
4564
+ append_event(events_path, "generation_smoke", "failed", "Live generation smoke test failed; ZeroGPU duration was not measured", generation_smoke)
4565
  else:
4566
  generation_smoke = measured_zero_gpu_recommendation(None) | {"status": "skipped", "expected_output_type": expected_output_type}
4567
  write_json(run_dir / "tests" / "generation_smoke.json", generation_smoke)
 
4571
  # hardware requests failed, classify the run honestly as needing manual
4572
  # hardware instead of pretending CPU/default hardware is enough. the existing-Space validation workflow
4573
  # can then smoke-test generation after the user sets a GPU manually.
4574
+ manual_hw_required = selected_hardware == "default-cpu-or-existing" and inference_gate.get("status") not in {"technical_blocker", "technical_blocker_boot_only", "health_only"} and (
4575
  inference_gate.get("implementation_signals", {}).get("has_spaces_gpu")
4576
  or inference_gate.get("implementation_signals", {}).get("has_torch")
4577
  or any((a.get("manual_action_required") for a in hardware_attempts if isinstance(a, dict)))
 
4688
  collect_pi_traces(run_dir, events_path)
4689
  except Exception:
4690
  pass
4691
+ terminal_status = "auth_refresh_required" if isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc) else "failed"
4692
+ details = {"error": str(exc)}
4693
+ if terminal_status == "auth_refresh_required":
4694
+ details["failure_type"] = "auth_refresh_required"
4695
+ repair_outcome = load_json_if_exists(run_dir / "repair_outcome.json") if (run_dir / "repair_outcome.json").exists() else {}
4696
+ if isinstance(repair_outcome, dict) and repair_outcome:
4697
+ details["repair_outcome"] = repair_outcome
4698
+ details["failure_type"] = repair_outcome.get("failure_type") or details.get("failure_type", "")
4699
+ fail(run_dir, events_path, "Universal model-card builder worker failed", details, status=terminal_status)
4700
 
4701
 
4702
  if __name__ == "__main__":
 
4706
 
4707
 
4708
  VALIDATE_EXISTING_SPACE_WORKER_SCRIPT = r'''
4709
+ import base64
4710
  import json
4711
  import os
4712
  import re
 
4924
  stripped = line.strip()
4925
  if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
4926
  continue
4927
+ name = re.split(r"[<>=!~;\[]", stripped, maxsplit=1)[0].strip().lower().replace("_", "-")
4928
  if name == wanted:
4929
  return True
4930
  return False
 
5110
  return "cancelled"
5111
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
5112
  return "manual_action_required"
5113
+ if status in {"technical_blocker", "technical_blocker_boot_only"}:
5114
  return "technical_blocker"
5115
  if full_inference_verified:
5116
  return "success"
 
5321
  return {"enabled": True, "written": True, "path": str(dest), "publish_mode": "backend"}
5322
 
5323
 
5324
+
5325
+
5326
+ class AuthRefreshRequired(RuntimeError):
5327
+ """Raised when an OAuth/JWT token is expired or too close to expiry."""
5328
+
5329
+
5330
+ def _decode_jwt_claims_unverified(token: str) -> dict:
5331
+ """Decode JWT claims without verification, only to inspect non-sensitive expiry metadata.
5332
+
5333
+ This never returns or writes the raw token. Opaque PAT-style tokens are supported by
5334
+ returning an empty dict so they remain usable with an `unknown` expiry state.
5335
+ """
5336
+ try:
5337
+ parts = (token or "").split(".")
5338
+ if len(parts) < 2:
5339
+ return {}
5340
+ payload = parts[1]
5341
+ payload += "=" * ((4 - len(payload) % 4) % 4)
5342
+ raw = base64.urlsafe_b64decode(payload.encode("utf-8"))
5343
+ data = json.loads(raw.decode("utf-8"))
5344
+ return data if isinstance(data, dict) else {}
5345
+ except Exception:
5346
+ return {}
5347
+
5348
+
5349
+ def token_expiry_status(token: str, *, minimum_required_seconds: int = 0) -> dict:
5350
+ issued_at = now()
5351
+ if not token:
5352
+ return {
5353
+ "schema_version": "auth_status.v1",
5354
+ "checked_at": issued_at,
5355
+ "token_present": False,
5356
+ "token_kind": "missing",
5357
+ "expiry_known": False,
5358
+ "status": "missing",
5359
+ "safe_for_phase": False,
5360
+ "minimum_required_seconds": minimum_required_seconds,
5361
+ }
5362
+ claims = _decode_jwt_claims_unverified(token)
5363
+ exp = claims.get("exp") if isinstance(claims, dict) else None
5364
+ iat = claims.get("iat") if isinstance(claims, dict) else None
5365
+ token_kind = "oauth_jwt" if exp is not None else ("jwt_without_exp" if claims else "opaque_or_unknown")
5366
+ payload = {
5367
+ "schema_version": "auth_status.v1",
5368
+ "checked_at": issued_at,
5369
+ "token_present": True,
5370
+ "token_kind": token_kind,
5371
+ "expiry_known": exp is not None,
5372
+ "minimum_required_seconds": int(minimum_required_seconds or 0),
5373
+ "token_value": "[REDACTED]",
5374
+ }
5375
+ if iat is not None:
5376
+ try:
5377
+ payload["issued_at"] = datetime.fromtimestamp(int(iat), tz=timezone.utc).isoformat()
5378
+ except Exception:
5379
+ pass
5380
+ if exp is None:
5381
+ payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "unknown_expiry"})
5382
+ return payload
5383
+ try:
5384
+ exp_int = int(exp)
5385
+ seconds_left = exp_int - int(time.time())
5386
+ payload.update({
5387
+ "expires_at": datetime.fromtimestamp(exp_int, tz=timezone.utc).isoformat(),
5388
+ "seconds_until_expiry": seconds_left,
5389
+ })
5390
+ except Exception:
5391
+ payload.update({"status": "unknown", "safe_for_phase": True, "auth_risk": "invalid_exp_claim"})
5392
+ return payload
5393
+ if seconds_left <= 0:
5394
+ payload.update({"status": "expired", "safe_for_phase": False, "auth_risk": "expired"})
5395
+ elif minimum_required_seconds and seconds_left < minimum_required_seconds:
5396
+ payload.update({"status": "expires_soon", "safe_for_phase": False, "auth_risk": "expires_before_phase_budget"})
5397
+ else:
5398
+ payload.update({"status": "ok", "safe_for_phase": True, "auth_risk": "ok"})
5399
+ return payload
5400
+
5401
+
5402
+ def write_auth_probe(run_dir: Path, events_path: Path | None, phase: str, token: str, *, minimum_required_seconds: int = 0, raise_on_unsafe: bool = False) -> dict:
5403
+ payload = token_expiry_status(token, minimum_required_seconds=minimum_required_seconds)
5404
+ payload["phase"] = phase
5405
+ safe_payload = {k: v for k, v in payload.items() if k != "token_value"}
5406
+ try:
5407
+ probes_dir = run_dir / "auth_probes"
5408
+ probes_dir.mkdir(parents=True, exist_ok=True)
5409
+ write_json(probes_dir / f"{phase}.json", safe_payload)
5410
+ write_json(run_dir / "auth_status.json", safe_payload)
5411
+ except Exception:
5412
+ pass
5413
+ status = str(payload.get("status") or "unknown")
5414
+ if events_path:
5415
+ event_status = "success" if payload.get("safe_for_phase") else "failed"
5416
+ if status == "unknown":
5417
+ event_status = "warning"
5418
+ append_event(
5419
+ events_path,
5420
+ "auth_probe",
5421
+ event_status,
5422
+ f"HF OAuth/token expiry check for {phase}: {status}",
5423
+ safe_payload,
5424
+ )
5425
+ if raise_on_unsafe and not payload.get("safe_for_phase"):
5426
+ raise AuthRefreshRequired(f"HF auth token is {status} before {phase}; refresh sign-in before continuing.")
5427
+ return payload
5428
+
5429
+
5430
+ def is_auth_expired_error(error: Exception | str) -> bool:
5431
+ text = str(error or "").lower()
5432
+ return any(marker in text for marker in [
5433
+ "oauth token has expired",
5434
+ "exp claim timestamp check failed",
5435
+ "token has expired",
5436
+ "jwt expired",
5437
+ ])
5438
+
5439
+
5440
+ def minimum_auth_seconds_for_phase(phase: str) -> int:
5441
+ env_key = "ASF_AUTH_MIN_SECONDS_" + re.sub(r"[^A-Z0-9]+", "_", phase.upper()).strip("_")
5442
+ raw = os.environ.get(env_key) or os.environ.get("ASF_AUTH_MIN_SECONDS", "")
5443
+ if raw:
5444
+ try:
5445
+ return max(0, int(raw))
5446
+ except Exception:
5447
+ pass
5448
+ defaults = {
5449
+ "before_space_create": 900,
5450
+ "before_upload": 900,
5451
+ "before_initial_validation": 1800,
5452
+ "before_repair": 1800,
5453
+ "before_repair_upload": 1800,
5454
+ "before_repair_validation": 1800,
5455
+ "before_linked_space_test": 1800,
5456
+ }
5457
+ return defaults.get(phase, 0)
5458
+
5459
  def ensure_hf_token_context(run_dir: Path, events_path: Path | None = None) -> dict:
5460
  """Ensure Pi and HF tooling see the same private HF token safely.
5461
 
 
5469
  os.environ.setdefault("HF_TOKEN", token)
5470
  os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", token)
5471
  os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
5472
+ expiry = token_expiry_status(token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_initial_validation"))
5473
  payload = {
5474
  "hf_token_present": bool(token),
5475
  "hf_token_length": len(token) if token else 0,
 
5478
  "bucket_source": os.environ.get("BUCKET_SOURCE") or "",
5479
  "target_space_id": os.environ.get("TARGET_SPACE_ID") or "",
5480
  "token_value": "[REDACTED]" if token else "",
5481
+ "token_kind": expiry.get("token_kind"),
5482
+ "expiry_known": expiry.get("expiry_known"),
5483
+ "expires_at": expiry.get("expires_at"),
5484
+ "seconds_until_expiry_at_job_start": expiry.get("seconds_until_expiry"),
5485
+ "auth_risk": expiry.get("auth_risk"),
5486
+ "safe_for_long_build": bool(expiry.get("safe_for_phase")),
5487
+ "minimum_required_seconds": expiry.get("minimum_required_seconds"),
5488
  }
5489
  run_dir.mkdir(parents=True, exist_ok=True)
5490
  write_json(run_dir / "token_context.json", payload)
 
6792
  write_json(state_path, {"run_id": run_id, "kind": "linked_space_validation", "status": "running", "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "created_by": username, "updated_at": now()})
6793
  if not token:
6794
  raise RuntimeError("HF_TOKEN is missing")
6795
+ ensure_hf_token_context(run_dir, events_path)
6796
+ write_auth_probe(run_dir, events_path, "before_linked_space_test", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_linked_space_test"), raise_on_unsafe=True)
6797
  if not TARGET_RE.match(target_space_id):
6798
  raise ValueError("TARGET_SPACE_ID must look like owner/space-name")
6799
  if not parent_build_run_id:
 
6894
  append_event(events_path, "done", "full_inference_success", "Existing Space validation completed", {"latency_seconds": smoke.get("latency_seconds")})
6895
  except Exception as exc:
6896
  collect_space_logs(target_space_id, token or "", run_dir, events_path)
6897
+ auth_blocked = isinstance(exc, AuthRefreshRequired) or is_auth_expired_error(exc)
6898
  details = {"error": str(exc)[:4000]}
6899
+ if auth_blocked:
6900
+ details["status_reason"] = "auth_refresh_required"
6901
+ details["failure_type"] = "auth_refresh_required"
6902
+ terminal_status = "auth_refresh_required" if auth_blocked else "failed"
6903
+ failure_state = {"run_id": run_id, "kind": "linked_space_validation", "status": terminal_status, "message": "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else str(exc), "target_space": target_space_id, "parent_build_run_id": parent_build_run_id, "source_kind": "build_run_prefill", "details": details, "updated_at": now()}
6904
  write_json(state_path, failure_state)
6905
+ write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed")
6906
  if parent_build_run_id:
6907
  parent_dir = output_root / "runs" / parent_build_run_id
6908
  parent_dir.mkdir(parents=True, exist_ok=True)
6909
+ failed_status = {"status": terminal_status, "validation_run_id": run_id, "parent_build_run_id": parent_build_run_id, "target_space": target_space_id, "details": details, "updated_at": now(), "effective_status": "unchanged"}
6910
  linked_path = parent_dir / "linked_validations.json"
6911
  linked = read_json(linked_path, {"parent_build_run_id": parent_build_run_id, "validations": []}) or {"parent_build_run_id": parent_build_run_id, "validations": []}
6912
  validations = linked.get("validations") if isinstance(linked, dict) else []
 
6921
  write_json(parent_dir / "manual_validation_status.json", failed_status)
6922
  write_json(linked_path, linked)
6923
  (run_dir / "report.md").write_text(f"# Existing Space Validation Failed\n\n```json\n{json.dumps(details, indent=2, ensure_ascii=False)}\n```\n", encoding="utf-8")
6924
+ append_event(events_path, "failure", "failed", "HF OAuth token expired or will expire too soon; refresh sign-in before continuing" if auth_blocked else "Existing Space validation failed", details)
6925
  try:
6926
  publish_eval_record(run_dir, phase="failure", events_path=events_path)
6927
  append_event(events_path, "anonymous_eval", "success", "Published anonymized failed validation evaluation record locally for backend archive publishing", {"enabled": True, "publish_mode": "backend", "archive_publish_confirmed": False})
 
6930
  # v190.33: also rewrite failure summary as the final visible file update
6931
  # for linked validations, so a failed validation cannot remain listed
6932
  # as running after the worker has already terminalized state.json.
6933
+ write_final_summary(run_dir, failure_state, {}, {"status": terminal_status, "failure_type": details.get("failure_type", "linked_validation_failed") if isinstance(details, dict) else "linked_validation_failed"}, status=terminal_status, message=str(failure_state.get("message") or details.get("error", "Existing Space validation failed")) if isinstance(details, dict) else "Existing Space validation failed")
6934
+ append_event(events_path, "summary_write", "success", "Wrote terminal failed validation summary", {"status": terminal_status})
6935
  raise SystemExit(1)
6936
 
6937