fffiloni commited on
Commit
da792f9
·
verified ·
1 Parent(s): 50e67f0

Upload 16 files

Browse files
src/bucket.py CHANGED
@@ -1820,6 +1820,7 @@ def read_run_bundle(run_id: str, *, bucket_source: str, token: str | None = None
1820
  "build_error_observation": _safe_read_json(f"{paths.root}/build_error_observation.json", token=token),
1821
  "repair_decision": _safe_read_json(f"{paths.root}/repair/REPAIR_DECISION.json", token=token),
1822
  "blockage": _safe_read_json(f"{paths.root}/repair/BLOCKAGE.json", token=token),
 
1823
  "repair_attempts": _safe_read_json(f"{paths.root}/repair/repair_attempts.json", token=token),
1824
  "artifact_manifest": _safe_read_json(f"{paths.root}/artifact_manifest.json", token=token),
1825
  "files": _list_run_files(run_id, bucket_source=bucket_source, token=token) if include_heavy else [],
@@ -1906,6 +1907,7 @@ def _run_list_summary_needs_enrichment(run_id: str, summary_file: dict[str, Any]
1906
  "technical_blocker",
1907
  "technical_blocker_boot_only",
1908
  "blocked",
 
1909
  "failed",
1910
  "failure",
1911
  "error",
@@ -1981,7 +1983,7 @@ def _build_run_list_item_from_files(run_id: str, *, root: str, bucket_source: st
1981
  item["effective_status"] = post_build_validation_status.get("effective_status") or item.get("effective_status")
1982
  if linked_validations:
1983
  rows = linked_validations.get("validations") if isinstance(linked_validations, dict) else []
1984
- terminal_tokens = {"success", "passed", "succeeded", "full_inference_success", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation", "manual_validation_passed", "manual_validated", "partial_validation", "manual_hardware_required", "generated_needs_manual_hardware", "technical_blocker", "technical_blocker_boot_only", "blocked", "completed_with_warnings", "failed", "failure", "error", "auth_refresh_required", "stale", "stopped", "cancelled", "canceled"}
1985
  accounted_rows = [row for row in rows if isinstance(row, dict) and str(row.get("status") or row.get("validation_status") or row.get("result_status") or row.get("effective_status") or "").lower() in terminal_tokens] if isinstance(rows, list) else []
1986
  item["linked_validations"] = linked_validations
1987
  item["linked_validation_count"] = len(accounted_rows)
 
1820
  "build_error_observation": _safe_read_json(f"{paths.root}/build_error_observation.json", token=token),
1821
  "repair_decision": _safe_read_json(f"{paths.root}/repair/REPAIR_DECISION.json", token=token),
1822
  "blockage": _safe_read_json(f"{paths.root}/repair/BLOCKAGE.json", token=token),
1823
+ "provider_quota_blocker": _safe_read_json(f"{paths.root}/PROVIDER_QUOTA_BLOCKER.json", token=token) or _safe_read_json(f"{paths.root}/provider_quota_blocker.json", token=token),
1824
  "repair_attempts": _safe_read_json(f"{paths.root}/repair/repair_attempts.json", token=token),
1825
  "artifact_manifest": _safe_read_json(f"{paths.root}/artifact_manifest.json", token=token),
1826
  "files": _list_run_files(run_id, bucket_source=bucket_source, token=token) if include_heavy else [],
 
1907
  "technical_blocker",
1908
  "technical_blocker_boot_only",
1909
  "blocked",
1910
+ "provider_quota_blocked",
1911
  "failed",
1912
  "failure",
1913
  "error",
 
1983
  item["effective_status"] = post_build_validation_status.get("effective_status") or item.get("effective_status")
1984
  if linked_validations:
1985
  rows = linked_validations.get("validations") if isinstance(linked_validations, dict) else []
1986
+ terminal_tokens = {"success", "passed", "succeeded", "full_inference_success", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation", "manual_validation_passed", "manual_validated", "partial_validation", "manual_hardware_required", "generated_needs_manual_hardware", "technical_blocker", "technical_blocker_boot_only", "blocked", "completed_with_warnings", "failed", "failure", "error", "auth_refresh_required", "provider_quota_blocked", "stale", "stopped", "cancelled", "canceled"}
1987
  accounted_rows = [row for row in rows if isinstance(row, dict) and str(row.get("status") or row.get("validation_status") or row.get("result_status") or row.get("effective_status") or "").lower() in terminal_tokens] if isinstance(rows, list) else []
1988
  item["linked_validations"] = linked_validations
1989
  item["linked_validation_count"] = len(accounted_rows)
src/effective_status.py CHANGED
@@ -43,7 +43,8 @@ MANUAL_BUILD_TOKENS = {
43
  "manual_action_required",
44
  "waiting_manual_action",
45
  }
46
- BLOCKED_BUILD_TOKENS = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
 
47
 
48
 
49
  def _lower(value: Any) -> str:
@@ -193,6 +194,8 @@ def _normalize_build_status(value: Any) -> str:
193
  return "full_inference_success"
194
  if status in {"repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}:
195
  return "failed"
 
 
196
  if status in {"cancelled", "canceled"}:
197
  return "stopped"
198
  if status in {"full_inference_candidate_health_passed", "health_only", "partial", "completed_with_warnings"}:
@@ -295,6 +298,8 @@ def compute_effective_run_status(
295
  display_label = "Partial validation"
296
  elif display_status == "manual_hardware_required":
297
  display_label = "Manual hardware required"
 
 
298
  elif display_status in BLOCKED_BUILD_TOKENS:
299
  display_label = "Diagnostic Space"
300
  elif display_status in FAILED_BUILD_TOKENS:
 
43
  "manual_action_required",
44
  "waiting_manual_action",
45
  }
46
+ BLOCKED_BUILD_TOKENS = {"technical_blocker", "technical_blocker_boot_only", "blocked", "provider_quota_blocked"}
47
+ PROVIDER_QUOTA_TOKENS = {"provider_quota_blocked"}
48
 
49
 
50
  def _lower(value: Any) -> str:
 
194
  return "full_inference_success"
195
  if status in {"repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}:
196
  return "failed"
197
+ if status in {"provider_quota_blocked"}:
198
+ return "provider_quota_blocked"
199
  if status in {"cancelled", "canceled"}:
200
  return "stopped"
201
  if status in {"full_inference_candidate_health_passed", "health_only", "partial", "completed_with_warnings"}:
 
298
  display_label = "Partial validation"
299
  elif display_status == "manual_hardware_required":
300
  display_label = "Manual hardware required"
301
+ elif display_status == "provider_quota_blocked":
302
+ display_label = "Provider quota blocked"
303
  elif display_status in BLOCKED_BUILD_TOKENS:
304
  display_label = "Diagnostic Space"
305
  elif display_status in FAILED_BUILD_TOKENS:
src/eval_archive.py CHANGED
@@ -9,7 +9,7 @@ from .bucket import RunPaths, read_json, read_text, write_json
9
  from .eval_config import effective_eval_config
10
  from .security import redact
11
 
12
- _TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"}
13
 
14
 
15
  def _now() -> str:
 
9
  from .eval_config import effective_eval_config
10
  from .security import redact
11
 
12
+ _TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required", "provider_quota_blocked"}
13
 
14
 
15
  def _now() -> str:
src/progress.py CHANGED
@@ -96,6 +96,7 @@ STEP_LABELS = {
96
  }
97
 
98
  STEP_ALIASES = {
 
99
  "bucket_ready": "bootstrap",
100
  "job_launched": "bootstrap",
101
  "hardware_preferred": "hardware_strategy",
@@ -129,6 +130,7 @@ PARTIAL_STATUSES = {
129
  }
130
  RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
131
  FAILED_STATUSES = {"failed", "error", "failure", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
 
132
  CANCELLED_STATUSES = {"cancelled", "canceled"}
133
 
134
 
@@ -150,7 +152,7 @@ def _canonical_step(step: str | None) -> str | None:
150
 
151
  def _is_terminal_status(status: str | None) -> bool:
152
  s = str(status or "").lower()
153
- return s in DONE_STATUSES or s in PARTIAL_STATUSES or s in FAILED_STATUSES or s in CANCELLED_STATUSES
154
 
155
 
156
  def _progress_for_index(index: int) -> int:
@@ -243,6 +245,8 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
243
  terminal_status = "cancelled"
244
  elif status_from_state in PARTIAL_STATUSES:
245
  terminal_status = status_from_state
 
 
246
  elif status_from_state in DONE_STATUSES:
247
  terminal_status = status_from_state
248
  elif status_from_state in FAILED_STATUSES:
@@ -318,7 +322,7 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
318
 
319
  if overall_status in FAILED_STATUSES:
320
  progress = max(_progress_for_index(furthest_index), 3)
321
- elif overall_status in CANCELLED_STATUSES:
322
  progress = max(_progress_for_index(furthest_index), 3)
323
  elif overall_status in DONE_STATUSES or overall_status in PARTIAL_STATUSES or current_step == "done":
324
  progress = 100
@@ -350,7 +354,7 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
350
  "current_step": current_step,
351
  "current_step_label": STEP_LABELS.get(current_step, current_step),
352
  "last_event": last_message or "No events yet",
353
- "visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))),
354
  "elapsed_seconds": max(0, elapsed),
355
  "eta_seconds": None,
356
  "timeline": timeline,
 
96
  }
97
 
98
  STEP_ALIASES = {
99
+ "provider_quota": "pi_run",
100
  "bucket_ready": "bootstrap",
101
  "job_launched": "bootstrap",
102
  "hardware_preferred": "hardware_strategy",
 
130
  }
131
  RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
132
  FAILED_STATUSES = {"failed", "error", "failure", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
133
+ BLOCKED_STATUSES = {"provider_quota_blocked"}
134
  CANCELLED_STATUSES = {"cancelled", "canceled"}
135
 
136
 
 
152
 
153
  def _is_terminal_status(status: str | None) -> bool:
154
  s = str(status or "").lower()
155
+ return s in DONE_STATUSES or s in PARTIAL_STATUSES or s in FAILED_STATUSES or s in BLOCKED_STATUSES or s in CANCELLED_STATUSES
156
 
157
 
158
  def _progress_for_index(index: int) -> int:
 
245
  terminal_status = "cancelled"
246
  elif status_from_state in PARTIAL_STATUSES:
247
  terminal_status = status_from_state
248
+ elif status_from_state in BLOCKED_STATUSES:
249
+ terminal_status = status_from_state
250
  elif status_from_state in DONE_STATUSES:
251
  terminal_status = status_from_state
252
  elif status_from_state in FAILED_STATUSES:
 
322
 
323
  if overall_status in FAILED_STATUSES:
324
  progress = max(_progress_for_index(furthest_index), 3)
325
+ elif overall_status in CANCELLED_STATUSES or overall_status in BLOCKED_STATUSES:
326
  progress = max(_progress_for_index(furthest_index), 3)
327
  elif overall_status in DONE_STATUSES or overall_status in PARTIAL_STATUSES or current_step == "done":
328
  progress = 100
 
354
  "current_step": current_step,
355
  "current_step_label": STEP_LABELS.get(current_step, current_step),
356
  "last_event": last_message or "No events yet",
357
+ "visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES or overall_status in BLOCKED_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))),
358
  "elapsed_seconds": max(0, elapsed),
359
  "eta_seconds": None,
360
  "timeline": timeline,
src/timeline_model.py CHANGED
@@ -31,7 +31,7 @@ PHASE_LABELS = {
31
  PHASE_STEPS = {
32
  "start": {"token_context", "bootstrap", "dependencies", "auth", "workspace", "node", "pi_install"},
33
  "model": {"model_analysis", "model_prescan"},
34
- "agent": {"pi_config", "pi_run", "traces", "pi_model_resolution", "pi_verification"},
35
  "hardware": {"hardware_strategy", "create_space_hardware", "create_space"},
36
  "deploy": {"metadata_sanitize", "requirements_sanitize", "upload_files", "space_logs", "space_runtime", "live_wait"},
37
  "live_validation": {"endpoint_discovery", "api_validation", "generation_smoke", "inference_gate"},
@@ -51,7 +51,7 @@ PHASE_STEPS = {
51
  "repair_validation",
52
  },
53
  "archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
54
- "done": {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"},
55
  }
56
 
57
  STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
@@ -62,7 +62,8 @@ FAILED_STATUSES = {"failed", "failure", "error"}
62
  RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
63
  CANCELLED_STATUSES = {"cancelled", "canceled"}
64
  MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
65
- BLOCKED_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
 
66
  AUTH_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
67
 
68
  TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status")
@@ -783,6 +784,8 @@ def _normalize_verdict_token(value: Any) -> str:
783
  return "partial_validation"
784
  if normalized in MANUAL_STATUSES:
785
  return "manual_action_required"
 
 
786
  if normalized in BLOCKED_STATUSES:
787
  return "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
788
  if normalized in FAILED_STATUSES:
@@ -846,10 +849,14 @@ def _final_visual_status(verdict: str) -> str:
846
  return "success"
847
  if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
848
  return "warn"
 
 
849
  if verdict in {"failed", "failure", "technical_blocker", "technical_blocker_boot_only"}:
850
  return "error"
851
  if verdict == "auth_refresh_required":
852
  return "warn"
 
 
853
  if verdict in {"manual_action_required", "cancelled"}:
854
  return "stopped"
855
  return "running"
@@ -866,6 +873,8 @@ def _status_label(verdict: str) -> tuple[str, str]:
866
  return "Technical blocker", "The run found a technical blocker."
867
  if verdict == "auth_refresh_required":
868
  return "Auth refresh required", "HF OAuth expired or is too close to expiry; sign in again before retrying validation."
 
 
869
  if verdict == "manual_action_required":
870
  return "Manual action required", "The run needs user action before validation can continue."
871
  if verdict == "failed":
 
31
  PHASE_STEPS = {
32
  "start": {"token_context", "bootstrap", "dependencies", "auth", "workspace", "node", "pi_install"},
33
  "model": {"model_analysis", "model_prescan"},
34
+ "agent": {"pi_config", "pi_run", "provider_quota", "traces", "pi_model_resolution", "pi_verification"},
35
  "hardware": {"hardware_strategy", "create_space_hardware", "create_space"},
36
  "deploy": {"metadata_sanitize", "requirements_sanitize", "upload_files", "space_logs", "space_runtime", "live_wait"},
37
  "live_validation": {"endpoint_discovery", "api_validation", "generation_smoke", "inference_gate"},
 
51
  "repair_validation",
52
  },
53
  "archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
54
+ "done": {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "provider_quota", "manual_hardware_required"},
55
  }
56
 
57
  STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
 
62
  RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
63
  CANCELLED_STATUSES = {"cancelled", "canceled"}
64
  MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
65
+ BLOCKED_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked", "provider_quota_blocked"}
66
+ PROVIDER_QUOTA_STATUSES = {"provider_quota_blocked"}
67
  AUTH_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
68
 
69
  TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status")
 
784
  return "partial_validation"
785
  if normalized in MANUAL_STATUSES:
786
  return "manual_action_required"
787
+ if normalized in PROVIDER_QUOTA_STATUSES:
788
+ return "provider_quota_blocked"
789
  if normalized in BLOCKED_STATUSES:
790
  return "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
791
  if normalized in FAILED_STATUSES:
 
849
  return "success"
850
  if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
851
  return "warn"
852
+ if verdict in {"provider_quota_blocked"}:
853
+ return "warn"
854
  if verdict in {"failed", "failure", "technical_blocker", "technical_blocker_boot_only"}:
855
  return "error"
856
  if verdict == "auth_refresh_required":
857
  return "warn"
858
+ if verdict == "provider_quota_blocked":
859
+ return "warn"
860
  if verdict in {"manual_action_required", "cancelled"}:
861
  return "stopped"
862
  return "running"
 
873
  return "Technical blocker", "The run found a technical blocker."
874
  if verdict == "auth_refresh_required":
875
  return "Auth refresh required", "HF OAuth expired or is too close to expiry; sign in again before retrying validation."
876
+ if verdict == "provider_quota_blocked":
877
+ return "Provider quota blocked", "HF Inference Providers quota or monthly spending limit blocked Pi before demo generation."
878
  if verdict == "manual_action_required":
879
  return "Manual action required", "The run needs user action before validation can continue."
880
  if verdict == "failed":
src/version.py CHANGED
@@ -1,7 +1,7 @@
1
  from __future__ import annotations
2
 
3
- ASF_APP_VERSION = "v198.26.127"
4
- ASF_RELEASE_NAME = "Agentic Space Factory v198.26.127"
5
 
6
 
7
  def resolve_app_version(value: str | None = None) -> str:
 
1
  from __future__ import annotations
2
 
3
+ ASF_APP_VERSION = "v198.26.130"
4
+ ASF_RELEASE_NAME = "Agentic Space Factory v198.26.130"
5
 
6
 
7
  def resolve_app_version(value: str | None = None) -> str:
src/view_models.py CHANGED
@@ -44,11 +44,12 @@ MANUAL_RAW_STATUSES = {
44
  FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
45
  CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
46
  BLOCKED_RAW_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
 
47
  RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
48
  AUTH_RAW_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
49
 
50
  AUTHORITATIVE_TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status", "post_repair_validation", "failure_type")
51
- TERMINAL_DISPLAY_STATUSES = SUCCESS_RAW_STATUSES | PARTIAL_RAW_STATUSES | MANUAL_RAW_STATUSES | FAILED_RAW_STATUSES | CANCELLED_RAW_STATUSES | BLOCKED_RAW_STATUSES | AUTH_RAW_STATUSES | {"stale", "stopped", "succeeded", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation"}
52
 
53
  STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
54
 
@@ -203,7 +204,7 @@ def _first_authoritative_terminal_status(bundle: dict[str, Any]) -> tuple[str, s
203
  for key in AUTHORITATIVE_TERMINAL_SOURCE_KEYS:
204
  normalized = _normalize_terminal_status(source.get(key))
205
  if normalized:
206
- if normalized in BLOCKED_RAW_STATUSES | FAILED_RAW_STATUSES | MANUAL_RAW_STATUSES | AUTH_RAW_STATUSES | CANCELLED_RAW_STATUSES | {"stopped", "stale"}:
207
  return normalized, f"{source_name}.{key}"
208
  if normalized == "full_inference_success" and blocking_seen:
209
  continue
@@ -245,6 +246,8 @@ def _status_model_from_terminal_status(status: str, *, source: str = "") -> dict
245
  global_status, verdict = "partial", normalized if normalized not in {"partial", "health_only", "full_inference_candidate_health_passed", "completed_with_warnings"} else "partial_validation"
246
  elif normalized in MANUAL_RAW_STATUSES:
247
  global_status, verdict = "waiting_manual_action", "manual_action_required"
 
 
248
  elif normalized in BLOCKED_RAW_STATUSES:
249
  global_status, verdict = "blocked", "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
250
  elif normalized in FAILED_RAW_STATUSES:
@@ -310,6 +313,9 @@ def normalize_run_status(
310
  elif manual:
311
  global_status = "waiting_manual_action"
312
  verdict = "manual_action_required"
 
 
 
313
  elif blocker:
314
  global_status = "blocked"
315
  verdict = "technical_blocker_boot_only" if "technical_blocker_boot_only" in statuses else "technical_blocker"
@@ -479,7 +485,12 @@ def build_diagnostics(bundle: dict[str, Any], status_model: dict[str, Any], phas
479
  issue_title = ""
480
  issue_detail = ""
481
  issue_status = ""
482
- if blocker_items:
 
 
 
 
 
483
  first = blocker_items[0]
484
  issue_title = _first_nonempty(first.get("type"), first.get("name"), "Technical blocker") if isinstance(first, dict) else "Technical blocker"
485
  issue_detail = _first_nonempty(first.get("claim"), first.get("reason"), first.get("message")) if isinstance(first, dict) else str(first)
 
44
  FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
45
  CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
46
  BLOCKED_RAW_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
47
+ PROVIDER_QUOTA_RAW_STATUSES = {"provider_quota_blocked"}
48
  RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
49
  AUTH_RAW_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
50
 
51
  AUTHORITATIVE_TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status", "post_repair_validation", "failure_type")
52
+ TERMINAL_DISPLAY_STATUSES = SUCCESS_RAW_STATUSES | PARTIAL_RAW_STATUSES | MANUAL_RAW_STATUSES | FAILED_RAW_STATUSES | CANCELLED_RAW_STATUSES | BLOCKED_RAW_STATUSES | PROVIDER_QUOTA_RAW_STATUSES | AUTH_RAW_STATUSES | {"stale", "stopped", "succeeded", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation"}
53
 
54
  STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
55
 
 
204
  for key in AUTHORITATIVE_TERMINAL_SOURCE_KEYS:
205
  normalized = _normalize_terminal_status(source.get(key))
206
  if normalized:
207
+ if normalized in BLOCKED_RAW_STATUSES | PROVIDER_QUOTA_RAW_STATUSES | FAILED_RAW_STATUSES | MANUAL_RAW_STATUSES | AUTH_RAW_STATUSES | CANCELLED_RAW_STATUSES | {"stopped", "stale"}:
208
  return normalized, f"{source_name}.{key}"
209
  if normalized == "full_inference_success" and blocking_seen:
210
  continue
 
246
  global_status, verdict = "partial", normalized if normalized not in {"partial", "health_only", "full_inference_candidate_health_passed", "completed_with_warnings"} else "partial_validation"
247
  elif normalized in MANUAL_RAW_STATUSES:
248
  global_status, verdict = "waiting_manual_action", "manual_action_required"
249
+ elif normalized in PROVIDER_QUOTA_RAW_STATUSES:
250
+ global_status, verdict = "blocked", "provider_quota_blocked"
251
  elif normalized in BLOCKED_RAW_STATUSES:
252
  global_status, verdict = "blocked", "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
253
  elif normalized in FAILED_RAW_STATUSES:
 
313
  elif manual:
314
  global_status = "waiting_manual_action"
315
  verdict = "manual_action_required"
316
+ elif statuses.intersection(PROVIDER_QUOTA_RAW_STATUSES):
317
+ global_status = "blocked"
318
+ verdict = "provider_quota_blocked"
319
  elif blocker:
320
  global_status = "blocked"
321
  verdict = "technical_blocker_boot_only" if "technical_blocker_boot_only" in statuses else "technical_blocker"
 
485
  issue_title = ""
486
  issue_detail = ""
487
  issue_status = ""
488
+ provider_quota = bundle.get("provider_quota_blocker") or (bundle.get("state") or {}).get("provider_quota_blocker") or ((bundle.get("state") or {}).get("details") or {}).get("provider_quota_blocker")
489
+ if isinstance(provider_quota, dict) and provider_quota:
490
+ issue_title = provider_quota.get("title") or "Provider quota blocked"
491
+ issue_detail = provider_quota.get("user_message") or provider_quota.get("summary") or "Hugging Face Inference Providers quota or billing blocked Pi before ASF could generate the demo."
492
+ issue_status = "action_required"
493
+ elif blocker_items:
494
  first = blocker_items[0]
495
  issue_title = _first_nonempty(first.get("type"), first.get("name"), "Technical blocker") if isinstance(first, dict) else "Technical blocker"
496
  issue_detail = _first_nonempty(first.get("claim"), first.get("reason"), first.get("message")) if isinstance(first, dict) else str(first)
src/worker_payload.py CHANGED
@@ -40,8 +40,8 @@ DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large"
40
  DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2"
41
  MAX_PI_REPAIR_ATTEMPTS = 3
42
  DEFAULT_FACTORY_UPLOAD_RETRY_BUDGET = 1
43
- APP_VERSION = "v198.26.127"
44
- app_version = "v198.26.127"
45
 
46
  # Internal agent/recovery files may be needed inside the transient Pi
47
  # workspace, but they should not be published to the generated Space or shown
@@ -630,6 +630,7 @@ def build_runtime_upload_payload(workspace: Path, run_dir: Path, events_path: Pa
630
  # candidate and stamp the app so /health can prove which candidate is
631
  # actually being served. No further runtime mutation is allowed after this.
632
  freeze_release_candidate(payload_dir, workspace, run_dir, events_path, manifest, target_space_id=os.environ.get("TARGET_SPACE_ID", ""))
 
633
  manifest_path = run_dir / "runtime_upload_payload_manifest.json"
634
  write_json(manifest_path, manifest)
635
 
@@ -739,6 +740,46 @@ def _current_repair_attempt_for_release(run_dir: Path) -> int | None:
739
  return None
740
 
741
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
742
  def _insert_asf_release_constant(app_text: str, release: dict) -> str:
743
  marker = "# ASF runtime release identity stamp"
744
  if marker in app_text:
@@ -772,48 +813,90 @@ def _insert_asf_release_constant(app_text: str, release: dict) -> str:
772
  "\n"
773
  )
774
  lines = app_text.splitlines(True)
775
- insert_at = 0
776
- # Preserve shebang/encoding/comments and place after the import block where possible.
777
- for i, line in enumerate(lines[:80]):
778
- stripped = line.strip()
779
- if stripped.startswith("import ") or stripped.startswith("from "):
780
- insert_at = i + 1
781
  lines.insert(insert_at, block)
782
  return "".join(lines)
783
 
784
 
785
- def _wrap_health_functions_with_release_identity(app_text: str, run_dir: Path | None = None) -> tuple[str, list[str]]:
786
  try:
787
  tree = ast.parse(app_text)
788
  except Exception:
789
- return app_text, []
790
- wrappers: list[tuple[int, str]] = []
791
- names = []
792
- for node in ast.walk(tree):
793
  if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
794
  name = str(node.name or "")
795
  lname = name.lower()
796
  if lname in {"health", "health_check", "runtime_health", "get_health"} or ("health" in lname and not lname.startswith("_asf")):
797
  if getattr(node, "end_lineno", None):
798
  names.append(name)
799
- wrappers.append((int(node.end_lineno), name))
 
 
 
 
 
 
 
 
800
  if not wrappers:
801
  return app_text, []
802
  lines = app_text.splitlines(True)
803
- for end_lineno, name in sorted(wrappers, reverse=True):
804
  alias = f"_asf_original_{name}"
805
- wrapper = (
806
- f"\ntry:\n"
807
- f" {alias} = {name}\n"
808
- f" def {name}(*args, **kwargs):\n"
809
- f" return _asf_stamp_health_payload({alias}(*args, **kwargs))\n"
810
- f"except Exception:\n"
811
- f" pass\n\n"
812
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
813
  lines.insert(end_lineno, wrapper)
814
  return "".join(lines), names
815
 
816
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
817
  def patch_runtime_payload_with_release_identity(payload_dir: Path, release: dict, run_dir: Path, events_path: Path) -> dict:
818
  payload_dir.mkdir(parents=True, exist_ok=True)
819
  write_json(payload_dir / "ASF_RELEASE.json", release)
@@ -2487,6 +2570,7 @@ def eval_terminal_status(status: str, phase: str = "") -> bool:
2487
  "partial_validation",
2488
  "technical_blocker",
2489
  "manual_hardware_required",
 
2490
  }
2491
 
2492
 
@@ -2541,6 +2625,8 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
2541
  return "cancelled"
2542
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
2543
  return "manual_action_required"
 
 
2544
  if status in {"technical_blocker", "technical_blocker_boot_only"}:
2545
  return "technical_blocker"
2546
  if full_inference_verified:
@@ -2925,7 +3011,10 @@ def run_final_cleanup_if_needed(run_dir: Path, events_path: Path, final_state: d
2925
  health_semantic_passed = signals.get("health_semantic_passed")
2926
  if health_semantic_passed is None:
2927
  health_semantic_passed = signals.get("health_passed")
2928
- if runtime_recovery.get("triggered") and runtime_recovery.get("recovery_attempted") and not runtime_recovery.get("recovery_exhausted"):
 
 
 
2929
  decision = {"should_pause": False, "reason": "runtime_recovery_pending", "final_status": final_status, "selected_hardware": selected_hardware}
2930
  else:
2931
  decision = should_pause_generated_space(final_status, target_space, selected_hardware, health_semantic_passed=health_semantic_passed if isinstance(health_semantic_passed, bool) else None)
@@ -2949,9 +3038,140 @@ def run_final_cleanup_if_needed(run_dir: Path, events_path: Path, final_state: d
2949
  write_json(run_dir / "state.json", state)
2950
  return cleanup
2951
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2952
  def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):
2953
  safe = safe_details(details)
2954
- append_event(events_path, "failure", "failed", message, safe)
 
 
 
 
 
 
 
 
 
 
 
 
2955
  existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
2956
  if not isinstance(existing_state, dict):
2957
  existing_state = {}
@@ -2969,11 +3189,21 @@ def fail(run_dir: Path, events_path: Path, message: str, details: dict | None =
2969
  "details": safe,
2970
  "repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {},
2971
  }
2972
- # Preserve the target Space when the worker fails after repository creation.
 
 
2973
  target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
2974
- if target_space:
 
2975
  failure_state["target_space"] = target_space
2976
  failure_state["target_space_url"] = f"https://huggingface.co/spaces/{target_space}"
 
 
 
 
 
 
 
2977
  write_json(run_dir / "state.json", failure_state)
2978
  cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, failure_state)
2979
  failure_state["cleanup"] = cleanup_status
@@ -9360,9 +9590,11 @@ def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None
9360
  runtime_upload_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {}
9361
  space_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {}
9362
  runtime_history = runtime_upload_epoch.get("history") if isinstance(runtime_upload_epoch.get("history"), list) else []
9363
- target_space = state.get("target_space") or state.get("target_space_id") or runtime_upload_epoch.get("target_space_id") or os.environ.get("TARGET_SPACE_ID", "")
9364
  runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history)
9365
  space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url")))
 
 
9366
  payload = {
9367
  "schema_version": "final_status_reconciliation.v198_26_9",
9368
  "status": status,
@@ -9370,8 +9602,10 @@ def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None
9370
  "message": message or state.get("message") or "",
9371
  "job_exit_code": int(job_exit_code if job_exit_code is not None else (0 if status == "full_inference_success" else 1)),
9372
  "target_space": target_space,
 
9373
  "target_space_url": f"https://huggingface.co/spaces/{target_space}" if target_space else "",
9374
- "space_created": bool(target_space and (runtime_uploaded or space_runtime_known)),
 
9375
  "runtime_uploaded": bool(runtime_uploaded),
9376
  "space_uploaded": bool(runtime_uploaded),
9377
  "space_runtime_known": bool(space_runtime_known),
@@ -15488,6 +15722,12 @@ def main():
15488
  )
15489
  (run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8")
15490
  if code != 0:
 
 
 
 
 
 
15491
  append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]})
15492
  collect_pi_traces(run_dir, events_path)
15493
  fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]})
@@ -16688,6 +16928,8 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
16688
  return "cancelled"
16689
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
16690
  return "manual_action_required"
 
 
16691
  if status in {"technical_blocker", "technical_blocker_boot_only"}:
16692
  return "technical_blocker"
16693
  if full_inference_verified:
@@ -20307,7 +20549,7 @@ def smoke_generate(target_space_id: str, token: str, run_dir: Path, events_path:
20307
  # write_json(run_dir / "tests" / "payload_source.json", payload_source_record)
20308
  # write_json(run_dir / "tests" / "replay_source.json", replay_source)
20309
  # write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight)
20310
- app_version = "v198.26.127"
20311
  engine_version = "unified_gradio_validation_harness_v198_25_3"
20312
  parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip()
20313
  validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"
 
40
  DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2"
41
  MAX_PI_REPAIR_ATTEMPTS = 3
42
  DEFAULT_FACTORY_UPLOAD_RETRY_BUDGET = 1
43
+ APP_VERSION = "v198.26.130"
44
+ app_version = "v198.26.130"
45
 
46
  # Internal agent/recovery files may be needed inside the transient Pi
47
  # workspace, but they should not be published to the generated Space or shown
 
630
  # candidate and stamp the app so /health can prove which candidate is
631
  # actually being served. No further runtime mutation is allowed after this.
632
  freeze_release_candidate(payload_dir, workspace, run_dir, events_path, manifest, target_space_id=os.environ.get("TARGET_SPACE_ID", ""))
633
+ release_candidate_payload_compile_guard(payload_dir, run_dir, events_path, reason="runtime_upload_payload_post_release_stamp")
634
  manifest_path = run_dir / "runtime_upload_payload_manifest.json"
635
  write_json(manifest_path, manifest)
636
 
 
740
  return None
741
 
742
 
743
+ def _release_identity_insert_index(app_text: str) -> int:
744
+ """Return a safe top-level insertion point for the ASF runtime stamp.
745
+
746
+ v198.26.127 inserted after any line whose stripped text looked like an
747
+ import. That accidentally matched imports inside top-level try/except
748
+ blocks, producing invalid Python such as `try: import torch; ASF_STAMP;
749
+ except ...`. Only top-level module body statements are safe anchors.
750
+ """
751
+ try:
752
+ tree = ast.parse(app_text)
753
+ lines = app_text.splitlines(True)
754
+ insert_after = 0
755
+ for idx, node in enumerate(tree.body):
756
+ if idx == 0 and isinstance(node, ast.Expr) and isinstance(getattr(node, "value", None), ast.Constant) and isinstance(node.value.value, str):
757
+ insert_after = int(getattr(node, "end_lineno", node.lineno) or node.lineno)
758
+ continue
759
+ if isinstance(node, ast.ImportFrom) and getattr(node, "module", "") == "__future__":
760
+ insert_after = int(getattr(node, "end_lineno", node.lineno) or node.lineno)
761
+ continue
762
+ if isinstance(node, (ast.Import, ast.ImportFrom)) and int(getattr(node, "col_offset", 0) or 0) == 0:
763
+ insert_after = int(getattr(node, "end_lineno", node.lineno) or node.lineno)
764
+ continue
765
+ break
766
+ return max(0, min(insert_after, len(lines)))
767
+ except Exception:
768
+ lines = app_text.splitlines(True)
769
+ insert_after = 0
770
+ # Conservative fallback: only unindented imports/future imports count.
771
+ for i, line in enumerate(lines[:120]):
772
+ if line.startswith("import ") or line.startswith("from "):
773
+ insert_after = i + 1
774
+ continue
775
+ stripped = line.strip()
776
+ if not stripped or stripped.startswith("#"):
777
+ continue
778
+ if insert_after:
779
+ break
780
+ return insert_after
781
+
782
+
783
  def _insert_asf_release_constant(app_text: str, release: dict) -> str:
784
  marker = "# ASF runtime release identity stamp"
785
  if marker in app_text:
 
813
  "\n"
814
  )
815
  lines = app_text.splitlines(True)
816
+ insert_at = _release_identity_insert_index(app_text)
 
 
 
 
 
817
  lines.insert(insert_at, block)
818
  return "".join(lines)
819
 
820
 
821
+ def _top_level_health_functions(app_text: str) -> tuple[list[tuple[int, str, bool]], list[str]]:
822
  try:
823
  tree = ast.parse(app_text)
824
  except Exception:
825
+ return [], []
826
+ wrappers: list[tuple[int, str, bool]] = []
827
+ names: list[str] = []
828
+ for node in tree.body:
829
  if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
830
  name = str(node.name or "")
831
  lname = name.lower()
832
  if lname in {"health", "health_check", "runtime_health", "get_health"} or ("health" in lname and not lname.startswith("_asf")):
833
  if getattr(node, "end_lineno", None):
834
  names.append(name)
835
+ wrappers.append((int(node.end_lineno), name, isinstance(node, ast.AsyncFunctionDef)))
836
+ return wrappers, names
837
+
838
+
839
+ def _wrap_health_functions_with_release_identity(app_text: str, run_dir: Path | None = None) -> tuple[str, list[str]]:
840
+ # Only wrap top-level health functions. Nested functions or class methods
841
+ # must not receive top-level wrapper assignments based on their end_lineno,
842
+ # because that can inject code into an unrelated block and break generated apps.
843
+ wrappers, names = _top_level_health_functions(app_text)
844
  if not wrappers:
845
  return app_text, []
846
  lines = app_text.splitlines(True)
847
+ for end_lineno, name, is_async in sorted(wrappers, reverse=True):
848
  alias = f"_asf_original_{name}"
849
+ if is_async:
850
+ wrapper = (
851
+ f"\ntry:\n"
852
+ f" {alias} = {name}\n"
853
+ f" async def {name}(*args, **kwargs):\n"
854
+ f" _asf_result = {alias}(*args, **kwargs)\n"
855
+ f" if hasattr(_asf_result, '__await__'):\n"
856
+ f" _asf_result = await _asf_result\n"
857
+ f" return _asf_stamp_health_payload(_asf_result)\n"
858
+ f"except Exception:\n"
859
+ f" pass\n\n"
860
+ )
861
+ else:
862
+ wrapper = (
863
+ f"\ntry:\n"
864
+ f" {alias} = {name}\n"
865
+ f" def {name}(*args, **kwargs):\n"
866
+ f" return _asf_stamp_health_payload({alias}(*args, **kwargs))\n"
867
+ f"except Exception:\n"
868
+ f" pass\n\n"
869
+ )
870
  lines.insert(end_lineno, wrapper)
871
  return "".join(lines), names
872
 
873
 
874
+ def release_candidate_payload_compile_guard(payload_dir: Path, run_dir: Path, events_path: Path | None, *, reason: str = "release_candidate_post_stamp") -> dict:
875
+ app_path = payload_dir / "app.py"
876
+ result = {
877
+ "schema_version": "release_candidate_payload_compile_guard.v198_26_129",
878
+ "reason": reason,
879
+ "checked_file": "runtime_upload_payload/app.py",
880
+ "passed": False,
881
+ "error": "",
882
+ "updated_at": now(),
883
+ }
884
+ if not app_path.exists():
885
+ result["error"] = "app.py missing from runtime upload payload"
886
+ else:
887
+ try:
888
+ compile(app_path.read_text(encoding="utf-8", errors="ignore"), str(app_path), "exec")
889
+ result["passed"] = True
890
+ except Exception as exc:
891
+ result["error"] = f"{type(exc).__name__}: {exc}"
892
+ write_json(run_dir / "release" / "RELEASE_CANDIDATE_COMPILE_GUARD.json", result)
893
+ if events_path is not None:
894
+ append_event(events_path, "release_candidate_compile_guard", "success" if result.get("passed") else "failed", "Checked stamped runtime payload app.py before upload", result)
895
+ if not result.get("passed"):
896
+ raise RuntimeError("release_candidate_payload_compile_failed: " + str(result.get("error") or "unknown"))
897
+ return result
898
+
899
+
900
  def patch_runtime_payload_with_release_identity(payload_dir: Path, release: dict, run_dir: Path, events_path: Path) -> dict:
901
  payload_dir.mkdir(parents=True, exist_ok=True)
902
  write_json(payload_dir / "ASF_RELEASE.json", release)
 
2570
  "partial_validation",
2571
  "technical_blocker",
2572
  "manual_hardware_required",
2573
+ "provider_quota_blocked",
2574
  }
2575
 
2576
 
 
2625
  return "cancelled"
2626
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
2627
  return "manual_action_required"
2628
+ if status == "provider_quota_blocked":
2629
+ return "provider_quota_blocked"
2630
  if status in {"technical_blocker", "technical_blocker_boot_only"}:
2631
  return "technical_blocker"
2632
  if full_inference_verified:
 
3011
  health_semantic_passed = signals.get("health_semantic_passed")
3012
  if health_semantic_passed is None:
3013
  health_semantic_passed = signals.get("health_passed")
3014
+ space_creation = generated_space_creation_evidence(run_dir, events_path)
3015
+ if target_space and not space_creation.get("space_created"):
3016
+ decision = {"should_pause": False, "reason": "target_space_not_created", "final_status": final_status, "selected_hardware": selected_hardware, "target_space_planned": target_space, "space_creation_evidence": space_creation}
3017
+ elif runtime_recovery.get("triggered") and runtime_recovery.get("recovery_attempted") and not runtime_recovery.get("recovery_exhausted"):
3018
  decision = {"should_pause": False, "reason": "runtime_recovery_pending", "final_status": final_status, "selected_hardware": selected_hardware}
3019
  else:
3020
  decision = should_pause_generated_space(final_status, target_space, selected_hardware, health_semantic_passed=health_semantic_passed if isinstance(health_semantic_passed, bool) else None)
 
3038
  write_json(run_dir / "state.json", state)
3039
  return cleanup
3040
 
3041
+
3042
+ PROVIDER_QUOTA_BLOCKED_STATUS = "provider_quota_blocked"
3043
+ PROVIDER_QUOTA_MARKERS = (
3044
+ "monthly spending limit for inference providers",
3045
+ "exceeded your monthly spending limit",
3046
+ "spending limit for inference providers",
3047
+ "inference providers quota",
3048
+ "provider quota",
3049
+ "quota exceeded",
3050
+ "insufficient credits",
3051
+ "billing limit",
3052
+ "payment required",
3053
+ )
3054
+
3055
+
3056
+ def classify_provider_quota_blocker(text: str | None, *, phase: str = "pi_run", returncode: int | None = None) -> dict:
3057
+ """Return a user-facing blocker when HF Inference Providers quota/billing stopped Pi.
3058
+
3059
+ This is not a generated Space failure. It happens before ASF can produce or
3060
+ validate a demo, so the UI must show an external provider/quota blocker
3061
+ instead of a generic model/build failure.
3062
+ """
3063
+ raw = str(text or "")
3064
+ low = raw.lower()
3065
+ if not raw:
3066
+ return {}
3067
+ matched = next((marker for marker in PROVIDER_QUOTA_MARKERS if marker in low), "")
3068
+ # HF provider errors often include a structured 403 and the exact spending
3069
+ # limit phrase. Keep this conservative so unrelated model errors still go
3070
+ # through the normal repair path.
3071
+ if not matched and not ("403" in low and "inference provider" in low and any(k in low for k in ("quota", "billing", "spending", "credits", "limit"))):
3072
+ return {}
3073
+ excerpt = redact_text(raw[-2200:])
3074
+ return {
3075
+ "schema_version": "provider_quota_blocker.v198_26_130",
3076
+ "status": PROVIDER_QUOTA_BLOCKED_STATUS,
3077
+ "category": "hf_inference_provider_quota",
3078
+ "phase": phase,
3079
+ "returncode": int(returncode) if returncode is not None else None,
3080
+ "matched_marker": matched or "403 inference provider quota/billing marker",
3081
+ "title": "Provider quota blocked",
3082
+ "summary": "Hugging Face Inference Providers quota or monthly spending limit blocked Pi before ASF could generate the demo.",
3083
+ "user_message": "ASF could not start the Pi coding pass because the Hugging Face Inference Providers quota or monthly spending limit was reached. No Space was uploaded and no model validation was attempted.",
3084
+ "suggested_action": "Check Hugging Face billing/quota settings or switch the Pi provider/model, then retry the build.",
3085
+ "retryable_after_user_action": True,
3086
+ "model_failure": False,
3087
+ "space_failure": False,
3088
+ "space_upload_attempted": False,
3089
+ "output_tail": excerpt,
3090
+ "updated_at": now(),
3091
+ }
3092
+
3093
+
3094
+ def write_provider_quota_blocker(run_dir: Path, events_path: Path, blocker: dict) -> dict:
3095
+ payload = {**(blocker or {}), "updated_at": now()}
3096
+ write_json(run_dir / "PROVIDER_QUOTA_BLOCKER.json", payload)
3097
+ write_json(run_dir / "provider_quota_blocker.json", payload)
3098
+ append_event(
3099
+ events_path,
3100
+ "provider_quota",
3101
+ "blocked",
3102
+ payload.get("user_message") or "Hugging Face Inference Providers quota blocked Pi before Space upload",
3103
+ {k: payload.get(k) for k in ("category", "phase", "returncode", "matched_marker", "suggested_action")},
3104
+ )
3105
+ write_live_status(run_dir, stage="provider_quota", status=PROVIDER_QUOTA_BLOCKED_STATUS, message=payload.get("user_message") or payload.get("summary") or "Provider quota blocked", data=payload)
3106
+ return payload
3107
+
3108
+
3109
+ def generated_space_creation_evidence(run_dir: Path, events_path: Path | None = None) -> dict:
3110
+ """Return whether ASF has evidence that the target Space repo actually exists.
3111
+
3112
+ target_space_id may be planned before Pi runs. Cleanup/autopause must not act
3113
+ on that planned id unless create/upload/runtime evidence proves the repo was
3114
+ actually created or reused by this run.
3115
+ """
3116
+ evidence = {"space_created": False, "source": "none", "reason": "no_create_space_or_upload_evidence"}
3117
+ try:
3118
+ runtime_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {}
3119
+ if isinstance(runtime_epoch, dict) and (runtime_epoch.get("last_upload_completed_at") or runtime_epoch.get("upload_sequence") or runtime_epoch.get("history")):
3120
+ return {"space_created": True, "source": "runtime_upload_epoch", "reason": "runtime_upload_recorded"}
3121
+ except Exception:
3122
+ pass
3123
+ for artifact, source in (("space_runtime.json", "space_runtime"), ("hardware_strategy.json", "hardware_strategy")):
3124
+ try:
3125
+ payload = load_json_if_exists(run_dir / artifact) if (run_dir / artifact).exists() else {}
3126
+ if not isinstance(payload, dict) or not payload:
3127
+ continue
3128
+ if source == "space_runtime" and (payload.get("stage") or payload.get("runtime_status") or payload.get("status") or payload.get("url")):
3129
+ return {"space_created": True, "source": source, "reason": "runtime_artifact_present"}
3130
+ if source == "hardware_strategy" and payload.get("created") is True:
3131
+ return {"space_created": True, "source": source, "reason": "hardware_strategy_created_true"}
3132
+ except Exception:
3133
+ pass
3134
+ try:
3135
+ path = events_path or (run_dir / "events.jsonl")
3136
+ if path and path.exists():
3137
+ for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
3138
+ if not line.strip():
3139
+ continue
3140
+ try:
3141
+ event = json.loads(line)
3142
+ except Exception:
3143
+ continue
3144
+ if not isinstance(event, dict):
3145
+ continue
3146
+ step = str(event.get("step") or "").lower()
3147
+ status = str(event.get("status") or "").lower()
3148
+ data = event.get("data") if isinstance(event.get("data"), dict) else {}
3149
+ if step == "create_space" and status in {"success", "warning"}:
3150
+ return {"space_created": True, "source": "events.create_space", "reason": status}
3151
+ if step in {"upload_files", "repair_upload"} and status == "success":
3152
+ return {"space_created": True, "source": f"events.{step}", "reason": "upload_success"}
3153
+ if data.get("created") is True or data.get("ok") is True and step == "create_space":
3154
+ return {"space_created": True, "source": "events.create_space.data", "reason": "created_or_ok_true"}
3155
+ except Exception:
3156
+ pass
3157
+ return evidence
3158
+
3159
+
3160
  def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):
3161
  safe = safe_details(details)
3162
+ quota_source = "\n".join([str(message or ""), str(safe.get("output_tail") or ""), str(safe.get("error") or "")])
3163
+ quota_blocker = safe.get("provider_quota_blocker") if isinstance(safe.get("provider_quota_blocker"), dict) else classify_provider_quota_blocker(quota_source, phase=str(safe.get("phase") or safe.get("step") or "failure"), returncode=safe.get("returncode") if isinstance(safe.get("returncode"), int) else None)
3164
+ if quota_blocker:
3165
+ status = PROVIDER_QUOTA_BLOCKED_STATUS
3166
+ safe["provider_quota_blocker"] = quota_blocker
3167
+ safe.setdefault("failure_type", PROVIDER_QUOTA_BLOCKED_STATUS)
3168
+ safe.setdefault("failure_owner", "hf_inference_provider")
3169
+ message = quota_blocker.get("user_message") or message
3170
+ try:
3171
+ write_provider_quota_blocker(run_dir, events_path, quota_blocker)
3172
+ except Exception:
3173
+ pass
3174
+ append_event(events_path, "failure", "blocked" if status == PROVIDER_QUOTA_BLOCKED_STATUS else "failed", message, safe)
3175
  existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
3176
  if not isinstance(existing_state, dict):
3177
  existing_state = {}
 
3189
  "details": safe,
3190
  "repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {},
3191
  }
3192
+ # Preserve target Space links only when the worker has evidence that the repo
3193
+ # was actually created/reused. Before Pi finishes, TARGET_SPACE_ID is only a
3194
+ # plan and cleanup/UI links must not treat it as a real Space.
3195
  target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
3196
+ space_creation = generated_space_creation_evidence(run_dir, events_path)
3197
+ if target_space and space_creation.get("space_created"):
3198
  failure_state["target_space"] = target_space
3199
  failure_state["target_space_url"] = f"https://huggingface.co/spaces/{target_space}"
3200
+ failure_state["space_creation_evidence"] = space_creation
3201
+ elif target_space:
3202
+ failure_state.pop("target_space", None)
3203
+ failure_state.pop("target_space_id", None)
3204
+ failure_state.pop("target_space_url", None)
3205
+ failure_state["target_space_planned"] = target_space
3206
+ failure_state["space_creation_evidence"] = space_creation
3207
  write_json(run_dir / "state.json", failure_state)
3208
  cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, failure_state)
3209
  failure_state["cleanup"] = cleanup_status
 
9590
  runtime_upload_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {}
9591
  space_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {}
9592
  runtime_history = runtime_upload_epoch.get("history") if isinstance(runtime_upload_epoch.get("history"), list) else []
9593
+ planned_target_space = state.get("target_space") or state.get("target_space_id") or runtime_upload_epoch.get("target_space_id") or os.environ.get("TARGET_SPACE_ID", "")
9594
  runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history)
9595
  space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url")))
9596
+ space_creation = generated_space_creation_evidence(run_dir, events_path)
9597
+ target_space = planned_target_space if space_creation.get("space_created") else ""
9598
  payload = {
9599
  "schema_version": "final_status_reconciliation.v198_26_9",
9600
  "status": status,
 
9602
  "message": message or state.get("message") or "",
9603
  "job_exit_code": int(job_exit_code if job_exit_code is not None else (0 if status == "full_inference_success" else 1)),
9604
  "target_space": target_space,
9605
+ "target_space_planned": planned_target_space if not target_space else "",
9606
  "target_space_url": f"https://huggingface.co/spaces/{target_space}" if target_space else "",
9607
+ "space_created": bool(target_space and space_creation.get("space_created")),
9608
+ "space_creation_evidence": space_creation,
9609
  "runtime_uploaded": bool(runtime_uploaded),
9610
  "space_uploaded": bool(runtime_uploaded),
9611
  "space_runtime_known": bool(space_runtime_known),
 
15722
  )
15723
  (run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8")
15724
  if code != 0:
15725
+ quota_blocker = classify_provider_quota_blocker(pi_out, phase="pi_run", returncode=code)
15726
+ if quota_blocker:
15727
+ append_event(events_path, "pi_run", "blocked", "Pi was blocked by Hugging Face Inference Providers quota/billing", {"returncode": code, "provider_quota_blocked": True, "output_tail": pi_out[-4000:]})
15728
+ write_provider_quota_blocker(run_dir, events_path, quota_blocker)
15729
+ collect_pi_traces(run_dir, events_path)
15730
+ fail(run_dir, events_path, quota_blocker.get("user_message") or "Pi provider quota blocked before Space upload", {"returncode": code, "output_tail": pi_out[-4000:], "provider_quota_blocker": quota_blocker, "phase": "pi_run"}, status=PROVIDER_QUOTA_BLOCKED_STATUS)
15731
  append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]})
15732
  collect_pi_traces(run_dir, events_path)
15733
  fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]})
 
16928
  return "cancelled"
16929
  if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
16930
  return "manual_action_required"
16931
+ if status == "provider_quota_blocked":
16932
+ return "provider_quota_blocked"
16933
  if status in {"technical_blocker", "technical_blocker_boot_only"}:
16934
  return "technical_blocker"
16935
  if full_inference_verified:
 
20549
  # write_json(run_dir / "tests" / "payload_source.json", payload_source_record)
20550
  # write_json(run_dir / "tests" / "replay_source.json", replay_source)
20551
  # write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight)
20552
+ app_version = "v198.26.130"
20553
  engine_version = "unified_gradio_validation_harness_v198_25_3"
20554
  parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip()
20555
  validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"