Upload 16 files
Browse files- src/bucket.py +3 -1
- src/effective_status.py +6 -1
- src/eval_archive.py +1 -1
- src/progress.py +7 -3
- src/timeline_model.py +12 -3
- src/version.py +2 -2
- src/view_models.py +14 -3
- src/worker_payload.py +272 -30
src/bucket.py
CHANGED
|
@@ -1820,6 +1820,7 @@ def read_run_bundle(run_id: str, *, bucket_source: str, token: str | None = None
|
|
| 1820 |
"build_error_observation": _safe_read_json(f"{paths.root}/build_error_observation.json", token=token),
|
| 1821 |
"repair_decision": _safe_read_json(f"{paths.root}/repair/REPAIR_DECISION.json", token=token),
|
| 1822 |
"blockage": _safe_read_json(f"{paths.root}/repair/BLOCKAGE.json", token=token),
|
|
|
|
| 1823 |
"repair_attempts": _safe_read_json(f"{paths.root}/repair/repair_attempts.json", token=token),
|
| 1824 |
"artifact_manifest": _safe_read_json(f"{paths.root}/artifact_manifest.json", token=token),
|
| 1825 |
"files": _list_run_files(run_id, bucket_source=bucket_source, token=token) if include_heavy else [],
|
|
@@ -1906,6 +1907,7 @@ def _run_list_summary_needs_enrichment(run_id: str, summary_file: dict[str, Any]
|
|
| 1906 |
"technical_blocker",
|
| 1907 |
"technical_blocker_boot_only",
|
| 1908 |
"blocked",
|
|
|
|
| 1909 |
"failed",
|
| 1910 |
"failure",
|
| 1911 |
"error",
|
|
@@ -1981,7 +1983,7 @@ def _build_run_list_item_from_files(run_id: str, *, root: str, bucket_source: st
|
|
| 1981 |
item["effective_status"] = post_build_validation_status.get("effective_status") or item.get("effective_status")
|
| 1982 |
if linked_validations:
|
| 1983 |
rows = linked_validations.get("validations") if isinstance(linked_validations, dict) else []
|
| 1984 |
-
terminal_tokens = {"success", "passed", "succeeded", "full_inference_success", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation", "manual_validation_passed", "manual_validated", "partial_validation", "manual_hardware_required", "generated_needs_manual_hardware", "technical_blocker", "technical_blocker_boot_only", "blocked", "completed_with_warnings", "failed", "failure", "error", "auth_refresh_required", "stale", "stopped", "cancelled", "canceled"}
|
| 1985 |
accounted_rows = [row for row in rows if isinstance(row, dict) and str(row.get("status") or row.get("validation_status") or row.get("result_status") or row.get("effective_status") or "").lower() in terminal_tokens] if isinstance(rows, list) else []
|
| 1986 |
item["linked_validations"] = linked_validations
|
| 1987 |
item["linked_validation_count"] = len(accounted_rows)
|
|
|
|
| 1820 |
"build_error_observation": _safe_read_json(f"{paths.root}/build_error_observation.json", token=token),
|
| 1821 |
"repair_decision": _safe_read_json(f"{paths.root}/repair/REPAIR_DECISION.json", token=token),
|
| 1822 |
"blockage": _safe_read_json(f"{paths.root}/repair/BLOCKAGE.json", token=token),
|
| 1823 |
+
"provider_quota_blocker": _safe_read_json(f"{paths.root}/PROVIDER_QUOTA_BLOCKER.json", token=token) or _safe_read_json(f"{paths.root}/provider_quota_blocker.json", token=token),
|
| 1824 |
"repair_attempts": _safe_read_json(f"{paths.root}/repair/repair_attempts.json", token=token),
|
| 1825 |
"artifact_manifest": _safe_read_json(f"{paths.root}/artifact_manifest.json", token=token),
|
| 1826 |
"files": _list_run_files(run_id, bucket_source=bucket_source, token=token) if include_heavy else [],
|
|
|
|
| 1907 |
"technical_blocker",
|
| 1908 |
"technical_blocker_boot_only",
|
| 1909 |
"blocked",
|
| 1910 |
+
"provider_quota_blocked",
|
| 1911 |
"failed",
|
| 1912 |
"failure",
|
| 1913 |
"error",
|
|
|
|
| 1983 |
item["effective_status"] = post_build_validation_status.get("effective_status") or item.get("effective_status")
|
| 1984 |
if linked_validations:
|
| 1985 |
rows = linked_validations.get("validations") if isinstance(linked_validations, dict) else []
|
| 1986 |
+
terminal_tokens = {"success", "passed", "succeeded", "full_inference_success", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation", "manual_validation_passed", "manual_validated", "partial_validation", "manual_hardware_required", "generated_needs_manual_hardware", "technical_blocker", "technical_blocker_boot_only", "blocked", "completed_with_warnings", "failed", "failure", "error", "auth_refresh_required", "provider_quota_blocked", "stale", "stopped", "cancelled", "canceled"}
|
| 1987 |
accounted_rows = [row for row in rows if isinstance(row, dict) and str(row.get("status") or row.get("validation_status") or row.get("result_status") or row.get("effective_status") or "").lower() in terminal_tokens] if isinstance(rows, list) else []
|
| 1988 |
item["linked_validations"] = linked_validations
|
| 1989 |
item["linked_validation_count"] = len(accounted_rows)
|
src/effective_status.py
CHANGED
|
@@ -43,7 +43,8 @@ MANUAL_BUILD_TOKENS = {
|
|
| 43 |
"manual_action_required",
|
| 44 |
"waiting_manual_action",
|
| 45 |
}
|
| 46 |
-
BLOCKED_BUILD_TOKENS = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
|
|
|
|
| 47 |
|
| 48 |
|
| 49 |
def _lower(value: Any) -> str:
|
|
@@ -193,6 +194,8 @@ def _normalize_build_status(value: Any) -> str:
|
|
| 193 |
return "full_inference_success"
|
| 194 |
if status in {"repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}:
|
| 195 |
return "failed"
|
|
|
|
|
|
|
| 196 |
if status in {"cancelled", "canceled"}:
|
| 197 |
return "stopped"
|
| 198 |
if status in {"full_inference_candidate_health_passed", "health_only", "partial", "completed_with_warnings"}:
|
|
@@ -295,6 +298,8 @@ def compute_effective_run_status(
|
|
| 295 |
display_label = "Partial validation"
|
| 296 |
elif display_status == "manual_hardware_required":
|
| 297 |
display_label = "Manual hardware required"
|
|
|
|
|
|
|
| 298 |
elif display_status in BLOCKED_BUILD_TOKENS:
|
| 299 |
display_label = "Diagnostic Space"
|
| 300 |
elif display_status in FAILED_BUILD_TOKENS:
|
|
|
|
| 43 |
"manual_action_required",
|
| 44 |
"waiting_manual_action",
|
| 45 |
}
|
| 46 |
+
BLOCKED_BUILD_TOKENS = {"technical_blocker", "technical_blocker_boot_only", "blocked", "provider_quota_blocked"}
|
| 47 |
+
PROVIDER_QUOTA_TOKENS = {"provider_quota_blocked"}
|
| 48 |
|
| 49 |
|
| 50 |
def _lower(value: Any) -> str:
|
|
|
|
| 194 |
return "full_inference_success"
|
| 195 |
if status in {"repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}:
|
| 196 |
return "failed"
|
| 197 |
+
if status in {"provider_quota_blocked"}:
|
| 198 |
+
return "provider_quota_blocked"
|
| 199 |
if status in {"cancelled", "canceled"}:
|
| 200 |
return "stopped"
|
| 201 |
if status in {"full_inference_candidate_health_passed", "health_only", "partial", "completed_with_warnings"}:
|
|
|
|
| 298 |
display_label = "Partial validation"
|
| 299 |
elif display_status == "manual_hardware_required":
|
| 300 |
display_label = "Manual hardware required"
|
| 301 |
+
elif display_status == "provider_quota_blocked":
|
| 302 |
+
display_label = "Provider quota blocked"
|
| 303 |
elif display_status in BLOCKED_BUILD_TOKENS:
|
| 304 |
display_label = "Diagnostic Space"
|
| 305 |
elif display_status in FAILED_BUILD_TOKENS:
|
src/eval_archive.py
CHANGED
|
@@ -9,7 +9,7 @@ from .bucket import RunPaths, read_json, read_text, write_json
|
|
| 9 |
from .eval_config import effective_eval_config
|
| 10 |
from .security import redact
|
| 11 |
|
| 12 |
-
_TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"}
|
| 13 |
|
| 14 |
|
| 15 |
def _now() -> str:
|
|
|
|
| 9 |
from .eval_config import effective_eval_config
|
| 10 |
from .security import redact
|
| 11 |
|
| 12 |
+
_TERMINAL = {"success", "done", "completed", "failed", "failure", "error", "cancelled", "canceled", "manual", "stale", "partial", "partial_validation", "completed_with_warnings", "success_with_warnings", "validated", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_manual_validation", "validated_after_stale_run", "manual_validation_passed", "full_inference_success", "full_inference_candidate_health_passed", "health_only", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required", "provider_quota_blocked"}
|
| 13 |
|
| 14 |
|
| 15 |
def _now() -> str:
|
src/progress.py
CHANGED
|
@@ -96,6 +96,7 @@ STEP_LABELS = {
|
|
| 96 |
}
|
| 97 |
|
| 98 |
STEP_ALIASES = {
|
|
|
|
| 99 |
"bucket_ready": "bootstrap",
|
| 100 |
"job_launched": "bootstrap",
|
| 101 |
"hardware_preferred": "hardware_strategy",
|
|
@@ -129,6 +130,7 @@ PARTIAL_STATUSES = {
|
|
| 129 |
}
|
| 130 |
RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
|
| 131 |
FAILED_STATUSES = {"failed", "error", "failure", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
|
|
|
|
| 132 |
CANCELLED_STATUSES = {"cancelled", "canceled"}
|
| 133 |
|
| 134 |
|
|
@@ -150,7 +152,7 @@ def _canonical_step(step: str | None) -> str | None:
|
|
| 150 |
|
| 151 |
def _is_terminal_status(status: str | None) -> bool:
|
| 152 |
s = str(status or "").lower()
|
| 153 |
-
return s in DONE_STATUSES or s in PARTIAL_STATUSES or s in FAILED_STATUSES or s in CANCELLED_STATUSES
|
| 154 |
|
| 155 |
|
| 156 |
def _progress_for_index(index: int) -> int:
|
|
@@ -243,6 +245,8 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
|
|
| 243 |
terminal_status = "cancelled"
|
| 244 |
elif status_from_state in PARTIAL_STATUSES:
|
| 245 |
terminal_status = status_from_state
|
|
|
|
|
|
|
| 246 |
elif status_from_state in DONE_STATUSES:
|
| 247 |
terminal_status = status_from_state
|
| 248 |
elif status_from_state in FAILED_STATUSES:
|
|
@@ -318,7 +322,7 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
|
|
| 318 |
|
| 319 |
if overall_status in FAILED_STATUSES:
|
| 320 |
progress = max(_progress_for_index(furthest_index), 3)
|
| 321 |
-
elif overall_status in CANCELLED_STATUSES:
|
| 322 |
progress = max(_progress_for_index(furthest_index), 3)
|
| 323 |
elif overall_status in DONE_STATUSES or overall_status in PARTIAL_STATUSES or current_step == "done":
|
| 324 |
progress = 100
|
|
@@ -350,7 +354,7 @@ def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str
|
|
| 350 |
"current_step": current_step,
|
| 351 |
"current_step_label": STEP_LABELS.get(current_step, current_step),
|
| 352 |
"last_event": last_message or "No events yet",
|
| 353 |
-
"visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))),
|
| 354 |
"elapsed_seconds": max(0, elapsed),
|
| 355 |
"eta_seconds": None,
|
| 356 |
"timeline": timeline,
|
|
|
|
| 96 |
}
|
| 97 |
|
| 98 |
STEP_ALIASES = {
|
| 99 |
+
"provider_quota": "pi_run",
|
| 100 |
"bucket_ready": "bootstrap",
|
| 101 |
"job_launched": "bootstrap",
|
| 102 |
"hardware_preferred": "hardware_strategy",
|
|
|
|
| 130 |
}
|
| 131 |
RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
|
| 132 |
FAILED_STATUSES = {"failed", "error", "failure", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
|
| 133 |
+
BLOCKED_STATUSES = {"provider_quota_blocked"}
|
| 134 |
CANCELLED_STATUSES = {"cancelled", "canceled"}
|
| 135 |
|
| 136 |
|
|
|
|
| 152 |
|
| 153 |
def _is_terminal_status(status: str | None) -> bool:
|
| 154 |
s = str(status or "").lower()
|
| 155 |
+
return s in DONE_STATUSES or s in PARTIAL_STATUSES or s in FAILED_STATUSES or s in BLOCKED_STATUSES or s in CANCELLED_STATUSES
|
| 156 |
|
| 157 |
|
| 158 |
def _progress_for_index(index: int) -> int:
|
|
|
|
| 245 |
terminal_status = "cancelled"
|
| 246 |
elif status_from_state in PARTIAL_STATUSES:
|
| 247 |
terminal_status = status_from_state
|
| 248 |
+
elif status_from_state in BLOCKED_STATUSES:
|
| 249 |
+
terminal_status = status_from_state
|
| 250 |
elif status_from_state in DONE_STATUSES:
|
| 251 |
terminal_status = status_from_state
|
| 252 |
elif status_from_state in FAILED_STATUSES:
|
|
|
|
| 322 |
|
| 323 |
if overall_status in FAILED_STATUSES:
|
| 324 |
progress = max(_progress_for_index(furthest_index), 3)
|
| 325 |
+
elif overall_status in CANCELLED_STATUSES or overall_status in BLOCKED_STATUSES:
|
| 326 |
progress = max(_progress_for_index(furthest_index), 3)
|
| 327 |
elif overall_status in DONE_STATUSES or overall_status in PARTIAL_STATUSES or current_step == "done":
|
| 328 |
progress = 100
|
|
|
|
| 354 |
"current_step": current_step,
|
| 355 |
"current_step_label": STEP_LABELS.get(current_step, current_step),
|
| 356 |
"last_event": last_message or "No events yet",
|
| 357 |
+
"visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES or overall_status in BLOCKED_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))),
|
| 358 |
"elapsed_seconds": max(0, elapsed),
|
| 359 |
"eta_seconds": None,
|
| 360 |
"timeline": timeline,
|
src/timeline_model.py
CHANGED
|
@@ -31,7 +31,7 @@ PHASE_LABELS = {
|
|
| 31 |
PHASE_STEPS = {
|
| 32 |
"start": {"token_context", "bootstrap", "dependencies", "auth", "workspace", "node", "pi_install"},
|
| 33 |
"model": {"model_analysis", "model_prescan"},
|
| 34 |
-
"agent": {"pi_config", "pi_run", "traces", "pi_model_resolution", "pi_verification"},
|
| 35 |
"hardware": {"hardware_strategy", "create_space_hardware", "create_space"},
|
| 36 |
"deploy": {"metadata_sanitize", "requirements_sanitize", "upload_files", "space_logs", "space_runtime", "live_wait"},
|
| 37 |
"live_validation": {"endpoint_discovery", "api_validation", "generation_smoke", "inference_gate"},
|
|
@@ -51,7 +51,7 @@ PHASE_STEPS = {
|
|
| 51 |
"repair_validation",
|
| 52 |
},
|
| 53 |
"archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
|
| 54 |
-
"done": {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required"},
|
| 55 |
}
|
| 56 |
|
| 57 |
STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
|
|
@@ -62,7 +62,8 @@ FAILED_STATUSES = {"failed", "failure", "error"}
|
|
| 62 |
RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
|
| 63 |
CANCELLED_STATUSES = {"cancelled", "canceled"}
|
| 64 |
MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
|
| 65 |
-
BLOCKED_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
|
|
|
|
| 66 |
AUTH_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
|
| 67 |
|
| 68 |
TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status")
|
|
@@ -783,6 +784,8 @@ def _normalize_verdict_token(value: Any) -> str:
|
|
| 783 |
return "partial_validation"
|
| 784 |
if normalized in MANUAL_STATUSES:
|
| 785 |
return "manual_action_required"
|
|
|
|
|
|
|
| 786 |
if normalized in BLOCKED_STATUSES:
|
| 787 |
return "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
|
| 788 |
if normalized in FAILED_STATUSES:
|
|
@@ -846,10 +849,14 @@ def _final_visual_status(verdict: str) -> str:
|
|
| 846 |
return "success"
|
| 847 |
if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
|
| 848 |
return "warn"
|
|
|
|
|
|
|
| 849 |
if verdict in {"failed", "failure", "technical_blocker", "technical_blocker_boot_only"}:
|
| 850 |
return "error"
|
| 851 |
if verdict == "auth_refresh_required":
|
| 852 |
return "warn"
|
|
|
|
|
|
|
| 853 |
if verdict in {"manual_action_required", "cancelled"}:
|
| 854 |
return "stopped"
|
| 855 |
return "running"
|
|
@@ -866,6 +873,8 @@ def _status_label(verdict: str) -> tuple[str, str]:
|
|
| 866 |
return "Technical blocker", "The run found a technical blocker."
|
| 867 |
if verdict == "auth_refresh_required":
|
| 868 |
return "Auth refresh required", "HF OAuth expired or is too close to expiry; sign in again before retrying validation."
|
|
|
|
|
|
|
| 869 |
if verdict == "manual_action_required":
|
| 870 |
return "Manual action required", "The run needs user action before validation can continue."
|
| 871 |
if verdict == "failed":
|
|
|
|
| 31 |
PHASE_STEPS = {
|
| 32 |
"start": {"token_context", "bootstrap", "dependencies", "auth", "workspace", "node", "pi_install"},
|
| 33 |
"model": {"model_analysis", "model_prescan"},
|
| 34 |
+
"agent": {"pi_config", "pi_run", "provider_quota", "traces", "pi_model_resolution", "pi_verification"},
|
| 35 |
"hardware": {"hardware_strategy", "create_space_hardware", "create_space"},
|
| 36 |
"deploy": {"metadata_sanitize", "requirements_sanitize", "upload_files", "space_logs", "space_runtime", "live_wait"},
|
| 37 |
"live_validation": {"endpoint_discovery", "api_validation", "generation_smoke", "inference_gate"},
|
|
|
|
| 51 |
"repair_validation",
|
| 52 |
},
|
| 53 |
"archive": {"report_write", "artifact_manifest", "anonymous_eval", "eval_publish", "eval_archive"},
|
| 54 |
+
"done": {"done", "failure", "technical_blocker", "technical_blocker_boot_only", "provider_quota", "manual_hardware_required"},
|
| 55 |
}
|
| 56 |
|
| 57 |
STEP_TO_PHASE = {step: phase for phase, steps in PHASE_STEPS.items() for step in steps}
|
|
|
|
| 62 |
RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"}
|
| 63 |
CANCELLED_STATUSES = {"cancelled", "canceled"}
|
| 64 |
MANUAL_STATUSES = {"manual_hardware_required", "manual_action_required", "generated_needs_manual_hardware", "waiting_manual_hardware"}
|
| 65 |
+
BLOCKED_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked", "provider_quota_blocked"}
|
| 66 |
+
PROVIDER_QUOTA_STATUSES = {"provider_quota_blocked"}
|
| 67 |
AUTH_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
|
| 68 |
|
| 69 |
TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status")
|
|
|
|
| 784 |
return "partial_validation"
|
| 785 |
if normalized in MANUAL_STATUSES:
|
| 786 |
return "manual_action_required"
|
| 787 |
+
if normalized in PROVIDER_QUOTA_STATUSES:
|
| 788 |
+
return "provider_quota_blocked"
|
| 789 |
if normalized in BLOCKED_STATUSES:
|
| 790 |
return "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
|
| 791 |
if normalized in FAILED_STATUSES:
|
|
|
|
| 849 |
return "success"
|
| 850 |
if verdict in {"partial_validation", "partial", "health_only", "completed_with_warnings"}:
|
| 851 |
return "warn"
|
| 852 |
+
if verdict in {"provider_quota_blocked"}:
|
| 853 |
+
return "warn"
|
| 854 |
if verdict in {"failed", "failure", "technical_blocker", "technical_blocker_boot_only"}:
|
| 855 |
return "error"
|
| 856 |
if verdict == "auth_refresh_required":
|
| 857 |
return "warn"
|
| 858 |
+
if verdict == "provider_quota_blocked":
|
| 859 |
+
return "warn"
|
| 860 |
if verdict in {"manual_action_required", "cancelled"}:
|
| 861 |
return "stopped"
|
| 862 |
return "running"
|
|
|
|
| 873 |
return "Technical blocker", "The run found a technical blocker."
|
| 874 |
if verdict == "auth_refresh_required":
|
| 875 |
return "Auth refresh required", "HF OAuth expired or is too close to expiry; sign in again before retrying validation."
|
| 876 |
+
if verdict == "provider_quota_blocked":
|
| 877 |
+
return "Provider quota blocked", "HF Inference Providers quota or monthly spending limit blocked Pi before demo generation."
|
| 878 |
if verdict == "manual_action_required":
|
| 879 |
return "Manual action required", "The run needs user action before validation can continue."
|
| 880 |
if verdict == "failed":
|
src/version.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
ASF_APP_VERSION = "v198.26.
|
| 4 |
-
ASF_RELEASE_NAME = "Agentic Space Factory v198.26.
|
| 5 |
|
| 6 |
|
| 7 |
def resolve_app_version(value: str | None = None) -> str:
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
ASF_APP_VERSION = "v198.26.130"
|
| 4 |
+
ASF_RELEASE_NAME = "Agentic Space Factory v198.26.130"
|
| 5 |
|
| 6 |
|
| 7 |
def resolve_app_version(value: str | None = None) -> str:
|
src/view_models.py
CHANGED
|
@@ -44,11 +44,12 @@ MANUAL_RAW_STATUSES = {
|
|
| 44 |
FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
|
| 45 |
CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
|
| 46 |
BLOCKED_RAW_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
|
|
|
|
| 47 |
RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
|
| 48 |
AUTH_RAW_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
|
| 49 |
|
| 50 |
AUTHORITATIVE_TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status", "post_repair_validation", "failure_type")
|
| 51 |
-
TERMINAL_DISPLAY_STATUSES = SUCCESS_RAW_STATUSES | PARTIAL_RAW_STATUSES | MANUAL_RAW_STATUSES | FAILED_RAW_STATUSES | CANCELLED_RAW_STATUSES | BLOCKED_RAW_STATUSES | AUTH_RAW_STATUSES | {"stale", "stopped", "succeeded", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation"}
|
| 52 |
|
| 53 |
STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
|
| 54 |
|
|
@@ -203,7 +204,7 @@ def _first_authoritative_terminal_status(bundle: dict[str, Any]) -> tuple[str, s
|
|
| 203 |
for key in AUTHORITATIVE_TERMINAL_SOURCE_KEYS:
|
| 204 |
normalized = _normalize_terminal_status(source.get(key))
|
| 205 |
if normalized:
|
| 206 |
-
if normalized in BLOCKED_RAW_STATUSES | FAILED_RAW_STATUSES | MANUAL_RAW_STATUSES | AUTH_RAW_STATUSES | CANCELLED_RAW_STATUSES | {"stopped", "stale"}:
|
| 207 |
return normalized, f"{source_name}.{key}"
|
| 208 |
if normalized == "full_inference_success" and blocking_seen:
|
| 209 |
continue
|
|
@@ -245,6 +246,8 @@ def _status_model_from_terminal_status(status: str, *, source: str = "") -> dict
|
|
| 245 |
global_status, verdict = "partial", normalized if normalized not in {"partial", "health_only", "full_inference_candidate_health_passed", "completed_with_warnings"} else "partial_validation"
|
| 246 |
elif normalized in MANUAL_RAW_STATUSES:
|
| 247 |
global_status, verdict = "waiting_manual_action", "manual_action_required"
|
|
|
|
|
|
|
| 248 |
elif normalized in BLOCKED_RAW_STATUSES:
|
| 249 |
global_status, verdict = "blocked", "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
|
| 250 |
elif normalized in FAILED_RAW_STATUSES:
|
|
@@ -310,6 +313,9 @@ def normalize_run_status(
|
|
| 310 |
elif manual:
|
| 311 |
global_status = "waiting_manual_action"
|
| 312 |
verdict = "manual_action_required"
|
|
|
|
|
|
|
|
|
|
| 313 |
elif blocker:
|
| 314 |
global_status = "blocked"
|
| 315 |
verdict = "technical_blocker_boot_only" if "technical_blocker_boot_only" in statuses else "technical_blocker"
|
|
@@ -479,7 +485,12 @@ def build_diagnostics(bundle: dict[str, Any], status_model: dict[str, Any], phas
|
|
| 479 |
issue_title = ""
|
| 480 |
issue_detail = ""
|
| 481 |
issue_status = ""
|
| 482 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 483 |
first = blocker_items[0]
|
| 484 |
issue_title = _first_nonempty(first.get("type"), first.get("name"), "Technical blocker") if isinstance(first, dict) else "Technical blocker"
|
| 485 |
issue_detail = _first_nonempty(first.get("claim"), first.get("reason"), first.get("message")) if isinstance(first, dict) else str(first)
|
|
|
|
| 44 |
FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"}
|
| 45 |
CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
|
| 46 |
BLOCKED_RAW_STATUSES = {"technical_blocker", "technical_blocker_boot_only", "blocked"}
|
| 47 |
+
PROVIDER_QUOTA_RAW_STATUSES = {"provider_quota_blocked"}
|
| 48 |
RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
|
| 49 |
AUTH_RAW_STATUSES = {"auth_refresh_required", "oauth_expired", "auth_expired", "repair_validation_inconclusive_auth", "inconclusive_auth_expired"}
|
| 50 |
|
| 51 |
AUTHORITATIVE_TERMINAL_SOURCE_KEYS = ("status", "final_status", "effective_status", "effective_verdict", "display_status", "verdict", "result_status", "validation_status", "post_repair_validation", "failure_type")
|
| 52 |
+
TERMINAL_DISPLAY_STATUSES = SUCCESS_RAW_STATUSES | PARTIAL_RAW_STATUSES | MANUAL_RAW_STATUSES | FAILED_RAW_STATUSES | CANCELLED_RAW_STATUSES | BLOCKED_RAW_STATUSES | PROVIDER_QUOTA_RAW_STATUSES | AUTH_RAW_STATUSES | {"stale", "stopped", "succeeded", "validated_after_space_test", "validated_after_manual_space_test", "success_with_expert_intervention", "expert_intervention_validation_passed", "recovered_by_space_test", "recovered_by_manual_validation"}
|
| 53 |
|
| 54 |
STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
|
| 55 |
|
|
|
|
| 204 |
for key in AUTHORITATIVE_TERMINAL_SOURCE_KEYS:
|
| 205 |
normalized = _normalize_terminal_status(source.get(key))
|
| 206 |
if normalized:
|
| 207 |
+
if normalized in BLOCKED_RAW_STATUSES | PROVIDER_QUOTA_RAW_STATUSES | FAILED_RAW_STATUSES | MANUAL_RAW_STATUSES | AUTH_RAW_STATUSES | CANCELLED_RAW_STATUSES | {"stopped", "stale"}:
|
| 208 |
return normalized, f"{source_name}.{key}"
|
| 209 |
if normalized == "full_inference_success" and blocking_seen:
|
| 210 |
continue
|
|
|
|
| 246 |
global_status, verdict = "partial", normalized if normalized not in {"partial", "health_only", "full_inference_candidate_health_passed", "completed_with_warnings"} else "partial_validation"
|
| 247 |
elif normalized in MANUAL_RAW_STATUSES:
|
| 248 |
global_status, verdict = "waiting_manual_action", "manual_action_required"
|
| 249 |
+
elif normalized in PROVIDER_QUOTA_RAW_STATUSES:
|
| 250 |
+
global_status, verdict = "blocked", "provider_quota_blocked"
|
| 251 |
elif normalized in BLOCKED_RAW_STATUSES:
|
| 252 |
global_status, verdict = "blocked", "technical_blocker_boot_only" if normalized == "technical_blocker_boot_only" else "technical_blocker"
|
| 253 |
elif normalized in FAILED_RAW_STATUSES:
|
|
|
|
| 313 |
elif manual:
|
| 314 |
global_status = "waiting_manual_action"
|
| 315 |
verdict = "manual_action_required"
|
| 316 |
+
elif statuses.intersection(PROVIDER_QUOTA_RAW_STATUSES):
|
| 317 |
+
global_status = "blocked"
|
| 318 |
+
verdict = "provider_quota_blocked"
|
| 319 |
elif blocker:
|
| 320 |
global_status = "blocked"
|
| 321 |
verdict = "technical_blocker_boot_only" if "technical_blocker_boot_only" in statuses else "technical_blocker"
|
|
|
|
| 485 |
issue_title = ""
|
| 486 |
issue_detail = ""
|
| 487 |
issue_status = ""
|
| 488 |
+
provider_quota = bundle.get("provider_quota_blocker") or (bundle.get("state") or {}).get("provider_quota_blocker") or ((bundle.get("state") or {}).get("details") or {}).get("provider_quota_blocker")
|
| 489 |
+
if isinstance(provider_quota, dict) and provider_quota:
|
| 490 |
+
issue_title = provider_quota.get("title") or "Provider quota blocked"
|
| 491 |
+
issue_detail = provider_quota.get("user_message") or provider_quota.get("summary") or "Hugging Face Inference Providers quota or billing blocked Pi before ASF could generate the demo."
|
| 492 |
+
issue_status = "action_required"
|
| 493 |
+
elif blocker_items:
|
| 494 |
first = blocker_items[0]
|
| 495 |
issue_title = _first_nonempty(first.get("type"), first.get("name"), "Technical blocker") if isinstance(first, dict) else "Technical blocker"
|
| 496 |
issue_detail = _first_nonempty(first.get("claim"), first.get("reason"), first.get("message")) if isinstance(first, dict) else str(first)
|
src/worker_payload.py
CHANGED
|
@@ -40,8 +40,8 @@ DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large"
|
|
| 40 |
DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2"
|
| 41 |
MAX_PI_REPAIR_ATTEMPTS = 3
|
| 42 |
DEFAULT_FACTORY_UPLOAD_RETRY_BUDGET = 1
|
| 43 |
-
APP_VERSION = "v198.26.
|
| 44 |
-
app_version = "v198.26.
|
| 45 |
|
| 46 |
# Internal agent/recovery files may be needed inside the transient Pi
|
| 47 |
# workspace, but they should not be published to the generated Space or shown
|
|
@@ -630,6 +630,7 @@ def build_runtime_upload_payload(workspace: Path, run_dir: Path, events_path: Pa
|
|
| 630 |
# candidate and stamp the app so /health can prove which candidate is
|
| 631 |
# actually being served. No further runtime mutation is allowed after this.
|
| 632 |
freeze_release_candidate(payload_dir, workspace, run_dir, events_path, manifest, target_space_id=os.environ.get("TARGET_SPACE_ID", ""))
|
|
|
|
| 633 |
manifest_path = run_dir / "runtime_upload_payload_manifest.json"
|
| 634 |
write_json(manifest_path, manifest)
|
| 635 |
|
|
@@ -739,6 +740,46 @@ def _current_repair_attempt_for_release(run_dir: Path) -> int | None:
|
|
| 739 |
return None
|
| 740 |
|
| 741 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
def _insert_asf_release_constant(app_text: str, release: dict) -> str:
|
| 743 |
marker = "# ASF runtime release identity stamp"
|
| 744 |
if marker in app_text:
|
|
@@ -772,48 +813,90 @@ def _insert_asf_release_constant(app_text: str, release: dict) -> str:
|
|
| 772 |
"\n"
|
| 773 |
)
|
| 774 |
lines = app_text.splitlines(True)
|
| 775 |
-
insert_at =
|
| 776 |
-
# Preserve shebang/encoding/comments and place after the import block where possible.
|
| 777 |
-
for i, line in enumerate(lines[:80]):
|
| 778 |
-
stripped = line.strip()
|
| 779 |
-
if stripped.startswith("import ") or stripped.startswith("from "):
|
| 780 |
-
insert_at = i + 1
|
| 781 |
lines.insert(insert_at, block)
|
| 782 |
return "".join(lines)
|
| 783 |
|
| 784 |
|
| 785 |
-
def
|
| 786 |
try:
|
| 787 |
tree = ast.parse(app_text)
|
| 788 |
except Exception:
|
| 789 |
-
return
|
| 790 |
-
wrappers: list[tuple[int, str]] = []
|
| 791 |
-
names = []
|
| 792 |
-
for node in
|
| 793 |
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
| 794 |
name = str(node.name or "")
|
| 795 |
lname = name.lower()
|
| 796 |
if lname in {"health", "health_check", "runtime_health", "get_health"} or ("health" in lname and not lname.startswith("_asf")):
|
| 797 |
if getattr(node, "end_lineno", None):
|
| 798 |
names.append(name)
|
| 799 |
-
wrappers.append((int(node.end_lineno), name))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 800 |
if not wrappers:
|
| 801 |
return app_text, []
|
| 802 |
lines = app_text.splitlines(True)
|
| 803 |
-
for end_lineno, name in sorted(wrappers, reverse=True):
|
| 804 |
alias = f"_asf_original_{name}"
|
| 805 |
-
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
|
| 812 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 813 |
lines.insert(end_lineno, wrapper)
|
| 814 |
return "".join(lines), names
|
| 815 |
|
| 816 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 817 |
def patch_runtime_payload_with_release_identity(payload_dir: Path, release: dict, run_dir: Path, events_path: Path) -> dict:
|
| 818 |
payload_dir.mkdir(parents=True, exist_ok=True)
|
| 819 |
write_json(payload_dir / "ASF_RELEASE.json", release)
|
|
@@ -2487,6 +2570,7 @@ def eval_terminal_status(status: str, phase: str = "") -> bool:
|
|
| 2487 |
"partial_validation",
|
| 2488 |
"technical_blocker",
|
| 2489 |
"manual_hardware_required",
|
|
|
|
| 2490 |
}
|
| 2491 |
|
| 2492 |
|
|
@@ -2541,6 +2625,8 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
|
|
| 2541 |
return "cancelled"
|
| 2542 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 2543 |
return "manual_action_required"
|
|
|
|
|
|
|
| 2544 |
if status in {"technical_blocker", "technical_blocker_boot_only"}:
|
| 2545 |
return "technical_blocker"
|
| 2546 |
if full_inference_verified:
|
|
@@ -2925,7 +3011,10 @@ def run_final_cleanup_if_needed(run_dir: Path, events_path: Path, final_state: d
|
|
| 2925 |
health_semantic_passed = signals.get("health_semantic_passed")
|
| 2926 |
if health_semantic_passed is None:
|
| 2927 |
health_semantic_passed = signals.get("health_passed")
|
| 2928 |
-
|
|
|
|
|
|
|
|
|
|
| 2929 |
decision = {"should_pause": False, "reason": "runtime_recovery_pending", "final_status": final_status, "selected_hardware": selected_hardware}
|
| 2930 |
else:
|
| 2931 |
decision = should_pause_generated_space(final_status, target_space, selected_hardware, health_semantic_passed=health_semantic_passed if isinstance(health_semantic_passed, bool) else None)
|
|
@@ -2949,9 +3038,140 @@ def run_final_cleanup_if_needed(run_dir: Path, events_path: Path, final_state: d
|
|
| 2949 |
write_json(run_dir / "state.json", state)
|
| 2950 |
return cleanup
|
| 2951 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2952 |
def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):
|
| 2953 |
safe = safe_details(details)
|
| 2954 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2955 |
existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
|
| 2956 |
if not isinstance(existing_state, dict):
|
| 2957 |
existing_state = {}
|
|
@@ -2969,11 +3189,21 @@ def fail(run_dir: Path, events_path: Path, message: str, details: dict | None =
|
|
| 2969 |
"details": safe,
|
| 2970 |
"repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {},
|
| 2971 |
}
|
| 2972 |
-
# Preserve
|
|
|
|
|
|
|
| 2973 |
target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
|
| 2974 |
-
|
|
|
|
| 2975 |
failure_state["target_space"] = target_space
|
| 2976 |
failure_state["target_space_url"] = f"https://huggingface.co/spaces/{target_space}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2977 |
write_json(run_dir / "state.json", failure_state)
|
| 2978 |
cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, failure_state)
|
| 2979 |
failure_state["cleanup"] = cleanup_status
|
|
@@ -9360,9 +9590,11 @@ def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None
|
|
| 9360 |
runtime_upload_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {}
|
| 9361 |
space_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {}
|
| 9362 |
runtime_history = runtime_upload_epoch.get("history") if isinstance(runtime_upload_epoch.get("history"), list) else []
|
| 9363 |
-
|
| 9364 |
runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history)
|
| 9365 |
space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url")))
|
|
|
|
|
|
|
| 9366 |
payload = {
|
| 9367 |
"schema_version": "final_status_reconciliation.v198_26_9",
|
| 9368 |
"status": status,
|
|
@@ -9370,8 +9602,10 @@ def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None
|
|
| 9370 |
"message": message or state.get("message") or "",
|
| 9371 |
"job_exit_code": int(job_exit_code if job_exit_code is not None else (0 if status == "full_inference_success" else 1)),
|
| 9372 |
"target_space": target_space,
|
|
|
|
| 9373 |
"target_space_url": f"https://huggingface.co/spaces/{target_space}" if target_space else "",
|
| 9374 |
-
"space_created": bool(target_space and (
|
|
|
|
| 9375 |
"runtime_uploaded": bool(runtime_uploaded),
|
| 9376 |
"space_uploaded": bool(runtime_uploaded),
|
| 9377 |
"space_runtime_known": bool(space_runtime_known),
|
|
@@ -15488,6 +15722,12 @@ def main():
|
|
| 15488 |
)
|
| 15489 |
(run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8")
|
| 15490 |
if code != 0:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15491 |
append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]})
|
| 15492 |
collect_pi_traces(run_dir, events_path)
|
| 15493 |
fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]})
|
|
@@ -16688,6 +16928,8 @@ def eval_verdict(status: str, phase: str, health_passed: bool, smoke_passed: boo
|
|
| 16688 |
return "cancelled"
|
| 16689 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 16690 |
return "manual_action_required"
|
|
|
|
|
|
|
| 16691 |
if status in {"technical_blocker", "technical_blocker_boot_only"}:
|
| 16692 |
return "technical_blocker"
|
| 16693 |
if full_inference_verified:
|
|
@@ -20307,7 +20549,7 @@ def smoke_generate(target_space_id: str, token: str, run_dir: Path, events_path:
|
|
| 20307 |
# write_json(run_dir / "tests" / "payload_source.json", payload_source_record)
|
| 20308 |
# write_json(run_dir / "tests" / "replay_source.json", replay_source)
|
| 20309 |
# write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight)
|
| 20310 |
-
app_version = "v198.26.
|
| 20311 |
engine_version = "unified_gradio_validation_harness_v198_25_3"
|
| 20312 |
parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip()
|
| 20313 |
validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"
|
|
|
|
| 40 |
DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2"
|
| 41 |
MAX_PI_REPAIR_ATTEMPTS = 3
|
| 42 |
DEFAULT_FACTORY_UPLOAD_RETRY_BUDGET = 1
|
| 43 |
+
APP_VERSION = "v198.26.130"
|
| 44 |
+
app_version = "v198.26.130"
|
| 45 |
|
| 46 |
# Internal agent/recovery files may be needed inside the transient Pi
|
| 47 |
# workspace, but they should not be published to the generated Space or shown
|
|
|
|
| 630 |
# candidate and stamp the app so /health can prove which candidate is
|
| 631 |
# actually being served. No further runtime mutation is allowed after this.
|
| 632 |
freeze_release_candidate(payload_dir, workspace, run_dir, events_path, manifest, target_space_id=os.environ.get("TARGET_SPACE_ID", ""))
|
| 633 |
+
release_candidate_payload_compile_guard(payload_dir, run_dir, events_path, reason="runtime_upload_payload_post_release_stamp")
|
| 634 |
manifest_path = run_dir / "runtime_upload_payload_manifest.json"
|
| 635 |
write_json(manifest_path, manifest)
|
| 636 |
|
|
|
|
| 740 |
return None
|
| 741 |
|
| 742 |
|
| 743 |
+
def _release_identity_insert_index(app_text: str) -> int:
|
| 744 |
+
"""Return a safe top-level insertion point for the ASF runtime stamp.
|
| 745 |
+
|
| 746 |
+
v198.26.127 inserted after any line whose stripped text looked like an
|
| 747 |
+
import. That accidentally matched imports inside top-level try/except
|
| 748 |
+
blocks, producing invalid Python such as `try: import torch; ASF_STAMP;
|
| 749 |
+
except ...`. Only top-level module body statements are safe anchors.
|
| 750 |
+
"""
|
| 751 |
+
try:
|
| 752 |
+
tree = ast.parse(app_text)
|
| 753 |
+
lines = app_text.splitlines(True)
|
| 754 |
+
insert_after = 0
|
| 755 |
+
for idx, node in enumerate(tree.body):
|
| 756 |
+
if idx == 0 and isinstance(node, ast.Expr) and isinstance(getattr(node, "value", None), ast.Constant) and isinstance(node.value.value, str):
|
| 757 |
+
insert_after = int(getattr(node, "end_lineno", node.lineno) or node.lineno)
|
| 758 |
+
continue
|
| 759 |
+
if isinstance(node, ast.ImportFrom) and getattr(node, "module", "") == "__future__":
|
| 760 |
+
insert_after = int(getattr(node, "end_lineno", node.lineno) or node.lineno)
|
| 761 |
+
continue
|
| 762 |
+
if isinstance(node, (ast.Import, ast.ImportFrom)) and int(getattr(node, "col_offset", 0) or 0) == 0:
|
| 763 |
+
insert_after = int(getattr(node, "end_lineno", node.lineno) or node.lineno)
|
| 764 |
+
continue
|
| 765 |
+
break
|
| 766 |
+
return max(0, min(insert_after, len(lines)))
|
| 767 |
+
except Exception:
|
| 768 |
+
lines = app_text.splitlines(True)
|
| 769 |
+
insert_after = 0
|
| 770 |
+
# Conservative fallback: only unindented imports/future imports count.
|
| 771 |
+
for i, line in enumerate(lines[:120]):
|
| 772 |
+
if line.startswith("import ") or line.startswith("from "):
|
| 773 |
+
insert_after = i + 1
|
| 774 |
+
continue
|
| 775 |
+
stripped = line.strip()
|
| 776 |
+
if not stripped or stripped.startswith("#"):
|
| 777 |
+
continue
|
| 778 |
+
if insert_after:
|
| 779 |
+
break
|
| 780 |
+
return insert_after
|
| 781 |
+
|
| 782 |
+
|
| 783 |
def _insert_asf_release_constant(app_text: str, release: dict) -> str:
|
| 784 |
marker = "# ASF runtime release identity stamp"
|
| 785 |
if marker in app_text:
|
|
|
|
| 813 |
"\n"
|
| 814 |
)
|
| 815 |
lines = app_text.splitlines(True)
|
| 816 |
+
insert_at = _release_identity_insert_index(app_text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 817 |
lines.insert(insert_at, block)
|
| 818 |
return "".join(lines)
|
| 819 |
|
| 820 |
|
| 821 |
+
def _top_level_health_functions(app_text: str) -> tuple[list[tuple[int, str, bool]], list[str]]:
|
| 822 |
try:
|
| 823 |
tree = ast.parse(app_text)
|
| 824 |
except Exception:
|
| 825 |
+
return [], []
|
| 826 |
+
wrappers: list[tuple[int, str, bool]] = []
|
| 827 |
+
names: list[str] = []
|
| 828 |
+
for node in tree.body:
|
| 829 |
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
| 830 |
name = str(node.name or "")
|
| 831 |
lname = name.lower()
|
| 832 |
if lname in {"health", "health_check", "runtime_health", "get_health"} or ("health" in lname and not lname.startswith("_asf")):
|
| 833 |
if getattr(node, "end_lineno", None):
|
| 834 |
names.append(name)
|
| 835 |
+
wrappers.append((int(node.end_lineno), name, isinstance(node, ast.AsyncFunctionDef)))
|
| 836 |
+
return wrappers, names
|
| 837 |
+
|
| 838 |
+
|
| 839 |
+
def _wrap_health_functions_with_release_identity(app_text: str, run_dir: Path | None = None) -> tuple[str, list[str]]:
|
| 840 |
+
# Only wrap top-level health functions. Nested functions or class methods
|
| 841 |
+
# must not receive top-level wrapper assignments based on their end_lineno,
|
| 842 |
+
# because that can inject code into an unrelated block and break generated apps.
|
| 843 |
+
wrappers, names = _top_level_health_functions(app_text)
|
| 844 |
if not wrappers:
|
| 845 |
return app_text, []
|
| 846 |
lines = app_text.splitlines(True)
|
| 847 |
+
for end_lineno, name, is_async in sorted(wrappers, reverse=True):
|
| 848 |
alias = f"_asf_original_{name}"
|
| 849 |
+
if is_async:
|
| 850 |
+
wrapper = (
|
| 851 |
+
f"\ntry:\n"
|
| 852 |
+
f" {alias} = {name}\n"
|
| 853 |
+
f" async def {name}(*args, **kwargs):\n"
|
| 854 |
+
f" _asf_result = {alias}(*args, **kwargs)\n"
|
| 855 |
+
f" if hasattr(_asf_result, '__await__'):\n"
|
| 856 |
+
f" _asf_result = await _asf_result\n"
|
| 857 |
+
f" return _asf_stamp_health_payload(_asf_result)\n"
|
| 858 |
+
f"except Exception:\n"
|
| 859 |
+
f" pass\n\n"
|
| 860 |
+
)
|
| 861 |
+
else:
|
| 862 |
+
wrapper = (
|
| 863 |
+
f"\ntry:\n"
|
| 864 |
+
f" {alias} = {name}\n"
|
| 865 |
+
f" def {name}(*args, **kwargs):\n"
|
| 866 |
+
f" return _asf_stamp_health_payload({alias}(*args, **kwargs))\n"
|
| 867 |
+
f"except Exception:\n"
|
| 868 |
+
f" pass\n\n"
|
| 869 |
+
)
|
| 870 |
lines.insert(end_lineno, wrapper)
|
| 871 |
return "".join(lines), names
|
| 872 |
|
| 873 |
|
| 874 |
+
def release_candidate_payload_compile_guard(payload_dir: Path, run_dir: Path, events_path: Path | None, *, reason: str = "release_candidate_post_stamp") -> dict:
|
| 875 |
+
app_path = payload_dir / "app.py"
|
| 876 |
+
result = {
|
| 877 |
+
"schema_version": "release_candidate_payload_compile_guard.v198_26_129",
|
| 878 |
+
"reason": reason,
|
| 879 |
+
"checked_file": "runtime_upload_payload/app.py",
|
| 880 |
+
"passed": False,
|
| 881 |
+
"error": "",
|
| 882 |
+
"updated_at": now(),
|
| 883 |
+
}
|
| 884 |
+
if not app_path.exists():
|
| 885 |
+
result["error"] = "app.py missing from runtime upload payload"
|
| 886 |
+
else:
|
| 887 |
+
try:
|
| 888 |
+
compile(app_path.read_text(encoding="utf-8", errors="ignore"), str(app_path), "exec")
|
| 889 |
+
result["passed"] = True
|
| 890 |
+
except Exception as exc:
|
| 891 |
+
result["error"] = f"{type(exc).__name__}: {exc}"
|
| 892 |
+
write_json(run_dir / "release" / "RELEASE_CANDIDATE_COMPILE_GUARD.json", result)
|
| 893 |
+
if events_path is not None:
|
| 894 |
+
append_event(events_path, "release_candidate_compile_guard", "success" if result.get("passed") else "failed", "Checked stamped runtime payload app.py before upload", result)
|
| 895 |
+
if not result.get("passed"):
|
| 896 |
+
raise RuntimeError("release_candidate_payload_compile_failed: " + str(result.get("error") or "unknown"))
|
| 897 |
+
return result
|
| 898 |
+
|
| 899 |
+
|
| 900 |
def patch_runtime_payload_with_release_identity(payload_dir: Path, release: dict, run_dir: Path, events_path: Path) -> dict:
|
| 901 |
payload_dir.mkdir(parents=True, exist_ok=True)
|
| 902 |
write_json(payload_dir / "ASF_RELEASE.json", release)
|
|
|
|
| 2570 |
"partial_validation",
|
| 2571 |
"technical_blocker",
|
| 2572 |
"manual_hardware_required",
|
| 2573 |
+
"provider_quota_blocked",
|
| 2574 |
}
|
| 2575 |
|
| 2576 |
|
|
|
|
| 2625 |
return "cancelled"
|
| 2626 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 2627 |
return "manual_action_required"
|
| 2628 |
+
if status == "provider_quota_blocked":
|
| 2629 |
+
return "provider_quota_blocked"
|
| 2630 |
if status in {"technical_blocker", "technical_blocker_boot_only"}:
|
| 2631 |
return "technical_blocker"
|
| 2632 |
if full_inference_verified:
|
|
|
|
| 3011 |
health_semantic_passed = signals.get("health_semantic_passed")
|
| 3012 |
if health_semantic_passed is None:
|
| 3013 |
health_semantic_passed = signals.get("health_passed")
|
| 3014 |
+
space_creation = generated_space_creation_evidence(run_dir, events_path)
|
| 3015 |
+
if target_space and not space_creation.get("space_created"):
|
| 3016 |
+
decision = {"should_pause": False, "reason": "target_space_not_created", "final_status": final_status, "selected_hardware": selected_hardware, "target_space_planned": target_space, "space_creation_evidence": space_creation}
|
| 3017 |
+
elif runtime_recovery.get("triggered") and runtime_recovery.get("recovery_attempted") and not runtime_recovery.get("recovery_exhausted"):
|
| 3018 |
decision = {"should_pause": False, "reason": "runtime_recovery_pending", "final_status": final_status, "selected_hardware": selected_hardware}
|
| 3019 |
else:
|
| 3020 |
decision = should_pause_generated_space(final_status, target_space, selected_hardware, health_semantic_passed=health_semantic_passed if isinstance(health_semantic_passed, bool) else None)
|
|
|
|
| 3038 |
write_json(run_dir / "state.json", state)
|
| 3039 |
return cleanup
|
| 3040 |
|
| 3041 |
+
|
| 3042 |
+
PROVIDER_QUOTA_BLOCKED_STATUS = "provider_quota_blocked"
|
| 3043 |
+
PROVIDER_QUOTA_MARKERS = (
|
| 3044 |
+
"monthly spending limit for inference providers",
|
| 3045 |
+
"exceeded your monthly spending limit",
|
| 3046 |
+
"spending limit for inference providers",
|
| 3047 |
+
"inference providers quota",
|
| 3048 |
+
"provider quota",
|
| 3049 |
+
"quota exceeded",
|
| 3050 |
+
"insufficient credits",
|
| 3051 |
+
"billing limit",
|
| 3052 |
+
"payment required",
|
| 3053 |
+
)
|
| 3054 |
+
|
| 3055 |
+
|
| 3056 |
+
def classify_provider_quota_blocker(text: str | None, *, phase: str = "pi_run", returncode: int | None = None) -> dict:
|
| 3057 |
+
"""Return a user-facing blocker when HF Inference Providers quota/billing stopped Pi.
|
| 3058 |
+
|
| 3059 |
+
This is not a generated Space failure. It happens before ASF can produce or
|
| 3060 |
+
validate a demo, so the UI must show an external provider/quota blocker
|
| 3061 |
+
instead of a generic model/build failure.
|
| 3062 |
+
"""
|
| 3063 |
+
raw = str(text or "")
|
| 3064 |
+
low = raw.lower()
|
| 3065 |
+
if not raw:
|
| 3066 |
+
return {}
|
| 3067 |
+
matched = next((marker for marker in PROVIDER_QUOTA_MARKERS if marker in low), "")
|
| 3068 |
+
# HF provider errors often include a structured 403 and the exact spending
|
| 3069 |
+
# limit phrase. Keep this conservative so unrelated model errors still go
|
| 3070 |
+
# through the normal repair path.
|
| 3071 |
+
if not matched and not ("403" in low and "inference provider" in low and any(k in low for k in ("quota", "billing", "spending", "credits", "limit"))):
|
| 3072 |
+
return {}
|
| 3073 |
+
excerpt = redact_text(raw[-2200:])
|
| 3074 |
+
return {
|
| 3075 |
+
"schema_version": "provider_quota_blocker.v198_26_130",
|
| 3076 |
+
"status": PROVIDER_QUOTA_BLOCKED_STATUS,
|
| 3077 |
+
"category": "hf_inference_provider_quota",
|
| 3078 |
+
"phase": phase,
|
| 3079 |
+
"returncode": int(returncode) if returncode is not None else None,
|
| 3080 |
+
"matched_marker": matched or "403 inference provider quota/billing marker",
|
| 3081 |
+
"title": "Provider quota blocked",
|
| 3082 |
+
"summary": "Hugging Face Inference Providers quota or monthly spending limit blocked Pi before ASF could generate the demo.",
|
| 3083 |
+
"user_message": "ASF could not start the Pi coding pass because the Hugging Face Inference Providers quota or monthly spending limit was reached. No Space was uploaded and no model validation was attempted.",
|
| 3084 |
+
"suggested_action": "Check Hugging Face billing/quota settings or switch the Pi provider/model, then retry the build.",
|
| 3085 |
+
"retryable_after_user_action": True,
|
| 3086 |
+
"model_failure": False,
|
| 3087 |
+
"space_failure": False,
|
| 3088 |
+
"space_upload_attempted": False,
|
| 3089 |
+
"output_tail": excerpt,
|
| 3090 |
+
"updated_at": now(),
|
| 3091 |
+
}
|
| 3092 |
+
|
| 3093 |
+
|
| 3094 |
+
def write_provider_quota_blocker(run_dir: Path, events_path: Path, blocker: dict) -> dict:
|
| 3095 |
+
payload = {**(blocker or {}), "updated_at": now()}
|
| 3096 |
+
write_json(run_dir / "PROVIDER_QUOTA_BLOCKER.json", payload)
|
| 3097 |
+
write_json(run_dir / "provider_quota_blocker.json", payload)
|
| 3098 |
+
append_event(
|
| 3099 |
+
events_path,
|
| 3100 |
+
"provider_quota",
|
| 3101 |
+
"blocked",
|
| 3102 |
+
payload.get("user_message") or "Hugging Face Inference Providers quota blocked Pi before Space upload",
|
| 3103 |
+
{k: payload.get(k) for k in ("category", "phase", "returncode", "matched_marker", "suggested_action")},
|
| 3104 |
+
)
|
| 3105 |
+
write_live_status(run_dir, stage="provider_quota", status=PROVIDER_QUOTA_BLOCKED_STATUS, message=payload.get("user_message") or payload.get("summary") or "Provider quota blocked", data=payload)
|
| 3106 |
+
return payload
|
| 3107 |
+
|
| 3108 |
+
|
| 3109 |
+
def generated_space_creation_evidence(run_dir: Path, events_path: Path | None = None) -> dict:
|
| 3110 |
+
"""Return whether ASF has evidence that the target Space repo actually exists.
|
| 3111 |
+
|
| 3112 |
+
target_space_id may be planned before Pi runs. Cleanup/autopause must not act
|
| 3113 |
+
on that planned id unless create/upload/runtime evidence proves the repo was
|
| 3114 |
+
actually created or reused by this run.
|
| 3115 |
+
"""
|
| 3116 |
+
evidence = {"space_created": False, "source": "none", "reason": "no_create_space_or_upload_evidence"}
|
| 3117 |
+
try:
|
| 3118 |
+
runtime_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {}
|
| 3119 |
+
if isinstance(runtime_epoch, dict) and (runtime_epoch.get("last_upload_completed_at") or runtime_epoch.get("upload_sequence") or runtime_epoch.get("history")):
|
| 3120 |
+
return {"space_created": True, "source": "runtime_upload_epoch", "reason": "runtime_upload_recorded"}
|
| 3121 |
+
except Exception:
|
| 3122 |
+
pass
|
| 3123 |
+
for artifact, source in (("space_runtime.json", "space_runtime"), ("hardware_strategy.json", "hardware_strategy")):
|
| 3124 |
+
try:
|
| 3125 |
+
payload = load_json_if_exists(run_dir / artifact) if (run_dir / artifact).exists() else {}
|
| 3126 |
+
if not isinstance(payload, dict) or not payload:
|
| 3127 |
+
continue
|
| 3128 |
+
if source == "space_runtime" and (payload.get("stage") or payload.get("runtime_status") or payload.get("status") or payload.get("url")):
|
| 3129 |
+
return {"space_created": True, "source": source, "reason": "runtime_artifact_present"}
|
| 3130 |
+
if source == "hardware_strategy" and payload.get("created") is True:
|
| 3131 |
+
return {"space_created": True, "source": source, "reason": "hardware_strategy_created_true"}
|
| 3132 |
+
except Exception:
|
| 3133 |
+
pass
|
| 3134 |
+
try:
|
| 3135 |
+
path = events_path or (run_dir / "events.jsonl")
|
| 3136 |
+
if path and path.exists():
|
| 3137 |
+
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
| 3138 |
+
if not line.strip():
|
| 3139 |
+
continue
|
| 3140 |
+
try:
|
| 3141 |
+
event = json.loads(line)
|
| 3142 |
+
except Exception:
|
| 3143 |
+
continue
|
| 3144 |
+
if not isinstance(event, dict):
|
| 3145 |
+
continue
|
| 3146 |
+
step = str(event.get("step") or "").lower()
|
| 3147 |
+
status = str(event.get("status") or "").lower()
|
| 3148 |
+
data = event.get("data") if isinstance(event.get("data"), dict) else {}
|
| 3149 |
+
if step == "create_space" and status in {"success", "warning"}:
|
| 3150 |
+
return {"space_created": True, "source": "events.create_space", "reason": status}
|
| 3151 |
+
if step in {"upload_files", "repair_upload"} and status == "success":
|
| 3152 |
+
return {"space_created": True, "source": f"events.{step}", "reason": "upload_success"}
|
| 3153 |
+
if data.get("created") is True or data.get("ok") is True and step == "create_space":
|
| 3154 |
+
return {"space_created": True, "source": "events.create_space.data", "reason": "created_or_ok_true"}
|
| 3155 |
+
except Exception:
|
| 3156 |
+
pass
|
| 3157 |
+
return evidence
|
| 3158 |
+
|
| 3159 |
+
|
| 3160 |
def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):
|
| 3161 |
safe = safe_details(details)
|
| 3162 |
+
quota_source = "\n".join([str(message or ""), str(safe.get("output_tail") or ""), str(safe.get("error") or "")])
|
| 3163 |
+
quota_blocker = safe.get("provider_quota_blocker") if isinstance(safe.get("provider_quota_blocker"), dict) else classify_provider_quota_blocker(quota_source, phase=str(safe.get("phase") or safe.get("step") or "failure"), returncode=safe.get("returncode") if isinstance(safe.get("returncode"), int) else None)
|
| 3164 |
+
if quota_blocker:
|
| 3165 |
+
status = PROVIDER_QUOTA_BLOCKED_STATUS
|
| 3166 |
+
safe["provider_quota_blocker"] = quota_blocker
|
| 3167 |
+
safe.setdefault("failure_type", PROVIDER_QUOTA_BLOCKED_STATUS)
|
| 3168 |
+
safe.setdefault("failure_owner", "hf_inference_provider")
|
| 3169 |
+
message = quota_blocker.get("user_message") or message
|
| 3170 |
+
try:
|
| 3171 |
+
write_provider_quota_blocker(run_dir, events_path, quota_blocker)
|
| 3172 |
+
except Exception:
|
| 3173 |
+
pass
|
| 3174 |
+
append_event(events_path, "failure", "blocked" if status == PROVIDER_QUOTA_BLOCKED_STATUS else "failed", message, safe)
|
| 3175 |
existing_state = load_json_if_exists(run_dir / "state.json") if (run_dir / "state.json").exists() else {}
|
| 3176 |
if not isinstance(existing_state, dict):
|
| 3177 |
existing_state = {}
|
|
|
|
| 3189 |
"details": safe,
|
| 3190 |
"repair_outcome": repair_outcome if isinstance(repair_outcome, dict) else {},
|
| 3191 |
}
|
| 3192 |
+
# Preserve target Space links only when the worker has evidence that the repo
|
| 3193 |
+
# was actually created/reused. Before Pi finishes, TARGET_SPACE_ID is only a
|
| 3194 |
+
# plan and cleanup/UI links must not treat it as a real Space.
|
| 3195 |
target_space = failure_state.get("target_space") or os.environ.get("TARGET_SPACE_ID") or ""
|
| 3196 |
+
space_creation = generated_space_creation_evidence(run_dir, events_path)
|
| 3197 |
+
if target_space and space_creation.get("space_created"):
|
| 3198 |
failure_state["target_space"] = target_space
|
| 3199 |
failure_state["target_space_url"] = f"https://huggingface.co/spaces/{target_space}"
|
| 3200 |
+
failure_state["space_creation_evidence"] = space_creation
|
| 3201 |
+
elif target_space:
|
| 3202 |
+
failure_state.pop("target_space", None)
|
| 3203 |
+
failure_state.pop("target_space_id", None)
|
| 3204 |
+
failure_state.pop("target_space_url", None)
|
| 3205 |
+
failure_state["target_space_planned"] = target_space
|
| 3206 |
+
failure_state["space_creation_evidence"] = space_creation
|
| 3207 |
write_json(run_dir / "state.json", failure_state)
|
| 3208 |
cleanup_status = run_final_cleanup_if_needed(run_dir, events_path, failure_state)
|
| 3209 |
failure_state["cleanup"] = cleanup_status
|
|
|
|
| 9590 |
runtime_upload_epoch = load_json_if_exists(run_dir / "runtime_upload_epoch.json") if (run_dir / "runtime_upload_epoch.json").exists() else {}
|
| 9591 |
space_runtime = load_json_if_exists(run_dir / "space_runtime.json") if (run_dir / "space_runtime.json").exists() else {}
|
| 9592 |
runtime_history = runtime_upload_epoch.get("history") if isinstance(runtime_upload_epoch.get("history"), list) else []
|
| 9593 |
+
planned_target_space = state.get("target_space") or state.get("target_space_id") or runtime_upload_epoch.get("target_space_id") or os.environ.get("TARGET_SPACE_ID", "")
|
| 9594 |
runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history)
|
| 9595 |
space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url")))
|
| 9596 |
+
space_creation = generated_space_creation_evidence(run_dir, events_path)
|
| 9597 |
+
target_space = planned_target_space if space_creation.get("space_created") else ""
|
| 9598 |
payload = {
|
| 9599 |
"schema_version": "final_status_reconciliation.v198_26_9",
|
| 9600 |
"status": status,
|
|
|
|
| 9602 |
"message": message or state.get("message") or "",
|
| 9603 |
"job_exit_code": int(job_exit_code if job_exit_code is not None else (0 if status == "full_inference_success" else 1)),
|
| 9604 |
"target_space": target_space,
|
| 9605 |
+
"target_space_planned": planned_target_space if not target_space else "",
|
| 9606 |
"target_space_url": f"https://huggingface.co/spaces/{target_space}" if target_space else "",
|
| 9607 |
+
"space_created": bool(target_space and space_creation.get("space_created")),
|
| 9608 |
+
"space_creation_evidence": space_creation,
|
| 9609 |
"runtime_uploaded": bool(runtime_uploaded),
|
| 9610 |
"space_uploaded": bool(runtime_uploaded),
|
| 9611 |
"space_runtime_known": bool(space_runtime_known),
|
|
|
|
| 15722 |
)
|
| 15723 |
(run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8")
|
| 15724 |
if code != 0:
|
| 15725 |
+
quota_blocker = classify_provider_quota_blocker(pi_out, phase="pi_run", returncode=code)
|
| 15726 |
+
if quota_blocker:
|
| 15727 |
+
append_event(events_path, "pi_run", "blocked", "Pi was blocked by Hugging Face Inference Providers quota/billing", {"returncode": code, "provider_quota_blocked": True, "output_tail": pi_out[-4000:]})
|
| 15728 |
+
write_provider_quota_blocker(run_dir, events_path, quota_blocker)
|
| 15729 |
+
collect_pi_traces(run_dir, events_path)
|
| 15730 |
+
fail(run_dir, events_path, quota_blocker.get("user_message") or "Pi provider quota blocked before Space upload", {"returncode": code, "output_tail": pi_out[-4000:], "provider_quota_blocker": quota_blocker, "phase": "pi_run"}, status=PROVIDER_QUOTA_BLOCKED_STATUS)
|
| 15731 |
append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]})
|
| 15732 |
collect_pi_traces(run_dir, events_path)
|
| 15733 |
fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]})
|
|
|
|
| 16928 |
return "cancelled"
|
| 16929 |
if status == "manual_hardware_required" or inference_gate.get("manual_hardware_required"):
|
| 16930 |
return "manual_action_required"
|
| 16931 |
+
if status == "provider_quota_blocked":
|
| 16932 |
+
return "provider_quota_blocked"
|
| 16933 |
if status in {"technical_blocker", "technical_blocker_boot_only"}:
|
| 16934 |
return "technical_blocker"
|
| 16935 |
if full_inference_verified:
|
|
|
|
| 20549 |
# write_json(run_dir / "tests" / "payload_source.json", payload_source_record)
|
| 20550 |
# write_json(run_dir / "tests" / "replay_source.json", replay_source)
|
| 20551 |
# write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight)
|
| 20552 |
+
app_version = "v198.26.130"
|
| 20553 |
engine_version = "unified_gradio_validation_harness_v198_25_3"
|
| 20554 |
parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip()
|
| 20555 |
validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"
|