Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Upload 15 files
Browse files- src/bucket.py +22 -10
- src/timeline_model.py +73 -3
- src/version.py +2 -2
- src/view_models.py +133 -4
- src/worker_payload.py +5 -5
src/bucket.py
CHANGED
|
@@ -1285,7 +1285,19 @@ def summarize_run_bundle(run_id: str, bundle: dict[str, Any], *, bucket_source:
|
|
| 1285 |
gate = bundle.get("inference_gate") or {}
|
| 1286 |
smoke = bundle.get("generation_smoke") or {}
|
| 1287 |
hardware = bundle.get("hardware_strategy") or {}
|
| 1288 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1289 |
eval_publish_status = bundle.get("eval_publish_status") or {}
|
| 1290 |
final_status_reconciliation = bundle.get("final_status_reconciliation") or {}
|
| 1291 |
eval_record = bundle.get("eval_record") or {}
|
|
@@ -1339,15 +1351,15 @@ def summarize_run_bundle(run_id: str, bundle: dict[str, Any], *, bucket_source:
|
|
| 1339 |
"updated_at": state.get("updated_at") or state.get("created_at") or launch.get("updated_at") or launch.get("created_at") or summary_file.get("updated_at") or summary_file.get("created_at") or "",
|
| 1340 |
"selected_hardware": hardware.get("selected_hardware") or state.get("selected_hardware") or launch.get("preferred_space_hardware") or summary_file.get("selected_hardware") or state.get("hardware") or "",
|
| 1341 |
"manual_hardware_required": bool(gate.get("manual_hardware_required") or hardware.get("manual_action_required") or launch.get("manual_hardware_required")),
|
| 1342 |
-
"health_passed": bool((gate.get("implementation_signals") or {}).get("health_passed") or smoke.get("health_passed")),
|
| 1343 |
-
"smoke_test_passed": bool(smoke.get("ok") or smoke.get("status") == "success"),
|
| 1344 |
-
"latency_seconds": smoke.get("latency_seconds") or smoke.get("observed_latency_seconds") or
|
| 1345 |
-
"observed_latency_seconds": smoke.get("observed_latency_seconds") or smoke.get("latency_seconds") or
|
| 1346 |
-
"recommended_zero_gpu_duration_seconds": smoke.get("recommended_zero_gpu_duration_seconds") or smoke.get("recommended_zerogpu_duration_seconds") or
|
| 1347 |
-
"recommended_zerogpu_duration_seconds": smoke.get("recommended_zerogpu_duration_seconds") or smoke.get("recommended_zero_gpu_duration_seconds") or
|
| 1348 |
-
"recommendation_source": smoke.get("recommendation_source") or
|
| 1349 |
-
"recommendation_hardware":
|
| 1350 |
-
"hardware_used_for_validation": manual_validation.get("hardware_used_for_validation") or manual_validation.get("recommendation_hardware") or "",
|
| 1351 |
"manual_validation_status": manual_validation,
|
| 1352 |
"manual_validation_passed": bool(manual_validation.get("status") == "success"),
|
| 1353 |
"post_build_status": effective_run_status.get("post_build_status") or "none",
|
|
|
|
| 1285 |
gate = bundle.get("inference_gate") or {}
|
| 1286 |
smoke = bundle.get("generation_smoke") or {}
|
| 1287 |
hardware = bundle.get("hardware_strategy") or {}
|
| 1288 |
+
post_build_validation = bundle.get("post_build_validation_status") or {}
|
| 1289 |
+
manual_validation = post_build_validation or bundle.get("manual_validation_status") or {}
|
| 1290 |
+
linked_validations = bundle.get("linked_validations") or {}
|
| 1291 |
+
linked_rows = linked_validations.get("validations") if isinstance(linked_validations, dict) else []
|
| 1292 |
+
if isinstance(linked_rows, list):
|
| 1293 |
+
linked_successes = [row for row in linked_rows if isinstance(row, dict) and str(row.get("status") or row.get("effective_status") or "").lower() in {"success", "passed", "succeeded", "full_inference_success", "validated_after_space_test", "validated_after_manual_space_test"}]
|
| 1294 |
+
if linked_successes and str(manual_validation.get("status") or "").lower() != "success":
|
| 1295 |
+
linked_successes.sort(key=lambda row: str(row.get("validated_at") or row.get("updated_at") or row.get("created_at") or ""), reverse=True)
|
| 1296 |
+
manual_validation = linked_successes[0]
|
| 1297 |
+
gate_smoke = gate.get("generation_smoke") if isinstance(gate.get("generation_smoke"), dict) else {}
|
| 1298 |
+
zero_gpu_recommendation = gate.get("zero_gpu_duration_recommendation") if isinstance(gate, dict) else None
|
| 1299 |
+
if isinstance(zero_gpu_recommendation, dict):
|
| 1300 |
+
zero_gpu_recommendation = zero_gpu_recommendation.get("recommended_zero_gpu_duration_seconds") or zero_gpu_recommendation.get("recommended_zerogpu_duration_seconds") or zero_gpu_recommendation.get("seconds")
|
| 1301 |
eval_publish_status = bundle.get("eval_publish_status") or {}
|
| 1302 |
final_status_reconciliation = bundle.get("final_status_reconciliation") or {}
|
| 1303 |
eval_record = bundle.get("eval_record") or {}
|
|
|
|
| 1351 |
"updated_at": state.get("updated_at") or state.get("created_at") or launch.get("updated_at") or launch.get("created_at") or summary_file.get("updated_at") or summary_file.get("created_at") or "",
|
| 1352 |
"selected_hardware": hardware.get("selected_hardware") or state.get("selected_hardware") or launch.get("preferred_space_hardware") or summary_file.get("selected_hardware") or state.get("hardware") or "",
|
| 1353 |
"manual_hardware_required": bool(gate.get("manual_hardware_required") or hardware.get("manual_action_required") or launch.get("manual_hardware_required")),
|
| 1354 |
+
"health_passed": bool((gate.get("implementation_signals") or {}).get("health_passed") or smoke.get("health_passed") or gate_smoke.get("health_passed")),
|
| 1355 |
+
"smoke_test_passed": bool(smoke.get("ok") or smoke.get("status") == "success" or gate_smoke.get("status") == "success"),
|
| 1356 |
+
"latency_seconds": manual_validation.get("latency_seconds") or manual_validation.get("observed_latency_seconds") or smoke.get("latency_seconds") or smoke.get("observed_latency_seconds") or gate_smoke.get("latency_seconds") or gate_smoke.get("observed_latency_seconds"),
|
| 1357 |
+
"observed_latency_seconds": manual_validation.get("observed_latency_seconds") or manual_validation.get("latency_seconds") or smoke.get("observed_latency_seconds") or smoke.get("latency_seconds") or gate_smoke.get("observed_latency_seconds") or gate_smoke.get("latency_seconds"),
|
| 1358 |
+
"recommended_zero_gpu_duration_seconds": manual_validation.get("recommended_zero_gpu_duration_seconds") or manual_validation.get("recommended_zerogpu_duration_seconds") or smoke.get("recommended_zero_gpu_duration_seconds") or smoke.get("recommended_zerogpu_duration_seconds") or gate_smoke.get("recommended_zero_gpu_duration_seconds") or gate_smoke.get("recommended_zerogpu_duration_seconds") or zero_gpu_recommendation,
|
| 1359 |
+
"recommended_zerogpu_duration_seconds": manual_validation.get("recommended_zerogpu_duration_seconds") or manual_validation.get("recommended_zero_gpu_duration_seconds") or smoke.get("recommended_zerogpu_duration_seconds") or smoke.get("recommended_zero_gpu_duration_seconds") or gate_smoke.get("recommended_zerogpu_duration_seconds") or gate_smoke.get("recommended_zero_gpu_duration_seconds") or zero_gpu_recommendation,
|
| 1360 |
+
"recommendation_source": manual_validation.get("recommendation_source") or smoke.get("recommendation_source") or gate_smoke.get("recommendation_source") or ("linked_space_test" if str(manual_validation.get("status") or "").lower() == "success" else ""),
|
| 1361 |
+
"recommendation_hardware": manual_validation.get("recommendation_hardware") or manual_validation.get("hardware_used_for_validation") or smoke.get("recommendation_hardware") or gate_smoke.get("recommendation_hardware") or state.get("selected_hardware") or "",
|
| 1362 |
+
"hardware_used_for_validation": manual_validation.get("hardware_used_for_validation") or manual_validation.get("recommendation_hardware") or state.get("selected_hardware") or "",
|
| 1363 |
"manual_validation_status": manual_validation,
|
| 1364 |
"manual_validation_passed": bool(manual_validation.get("status") == "success"),
|
| 1365 |
"post_build_status": effective_run_status.get("post_build_status") or "none",
|
src/timeline_model.py
CHANGED
|
@@ -160,6 +160,66 @@ def _latest_event_by_step(bundle: dict[str, Any], step_name: str) -> dict[str, A
|
|
| 160 |
return {}
|
| 161 |
|
| 162 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
def build_live_validation_model(bundle: dict[str, Any]) -> dict[str, Any]:
|
| 164 |
"""Compact live validation telemetry for the UI.
|
| 165 |
|
|
@@ -281,6 +341,7 @@ def build_live_validation_model(bundle: dict[str, Any]) -> dict[str, Any]:
|
|
| 281 |
"live_status_updated_at": live_status.get("updated_at") or "",
|
| 282 |
"message": smoke.get("error") or smoke.get("message") or live_message or api_event.get("message") or runtime_error or "",
|
| 283 |
"next_action": next_action,
|
|
|
|
| 284 |
}
|
| 285 |
|
| 286 |
|
|
@@ -508,7 +569,11 @@ def _build_phase_details(bundle: dict[str, Any], phase: str) -> list[dict[str, A
|
|
| 508 |
elif phase == "live_validation":
|
| 509 |
live = build_live_validation_model(bundle)
|
| 510 |
runtime_stage = live.get("runtime_stage") or ""
|
| 511 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
| 512 |
details.append({"label": f"Space runtime: {runtime_stage}", "status": "info"})
|
| 513 |
if live.get("health") == "passed":
|
| 514 |
details.append({"label": "Health passed", "status": "complete"})
|
|
@@ -677,14 +742,19 @@ def _phase_status(bundle: dict[str, Any], phase: str, verdict: str) -> str:
|
|
| 677 |
if phase == "agent":
|
| 678 |
latest_by_step = _latest_events_by_step(bundle, phase)
|
| 679 |
pi_run = latest_by_step.get("pi_run") or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
if _lower(pi_run.get("status")) in RUNNING_STATUSES and not terminal:
|
| 681 |
return "running"
|
| 682 |
if terminal:
|
| 683 |
material_events = [e for e in _events_for_phase(bundle, phase) if _lower(e.get("step")) != "pi_model_resolution"]
|
| 684 |
material_statuses = {_lower(e.get("status")) for e in material_events}
|
| 685 |
-
if material_statuses.intersection(FAILED_STATUSES):
|
| 686 |
return "failed"
|
| 687 |
-
if material_statuses.intersection(RUNNING_STATUSES):
|
| 688 |
return "running"
|
| 689 |
if material_statuses.intersection(SUCCESS_STATUSES) or _has_downstream_proof(bundle, phase):
|
| 690 |
return "complete"
|
|
|
|
| 160 |
return {}
|
| 161 |
|
| 162 |
|
| 163 |
+
|
| 164 |
+
def _runtime_history(bundle: dict[str, Any], *, live: dict[str, Any], runtime: dict[str, Any], smoke: dict[str, Any], gate: dict[str, Any]) -> list[dict[str, Any]]:
|
| 165 |
+
"""Return a compact build→runtime→API→smoke history for Live test.
|
| 166 |
+
|
| 167 |
+
v198.26.9 keeps this curated: it restores the building/runtime feedback
|
| 168 |
+
users need without dumping raw events into Active Run.
|
| 169 |
+
"""
|
| 170 |
+
history: list[dict[str, Any]] = []
|
| 171 |
+
seen: set[str] = set()
|
| 172 |
+
|
| 173 |
+
def add(stage: str, label: str, status: str = "info", ts: Any = "", detail: Any = "") -> None:
|
| 174 |
+
key = f"{stage}:{label}:{status}"
|
| 175 |
+
if key in seen:
|
| 176 |
+
return
|
| 177 |
+
seen.add(key)
|
| 178 |
+
history.append({"stage": stage, "label": label, "status": status, "ts": ts or "", "detail": str(detail or "")[:500]})
|
| 179 |
+
|
| 180 |
+
for event in _events(bundle):
|
| 181 |
+
step = _lower(event.get("step"))
|
| 182 |
+
status = _lower(event.get("status")) or "info"
|
| 183 |
+
ts = event.get("ts") or event.get("created_at") or ""
|
| 184 |
+
msg = event.get("message") or event.get("step") or ""
|
| 185 |
+
if step in {"create_space", "create_space_hardware"}:
|
| 186 |
+
add("space_created", "Space created" if status in SUCCESS_STATUSES else "Creating Space", status, ts, msg)
|
| 187 |
+
elif step == "upload_files":
|
| 188 |
+
add("runtime_uploaded", "Runtime uploaded" if status in SUCCESS_STATUSES else "Uploading runtime", status, ts, msg)
|
| 189 |
+
elif step in {"space_runtime", "live_wait"}:
|
| 190 |
+
lower_msg = _lower(msg)
|
| 191 |
+
if "building" in lower_msg or "build" in lower_msg:
|
| 192 |
+
add("space_building", "Space is building", status, ts, msg)
|
| 193 |
+
elif "running" in lower_msg or "runtime" in lower_msg or step == "space_runtime":
|
| 194 |
+
add("space_running", "Space runtime observed", status, ts, msg)
|
| 195 |
+
else:
|
| 196 |
+
add("space_runtime", "Space runtime check", status, ts, msg)
|
| 197 |
+
elif step in {"endpoint_discovery", "api_validation"}:
|
| 198 |
+
add("api_schema", "Gradio API checked" if status in SUCCESS_STATUSES else "Checking Gradio API", status, ts, msg)
|
| 199 |
+
elif step == "generation_smoke":
|
| 200 |
+
add("generation_smoke", "Generation smoke passed" if status in SUCCESS_STATUSES else "Generation smoke running" if status in RUNNING_STATUSES else "Generation smoke checked", status, ts, msg)
|
| 201 |
+
elif step == "inference_gate":
|
| 202 |
+
add("inference_gate", "Inference gate resolved", status, ts, msg)
|
| 203 |
+
|
| 204 |
+
runtime_stage = str(runtime.get("stage") or runtime.get("status") or runtime.get("runtime_status") or "").strip()
|
| 205 |
+
upper_stage = runtime_stage.upper()
|
| 206 |
+
if upper_stage:
|
| 207 |
+
if "BUILD" in upper_stage and not any(h["stage"] == "space_building" for h in history):
|
| 208 |
+
add("space_building", "Space is building", "running", runtime.get("updated_at") or "", runtime_stage)
|
| 209 |
+
if "RUNNING" in upper_stage and not any(h["stage"] == "space_running" for h in history):
|
| 210 |
+
add("space_running", "Space runtime is running", "success", runtime.get("updated_at") or "", runtime_stage)
|
| 211 |
+
if "ERROR" in upper_stage:
|
| 212 |
+
add("space_runtime_error", "Space runtime error", "failed", runtime.get("updated_at") or "", runtime.get("error") or runtime_stage)
|
| 213 |
+
if live.get("stage") and not history:
|
| 214 |
+
add(str(live.get("stage")), str(live.get("message") or live.get("stage")), str(live.get("status") or "info"), live.get("updated_at") or "")
|
| 215 |
+
endpoints = gate.get("endpoints") or gate.get("api_endpoints") or gate.get("named_endpoints") or []
|
| 216 |
+
selected = str(smoke.get("api_name") or gate.get("api_name") or gate.get("selected_endpoint") or gate.get("selected_api_name") or "").strip()
|
| 217 |
+
if (selected or (isinstance(endpoints, list) and endpoints)) and not any(h["stage"] == "api_schema" for h in history):
|
| 218 |
+
add("api_schema", f"Gradio API detected{': ' + selected if selected else ''}", "success")
|
| 219 |
+
if (_lower(smoke.get("status")) == "success" or smoke.get("ok") is True or (gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True) and not any(h["stage"] == "generation_smoke" for h in history):
|
| 220 |
+
add("generation_smoke", "Generation smoke passed", "success", smoke.get("updated_at") or "", smoke.get("api_name") or "")
|
| 221 |
+
return history[-10:]
|
| 222 |
+
|
| 223 |
def build_live_validation_model(bundle: dict[str, Any]) -> dict[str, Any]:
|
| 224 |
"""Compact live validation telemetry for the UI.
|
| 225 |
|
|
|
|
| 341 |
"live_status_updated_at": live_status.get("updated_at") or "",
|
| 342 |
"message": smoke.get("error") or smoke.get("message") or live_message or api_event.get("message") or runtime_error or "",
|
| 343 |
"next_action": next_action,
|
| 344 |
+
"runtime_history": _runtime_history(bundle, live=live_status, runtime=runtime, smoke=smoke, gate=gate),
|
| 345 |
}
|
| 346 |
|
| 347 |
|
|
|
|
| 569 |
elif phase == "live_validation":
|
| 570 |
live = build_live_validation_model(bundle)
|
| 571 |
runtime_stage = live.get("runtime_stage") or ""
|
| 572 |
+
runtime_history = live.get("runtime_history") if isinstance(live.get("runtime_history"), list) else []
|
| 573 |
+
for item in runtime_history:
|
| 574 |
+
if isinstance(item, dict) and item.get("label"):
|
| 575 |
+
details.append({"label": str(item.get("label")), "status": str(item.get("status") or "info")})
|
| 576 |
+
if runtime_stage and not runtime_history:
|
| 577 |
details.append({"label": f"Space runtime: {runtime_stage}", "status": "info"})
|
| 578 |
if live.get("health") == "passed":
|
| 579 |
details.append({"label": "Health passed", "status": "complete"})
|
|
|
|
| 742 |
if phase == "agent":
|
| 743 |
latest_by_step = _latest_events_by_step(bundle, phase)
|
| 744 |
pi_run = latest_by_step.get("pi_run") or {}
|
| 745 |
+
# v198.26.9: terminal success with downstream proof closes Agent. An
|
| 746 |
+
# old pi_run=running signal must not keep the visible dot pulsing after
|
| 747 |
+
# deploy/live/archive/final success has already happened.
|
| 748 |
+
if terminal and _final_visual_status(verdict) == "success" and _has_downstream_proof(bundle, phase):
|
| 749 |
+
return "complete"
|
| 750 |
if _lower(pi_run.get("status")) in RUNNING_STATUSES and not terminal:
|
| 751 |
return "running"
|
| 752 |
if terminal:
|
| 753 |
material_events = [e for e in _events_for_phase(bundle, phase) if _lower(e.get("step")) != "pi_model_resolution"]
|
| 754 |
material_statuses = {_lower(e.get("status")) for e in material_events}
|
| 755 |
+
if material_statuses.intersection(FAILED_STATUSES) and _final_visual_status(verdict) != "success":
|
| 756 |
return "failed"
|
| 757 |
+
if material_statuses.intersection(RUNNING_STATUSES) and not _has_downstream_proof(bundle, phase):
|
| 758 |
return "running"
|
| 759 |
if material_statuses.intersection(SUCCESS_STATUSES) or _has_downstream_proof(bundle, phase):
|
| 760 |
return "complete"
|
src/version.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
ASF_APP_VERSION = "v198.26.
|
| 4 |
-
ASF_RELEASE_NAME = "Agentic Space Factory v198.26.
|
| 5 |
|
| 6 |
|
| 7 |
def resolve_app_version(value: str | None = None) -> str:
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
ASF_APP_VERSION = "v198.26.9"
|
| 4 |
+
ASF_RELEASE_NAME = "Agentic Space Factory v198.26.9"
|
| 5 |
|
| 6 |
|
| 7 |
def resolve_app_version(value: str | None = None) -> str:
|
src/view_models.py
CHANGED
|
@@ -544,7 +544,7 @@ def _space_target_url_from_bundle(bundle: dict[str, Any], target: str = "") -> s
|
|
| 544 |
def build_space_link_state(bundle: dict[str, Any], *, target: str = "", target_url: str = "") -> dict[str, Any]:
|
| 545 |
"""Return durable Space link readiness for Active Run.
|
| 546 |
|
| 547 |
-
v198.26.
|
| 548 |
depend on the frontend receiving the create/upload event inside a truncated
|
| 549 |
recent-events window. If a runtime upload epoch or Space runtime probe is
|
| 550 |
present, a terminal failed run still gets Open Space / Settings links.
|
|
@@ -582,7 +582,7 @@ def build_space_link_state(bundle: dict[str, Any], *, target: str = "", target_u
|
|
| 582 |
if space_runtime_known:
|
| 583 |
sources.append("space_runtime")
|
| 584 |
return {
|
| 585 |
-
"schema_version": "space_link_state.
|
| 586 |
"target_space": target,
|
| 587 |
"target_space_id": target,
|
| 588 |
"target_space_known": bool(target),
|
|
@@ -602,6 +602,127 @@ def _space_links_ready_from_bundle(bundle: dict[str, Any]) -> bool:
|
|
| 602 |
return bool(build_space_link_state(bundle).get("links_ready"))
|
| 603 |
|
| 604 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 605 |
def _known_gradio_endpoint_info(bundle: dict[str, Any]) -> dict[str, Any]:
|
| 606 |
smoke = bundle.get("generation_smoke") or {}
|
| 607 |
gate = bundle.get("inference_gate") or {}
|
|
@@ -736,6 +857,7 @@ def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]
|
|
| 736 |
def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
|
| 737 |
smoke = bundle.get("generation_smoke") or {}
|
| 738 |
summary = bundle.get("summary") or {}
|
|
|
|
| 739 |
policy = build_space_test_policy(bundle, status_model)
|
| 740 |
# v191.12 regression guard: a contract-declared boot-only blocker must not
|
| 741 |
# resurrect the historical /generate fallback in the display model. Keep the
|
|
@@ -750,8 +872,12 @@ def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any])
|
|
| 750 |
"target_space": policy.get("target_space") or summary.get("target_space") or "",
|
| 751 |
"target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "",
|
| 752 |
"endpoint": endpoint,
|
| 753 |
-
"status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending",
|
| 754 |
-
"latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 755 |
"expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
|
| 756 |
"verdict": status_model["verdict"],
|
| 757 |
"output_artifact": smoke.get("output_artifact") or smoke.get("artifact_url") or "",
|
|
@@ -794,6 +920,7 @@ def build_run_view_model(
|
|
| 794 |
target = _space_target_from_bundle(bundle)
|
| 795 |
target_url = _space_target_url_from_bundle(bundle, target)
|
| 796 |
space_link_state = build_space_link_state(bundle, target=target, target_url=target_url)
|
|
|
|
| 797 |
links = {
|
| 798 |
"job_url": _job_url_from_view_sources(bucket_source=bucket_source, summary=summary, state=state, launch=bundle.get("launch") or {}),
|
| 799 |
"target_space_url": target_url,
|
|
@@ -834,6 +961,8 @@ def build_run_view_model(
|
|
| 834 |
"space_test": {**build_space_test_model(bundle, status_model), "linked_validation": manual_validation, "mode": "linked_build_run"},
|
| 835 |
"space_test_policy": space_test_policy,
|
| 836 |
"space_link_state": space_link_state,
|
|
|
|
|
|
|
| 837 |
"manual_validation": manual_validation,
|
| 838 |
"post_build_validation": effective_run_status.get("post_build_validation") or {},
|
| 839 |
"display_status": effective_run_status.get("display_status") or status_model.get("global_status"),
|
|
|
|
| 544 |
def build_space_link_state(bundle: dict[str, Any], *, target: str = "", target_url: str = "") -> dict[str, Any]:
|
| 545 |
"""Return durable Space link readiness for Active Run.
|
| 546 |
|
| 547 |
+
v198.26.9 invariant: Space links are a persisted run fact. They must not
|
| 548 |
depend on the frontend receiving the create/upload event inside a truncated
|
| 549 |
recent-events window. If a runtime upload epoch or Space runtime probe is
|
| 550 |
present, a terminal failed run still gets Open Space / Settings links.
|
|
|
|
| 582 |
if space_runtime_known:
|
| 583 |
sources.append("space_runtime")
|
| 584 |
return {
|
| 585 |
+
"schema_version": "space_link_state.v198_26_9",
|
| 586 |
"target_space": target,
|
| 587 |
"target_space_id": target,
|
| 588 |
"target_space_known": bool(target),
|
|
|
|
| 602 |
return bool(build_space_link_state(bundle).get("links_ready"))
|
| 603 |
|
| 604 |
|
| 605 |
+
|
| 606 |
+
def _number_from_value(value: Any) -> float | None:
|
| 607 |
+
if value is None or value == "":
|
| 608 |
+
return None
|
| 609 |
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
| 610 |
+
number = float(value)
|
| 611 |
+
return number if number > 0 else None
|
| 612 |
+
if isinstance(value, dict):
|
| 613 |
+
# Common shape: {"recommended_zero_gpu_duration_seconds": 40}
|
| 614 |
+
for key in (
|
| 615 |
+
"seconds",
|
| 616 |
+
"duration_seconds",
|
| 617 |
+
"recommended_zero_gpu_duration_seconds",
|
| 618 |
+
"recommended_zerogpu_duration_seconds",
|
| 619 |
+
"recommended_duration_seconds",
|
| 620 |
+
"latency_seconds",
|
| 621 |
+
"observed_latency_seconds",
|
| 622 |
+
):
|
| 623 |
+
found = _number_from_value(value.get(key))
|
| 624 |
+
if found is not None:
|
| 625 |
+
return found
|
| 626 |
+
return None
|
| 627 |
+
text = str(value).strip().lower().replace("seconds", "").replace("second", "").replace("sec", "").replace("s", "")
|
| 628 |
+
try:
|
| 629 |
+
number = float(text)
|
| 630 |
+
return number if number > 0 else None
|
| 631 |
+
except Exception:
|
| 632 |
+
return None
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
def _first_number(*values: Any) -> float | None:
|
| 636 |
+
for value in values:
|
| 637 |
+
number = _number_from_value(value)
|
| 638 |
+
if number is not None:
|
| 639 |
+
return number
|
| 640 |
+
return None
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
def _latest_successful_linked_validation(bundle: dict[str, Any]) -> dict[str, Any]:
|
| 644 |
+
linked = bundle.get("linked_validations") or {}
|
| 645 |
+
rows = linked.get("validations") if isinstance(linked, dict) else []
|
| 646 |
+
if not isinstance(rows, list):
|
| 647 |
+
return {}
|
| 648 |
+
successful = [row for row in rows if isinstance(row, dict) and _lower(row.get("status") or row.get("effective_status")) in {"success", "passed", "succeeded", "full_inference_success", "validated_after_space_test", "validated_after_manual_space_test"}]
|
| 649 |
+
if not successful:
|
| 650 |
+
return {}
|
| 651 |
+
successful.sort(key=lambda row: str(row.get("validated_at") or row.get("updated_at") or row.get("created_at") or ""), reverse=True)
|
| 652 |
+
return successful[0]
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def build_validation_metrics(bundle: dict[str, Any]) -> dict[str, Any]:
|
| 656 |
+
"""Durable validation/latency model for Active Run.
|
| 657 |
+
|
| 658 |
+
v198.26.9 invariant: latency and ZeroGPU duration are post-run facts, not
|
| 659 |
+
incidental event text. They are resolved once from persisted automatic
|
| 660 |
+
smoke artifacts and linked Space Test artifacts, then exposed by /view and
|
| 661 |
+
/progress so the frontend does not have to guess.
|
| 662 |
+
"""
|
| 663 |
+
bundle = bundle or {}
|
| 664 |
+
summary = bundle.get("summary") or {}
|
| 665 |
+
state = bundle.get("state") or {}
|
| 666 |
+
smoke = bundle.get("generation_smoke") or {}
|
| 667 |
+
gate = bundle.get("inference_gate") or {}
|
| 668 |
+
gate_smoke = gate.get("generation_smoke") if isinstance(gate.get("generation_smoke"), dict) else {}
|
| 669 |
+
post_build = bundle.get("post_build_validation_status") or {}
|
| 670 |
+
manual = bundle.get("manual_validation_status") or {}
|
| 671 |
+
linked_success = _latest_successful_linked_validation(bundle)
|
| 672 |
+
gate_recommendation = gate.get("zero_gpu_duration_recommendation") if isinstance(gate, dict) else None
|
| 673 |
+
|
| 674 |
+
# Linked Space Test success is a stronger post-build measurement than the
|
| 675 |
+
# original automatic build smoke. It should update the parent Active Run.
|
| 676 |
+
source_order = [linked_success, post_build if _lower(post_build.get("status")) == "success" else {}, manual if _lower(manual.get("status")) == "success" else {}, smoke, gate_smoke, gate, summary, state]
|
| 677 |
+
latency = _first_number(*(src.get("latency_seconds") for src in source_order if isinstance(src, dict)), *(src.get("observed_latency_seconds") for src in source_order if isinstance(src, dict)))
|
| 678 |
+
duration = _first_number(
|
| 679 |
+
*(src.get("recommended_zero_gpu_duration_seconds") for src in source_order if isinstance(src, dict)),
|
| 680 |
+
*(src.get("recommended_zerogpu_duration_seconds") for src in source_order if isinstance(src, dict)),
|
| 681 |
+
*(src.get("recommended_duration_seconds") for src in source_order if isinstance(src, dict)),
|
| 682 |
+
gate_recommendation,
|
| 683 |
+
)
|
| 684 |
+
api_name = _first_nonempty(*(src.get("api_name") for src in source_order if isinstance(src, dict)), gate.get("selected_api_name"), gate.get("selected_endpoint"))
|
| 685 |
+
hardware = _first_nonempty(
|
| 686 |
+
*(src.get("hardware_used_for_validation") for src in source_order if isinstance(src, dict)),
|
| 687 |
+
*(src.get("recommendation_hardware") for src in source_order if isinstance(src, dict)),
|
| 688 |
+
smoke.get("hardware"),
|
| 689 |
+
gate.get("hardware"),
|
| 690 |
+
summary.get("selected_hardware"),
|
| 691 |
+
state.get("selected_hardware"),
|
| 692 |
+
)
|
| 693 |
+
output_artifact = _first_nonempty(*(src.get("output_artifact") or src.get("artifact_url") for src in source_order if isinstance(src, dict)))
|
| 694 |
+
source = ""
|
| 695 |
+
if linked_success or _lower(post_build.get("status")) == "success" or _lower(manual.get("status")) == "success":
|
| 696 |
+
source = "linked_space_test"
|
| 697 |
+
elif smoke.get("status") == "success" or smoke.get("ok") is True:
|
| 698 |
+
source = "automatic_generation_smoke"
|
| 699 |
+
elif gate_smoke or gate:
|
| 700 |
+
source = "inference_gate"
|
| 701 |
+
source = _first_nonempty(*(src.get("recommendation_source") for src in source_order if isinstance(src, dict)), source)
|
| 702 |
+
health_passed = bool((gate.get("implementation_signals") or {}).get("health_passed") or smoke.get("health_passed") or gate_smoke.get("health_passed"))
|
| 703 |
+
smoke_passed = bool(smoke.get("ok") or _lower(smoke.get("status")) == "success" or _lower(gate_smoke.get("status")) == "success" or (gate.get("implementation_signals") or {}).get("generation_smoke_passed") is True)
|
| 704 |
+
validation_run_id = _first_nonempty(linked_success.get("validation_run_id"), linked_success.get("run_id"), post_build.get("validation_run_id"), manual.get("validation_run_id"))
|
| 705 |
+
metrics = {
|
| 706 |
+
"schema_version": "validation_metrics.v198_26_9",
|
| 707 |
+
"available": bool(latency or duration or smoke_passed or health_passed or validation_run_id),
|
| 708 |
+
"source": source,
|
| 709 |
+
"latency_seconds": latency,
|
| 710 |
+
"observed_latency_seconds": latency,
|
| 711 |
+
"recommended_zero_gpu_duration_seconds": duration,
|
| 712 |
+
"recommended_zerogpu_duration_seconds": duration,
|
| 713 |
+
"hardware_used_for_validation": hardware,
|
| 714 |
+
"recommendation_hardware": hardware,
|
| 715 |
+
"api_name": api_name,
|
| 716 |
+
"output_artifact": output_artifact,
|
| 717 |
+
"health_passed": health_passed,
|
| 718 |
+
"generation_smoke_passed": smoke_passed,
|
| 719 |
+
"automatic_smoke_passed": bool(smoke.get("ok") or _lower(smoke.get("status")) == "success"),
|
| 720 |
+
"linked_space_test_passed": bool(linked_success or _lower(post_build.get("status")) == "success" or _lower(manual.get("status")) == "success"),
|
| 721 |
+
"validation_run_id": validation_run_id,
|
| 722 |
+
"updated_at": _first_nonempty(linked_success.get("validated_at"), linked_success.get("updated_at"), post_build.get("validated_at"), post_build.get("updated_at"), manual.get("validated_at"), manual.get("updated_at"), summary.get("updated_at"), state.get("updated_at")),
|
| 723 |
+
}
|
| 724 |
+
return metrics
|
| 725 |
+
|
| 726 |
def _known_gradio_endpoint_info(bundle: dict[str, Any]) -> dict[str, Any]:
|
| 727 |
smoke = bundle.get("generation_smoke") or {}
|
| 728 |
gate = bundle.get("inference_gate") or {}
|
|
|
|
| 857 |
def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
|
| 858 |
smoke = bundle.get("generation_smoke") or {}
|
| 859 |
summary = bundle.get("summary") or {}
|
| 860 |
+
metrics = build_validation_metrics(bundle)
|
| 861 |
policy = build_space_test_policy(bundle, status_model)
|
| 862 |
# v191.12 regression guard: a contract-declared boot-only blocker must not
|
| 863 |
# resurrect the historical /generate fallback in the display model. Keep the
|
|
|
|
| 872 |
"target_space": policy.get("target_space") or summary.get("target_space") or "",
|
| 873 |
"target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "",
|
| 874 |
"endpoint": endpoint,
|
| 875 |
+
"status": "passed" if metrics.get("generation_smoke_passed") or smoke.get("ok") or smoke.get("status") == "success" else "pending",
|
| 876 |
+
"latency_seconds": metrics.get("latency_seconds") or smoke.get("latency_seconds") or summary.get("latency_seconds"),
|
| 877 |
+
"observed_latency_seconds": metrics.get("observed_latency_seconds"),
|
| 878 |
+
"recommended_zero_gpu_duration_seconds": metrics.get("recommended_zero_gpu_duration_seconds"),
|
| 879 |
+
"recommended_zerogpu_duration_seconds": metrics.get("recommended_zerogpu_duration_seconds"),
|
| 880 |
+
"validation_metrics": metrics,
|
| 881 |
"expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
|
| 882 |
"verdict": status_model["verdict"],
|
| 883 |
"output_artifact": smoke.get("output_artifact") or smoke.get("artifact_url") or "",
|
|
|
|
| 920 |
target = _space_target_from_bundle(bundle)
|
| 921 |
target_url = _space_target_url_from_bundle(bundle, target)
|
| 922 |
space_link_state = build_space_link_state(bundle, target=target, target_url=target_url)
|
| 923 |
+
validation_metrics = build_validation_metrics({**bundle, "summary": summary})
|
| 924 |
links = {
|
| 925 |
"job_url": _job_url_from_view_sources(bucket_source=bucket_source, summary=summary, state=state, launch=bundle.get("launch") or {}),
|
| 926 |
"target_space_url": target_url,
|
|
|
|
| 961 |
"space_test": {**build_space_test_model(bundle, status_model), "linked_validation": manual_validation, "mode": "linked_build_run"},
|
| 962 |
"space_test_policy": space_test_policy,
|
| 963 |
"space_link_state": space_link_state,
|
| 964 |
+
"validation_metrics": validation_metrics,
|
| 965 |
+
"run_documents": bundle.get("run_documents") or [],
|
| 966 |
"manual_validation": manual_validation,
|
| 967 |
"post_build_validation": effective_run_status.get("post_build_validation") or {},
|
| 968 |
"display_status": effective_run_status.get("display_status") or status_model.get("global_status"),
|
src/worker_payload.py
CHANGED
|
@@ -35,8 +35,8 @@ DEFAULT_PREFERRED_SPACE_HARDWARE = "zero-a10g"
|
|
| 35 |
DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large"
|
| 36 |
DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2"
|
| 37 |
MAX_PI_REPAIR_ATTEMPTS = 3
|
| 38 |
-
APP_VERSION = "v198.26.
|
| 39 |
-
app_version = "v198.26.
|
| 40 |
|
| 41 |
# Internal agent/recovery files may be needed inside the transient Pi
|
| 42 |
# workspace, but they should not be published to the generated Space or shown
|
|
@@ -5254,7 +5254,7 @@ def reconcile_final_success_state(run_dir: Path, events_path: Path | None, final
|
|
| 5254 |
def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None, final_state: dict, *, status: str, message: str = "", details: dict | None = None, job_exit_code: int | None = None) -> dict:
|
| 5255 |
"""Persist a terminal truth artifact for every final outcome.
|
| 5256 |
|
| 5257 |
-
v198.26.
|
| 5258 |
of state, repair_outcome and recent events. Success already writes
|
| 5259 |
final_status_reconciliation.json; failed/auth/cancelled terminal paths now
|
| 5260 |
write the same artifact so Active Run can converge deterministically.
|
|
@@ -5268,7 +5268,7 @@ def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None
|
|
| 5268 |
runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history)
|
| 5269 |
space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url")))
|
| 5270 |
payload = {
|
| 5271 |
-
"schema_version": "final_status_reconciliation.
|
| 5272 |
"status": status,
|
| 5273 |
"terminal": True,
|
| 5274 |
"message": message or state.get("message") or "",
|
|
@@ -12503,7 +12503,7 @@ def smoke_generate(target_space_id: str, token: str, run_dir: Path, events_path:
|
|
| 12503 |
# write_json(run_dir / "tests" / "payload_source.json", payload_source_record)
|
| 12504 |
# write_json(run_dir / "tests" / "replay_source.json", replay_source)
|
| 12505 |
# write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight)
|
| 12506 |
-
app_version = "v198.26.
|
| 12507 |
engine_version = "unified_gradio_validation_harness_v198_25_3"
|
| 12508 |
parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip()
|
| 12509 |
validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"
|
|
|
|
| 35 |
DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large"
|
| 36 |
DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2"
|
| 37 |
MAX_PI_REPAIR_ATTEMPTS = 3
|
| 38 |
+
APP_VERSION = "v198.26.9"
|
| 39 |
+
app_version = "v198.26.9"
|
| 40 |
|
| 41 |
# Internal agent/recovery files may be needed inside the transient Pi
|
| 42 |
# workspace, but they should not be published to the generated Space or shown
|
|
|
|
| 5254 |
def write_terminal_status_reconciliation(run_dir: Path, events_path: Path | None, final_state: dict, *, status: str, message: str = "", details: dict | None = None, job_exit_code: int | None = None) -> dict:
|
| 5255 |
"""Persist a terminal truth artifact for every final outcome.
|
| 5256 |
|
| 5257 |
+
v198.26.9: the UI must not have to infer failed/stopped states from a mix
|
| 5258 |
of state, repair_outcome and recent events. Success already writes
|
| 5259 |
final_status_reconciliation.json; failed/auth/cancelled terminal paths now
|
| 5260 |
write the same artifact so Active Run can converge deterministically.
|
|
|
|
| 5268 |
runtime_uploaded = bool(runtime_upload_epoch.get("last_upload_completed_at") or runtime_upload_epoch.get("upload_sequence") or runtime_history)
|
| 5269 |
space_runtime_known = bool(isinstance(space_runtime, dict) and (space_runtime.get("stage") or space_runtime.get("status") or space_runtime.get("runtime_status") or space_runtime.get("updated_at") or space_runtime.get("url")))
|
| 5270 |
payload = {
|
| 5271 |
+
"schema_version": "final_status_reconciliation.v198_26_9",
|
| 5272 |
"status": status,
|
| 5273 |
"terminal": True,
|
| 5274 |
"message": message or state.get("message") or "",
|
|
|
|
| 12503 |
# write_json(run_dir / "tests" / "payload_source.json", payload_source_record)
|
| 12504 |
# write_json(run_dir / "tests" / "replay_source.json", replay_source)
|
| 12505 |
# write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight)
|
| 12506 |
+
app_version = "v198.26.9"
|
| 12507 |
engine_version = "unified_gradio_validation_harness_v198_25_3"
|
| 12508 |
parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip()
|
| 12509 |
validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"
|