from __future__ import annotations from datetime import datetime, timezone from typing import Any # Exact worker-facing steps emitted by src/worker_payload.py. # Keep this list close to the Job logs/events rather than a marketing pipeline. STEP_ORDER = [ "bootstrap", "dependencies", "auth", "model_analysis", "workspace", "node", "pi_install", "pi_config", "pi_run", "pi_model_resolution", "pi_verification", "metadata_sanitize", "requirements_sanitize", "hardware_strategy", "create_space_hardware", "create_space", "failure_detected", "failure_diagnosis", "pi_diagnosis", "repair_decision", "wait_for_logs", "factory_rebuild", "repair", "repair_diagnosis", "repair_brief", "repair_plan", "repair_patch", "repair_upload", "repair_validation", "technical_blocker", "technical_blocker_boot_only", "manual_hardware_required", "upload_files", "space_runtime", "space_logs", "api_validation", "live_wait", "generation_smoke", "inference_gate", "report_write", "done", "failure", ] STEP_LABELS = { "bootstrap": "Bootstrap", "dependencies": "Dependencies", "auth": "Auth", "model_analysis": "Model analysis", "workspace": "Workspace", "node": "Node/npm", "pi_install": "Pi install", "pi_config": "Pi config", "pi_run": "Pi run", "pi_model_resolution": "Pi model", "pi_verification": "Pi verification", "metadata_sanitize": "Metadata sanitize", "requirements_sanitize": "Requirements sanitize", "hardware_strategy": "Hardware strategy", "create_space_hardware": "Create with hardware", "create_space": "Create Space", "failure_detected": "Failure detected", "failure_diagnosis": "Failure diagnosis", "pi_diagnosis": "Pi diagnosis", "repair_decision": "Repair decision", "wait_for_logs": "Wait for logs", "factory_rebuild": "Factory rebuild", "repair": "Repair start", "repair_diagnosis": "Patch allowed", "repair_brief": "Repair brief", "repair_plan": "Repair plan", "repair_patch": "Repair patch", "repair_upload": "Repair upload", "repair_validation": "Repair validation", "technical_blocker": "Technical blocker", "technical_blocker_boot_only": "Boot-only blocker", "manual_hardware_required": "Manual hardware", "upload_files": "Upload files", "space_runtime": "Space runtime", "space_logs": "Space logs", "api_validation": "API validation", "live_wait": "Live wait", "generation_smoke": "Generation smoke", "inference_gate": "Inference gate", "report_write": "Report", "done": "Done", "failure": "Failure", } STEP_ALIASES = { "bucket_ready": "bootstrap", "job_launched": "bootstrap", "hardware_preferred": "hardware_strategy", "hardware_fallback": "hardware_strategy", "hardware": "hardware_strategy", "build": "space_runtime", "diagnose": "failure_diagnosis", "diagnosis": "failure_diagnosis", "pi_decision": "repair_decision", "decision": "repair_decision", "patch": "repair_patch", "repair_failed": "repair_validation", } DONE_STATUSES = { "success", "done", "completed", "passed", "full_inference_success", "manual_hardware_required", "technical_blocker", "technical_blocker_boot_only", } PARTIAL_STATUSES = { "full_inference_candidate_health_passed", "health_only", "partial", "partial_validation", "completed_with_warnings", } RUNNING_STATUSES = {"started", "running", "waiting", "pending", "scheduled"} FAILED_STATUSES = {"failed", "error", "failure", "repair_validation_failed", "upload_failed", "smoke_still_failed", "repair_failed_same_error", "repair_exhausted", "no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard"} CANCELLED_STATUSES = {"cancelled", "canceled"} def _parse_ts(ts: str | None) -> datetime | None: if not ts: return None try: return datetime.fromisoformat(ts.replace("Z", "+00:00")) except Exception: return None def _canonical_step(step: str | None) -> str | None: if not step: return None value = str(step) return STEP_ALIASES.get(value, value if value in STEP_ORDER else None) def _is_terminal_status(status: str | None) -> bool: s = str(status or "").lower() return s in DONE_STATUSES or s in PARTIAL_STATUSES or s in FAILED_STATUSES or s in CANCELLED_STATUSES def _progress_for_index(index: int) -> int: if index <= 0: return 3 if len(STEP_ORDER) <= 1: return 0 return int(round((index / (len(STEP_ORDER) - 1)) * 100)) def progress_from_events(events: list[dict[str, Any]] | None, *, state: dict[str, Any] | None = None) -> dict[str, Any]: """Build a monotonic live progress model from worker events + Job state. The UI was confusing when late/partial bucket events made steps appear to move backwards. This model treats the furthest observed real worker step as the source of truth, marks all earlier steps done, keeps only that step running while the Job is active, and freezes it as stopped on terminal non-success statuses. """ events = events or [] state = state or {} step_status = {step: "pending" for step in STEP_ORDER} last_event: dict[str, Any] | None = None terminal_status = None first_ts = None last_ts = None furthest_index = -1 current_step = "bootstrap" current_status = "" current_event: dict[str, Any] | None = None latest_signal_step = "" latest_signal_status = "" latest_signal_event: dict[str, Any] | None = None latest_signal_ts: datetime | None = None for event in events: if not isinstance(event, dict): continue step = _canonical_step(event.get("step")) status = str(event.get("status") or "").lower() ts = _parse_ts(event.get("ts")) if ts and not first_ts: first_ts = ts if ts: last_ts = ts if not step: continue last_event = event idx = STEP_ORDER.index(step) # Never let older / lower-index failed events poison the whole run once # later validation or repair steps have appeared. if idx >= furthest_index: furthest_index = idx current_step = step current_status = status current_event = event if ts is None or latest_signal_ts is None or ts >= latest_signal_ts: latest_signal_ts = ts latest_signal_step = step latest_signal_status = status latest_signal_event = event if latest_signal_status in RUNNING_STATUSES: current_step = latest_signal_step or current_step current_status = latest_signal_status current_event = latest_signal_event or current_event if current_step in STEP_ORDER: furthest_index = STEP_ORDER.index(current_step) terminal_status = None if current_step == "done" and current_status: terminal_status = current_status elif current_step == "failure" and current_status: terminal_status = "failed" if current_status in FAILED_STATUSES or current_status in DONE_STATUSES else current_status elif current_status in CANCELLED_STATUSES: terminal_status = "cancelled" elif current_status in FAILED_STATUSES: terminal_status = "failed" repair = state.get("repair_outcome") if isinstance(state.get("repair_outcome"), dict) else {} repair_status = str((repair or {}).get("post_repair_validation") or (repair or {}).get("failure_type") or (repair or {}).get("status") or "").lower() status_from_state = str(state.get("status") or state.get("gate_status") or "").lower() if repair_status in FAILED_STATUSES: status_from_state = "failed" elif repair_status in DONE_STATUSES: status_from_state = repair_status job_stage = str(state.get("job_stage") or "").lower() if status_from_state in CANCELLED_STATUSES: terminal_status = "cancelled" elif status_from_state in PARTIAL_STATUSES: terminal_status = status_from_state elif status_from_state in DONE_STATUSES: terminal_status = status_from_state elif status_from_state in FAILED_STATUSES: terminal_status = "failed" if current_status not in FAILED_STATUSES and current_step != "failure": current_step = "failure" current_status = "failed" furthest_index = STEP_ORDER.index("failure") elif job_stage in CANCELLED_STATUSES: terminal_status = "cancelled" elif job_stage in FAILED_STATUSES: # A terminal HF Job failure must produce an explicit red timeline point. terminal_status = "failed" if current_status not in FAILED_STATUSES and current_step != "failure": current_step = "failure" current_status = "failed" furthest_index = STEP_ORDER.index("failure") if terminal_status: overall_status = terminal_status elif current_status in RUNNING_STATUSES: overall_status = "running" elif status_from_state in RUNNING_STATUSES: overall_status = "running" elif events: overall_status = "running" else: overall_status = "not_started" terminal = _is_terminal_status(overall_status) if furthest_index < 0: furthest_index = 0 if overall_status in DONE_STATUSES or overall_status in PARTIAL_STATUSES or current_step == "done": current_step = "done" furthest_index = STEP_ORDER.index("done") for step in STEP_ORDER: if step != "failure": step_status[step] = "done" elif overall_status in FAILED_STATUSES: failure_idx = STEP_ORDER.index("failure") if current_step in STEP_ORDER and current_step != "failure": # Preserve the failed worker step when it is known; otherwise expose # the explicit Failure stage so the UI always has a red dot. failed_idx = STEP_ORDER.index(current_step) else: failed_idx = failure_idx current_step = "failure" furthest_index = max(furthest_index, failed_idx) for idx, step in enumerate(STEP_ORDER): if idx < failed_idx: step_status[step] = "done" elif idx == failed_idx: step_status[step] = "failed" else: step_status[step] = "pending" step_status["failure"] = "failed" else: for idx, step in enumerate(STEP_ORDER): if idx < furthest_index: step_status[step] = "done" elif idx == furthest_index: if overall_status in FAILED_STATUSES: step_status[step] = "failed" elif overall_status in CANCELLED_STATUSES or (terminal and overall_status not in DONE_STATUSES): step_status[step] = "stopped" elif overall_status in {"not_started", "idle"}: step_status[step] = "pending" else: step_status[step] = "running" else: step_status[step] = "pending" if overall_status in FAILED_STATUSES: progress = max(_progress_for_index(furthest_index), 3) elif overall_status in CANCELLED_STATUSES: progress = max(_progress_for_index(furthest_index), 3) elif overall_status in DONE_STATUSES or overall_status in PARTIAL_STATUSES or current_step == "done": progress = 100 else: progress = max(3, min(99, _progress_for_index(furthest_index))) now = datetime.now(timezone.utc) started = first_ts or _parse_ts(str(state.get("started_at") or state.get("created_at") or "")) if started: end = (last_ts if terminal and last_ts else now) elapsed = int((end - started).total_seconds()) else: elapsed = 0 timeline = [ {"step": step, "label": STEP_LABELS[step], "status": step_status[step]} for step in STEP_ORDER ] last_message = None if current_event: last_message = current_event.get("message") or current_event.get("step") elif last_event: last_message = last_event.get("message") or last_event.get("step") return { "status": overall_status, "progress": int(max(0, min(100, progress))), "current_step": current_step, "current_step_label": STEP_LABELS.get(current_step, current_step), "last_event": last_message or "No events yet", "visual_status": "error" if overall_status in FAILED_STATUSES else ("warn" if overall_status in PARTIAL_STATUSES else ("stopped" if overall_status in CANCELLED_STATUSES else ("success" if overall_status in DONE_STATUSES and overall_status not in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("stopped" if overall_status in {"manual_hardware_required", "technical_blocker", "technical_blocker_boot_only"} else ("running" if overall_status == "running" else "neutral"))))), "elapsed_seconds": max(0, elapsed), "eta_seconds": None, "timeline": timeline, "last_event_raw": last_event or {}, }