fffiloni's picture
Upload 2 files
cfac53d verified
Raw
History Blame
20.5 kB
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
from .progress import STEP_ALIASES, STEP_LABELS, STEP_ORDER
from .config import settings
PRODUCT_STEPS = [{"id": step, "label": STEP_LABELS[step]} for step in STEP_ORDER]
def _runs_prefix() -> str:
return settings.bucket_runs_prefix.strip().strip("/") or "runs"
TERMINAL_GLOBAL_STATUSES = {"succeeded", "failed", "cancelled", "blocked", "waiting_manual_action"}
SUCCESS_RAW_STATUSES = {
"success",
"done",
"completed",
"passed",
"full_inference_success",
"full_inference_candidate_health_passed",
"repair_success",
}
MANUAL_RAW_STATUSES = {
"manual_hardware_required",
"generated_needs_manual_hardware",
"waiting_manual_hardware",
"manual_action_required",
}
FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed"}
CANCELLED_RAW_STATUSES = {"cancelled", "canceled"}
BLOCKED_RAW_STATUSES = {"technical_blocker", "blocked", "health_only"}
RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"}
STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()}
def parse_ts(value: Any) -> datetime | None:
if not value:
return None
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
except Exception:
return None
def _lower(value: Any) -> str:
return str(value or "").strip().lower()
def _first_nonempty(*values: Any) -> str:
for value in values:
if value is not None and str(value).strip():
return str(value)
return ""
def _owner_from_bucket_source(bucket_source: str | None) -> str:
if not bucket_source:
return ""
return str(bucket_source).split("/", 1)[0].strip()
def _job_url_from_view_sources(*, bucket_source: str, summary: dict[str, Any], state: dict[str, Any], launch: dict[str, Any]) -> str:
explicit = summary.get("job_url") or state.get("job_url") or launch.get("job_url")
if explicit:
return str(explicit)
job_id = summary.get("job_id") or state.get("job_id") or launch.get("job_id")
owner = (
summary.get("created_by")
or summary.get("username")
or summary.get("owner")
or state.get("created_by")
or state.get("username")
or state.get("owner")
or launch.get("created_by")
or launch.get("username")
or launch.get("owner")
or _owner_from_bucket_source(bucket_source)
)
if job_id and owner:
return f"https://huggingface.co/jobs/{owner}/{job_id}"
return ""
def _latest_activity_at(bundle: dict[str, Any], summary: dict[str, Any], state: dict[str, Any], launch: dict[str, Any]) -> datetime | None:
candidates: list[datetime] = []
for event in bundle.get("events") or []:
ts = parse_ts(event.get("ts") if isinstance(event, dict) else None)
if ts:
candidates.append(ts)
for source in (summary, state, launch, bundle.get("summary_file") or {}):
for key in ("updated_at", "finished_at", "created_at", "started_at"):
ts = parse_ts(source.get(key) if isinstance(source, dict) else None)
if ts:
candidates.append(ts)
return max(candidates) if candidates else None
def _created_at(summary: dict[str, Any], state: dict[str, Any], launch: dict[str, Any]) -> datetime | None:
for source in (summary, state, launch):
ts = parse_ts(source.get("created_at") if isinstance(source, dict) else None)
if ts:
return ts
return None
def _raw_statuses(bundle: dict[str, Any]) -> set[str]:
summary = bundle.get("summary") or {}
state = bundle.get("state") or {}
launch = bundle.get("launch") or {}
gate = bundle.get("inference_gate") or {}
smoke = bundle.get("generation_smoke") or {}
hardware = bundle.get("hardware_strategy") or {}
blockers = bundle.get("technical_blockers") or {}
values = {
_lower(summary.get("status")),
_lower(state.get("status")),
_lower(state.get("gate_status")),
_lower(launch.get("status")),
_lower(gate.get("status")),
_lower(smoke.get("status")),
_lower(hardware.get("status")),
_lower(blockers.get("status")),
}
# Event-level statuses describe individual steps and must not by themselves
# turn a whole run into success/failure. The product phase is derived from
# events separately in derive_product_phase().
return {v for v in values if v}
def requires_manual_action(bundle: dict[str, Any]) -> bool:
summary = bundle.get("summary") or {}
gate = bundle.get("inference_gate") or {}
hardware = bundle.get("hardware_strategy") or {}
statuses = _raw_statuses(bundle)
return bool(
summary.get("manual_hardware_required")
or gate.get("manual_hardware_required")
or hardware.get("manual_action_required")
or statuses.intersection(MANUAL_RAW_STATUSES)
)
def has_technical_blocker(bundle: dict[str, Any]) -> bool:
blockers = bundle.get("technical_blockers") or {}
blocker_items = blockers.get("blockers") if isinstance(blockers, dict) else None
return bool(blocker_items or _raw_statuses(bundle).intersection(BLOCKED_RAW_STATUSES))
def normalize_run_status(
bundle: dict[str, Any],
*,
now: datetime | None = None,
stale_after: timedelta = timedelta(hours=6),
) -> dict[str, Any]:
"""Return canonical status flags for the product UI."""
now = now or datetime.now(timezone.utc)
summary = bundle.get("summary") or {}
state = bundle.get("state") or {}
launch = bundle.get("launch") or {}
statuses = _raw_statuses(bundle)
manual = requires_manual_action(bundle)
blocker = has_technical_blocker(bundle)
if statuses.intersection(CANCELLED_RAW_STATUSES):
global_status = "cancelled"
verdict = "cancelled"
elif manual:
global_status = "waiting_manual_action"
verdict = "manual_action_required"
elif statuses.intersection(SUCCESS_RAW_STATUSES):
global_status = "succeeded"
verdict = "passed"
elif statuses.intersection(FAILED_RAW_STATUSES):
global_status = "failed"
verdict = "failed"
elif blocker:
global_status = "blocked"
verdict = "technical_blocker"
elif statuses.intersection(RUNNING_RAW_STATUSES) or bundle.get("events") or launch:
global_status = "running"
verdict = "pending"
else:
global_status = "unknown"
verdict = "unknown"
latest = _latest_activity_at(bundle, summary, state, launch)
created = _created_at(summary, state, launch)
reference = latest or created
is_terminal = global_status in TERMINAL_GLOBAL_STATUSES or global_status in {"succeeded"}
is_stale = False
if reference and not is_terminal and now - reference > stale_after:
global_status = "stale"
verdict = "stale"
is_stale = True
return {
"global_status": global_status,
"raw_status": _first_nonempty(summary.get("status"), state.get("status"), launch.get("status"), "unknown"),
"verdict": verdict,
"requires_manual_action": manual,
"manual_action_type": "hardware" if manual else "",
"has_technical_blocker": blocker,
"has_target_space": bool(summary.get("target_space") or state.get("target_space") or launch.get("target_space")),
"has_job_url": bool(summary.get("job_url") or state.get("job_url") or launch.get("job_url") or summary.get("job_id") or state.get("job_id") or launch.get("job_id")),
"has_live_api_result": bool((bundle.get("generation_smoke") or {}).get("ok") or (bundle.get("generation_smoke") or {}).get("status") == "success"),
"is_terminal": global_status in {"succeeded", "failed", "blocked", "waiting_manual_action"},
"is_pollable": global_status in {"queued", "running", "validating", "unknown"},
"is_stale": is_stale,
"latest_activity_at": latest.isoformat() if latest else "",
}
def derive_product_phase(bundle: dict[str, Any], status_model: dict[str, Any]) -> str:
if status_model["global_status"] == "succeeded":
return "done"
if status_model["global_status"] in {"failed", "blocked", "stale", "cancelled"}:
return "failure"
if status_model["requires_manual_action"]:
return "inference_gate"
events = [e for e in (bundle.get("events") or []) if isinstance(e, dict)]
for event in reversed(events):
step = _lower(event.get("step"))
message = _lower(event.get("message"))
status = _lower(event.get("status"))
if "patch" in step or "repair" in step or "patched" in message or "restart" in message:
return "hardware_strategy"
if "missing" in message or "error" in message or status in FAILED_RAW_STATUSES:
return "hardware_strategy"
phase = STEP_TO_PHASE.get(step)
if phase:
return phase
summary = bundle.get("summary") or {}
if summary.get("target_space"):
return "hardware_strategy"
if bundle.get("launch"):
return "bootstrap"
return "bootstrap"
def build_pipeline(phase: str, status_model: dict[str, Any]) -> list[dict[str, Any]]:
phase_ids = [s["id"] for s in PRODUCT_STEPS]
current_index = phase_ids.index(phase) if phase in phase_ids else 0
global_status = status_model["global_status"]
pipeline: list[dict[str, Any]] = []
for index, step in enumerate(PRODUCT_STEPS):
if global_status == "succeeded":
step_status = "completed"
elif index < current_index:
step_status = "completed"
elif index == current_index:
if global_status == "failed":
step_status = "failed"
elif global_status in {"blocked", "waiting_manual_action", "stale"}:
step_status = "blocked"
else:
step_status = "running"
else:
step_status = "pending"
pipeline.append({**step, "status": step_status})
return pipeline
def _agent_for_event(event: dict[str, Any]) -> str:
step = _lower(event.get("step"))
if step in {"model_analysis", "bucket_ready", "job_launched", "bootstrap"}:
return "Planner Agent"
if step in {"workspace", "node", "pi_install", "pi_config", "pi_run", "repair", "patch"}:
return "Coder Agent"
if step in {"create_space", "upload_files", "hardware", "hardware_preferred", "hardware_fallback"}:
return "Hub Agent"
if step in {"api_validation", "generation_smoke", "inference_gate"}:
return "Tester Agent"
if _lower(event.get("status")) in FAILED_RAW_STATUSES:
return "Diagnostician Agent"
return "Factory Agent"
def _severity_for_event(event: dict[str, Any]) -> str:
status = _lower(event.get("status"))
message = _lower(event.get("message"))
if status in FAILED_RAW_STATUSES or "traceback" in message or "modulenotfound" in message:
return "error"
if status in {"warning", "manual_hardware_required"} or "manual" in message:
return "warning"
if status in SUCCESS_RAW_STATUSES:
return "success"
if status in RUNNING_RAW_STATUSES:
return "running"
return "info"
def build_activity_feed(bundle: dict[str, Any], *, limit: int = 40) -> list[dict[str, Any]]:
events = [e for e in (bundle.get("events") or []) if isinstance(e, dict)]
if not events and bundle.get("launch"):
launch = bundle.get("launch") or {}
events = [
{
"ts": launch.get("created_at") or "",
"step": "job_launched",
"status": launch.get("status") or "running",
"message": f"Build job launched for {launch.get('target_space') or 'target Space'}",
}
]
feed: list[dict[str, Any]] = []
for event in events[-limit:]:
feed.append(
{
"ts": event.get("ts") or event.get("created_at") or "",
"step": event.get("step") or "",
"status": event.get("status") or "",
"message": event.get("message") or event.get("step") or "Event received",
"severity": _severity_for_event(event),
"agent": _agent_for_event(event),
}
)
return feed
def build_diagnostics(bundle: dict[str, Any], status_model: dict[str, Any], phase: str) -> dict[str, Any]:
summary = bundle.get("summary") or {}
smoke = bundle.get("generation_smoke") or {}
gate = bundle.get("inference_gate") or {}
hardware = bundle.get("hardware_strategy") or {}
blockers = bundle.get("technical_blockers") or {}
blocker_items = blockers.get("blockers") if isinstance(blockers, dict) else []
issue_title = ""
issue_detail = ""
issue_status = ""
if blocker_items:
first = blocker_items[0]
issue_title = _first_nonempty(first.get("type"), first.get("name"), "Technical blocker") if isinstance(first, dict) else "Technical blocker"
issue_detail = _first_nonempty(first.get("claim"), first.get("reason"), first.get("message")) if isinstance(first, dict) else str(first)
issue_status = "open"
elif status_model["requires_manual_action"]:
issue_title = "Manual hardware required"
issue_detail = "Automatic hardware selection was not available. Choose hardware in Space settings, then run Space Test."
issue_status = "action_required"
elif status_model["global_status"] == "failed":
issue_title = "Run failed"
issue_detail = "Inspect logs and run artifacts for the failing step."
issue_status = "open"
health_passed = bool((gate.get("implementation_signals") or {}).get("health_passed") or smoke.get("health_passed"))
smoke_ok = bool(smoke.get("ok") or smoke.get("status") == "success")
build_ok = phase in {"api_validation", "live_wait", "generation_smoke", "inference_gate", "report_write", "done"} or status_model["global_status"] == "succeeded"
return {
"build_status": "passed" if build_ok else ("blocked" if status_model["global_status"] in {"blocked", "waiting_manual_action", "failed"} else "building"),
"api_status": "passed" if smoke_ok else ("blocked" if status_model["global_status"] in {"blocked", "failed"} else "pending"),
"tests_status": "passed" if smoke_ok or health_passed else ("blocked" if status_model["global_status"] in {"blocked", "failed"} else "pending"),
"verdict_status": status_model["verdict"],
"zerogpu_rules": [
{"label": "Gradio interface", "status": "ok"},
{"label": "Private Space", "status": "ok" if summary.get("target_space") else "pending"},
{"label": "ZeroGPU-first strategy", "status": "ok" if "zero" in _lower(summary.get("selected_hardware") or hardware.get("preferred_space_hardware") or "zero") else "pending"},
{"label": "Live API verification", "status": "ok" if smoke_ok else "pending"},
],
"issue": {"title": issue_title, "detail": issue_detail, "status": issue_status},
}
def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]:
smoke = bundle.get("generation_smoke") or {}
summary = bundle.get("summary") or {}
return {
"target_space": summary.get("target_space") or "",
"target_space_url": summary.get("target_space_url") or "",
"endpoint": smoke.get("api_name") or smoke.get("endpoint") or "/generate",
"status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending",
"latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"),
"expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "",
"verdict": status_model["verdict"],
"output_artifact": smoke.get("output_artifact") or smoke.get("artifact_url") or "",
}
def build_run_view_model(
run_id: str,
bundle: dict[str, Any],
*,
bucket_source: str,
now: datetime | None = None,
) -> dict[str, Any]:
summary = bundle.get("summary") or {}
state = bundle.get("state") or {}
status_model = normalize_run_status(bundle, now=now)
phase = derive_product_phase(bundle, status_model)
elapsed_seconds = None
current_now = now or datetime.now(timezone.utc)
created = parse_ts(summary.get("started_at") or state.get("started_at") or summary.get("created_at") or state.get("created_at"))
latest = parse_ts(status_model.get("latest_activity_at"))
if created:
end = latest if status_model.get("is_terminal") and latest else current_now
elapsed_seconds = int((end - created).total_seconds())
links = {
"job_url": _job_url_from_view_sources(bucket_source=bucket_source, summary=summary, state=state, launch=bundle.get("launch") or {}),
"target_space_url": summary.get("target_space_url") or "",
"target_space_settings_url": f"{summary.get('target_space_url')}/settings" if summary.get("target_space_url") else "",
"artifacts_url": summary.get("artifacts_url") or f"https://huggingface.co/buckets/{bucket_source}/tree/{_runs_prefix()}/{run_id}",
}
return {
"schema_version": "run_view_model.v1",
"run_id": run_id,
"bucket_source": bucket_source,
"header": {
"title": _first_nonempty(summary.get("title"), f"Build Space for {summary.get('model_id')}", run_id),
"status": status_model["global_status"],
"status_label": status_model["global_status"].replace("_", " ").title(),
"raw_status": status_model["raw_status"],
"space": summary.get("target_space") or "",
"space_url": summary.get("target_space_url") or "",
"current_phase": phase,
"current_phase_label": next((s["label"] for s in PRODUCT_STEPS if s["id"] == phase), phase),
"elapsed_seconds": max(0, elapsed_seconds or 0),
"started_at": summary.get("created_at") or state.get("created_at") or "",
"updated_at": status_model["latest_activity_at"],
},
"status_model": status_model,
"pipeline": build_pipeline(phase, status_model),
"activity": build_activity_feed(bundle),
"diagnostics": build_diagnostics(bundle, status_model, phase),
"space_test": build_space_test_model(bundle, status_model),
"actions": {
"can_resume": status_model["global_status"] in {"running", "stale", "unknown"},
"can_stop": status_model["global_status"] in {"running", "queued", "unknown"},
"can_open_space": bool(summary.get("target_space_url")),
"can_validate": bool(summary.get("target_space")),
"requires_manual_action": status_model["requires_manual_action"],
},
"links": links,
}
def is_resumable_summary(summary: dict[str, Any], *, now: datetime | None = None, stale_after: timedelta = timedelta(hours=6)) -> bool:
"""Best-effort resumability test for lightweight /api/runs summaries."""
status = _lower(summary.get("status"))
if status in SUCCESS_RAW_STATUSES or status in FAILED_RAW_STATUSES or status in BLOCKED_RAW_STATUSES or status in MANUAL_RAW_STATUSES:
return False
if "success" in status or "failed" in status or "blocker" in status or "manual" in status:
return False
now = now or datetime.now(timezone.utc)
latest = parse_ts(summary.get("updated_at") or summary.get("created_at"))
if latest and now - latest > stale_after:
return False
return status in RUNNING_RAW_STATUSES or status in {"unknown", ""}
def find_latest_resumable_run(summaries: list[dict[str, Any]], *, now: datetime | None = None) -> dict[str, Any] | None:
candidates = [s for s in summaries if isinstance(s, dict) and is_resumable_summary(s, now=now)]
if not candidates:
return None
return sorted(candidates, key=lambda s: str(s.get("updated_at") or s.get("created_at") or s.get("run_id")), reverse=True)[0]