from __future__ import annotations from datetime import datetime, timedelta, timezone from typing import Any from .progress import STEP_ALIASES, STEP_LABELS, STEP_ORDER from .timeline_model import build_run_timeline_model from .config import settings PRODUCT_STEPS = [{"id": step, "label": STEP_LABELS[step]} for step in STEP_ORDER] def _runs_prefix() -> str: return settings.bucket_runs_prefix.strip().strip("/") or "runs" TERMINAL_GLOBAL_STATUSES = {"succeeded", "partial", "failed", "cancelled", "blocked", "waiting_manual_action"} SUCCESS_RAW_STATUSES = { "success", "done", "completed", "passed", "full_inference_success", "repair_success", } PARTIAL_RAW_STATUSES = { "full_inference_candidate_health_passed", "health_only", "partial", "partial_validation", "completed_with_warnings", } MANUAL_RAW_STATUSES = { "manual_hardware_required", "generated_needs_manual_hardware", "waiting_manual_hardware", "manual_action_required", } FAILED_RAW_STATUSES = {"failed", "failure", "error", "repair_failed"} CANCELLED_RAW_STATUSES = {"cancelled", "canceled"} BLOCKED_RAW_STATUSES = {"technical_blocker", "blocked", "health_only"} RUNNING_RAW_STATUSES = {"started", "running", "waiting", "queued", "pending", "scheduled"} STEP_TO_PHASE = {step: step for step in STEP_ORDER} | {alias: canonical for alias, canonical in STEP_ALIASES.items()} def parse_ts(value: Any) -> datetime | None: if not value: return None if isinstance(value, datetime): return value if value.tzinfo else value.replace(tzinfo=timezone.utc) try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) except Exception: return None def _lower(value: Any) -> str: return str(value or "").strip().lower() def _first_nonempty(*values: Any) -> str: for value in values: if value is not None and str(value).strip(): return str(value) return "" def _owner_from_bucket_source(bucket_source: str | None) -> str: if not bucket_source: return "" return str(bucket_source).split("/", 1)[0].strip() def _job_url_from_view_sources(*, bucket_source: str, summary: dict[str, Any], state: dict[str, Any], launch: dict[str, Any]) -> str: explicit = summary.get("job_url") or state.get("job_url") or launch.get("job_url") if explicit: return str(explicit) job_id = summary.get("job_id") or state.get("job_id") or launch.get("job_id") owner = ( summary.get("created_by") or summary.get("username") or summary.get("owner") or state.get("created_by") or state.get("username") or state.get("owner") or launch.get("created_by") or launch.get("username") or launch.get("owner") or _owner_from_bucket_source(bucket_source) ) if job_id and owner: return f"https://huggingface.co/jobs/{owner}/{job_id}" return "" def _latest_activity_at(bundle: dict[str, Any], summary: dict[str, Any], state: dict[str, Any], launch: dict[str, Any]) -> datetime | None: candidates: list[datetime] = [] for event in bundle.get("events") or []: ts = parse_ts(event.get("ts") if isinstance(event, dict) else None) if ts: candidates.append(ts) for source in (summary, state, launch, bundle.get("summary_file") or {}): for key in ("updated_at", "finished_at", "created_at", "started_at"): ts = parse_ts(source.get(key) if isinstance(source, dict) else None) if ts: candidates.append(ts) return max(candidates) if candidates else None def _created_at(summary: dict[str, Any], state: dict[str, Any], launch: dict[str, Any]) -> datetime | None: for source in (summary, state, launch): ts = parse_ts(source.get("created_at") if isinstance(source, dict) else None) if ts: return ts return None def _raw_statuses(bundle: dict[str, Any]) -> set[str]: summary = bundle.get("summary") or {} state = bundle.get("state") or {} launch = bundle.get("launch") or {} gate = bundle.get("inference_gate") or {} smoke = bundle.get("generation_smoke") or {} hardware = bundle.get("hardware_strategy") or {} blockers = bundle.get("technical_blockers") or {} values = { _lower(summary.get("status")), _lower(state.get("status")), _lower(state.get("gate_status")), _lower(launch.get("status")), _lower(gate.get("status")), _lower(smoke.get("status")), _lower(hardware.get("status")), _lower(blockers.get("status")), } # Event-level statuses describe individual steps and must not by themselves # turn a whole run into success/failure. The product phase is derived from # events separately in derive_product_phase(). return {v for v in values if v} def requires_manual_action(bundle: dict[str, Any]) -> bool: summary = bundle.get("summary") or {} gate = bundle.get("inference_gate") or {} hardware = bundle.get("hardware_strategy") or {} statuses = _raw_statuses(bundle) return bool( summary.get("manual_hardware_required") or gate.get("manual_hardware_required") or hardware.get("manual_action_required") or statuses.intersection(MANUAL_RAW_STATUSES) ) def has_technical_blocker(bundle: dict[str, Any]) -> bool: blockers = bundle.get("technical_blockers") or {} blocker_items = blockers.get("blockers") if isinstance(blockers, dict) else None return bool(blocker_items or _raw_statuses(bundle).intersection(BLOCKED_RAW_STATUSES)) def normalize_run_status( bundle: dict[str, Any], *, now: datetime | None = None, stale_after: timedelta = timedelta(hours=6), ) -> dict[str, Any]: """Return canonical status flags for the product UI.""" now = now or datetime.now(timezone.utc) summary = bundle.get("summary") or {} state = bundle.get("state") or {} launch = bundle.get("launch") or {} statuses = _raw_statuses(bundle) manual = requires_manual_action(bundle) blocker = has_technical_blocker(bundle) if statuses.intersection(CANCELLED_RAW_STATUSES): global_status = "cancelled" verdict = "cancelled" elif manual: global_status = "waiting_manual_action" verdict = "manual_action_required" elif statuses.intersection(SUCCESS_RAW_STATUSES): global_status = "succeeded" verdict = "passed" elif statuses.intersection(PARTIAL_RAW_STATUSES): global_status = "partial" verdict = "partial_validation" elif statuses.intersection(FAILED_RAW_STATUSES): global_status = "failed" verdict = "failed" elif blocker: global_status = "blocked" verdict = "technical_blocker" elif statuses.intersection(RUNNING_RAW_STATUSES) or bundle.get("events") or launch: global_status = "running" verdict = "pending" else: global_status = "unknown" verdict = "unknown" latest = _latest_activity_at(bundle, summary, state, launch) created = _created_at(summary, state, launch) reference = latest or created is_terminal = global_status in TERMINAL_GLOBAL_STATUSES or global_status in {"succeeded"} is_stale = False if reference and not is_terminal and now - reference > stale_after: global_status = "stale" verdict = "stale" is_stale = True return { "global_status": global_status, "raw_status": _first_nonempty(summary.get("status"), state.get("status"), launch.get("status"), "unknown"), "verdict": verdict, "requires_manual_action": manual, "manual_action_type": "hardware" if manual else "", "has_technical_blocker": blocker, "has_target_space": bool(summary.get("target_space") or state.get("target_space") or launch.get("target_space")), "has_job_url": bool(summary.get("job_url") or state.get("job_url") or launch.get("job_url") or summary.get("job_id") or state.get("job_id") or launch.get("job_id")), "has_live_api_result": bool((bundle.get("generation_smoke") or {}).get("ok") or (bundle.get("generation_smoke") or {}).get("status") == "success"), "is_terminal": global_status in {"succeeded", "partial", "failed", "blocked", "waiting_manual_action", "cancelled"}, "is_pollable": global_status in {"queued", "running", "validating", "unknown"}, "is_stale": is_stale, "latest_activity_at": latest.isoformat() if latest else "", } def derive_product_phase(bundle: dict[str, Any], status_model: dict[str, Any]) -> str: if status_model["global_status"] in {"succeeded", "partial"}: return "done" if status_model["global_status"] in {"failed", "blocked", "stale", "cancelled"}: return "failure" if status_model["requires_manual_action"]: return "inference_gate" events = [e for e in (bundle.get("events") or []) if isinstance(e, dict)] for event in reversed(events): step = _lower(event.get("step")) message = _lower(event.get("message")) status = _lower(event.get("status")) if "patch" in step or "repair" in step or "patched" in message or "restart" in message: return "hardware_strategy" if "missing" in message or "error" in message or status in FAILED_RAW_STATUSES: return "hardware_strategy" phase = STEP_TO_PHASE.get(step) if phase: return phase summary = bundle.get("summary") or {} if summary.get("target_space"): return "hardware_strategy" if bundle.get("launch"): return "bootstrap" return "bootstrap" def build_pipeline(phase: str, status_model: dict[str, Any]) -> list[dict[str, Any]]: phase_ids = [s["id"] for s in PRODUCT_STEPS] current_index = phase_ids.index(phase) if phase in phase_ids else 0 global_status = status_model["global_status"] pipeline: list[dict[str, Any]] = [] for index, step in enumerate(PRODUCT_STEPS): if global_status == "succeeded": step_status = "completed" elif index < current_index: step_status = "completed" elif index == current_index: if global_status == "failed": step_status = "failed" elif global_status in {"blocked", "waiting_manual_action", "stale"}: step_status = "blocked" else: step_status = "running" else: step_status = "pending" pipeline.append({**step, "status": step_status}) return pipeline def _agent_for_event(event: dict[str, Any]) -> str: step = _lower(event.get("step")) if step in {"model_analysis", "bucket_ready", "job_launched", "bootstrap"}: return "Planner Agent" if step in {"workspace", "node", "pi_install", "pi_config", "pi_run", "repair", "patch"}: return "Coder Agent" if step in {"create_space", "upload_files", "hardware", "hardware_preferred", "hardware_fallback"}: return "Hub Agent" if step in {"api_validation", "generation_smoke", "inference_gate"}: return "Tester Agent" if _lower(event.get("status")) in FAILED_RAW_STATUSES: return "Diagnostician Agent" return "Factory Agent" def _severity_for_event(event: dict[str, Any]) -> str: status = _lower(event.get("status")) message = _lower(event.get("message")) if status in FAILED_RAW_STATUSES or "traceback" in message or "modulenotfound" in message: return "error" if status in {"warning", "manual_hardware_required"} or "manual" in message: return "warning" if status in SUCCESS_RAW_STATUSES: return "success" if status in PARTIAL_RAW_STATUSES: return "warning" if status in RUNNING_RAW_STATUSES: return "running" return "info" def build_activity_feed(bundle: dict[str, Any], *, limit: int = 40) -> list[dict[str, Any]]: events = [e for e in (bundle.get("events") or []) if isinstance(e, dict)] if not events and bundle.get("launch"): launch = bundle.get("launch") or {} events = [ { "ts": launch.get("created_at") or "", "step": "job_launched", "status": launch.get("status") or "running", "message": f"Build job launched for {launch.get('target_space') or 'target Space'}", } ] feed: list[dict[str, Any]] = [] for event in events[-limit:]: feed.append( { "ts": event.get("ts") or event.get("created_at") or "", "step": event.get("step") or "", "status": event.get("status") or "", "message": event.get("message") or event.get("step") or "Event received", "severity": _severity_for_event(event), "agent": _agent_for_event(event), } ) return feed def build_diagnostics(bundle: dict[str, Any], status_model: dict[str, Any], phase: str) -> dict[str, Any]: summary = bundle.get("summary") or {} smoke = bundle.get("generation_smoke") or {} gate = bundle.get("inference_gate") or {} hardware = bundle.get("hardware_strategy") or {} blockers = bundle.get("technical_blockers") or {} blocker_items = blockers.get("blockers") if isinstance(blockers, dict) else [] issue_title = "" issue_detail = "" issue_status = "" if blocker_items: first = blocker_items[0] issue_title = _first_nonempty(first.get("type"), first.get("name"), "Technical blocker") if isinstance(first, dict) else "Technical blocker" issue_detail = _first_nonempty(first.get("claim"), first.get("reason"), first.get("message")) if isinstance(first, dict) else str(first) issue_status = "open" elif status_model["requires_manual_action"]: issue_title = "Manual hardware required" issue_detail = "Automatic hardware selection was not available. Choose hardware in Space settings, then run Space Test." issue_status = "action_required" elif status_model["global_status"] == "failed": issue_title = "Run failed" issue_detail = "Inspect logs and run artifacts for the failing step." issue_status = "open" health_passed = bool((gate.get("implementation_signals") or {}).get("health_passed") or smoke.get("health_passed")) smoke_ok = bool(smoke.get("ok") or smoke.get("status") == "success") build_ok = phase in {"api_validation", "live_wait", "generation_smoke", "inference_gate", "report_write", "done"} or status_model["global_status"] == "succeeded" return { "build_status": "passed" if build_ok else ("blocked" if status_model["global_status"] in {"blocked", "waiting_manual_action", "failed"} else "building"), "api_status": "passed" if smoke_ok else ("blocked" if status_model["global_status"] in {"blocked", "failed"} else "pending"), "tests_status": "passed" if smoke_ok or health_passed else ("blocked" if status_model["global_status"] in {"blocked", "failed"} else "pending"), "verdict_status": status_model["verdict"], "zerogpu_rules": [ {"label": "Gradio interface", "status": "ok"}, {"label": "Private Space", "status": "ok" if summary.get("target_space") else "pending"}, {"label": "ZeroGPU-first strategy", "status": "ok" if "zero" in _lower(summary.get("selected_hardware") or hardware.get("preferred_space_hardware") or "zero") else "pending"}, {"label": "Live API verification", "status": "ok" if smoke_ok else "pending"}, ], "issue": {"title": issue_title, "detail": issue_detail, "status": issue_status}, } def _space_target_from_bundle(bundle: dict[str, Any]) -> str: summary = bundle.get("summary") or {} state = bundle.get("state") or {} launch = bundle.get("launch") or {} return _first_nonempty( summary.get("target_space"), state.get("target_space"), launch.get("target_space"), summary.get("target_space_id"), ) def _space_target_url_from_bundle(bundle: dict[str, Any], target: str = "") -> str: summary = bundle.get("summary") or {} state = bundle.get("state") or {} launch = bundle.get("launch") or {} explicit = _first_nonempty(summary.get("target_space_url"), state.get("target_space_url"), launch.get("target_space_url")) if explicit: return explicit return f"https://huggingface.co/spaces/{target}" if target else "" def _known_gradio_endpoint_info(bundle: dict[str, Any]) -> dict[str, Any]: smoke = bundle.get("generation_smoke") or {} gate = bundle.get("inference_gate") or {} live = bundle.get("live_status") or {} schema = bundle.get("api_schema") or bundle.get("gradio_schema") or {} endpoints: list[Any] = [] for source in (smoke, gate, schema, live): if not isinstance(source, dict): continue for key in ("discovered_api_names", "api_names", "endpoints", "api_endpoints", "named_endpoints"): value = source.get(key) if isinstance(value, list): endpoints.extend(value) endpoint = _first_nonempty(smoke.get("api_name"), smoke.get("endpoint"), gate.get("api_name"), schema.get("selected_api_name")) endpoint_count = len([e for e in endpoints if e]) endpoint_known = bool(endpoint and endpoint != "/generate") or endpoint_count > 0 or bool(smoke.get("status") == "success") return {"endpoint": endpoint or "/generate", "endpoint_count": endpoint_count, "endpoint_known": endpoint_known, "endpoints": endpoints[:20]} def build_space_test_policy(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]: """Return the canonical linked Space Test policy for a Build Run. Space Test is linked-only: parent Build Run and target Space are mandatory, but a known endpoint is optional because the validation worker can discover Gradio endpoints before generating a smoke-test payload. """ summary = bundle.get("summary") or {} state = bundle.get("state") or {} smoke = bundle.get("generation_smoke") or {} runtime = bundle.get("space_runtime") or {} target = _space_target_from_bundle(bundle) target_url = _space_target_url_from_bundle(bundle, target) raw_status = _lower(status_model.get("raw_status") or summary.get("status") or state.get("status")) global_status = _lower(status_model.get("global_status")) verdict = _lower(status_model.get("verdict")) endpoint_info = _known_gradio_endpoint_info(bundle) requires_discovery = bool(target and not endpoint_info.get("endpoint_known")) runtime_stage = _lower(runtime.get("stage") or runtime.get("status")) if isinstance(runtime, dict) else "" health_passed = bool((bundle.get("inference_gate") or {}).get("implementation_signals", {}).get("health_passed") or smoke.get("health_passed")) def policy(mode: str, enabled: bool, reason: str, label: str, message: str, *, on_success: str = "unchanged", on_failure: str = "unchanged", allow_replay: bool = False, allow_recovery: bool = False) -> dict[str, Any]: return { "schema_version": "space_test_policy.v1", "mode": mode, "enabled": bool(enabled), "reason": reason, "label": label, "message": message, "target_space": target, "target_space_url": target_url, "parent_build_run_id": summary.get("run_id") or state.get("run_id") or "", "endpoint": endpoint_info.get("endpoint") or "/generate", "endpoint_known": bool(endpoint_info.get("endpoint_known")), "endpoint_count": endpoint_info.get("endpoint_count") or 0, "requires_endpoint_discovery": bool(requires_discovery and enabled), "can_update_parent_effective_status": on_success != "unchanged", "effective_status_on_success": on_success, "effective_status_on_failure": on_failure, "allow_replay": bool(allow_replay), "allow_recovery": bool(allow_recovery), } if not target: return policy("unavailable", False, "no_target_space", "Space Test unavailable", "No generated Space is available to validate.") if global_status in {"running", "unknown"}: return policy("blocked", False, "build_not_terminal", "Available after final Space state", "Space Test becomes available after the Build Run reaches a final Space state.") if global_status == "cancelled" or raw_status in {"stopped", "cancelled", "canceled"}: return policy("unavailable", False, "run_stopped", "Space Test unavailable", "This run was stopped. Start a new build before validation.") if status_model.get("manual_validation_passed") or _lower(status_model.get("effective_status")) in {"validated_after_manual_space_test", "manual_validation_passed", "validated"}: return policy("replay", True, "manual_validation_already_passed", "Replay linked validation", "This Build Run already has a successful linked Space Test. You can replay the validation.", allow_replay=True) if raw_status in {"completed_with_warnings", "success_with_warnings"}: return policy("replay", True, "completed_with_warnings", "Replay validation", "Build completed with warnings. Space Test can replay or confirm the validated endpoint.", allow_replay=True) if global_status == "succeeded" or verdict in {"passed", "success"}: return policy("replay", True, "automatic_smoke_passed", "Replay automatic smoke", "Automatic smoke test passed. You can replay the validated request.", allow_replay=True) if global_status == "partial" or verdict == "partial_validation" or raw_status in PARTIAL_RAW_STATUSES: return policy("complete", True, "partial_generation_not_verified", "Complete validation", "Automatic smoke did not verify generation. Run linked Space Test to complete validation.", on_success="validated_after_manual_space_test") if global_status == "waiting_manual_action" or status_model.get("requires_manual_action"): reachable = "running" in runtime_stage or health_passed or endpoint_info.get("endpoint_known") if reachable: return policy("complete", True, "manual_hardware_action_possible", "Validate after manual action", "Manual hardware action may be required. If the Space is now running, run linked validation.", on_success="validated_after_manual_space_test") return policy("blocked", False, "manual_hardware_required", "Manual hardware action required", "Complete the hardware action in Space Settings before validation.") if global_status == "failed": return policy("recover", True, "failed_after_space_creation", "Recover validation", "The generated Space previously failed validation. If it is now reachable, run linked Space Test to check recovery.", on_success="recovered_by_manual_validation", allow_recovery=True) if global_status == "stale": return policy("recover", True, "stale_with_target_space", "Recover stale run", "The original run became stale. If the Space is now available, run linked validation to recover the result.", on_success="recovered_by_manual_validation", allow_recovery=True) return policy("blocked", False, "unsupported_build_state", "Space Test unavailable", "This Build Run state is not eligible for linked Space Test yet.") def build_space_test_model(bundle: dict[str, Any], status_model: dict[str, Any]) -> dict[str, Any]: smoke = bundle.get("generation_smoke") or {} summary = bundle.get("summary") or {} policy = build_space_test_policy(bundle, status_model) return { "target_space": policy.get("target_space") or summary.get("target_space") or "", "target_space_url": policy.get("target_space_url") or summary.get("target_space_url") or "", "endpoint": policy.get("endpoint") or smoke.get("api_name") or smoke.get("endpoint") or "/generate", "status": "passed" if smoke.get("ok") or smoke.get("status") == "success" else "pending", "latency_seconds": smoke.get("latency_seconds") or summary.get("latency_seconds"), "expected_output_type": summary.get("expected_output_type") or smoke.get("expected_output_type") or "", "verdict": status_model["verdict"], "output_artifact": smoke.get("output_artifact") or smoke.get("artifact_url") or "", "policy": policy, } def build_run_view_model( run_id: str, bundle: dict[str, Any], *, bucket_source: str, now: datetime | None = None, ) -> dict[str, Any]: summary = bundle.get("summary") or {} state = bundle.get("state") or {} manual_validation = bundle.get("manual_validation_status") or summary.get("manual_validation_status") or {} status_model = normalize_run_status(bundle, now=now) if manual_validation.get("status") == "success": status_model = {**status_model, "manual_validation_passed": True, "effective_status": manual_validation.get("effective_status") or "validated_after_manual_space_test", "effective_verdict": "validated_after_manual_space_test"} phase = derive_product_phase(bundle, status_model) space_test_policy = build_space_test_policy(bundle, status_model) elapsed_seconds = None current_now = now or datetime.now(timezone.utc) created = parse_ts(summary.get("started_at") or state.get("started_at") or summary.get("created_at") or state.get("created_at")) latest = parse_ts(status_model.get("latest_activity_at")) if created: end = latest if status_model.get("is_terminal") and latest else current_now elapsed_seconds = int((end - created).total_seconds()) links = { "job_url": _job_url_from_view_sources(bucket_source=bucket_source, summary=summary, state=state, launch=bundle.get("launch") or {}), "target_space_url": summary.get("target_space_url") or "", "target_space_settings_url": f"{summary.get('target_space_url')}/settings" if summary.get("target_space_url") else "", "artifacts_url": summary.get("artifacts_url") or f"https://huggingface.co/buckets/{bucket_source}/tree/{_runs_prefix()}/{run_id}", } return { "schema_version": "run_view_model.v1", "run_id": run_id, "bucket_source": bucket_source, "header": { "title": _first_nonempty(summary.get("title"), f"Build Space for {summary.get('model_id')}", run_id), "status": status_model["global_status"], "status_label": status_model["global_status"].replace("_", " ").title(), "raw_status": status_model["raw_status"], "space": summary.get("target_space") or "", "space_url": summary.get("target_space_url") or "", "current_phase": phase, "current_phase_label": next((s["label"] for s in PRODUCT_STEPS if s["id"] == phase), phase), "elapsed_seconds": max(0, elapsed_seconds or 0), "started_at": summary.get("created_at") or state.get("created_at") or "", "updated_at": status_model["latest_activity_at"], }, "status_model": status_model, "pipeline": build_pipeline(phase, status_model), "timeline_model": build_run_timeline_model(bundle), "activity": build_activity_feed(bundle), "diagnostics": build_diagnostics(bundle, status_model, phase), "space_test": {**build_space_test_model(bundle, status_model), "linked_validation": manual_validation, "mode": "linked_build_run"}, "space_test_policy": space_test_policy, "manual_validation": manual_validation, "effective_verdict": status_model.get("effective_verdict") or status_model["verdict"], "actions": { "can_resume": status_model["global_status"] in {"running", "stale", "unknown"}, "can_stop": status_model["global_status"] in {"running", "queued", "unknown"}, "can_open_space": bool(summary.get("target_space_url")), "can_validate": bool(space_test_policy.get("enabled")), "requires_manual_action": status_model["requires_manual_action"], }, "links": links, } def is_resumable_summary(summary: dict[str, Any], *, now: datetime | None = None, stale_after: timedelta = timedelta(hours=6)) -> bool: """Best-effort resumability test for lightweight /api/runs summaries.""" status = _lower(summary.get("status")) if status in SUCCESS_RAW_STATUSES or status in PARTIAL_RAW_STATUSES or status in FAILED_RAW_STATUSES or status in BLOCKED_RAW_STATUSES or status in MANUAL_RAW_STATUSES: return False if "success" in status or "failed" in status or "blocker" in status or "manual" in status: return False now = now or datetime.now(timezone.utc) latest = parse_ts(summary.get("updated_at") or summary.get("created_at")) if latest and now - latest > stale_after: return False return status in RUNNING_RAW_STATUSES or status in {"unknown", ""} def find_latest_resumable_run(summaries: list[dict[str, Any]], *, now: datetime | None = None) -> dict[str, Any] | None: candidates = [s for s in summaries if isinstance(s, dict) and is_resumable_summary(s, now=now)] if not candidates: return None return sorted(candidates, key=lambda s: str(s.get("updated_at") or s.get("created_at") or s.get("run_id")), reverse=True)[0]