from __future__ import annotations import json from datetime import datetime, timezone from dataclasses import dataclass from typing import Any from huggingface_hub import HfFileSystem, bucket_info, create_bucket from .config import bucket_uri_from_source, normalize_bucket_name, settings, user_bucket_source from .security import redact @dataclass(frozen=True) class RunPaths: run_id: str bucket_source: str @property def bucket_uri(self) -> str: return bucket_uri_from_source(self.bucket_source) @property def root(self) -> str: return f"{self.bucket_uri}/runs/{self.run_id}" @property def state(self) -> str: return f"{self.root}/state.json" @property def events(self) -> str: return f"{self.root}/events.jsonl" @property def report(self) -> str: return f"{self.root}/report.md" def _fs(token: str | None = None) -> HfFileSystem: return HfFileSystem(token=token) def check_user_bucket(*, username: str, bucket_name: str | None = None, token: str | None = None) -> dict[str, Any]: """Return bucket readiness for the signed-in user without creating resources.""" bucket_source = user_bucket_source(username=username, bucket_name=bucket_name) bucket_uri = bucket_uri_from_source(bucket_source) try: info = bucket_info(bucket_source, token=token) return { "ok": True, "exists": True, "bucket_source": bucket_source, "bucket_uri": bucket_uri, "name": getattr(info, "name", normalize_bucket_name(bucket_name)), "private": getattr(info, "private", None), } except Exception as exc: # noqa: BLE001 - report readable status in UI error = str(exc) not_found = any(marker in error.lower() for marker in ["404", "not found", "repository not found", "bucket not found"]) return { "ok": False, "exists": False if not_found else None, "bucket_source": bucket_source, "bucket_uri": bucket_uri, "error": error, } def create_user_bucket(*, username: str, bucket_name: str | None = None, token: str | None = None) -> dict[str, Any]: """Create the signed-in user's private run bucket, then return readiness.""" bucket_source = user_bucket_source(username=username, bucket_name=bucket_name) try: url = create_bucket(bucket_source, private=True, exist_ok=True, token=token) except Exception as exc: # noqa: BLE001 return { "ok": False, "exists": None, "bucket_source": bucket_source, "bucket_uri": bucket_uri_from_source(bucket_source), "error": str(exc), } status = check_user_bucket(username=username, bucket_name=bucket_name, token=token) status["created_or_existing"] = True status["create_url"] = str(url) return status def assert_user_bucket_ready(*, username: str, bucket_name: str | None = None, token: str | None = None) -> dict[str, Any]: """Raise a clear error if the user's run bucket cannot be used.""" status = check_user_bucket(username=username, bucket_name=bucket_name, token=token) if not status.get("ok"): source = status.get("bucket_source") or user_bucket_source(username=username, bucket_name=bucket_name) error = status.get("error") or "Bucket does not exist or is not accessible." raise ValueError( f"Run bucket `{source}` is not ready. Click 'Create private run bucket' first, " f"or create it manually in Hugging Face Storage Buckets. Details: {error}" ) return status def read_text(path: str, token: str | None = None) -> str | None: fs = _fs(token) try: with fs.open(path, "r") as f: return f.read() except FileNotFoundError: return None except Exception as exc: # noqa: BLE001 - surface readable error in UI return f"[Could not read {path}: {exc}]" def read_json(path: str, token: str | None = None) -> dict[str, Any] | None: content = read_text(path, token=token) if not content or content.startswith("[Could not read"): return None try: return json.loads(content) except json.JSONDecodeError: return {"_error": "Invalid JSON", "raw": redact(content)} def write_json(path: str, payload: dict[str, Any], token: str | None = None) -> None: """Write a JSON document to a bucket path using HfFileSystem. Used by API routes to persist launch metadata immediately after a Job is created, before the worker has had a chance to write state.json. """ fs = _fs(token) with fs.open(path, "w") as f: f.write(json.dumps(payload, indent=2, ensure_ascii=False)) def append_run_event(run_id: str, *, bucket_source: str, step: str, status: str, message: str, details: dict[str, Any] | None = None, token: str | None = None) -> dict[str, Any]: """Append one JSONL event to a run's events.jsonl in the bucket.""" paths = RunPaths(run_id, bucket_source=bucket_source) event = { "ts": datetime.now(timezone.utc).isoformat(), "step": step, "status": status, "message": message, "details": details or {}, } fs = _fs(token) try: existing = read_text(paths.events, token=token) or "" if existing and not existing.endswith("\n"): existing += "\n" except Exception: existing = "" with fs.open(paths.events, "w") as f: f.write(existing + json.dumps(event, ensure_ascii=False) + "\n") return event def delete_run_folder(run_id: str, *, bucket_source: str, token: str | None = None) -> None: """Delete a run folder from the private bucket.""" paths = RunPaths(run_id, bucket_source=bucket_source) _fs(token).rm(paths.root, recursive=True) def write_launch_metadata(run_id: str, *, bucket_source: str, payload: dict[str, Any], token: str | None = None) -> None: """Persist enough metadata immediately for the Run Explorer. Workers can take time before writing state.json/events.jsonl. The custom UI should still be able to show a just-launched run, open its Job, and resume polling after a page refresh. Store launch.json plus a lightweight summary.json and minimal state.json fallback. The worker may overwrite state.json later with richer information. """ paths = RunPaths(run_id, bucket_source=bucket_source) launch = dict(payload) launch.setdefault("run_id", run_id) launch.setdefault("bucket_source", bucket_source) launch.setdefault("status", "running") write_json(f"{paths.root}/launch.json", launch, token=token) summary = { "run_id": run_id, "bucket_source": bucket_source, "kind": launch.get("kind") or "unknown", "status": launch.get("status") or "running", "model_id": launch.get("model_id") or "", "target_space": launch.get("target_space") or "", "target_space_url": launch.get("target_space_url") or (f"https://huggingface.co/spaces/{launch.get('target_space')}" if launch.get("target_space") else ""), "job_id": launch.get("job_id") or "", "job_url": launch.get("job_url") or "", "created_by": launch.get("created_by") or launch.get("username") or "", "created_at": launch.get("created_at") or "", "updated_at": launch.get("updated_at") or launch.get("created_at") or "", "selected_hardware": launch.get("preferred_space_hardware") or launch.get("selected_hardware") or "", "artifacts_url": f"https://huggingface.co/buckets/{bucket_source}/tree/runs/{run_id}", } write_json(f"{paths.root}/summary.json", summary, token=token) # Minimal state fallback: progress polling can treat it as running until # the worker writes a definitive state.json. write_json( f"{paths.root}/state.json", { "run_id": run_id, "kind": summary["kind"], "status": summary["status"], "model_id": summary["model_id"], "target_space": summary["target_space"], "target_space_url": summary["target_space_url"], "job_id": summary["job_id"], "job_url": summary["job_url"], "created_by": summary["created_by"], "created_at": summary["created_at"], "updated_at": summary["updated_at"], }, token=token, ) def read_events(run_id: str, *, bucket_source: str, token: str | None = None) -> list[dict[str, Any]]: paths = RunPaths(run_id, bucket_source=bucket_source) content = read_text(paths.events, token=token) if not content: return [] events: list[dict[str, Any]] = [] for line in content.splitlines(): line = line.strip() if not line: continue try: events.append(json.loads(line)) except json.JSONDecodeError: events.append({"step": "parse_events", "status": "warning", "message": redact(line)}) return events def _safe_read_json(path: str, token: str | None = None) -> dict[str, Any]: return read_json(path, token=token) or {} def _safe_read_text(path: str, token: str | None = None) -> str: return redact(read_text(path, token=token) or "") def _owner_from_bucket_source(bucket_source: str | None) -> str: if not bucket_source: return "" return str(bucket_source).split("/", 1)[0].strip() def _job_url_from_sources(*, bucket_source: str, summary_file: dict[str, Any], launch: dict[str, Any], state: dict[str, Any]) -> str: """Return a stable HF Job URL from any metadata shape we have. Older run bundles may have job_id in summary.json, created_by in launch.json, or only the bucket owner available. The UI should still expose the Job link whenever a job_id is known. """ explicit = summary_file.get("job_url") or launch.get("job_url") or state.get("job_url") if explicit: return str(explicit) job_id = summary_file.get("job_id") or launch.get("job_id") or state.get("job_id") owner = ( summary_file.get("created_by") or summary_file.get("username") or summary_file.get("owner") or launch.get("created_by") or launch.get("username") or launch.get("owner") or state.get("created_by") or state.get("username") or state.get("owner") or _owner_from_bucket_source(bucket_source) ) if job_id and owner: return f"https://huggingface.co/jobs/{owner}/{job_id}" return "" def _job_url_from_launch_or_state(*, launch: dict[str, Any], state: dict[str, Any]) -> str: # Backward-compatible wrapper for older call sites/tests. return _job_url_from_sources(bucket_source="", summary_file={}, launch=launch, state=state) def _job_id_from_launch_or_state(*, launch: dict[str, Any], state: dict[str, Any]) -> str: return str(launch.get("job_id") or state.get("job_id") or "") def _list_run_files( run_id: str, *, bucket_source: str, token: str | None = None, prefixes: tuple[str, ...] = ("generated", "tests", "logs", "artifacts", "traces", "repair"), max_files: int = 120, ) -> list[dict[str, Any]]: """Return a compact, best-effort file index for a run. The Run Explorer should not fail when a Bucket contains partial or old runs. This function therefore treats every listing error as non-fatal and limits recursion so the UI stays responsive even when traces are large. """ fs = _fs(token) base = f"{bucket_uri_from_source(bucket_source)}/runs/{run_id}" files: list[dict[str, Any]] = [] def walk(prefix: str, depth: int = 0) -> None: if len(files) >= max_files or depth > 3: return path = f"{base}/{prefix}" try: entries = fs.ls(path, detail=True) except Exception: return for entry in entries: if len(files) >= max_files: break name = entry.get("name") if isinstance(entry, dict) else str(entry) if not name: continue typ = str(entry.get("type") or "") if isinstance(entry, dict) else "" size = entry.get("size") if isinstance(entry, dict) else None rel = name.replace(base + "/", "", 1) if typ == "directory": walk(rel, depth + 1) continue files.append( { "path": rel, "name": rel.split("/")[-1], "size": size, "url": f"https://huggingface.co/buckets/{bucket_source}/blob/runs/{run_id}/{rel}", } ) for prefix in prefixes: walk(prefix) return files def _normalize_model_name(value: str | None) -> str: return re.sub(r"[^a-z0-9]+", "", (value or "").lower()) def _extract_pi_models_from_text(text: str) -> list[str]: if not text: return [] patterns = [ r"\bQwen/[A-Za-z0-9_.-]+", r"\bQwen(?:2(?:\.5)?|3)?[-_/A-Za-z0-9.]*Coder[-_/A-Za-z0-9.]*", r"\bKimi[-_/A-Za-z0-9.]+", r"\bMoonshotAI/[A-Za-z0-9_.-]+", r"\bClaude[-_/A-Za-z0-9.]+", r"\bGPT[-_/A-Za-z0-9.]+", r"\bDeepSeek[-_/A-Za-z0-9.]+", r"\b[Mm]odel(?:\\s+used|\\s+selected|\\s*:)?\\s*[:=]?\\s*([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", ] found: list[str] = [] for pattern in patterns: for match in re.finditer(pattern, text, flags=re.IGNORECASE): value = match.group(1) if match.groups() else match.group(0) value = value.strip().strip('",` ') if value and value not in found: found.append(value) return found[:12] def _trace_based_pi_model_resolution(run_id: str, bundle: dict[str, Any], *, bucket_source: str, token: str | None = None) -> dict[str, Any]: state = bundle.get("state") or {} launch = bundle.get("launch") or {} requested = ( state.get("pi_model") or launch.get("pi_model") or launch.get("PI_MODEL") or state.get("pi_model_resolution", {}).get("requested_model") or "" ) configured = state.get("pi_model_resolution", {}).get("configured_model") or requested root = RunPaths(run_id, bucket_source=bucket_source).root trace_texts: list[str] = [] try: fs = _fs(token) for pattern in ( f"{root}/traces/redacted/**/*.jsonl", f"{root}/traces/redacted/*.jsonl", f"{root}/traces/raw/**/*.jsonl", f"{root}/logs/pi_output.txt", f"{root}/logs/pi_live_output.txt", ): for path in fs.glob(pattern): try: trace_texts.append(_safe_read_text(str(path), token=token)[:250000]) except Exception: pass except Exception: pass observed = _extract_pi_models_from_text("\n".join(trace_texts)) n_requested = _normalize_model_name(requested) n_configured = _normalize_model_name(configured) effective = "" for raw in observed: n_raw = _normalize_model_name(raw) if n_raw and n_raw not in {n_requested, n_configured}: effective = raw break if not effective and observed: effective = observed[0] mismatch = bool(effective and _normalize_model_name(effective) not in {n_requested, n_configured}) if not requested and not observed: return {} return { "requested_model": requested, "configured_model": configured or requested, "observed_models": observed, "effective_model": effective or configured or requested, "provider": "huggingface", "mismatch": mismatch, "source": "published_pi_traces", } def summarize_run_bundle(run_id: str, bundle: dict[str, Any], *, bucket_source: str) -> dict[str, Any]: summary_file = bundle.get("summary_file") or {} state = bundle.get("state") or {} launch = bundle.get("launch") or {} gate = bundle.get("inference_gate") or {} smoke = bundle.get("generation_smoke") or {} hardware = bundle.get("hardware_strategy") or {} target_space = state.get("target_space") or launch.get("target_space") or summary_file.get("target_space") or "" status = state.get("status") or launch.get("status") or summary_file.get("status") or gate.get("status") or smoke.get("status") or "unknown" return { "run_id": run_id, "kind": state.get("kind") or launch.get("kind") or summary_file.get("kind") or "unknown", "status": status, "model_id": state.get("model_id") or launch.get("model_id") or summary_file.get("model_id") or state.get("model") or "", "target_space": target_space, "target_space_url": state.get("target_space_url") or launch.get("target_space_url") or summary_file.get("target_space_url") or (f"https://huggingface.co/spaces/{target_space}" if target_space else ""), "job_id": state.get("job_id") or launch.get("job_id") or summary_file.get("job_id") or "", "job_url": _job_url_from_sources(bucket_source=bucket_source, summary_file=summary_file, launch=launch, state=state), "created_at": state.get("created_at") or launch.get("created_at") or summary_file.get("created_at") or "", "updated_at": state.get("updated_at") or state.get("created_at") or launch.get("updated_at") or launch.get("created_at") or summary_file.get("updated_at") or summary_file.get("created_at") or "", "selected_hardware": hardware.get("selected_hardware") or state.get("selected_hardware") or launch.get("preferred_space_hardware") or summary_file.get("selected_hardware") or state.get("hardware") or "", "manual_hardware_required": bool(gate.get("manual_hardware_required") or hardware.get("manual_action_required") or launch.get("manual_hardware_required")), "health_passed": bool((gate.get("implementation_signals") or {}).get("health_passed") or smoke.get("health_passed")), "smoke_test_passed": bool(smoke.get("ok") or smoke.get("status") == "success"), "latency_seconds": smoke.get("latency_seconds"), "expected_output_type": smoke.get("expected_output_type") or state.get("expected_output_type") or launch.get("expected_output_type") or "", "pi_model_resolution": bundle.get("pi_model_resolution") or state.get("pi_model_resolution") or {}, "artifacts_url": f"https://huggingface.co/buckets/{bucket_source}/tree/runs/{run_id}", } def read_run_bundle(run_id: str, *, bucket_source: str, token: str | None = None, include_heavy: bool = True) -> dict[str, Any]: paths = RunPaths(run_id, bucket_source=bucket_source) bundle = { "paths": { "root": paths.root, "state": paths.state, "events": paths.events, "report": paths.report, }, "summary_file": _safe_read_json(f"{paths.root}/summary.json", token=token), "launch": _safe_read_json(f"{paths.root}/launch.json", token=token), "state": read_json(paths.state, token=token) or {}, "events": read_events(run_id, bucket_source=bucket_source, token=token), "report": _safe_read_text(paths.report, token=token) if include_heavy else "", "inference_gate": _safe_read_json(f"{paths.root}/inference_gate.json", token=token), "generation_smoke": _safe_read_json(f"{paths.root}/tests/generation_smoke.json", token=token) or _safe_read_json(f"{paths.root}/generation_smoke.json", token=token), "hardware_strategy": _safe_read_json(f"{paths.root}/hardware_strategy.json", token=token), "hardware_attempts": _safe_read_json(f"{paths.root}/hardware_attempts.json", token=token), "technical_blockers": _safe_read_json(f"{paths.root}/TECHNICAL_BLOCKERS.json", token=token), "model_analysis": _safe_read_json(f"{paths.root}/model_analysis.json", token=token), "pi_model_resolution": _safe_read_json(f"{paths.root}/pi_model_resolution.json", token=token) or (read_json(paths.state, token=token) or {}).get("pi_model_resolution") or {}, "space_runtime": _safe_read_json(f"{paths.root}/space_runtime.json", token=token), "files": _list_run_files(run_id, bucket_source=bucket_source, token=token) if include_heavy else [], } if not bundle.get("pi_model_resolution"): bundle["pi_model_resolution"] = _trace_based_pi_model_resolution(run_id, bundle, bucket_source=bucket_source, token=token) bundle["summary"] = summarize_run_bundle(run_id, bundle, bucket_source=bucket_source) return bundle def _run_id_from_path(path: str) -> str: return path.rstrip('/').split('/')[-1] def _discover_run_ids(root: str, *, token: str | None = None, limit: int = 300) -> list[str]: """Discover run ids in a bucket even when the run folder is partial. HfFileSystem may expose object-store prefixes slightly differently between Buckets/runtime versions. Use ls plus a few targeted globs so running Jobs with only launch.json/summary.json are still shown in the Run Explorer. """ fs = _fs(token) run_ids: set[str] = set() def add_from_path(path: str) -> None: if "/runs/" in path: tail = path.split("/runs/", 1)[1] else: tail = path.replace(root.rstrip("/") + "/", "", 1) run_id = tail.strip("/").split("/", 1)[0] if run_id and run_id != "runs": run_ids.add(run_id) try: for entry in fs.ls(root, detail=True): name = entry.get("name") if isinstance(entry, dict) else str(entry) if name: add_from_path(name) except Exception: pass for pattern in (f"{root}/*", f"{root}/*/summary.json", f"{root}/*/launch.json", f"{root}/*/state.json"): try: for path in fs.glob(pattern): add_from_path(str(path)) if len(run_ids) >= limit: break except Exception: continue return sorted(run_ids, reverse=True)[:limit] def list_recent_runs( *, bucket_source: str, token: str | None = None, limit: int = 50, query: str | None = None, status: str | None = None, ) -> list[dict[str, Any]]: """List recent run summaries from a user's bucket. Best effort: a run can be visible as soon as launch.json/summary.json exists, before the worker writes full state/events. This keeps running Jobs visible and makes Job links available immediately. """ root = f"{bucket_uri_from_source(bucket_source)}/runs" run_ids = _discover_run_ids(root, token=token) if not run_ids: return [] runs: list[dict[str, Any]] = [] for run_id in run_ids: summary_file = read_json(f"{root}/{run_id}/summary.json", token=token) or {} launch = read_json(f"{root}/{run_id}/launch.json", token=token) or {} state = read_json(f"{root}/{run_id}/state.json", token=token) or {} gate = read_json(f"{root}/{run_id}/inference_gate.json", token=token) or {} smoke = read_json(f"{root}/{run_id}/tests/generation_smoke.json", token=token) or read_json(f"{root}/{run_id}/generation_smoke.json", token=token) or {} hardware = read_json(f"{root}/{run_id}/hardware_strategy.json", token=token) or {} pi_model_resolution = read_json(f"{root}/{run_id}/pi_model_resolution.json", token=token) or state.get("pi_model_resolution") or {} partial_bundle = { "summary_file": summary_file, "launch": launch, "state": state, "inference_gate": gate, "generation_smoke": smoke, "hardware_strategy": hardware, "pi_model_resolution": pi_model_resolution, } item = summarize_run_bundle(run_id, partial_bundle, bucket_source=bucket_source) item["bucket_source"] = bucket_source haystack = " ".join(str(item.get(k, "")) for k in ["run_id", "model_id", "target_space", "status", "kind", "job_id"]).lower() if query and query.lower() not in haystack: continue if status and status not in {"all", ""} and item["status"] != status: continue runs.append(item) runs.sort(key=lambda r: str(r.get("updated_at") or r.get("created_at") or r.get("run_id")), reverse=True) return runs[: max(1, min(int(limit or 50), 200))]