Upload 6 files
Browse files- src/bucket.py +89 -0
- src/config.py +22 -0
- src/jobs.py +140 -0
- src/runs.py +23 -0
- src/security.py +24 -0
- src/worker_payload.py +587 -0
src/bucket.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from huggingface_hub import HfFileSystem
|
| 8 |
+
|
| 9 |
+
from .config import settings
|
| 10 |
+
from .security import redact
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass(frozen=True)
|
| 14 |
+
class RunPaths:
|
| 15 |
+
run_id: str
|
| 16 |
+
bucket_uri: str = settings.bucket_uri
|
| 17 |
+
|
| 18 |
+
@property
|
| 19 |
+
def root(self) -> str:
|
| 20 |
+
return f"{self.bucket_uri}/runs/{self.run_id}"
|
| 21 |
+
|
| 22 |
+
@property
|
| 23 |
+
def state(self) -> str:
|
| 24 |
+
return f"{self.root}/state.json"
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
def events(self) -> str:
|
| 28 |
+
return f"{self.root}/events.jsonl"
|
| 29 |
+
|
| 30 |
+
@property
|
| 31 |
+
def report(self) -> str:
|
| 32 |
+
return f"{self.root}/report.md"
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _fs(token: str | None = None) -> HfFileSystem:
|
| 36 |
+
return HfFileSystem(token=token)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def read_text(path: str, token: str | None = None) -> str | None:
|
| 40 |
+
fs = _fs(token)
|
| 41 |
+
try:
|
| 42 |
+
with fs.open(path, "r") as f:
|
| 43 |
+
return f.read()
|
| 44 |
+
except FileNotFoundError:
|
| 45 |
+
return None
|
| 46 |
+
except Exception as exc: # noqa: BLE001 - surface readable error in UI
|
| 47 |
+
return f"[Could not read {path}: {exc}]"
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def read_json(path: str, token: str | None = None) -> dict[str, Any] | None:
|
| 51 |
+
content = read_text(path, token=token)
|
| 52 |
+
if not content or content.startswith("[Could not read"):
|
| 53 |
+
return None
|
| 54 |
+
try:
|
| 55 |
+
return json.loads(content)
|
| 56 |
+
except json.JSONDecodeError:
|
| 57 |
+
return {"_error": "Invalid JSON", "raw": redact(content)}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def read_events(run_id: str, token: str | None = None) -> list[dict[str, Any]]:
|
| 61 |
+
paths = RunPaths(run_id)
|
| 62 |
+
content = read_text(paths.events, token=token)
|
| 63 |
+
if not content:
|
| 64 |
+
return []
|
| 65 |
+
events: list[dict[str, Any]] = []
|
| 66 |
+
for line in content.splitlines():
|
| 67 |
+
line = line.strip()
|
| 68 |
+
if not line:
|
| 69 |
+
continue
|
| 70 |
+
try:
|
| 71 |
+
events.append(json.loads(line))
|
| 72 |
+
except json.JSONDecodeError:
|
| 73 |
+
events.append({"step": "parse_events", "status": "warning", "message": redact(line)})
|
| 74 |
+
return events
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def read_run_bundle(run_id: str, token: str | None = None) -> dict[str, Any]:
|
| 78 |
+
paths = RunPaths(run_id)
|
| 79 |
+
return {
|
| 80 |
+
"paths": {
|
| 81 |
+
"root": paths.root,
|
| 82 |
+
"state": paths.state,
|
| 83 |
+
"events": paths.events,
|
| 84 |
+
"report": paths.report,
|
| 85 |
+
},
|
| 86 |
+
"state": read_json(paths.state, token=token),
|
| 87 |
+
"events": read_events(run_id, token=token),
|
| 88 |
+
"report": redact(read_text(paths.report, token=token) or ""),
|
| 89 |
+
}
|
src/config.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass(frozen=True)
|
| 8 |
+
class Settings:
|
| 9 |
+
"""Runtime configuration for the orchestrator Space."""
|
| 10 |
+
|
| 11 |
+
bucket_source: str = os.getenv("SPACE_FACTORY_BUCKET_SOURCE", "fffiloni/space-factory-runs")
|
| 12 |
+
bucket_mount: str = os.getenv("SPACE_FACTORY_BUCKET_MOUNT", "/output")
|
| 13 |
+
job_flavor: str = os.getenv("SPACE_FACTORY_JOB_FLAVOR", "cpu-basic")
|
| 14 |
+
job_timeout: str = os.getenv("SPACE_FACTORY_JOB_TIMEOUT", "15m")
|
| 15 |
+
job_image: str = os.getenv("SPACE_FACTORY_JOB_IMAGE", "python:3.12")
|
| 16 |
+
|
| 17 |
+
@property
|
| 18 |
+
def bucket_uri(self) -> str:
|
| 19 |
+
return f"hf://buckets/{self.bucket_source}"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
settings = Settings()
|
src/jobs.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
from huggingface_hub import Volume, fetch_job_logs, inspect_job, run_job
|
| 7 |
+
|
| 8 |
+
from .config import settings
|
| 9 |
+
from .runs import make_run_id, utc_now_iso, validate_run_id
|
| 10 |
+
from .worker_payload import (
|
| 11 |
+
encoded_create_space_worker_script,
|
| 12 |
+
encoded_worker_script,
|
| 13 |
+
python_decode_and_run_command,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
SPACE_SLUG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}$")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _base_env(*, run_id: str, username: str, worker_script_b64: str) -> dict[str, str]:
|
| 20 |
+
return {
|
| 21 |
+
"RUN_ID": run_id,
|
| 22 |
+
"HF_USERNAME": username or "unknown",
|
| 23 |
+
"BUCKET_SOURCE": settings.bucket_source,
|
| 24 |
+
"OUTPUT_ROOT": settings.bucket_mount,
|
| 25 |
+
"WORKER_SCRIPT_B64": worker_script_b64,
|
| 26 |
+
"LAUNCHED_AT": utc_now_iso(),
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _launch_job(*, token: str, env: dict[str, str]) -> Any:
|
| 31 |
+
return run_job(
|
| 32 |
+
image=settings.job_image,
|
| 33 |
+
command=python_decode_and_run_command(),
|
| 34 |
+
flavor=settings.job_flavor,
|
| 35 |
+
timeout=settings.job_timeout,
|
| 36 |
+
env=env,
|
| 37 |
+
secrets={"HF_TOKEN": token},
|
| 38 |
+
volumes=[Volume(type="bucket", source=settings.bucket_source, mount_path=settings.bucket_mount)],
|
| 39 |
+
token=token,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _job_result(job: Any, *, run_id: str, kind: str, extra: dict[str, Any] | None = None) -> dict[str, Any]:
|
| 44 |
+
payload: dict[str, Any] = {
|
| 45 |
+
"run_id": run_id,
|
| 46 |
+
"kind": kind,
|
| 47 |
+
"job_id": job.id,
|
| 48 |
+
"job_url": getattr(job, "url", None),
|
| 49 |
+
"status": getattr(getattr(job, "status", None), "stage", None),
|
| 50 |
+
"bucket_source": settings.bucket_source,
|
| 51 |
+
"bucket_uri": settings.bucket_uri,
|
| 52 |
+
}
|
| 53 |
+
if extra:
|
| 54 |
+
payload.update(extra)
|
| 55 |
+
return payload
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def launch_hello_job(*, token: str, username: str, run_id: str | None = None) -> dict[str, Any]:
|
| 59 |
+
"""Launch the Phase 1 HF Job that only writes state/events/report to the bucket."""
|
| 60 |
+
if not token:
|
| 61 |
+
raise ValueError("Missing OAuth token. Please sign in with Hugging Face first.")
|
| 62 |
+
|
| 63 |
+
safe_run_id = validate_run_id(run_id) if run_id else make_run_id("hello")
|
| 64 |
+
env = _base_env(run_id=safe_run_id, username=username, worker_script_b64=encoded_worker_script())
|
| 65 |
+
job = _launch_job(token=token, env=env)
|
| 66 |
+
return _job_result(job, run_id=safe_run_id, kind="hello_job")
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def normalize_target_space(*, username: str, target_slug: str | None, run_id: str) -> str:
|
| 70 |
+
"""Return `username/slug`, constrained to the signed-in user's namespace for V2."""
|
| 71 |
+
slug = (target_slug or "").strip()
|
| 72 |
+
if not slug:
|
| 73 |
+
slug = f"space-factory-{run_id}".lower()[:80]
|
| 74 |
+
# If user pasted a full repo id, only allow their own namespace in Phase 2.
|
| 75 |
+
if "/" in slug:
|
| 76 |
+
namespace, repo = slug.split("/", 1)
|
| 77 |
+
if namespace != username:
|
| 78 |
+
raise ValueError("For Phase 2, the target Space must be created in your own namespace.")
|
| 79 |
+
slug = repo
|
| 80 |
+
if not SPACE_SLUG_RE.match(slug):
|
| 81 |
+
raise ValueError("Invalid target Space name. Use letters, numbers, dots, underscores, or dashes.")
|
| 82 |
+
return f"{username}/{slug}"
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def launch_create_private_space_job(
|
| 86 |
+
*,
|
| 87 |
+
token: str,
|
| 88 |
+
username: str,
|
| 89 |
+
target_slug: str | None = None,
|
| 90 |
+
run_id: str | None = None,
|
| 91 |
+
) -> dict[str, Any]:
|
| 92 |
+
"""Launch the Phase 2 Job: create a private target Gradio Space and validate it live."""
|
| 93 |
+
if not token:
|
| 94 |
+
raise ValueError("Missing OAuth token. Please sign in with Hugging Face first.")
|
| 95 |
+
safe_run_id = validate_run_id(run_id) if run_id else make_run_id("space")
|
| 96 |
+
target_space_id = normalize_target_space(username=username, target_slug=target_slug, run_id=safe_run_id)
|
| 97 |
+
|
| 98 |
+
env = _base_env(
|
| 99 |
+
run_id=safe_run_id,
|
| 100 |
+
username=username,
|
| 101 |
+
worker_script_b64=encoded_create_space_worker_script(),
|
| 102 |
+
)
|
| 103 |
+
env["TARGET_SPACE_ID"] = target_space_id
|
| 104 |
+
job = _launch_job(token=token, env=env)
|
| 105 |
+
return _job_result(
|
| 106 |
+
job,
|
| 107 |
+
run_id=safe_run_id,
|
| 108 |
+
kind="create_private_space",
|
| 109 |
+
extra={"target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}"},
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def inspect_job_safe(job_id: str, token: str | None = None) -> dict[str, Any]:
|
| 114 |
+
if not job_id:
|
| 115 |
+
return {"error": "Missing job_id"}
|
| 116 |
+
try:
|
| 117 |
+
info = inspect_job(job_id=job_id, token=token)
|
| 118 |
+
status = getattr(info, "status", None)
|
| 119 |
+
return {
|
| 120 |
+
"id": info.id,
|
| 121 |
+
"url": getattr(info, "url", None),
|
| 122 |
+
"stage": getattr(status, "stage", None),
|
| 123 |
+
"message": getattr(status, "message", None),
|
| 124 |
+
"flavor": getattr(info, "flavor", None),
|
| 125 |
+
"created_at": str(getattr(info, "created_at", "")),
|
| 126 |
+
"started_at": str(getattr(info, "started_at", "")),
|
| 127 |
+
"finished_at": str(getattr(info, "finished_at", "")),
|
| 128 |
+
}
|
| 129 |
+
except Exception as exc: # noqa: BLE001
|
| 130 |
+
return {"error": str(exc)}
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def fetch_recent_logs_safe(job_id: str, token: str | None = None, max_lines: int = 120) -> str:
|
| 134 |
+
if not job_id:
|
| 135 |
+
return ""
|
| 136 |
+
try:
|
| 137 |
+
logs = list(fetch_job_logs(job_id=job_id, token=token))
|
| 138 |
+
return "\n".join(str(line).rstrip("\n") for line in logs[-max_lines:])
|
| 139 |
+
except Exception as exc: # noqa: BLE001
|
| 140 |
+
return f"Could not fetch job logs: {exc}"
|
src/runs.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
import uuid
|
| 5 |
+
from datetime import datetime, timezone
|
| 6 |
+
|
| 7 |
+
RUN_ID_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,80}$")
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def utc_now_iso() -> str:
|
| 11 |
+
return datetime.now(timezone.utc).isoformat()
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def make_run_id(prefix: str = "run") -> str:
|
| 15 |
+
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
| 16 |
+
return f"{prefix}-{stamp}-{uuid.uuid4().hex[:8]}"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def validate_run_id(run_id: str) -> str:
|
| 20 |
+
cleaned = (run_id or "").strip()
|
| 21 |
+
if not RUN_ID_RE.match(cleaned):
|
| 22 |
+
raise ValueError("Invalid run_id. Use 3-80 characters: letters, numbers, dots, underscores or dashes.")
|
| 23 |
+
return cleaned
|
src/security.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
|
| 5 |
+
SECRET_PATTERNS = [
|
| 6 |
+
re.compile(r"hf_[A-Za-z0-9_\-]{20,}"),
|
| 7 |
+
re.compile(r"Bearer\s+[A-Za-z0-9_\.\-]+", re.IGNORECASE),
|
| 8 |
+
re.compile(r"(HF_TOKEN|OAUTH_TOKEN|ACCESS_TOKEN|AUTHORIZATION|PASSWORD|SECRET)\s*[:=]\s*[^\s]+", re.IGNORECASE),
|
| 9 |
+
]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def redact(text: str | None) -> str:
|
| 13 |
+
"""Best-effort redaction for logs/reports shown in the UI.
|
| 14 |
+
|
| 15 |
+
This is intentionally conservative. It is not a complete DLP system,
|
| 16 |
+
but it protects against obvious token leaks in first-version outputs.
|
| 17 |
+
"""
|
| 18 |
+
if not text:
|
| 19 |
+
return ""
|
| 20 |
+
|
| 21 |
+
redacted = text
|
| 22 |
+
for pattern in SECRET_PATTERNS:
|
| 23 |
+
redacted = pattern.sub("[REDACTED]", redacted)
|
| 24 |
+
return redacted
|
src/worker_payload.py
ADDED
|
@@ -0,0 +1,587 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import base64
|
| 4 |
+
import textwrap
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
HELLO_WORKER_SCRIPT = r'''
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
from datetime import datetime, timezone
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def now():
|
| 15 |
+
return datetime.now(timezone.utc).isoformat()
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def write_json(path: Path, payload: dict):
|
| 19 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 20 |
+
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):
|
| 24 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 25 |
+
event = {
|
| 26 |
+
"ts": now(),
|
| 27 |
+
"step": step,
|
| 28 |
+
"status": status,
|
| 29 |
+
"message": message,
|
| 30 |
+
"data": data or {},
|
| 31 |
+
}
|
| 32 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 33 |
+
with path.open("a", encoding="utf-8") as f:
|
| 34 |
+
f.write(line + "\n")
|
| 35 |
+
# Keep HF Job logs useful as well as Bucket events.
|
| 36 |
+
print(line, flush=True)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def main():
|
| 40 |
+
run_id = os.environ["RUN_ID"]
|
| 41 |
+
hf_username = os.environ.get("HF_USERNAME", "unknown")
|
| 42 |
+
bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")
|
| 43 |
+
output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))
|
| 44 |
+
job_id = os.environ.get("JOB_ID")
|
| 45 |
+
accelerator = os.environ.get("ACCELERATOR") or "none"
|
| 46 |
+
cpu_cores = os.environ.get("CPU_CORES")
|
| 47 |
+
memory = os.environ.get("MEMORY")
|
| 48 |
+
has_hf_token = bool(os.environ.get("HF_TOKEN"))
|
| 49 |
+
|
| 50 |
+
run_dir = output_root / "runs" / run_id
|
| 51 |
+
state_path = run_dir / "state.json"
|
| 52 |
+
events_path = run_dir / "events.jsonl"
|
| 53 |
+
report_path = run_dir / "report.md"
|
| 54 |
+
|
| 55 |
+
append_event(events_path, "bootstrap", "started", "HF Job started")
|
| 56 |
+
append_event(
|
| 57 |
+
events_path,
|
| 58 |
+
"environment",
|
| 59 |
+
"success",
|
| 60 |
+
"Collected non-sensitive job environment metadata",
|
| 61 |
+
{
|
| 62 |
+
"job_id": job_id,
|
| 63 |
+
"accelerator": accelerator,
|
| 64 |
+
"cpu_cores": cpu_cores,
|
| 65 |
+
"memory": memory,
|
| 66 |
+
"has_hf_token": has_hf_token,
|
| 67 |
+
},
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
state = {
|
| 71 |
+
"run_id": run_id,
|
| 72 |
+
"status": "success",
|
| 73 |
+
"kind": "hello_job",
|
| 74 |
+
"message": "Hello from HF Job. OAuth → Job → Bucket write succeeded.",
|
| 75 |
+
"created_at": now(),
|
| 76 |
+
"updated_at": now(),
|
| 77 |
+
"created_by": hf_username,
|
| 78 |
+
"bucket_source": bucket_source,
|
| 79 |
+
"job_id": job_id,
|
| 80 |
+
"accelerator": accelerator,
|
| 81 |
+
"cpu_cores": cpu_cores,
|
| 82 |
+
"memory": memory,
|
| 83 |
+
"has_hf_token": has_hf_token,
|
| 84 |
+
"security_notes": [
|
| 85 |
+
"HF_TOKEN was not printed.",
|
| 86 |
+
"This run does not create or publish any repository.",
|
| 87 |
+
"The bucket should remain private.",
|
| 88 |
+
],
|
| 89 |
+
}
|
| 90 |
+
write_json(state_path, state)
|
| 91 |
+
append_event(events_path, "state_write", "success", "Wrote state.json")
|
| 92 |
+
|
| 93 |
+
report = f"""# Agentic Space Factory — Hello Job Report
|
| 94 |
+
|
| 95 |
+
Run ID: `{run_id}`
|
| 96 |
+
|
| 97 |
+
Status: **success**
|
| 98 |
+
|
| 99 |
+
This first worker validated the critical foundation:
|
| 100 |
+
|
| 101 |
+
```text
|
| 102 |
+
OAuth user → HF Job → mounted Storage Bucket → state/events/report write
|
| 103 |
+
```
|
| 104 |
+
|
| 105 |
+
## Non-sensitive job metadata
|
| 106 |
+
|
| 107 |
+
- Job ID: `{job_id}`
|
| 108 |
+
- User: `{hf_username}`
|
| 109 |
+
- Bucket: `{bucket_source}`
|
| 110 |
+
- Accelerator: `{accelerator}`
|
| 111 |
+
- CPU cores: `{cpu_cores}`
|
| 112 |
+
- Memory: `{memory}`
|
| 113 |
+
- HF token present in job env: `{has_hf_token}`
|
| 114 |
+
|
| 115 |
+
## Security posture
|
| 116 |
+
|
| 117 |
+
- The token was passed as a secret and was not printed.
|
| 118 |
+
- This run did not create or modify any Hugging Face repository.
|
| 119 |
+
- This run did not publish anything publicly.
|
| 120 |
+
|
| 121 |
+
## Next implementation step
|
| 122 |
+
|
| 123 |
+
The next increment should create a private target Gradio Space and validate it with `gradio_client` before reporting success.
|
| 124 |
+
"""
|
| 125 |
+
report_path.write_text(report, encoding="utf-8")
|
| 126 |
+
append_event(events_path, "report_write", "success", "Wrote report.md")
|
| 127 |
+
append_event(events_path, "done", "success", "Hello Job completed")
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
if __name__ == "__main__":
|
| 131 |
+
main()
|
| 132 |
+
'''
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
CREATE_SPACE_WORKER_SCRIPT = r'''
|
| 136 |
+
import json
|
| 137 |
+
import os
|
| 138 |
+
import re
|
| 139 |
+
import subprocess
|
| 140 |
+
import sys
|
| 141 |
+
import time
|
| 142 |
+
from datetime import datetime, timezone
|
| 143 |
+
from pathlib import Path
|
| 144 |
+
from textwrap import dedent
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
TARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$")
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def now():
|
| 151 |
+
return datetime.now(timezone.utc).isoformat()
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def write_json(path: Path, payload: dict):
|
| 155 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 156 |
+
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):
|
| 160 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 161 |
+
event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}}
|
| 162 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 163 |
+
with path.open("a", encoding="utf-8") as f:
|
| 164 |
+
f.write(line + "\n")
|
| 165 |
+
# Keep HF Job logs useful as well as Bucket events.
|
| 166 |
+
print(line, flush=True)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):
|
| 170 |
+
state_path = run_dir / "state.json"
|
| 171 |
+
append_event(events_path, "failure", "failed", message, details or {})
|
| 172 |
+
write_json(state_path, {
|
| 173 |
+
"run_id": os.environ.get("RUN_ID"),
|
| 174 |
+
"kind": "create_private_space",
|
| 175 |
+
"status": status,
|
| 176 |
+
"message": message,
|
| 177 |
+
"updated_at": now(),
|
| 178 |
+
"details": details or {},
|
| 179 |
+
})
|
| 180 |
+
report = f"""# Agentic Space Factory — Private Space Creation Report
|
| 181 |
+
|
| 182 |
+
Status: **{status}**
|
| 183 |
+
|
| 184 |
+
{message}
|
| 185 |
+
|
| 186 |
+
```json
|
| 187 |
+
{json.dumps(details or {}, indent=2, ensure_ascii=False)}
|
| 188 |
+
```
|
| 189 |
+
"""
|
| 190 |
+
(run_dir / "report.md").write_text(report, encoding="utf-8")
|
| 191 |
+
raise SystemExit(1)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def pip_install(events_path: Path):
|
| 195 |
+
append_event(events_path, "dependencies", "started", "Installing worker dependencies")
|
| 196 |
+
cmd = [sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0"]
|
| 197 |
+
result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
| 198 |
+
if result.returncode != 0:
|
| 199 |
+
append_event(events_path, "dependencies", "failed", "Dependency installation failed", {"output_tail": result.stdout[-4000:]})
|
| 200 |
+
raise RuntimeError(result.stdout)
|
| 201 |
+
append_event(events_path, "dependencies", "success", "Worker dependencies installed")
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def target_files(target_space_id: str) -> dict[str, str]:
|
| 205 |
+
app_py = dedent(f"""
|
| 206 |
+
import gradio as gr
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def greet(name: str) -> str:
|
| 210 |
+
name = (name or "friend").strip() or "friend"
|
| 211 |
+
return f"Hello {{name}} — this private Space was generated by Agentic Space Factory."
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
demo = gr.Interface(
|
| 215 |
+
fn=greet,
|
| 216 |
+
inputs=gr.Textbox(label="Name", value="Hugging Face"),
|
| 217 |
+
outputs=gr.Textbox(label="Result"),
|
| 218 |
+
title="Generated private Space",
|
| 219 |
+
description="A minimal Gradio Space created by an HF Job, then validated through the live Gradio API.",
|
| 220 |
+
examples=[["Hugging Face"], ["Agentic Space Factory"]],
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
if __name__ == "__main__":
|
| 225 |
+
demo.launch()
|
| 226 |
+
""").strip() + "\n"
|
| 227 |
+
|
| 228 |
+
readme = dedent(f"""
|
| 229 |
+
---
|
| 230 |
+
title: Generated Private Space
|
| 231 |
+
emoji: 🧪
|
| 232 |
+
colorFrom: blue
|
| 233 |
+
colorTo: purple
|
| 234 |
+
sdk: gradio
|
| 235 |
+
app_file: app.py
|
| 236 |
+
python_version: "3.11"
|
| 237 |
+
pinned: false
|
| 238 |
+
---
|
| 239 |
+
|
| 240 |
+
# Generated Private Space
|
| 241 |
+
|
| 242 |
+
This private Space was generated by **Agentic Space Factory**.
|
| 243 |
+
|
| 244 |
+
Target repo: `{target_space_id}`
|
| 245 |
+
|
| 246 |
+
This Phase 2 version intentionally creates only a safe hello-world Gradio app.
|
| 247 |
+
Later phases will add Pi, model-card analysis, ZeroGPU templates, and automatic repair.
|
| 248 |
+
""").strip() + "\n"
|
| 249 |
+
|
| 250 |
+
requirements = "gradio>=5.0.0\n"
|
| 251 |
+
return {"app.py": app_py, "README.md": readme, "requirements.txt": requirements}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def save_generated_files(run_dir: Path, files: dict[str, str]):
|
| 255 |
+
generated_dir = run_dir / "generated"
|
| 256 |
+
generated_dir.mkdir(parents=True, exist_ok=True)
|
| 257 |
+
for filename, content in files.items():
|
| 258 |
+
(generated_dir / filename).write_text(content, encoding="utf-8")
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def create_and_upload_space(api, token: str, target_space_id: str, files: dict[str, str], events_path: Path):
|
| 262 |
+
append_event(events_path, "create_space", "started", f"Creating private target Space {target_space_id}")
|
| 263 |
+
try:
|
| 264 |
+
api.create_repo(
|
| 265 |
+
repo_id=target_space_id,
|
| 266 |
+
repo_type="space",
|
| 267 |
+
space_sdk="gradio",
|
| 268 |
+
private=True,
|
| 269 |
+
exist_ok=False,
|
| 270 |
+
token=token,
|
| 271 |
+
)
|
| 272 |
+
append_event(events_path, "create_space", "success", "Private target Space created", {"target_space": target_space_id})
|
| 273 |
+
except Exception as exc:
|
| 274 |
+
# If it already exists, fail safely instead of overwriting user resources unexpectedly.
|
| 275 |
+
append_event(events_path, "create_space", "failed", "Could not create target Space", {"error": str(exc)})
|
| 276 |
+
raise
|
| 277 |
+
|
| 278 |
+
append_event(events_path, "upload_files", "started", "Uploading generated files to target Space")
|
| 279 |
+
for path_in_repo, content in files.items():
|
| 280 |
+
api.upload_file(
|
| 281 |
+
path_or_fileobj=content.encode("utf-8"),
|
| 282 |
+
path_in_repo=path_in_repo,
|
| 283 |
+
repo_id=target_space_id,
|
| 284 |
+
repo_type="space",
|
| 285 |
+
token=token,
|
| 286 |
+
)
|
| 287 |
+
append_event(events_path, "upload_files", "success", f"Uploaded {path_in_repo}")
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def make_gradio_client(target_space_id: str, token: str):
|
| 291 |
+
"""Create a Gradio Client across gradio_client versions.
|
| 292 |
+
|
| 293 |
+
gradio_client 2.x uses `token=...`; older/newer docs often mention
|
| 294 |
+
`hf_token=...`; some versions expose `api_key` or `headers`. Using
|
| 295 |
+
signature introspection prevents a permanent wait loop on a TypeError.
|
| 296 |
+
"""
|
| 297 |
+
import inspect
|
| 298 |
+
from gradio_client import Client
|
| 299 |
+
|
| 300 |
+
params = inspect.signature(Client).parameters
|
| 301 |
+
if "token" in params:
|
| 302 |
+
return Client(target_space_id, token=token)
|
| 303 |
+
if "hf_token" in params:
|
| 304 |
+
return Client(target_space_id, hf_token=token)
|
| 305 |
+
if "api_key" in params:
|
| 306 |
+
return Client(target_space_id, api_key=token)
|
| 307 |
+
if "headers" in params:
|
| 308 |
+
return Client(target_space_id, headers={"Authorization": f"Bearer {token}"})
|
| 309 |
+
# Last-resort fallback: if the process is logged in via HF_TOKEN/HF CLI,
|
| 310 |
+
# some client versions can pick credentials from the environment/cache.
|
| 311 |
+
return Client(target_space_id)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def get_api_schema(client):
|
| 315 |
+
try:
|
| 316 |
+
return client.view_api(return_format="dict")
|
| 317 |
+
except TypeError:
|
| 318 |
+
return client.view_api()
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def extract_api_names(api_schema) -> list[str]:
|
| 322 |
+
"""Best-effort extraction across gradio_client schema formats.
|
| 323 |
+
|
| 324 |
+
Gradio/Gradio Client versions differ: an Interface can expose `/predict`,
|
| 325 |
+
`/greet`, or another named endpoint. For the generated hello app the live
|
| 326 |
+
Job logs show `/greet`, so validation must discover endpoints instead of
|
| 327 |
+
hardcoding `/predict`.
|
| 328 |
+
"""
|
| 329 |
+
names: list[str] = []
|
| 330 |
+
|
| 331 |
+
def add(value):
|
| 332 |
+
if not value or not isinstance(value, str):
|
| 333 |
+
return
|
| 334 |
+
name = value if value.startswith("/") else f"/{value}"
|
| 335 |
+
if name not in names:
|
| 336 |
+
names.append(name)
|
| 337 |
+
|
| 338 |
+
def walk(obj):
|
| 339 |
+
if isinstance(obj, dict):
|
| 340 |
+
for key, value in obj.items():
|
| 341 |
+
if key in {"api_name", "apiName"}:
|
| 342 |
+
add(value)
|
| 343 |
+
# Some schemas use endpoint paths as keys, for example `/greet`.
|
| 344 |
+
if isinstance(key, str) and key.startswith("/"):
|
| 345 |
+
add(key)
|
| 346 |
+
walk(value)
|
| 347 |
+
elif isinstance(obj, list):
|
| 348 |
+
for item in obj:
|
| 349 |
+
walk(item)
|
| 350 |
+
|
| 351 |
+
walk(api_schema)
|
| 352 |
+
return names
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def predict_with_available_endpoint(client, api_schema, value: str):
|
| 356 |
+
candidates = extract_api_names(api_schema)
|
| 357 |
+
for fallback in ["/greet", "/predict"]:
|
| 358 |
+
if fallback not in candidates:
|
| 359 |
+
candidates.append(fallback)
|
| 360 |
+
|
| 361 |
+
errors = []
|
| 362 |
+
for api_name in candidates:
|
| 363 |
+
try:
|
| 364 |
+
return api_name, client.predict(value, api_name=api_name)
|
| 365 |
+
except Exception as exc:
|
| 366 |
+
errors.append({"api_name": api_name, "error": str(exc)[-500:]})
|
| 367 |
+
|
| 368 |
+
# Last fallback for old/simple gradio_client versions where api_name may be optional.
|
| 369 |
+
try:
|
| 370 |
+
return None, client.predict(value)
|
| 371 |
+
except Exception as exc:
|
| 372 |
+
errors.append({"api_name": None, "error": str(exc)[-500:]})
|
| 373 |
+
raise RuntimeError(f"No candidate Gradio endpoint worked: {json.dumps(errors, ensure_ascii=False)}")
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
def validate_live_api(target_space_id: str, token: str, events_path: Path, tests_dir: Path, timeout_seconds: int = 360):
|
| 377 |
+
tests_dir.mkdir(parents=True, exist_ok=True)
|
| 378 |
+
deadline = time.time() + timeout_seconds
|
| 379 |
+
last_error = None
|
| 380 |
+
attempt = 0
|
| 381 |
+
append_event(events_path, "api_validation", "started", "Waiting for live Gradio API to become available")
|
| 382 |
+
|
| 383 |
+
while time.time() < deadline:
|
| 384 |
+
attempt += 1
|
| 385 |
+
try:
|
| 386 |
+
client = make_gradio_client(target_space_id, token)
|
| 387 |
+
api_schema = get_api_schema(client)
|
| 388 |
+
api_names = extract_api_names(api_schema)
|
| 389 |
+
write_json(tests_dir / "api_schema.json", {"schema": api_schema, "api_names": api_names})
|
| 390 |
+
used_api_name, result = predict_with_available_endpoint(client, api_schema, "Agentic Space Factory")
|
| 391 |
+
result_text = str(result)
|
| 392 |
+
ok = "Agentic Space Factory" in result_text and "Hello" in result_text
|
| 393 |
+
payload = {
|
| 394 |
+
"attempt": attempt,
|
| 395 |
+
"target_space": target_space_id,
|
| 396 |
+
"api_test_passed": ok,
|
| 397 |
+
"api_name": used_api_name,
|
| 398 |
+
"discovered_api_names": api_names,
|
| 399 |
+
"result": result_text,
|
| 400 |
+
"validated_at": now(),
|
| 401 |
+
}
|
| 402 |
+
write_json(tests_dir / "test_result.json", payload)
|
| 403 |
+
if ok:
|
| 404 |
+
append_event(
|
| 405 |
+
events_path,
|
| 406 |
+
"api_validation",
|
| 407 |
+
"success",
|
| 408 |
+
"Live Gradio API test passed",
|
| 409 |
+
{"attempt": attempt, "api_name": used_api_name, "discovered_api_names": api_names},
|
| 410 |
+
)
|
| 411 |
+
return payload
|
| 412 |
+
last_error = f"Unexpected API result from {used_api_name}: {result_text}"
|
| 413 |
+
except Exception as exc:
|
| 414 |
+
last_error = str(exc)
|
| 415 |
+
append_event(events_path, "api_validation", "waiting", "Live API not ready yet", {"attempt": attempt, "error": last_error[-1000:]})
|
| 416 |
+
time.sleep(20)
|
| 417 |
+
|
| 418 |
+
payload = {
|
| 419 |
+
"target_space": target_space_id,
|
| 420 |
+
"api_test_passed": False,
|
| 421 |
+
"error": last_error,
|
| 422 |
+
"validated_at": now(),
|
| 423 |
+
}
|
| 424 |
+
write_json(tests_dir / "test_result.json", payload)
|
| 425 |
+
raise RuntimeError(f"Live API validation did not pass before timeout: {last_error}")
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
def main():
|
| 429 |
+
run_id = os.environ["RUN_ID"]
|
| 430 |
+
hf_username = os.environ.get("HF_USERNAME", "unknown")
|
| 431 |
+
bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")
|
| 432 |
+
output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))
|
| 433 |
+
target_space_id = os.environ["TARGET_SPACE_ID"]
|
| 434 |
+
token = os.environ.get("HF_TOKEN")
|
| 435 |
+
|
| 436 |
+
run_dir = output_root / "runs" / run_id
|
| 437 |
+
events_path = run_dir / "events.jsonl"
|
| 438 |
+
state_path = run_dir / "state.json"
|
| 439 |
+
report_path = run_dir / "report.md"
|
| 440 |
+
target_json_path = run_dir / "target_space.json"
|
| 441 |
+
|
| 442 |
+
append_event(events_path, "bootstrap", "started", "Private Space creation worker started")
|
| 443 |
+
write_json(state_path, {
|
| 444 |
+
"run_id": run_id,
|
| 445 |
+
"kind": "create_private_space",
|
| 446 |
+
"status": "running",
|
| 447 |
+
"message": "Creating private target Space",
|
| 448 |
+
"target_space": target_space_id,
|
| 449 |
+
"created_by": hf_username,
|
| 450 |
+
"bucket_source": bucket_source,
|
| 451 |
+
"created_at": now(),
|
| 452 |
+
"updated_at": now(),
|
| 453 |
+
})
|
| 454 |
+
|
| 455 |
+
if not token:
|
| 456 |
+
fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets")
|
| 457 |
+
if not TARGET_RE.match(target_space_id):
|
| 458 |
+
fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space": target_space_id})
|
| 459 |
+
if not target_space_id.startswith(f"{hf_username}/"):
|
| 460 |
+
fail(run_dir, events_path, "For Phase 2, target Space must be in the signed-in user's namespace", {"target_space": target_space_id, "username": hf_username})
|
| 461 |
+
|
| 462 |
+
try:
|
| 463 |
+
pip_install(events_path)
|
| 464 |
+
from huggingface_hub import HfApi
|
| 465 |
+
|
| 466 |
+
api = HfApi(token=token)
|
| 467 |
+
whoami = api.whoami(token=token)
|
| 468 |
+
append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})
|
| 469 |
+
|
| 470 |
+
files = target_files(target_space_id)
|
| 471 |
+
save_generated_files(run_dir, files)
|
| 472 |
+
append_event(events_path, "generate_files", "success", "Generated minimal Gradio Space files", {"files": list(files)})
|
| 473 |
+
|
| 474 |
+
create_and_upload_space(api, token, target_space_id, files, events_path)
|
| 475 |
+
write_json(target_json_path, {
|
| 476 |
+
"target_space": target_space_id,
|
| 477 |
+
"url": f"https://huggingface.co/spaces/{target_space_id}",
|
| 478 |
+
"private": True,
|
| 479 |
+
"sdk": "gradio",
|
| 480 |
+
"created_by": hf_username,
|
| 481 |
+
})
|
| 482 |
+
|
| 483 |
+
validation = validate_live_api(target_space_id, token, events_path, run_dir / "tests")
|
| 484 |
+
|
| 485 |
+
final_state = {
|
| 486 |
+
"run_id": run_id,
|
| 487 |
+
"kind": "create_private_space",
|
| 488 |
+
"status": "success",
|
| 489 |
+
"message": "Private Gradio Space created and validated through the live API.",
|
| 490 |
+
"target_space": target_space_id,
|
| 491 |
+
"target_space_url": f"https://huggingface.co/spaces/{target_space_id}",
|
| 492 |
+
"created_by": hf_username,
|
| 493 |
+
"bucket_source": bucket_source,
|
| 494 |
+
"validation": validation,
|
| 495 |
+
"updated_at": now(),
|
| 496 |
+
"security_notes": [
|
| 497 |
+
"The target Space was created as private.",
|
| 498 |
+
"The HF token was not printed or written to report files.",
|
| 499 |
+
"Success was declared only after a live Gradio API test passed.",
|
| 500 |
+
],
|
| 501 |
+
}
|
| 502 |
+
write_json(state_path, final_state)
|
| 503 |
+
|
| 504 |
+
report = f"""# Agentic Space Factory — Private Space Creation Report
|
| 505 |
+
|
| 506 |
+
Run ID: `{run_id}`
|
| 507 |
+
|
| 508 |
+
Status: **success**
|
| 509 |
+
|
| 510 |
+
Created private Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_id})
|
| 511 |
+
|
| 512 |
+
## What happened
|
| 513 |
+
|
| 514 |
+
```text
|
| 515 |
+
OAuth user → HF Job → private Space creation → file upload → live Gradio API validation → Bucket report
|
| 516 |
+
```
|
| 517 |
+
|
| 518 |
+
## Generated files
|
| 519 |
+
|
| 520 |
+
- `app.py`
|
| 521 |
+
- `requirements.txt`
|
| 522 |
+
- `README.md`
|
| 523 |
+
|
| 524 |
+
Copies are stored in:
|
| 525 |
+
|
| 526 |
+
```text
|
| 527 |
+
runs/{run_id}/generated/
|
| 528 |
+
```
|
| 529 |
+
|
| 530 |
+
## Live API validation
|
| 531 |
+
|
| 532 |
+
```json
|
| 533 |
+
{json.dumps(validation, indent=2, ensure_ascii=False)}
|
| 534 |
+
```
|
| 535 |
+
|
| 536 |
+
## Security posture
|
| 537 |
+
|
| 538 |
+
- The target Space was created as private.
|
| 539 |
+
- No token was printed or intentionally persisted.
|
| 540 |
+
- Success was declared only after the live Gradio API returned the expected output.
|
| 541 |
+
|
| 542 |
+
## Next step
|
| 543 |
+
|
| 544 |
+
Phase 3 should introduce Pi inside the Job and ask it to modify/repair this simple Space while preserving the live API validation gate.
|
| 545 |
+
"""
|
| 546 |
+
report_path.write_text(report, encoding="utf-8")
|
| 547 |
+
append_event(events_path, "report_write", "success", "Wrote report.md")
|
| 548 |
+
append_event(events_path, "done", "success", "Private Space creation worker completed")
|
| 549 |
+
except Exception as exc:
|
| 550 |
+
fail(run_dir, events_path, "Private Space creation worker failed", {"error": str(exc)})
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
if __name__ == "__main__":
|
| 554 |
+
main()
|
| 555 |
+
'''
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def _encode(script: str) -> str:
|
| 559 |
+
return base64.b64encode(script.encode("utf-8")).decode("ascii")
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
def encoded_worker_script() -> str:
|
| 563 |
+
"""Return the base64-encoded Phase 1 hello worker script."""
|
| 564 |
+
return _encode(HELLO_WORKER_SCRIPT)
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def encoded_create_space_worker_script() -> str:
|
| 568 |
+
"""Return the base64-encoded Phase 2 private Space creation worker script."""
|
| 569 |
+
return _encode(CREATE_SPACE_WORKER_SCRIPT)
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def python_decode_and_run_command() -> list[str]:
|
| 573 |
+
"""Command list for `run_job`.
|
| 574 |
+
|
| 575 |
+
The Job image only needs Python. The script is passed via env as base64 and
|
| 576 |
+
executed from /tmp, which avoids persisting code or exposing secrets.
|
| 577 |
+
"""
|
| 578 |
+
runner = textwrap.dedent(
|
| 579 |
+
"""
|
| 580 |
+
import base64, os, pathlib, subprocess, sys
|
| 581 |
+
script = base64.b64decode(os.environ['WORKER_SCRIPT_B64']).decode('utf-8')
|
| 582 |
+
path = pathlib.Path('/tmp/space_factory_worker.py')
|
| 583 |
+
path.write_text(script, encoding='utf-8')
|
| 584 |
+
raise SystemExit(subprocess.call([sys.executable, str(path)]))
|
| 585 |
+
"""
|
| 586 |
+
).strip()
|
| 587 |
+
return ["python", "-c", runner]
|