Upload 6 files
Browse files- src/jobs.py +83 -3
- src/worker_payload.py +513 -0
src/jobs.py
CHANGED
|
@@ -11,6 +11,8 @@ from .worker_payload import (
|
|
| 11 |
encoded_create_space_worker_script,
|
| 12 |
encoded_pi_gist_worker_script,
|
| 13 |
encoded_pi_model_card_worker_script,
|
|
|
|
|
|
|
| 14 |
encoded_pi_space_worker_script,
|
| 15 |
encoded_worker_script,
|
| 16 |
python_decode_and_run_command,
|
|
@@ -30,12 +32,12 @@ def _base_env(*, run_id: str, username: str, worker_script_b64: str) -> dict[str
|
|
| 30 |
}
|
| 31 |
|
| 32 |
|
| 33 |
-
def _launch_job(*, token: str, env: dict[str, str]) -> Any:
|
| 34 |
return run_job(
|
| 35 |
image=settings.job_image,
|
| 36 |
command=python_decode_and_run_command(),
|
| 37 |
-
flavor=settings.job_flavor,
|
| 38 |
-
timeout=settings.job_timeout,
|
| 39 |
env=env,
|
| 40 |
secrets={"HF_TOKEN": token},
|
| 41 |
volumes=[Volume(type="bucket", source=settings.bucket_source, mount_path=settings.bucket_mount)],
|
|
@@ -222,6 +224,84 @@ def launch_pi_model_card_job(
|
|
| 222 |
)
|
| 223 |
|
| 224 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
def inspect_job_safe(job_id: str, token: str | None = None) -> dict[str, Any]:
|
| 226 |
if not job_id:
|
| 227 |
return {"error": "Missing job_id"}
|
|
|
|
| 11 |
encoded_create_space_worker_script,
|
| 12 |
encoded_pi_gist_worker_script,
|
| 13 |
encoded_pi_model_card_worker_script,
|
| 14 |
+
encoded_runtime_recommender_worker_script,
|
| 15 |
+
encoded_longcat_article_worker_script,
|
| 16 |
encoded_pi_space_worker_script,
|
| 17 |
encoded_worker_script,
|
| 18 |
python_decode_and_run_command,
|
|
|
|
| 32 |
}
|
| 33 |
|
| 34 |
|
| 35 |
+
def _launch_job(*, token: str, env: dict[str, str], flavor: str | None = None, timeout: str | None = None) -> Any:
|
| 36 |
return run_job(
|
| 37 |
image=settings.job_image,
|
| 38 |
command=python_decode_and_run_command(),
|
| 39 |
+
flavor=flavor or settings.job_flavor,
|
| 40 |
+
timeout=timeout or settings.job_timeout,
|
| 41 |
env=env,
|
| 42 |
secrets={"HF_TOKEN": token},
|
| 43 |
volumes=[Volume(type="bucket", source=settings.bucket_source, mount_path=settings.bucket_mount)],
|
|
|
|
| 224 |
)
|
| 225 |
|
| 226 |
|
| 227 |
+
|
| 228 |
+
def launch_runtime_recommender_job(
|
| 229 |
+
*,
|
| 230 |
+
token: str,
|
| 231 |
+
username: str,
|
| 232 |
+
model_id: str | None = None,
|
| 233 |
+
run_id: str | None = None,
|
| 234 |
+
) -> dict[str, Any]:
|
| 235 |
+
"""Launch the Phase 6 Job: analyze model metadata and recommend safe runtime/hardware before building."""
|
| 236 |
+
if not token:
|
| 237 |
+
raise ValueError("Missing OAuth token. Please sign in with Hugging Face first.")
|
| 238 |
+
safe_run_id = validate_run_id(run_id) if run_id else make_run_id("runtime")
|
| 239 |
+
clean_model_id = (model_id or "").strip().replace("https://huggingface.co/", "").strip("/")
|
| 240 |
+
if "/" not in clean_model_id:
|
| 241 |
+
raise ValueError("Model ID must look like owner/model-name.")
|
| 242 |
+
env = _base_env(
|
| 243 |
+
run_id=safe_run_id,
|
| 244 |
+
username=username,
|
| 245 |
+
worker_script_b64=encoded_runtime_recommender_worker_script(),
|
| 246 |
+
)
|
| 247 |
+
env["MODEL_ID"] = clean_model_id
|
| 248 |
+
job = _launch_job(token=token, env=env)
|
| 249 |
+
return _job_result(
|
| 250 |
+
job,
|
| 251 |
+
run_id=safe_run_id,
|
| 252 |
+
kind="runtime_recommender",
|
| 253 |
+
extra={"model_id": clean_model_id},
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def launch_longcat_article_job(
|
| 258 |
+
*,
|
| 259 |
+
token: str,
|
| 260 |
+
username: str,
|
| 261 |
+
target_slug: str | None = None,
|
| 262 |
+
model_id: str | None = None,
|
| 263 |
+
pi_model: str | None = None,
|
| 264 |
+
preferred_space_hardware: str | None = None,
|
| 265 |
+
fallback_space_hardware: str | None = None,
|
| 266 |
+
allow_fixed_gpu_fallback: bool = True,
|
| 267 |
+
run_id: str | None = None,
|
| 268 |
+
) -> dict[str, Any]:
|
| 269 |
+
"""Launch Phase 7: attempt an article-style LongCat Space with ZeroGPU + fixed GPU fallback."""
|
| 270 |
+
if not token:
|
| 271 |
+
raise ValueError("Missing OAuth token. Please sign in with Hugging Face first.")
|
| 272 |
+
safe_run_id = validate_run_id(run_id) if run_id else make_run_id("longcat")
|
| 273 |
+
target_space_id = normalize_target_space(username=username, target_slug=target_slug, run_id=safe_run_id)
|
| 274 |
+
clean_model_id = (model_id or "meituan-longcat/LongCat-Video-Avatar-1.5").strip().replace("https://huggingface.co/", "").strip("/")
|
| 275 |
+
if "/" not in clean_model_id:
|
| 276 |
+
raise ValueError("Model ID must look like owner/model-name.")
|
| 277 |
+
|
| 278 |
+
env = _base_env(
|
| 279 |
+
run_id=safe_run_id,
|
| 280 |
+
username=username,
|
| 281 |
+
worker_script_b64=encoded_longcat_article_worker_script(),
|
| 282 |
+
)
|
| 283 |
+
env["TARGET_SPACE_ID"] = target_space_id
|
| 284 |
+
env["MODEL_ID"] = clean_model_id
|
| 285 |
+
env["PI_MODEL"] = (pi_model or "moonshotai/Kimi-K2.5").strip()
|
| 286 |
+
env["PREFERRED_SPACE_HARDWARE"] = (preferred_space_hardware or "zero-a10g").strip()
|
| 287 |
+
env["FALLBACK_SPACE_HARDWARE"] = (fallback_space_hardware or "l40sx1").strip()
|
| 288 |
+
env["ALLOW_FIXED_GPU_FALLBACK"] = "true" if allow_fixed_gpu_fallback else "false"
|
| 289 |
+
job = _launch_job(token=token, env=env, timeout="60m")
|
| 290 |
+
return _job_result(
|
| 291 |
+
job,
|
| 292 |
+
run_id=safe_run_id,
|
| 293 |
+
kind="longcat_article_reproduction",
|
| 294 |
+
extra={
|
| 295 |
+
"target_space": target_space_id,
|
| 296 |
+
"target_space_url": f"https://huggingface.co/spaces/{target_space_id}",
|
| 297 |
+
"model_id": clean_model_id,
|
| 298 |
+
"pi_model": env["PI_MODEL"],
|
| 299 |
+
"preferred_space_hardware": env["PREFERRED_SPACE_HARDWARE"],
|
| 300 |
+
"fallback_space_hardware": env["FALLBACK_SPACE_HARDWARE"],
|
| 301 |
+
"allow_fixed_gpu_fallback": allow_fixed_gpu_fallback,
|
| 302 |
+
},
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
def inspect_job_safe(job_id: str, token: str | None = None) -> dict[str, Any]:
|
| 306 |
if not job_id:
|
| 307 |
return {"error": "Missing job_id"}
|
src/worker_payload.py
CHANGED
|
@@ -568,6 +568,509 @@ def _encode(script: str) -> str:
|
|
| 568 |
|
| 569 |
PI_MODEL_CARD_WORKER_SCRIPT = 'import json\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\nimport time\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom textwrap import dedent\n\n\nTARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$")\nSUPPORTED_TASKS = {"text-generation", "text2text-generation", "fill-mask", "text-classification", "sentiment-analysis"}\n\n\ndef now():\n return datetime.now(timezone.utc).isoformat()\n\n\ndef write_json(path: Path, payload: dict):\n path.parent.mkdir(parents=True, exist_ok=True)\n path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\\n", encoding="utf-8")\n\n\ndef append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):\n path.parent.mkdir(parents=True, exist_ok=True)\n event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}}\n line = json.dumps(event, ensure_ascii=False)\n with path.open("a", encoding="utf-8") as f:\n f.write(line + "\\n")\n print(line, flush=True)\n\n\ndef redact_text(text: str | None) -> str:\n if not text:\n return ""\n value = text\n for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]:\n secret = os.environ.get(secret_name)\n if secret:\n value = value.replace(secret, "[REDACTED]")\n value = re.sub(r"Bearer\\s+[A-Za-z0-9_\\-.=]+", "Bearer [REDACTED]", value)\n value = re.sub(r"hf_[A-Za-z0-9_\\-]{10,}", "hf_[REDACTED]", value)\n return value\n\n\ndef safe_details(details: dict | None) -> dict:\n if not details:\n return {}\n try:\n return json.loads(redact_text(json.dumps(details, ensure_ascii=False)))\n except Exception:\n return {"redacted_details": redact_text(str(details))[-4000:]}\n\n\ndef fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):\n safe = safe_details(details)\n append_event(events_path, "failure", "failed", message, safe)\n write_json(run_dir / "state.json", {\n "run_id": os.environ.get("RUN_ID"),\n "kind": "pi_model_card",\n "status": status,\n "message": message,\n "updated_at": now(),\n "details": safe,\n })\n report = f"""# Agentic Space Factory — Model Card Space Report\n\nStatus: **{status}**\n\n{message}\n\n```json\n{json.dumps(safe, indent=2, ensure_ascii=False)}\n```\n"""\n (run_dir / "report.md").write_text(report, encoding="utf-8")\n raise SystemExit(1)\n\n\ndef run_cmd(cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600):\n result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\n return result.returncode, redact_text(result.stdout)\n\n\ndef install_python_deps(events_path: Path):\n append_event(events_path, "dependencies", "started", "Installing Python worker dependencies")\n code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0"], timeout=600)\n if code != 0:\n append_event(events_path, "dependencies", "failed", "Python dependency installation failed", {"output_tail": out[-4000:]})\n raise RuntimeError(out)\n append_event(events_path, "dependencies", "success", "Python worker dependencies installed")\n\n\ndef ensure_node(events_path: Path):\n node = shutil.which("node")\n npm = shutil.which("npm")\n if node and npm:\n _, node_v = run_cmd([node, "--version"], timeout=30)\n _, npm_v = run_cmd([npm, "--version"], timeout=30)\n append_event(events_path, "node", "success", "Node/npm already available", {"node": node_v.strip(), "npm": npm_v.strip()})\n return\n append_event(events_path, "node", "started", "Installing nodejs/npm through apt-get")\n code, out = run_cmd(["bash", "-lc", "apt-get update -qq && apt-get install -y -qq nodejs npm"], timeout=600)\n if code != 0:\n append_event(events_path, "node", "failed", "Could not install nodejs/npm", {"output_tail": out[-4000:]})\n raise RuntimeError(out)\n append_event(events_path, "node", "success", "Installed nodejs/npm")\n\n\ndef install_pi(events_path: Path):\n ensure_node(events_path)\n append_event(events_path, "pi_install", "started", "Installing Pi coding agent from npm")\n code, out = run_cmd(["npm", "install", "-g", "@mariozechner/pi-coding-agent"], timeout=900)\n if code != 0:\n append_event(events_path, "pi_install", "failed", "Pi npm installation failed", {"output_tail": out[-4000:]})\n raise RuntimeError(out)\n code, version = run_cmd(["pi", "--version"], timeout=60)\n append_event(events_path, "pi_install", "success", "Pi installed", {"version_output": version.strip()[-300:]})\n\n\ndef configure_pi(events_path: Path, model: str):\n pi_dir = Path.home() / ".pi" / "agent"\n pi_dir.mkdir(parents=True, exist_ok=True)\n (pi_dir / "auth.json").write_text(json.dumps({"huggingface": {"type": "api_key", "key": os.environ.get("HF_TOKEN", "")}}, indent=2), encoding="utf-8")\n (pi_dir / "settings.json").write_text(json.dumps({"model": model, "provider": "huggingface", "autoRun": True, "autoApply": True}, indent=2), encoding="utf-8")\n append_event(events_path, "pi_config", "success", "Configured Pi", {"model": model})\n\n\ndef sanitize_model_id(model_id: str) -> str:\n model_id = (model_id or "").strip().replace("https://huggingface.co/", "")\n model_id = model_id.split("?", 1)[0].strip("/")\n if not re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", model_id):\n raise ValueError("MODEL_ID must look like owner/model-name")\n return model_id\n\n\ndef analyze_model(model_id: str, token: str, run_dir: Path, events_path: Path) -> dict:\n from huggingface_hub import HfApi, hf_hub_download\n append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})\n api = HfApi(token=token)\n info = api.model_info(model_id, token=token, files_metadata=False)\n siblings = [getattr(s, "rfilename", "") for s in (info.siblings or [])]\n pipeline_tag = getattr(info, "pipeline_tag", None)\n library_name = getattr(info, "library_name", None)\n tags = list(getattr(info, "tags", []) or [])\n readme_excerpt = ""\n try:\n readme_path = hf_hub_download(repo_id=model_id, filename="README.md", token=token)\n readme_text = Path(readme_path).read_text(encoding="utf-8", errors="ignore")\n readme_excerpt = readme_text[:6000]\n except Exception as exc:\n readme_excerpt = f"Could not download README.md: {exc}"\n task = pipeline_tag or "text-generation"\n if task == "sentiment-analysis":\n task = "text-classification"\n supported = task in SUPPORTED_TASKS\n analysis = {\n "model_id": model_id,\n "pipeline_tag": pipeline_tag,\n "library_name": library_name,\n "tags": tags[:80],\n "siblings": siblings[:120],\n "selected_task": task,\n "template": "transformers_text_pipeline" if supported else "unsupported",\n "supported": supported,\n "confidence": 0.8 if supported else 0.25,\n "risks": [],\n "readme_excerpt": readme_excerpt,\n "evidence": [f"pipeline_tag={pipeline_tag}", f"library_name={library_name}", f"files={\', \'.join(siblings[:12])}"],\n }\n if not supported:\n analysis["risks"].append("Phase 5 only supports simple Transformers text pipeline tasks.")\n if "gated" in tags:\n analysis["risks"].append("Model appears gated; generated Space will not receive OAuth token as a secret in Phase 5.")\n write_json(run_dir / "model_analysis.json", analysis)\n append_event(events_path, "model_analysis", "success" if supported else "unsupported", "Model metadata analyzed", {"selected_task": task, "supported": supported, "confidence": analysis["confidence"]})\n return analysis\n\n\ndef render_app(model_id: str, task: str) -> str:\n return dedent(f\'\'\'\n import gradio as gr\n from transformers import pipeline\n\n MODEL_ID = {model_id!r}\n TASK = {task!r}\n\n pipe = pipeline(TASK, model=MODEL_ID)\n\n def run_model(text: str) -> str:\n text = (text or "Hello from Agentic Space Factory").strip() or "Hello from Agentic Space Factory"\n if TASK == "text-generation":\n result = pipe(text, max_new_tokens=32, do_sample=False)\n return result[0].get("generated_text", str(result))\n if TASK == "text2text-generation":\n result = pipe(text, max_new_tokens=64)\n return result[0].get("generated_text", str(result))\n if TASK == "fill-mask":\n mask = getattr(getattr(pipe, "tokenizer", None), "mask_token", None) or "<mask>"\n if mask not in text:\n text = f"Hugging Face is {{mask}}."\n result = pipe(text)\n return str(result[:3] if isinstance(result, list) else result)\n if TASK in {"text-classification", "sentiment-analysis"}:\n return str(pipe(text))\n return str(pipe(text))\n\n demo = gr.Interface(\n fn=run_model,\n inputs=gr.Textbox(label="Input", value="Hello from Agentic Space Factory"),\n outputs=gr.Textbox(label="Model output"),\n title=f"Model demo: {{MODEL_ID}}",\n description="Generated by Agentic Space Factory from model metadata. Pi adapted this model app.",\n examples=[["Hello from Agentic Space Factory"], ["Hugging Face is awesome"]],\n )\n\n if __name__ == "__main__":\n demo.launch()\n \'\'\').strip() + "\\n"\n\n\ndef render_readme(model_id: str, task: str, target_space_id: str) -> str:\n return dedent(f\'\'\'\n ---\n title: Model Card Generated Space\n emoji: 🤖\n colorFrom: green\n colorTo: blue\n sdk: gradio\n app_file: app.py\n python_version: "3.11"\n pinned: false\n ---\n\n # Model Card Generated Space\n\n This private Space was generated by Agentic Space Factory from `{model_id}`.\n\n - Target Space: `{target_space_id}`\n - Selected task: `{task}`\n - Template: `transformers_text_pipeline`\n\n Phase 5 is intentionally limited to simple Transformers text pipelines.\n \'\'\').strip() + "\\n"\n\n\ndef prepare_workspace(workspace: Path, run_dir: Path, model_id: str, task: str, target_space_id: str, analysis: dict, events_path: Path):\n workspace.mkdir(parents=True, exist_ok=True)\n (workspace / "app.py").write_text(render_app(model_id, task), encoding="utf-8")\n (workspace / "README.md").write_text(render_readme(model_id, task, target_space_id), encoding="utf-8")\n (workspace / "requirements.txt").write_text("gradio>=5.0.0\\nhuggingface_hub>=0.34.0,<1.0.0\\ntransformers>=4.45.0\\ntorch\\nsafetensors\\n", encoding="utf-8")\n goal = f"""You are running inside a Hugging Face Job as a coding agent.\n\nGoal: adapt the provided minimal Gradio app for the model `{model_id}` and task `{task}`.\n\nFirst, read the HF Spaces Agent Quickstart gist:\nhttps://gist.github.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d\n\nRules for this Phase 5 smoke test:\n- Work only in the current workspace.\n- Do not create, delete, publish, or modify Hugging Face repos. The wrapper will create/upload the private Space.\n- Preserve `app.py`, `README.md`, and `requirements.txt`.\n- Do not remove the `huggingface_hub>=0.34.0,<1.0.0` compatibility pin from requirements.txt.\n- Preserve the `run_model` function and a Gradio Interface or Blocks app.\n- Preserve the exact marker phrase: Pi adapted this model app.\n- Keep the app simple and CPU-friendly.\n- Do not print secrets.\n- Write a short summary to `PI_SUMMARY.md`.\n\nModel analysis:\n```json\n{json.dumps({k: v for k, v in analysis.items() if k != \'readme_excerpt\'}, indent=2, ensure_ascii=False)}\n```\n\nREADME excerpt:\n{analysis.get(\'readme_excerpt\', \'\')[:3000]}\n"""\n (workspace / "GOAL.md").write_text(goal, encoding="utf-8")\n save_generated_files(run_dir, workspace)\n append_event(events_path, "workspace", "success", "Prepared model app workspace", {"files": ["app.py", "README.md", "requirements.txt", "GOAL.md"]})\n\n\ndef save_generated_files(run_dir: Path, workspace: Path):\n generated_dir = run_dir / "generated"\n generated_dir.mkdir(parents=True, exist_ok=True)\n for filename in ["app.py", "README.md", "requirements.txt", "GOAL.md", "PI_SUMMARY.md"]:\n path = workspace / filename\n if path.exists():\n (generated_dir / filename).write_text(path.read_text(encoding="utf-8", errors="ignore"), encoding="utf-8")\n\n\ndef run_pi(workspace: Path, run_dir: Path, events_path: Path, model: str):\n append_event(events_path, "pi_run", "started", "Running Pi on model-card workspace", {"model": model})\n env = os.environ.copy()\n env["HF_TOKEN"] = os.environ.get("HF_TOKEN", "")\n code, out = run_cmd(["pi", "-p", (workspace / "GOAL.md").read_text(encoding="utf-8")], cwd=workspace, env=env, timeout=1800)\n logs_dir = run_dir / "logs"\n logs_dir.mkdir(parents=True, exist_ok=True)\n (logs_dir / "pi_output.txt").write_text(out, encoding="utf-8")\n save_generated_files(run_dir, workspace)\n if code != 0:\n append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": out[-4000:]})\n raise RuntimeError("Pi failed. See logs/pi_output.txt")\n app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")\n if "Pi adapted this model app" not in app_text:\n raise RuntimeError("Pi/app verification failed: expected marker phrase missing from app.py")\n append_event(events_path, "pi_run", "success", "Pi completed and preserved required marker")\n\n\ndef collect_pi_traces(run_dir: Path, events_path: Path):\n src = Path.home() / ".pi" / "agent" / "sessions"\n raw_dir = run_dir / "traces" / "raw"\n redacted_dir = run_dir / "traces" / "redacted"\n raw_dir.mkdir(parents=True, exist_ok=True)\n redacted_dir.mkdir(parents=True, exist_ok=True)\n count = 0\n if src.exists():\n for path in src.rglob("*.jsonl"):\n count += 1\n text = path.read_text(encoding="utf-8", errors="ignore")\n (raw_dir / path.name).write_text(redact_text(text), encoding="utf-8")\n (redacted_dir / path.name).write_text(redact_text(text), encoding="utf-8")\n append_event(events_path, "traces", "success", "Collected Pi traces", {"count": count})\n\n\ndef make_gradio_client(target_space_id: str, token: str):\n import inspect\n from gradio_client import Client\n params = inspect.signature(Client).parameters\n if "token" in params:\n return Client(target_space_id, token=token)\n if "hf_token" in params:\n return Client(target_space_id, hf_token=token)\n if "api_key" in params:\n return Client(target_space_id, api_key=token)\n if "headers" in params:\n return Client(target_space_id, headers={"Authorization": f"Bearer {token}"})\n return Client(target_space_id)\n\n\ndef get_api_schema(client):\n try:\n return client.view_api(return_format="dict")\n except TypeError:\n return client.view_api()\n\n\ndef extract_api_names(api_schema) -> list[str]:\n names = []\n def add(value):\n if not value or not isinstance(value, str):\n return\n name = value if value.startswith("/") else f"/{value}"\n if name not in names:\n names.append(name)\n def walk(obj):\n if isinstance(obj, dict):\n for key, value in obj.items():\n if key in {"api_name", "apiName"}:\n add(value)\n if isinstance(key, str) and key.startswith("/"):\n add(key)\n walk(value)\n elif isinstance(obj, list):\n for item in obj:\n walk(item)\n walk(api_schema)\n return names\n\n\ndef predict_with_available_endpoint(client, api_schema, value: str):\n candidates = extract_api_names(api_schema)\n for fallback in ["/run_model", "/predict", "/greet"]:\n if fallback not in candidates:\n candidates.append(fallback)\n errors = []\n for api_name in candidates:\n try:\n return api_name, client.predict(value, api_name=api_name)\n except Exception as exc:\n errors.append({"api_name": api_name, "error": str(exc)[-500:]})\n try:\n return None, client.predict(value)\n except Exception as exc:\n errors.append({"api_name": None, "error": str(exc)[-500:]})\n raise RuntimeError(f"No candidate Gradio endpoint worked: {json.dumps(errors, ensure_ascii=False)}")\n\n\ndef validate_live_api(target_space_id: str, token: str, events_path: Path, tests_dir: Path, timeout_seconds: int = 900):\n tests_dir.mkdir(parents=True, exist_ok=True)\n deadline = time.time() + timeout_seconds\n last_error = None\n attempt = 0\n append_event(events_path, "api_validation", "started", "Waiting for live model Gradio API to become available")\n while time.time() < deadline:\n attempt += 1\n try:\n client = make_gradio_client(target_space_id, token)\n api_schema = get_api_schema(client)\n api_names = extract_api_names(api_schema)\n write_json(tests_dir / "api_schema.json", {"schema": api_schema, "api_names": api_names})\n used_api_name, result = predict_with_available_endpoint(client, api_schema, "Hello from Agentic Space Factory")\n result_text = str(result)\n ok = bool(result_text and len(result_text.strip()) >= 2)\n payload = {"attempt": attempt, "target_space": target_space_id, "api_test_passed": ok, "api_name": used_api_name, "discovered_api_names": api_names, "result": result_text[:4000], "validated_at": now()}\n write_json(tests_dir / "test_result.json", payload)\n if ok:\n append_event(events_path, "api_validation", "success", "Live model API test passed", {"attempt": attempt, "api_name": used_api_name, "discovered_api_names": api_names})\n return payload\n last_error = f"Unexpected empty API result from {used_api_name}: {result_text}"\n except Exception as exc:\n last_error = str(exc)\n append_event(events_path, "api_validation", "waiting", "Live API not ready yet", {"attempt": attempt, "error": last_error[-1000:]})\n time.sleep(20)\n payload = {"target_space": target_space_id, "api_test_passed": False, "error": last_error, "validated_at": now()}\n write_json(tests_dir / "test_result.json", payload)\n raise RuntimeError(f"Live API validation did not pass before timeout: {last_error}")\n\n\ndef create_and_upload_space(api, token: str, target_space_id: str, workspace: Path, events_path: Path):\n append_event(events_path, "create_space", "started", f"Creating private target Space {target_space_id}")\n api.create_repo(repo_id=target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, token=token)\n append_event(events_path, "create_space", "success", "Private target Space created", {"target_space": target_space_id})\n append_event(events_path, "upload_files", "started", "Uploading model app files to target Space")\n for path_in_repo in ["app.py", "README.md", "requirements.txt"]:\n api.upload_file(path_or_fileobj=(workspace / path_in_repo).read_bytes(), path_in_repo=path_in_repo, repo_id=target_space_id, repo_type="space", token=token)\n append_event(events_path, "upload_files", "success", f"Uploaded {path_in_repo}")\n\n\ndef main():\n run_id = os.environ["RUN_ID"]\n hf_username = os.environ.get("HF_USERNAME", "unknown")\n bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")\n output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))\n target_space_id = os.environ["TARGET_SPACE_ID"]\n model_id = sanitize_model_id(os.environ.get("MODEL_ID", ""))\n pi_model = os.environ.get("PI_MODEL") or "moonshotai/Kimi-K2.5"\n token = os.environ.get("HF_TOKEN")\n run_dir = output_root / "runs" / run_id\n events_path = run_dir / "events.jsonl"\n state_path = run_dir / "state.json"\n workspace = Path("/tmp") / f"space-factory-model-{run_id}"\n append_event(events_path, "bootstrap", "started", "Pi model-card worker started", {"model_id": model_id})\n write_json(state_path, {"run_id": run_id, "kind": "pi_model_card", "status": "running", "message": "Analyzing model card and generating a private model demo Space", "model_id": model_id, "target_space": target_space_id, "created_by": hf_username, "bucket_source": bucket_source, "created_at": now(), "updated_at": now()})\n if not token:\n fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets")\n if not TARGET_RE.match(target_space_id):\n fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space": target_space_id})\n if not target_space_id.startswith(f"{hf_username}/"):\n fail(run_dir, events_path, "Target Space must be in the signed-in user\'s namespace", {"target_space": target_space_id, "username": hf_username})\n try:\n install_python_deps(events_path)\n from huggingface_hub import HfApi\n api = HfApi(token=token)\n whoami = api.whoami(token=token)\n append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})\n analysis = analyze_model(model_id, token, run_dir, events_path)\n if not analysis.get("supported"):\n fail(run_dir, events_path, "Model task is unsupported by Phase 5", {"model_analysis": {k: v for k, v in analysis.items() if k != "readme_excerpt"}}, status="unsupported")\n prepare_workspace(workspace, run_dir, model_id, analysis["selected_task"], target_space_id, analysis, events_path)\n install_pi(events_path)\n configure_pi(events_path, pi_model)\n run_pi(workspace, run_dir, events_path, pi_model)\n collect_pi_traces(run_dir, events_path)\n create_and_upload_space(api, token, target_space_id, workspace, events_path)\n write_json(run_dir / "target_space.json", {"target_space": target_space_id, "url": f"https://huggingface.co/spaces/{target_space_id}", "private": True, "sdk": "gradio", "created_by": hf_username, "model_id": model_id})\n validation = validate_live_api(target_space_id, token, events_path, run_dir / "tests")\n final_state = {"run_id": run_id, "kind": "pi_model_card", "status": "success", "message": "Model-card generated private Space created and validated through the live API.", "model_id": model_id, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "created_by": hf_username, "bucket_source": bucket_source, "model_analysis": {k: v for k, v in analysis.items() if k != "readme_excerpt"}, "validation": validation, "updated_at": now(), "security_notes": ["The target Space was created as private.", "The HF token was not printed or intentionally persisted.", "Phase 5 supports only simple public text pipeline models.", "Success was declared only after the wrapper live API test passed."]}\n write_json(state_path, final_state)\n report = f"""# Agentic Space Factory — Model Card Space Report\n\nRun ID: `{run_id}`\n\nStatus: **success**\n\nCreated private model demo Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_id})\n\n## Model\n\n- Model ID: `{model_id}`\n- Selected task: `{analysis[\'selected_task\']}`\n- Template: `{analysis[\'template\']}`\n- Pi model: `{pi_model}`\n\n## What happened\n\n```text\nOAuth user → HF Job → model metadata analysis → Pi adapts app.py → private Space creation → live API validation → Bucket report\n```\n\n## Live API validation\n\n```json\n{json.dumps(validation, indent=2, ensure_ascii=False)}\n```\n\n## Security posture\n\n- The target Space was created as private.\n- No token was printed or intentionally persisted.\n- Pi was instructed not to create/delete/publish repos; the wrapper performed Hub operations.\n- Success was declared only after the live API returned a non-empty result.\n\n## Next step\n\nPhase 6 should add a ZeroGPU Diffusers template and stricter model compatibility gating.\n"""\n (run_dir / "report.md").write_text(report, encoding="utf-8")\n append_event(events_path, "report_write", "success", "Wrote report.md")\n append_event(events_path, "done", "success", "Pi model-card worker completed")\n except SystemExit:\n raise\n except Exception as exc:\n collect_pi_traces(run_dir, events_path)\n fail(run_dir, events_path, "Pi model-card worker failed", {"error": str(exc)})\n\n\nif __name__ == "__main__":\n main()\n'
|
| 570 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
def encoded_worker_script() -> str:
|
| 572 |
"""Return the base64-encoded Phase 1 hello worker script."""
|
| 573 |
return _encode(HELLO_WORKER_SCRIPT)
|
|
@@ -593,6 +1096,16 @@ def encoded_pi_model_card_worker_script() -> str:
|
|
| 593 |
return _encode(PI_MODEL_CARD_WORKER_SCRIPT)
|
| 594 |
|
| 595 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
def python_decode_and_run_command() -> list[str]:
|
| 597 |
"""Command list for `run_job`.
|
| 598 |
|
|
|
|
| 568 |
|
| 569 |
PI_MODEL_CARD_WORKER_SCRIPT = 'import json\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\nimport time\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom textwrap import dedent\n\n\nTARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$")\nSUPPORTED_TASKS = {"text-generation", "text2text-generation", "fill-mask", "text-classification", "sentiment-analysis"}\n\n\ndef now():\n return datetime.now(timezone.utc).isoformat()\n\n\ndef write_json(path: Path, payload: dict):\n path.parent.mkdir(parents=True, exist_ok=True)\n path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\\n", encoding="utf-8")\n\n\ndef append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):\n path.parent.mkdir(parents=True, exist_ok=True)\n event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}}\n line = json.dumps(event, ensure_ascii=False)\n with path.open("a", encoding="utf-8") as f:\n f.write(line + "\\n")\n print(line, flush=True)\n\n\ndef redact_text(text: str | None) -> str:\n if not text:\n return ""\n value = text\n for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]:\n secret = os.environ.get(secret_name)\n if secret:\n value = value.replace(secret, "[REDACTED]")\n value = re.sub(r"Bearer\\s+[A-Za-z0-9_\\-.=]+", "Bearer [REDACTED]", value)\n value = re.sub(r"hf_[A-Za-z0-9_\\-]{10,}", "hf_[REDACTED]", value)\n return value\n\n\ndef safe_details(details: dict | None) -> dict:\n if not details:\n return {}\n try:\n return json.loads(redact_text(json.dumps(details, ensure_ascii=False)))\n except Exception:\n return {"redacted_details": redact_text(str(details))[-4000:]}\n\n\ndef fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):\n safe = safe_details(details)\n append_event(events_path, "failure", "failed", message, safe)\n write_json(run_dir / "state.json", {\n "run_id": os.environ.get("RUN_ID"),\n "kind": "pi_model_card",\n "status": status,\n "message": message,\n "updated_at": now(),\n "details": safe,\n })\n report = f"""# Agentic Space Factory — Model Card Space Report\n\nStatus: **{status}**\n\n{message}\n\n```json\n{json.dumps(safe, indent=2, ensure_ascii=False)}\n```\n"""\n (run_dir / "report.md").write_text(report, encoding="utf-8")\n raise SystemExit(1)\n\n\ndef run_cmd(cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600):\n result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\n return result.returncode, redact_text(result.stdout)\n\n\ndef install_python_deps(events_path: Path):\n append_event(events_path, "dependencies", "started", "Installing Python worker dependencies")\n code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0"], timeout=600)\n if code != 0:\n append_event(events_path, "dependencies", "failed", "Python dependency installation failed", {"output_tail": out[-4000:]})\n raise RuntimeError(out)\n append_event(events_path, "dependencies", "success", "Python worker dependencies installed")\n\n\ndef ensure_node(events_path: Path):\n node = shutil.which("node")\n npm = shutil.which("npm")\n if node and npm:\n _, node_v = run_cmd([node, "--version"], timeout=30)\n _, npm_v = run_cmd([npm, "--version"], timeout=30)\n append_event(events_path, "node", "success", "Node/npm already available", {"node": node_v.strip(), "npm": npm_v.strip()})\n return\n append_event(events_path, "node", "started", "Installing nodejs/npm through apt-get")\n code, out = run_cmd(["bash", "-lc", "apt-get update -qq && apt-get install -y -qq nodejs npm"], timeout=600)\n if code != 0:\n append_event(events_path, "node", "failed", "Could not install nodejs/npm", {"output_tail": out[-4000:]})\n raise RuntimeError(out)\n append_event(events_path, "node", "success", "Installed nodejs/npm")\n\n\ndef install_pi(events_path: Path):\n ensure_node(events_path)\n append_event(events_path, "pi_install", "started", "Installing Pi coding agent from npm")\n code, out = run_cmd(["npm", "install", "-g", "@mariozechner/pi-coding-agent"], timeout=900)\n if code != 0:\n append_event(events_path, "pi_install", "failed", "Pi npm installation failed", {"output_tail": out[-4000:]})\n raise RuntimeError(out)\n code, version = run_cmd(["pi", "--version"], timeout=60)\n append_event(events_path, "pi_install", "success", "Pi installed", {"version_output": version.strip()[-300:]})\n\n\ndef configure_pi(events_path: Path, model: str):\n pi_dir = Path.home() / ".pi" / "agent"\n pi_dir.mkdir(parents=True, exist_ok=True)\n (pi_dir / "auth.json").write_text(json.dumps({"huggingface": {"type": "api_key", "key": os.environ.get("HF_TOKEN", "")}}, indent=2), encoding="utf-8")\n (pi_dir / "settings.json").write_text(json.dumps({"model": model, "provider": "huggingface", "autoRun": True, "autoApply": True}, indent=2), encoding="utf-8")\n append_event(events_path, "pi_config", "success", "Configured Pi", {"model": model})\n\n\ndef sanitize_model_id(model_id: str) -> str:\n model_id = (model_id or "").strip().replace("https://huggingface.co/", "")\n model_id = model_id.split("?", 1)[0].strip("/")\n if not re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", model_id):\n raise ValueError("MODEL_ID must look like owner/model-name")\n return model_id\n\n\ndef analyze_model(model_id: str, token: str, run_dir: Path, events_path: Path) -> dict:\n from huggingface_hub import HfApi, hf_hub_download\n append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})\n api = HfApi(token=token)\n info = api.model_info(model_id, token=token, files_metadata=False)\n siblings = [getattr(s, "rfilename", "") for s in (info.siblings or [])]\n pipeline_tag = getattr(info, "pipeline_tag", None)\n library_name = getattr(info, "library_name", None)\n tags = list(getattr(info, "tags", []) or [])\n readme_excerpt = ""\n try:\n readme_path = hf_hub_download(repo_id=model_id, filename="README.md", token=token)\n readme_text = Path(readme_path).read_text(encoding="utf-8", errors="ignore")\n readme_excerpt = readme_text[:6000]\n except Exception as exc:\n readme_excerpt = f"Could not download README.md: {exc}"\n task = pipeline_tag or "text-generation"\n if task == "sentiment-analysis":\n task = "text-classification"\n supported = task in SUPPORTED_TASKS\n analysis = {\n "model_id": model_id,\n "pipeline_tag": pipeline_tag,\n "library_name": library_name,\n "tags": tags[:80],\n "siblings": siblings[:120],\n "selected_task": task,\n "template": "transformers_text_pipeline" if supported else "unsupported",\n "supported": supported,\n "confidence": 0.8 if supported else 0.25,\n "risks": [],\n "readme_excerpt": readme_excerpt,\n "evidence": [f"pipeline_tag={pipeline_tag}", f"library_name={library_name}", f"files={\', \'.join(siblings[:12])}"],\n }\n if not supported:\n analysis["risks"].append("Phase 5 only supports simple Transformers text pipeline tasks.")\n if "gated" in tags:\n analysis["risks"].append("Model appears gated; generated Space will not receive OAuth token as a secret in Phase 5.")\n write_json(run_dir / "model_analysis.json", analysis)\n append_event(events_path, "model_analysis", "success" if supported else "unsupported", "Model metadata analyzed", {"selected_task": task, "supported": supported, "confidence": analysis["confidence"]})\n return analysis\n\n\ndef render_app(model_id: str, task: str) -> str:\n return dedent(f\'\'\'\n import gradio as gr\n from transformers import pipeline\n\n MODEL_ID = {model_id!r}\n TASK = {task!r}\n\n pipe = pipeline(TASK, model=MODEL_ID)\n\n def run_model(text: str) -> str:\n text = (text or "Hello from Agentic Space Factory").strip() or "Hello from Agentic Space Factory"\n if TASK == "text-generation":\n result = pipe(text, max_new_tokens=32, do_sample=False)\n return result[0].get("generated_text", str(result))\n if TASK == "text2text-generation":\n result = pipe(text, max_new_tokens=64)\n return result[0].get("generated_text", str(result))\n if TASK == "fill-mask":\n mask = getattr(getattr(pipe, "tokenizer", None), "mask_token", None) or "<mask>"\n if mask not in text:\n text = f"Hugging Face is {{mask}}."\n result = pipe(text)\n return str(result[:3] if isinstance(result, list) else result)\n if TASK in {"text-classification", "sentiment-analysis"}:\n return str(pipe(text))\n return str(pipe(text))\n\n demo = gr.Interface(\n fn=run_model,\n inputs=gr.Textbox(label="Input", value="Hello from Agentic Space Factory"),\n outputs=gr.Textbox(label="Model output"),\n title=f"Model demo: {{MODEL_ID}}",\n description="Generated by Agentic Space Factory from model metadata. Pi adapted this model app.",\n examples=[["Hello from Agentic Space Factory"], ["Hugging Face is awesome"]],\n )\n\n if __name__ == "__main__":\n demo.launch()\n \'\'\').strip() + "\\n"\n\n\ndef render_readme(model_id: str, task: str, target_space_id: str) -> str:\n return dedent(f\'\'\'\n ---\n title: Model Card Generated Space\n emoji: 🤖\n colorFrom: green\n colorTo: blue\n sdk: gradio\n app_file: app.py\n python_version: "3.11"\n pinned: false\n ---\n\n # Model Card Generated Space\n\n This private Space was generated by Agentic Space Factory from `{model_id}`.\n\n - Target Space: `{target_space_id}`\n - Selected task: `{task}`\n - Template: `transformers_text_pipeline`\n\n Phase 5 is intentionally limited to simple Transformers text pipelines.\n \'\'\').strip() + "\\n"\n\n\ndef prepare_workspace(workspace: Path, run_dir: Path, model_id: str, task: str, target_space_id: str, analysis: dict, events_path: Path):\n workspace.mkdir(parents=True, exist_ok=True)\n (workspace / "app.py").write_text(render_app(model_id, task), encoding="utf-8")\n (workspace / "README.md").write_text(render_readme(model_id, task, target_space_id), encoding="utf-8")\n (workspace / "requirements.txt").write_text("gradio>=5.0.0\\nhuggingface_hub>=0.34.0,<1.0.0\\ntransformers>=4.45.0\\ntorch\\nsafetensors\\n", encoding="utf-8")\n goal = f"""You are running inside a Hugging Face Job as a coding agent.\n\nGoal: adapt the provided minimal Gradio app for the model `{model_id}` and task `{task}`.\n\nFirst, read the HF Spaces Agent Quickstart gist:\nhttps://gist.github.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d\n\nRules for this Phase 5 smoke test:\n- Work only in the current workspace.\n- Do not create, delete, publish, or modify Hugging Face repos. The wrapper will create/upload the private Space.\n- Preserve `app.py`, `README.md`, and `requirements.txt`.\n- Do not remove the `huggingface_hub>=0.34.0,<1.0.0` compatibility pin from requirements.txt.\n- Preserve the `run_model` function and a Gradio Interface or Blocks app.\n- Preserve the exact marker phrase: Pi adapted this model app.\n- Keep the app simple and CPU-friendly.\n- Do not print secrets.\n- Write a short summary to `PI_SUMMARY.md`.\n\nModel analysis:\n```json\n{json.dumps({k: v for k, v in analysis.items() if k != \'readme_excerpt\'}, indent=2, ensure_ascii=False)}\n```\n\nREADME excerpt:\n{analysis.get(\'readme_excerpt\', \'\')[:3000]}\n"""\n (workspace / "GOAL.md").write_text(goal, encoding="utf-8")\n save_generated_files(run_dir, workspace)\n append_event(events_path, "workspace", "success", "Prepared model app workspace", {"files": ["app.py", "README.md", "requirements.txt", "GOAL.md"]})\n\n\ndef save_generated_files(run_dir: Path, workspace: Path):\n generated_dir = run_dir / "generated"\n generated_dir.mkdir(parents=True, exist_ok=True)\n for filename in ["app.py", "README.md", "requirements.txt", "GOAL.md", "PI_SUMMARY.md"]:\n path = workspace / filename\n if path.exists():\n (generated_dir / filename).write_text(path.read_text(encoding="utf-8", errors="ignore"), encoding="utf-8")\n\n\ndef run_pi(workspace: Path, run_dir: Path, events_path: Path, model: str):\n append_event(events_path, "pi_run", "started", "Running Pi on model-card workspace", {"model": model})\n env = os.environ.copy()\n env["HF_TOKEN"] = os.environ.get("HF_TOKEN", "")\n code, out = run_cmd(["pi", "-p", (workspace / "GOAL.md").read_text(encoding="utf-8")], cwd=workspace, env=env, timeout=1800)\n logs_dir = run_dir / "logs"\n logs_dir.mkdir(parents=True, exist_ok=True)\n (logs_dir / "pi_output.txt").write_text(out, encoding="utf-8")\n save_generated_files(run_dir, workspace)\n if code != 0:\n append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": out[-4000:]})\n raise RuntimeError("Pi failed. See logs/pi_output.txt")\n app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")\n if "Pi adapted this model app" not in app_text:\n raise RuntimeError("Pi/app verification failed: expected marker phrase missing from app.py")\n append_event(events_path, "pi_run", "success", "Pi completed and preserved required marker")\n\n\ndef collect_pi_traces(run_dir: Path, events_path: Path):\n src = Path.home() / ".pi" / "agent" / "sessions"\n raw_dir = run_dir / "traces" / "raw"\n redacted_dir = run_dir / "traces" / "redacted"\n raw_dir.mkdir(parents=True, exist_ok=True)\n redacted_dir.mkdir(parents=True, exist_ok=True)\n count = 0\n if src.exists():\n for path in src.rglob("*.jsonl"):\n count += 1\n text = path.read_text(encoding="utf-8", errors="ignore")\n (raw_dir / path.name).write_text(redact_text(text), encoding="utf-8")\n (redacted_dir / path.name).write_text(redact_text(text), encoding="utf-8")\n append_event(events_path, "traces", "success", "Collected Pi traces", {"count": count})\n\n\ndef make_gradio_client(target_space_id: str, token: str):\n import inspect\n from gradio_client import Client\n params = inspect.signature(Client).parameters\n if "token" in params:\n return Client(target_space_id, token=token)\n if "hf_token" in params:\n return Client(target_space_id, hf_token=token)\n if "api_key" in params:\n return Client(target_space_id, api_key=token)\n if "headers" in params:\n return Client(target_space_id, headers={"Authorization": f"Bearer {token}"})\n return Client(target_space_id)\n\n\ndef get_api_schema(client):\n try:\n return client.view_api(return_format="dict")\n except TypeError:\n return client.view_api()\n\n\ndef extract_api_names(api_schema) -> list[str]:\n names = []\n def add(value):\n if not value or not isinstance(value, str):\n return\n name = value if value.startswith("/") else f"/{value}"\n if name not in names:\n names.append(name)\n def walk(obj):\n if isinstance(obj, dict):\n for key, value in obj.items():\n if key in {"api_name", "apiName"}:\n add(value)\n if isinstance(key, str) and key.startswith("/"):\n add(key)\n walk(value)\n elif isinstance(obj, list):\n for item in obj:\n walk(item)\n walk(api_schema)\n return names\n\n\ndef predict_with_available_endpoint(client, api_schema, value: str):\n candidates = extract_api_names(api_schema)\n for fallback in ["/run_model", "/predict", "/greet"]:\n if fallback not in candidates:\n candidates.append(fallback)\n errors = []\n for api_name in candidates:\n try:\n return api_name, client.predict(value, api_name=api_name)\n except Exception as exc:\n errors.append({"api_name": api_name, "error": str(exc)[-500:]})\n try:\n return None, client.predict(value)\n except Exception as exc:\n errors.append({"api_name": None, "error": str(exc)[-500:]})\n raise RuntimeError(f"No candidate Gradio endpoint worked: {json.dumps(errors, ensure_ascii=False)}")\n\n\ndef validate_live_api(target_space_id: str, token: str, events_path: Path, tests_dir: Path, timeout_seconds: int = 900):\n tests_dir.mkdir(parents=True, exist_ok=True)\n deadline = time.time() + timeout_seconds\n last_error = None\n attempt = 0\n append_event(events_path, "api_validation", "started", "Waiting for live model Gradio API to become available")\n while time.time() < deadline:\n attempt += 1\n try:\n client = make_gradio_client(target_space_id, token)\n api_schema = get_api_schema(client)\n api_names = extract_api_names(api_schema)\n write_json(tests_dir / "api_schema.json", {"schema": api_schema, "api_names": api_names})\n used_api_name, result = predict_with_available_endpoint(client, api_schema, "Hello from Agentic Space Factory")\n result_text = str(result)\n ok = bool(result_text and len(result_text.strip()) >= 2)\n payload = {"attempt": attempt, "target_space": target_space_id, "api_test_passed": ok, "api_name": used_api_name, "discovered_api_names": api_names, "result": result_text[:4000], "validated_at": now()}\n write_json(tests_dir / "test_result.json", payload)\n if ok:\n append_event(events_path, "api_validation", "success", "Live model API test passed", {"attempt": attempt, "api_name": used_api_name, "discovered_api_names": api_names})\n return payload\n last_error = f"Unexpected empty API result from {used_api_name}: {result_text}"\n except Exception as exc:\n last_error = str(exc)\n append_event(events_path, "api_validation", "waiting", "Live API not ready yet", {"attempt": attempt, "error": last_error[-1000:]})\n time.sleep(20)\n payload = {"target_space": target_space_id, "api_test_passed": False, "error": last_error, "validated_at": now()}\n write_json(tests_dir / "test_result.json", payload)\n raise RuntimeError(f"Live API validation did not pass before timeout: {last_error}")\n\n\ndef create_and_upload_space(api, token: str, target_space_id: str, workspace: Path, events_path: Path):\n append_event(events_path, "create_space", "started", f"Creating private target Space {target_space_id}")\n api.create_repo(repo_id=target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, token=token)\n append_event(events_path, "create_space", "success", "Private target Space created", {"target_space": target_space_id})\n append_event(events_path, "upload_files", "started", "Uploading model app files to target Space")\n for path_in_repo in ["app.py", "README.md", "requirements.txt"]:\n api.upload_file(path_or_fileobj=(workspace / path_in_repo).read_bytes(), path_in_repo=path_in_repo, repo_id=target_space_id, repo_type="space", token=token)\n append_event(events_path, "upload_files", "success", f"Uploaded {path_in_repo}")\n\n\ndef main():\n run_id = os.environ["RUN_ID"]\n hf_username = os.environ.get("HF_USERNAME", "unknown")\n bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")\n output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))\n target_space_id = os.environ["TARGET_SPACE_ID"]\n model_id = sanitize_model_id(os.environ.get("MODEL_ID", ""))\n pi_model = os.environ.get("PI_MODEL") or "moonshotai/Kimi-K2.5"\n token = os.environ.get("HF_TOKEN")\n run_dir = output_root / "runs" / run_id\n events_path = run_dir / "events.jsonl"\n state_path = run_dir / "state.json"\n workspace = Path("/tmp") / f"space-factory-model-{run_id}"\n append_event(events_path, "bootstrap", "started", "Pi model-card worker started", {"model_id": model_id})\n write_json(state_path, {"run_id": run_id, "kind": "pi_model_card", "status": "running", "message": "Analyzing model card and generating a private model demo Space", "model_id": model_id, "target_space": target_space_id, "created_by": hf_username, "bucket_source": bucket_source, "created_at": now(), "updated_at": now()})\n if not token:\n fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets")\n if not TARGET_RE.match(target_space_id):\n fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space": target_space_id})\n if not target_space_id.startswith(f"{hf_username}/"):\n fail(run_dir, events_path, "Target Space must be in the signed-in user\'s namespace", {"target_space": target_space_id, "username": hf_username})\n try:\n install_python_deps(events_path)\n from huggingface_hub import HfApi\n api = HfApi(token=token)\n whoami = api.whoami(token=token)\n append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})\n analysis = analyze_model(model_id, token, run_dir, events_path)\n if not analysis.get("supported"):\n fail(run_dir, events_path, "Model task is unsupported by Phase 5", {"model_analysis": {k: v for k, v in analysis.items() if k != "readme_excerpt"}}, status="unsupported")\n prepare_workspace(workspace, run_dir, model_id, analysis["selected_task"], target_space_id, analysis, events_path)\n install_pi(events_path)\n configure_pi(events_path, pi_model)\n run_pi(workspace, run_dir, events_path, pi_model)\n collect_pi_traces(run_dir, events_path)\n create_and_upload_space(api, token, target_space_id, workspace, events_path)\n write_json(run_dir / "target_space.json", {"target_space": target_space_id, "url": f"https://huggingface.co/spaces/{target_space_id}", "private": True, "sdk": "gradio", "created_by": hf_username, "model_id": model_id})\n validation = validate_live_api(target_space_id, token, events_path, run_dir / "tests")\n final_state = {"run_id": run_id, "kind": "pi_model_card", "status": "success", "message": "Model-card generated private Space created and validated through the live API.", "model_id": model_id, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "created_by": hf_username, "bucket_source": bucket_source, "model_analysis": {k: v for k, v in analysis.items() if k != "readme_excerpt"}, "validation": validation, "updated_at": now(), "security_notes": ["The target Space was created as private.", "The HF token was not printed or intentionally persisted.", "Phase 5 supports only simple public text pipeline models.", "Success was declared only after the wrapper live API test passed."]}\n write_json(state_path, final_state)\n report = f"""# Agentic Space Factory — Model Card Space Report\n\nRun ID: `{run_id}`\n\nStatus: **success**\n\nCreated private model demo Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_id})\n\n## Model\n\n- Model ID: `{model_id}`\n- Selected task: `{analysis[\'selected_task\']}`\n- Template: `{analysis[\'template\']}`\n- Pi model: `{pi_model}`\n\n## What happened\n\n```text\nOAuth user → HF Job → model metadata analysis → Pi adapts app.py → private Space creation → live API validation → Bucket report\n```\n\n## Live API validation\n\n```json\n{json.dumps(validation, indent=2, ensure_ascii=False)}\n```\n\n## Security posture\n\n- The target Space was created as private.\n- No token was printed or intentionally persisted.\n- Pi was instructed not to create/delete/publish repos; the wrapper performed Hub operations.\n- Success was declared only after the live API returned a non-empty result.\n\n## Next step\n\nPhase 6 should add a ZeroGPU Diffusers template and stricter model compatibility gating.\n"""\n (run_dir / "report.md").write_text(report, encoding="utf-8")\n append_event(events_path, "report_write", "success", "Wrote report.md")\n append_event(events_path, "done", "success", "Pi model-card worker completed")\n except SystemExit:\n raise\n except Exception as exc:\n collect_pi_traces(run_dir, events_path)\n fail(run_dir, events_path, "Pi model-card worker failed", {"error": str(exc)})\n\n\nif __name__ == "__main__":\n main()\n'
|
| 570 |
|
| 571 |
+
|
| 572 |
+
RUNTIME_RECOMMENDER_WORKER_SCRIPT = 'import json\nimport os\nimport re\nimport sys\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nimport subprocess\n\nSUPPORTED_TEXT_TASKS = {"text-generation", "text2text-generation", "fill-mask", "text-classification", "sentiment-analysis"}\nDIFFUSION_TASKS = {"text-to-image", "image-to-image", "image-to-video", "text-to-video"}\n\n\ndef now():\n return datetime.now(timezone.utc).isoformat()\n\n\ndef write_json(path: Path, payload: dict):\n path.parent.mkdir(parents=True, exist_ok=True)\n path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\\n", encoding="utf-8")\n\n\ndef append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):\n path.parent.mkdir(parents=True, exist_ok=True)\n event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}}\n line = json.dumps(event, ensure_ascii=False)\n with path.open("a", encoding="utf-8") as f:\n f.write(line + "\\n")\n print(line, flush=True)\n\n\ndef redact_text(text: str | None) -> str:\n if not text:\n return ""\n value = text\n for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]:\n secret = os.environ.get(secret_name)\n if secret:\n value = value.replace(secret, "[REDACTED]")\n value = re.sub(r"Bearer\\s+[A-Za-z0-9_\\-.=]+", "Bearer [REDACTED]", value)\n value = re.sub(r"hf_[A-Za-z0-9_\\-]{10,}", "hf_[REDACTED]", value)\n return value\n\n\ndef run_cmd(cmd: list[str], timeout: int = 600):\n result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\n return result.returncode, redact_text(result.stdout)\n\n\ndef fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):\n append_event(events_path, "failure", "failed", message, details or {})\n write_json(run_dir / "state.json", {\n "run_id": os.environ.get("RUN_ID"),\n "kind": "runtime_recommender",\n "status": status,\n "message": message,\n "updated_at": now(),\n "details": details or {},\n })\n (run_dir / "report.md").write_text(f"# Runtime Recommendation Report\\n\\nStatus: **{status}**\\n\\n{message}\\n\\n```json\\n{json.dumps(details or {}, indent=2, ensure_ascii=False)}\\n```\\n", encoding="utf-8")\n raise SystemExit(1)\n\n\ndef sanitize_model_id(model_id: str) -> str:\n model_id = (model_id or "").strip().replace("https://huggingface.co/", "")\n model_id = model_id.split("?", 1)[0].strip("/")\n if not re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", model_id):\n raise ValueError("MODEL_ID must look like owner/model-name")\n return model_id\n\n\ndef human_bytes(value: int | None) -> str:\n if value is None:\n return "unknown"\n size = float(value)\n for unit in ["B", "KB", "MB", "GB", "TB"]:\n if size < 1024 or unit == "TB":\n return f"{size:.1f} {unit}"\n size /= 1024\n return str(value)\n\n\ndef infer_task(pipeline_tag, library_name, tags, siblings):\n if pipeline_tag:\n return "text-classification" if pipeline_tag == "sentiment-analysis" else pipeline_tag\n if library_name == "diffusers" or "model_index.json" in siblings:\n return "text-to-image"\n if "config.json" in siblings:\n return "text-generation"\n return "unknown"\n\n\ndef recommend_runtime(task: str, library_name: str | None, tags: list[str], siblings: list[str], total_size: int | None):\n risks = []\n evidence = []\n supported_for_phase5 = task in SUPPORTED_TEXT_TASKS\n is_diffusion = task in DIFFUSION_TASKS or library_name == "diffusers" or "model_index.json" in siblings\n gated = any("gated" == t or "gated" in t for t in tags)\n trust_remote = any("trust_remote_code" in t or "custom_code" in t for t in tags)\n\n if gated:\n risks.append("Model appears gated; generated Spaces will need explicit secret/token handling before runtime validation.")\n if trust_remote:\n risks.append("Model may require custom code/trust_remote_code; keep out of V1 auto-build unless manually approved.")\n if total_size is not None:\n evidence.append(f"Estimated repo file size: {human_bytes(total_size)}")\n evidence.append(f"task={task}")\n evidence.append(f"library_name={library_name}")\n\n if is_diffusion:\n if total_size and total_size > 15 * 1024**3:\n return {\n "target_runtime": "manual-review",\n "target_space_hardware": "manual-review",\n "job_flavor": "cpu-basic",\n "confidence": 0.35,\n "supported_by_current_builder": False,\n "reason": "Diffusion/video-like model appears too large for the next ZeroGPU MVP without manual review.",\n "risks": risks + ["Large diffusion/video models may exceed ZeroGPU duration/memory assumptions."],\n "evidence": evidence,\n }\n return {\n "target_runtime": "zerogpu-candidate",\n "target_space_hardware": "zero-a10g / ZeroGPU if available for the user",\n "job_flavor": "cpu-basic",\n "confidence": 0.72,\n "supported_by_current_builder": False,\n "reason": "Diffusers-style model; good candidate for Phase 7 ZeroGPU template, but not for Phase 5 CPU text template.",\n "risks": risks + ["ZeroGPU template is not implemented yet in this version."],\n "evidence": evidence,\n }\n\n if task in SUPPORTED_TEXT_TASKS:\n if total_size is None:\n runtime = "cpu-basic"\n confidence = 0.62\n reason = "Simple text pipeline task; file sizes unavailable, starting with CPU is acceptable for a guarded MVP."\n elif total_size <= 500 * 1024**2:\n runtime = "cpu-basic"\n confidence = 0.82\n reason = "Small text model; CPU Basic should be acceptable for validation demos."\n elif total_size <= 2 * 1024**3:\n runtime = "cpu-upgrade recommended"\n confidence = 0.68\n reason = "Medium text model; CPU Basic may be slow, CPU Upgrade is safer for runtime."\n risks.append("CPU Basic may cold-start or infer slowly.")\n else:\n runtime = "manual-review or GPU/Endpoint"\n confidence = 0.38\n reason = "Large text model; not suitable for automatic CPU Space generation in this MVP."\n risks.append("Model appears too large for the current CPU-only template.")\n supported_for_phase5 = False\n return {\n "target_runtime": runtime,\n "target_space_hardware": runtime,\n "job_flavor": "cpu-basic",\n "confidence": confidence,\n "supported_by_current_builder": supported_for_phase5,\n "reason": reason,\n "risks": risks,\n "evidence": evidence,\n }\n\n return {\n "target_runtime": "unsupported",\n "target_space_hardware": "manual-review",\n "job_flavor": "cpu-basic",\n "confidence": 0.25,\n "supported_by_current_builder": False,\n "reason": "Task/library combination is outside the current safe templates.",\n "risks": risks + ["Unsupported by current builder templates."],\n "evidence": evidence,\n }\n\n\ndef main():\n run_id = os.environ["RUN_ID"]\n hf_username = os.environ.get("HF_USERNAME", "unknown")\n bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")\n output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))\n model_id = sanitize_model_id(os.environ.get("MODEL_ID", ""))\n token = os.environ.get("HF_TOKEN")\n run_dir = output_root / "runs" / run_id\n events_path = run_dir / "events.jsonl"\n state_path = run_dir / "state.json"\n\n append_event(events_path, "bootstrap", "started", "Runtime recommender worker started", {"model_id": model_id})\n write_json(state_path, {"run_id": run_id, "kind": "runtime_recommender", "status": "running", "message": "Analyzing model compatibility and runtime recommendation", "model_id": model_id, "created_by": hf_username, "bucket_source": bucket_source, "created_at": now(), "updated_at": now()})\n if not token:\n fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets")\n\n try:\n append_event(events_path, "dependencies", "started", "Installing Python worker dependencies")\n code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0"], timeout=600)\n if code != 0:\n fail(run_dir, events_path, "Python dependency installation failed", {"output_tail": out[-4000:]})\n append_event(events_path, "dependencies", "success", "Python worker dependencies installed")\n\n from huggingface_hub import HfApi, hf_hub_download\n api = HfApi(token=token)\n whoami = api.whoami(token=token)\n append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})\n\n append_event(events_path, "model_analysis", "started", "Fetching model metadata", {"model_id": model_id})\n info = api.model_info(model_id, token=token, files_metadata=True)\n siblings_objs = list(info.siblings or [])\n siblings = [getattr(s, "rfilename", "") for s in siblings_objs]\n sizes = [getattr(s, "size", None) for s in siblings_objs]\n total_size = sum(x for x in sizes if isinstance(x, int)) if any(isinstance(x, int) for x in sizes) else None\n pipeline_tag = getattr(info, "pipeline_tag", None)\n library_name = getattr(info, "library_name", None)\n tags = list(getattr(info, "tags", []) or [])\n task = infer_task(pipeline_tag, library_name, tags, siblings)\n readme_excerpt = ""\n try:\n readme_path = hf_hub_download(repo_id=model_id, filename="README.md", token=token)\n readme_excerpt = Path(readme_path).read_text(encoding="utf-8", errors="ignore")[:5000]\n except Exception as exc:\n readme_excerpt = f"README unavailable: {exc}"\n\n recommendation = recommend_runtime(task, library_name, tags, siblings, total_size)\n analysis = {\n "model_id": model_id,\n "pipeline_tag": pipeline_tag,\n "library_name": library_name,\n "tags": tags[:100],\n "siblings": siblings[:160],\n "estimated_total_file_size_bytes": total_size,\n "estimated_total_file_size_human": human_bytes(total_size),\n "selected_task": task,\n "readme_excerpt": readme_excerpt,\n "runtime_recommendation": recommendation,\n }\n write_json(run_dir / "model_analysis.json", analysis)\n write_json(run_dir / "runtime_recommendation.json", recommendation)\n append_event(events_path, "model_analysis", "success", "Model metadata analyzed", {"selected_task": task, "target_runtime": recommendation["target_runtime"], "confidence": recommendation["confidence"], "supported_by_current_builder": recommendation["supported_by_current_builder"]})\n\n status = "success" if recommendation["confidence"] >= 0.35 else "needs_review"\n final_state = {"run_id": run_id, "kind": "runtime_recommender", "status": status, "message": "Runtime recommendation completed", "model_id": model_id, "created_by": hf_username, "bucket_source": bucket_source, "model_analysis": {k: v for k, v in analysis.items() if k != "readme_excerpt"}, "updated_at": now()}\n write_json(state_path, final_state)\n report = f"""# Agentic Space Factory — Runtime Recommendation Report\n\nRun ID: `{run_id}`\n\nStatus: **{status}**\n\nModel: `{model_id}`\n\n## Recommendation\n\n```json\n{json.dumps(recommendation, indent=2, ensure_ascii=False)}\n```\n\n## Model metadata\n\n- Pipeline tag: `{pipeline_tag}`\n- Library: `{library_name}`\n- Selected task: `{task}`\n- Estimated total file size: `{human_bytes(total_size)}`\n\n## Interpretation\n\n- `supported_by_current_builder=true` means Phase 5 can attempt the current simple text-pipeline builder.\n- `zerogpu-candidate` means the model looks like a candidate for the upcoming ZeroGPU/Diffusers template, but should not be sent through the CPU text builder.\n- `manual-review` means the model should not be auto-built without an explicit user decision.\n\n## Next step\n\nUse this recommendation as a gate before launching Phase 5/Phase 7 builders.\n"""\n (run_dir / "report.md").write_text(report, encoding="utf-8")\n append_event(events_path, "report_write", "success", "Wrote report.md")\n append_event(events_path, "done", "success", "Runtime recommendation completed")\n except SystemExit:\n raise\n except Exception as exc:\n fail(run_dir, events_path, "Runtime recommender worker failed", {"error": str(exc)})\n\n\nif __name__ == "__main__":\n main()\n'
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
LONGCAT_ARTICLE_WORKER_SCRIPT = r'''
|
| 576 |
+
import json
|
| 577 |
+
import os
|
| 578 |
+
import re
|
| 579 |
+
import shutil
|
| 580 |
+
import subprocess
|
| 581 |
+
import sys
|
| 582 |
+
import time
|
| 583 |
+
from datetime import datetime, timezone
|
| 584 |
+
from pathlib import Path
|
| 585 |
+
from textwrap import dedent
|
| 586 |
+
|
| 587 |
+
TARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$")
|
| 588 |
+
GIST_URL = "https://gist.github.com/gary149/2aba2962375fa9ca56bb9ef53f00b73d"
|
| 589 |
+
DEFAULT_MODEL_ID = "meituan-longcat/LongCat-Video-Avatar-1.5"
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
def now():
|
| 593 |
+
return datetime.now(timezone.utc).isoformat()
|
| 594 |
+
|
| 595 |
+
|
| 596 |
+
def write_json(path: Path, payload: dict):
|
| 597 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 598 |
+
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
| 599 |
+
|
| 600 |
+
|
| 601 |
+
def append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):
|
| 602 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 603 |
+
event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}}
|
| 604 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 605 |
+
with path.open("a", encoding="utf-8") as f:
|
| 606 |
+
f.write(line + "\n")
|
| 607 |
+
print(line, flush=True)
|
| 608 |
+
|
| 609 |
+
|
| 610 |
+
def redact_text(text: str | None) -> str:
|
| 611 |
+
if not text:
|
| 612 |
+
return ""
|
| 613 |
+
value = text
|
| 614 |
+
for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]:
|
| 615 |
+
secret = os.environ.get(secret_name)
|
| 616 |
+
if secret:
|
| 617 |
+
value = value.replace(secret, "[REDACTED]")
|
| 618 |
+
value = re.sub(r"Bearer\s+[A-Za-z0-9_\-.=]+", "Bearer [REDACTED]", value)
|
| 619 |
+
value = re.sub(r"hf_[A-Za-z0-9_\-]{10,}", "hf_[REDACTED]", value)
|
| 620 |
+
return value
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def safe_details(details: dict | None) -> dict:
|
| 624 |
+
if not details:
|
| 625 |
+
return {}
|
| 626 |
+
try:
|
| 627 |
+
return json.loads(redact_text(json.dumps(details, ensure_ascii=False)))
|
| 628 |
+
except Exception:
|
| 629 |
+
return {"redacted_details": redact_text(str(details))[-4000:]}
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
def fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):
|
| 633 |
+
safe = safe_details(details)
|
| 634 |
+
append_event(events_path, "failure", "failed", message, safe)
|
| 635 |
+
write_json(run_dir / "state.json", {
|
| 636 |
+
"run_id": os.environ.get("RUN_ID"),
|
| 637 |
+
"kind": "longcat_article_reproduction",
|
| 638 |
+
"status": status,
|
| 639 |
+
"message": message,
|
| 640 |
+
"updated_at": now(),
|
| 641 |
+
"details": safe,
|
| 642 |
+
})
|
| 643 |
+
report = f"""# Agentic Space Factory — LongCat Article Reproduction Report
|
| 644 |
+
|
| 645 |
+
Status: **{status}**
|
| 646 |
+
|
| 647 |
+
{message}
|
| 648 |
+
|
| 649 |
+
```json
|
| 650 |
+
{json.dumps(safe, indent=2, ensure_ascii=False)}
|
| 651 |
+
```
|
| 652 |
+
"""
|
| 653 |
+
(run_dir / "report.md").write_text(report, encoding="utf-8")
|
| 654 |
+
raise SystemExit(1)
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
def run_cmd(cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600):
|
| 658 |
+
result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)
|
| 659 |
+
return result.returncode, redact_text(result.stdout)
|
| 660 |
+
|
| 661 |
+
|
| 662 |
+
def install_python_deps(events_path: Path):
|
| 663 |
+
append_event(events_path, "dependencies", "started", "Installing Python worker dependencies")
|
| 664 |
+
code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0"], timeout=600)
|
| 665 |
+
if code != 0:
|
| 666 |
+
append_event(events_path, "dependencies", "failed", "Python dependency installation failed", {"output_tail": out[-4000:]})
|
| 667 |
+
raise RuntimeError(out)
|
| 668 |
+
append_event(events_path, "dependencies", "success", "Python worker dependencies installed")
|
| 669 |
+
|
| 670 |
+
|
| 671 |
+
def ensure_node(events_path: Path):
|
| 672 |
+
node = shutil.which("node")
|
| 673 |
+
npm = shutil.which("npm")
|
| 674 |
+
if node and npm:
|
| 675 |
+
_, node_v = run_cmd([node, "--version"], timeout=30)
|
| 676 |
+
_, npm_v = run_cmd([npm, "--version"], timeout=30)
|
| 677 |
+
append_event(events_path, "node", "success", "Node/npm already available", {"node": node_v.strip(), "npm": npm_v.strip()})
|
| 678 |
+
return
|
| 679 |
+
append_event(events_path, "node", "started", "Installing nodejs/npm through apt-get")
|
| 680 |
+
code, out = run_cmd(["bash", "-lc", "apt-get update -qq && apt-get install -y -qq nodejs npm"], timeout=600)
|
| 681 |
+
if code != 0:
|
| 682 |
+
append_event(events_path, "node", "failed", "Could not install nodejs/npm", {"output_tail": out[-4000:]})
|
| 683 |
+
raise RuntimeError(out)
|
| 684 |
+
append_event(events_path, "node", "success", "Installed nodejs/npm")
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
def install_pi(events_path: Path):
|
| 688 |
+
ensure_node(events_path)
|
| 689 |
+
append_event(events_path, "pi_install", "started", "Installing Pi coding agent from npm")
|
| 690 |
+
code, out = run_cmd(["npm", "install", "-g", "@mariozechner/pi-coding-agent"], timeout=900)
|
| 691 |
+
if code != 0:
|
| 692 |
+
append_event(events_path, "pi_install", "failed", "Pi npm installation failed", {"output_tail": out[-4000:]})
|
| 693 |
+
raise RuntimeError(out)
|
| 694 |
+
code, version = run_cmd(["pi", "--version"], timeout=60)
|
| 695 |
+
append_event(events_path, "pi_install", "success", "Pi installed", {"version_output": version.strip()[-300:]})
|
| 696 |
+
|
| 697 |
+
|
| 698 |
+
def configure_pi(events_path: Path, model: str):
|
| 699 |
+
pi_dir = Path.home() / ".pi" / "agent"
|
| 700 |
+
pi_dir.mkdir(parents=True, exist_ok=True)
|
| 701 |
+
(pi_dir / "auth.json").write_text(json.dumps({"huggingface": {"type": "api_key", "key": os.environ.get("HF_TOKEN", "")}}, indent=2), encoding="utf-8")
|
| 702 |
+
(pi_dir / "settings.json").write_text(json.dumps({"model": model, "provider": "huggingface", "autoRun": True, "autoApply": True}, indent=2), encoding="utf-8")
|
| 703 |
+
append_event(events_path, "pi_config", "success", "Configured Pi", {"model": model})
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
def collect_pi_traces(run_dir: Path, events_path: Path):
|
| 707 |
+
traces_dir = Path.home() / ".pi" / "agent" / "sessions"
|
| 708 |
+
raw_dir = run_dir / "traces" / "raw"
|
| 709 |
+
redacted_dir = run_dir / "traces" / "redacted"
|
| 710 |
+
raw_dir.mkdir(parents=True, exist_ok=True)
|
| 711 |
+
redacted_dir.mkdir(parents=True, exist_ok=True)
|
| 712 |
+
count = 0
|
| 713 |
+
if traces_dir.exists():
|
| 714 |
+
for path in traces_dir.rglob("*.jsonl"):
|
| 715 |
+
rel = path.relative_to(traces_dir)
|
| 716 |
+
target_raw = raw_dir / rel
|
| 717 |
+
target_raw.parent.mkdir(parents=True, exist_ok=True)
|
| 718 |
+
text = path.read_text(encoding="utf-8", errors="ignore")
|
| 719 |
+
target_raw.write_text(text, encoding="utf-8")
|
| 720 |
+
target_redacted = redacted_dir / rel
|
| 721 |
+
target_redacted.parent.mkdir(parents=True, exist_ok=True)
|
| 722 |
+
target_redacted.write_text(redact_text(text), encoding="utf-8")
|
| 723 |
+
count += 1
|
| 724 |
+
append_event(events_path, "traces", "success", "Collected Pi traces", {"count": count})
|
| 725 |
+
return count
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
def sanitize_model_id(model_id: str) -> str:
|
| 729 |
+
model_id = (model_id or DEFAULT_MODEL_ID).strip().replace("https://huggingface.co/", "")
|
| 730 |
+
model_id = model_id.split("?", 1)[0].strip("/")
|
| 731 |
+
if not re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", model_id):
|
| 732 |
+
raise ValueError("MODEL_ID must look like owner/model-name")
|
| 733 |
+
return model_id
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
def make_gradio_client(target_space_id: str, token: str):
|
| 737 |
+
import inspect
|
| 738 |
+
from gradio_client import Client
|
| 739 |
+
params = inspect.signature(Client).parameters
|
| 740 |
+
if "token" in params:
|
| 741 |
+
return Client(target_space_id, token=token)
|
| 742 |
+
if "hf_token" in params:
|
| 743 |
+
return Client(target_space_id, hf_token=token)
|
| 744 |
+
if "api_key" in params:
|
| 745 |
+
return Client(target_space_id, api_key=token)
|
| 746 |
+
if "headers" in params:
|
| 747 |
+
return Client(target_space_id, headers={"Authorization": f"Bearer {token}"})
|
| 748 |
+
return Client(target_space_id)
|
| 749 |
+
|
| 750 |
+
|
| 751 |
+
def api_names_from_schema(schema) -> list[str]:
|
| 752 |
+
names: list[str] = []
|
| 753 |
+
if isinstance(schema, dict):
|
| 754 |
+
endpoints = schema.get("named_endpoints") or schema.get("endpoints") or {}
|
| 755 |
+
if isinstance(endpoints, dict):
|
| 756 |
+
for key, value in endpoints.items():
|
| 757 |
+
if isinstance(key, str) and key.startswith("/"):
|
| 758 |
+
names.append(key)
|
| 759 |
+
if isinstance(value, dict):
|
| 760 |
+
api_name = value.get("api_name")
|
| 761 |
+
if isinstance(api_name, str) and api_name.startswith("/"):
|
| 762 |
+
names.append(api_name)
|
| 763 |
+
if isinstance(schema.get("dependencies"), list):
|
| 764 |
+
for dep in schema["dependencies"]:
|
| 765 |
+
if isinstance(dep, dict):
|
| 766 |
+
api_name = dep.get("api_name")
|
| 767 |
+
if isinstance(api_name, str):
|
| 768 |
+
names.append(api_name if api_name.startswith("/") else f"/{api_name}")
|
| 769 |
+
return list(dict.fromkeys(names))
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
def validate_live_api(target_space_id: str, token: str, run_dir: Path, events_path: Path, timeout_s: int = 900):
|
| 773 |
+
append_event(events_path, "api_validation", "started", "Waiting for live Gradio API to become available")
|
| 774 |
+
deadline = time.time() + timeout_s
|
| 775 |
+
attempt = 0
|
| 776 |
+
last_error = None
|
| 777 |
+
while time.time() < deadline:
|
| 778 |
+
attempt += 1
|
| 779 |
+
try:
|
| 780 |
+
client = make_gradio_client(target_space_id, token)
|
| 781 |
+
schema = client.view_api(return_format="dict")
|
| 782 |
+
write_json(run_dir / "tests" / "api_schema.json", schema if isinstance(schema, dict) else {"schema": str(schema)})
|
| 783 |
+
discovered = api_names_from_schema(schema)
|
| 784 |
+
candidates = []
|
| 785 |
+
for name in ["/health", "/predict", "/greet"] + discovered:
|
| 786 |
+
if name not in candidates:
|
| 787 |
+
candidates.append(name)
|
| 788 |
+
errors = []
|
| 789 |
+
for api_name in candidates:
|
| 790 |
+
try:
|
| 791 |
+
if api_name == "/health":
|
| 792 |
+
result = client.predict(api_name=api_name)
|
| 793 |
+
elif api_name == "/greet":
|
| 794 |
+
result = client.predict("Agentic Space Factory", api_name=api_name)
|
| 795 |
+
else:
|
| 796 |
+
result = client.predict(api_name=api_name)
|
| 797 |
+
payload = {"status": "success", "attempt": attempt, "api_name": api_name, "discovered_api_names": discovered, "result_repr": repr(result)[:2000]}
|
| 798 |
+
write_json(run_dir / "tests" / "test_result.json", payload)
|
| 799 |
+
append_event(events_path, "api_validation", "success", "Live API validation passed", {"attempt": attempt, "api_name": api_name, "discovered_api_names": discovered})
|
| 800 |
+
return payload
|
| 801 |
+
except Exception as exc:
|
| 802 |
+
errors.append({"api_name": api_name, "error": str(exc)[:1000]})
|
| 803 |
+
last_error = "; ".join(f"{e['api_name']}: {e['error']}" for e in errors[:5]) or "No callable API endpoints found"
|
| 804 |
+
except Exception as exc:
|
| 805 |
+
last_error = str(exc)
|
| 806 |
+
append_event(events_path, "api_validation", "waiting", "Live API not ready yet", {"attempt": attempt, "error": last_error[-1500:] if last_error else None})
|
| 807 |
+
time.sleep(30)
|
| 808 |
+
raise RuntimeError(f"Live API validation did not pass before timeout: {last_error}")
|
| 809 |
+
|
| 810 |
+
|
| 811 |
+
def request_hardware(api, target_space_id: str, hardware: str, token: str, events_path: Path, step: str):
|
| 812 |
+
if not hardware:
|
| 813 |
+
return {"requested": False, "hardware": hardware, "ok": False, "error": "empty hardware"}
|
| 814 |
+
try:
|
| 815 |
+
from huggingface_hub import SpaceHardware
|
| 816 |
+
hw = SpaceHardware(hardware)
|
| 817 |
+
runtime = api.request_space_hardware(repo_id=target_space_id, hardware=hw, token=token)
|
| 818 |
+
payload = {"requested": True, "hardware": hardware, "ok": True, "runtime_stage": getattr(getattr(runtime, "stage", None), "value", str(getattr(runtime, "stage", None)))}
|
| 819 |
+
append_event(events_path, step, "success", f"Requested Space hardware {hardware}", payload)
|
| 820 |
+
return payload
|
| 821 |
+
except Exception as exc:
|
| 822 |
+
payload = {"requested": True, "hardware": hardware, "ok": False, "error": str(exc)[:2000]}
|
| 823 |
+
append_event(events_path, step, "failed", f"Could not request Space hardware {hardware}", payload)
|
| 824 |
+
return payload
|
| 825 |
+
|
| 826 |
+
|
| 827 |
+
def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool):
|
| 828 |
+
workspace.mkdir(parents=True, exist_ok=True)
|
| 829 |
+
app_py = f"""import gradio as gr
|
| 830 |
+
|
| 831 |
+
MODEL_ID = "{model_id}"
|
| 832 |
+
TARGET_SPACE_ID = "{target_space_id}"
|
| 833 |
+
|
| 834 |
+
|
| 835 |
+
def health():
|
| 836 |
+
return {{
|
| 837 |
+
"status": "booted",
|
| 838 |
+
"model_id": MODEL_ID,
|
| 839 |
+
"note": "Pi should replace this scaffold with a LongCat demo while preserving /health."
|
| 840 |
+
}}
|
| 841 |
+
|
| 842 |
+
|
| 843 |
+
def placeholder():
|
| 844 |
+
return "LongCat demo scaffold booted. The full model pipeline was not wired yet."
|
| 845 |
+
|
| 846 |
+
with gr.Blocks(title="LongCat Video Avatar — Agentic Space Factory") as demo:
|
| 847 |
+
gr.Markdown("# LongCat Video Avatar — Agentic Space Factory")
|
| 848 |
+
gr.Markdown("This private Space was generated by the Phase 7 article reproduction worker.")
|
| 849 |
+
gr.JSON(label="Health", value=health(), every=None)
|
| 850 |
+
gr.Button("Health check").click(fn=health, inputs=None, outputs=gr.JSON(), api_name="health")
|
| 851 |
+
gr.Button("Placeholder").click(fn=placeholder, inputs=None, outputs=gr.Textbox(), api_name="predict")
|
| 852 |
+
|
| 853 |
+
if __name__ == "__main__":
|
| 854 |
+
demo.launch()
|
| 855 |
+
"""
|
| 856 |
+
(workspace / "app.py").write_text(app_py, encoding="utf-8")
|
| 857 |
+
req = """gradio>=5.0.0
|
| 858 |
+
huggingface_hub>=0.34.0,<1.0.0
|
| 859 |
+
spaces
|
| 860 |
+
transformers>=4.45.0
|
| 861 |
+
diffusers
|
| 862 |
+
accelerate
|
| 863 |
+
safetensors
|
| 864 |
+
torch
|
| 865 |
+
opencv-python-headless
|
| 866 |
+
pillow
|
| 867 |
+
numpy
|
| 868 |
+
"""
|
| 869 |
+
(workspace / "requirements.txt").write_text(req, encoding="utf-8")
|
| 870 |
+
readme = f"""---
|
| 871 |
+
title: LongCat Video Avatar Agentic Factory
|
| 872 |
+
sdk: gradio
|
| 873 |
+
app_file: app.py
|
| 874 |
+
python_version: "3.10"
|
| 875 |
+
suggested_hardware: {preferred_hardware or fallback_hardware or "zero-a10g"}
|
| 876 |
+
---
|
| 877 |
+
|
| 878 |
+
# LongCat Video Avatar — Agentic Space Factory
|
| 879 |
+
|
| 880 |
+
Private generated Space for `{model_id}`.
|
| 881 |
+
|
| 882 |
+
This Space is created by Phase 7. It should remain private until manually reviewed.
|
| 883 |
+
"""
|
| 884 |
+
(workspace / "README.md").write_text(readme, encoding="utf-8")
|
| 885 |
+
goal = f"""You are Pi running inside a Hugging Face Job for Agentic Space Factory Phase 7.
|
| 886 |
+
|
| 887 |
+
Goal: attempt to reproduce the workflow from the Hugging Face article by building a private Space demo for:
|
| 888 |
+
|
| 889 |
+
MODEL_ID: {model_id}
|
| 890 |
+
TARGET_SPACE_ID: {target_space_id}
|
| 891 |
+
|
| 892 |
+
First read and follow the operational rules from this gist:
|
| 893 |
+
{GIST_URL}
|
| 894 |
+
|
| 895 |
+
Important safety and product constraints:
|
| 896 |
+
- The target Space must remain private.
|
| 897 |
+
- Do not delete any user resources.
|
| 898 |
+
- Do not print secrets or tokens.
|
| 899 |
+
- Work only inside the current workspace.
|
| 900 |
+
- The wrapper will create the private Space, request hardware, upload files, and validate the live API. Do not create/delete repos yourself in this Phase 7 worker.
|
| 901 |
+
- You should edit app.py, requirements.txt, and README.md to make the best possible LongCat demo for the model card.
|
| 902 |
+
- Preserve a cheap /health endpoint that does not run full video generation. The wrapper uses it for boot validation.
|
| 903 |
+
- If full generation is too risky or requires unavailable inputs, keep the app bootable and clearly document what remains incomplete.
|
| 904 |
+
- Prefer ZeroGPU-compatible code when possible, but allow fixed GPU runtime. Preferred hardware: {preferred_hardware}. Fallback hardware enabled: {allow_fallback}. Fallback hardware: {fallback_hardware}.
|
| 905 |
+
- If you use @spaces.GPU, decorate only the inference/generation function. Do not decorate /health.
|
| 906 |
+
- Keep the huggingface_hub pin in requirements.txt: huggingface_hub>=0.34.0,<1.0.0.
|
| 907 |
+
|
| 908 |
+
Deliverables:
|
| 909 |
+
- app.py must boot on Hugging Face Spaces.
|
| 910 |
+
- app.py must expose /health.
|
| 911 |
+
- README.md must explain the runtime strategy and limitations.
|
| 912 |
+
- Write a concise PI_SUMMARY.md with what you changed and whether full generation is implemented.
|
| 913 |
+
"""
|
| 914 |
+
(workspace / "GOAL.md").write_text(goal, encoding="utf-8")
|
| 915 |
+
return ["app.py", "requirements.txt", "README.md", "GOAL.md"]
|
| 916 |
+
|
| 917 |
+
|
| 918 |
+
def upload_workspace(api, workspace: Path, target_space_id: str, token: str, run_dir: Path, events_path: Path):
|
| 919 |
+
append_event(events_path, "upload_files", "started", "Uploading generated LongCat files")
|
| 920 |
+
gen_dir = run_dir / "generated"
|
| 921 |
+
gen_dir.mkdir(parents=True, exist_ok=True)
|
| 922 |
+
for filename in ["app.py", "README.md", "requirements.txt", "GOAL.md", "PI_SUMMARY.md"]:
|
| 923 |
+
src = workspace / filename
|
| 924 |
+
if src.exists():
|
| 925 |
+
shutil.copy2(src, gen_dir / filename)
|
| 926 |
+
for filename in ["app.py", "README.md", "requirements.txt"]:
|
| 927 |
+
src = workspace / filename
|
| 928 |
+
if not src.exists():
|
| 929 |
+
raise RuntimeError(f"Missing required generated file: {filename}")
|
| 930 |
+
api.upload_file(path_or_fileobj=str(src), path_in_repo=filename, repo_id=target_space_id, repo_type="space", token=token)
|
| 931 |
+
append_event(events_path, "upload_files", "success", f"Uploaded {filename}")
|
| 932 |
+
|
| 933 |
+
|
| 934 |
+
def main():
|
| 935 |
+
run_id = os.environ["RUN_ID"]
|
| 936 |
+
hf_username = os.environ.get("HF_USERNAME", "unknown")
|
| 937 |
+
bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")
|
| 938 |
+
output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))
|
| 939 |
+
target_space_id = os.environ.get("TARGET_SPACE_ID", "")
|
| 940 |
+
model_id = sanitize_model_id(os.environ.get("MODEL_ID", DEFAULT_MODEL_ID))
|
| 941 |
+
pi_model = os.environ.get("PI_MODEL", "moonshotai/Kimi-K2.5")
|
| 942 |
+
preferred_hardware = os.environ.get("PREFERRED_SPACE_HARDWARE", "zero-a10g")
|
| 943 |
+
fallback_hardware = os.environ.get("FALLBACK_SPACE_HARDWARE", "l40sx1")
|
| 944 |
+
allow_fixed_gpu_fallback = os.environ.get("ALLOW_FIXED_GPU_FALLBACK", "true").lower() in {"1", "true", "yes", "on"}
|
| 945 |
+
token = os.environ.get("HF_TOKEN")
|
| 946 |
+
|
| 947 |
+
run_dir = output_root / "runs" / run_id
|
| 948 |
+
events_path = run_dir / "events.jsonl"
|
| 949 |
+
state_path = run_dir / "state.json"
|
| 950 |
+
workspace = Path("/tmp/longcat_workspace")
|
| 951 |
+
|
| 952 |
+
append_event(events_path, "bootstrap", "started", "LongCat article reproduction worker started", {"model_id": model_id, "target_space_id": target_space_id})
|
| 953 |
+
write_json(state_path, {"run_id": run_id, "kind": "longcat_article_reproduction", "status": "running", "message": "Attempting LongCat article-style Space creation", "model_id": model_id, "target_space": target_space_id, "created_by": hf_username, "bucket_source": bucket_source, "created_at": now(), "updated_at": now()})
|
| 954 |
+
if not token:
|
| 955 |
+
fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets")
|
| 956 |
+
if not TARGET_RE.match(target_space_id):
|
| 957 |
+
fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space_id": target_space_id})
|
| 958 |
+
|
| 959 |
+
try:
|
| 960 |
+
install_python_deps(events_path)
|
| 961 |
+
from huggingface_hub import HfApi
|
| 962 |
+
api = HfApi(token=token)
|
| 963 |
+
whoami = api.whoami(token=token)
|
| 964 |
+
append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})
|
| 965 |
+
|
| 966 |
+
append_event(events_path, "model_analysis", "started", "Fetching LongCat model metadata", {"model_id": model_id})
|
| 967 |
+
info = api.model_info(model_id, token=token, files_metadata=True)
|
| 968 |
+
siblings = [getattr(s, "rfilename", "") for s in (info.siblings or [])]
|
| 969 |
+
analysis = {"model_id": model_id, "pipeline_tag": getattr(info, "pipeline_tag", None), "library_name": getattr(info, "library_name", None), "tags": list(getattr(info, "tags", []) or [])[:100], "siblings": siblings[:160], "article_target": model_id == DEFAULT_MODEL_ID, "preferred_hardware": preferred_hardware, "fallback_hardware": fallback_hardware, "allow_fixed_gpu_fallback": allow_fixed_gpu_fallback}
|
| 970 |
+
write_json(run_dir / "model_analysis.json", analysis)
|
| 971 |
+
append_event(events_path, "model_analysis", "success", "Model metadata fetched", {"pipeline_tag": analysis["pipeline_tag"], "library_name": analysis["library_name"]})
|
| 972 |
+
|
| 973 |
+
create_initial_workspace(workspace, model_id, target_space_id, preferred_hardware, fallback_hardware, allow_fixed_gpu_fallback)
|
| 974 |
+
append_event(events_path, "workspace", "success", "Prepared LongCat workspace", {"files": sorted(p.name for p in workspace.iterdir())})
|
| 975 |
+
|
| 976 |
+
install_pi(events_path)
|
| 977 |
+
configure_pi(events_path, pi_model)
|
| 978 |
+
append_event(events_path, "pi_run", "started", "Running Pi on LongCat workspace", {"model": pi_model})
|
| 979 |
+
code, pi_out = run_cmd(["pi", "-p", (workspace / "GOAL.md").read_text(encoding="utf-8")], cwd=workspace, timeout=2400)
|
| 980 |
+
(run_dir / "logs").mkdir(parents=True, exist_ok=True)
|
| 981 |
+
(run_dir / "logs" / "pi_output.txt").write_text(pi_out, encoding="utf-8")
|
| 982 |
+
if code != 0:
|
| 983 |
+
append_event(events_path, "pi_run", "failed", "Pi returned a non-zero exit code", {"returncode": code, "output_tail": pi_out[-4000:]})
|
| 984 |
+
collect_pi_traces(run_dir, events_path)
|
| 985 |
+
fail(run_dir, events_path, "Pi failed before Space upload", {"returncode": code, "output_tail": pi_out[-4000:]})
|
| 986 |
+
append_event(events_path, "pi_run", "success", "Pi completed LongCat workspace pass", {"output_tail": pi_out[-2000:]})
|
| 987 |
+
if not (workspace / "PI_SUMMARY.md").exists():
|
| 988 |
+
(workspace / "PI_SUMMARY.md").write_text("# Pi Summary\n\nPi did not create a PI_SUMMARY.md. See logs/pi_output.txt.\n", encoding="utf-8")
|
| 989 |
+
|
| 990 |
+
app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore")
|
| 991 |
+
if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text:
|
| 992 |
+
append_event(events_path, "pi_verification", "failed", "app.py does not appear to expose /health; injecting safe health endpoint is not implemented")
|
| 993 |
+
fail(run_dir, events_path, "Pi output did not preserve a /health endpoint")
|
| 994 |
+
append_event(events_path, "pi_verification", "success", "Pi output preserved health validation endpoint")
|
| 995 |
+
|
| 996 |
+
append_event(events_path, "create_space", "started", "Creating private LongCat target Space", {"target_space": target_space_id})
|
| 997 |
+
api.create_repo(repo_id=target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, token=token)
|
| 998 |
+
append_event(events_path, "create_space", "success", "Private target Space created", {"target_space": target_space_id})
|
| 999 |
+
|
| 1000 |
+
hardware_attempts = []
|
| 1001 |
+
preferred = request_hardware(api, target_space_id, preferred_hardware, token, events_path, "hardware_preferred")
|
| 1002 |
+
hardware_attempts.append(preferred)
|
| 1003 |
+
selected_hardware = preferred_hardware if preferred.get("ok") else None
|
| 1004 |
+
if not selected_hardware and allow_fixed_gpu_fallback and fallback_hardware:
|
| 1005 |
+
fallback = request_hardware(api, target_space_id, fallback_hardware, token, events_path, "hardware_fallback")
|
| 1006 |
+
hardware_attempts.append(fallback)
|
| 1007 |
+
selected_hardware = fallback_hardware if fallback.get("ok") else None
|
| 1008 |
+
if not selected_hardware:
|
| 1009 |
+
append_event(events_path, "hardware", "warning", "Could not request preferred/fallback hardware; Space may remain on default CPU", {"attempts": hardware_attempts})
|
| 1010 |
+
selected_hardware = "default-cpu-or-existing"
|
| 1011 |
+
write_json(run_dir / "hardware_attempts.json", {"selected_hardware": selected_hardware, "attempts": hardware_attempts})
|
| 1012 |
+
|
| 1013 |
+
upload_workspace(api, workspace, target_space_id, token, run_dir, events_path)
|
| 1014 |
+
validation = validate_live_api(target_space_id, token, run_dir, events_path, timeout_s=1200)
|
| 1015 |
+
collect_pi_traces(run_dir, events_path)
|
| 1016 |
+
|
| 1017 |
+
final_state = {"run_id": run_id, "kind": "longcat_article_reproduction", "status": "success", "message": "LongCat article-style Space created and boot/API health validated", "model_id": model_id, "target_space": target_space_id, "target_space_url": f"https://huggingface.co/spaces/{target_space_id}", "selected_hardware": selected_hardware, "hardware_attempts": hardware_attempts, "validation": validation, "updated_at": now(), "created_by": hf_username, "bucket_source": bucket_source}
|
| 1018 |
+
write_json(state_path, final_state)
|
| 1019 |
+
report = f"""# Agentic Space Factory — LongCat Article Reproduction Report
|
| 1020 |
+
|
| 1021 |
+
Run ID: `{run_id}`
|
| 1022 |
+
|
| 1023 |
+
Status: **success**
|
| 1024 |
+
|
| 1025 |
+
Target Space: https://huggingface.co/spaces/{target_space_id}
|
| 1026 |
+
|
| 1027 |
+
Model: `{model_id}`
|
| 1028 |
+
|
| 1029 |
+
## Hardware
|
| 1030 |
+
|
| 1031 |
+
Selected/requested hardware: `{selected_hardware}`
|
| 1032 |
+
|
| 1033 |
+
```json
|
| 1034 |
+
{json.dumps(hardware_attempts, indent=2, ensure_ascii=False)}
|
| 1035 |
+
```
|
| 1036 |
+
|
| 1037 |
+
## Validation
|
| 1038 |
+
|
| 1039 |
+
The wrapper validated the live Gradio API. This Phase 7 validation is intentionally a **boot/health API validation** first. Full video generation may still require manual review, real inputs, quota, and model-specific optimization.
|
| 1040 |
+
|
| 1041 |
+
```json
|
| 1042 |
+
{json.dumps(validation, indent=2, ensure_ascii=False)}
|
| 1043 |
+
```
|
| 1044 |
+
|
| 1045 |
+
## Pi summary
|
| 1046 |
+
|
| 1047 |
+
{(workspace / 'PI_SUMMARY.md').read_text(encoding='utf-8', errors='ignore') if (workspace / 'PI_SUMMARY.md').exists() else 'No PI_SUMMARY.md was produced.'}
|
| 1048 |
+
|
| 1049 |
+
## Safety
|
| 1050 |
+
|
| 1051 |
+
- The target Space was created private.
|
| 1052 |
+
- No public publication was attempted.
|
| 1053 |
+
- Raw traces should remain private; redacted traces are stored separately.
|
| 1054 |
+
- If fallback fixed GPU was used, review billing/hardware settings manually after the run.
|
| 1055 |
+
"""
|
| 1056 |
+
(run_dir / "report.md").write_text(report, encoding="utf-8")
|
| 1057 |
+
append_event(events_path, "report_write", "success", "Wrote report.md")
|
| 1058 |
+
append_event(events_path, "done", "success", "LongCat article reproduction worker completed", {"target_space": target_space_id, "selected_hardware": selected_hardware})
|
| 1059 |
+
except SystemExit:
|
| 1060 |
+
raise
|
| 1061 |
+
except Exception as exc:
|
| 1062 |
+
try:
|
| 1063 |
+
collect_pi_traces(run_dir, events_path)
|
| 1064 |
+
except Exception:
|
| 1065 |
+
pass
|
| 1066 |
+
fail(run_dir, events_path, "LongCat article reproduction worker failed", {"error": str(exc)})
|
| 1067 |
+
|
| 1068 |
+
|
| 1069 |
+
if __name__ == "__main__":
|
| 1070 |
+
main()
|
| 1071 |
+
'''
|
| 1072 |
+
|
| 1073 |
+
|
| 1074 |
def encoded_worker_script() -> str:
|
| 1075 |
"""Return the base64-encoded Phase 1 hello worker script."""
|
| 1076 |
return _encode(HELLO_WORKER_SCRIPT)
|
|
|
|
| 1096 |
return _encode(PI_MODEL_CARD_WORKER_SCRIPT)
|
| 1097 |
|
| 1098 |
|
| 1099 |
+
|
| 1100 |
+
def encoded_runtime_recommender_worker_script() -> str:
|
| 1101 |
+
"""Return the base64-encoded Phase 6 runtime recommender worker script."""
|
| 1102 |
+
return _encode(RUNTIME_RECOMMENDER_WORKER_SCRIPT)
|
| 1103 |
+
|
| 1104 |
+
|
| 1105 |
+
def encoded_longcat_article_worker_script() -> str:
|
| 1106 |
+
"""Return the base64-encoded Phase 7 LongCat article reproduction worker script."""
|
| 1107 |
+
return _encode(LONGCAT_ARTICLE_WORKER_SCRIPT)
|
| 1108 |
+
|
| 1109 |
def python_decode_and_run_command() -> list[str]:
|
| 1110 |
"""Command list for `run_job`.
|
| 1111 |
|