Upload 6 files

src/jobs.py

@@ -9,6 +9,7 @@ from .config import settings
 from .runs import make_run_id, utc_now_iso, validate_run_id
 from .worker_payload import (
     encoded_create_space_worker_script,
+    encoded_pi_space_worker_script,
     encoded_worker_script,
     python_decode_and_run_command,
 )
@@ -110,6 +111,40 @@ def launch_create_private_space_job(
     )
 
 
+def launch_pi_space_smoke_job(
+    *,
+    token: str,
+    username: str,
+    target_slug: str | None = None,
+    pi_model: str | None = None,
+    run_id: str | None = None,
+) -> dict[str, Any]:
+    """Launch the Phase 3 Job: install Pi, let Pi modify a minimal app, then create/validate a private Space."""
+    if not token:
+        raise ValueError("Missing OAuth token. Please sign in with Hugging Face first.")
+    safe_run_id = validate_run_id(run_id) if run_id else make_run_id("pi")
+    target_space_id = normalize_target_space(username=username, target_slug=target_slug, run_id=safe_run_id)
+
+    env = _base_env(
+        run_id=safe_run_id,
+        username=username,
+        worker_script_b64=encoded_pi_space_worker_script(),
+    )
+    env["TARGET_SPACE_ID"] = target_space_id
+    env["PI_MODEL"] = (pi_model or "moonshotai/Kimi-K2.5").strip()
+    job = _launch_job(token=token, env=env)
+    return _job_result(
+        job,
+        run_id=safe_run_id,
+        kind="pi_space_smoke",
+        extra={
+            "target_space": target_space_id,
+            "target_space_url": f"https://huggingface.co/spaces/{target_space_id}",
+            "pi_model": env["PI_MODEL"],
+        },
+    )
+
+
 def inspect_job_safe(job_id: str, token: str | None = None) -> dict[str, Any]:
     if not job_id:
         return {"error": "Missing job_id"}

src/worker_payload.py

@@ -555,6 +555,9 @@ if __name__ == "__main__":
 '''
 
 
+
+PI_SPACE_WORKER_SCRIPT = 'import json\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\nimport time\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom textwrap import dedent\n\n\nTARGET_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,95}/[A-Za-z0-9][A-Za-z0-9._-]{1,95}$")\n\n\ndef now():\n    return datetime.now(timezone.utc).isoformat()\n\n\ndef write_json(path: Path, payload: dict):\n    path.parent.mkdir(parents=True, exist_ok=True)\n    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\\n", encoding="utf-8")\n\n\ndef append_event(path: Path, step: str, status: str, message: str, data: dict | None = None):\n    path.parent.mkdir(parents=True, exist_ok=True)\n    event = {"ts": now(), "step": step, "status": status, "message": message, "data": data or {}}\n    line = json.dumps(event, ensure_ascii=False)\n    with path.open("a", encoding="utf-8") as f:\n        f.write(line + "\\n")\n    print(line, flush=True)\n\n\ndef redact_text(text: str | None) -> str:\n    if not text:\n        return ""\n    value = text\n    for secret_name in ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"]:\n        secret = os.environ.get(secret_name)\n        if secret:\n            value = value.replace(secret, "[REDACTED]")\n    value = re.sub(r"Bearer\\s+[A-Za-z0-9_\\-.=]+", "Bearer [REDACTED]", value)\n    value = re.sub(r"hf_[A-Za-z0-9_\\-]{10,}", "hf_[REDACTED]", value)\n    return value\n\n\ndef safe_details(details: dict | None) -> dict:\n    if not details:\n        return {}\n    try:\n        return json.loads(redact_text(json.dumps(details, ensure_ascii=False)))\n    except Exception:\n        return {"redacted_details": redact_text(str(details))[-4000:]}\n\n\ndef fail(run_dir: Path, events_path: Path, message: str, details: dict | None = None, status: str = "failed"):\n    safe = safe_details(details)\n    append_event(events_path, "failure", "failed", message, safe)\n    write_json(run_dir / "state.json", {\n        "run_id": os.environ.get("RUN_ID"),\n        "kind": "pi_space_smoke",\n        "status": status,\n        "message": message,\n        "updated_at": now(),\n        "details": safe,\n    })\n    report = f"""# Agentic Space Factory — Pi Smoke Test Report\n\nStatus: **{status}**\n\n{message}\n\n```json\n{json.dumps(safe, indent=2, ensure_ascii=False)}\n```\n"""\n    (run_dir / "report.md").write_text(report, encoding="utf-8")\n    raise SystemExit(1)\n\n\ndef run_cmd(cmd: list[str], *, cwd: Path | None = None, env: dict | None = None, timeout: int = 600):\n    result = subprocess.run(\n        cmd,\n        cwd=str(cwd) if cwd else None,\n        env=env,\n        text=True,\n        stdout=subprocess.PIPE,\n        stderr=subprocess.STDOUT,\n        timeout=timeout,\n    )\n    return result.returncode, redact_text(result.stdout)\n\n\ndef install_python_deps(events_path: Path):\n    append_event(events_path, "dependencies", "started", "Installing Python worker dependencies")\n    code, out = run_cmd([sys.executable, "-m", "pip", "install", "-q", "--upgrade", "huggingface_hub>=1.0.0", "gradio_client>=2.0.0"], timeout=600)\n    if code != 0:\n        append_event(events_path, "dependencies", "failed", "Python dependency installation failed", {"output_tail": out[-4000:]})\n        raise RuntimeError(out)\n    append_event(events_path, "dependencies", "success", "Python worker dependencies installed")\n\n\ndef ensure_node(events_path: Path):\n    node = shutil.which("node")\n    npm = shutil.which("npm")\n    if node and npm:\n        _, node_v = run_cmd([node, "--version"], timeout=30)\n        _, npm_v = run_cmd([npm, "--version"], timeout=30)\n        append_event(events_path, "node", "success", "Node/npm already available", {"node": node_v.strip(), "npm": npm_v.strip()})\n        return\n    append_event(events_path, "node", "started", "Installing nodejs/npm through apt-get")\n    code, out = run_cmd(["bash", "-lc", "apt-get update -qq && apt-get install -y -qq nodejs npm"], timeout=600)\n    if code != 0:\n        append_event(events_path, "node", "failed", "Could not install nodejs/npm", {"output_tail": out[-4000:]})\n        raise RuntimeError(out)\n    append_event(events_path, "node", "success", "Installed nodejs/npm")\n\n\ndef install_pi(events_path: Path):\n    ensure_node(events_path)\n    append_event(events_path, "pi_install", "started", "Installing Pi coding agent from npm")\n    code, out = run_cmd(["npm", "install", "-g", "@mariozechner/pi-coding-agent"], timeout=900)\n    if code != 0:\n        append_event(events_path, "pi_install", "failed", "Pi npm installation failed", {"output_tail": out[-4000:]})\n        raise RuntimeError(out)\n    code, version = run_cmd(["pi", "--version"], timeout=60)\n    append_event(events_path, "pi_install", "success", "Pi installed", {"version_output": version.strip()[-300:]})\n\n\ndef configure_pi(events_path: Path, model: str):\n    pi_dir = Path.home() / ".pi" / "agent"\n    pi_dir.mkdir(parents=True, exist_ok=True)\n    (pi_dir / "auth.json").write_text(json.dumps({"huggingface": {"type": "api_key", "key": os.environ.get("HF_TOKEN", "")}}, indent=2), encoding="utf-8")\n    (pi_dir / "settings.json").write_text(json.dumps({"defaultProvider": "huggingface", "defaultModel": model}, indent=2), encoding="utf-8")\n    append_event(events_path, "pi_config", "success", "Configured Pi for Hugging Face Inference Providers", {"model": model})\n\n\ndef initial_files(target_space_id: str) -> dict[str, str]:\n    app_py = dedent(\'\'\'\n    import gradio as gr\n\n\n    def greet(name: str) -> str:\n        name = (name or "friend").strip() or "friend"\n        return f"Hello {name} — this private Space was generated by Agentic Space Factory."\n\n\n    demo = gr.Interface(\n        fn=greet,\n        inputs=gr.Textbox(label="Name", value="Hugging Face"),\n        outputs=gr.Textbox(label="Result"),\n        title="Generated private Space",\n        description="A minimal Gradio Space created by an HF Job. Phase 3 asks Pi to modify it safely before upload.",\n        examples=[["Hugging Face"], ["Agentic Space Factory"]],\n    )\n\n\n    if __name__ == "__main__":\n        demo.launch()\n    \'\'\').strip() + "\\n"\n    readme = dedent(f\'\'\'\n    ---\n    title: Pi Modified Private Space\n    emoji: 🧪\n    colorFrom: green\n    colorTo: blue\n    sdk: gradio\n    app_file: app.py\n    python_version: "3.11"\n    pinned: false\n    ---\n\n    # Pi Modified Private Space\n\n    This private Space was generated by **Agentic Space Factory**.\n\n    Target repo: `{target_space_id}`\n\n    Phase 3 asks Pi to make a small, safe modification before the Space is uploaded and validated through the live Gradio API.\n    \'\'\').strip() + "\\n"\n    return {"app.py": app_py, "README.md": readme, "requirements.txt": "gradio>=5.0.0\\n"}\n\n\ndef write_workspace(workspace: Path, files: dict[str, str]):\n    workspace.mkdir(parents=True, exist_ok=True)\n    for name, content in files.items():\n        (workspace / name).write_text(content, encoding="utf-8")\n\n\ndef run_pi(events_path: Path, workspace: Path, target_space_id: str, model: str, run_dir: Path):\n    goal = f"""\nYou are running inside an ephemeral Hugging Face Job.\n\nTask:\nModify the minimal Gradio app in this directory to prove that Pi can safely edit project files before they are uploaded to a private Hugging Face Space.\n\nHard constraints:\n- Only edit app.py and README.md if needed.\n- Do not read, print, or store secrets or environment variables.\n- Preserve the Gradio interface and the greet(name: str) function.\n- The live API validation expects the output for input "Agentic Space Factory" to contain "Hello" and "Agentic Space Factory".\n- Add the exact phrase "Pi modified this app" to the returned greeting string.\n- Keep the app simple and reliable.\n- Do not create, modify, or delete Hugging Face resources.\n\nWhen done, stop. No extra explanation is required.\n\nTarget Space later: {target_space_id}\nModel configured for Pi: {model}\n""".strip()\n    (workspace / "GOAL.md").write_text(goal + "\\n", encoding="utf-8")\n    append_event(events_path, "pi_run", "started", "Running Pi in non-interactive print mode", {"model": model})\n    env = os.environ.copy()\n    env["HF_TOKEN"] = os.environ.get("HF_TOKEN", "")\n    code, out = run_cmd(["pi", "-p", goal], cwd=workspace, env=env, timeout=600)\n    (run_dir / "logs").mkdir(parents=True, exist_ok=True)\n    (run_dir / "logs" / "pi_output.txt").write_text(out, encoding="utf-8")\n    if code != 0:\n        append_event(events_path, "pi_run", "failed", "Pi exited with a non-zero status", {"exit_code": code, "output_tail": out[-4000:]})\n        raise RuntimeError(f"Pi failed with exit code {code}: {out[-2000:]}")\n    append_event(events_path, "pi_run", "success", "Pi completed", {"output_tail": out[-1000:]})\n\n\ndef collect_pi_traces(events_path: Path, run_dir: Path):\n    trace_root = Path.home() / ".pi" / "agent" / "sessions"\n    raw_dir = run_dir / "traces" / "raw"\n    redacted_dir = run_dir / "traces" / "redacted"\n    raw_dir.mkdir(parents=True, exist_ok=True)\n    redacted_dir.mkdir(parents=True, exist_ok=True)\n    copied = []\n    if trace_root.exists():\n        for path in trace_root.rglob("*.jsonl"):\n            rel = path.relative_to(trace_root)\n            raw_target = raw_dir / rel\n            red_target = redacted_dir / rel\n            raw_target.parent.mkdir(parents=True, exist_ok=True)\n            red_target.parent.mkdir(parents=True, exist_ok=True)\n            text = path.read_text(encoding="utf-8", errors="replace")\n            redacted = redact_text(text)\n            raw_target.write_text(redacted, encoding="utf-8")\n            red_target.write_text(redacted, encoding="utf-8")\n            copied.append(str(rel))\n    append_event(events_path, "pi_traces", "success", "Collected Pi traces", {"count": len(copied), "files": copied[:10]})\n    return copied\n\n\ndef assert_pi_modified(workspace: Path):\n    app_text = (workspace / "app.py").read_text(encoding="utf-8")\n    if "Pi modified this app" not in app_text:\n        raise RuntimeError("Pi did not add the required marker phrase to app.py")\n    if "def greet" not in app_text or "gr.Interface" not in app_text:\n        raise RuntimeError("Pi modification appears to have broken the minimal Gradio app")\n\n\ndef save_generated_files(run_dir: Path, workspace: Path):\n    generated_dir = run_dir / "generated"\n    generated_dir.mkdir(parents=True, exist_ok=True)\n    for filename in ["app.py", "README.md", "requirements.txt", "GOAL.md"]:\n        src = workspace / filename\n        if src.exists():\n            (generated_dir / filename).write_text(src.read_text(encoding="utf-8"), encoding="utf-8")\n\n\ndef create_and_upload_space(api, token: str, target_space_id: str, workspace: Path, events_path: Path):\n    append_event(events_path, "create_space", "started", f"Creating private target Space {target_space_id}")\n    api.create_repo(repo_id=target_space_id, repo_type="space", space_sdk="gradio", private=True, exist_ok=False, token=token)\n    append_event(events_path, "create_space", "success", "Private target Space created", {"target_space": target_space_id})\n    append_event(events_path, "upload_files", "started", "Uploading Pi-modified files to target Space")\n    for path_in_repo in ["app.py", "README.md", "requirements.txt"]:\n        content = (workspace / path_in_repo).read_bytes()\n        api.upload_file(path_or_fileobj=content, path_in_repo=path_in_repo, repo_id=target_space_id, repo_type="space", token=token)\n        append_event(events_path, "upload_files", "success", f"Uploaded {path_in_repo}")\n\n\ndef make_gradio_client(target_space_id: str, token: str):\n    import inspect\n    from gradio_client import Client\n    params = inspect.signature(Client).parameters\n    if "token" in params:\n        return Client(target_space_id, token=token)\n    if "hf_token" in params:\n        return Client(target_space_id, hf_token=token)\n    if "api_key" in params:\n        return Client(target_space_id, api_key=token)\n    if "headers" in params:\n        return Client(target_space_id, headers={"Authorization": f"Bearer {token}"})\n    return Client(target_space_id)\n\n\ndef get_api_schema(client):\n    try:\n        return client.view_api(return_format="dict")\n    except TypeError:\n        return client.view_api()\n\n\ndef extract_api_names(api_schema) -> list[str]:\n    names = []\n    def add(value):\n        if not value or not isinstance(value, str):\n            return\n        name = value if value.startswith("/") else f"/{value}"\n        if name not in names:\n            names.append(name)\n    def walk(obj):\n        if isinstance(obj, dict):\n            for key, value in obj.items():\n                if key in {"api_name", "apiName"}:\n                    add(value)\n                if isinstance(key, str) and key.startswith("/"):\n                    add(key)\n                walk(value)\n        elif isinstance(obj, list):\n            for item in obj:\n                walk(item)\n    walk(api_schema)\n    return names\n\n\ndef predict_with_available_endpoint(client, api_schema, value: str):\n    candidates = extract_api_names(api_schema)\n    for fallback in ["/greet", "/predict"]:\n        if fallback not in candidates:\n            candidates.append(fallback)\n    errors = []\n    for api_name in candidates:\n        try:\n            return api_name, client.predict(value, api_name=api_name)\n        except Exception as exc:\n            errors.append({"api_name": api_name, "error": str(exc)[-500:]})\n    try:\n        return None, client.predict(value)\n    except Exception as exc:\n        errors.append({"api_name": None, "error": str(exc)[-500:]})\n        raise RuntimeError(f"No candidate Gradio endpoint worked: {json.dumps(errors, ensure_ascii=False)}")\n\n\ndef validate_live_api(target_space_id: str, token: str, events_path: Path, tests_dir: Path, timeout_seconds: int = 420):\n    tests_dir.mkdir(parents=True, exist_ok=True)\n    deadline = time.time() + timeout_seconds\n    last_error = None\n    attempt = 0\n    append_event(events_path, "api_validation", "started", "Waiting for live Gradio API to become available")\n    while time.time() < deadline:\n        attempt += 1\n        try:\n            client = make_gradio_client(target_space_id, token)\n            api_schema = get_api_schema(client)\n            api_names = extract_api_names(api_schema)\n            write_json(tests_dir / "api_schema.json", {"schema": api_schema, "api_names": api_names})\n            used_api_name, result = predict_with_available_endpoint(client, api_schema, "Agentic Space Factory")\n            result_text = str(result)\n            ok = "Hello" in result_text and "Agentic Space Factory" in result_text and "Pi modified this app" in result_text\n            payload = {\n                "attempt": attempt,\n                "target_space": target_space_id,\n                "api_test_passed": ok,\n                "api_name": used_api_name,\n                "discovered_api_names": api_names,\n                "result": result_text,\n                "validated_at": now(),\n            }\n            write_json(tests_dir / "test_result.json", payload)\n            if ok:\n                append_event(events_path, "api_validation", "success", "Live Gradio API test passed", {"attempt": attempt, "api_name": used_api_name, "discovered_api_names": api_names})\n                return payload\n            last_error = f"Unexpected API result from {used_api_name}: {result_text}"\n        except Exception as exc:\n            last_error = str(exc)\n            append_event(events_path, "api_validation", "waiting", "Live API not ready yet", {"attempt": attempt, "error": last_error[-1000:]})\n        time.sleep(20)\n    payload = {"target_space": target_space_id, "api_test_passed": False, "error": last_error, "validated_at": now()}\n    write_json(tests_dir / "test_result.json", payload)\n    raise RuntimeError(f"Live API validation did not pass before timeout: {last_error}")\n\n\ndef main():\n    run_id = os.environ["RUN_ID"]\n    hf_username = os.environ.get("HF_USERNAME", "unknown")\n    bucket_source = os.environ.get("BUCKET_SOURCE", "unknown")\n    output_root = Path(os.environ.get("OUTPUT_ROOT", "/output"))\n    target_space_id = os.environ["TARGET_SPACE_ID"]\n    token = os.environ.get("HF_TOKEN")\n    pi_model = os.environ.get("PI_MODEL") or "moonshotai/Kimi-K2.5"\n    run_dir = output_root / "runs" / run_id\n    events_path = run_dir / "events.jsonl"\n    state_path = run_dir / "state.json"\n    report_path = run_dir / "report.md"\n    target_json_path = run_dir / "target_space.json"\n    workspace = Path("/tmp/space_factory_pi_workspace")\n\n    append_event(events_path, "bootstrap", "started", "Pi Space smoke worker started")\n    write_json(state_path, {\n        "run_id": run_id,\n        "kind": "pi_space_smoke",\n        "status": "running",\n        "message": "Running Pi smoke test before creating private target Space",\n        "target_space": target_space_id,\n        "pi_model": pi_model,\n        "created_by": hf_username,\n        "bucket_source": bucket_source,\n        "created_at": now(),\n        "updated_at": now(),\n    })\n    if not token:\n        fail(run_dir, events_path, "HF_TOKEN is missing from Job secrets")\n    if not TARGET_RE.match(target_space_id):\n        fail(run_dir, events_path, "Invalid TARGET_SPACE_ID", {"target_space": target_space_id})\n    if not target_space_id.startswith(f"{hf_username}/"):\n        fail(run_dir, events_path, "For Phase 3, target Space must be in the signed-in user\'s namespace", {"target_space": target_space_id, "username": hf_username})\n\n    try:\n        install_python_deps(events_path)\n        install_pi(events_path)\n        configure_pi(events_path, pi_model)\n        files = initial_files(target_space_id)\n        write_workspace(workspace, files)\n        append_event(events_path, "workspace", "success", "Prepared local workspace for Pi", {"files": list(files)})\n        run_pi(events_path, workspace, target_space_id, pi_model, run_dir)\n        trace_files = collect_pi_traces(events_path, run_dir)\n        assert_pi_modified(workspace)\n        append_event(events_path, "pi_validation", "success", "Pi made the required safe modification")\n        save_generated_files(run_dir, workspace)\n\n        from huggingface_hub import HfApi\n        api = HfApi(token=token)\n        whoami = api.whoami(token=token)\n        append_event(events_path, "auth", "success", "Authenticated inside Job", {"whoami_name": whoami.get("name")})\n        create_and_upload_space(api, token, target_space_id, workspace, events_path)\n        write_json(target_json_path, {"target_space": target_space_id, "url": f"https://huggingface.co/spaces/{target_space_id}", "private": True, "sdk": "gradio", "created_by": hf_username})\n        validation = validate_live_api(target_space_id, token, events_path, run_dir / "tests")\n        trace_files = collect_pi_traces(events_path, run_dir)\n        final_state = {\n            "run_id": run_id,\n            "kind": "pi_space_smoke",\n            "status": "success",\n            "message": "Pi modified a Gradio app, the private Space was created, and the live API validation passed.",\n            "target_space": target_space_id,\n            "target_space_url": f"https://huggingface.co/spaces/{target_space_id}",\n            "pi_model": pi_model,\n            "created_by": hf_username,\n            "bucket_source": bucket_source,\n            "validation": validation,\n            "trace_files": trace_files,\n            "updated_at": now(),\n            "security_notes": [\n                "The target Space was created as private.",\n                "The HF token was redacted from saved logs/traces.",\n                "Success was declared only after a live Gradio API test passed.",\n            ],\n        }\n        write_json(state_path, final_state)\n        report = f"""# Agentic Space Factory — Pi Smoke Test Report\n\nRun ID: `{run_id}`\n\nStatus: **success**\n\nCreated private Space: [`{target_space_id}`](https://huggingface.co/spaces/{target_space_id})\n\n## What happened\n\n```text\nOAuth user → HF Job → install Pi → Pi edits app.py �� private Space creation → file upload → live Gradio API validation → Bucket report\n```\n\n## Pi\n\n- Model: `{pi_model}`\n- Pi traces copied: `{len(trace_files)}`\n- Trace paths: `runs/{run_id}/traces/`\n\n## Live API validation\n\n```json\n{json.dumps(validation, indent=2, ensure_ascii=False)}\n```\n\n## Security posture\n\n- The target Space was created as private.\n- The token was not intentionally printed or persisted.\n- Saved Pi outputs/traces are redacted on a best-effort basis.\n- Success was declared only after the live Gradio API returned the expected Pi-modified output.\n\n## Next step\n\nPhase 4 should ask Pi to apply the HF Spaces Agent Quickstart gist against a generated Space, still with strict private-by-default and live API validation gates.\n"""\n        report_path.write_text(report, encoding="utf-8")\n        append_event(events_path, "report_write", "success", "Wrote report.md")\n        append_event(events_path, "done", "success", "Pi Space smoke worker completed")\n    except Exception as exc:\n        try:\n            collect_pi_traces(events_path, run_dir)\n        except Exception:\n            pass\n        fail(run_dir, events_path, "Pi Space smoke worker failed", {"error": str(exc)})\n\n\nif __name__ == "__main__":\n    main()\n'
+
 def _encode(script: str) -> str:
     return base64.b64encode(script.encode("utf-8")).decode("ascii")
 
@@ -569,6 +572,11 @@ def encoded_create_space_worker_script() -> str:
     return _encode(CREATE_SPACE_WORKER_SCRIPT)
 
 
+def encoded_pi_space_worker_script() -> str:
+    """Return the base64-encoded Phase 3 Pi smoke worker script."""
+    return _encode(PI_SPACE_WORKER_SCRIPT)
+
+
 def python_decode_and_run_command() -> list[str]:
     """Command list for `run_job`.