Upload 14 files

src/bucket.py

@@ -678,6 +678,7 @@ def _run_document_links(run_id: str, *, bucket_source: str, bundle: dict[str, An
     eval_publish_payload = bundle.get("eval_publish") if isinstance(bundle.get("eval_publish"), dict) else bundle.get("eval_publish_status") if isinstance(bundle.get("eval_publish_status"), dict) else {}
     eval_publish_present = bool(eval_publish_payload) or manifest_has("eval_publish_status.json") or "eval_publish_status.json" in file_paths or (bool(token) and _path_exists(f"{paths.root}/eval_publish_status.json", token=token))
     space_logs_index_present = manifest_has("logs/space_logs_index.json") or "logs/space_logs_index.json" in file_paths or (bool(token) and _path_exists(f"{paths.root}/logs/space_logs_index.json", token=token))
+    build_error_present = manifest_has("build_error_observation.json") or "build_error_observation.json" in file_paths or (bool(token) and _path_exists(f"{paths.root}/build_error_observation.json", token=token))
     repair_decision_present = manifest_has("repair/REPAIR_DECISION.json") or "repair/REPAIR_DECISION.json" in file_paths or (bool(token) and _path_exists(f"{paths.root}/repair/REPAIR_DECISION.json", token=token))
     blockage_present = manifest_has("repair/BLOCKAGE.json") or "repair/BLOCKAGE.json" in file_paths or (bool(token) and _path_exists(f"{paths.root}/repair/BLOCKAGE.json", token=token))
     repair_present = manifest_has("repair") or any(path.startswith("repair/") for path in file_paths) or (bool(token) and _path_exists(f"{paths.root}/repair/REPAIR_SUMMARY.md", token=token))
@@ -726,7 +727,7 @@ def _run_document_links(run_id: str, *, bucket_source: str, bundle: dict[str, An
             "icon": "◎",
             "present": eval_publish_present,
             "url": _bucket_file_url(bucket_source, run_id, "eval_publish_status.json"),
-            "tone": "warn" if eval_publish_present and not eval_publish_payload.get("published") else "neutral",
+            "tone": "success" if eval_publish_payload.get("published") else "pending" if eval_publish_present and str(eval_publish_payload.get("reason") or "") == "record_not_ready" else "warn" if eval_publish_present else "neutral",
         },
         {
             "id": "space_logs",
@@ -735,7 +736,16 @@ def _run_document_links(run_id: str, *, bucket_source: str, bundle: dict[str, An
             "icon": "▣",
             "present": space_logs_index_present,
             "url": _bucket_file_url(bucket_source, run_id, "logs/space_logs_index.json"),
-            "tone": "neutral",
+            "tone": "warn" if "failed" in status or "error" in status else "neutral",
+        },
+        {
+            "id": "build_error",
+            "label": "Build error",
+            "subtitle": "Observed build failure",
+            "icon": "⛔",
+            "present": build_error_present,
+            "url": _bucket_file_url(bucket_source, run_id, "build_error_observation.json"),
+            "tone": "warn",
         },
     ]
     if repair_decision_present:
@@ -786,6 +796,9 @@ def _run_document_links(run_id: str, *, bucket_source: str, bundle: dict[str, An
                 "tone": "warn",
             }
         )
+    if "failed" in status or "error" in status:
+        priority = {"report": 0, "blockage": 1, "build_error": 2, "space_logs": 3, "repair_decision": 4, "repair": 5, "blockers": 6, "pi_raw_trace": 7, "pi_redacted_trace": 8, "smoke": 9, "eval_publish": 10}
+        docs.sort(key=lambda item: priority.get(str(item.get("id") or ""), 50))
     return docs
 
 
@@ -938,6 +951,7 @@ def read_run_bundle(run_id: str, *, bucket_source: str, token: str | None = None
         "space_runtime": _safe_read_json(f"{paths.root}/space_runtime.json", token=token),
         "api_schema": _safe_read_json(f"{paths.root}/tests/api_schema.json", token=token) or _safe_read_json(f"{paths.root}/tests/generation_api_schema.json", token=token),
         "eval_publish_status": _safe_read_json(f"{paths.root}/eval_publish_status.json", token=token),
+        "build_error_observation": _safe_read_json(f"{paths.root}/build_error_observation.json", token=token),
         "repair_decision": _safe_read_json(f"{paths.root}/repair/REPAIR_DECISION.json", token=token),
         "blockage": _safe_read_json(f"{paths.root}/repair/BLOCKAGE.json", token=token),
         "artifact_manifest": _safe_read_json(f"{paths.root}/artifact_manifest.json", token=token),

src/timeline_model.py

@@ -144,6 +144,44 @@ def _eval_publish(bundle: dict[str, Any]) -> dict[str, Any]:
     return status if isinstance(status, dict) else {}
 
 
+def _build_error_observation(bundle: dict[str, Any]) -> dict[str, Any]:
+    observation = bundle.get("build_error_observation") or {}
+    return observation if isinstance(observation, dict) else {}
+
+
+def _final_blocker_label(bundle: dict[str, Any]) -> str:
+    observation = _build_error_observation(bundle)
+    reason = _lower(observation.get("reason"))
+    tail = str(observation.get("tail") or observation.get("first_error") or "")
+    if reason or "pyenv install 3.1" in tail or "BUILD FAILED" in tail:
+        if "pyenv install 3.1" in tail or "python3.1" in tail.replace(" ", ""):
+            return "Build failed: invalid Python version requested pyenv install 3.1"
+        return "Build failed after repair"
+    blockage = bundle.get("blockage") or {}
+    if isinstance(blockage, dict):
+        status = str(blockage.get("status") or "").replace("_", " ").strip()
+        decision = blockage.get("decision") if isinstance(blockage.get("decision"), dict) else {}
+        reason_text = str(decision.get("reason") or blockage.get("reason") or "").strip()
+        if reason_text:
+            return reason_text[:220]
+        if status:
+            return status.title()
+    return ""
+
+
+def _is_record_not_ready(status: dict[str, Any]) -> bool:
+    return _lower(status.get("reason")) == "record_not_ready"
+
+
+def _has_terminal_event(bundle: dict[str, Any]) -> bool:
+    return any(_lower(event.get("step")) in {"done", "failure", "technical_blocker", "manual_hardware_required"} for event in _events(bundle))
+
+
+def _latest_recovery_event(bundle: dict[str, Any]) -> dict[str, Any] | None:
+    events = _events_for_phase(bundle, "recovery")
+    return events[-1] if events else None
+
+
 def _phase_status_from_events(bundle: dict[str, Any], phase: str) -> str:
     latest_by_step = _latest_events_by_step(bundle, phase)
     if not latest_by_step:
@@ -250,15 +288,19 @@ def _logs_warning(bundle: dict[str, Any]) -> dict[str, Any] | None:
     return None
 
 
-def _archive_warning(bundle: dict[str, Any]) -> dict[str, Any] | None:
+def _archive_warning(bundle: dict[str, Any], *, terminal: bool) -> dict[str, Any] | None:
     status = _eval_publish(bundle)
-    if not status:
+    if not status or status.get("published") is True:
         return None
-    if status.get("published") is True:
+    # record_not_ready is the expected state while a run is still executing.
+    # It should not become a global warning until the run has actually ended.
+    if not terminal and _is_record_not_ready(status):
         return None
     if status.get("attempted"):
         return {"code": "eval_archive_not_published", "label": "Eval archive not published", "detail": str(status.get("reason") or "Backend archive copy did not complete.")}
-    return {"code": "eval_archive_pending", "label": "Eval archive pending", "detail": "Backend archive copy has not been confirmed yet."}
+    if terminal:
+        return {"code": "eval_archive_pending", "label": "Eval archive pending", "detail": "Backend archive copy has not been confirmed yet."}
+    return None
 
 
 def _build_phase_details(bundle: dict[str, Any], phase: str) -> list[dict[str, Any]]:
@@ -277,6 +319,9 @@ def _build_phase_details(bundle: dict[str, Any], phase: str) -> list[dict[str, A
         elif smoke:
             reason = smoke.get("failure_type") or smoke.get("error") or "Generation was not verified"
             details.append({"label": f"Generation not verified: {reason}", "status": "warning"})
+        blocker = _final_blocker_label(bundle)
+        if blocker:
+            details.append({"label": blocker, "status": "failed"})
         recommendation = smoke.get("recommended_zero_gpu_duration_seconds") or (_gate(bundle).get("zero_gpu_duration_recommendation") or {}).get("recommended_zero_gpu_duration_seconds")
         if recommendation:
             details.append({"label": f"ZeroGPU duration recommendation: {recommendation}s", "status": "complete"})
@@ -288,7 +333,10 @@ def _build_phase_details(bundle: dict[str, Any], phase: str) -> list[dict[str, A
                 if eval_status.get("archive_relative_path"):
                     details.append({"label": str(eval_status.get("archive_relative_path")), "status": "info"})
             elif eval_status.get("attempted"):
-                details.append({"label": f"Eval archive not published: {eval_status.get('reason') or 'unknown'}", "status": "warning"})
+                if _is_record_not_ready(eval_status):
+                    details.append({"label": "Eval archive pending until the run completes", "status": "pending"})
+                else:
+                    details.append({"label": f"Eval archive not published: {eval_status.get('reason') or 'unknown'}", "status": "warning"})
             else:
                 details.append({"label": "Eval archive pending", "status": "pending"})
     elif phase == "recovery":
@@ -296,14 +344,22 @@ def _build_phase_details(bundle: dict[str, Any], phase: str) -> list[dict[str, A
         if not steps.intersection(PHASE_STEPS["recovery"]):
             details.append({"label": "Recovery not needed", "status": "skipped"})
         else:
+            failed_steps = {
+                _lower(e.get("step"))
+                for e in _events(bundle)
+                if _lower(e.get("step")) in PHASE_STEPS["recovery"] and _lower(e.get("status")) in FAILED_STATUSES
+            }
             for label, candidates in [
                 ("Diagnose", {"failure_diagnosis", "pi_diagnosis"}),
                 ("Decide", {"repair_decision"}),
-                ("Act", {"repair", "repair_patch", "repair_upload"}),
+                ("Patch", {"repair", "repair_patch", "repair_upload"}),
                 ("Revalidate", {"repair_validation"}),
             ]:
-                status = "complete" if steps.intersection(candidates) else "pending"
+                status = "failed" if failed_steps.intersection(candidates) else "complete" if steps.intersection(candidates) else "pending"
                 details.append({"label": label, "status": status})
+            blocker = _final_blocker_label(bundle)
+            if blocker:
+                details.append({"label": blocker, "status": "failed"})
     else:
         latest = _latest_event_for_phase(bundle, phase)
         if latest:
@@ -321,6 +377,9 @@ def _phase_summary(bundle: dict[str, Any], phase: str, status: str) -> str:
         if warning:
             return warning["label"]
     if phase == "live_validation":
+        blocker = _final_blocker_label(bundle)
+        if blocker and _final_visual_status(_verdict(bundle)) == "error":
+            return "Live validation incomplete"
         signals = _signals(bundle)
         smoke = _smoke(bundle)
         if signals.get("generation_smoke_passed") is True or _lower(smoke.get("status")) == "success":
@@ -333,16 +392,24 @@ def _phase_summary(bundle: dict[str, Any], phase: str, status: str) -> str:
         steps = _event_steps(bundle)
         if not steps.intersection(PHASE_STEPS["recovery"]):
             return "Not needed"
+        if any(_lower(e.get("step")) == "repair_validation" and _lower(e.get("status")) in FAILED_STATUSES for e in _events(bundle)):
+            return "Patch attempted; final blocker remains"
         if any(_lower(e.get("step")) == "repair_validation" and _lower(e.get("status")) in SUCCESS_STATUSES for e in _events(bundle)):
             return "Repair revalidated"
-        return "Recovery attempted"
+        latest = _latest_recovery_event(bundle) or {}
+        latest_step = _lower(latest.get("step"))
+        if latest_step in {"repair_validation"}:
+            return "Revalidating repair"
+        if latest_step in {"repair", "repair_plan", "repair_patch", "repair_upload"}:
+            return "Repairing"
+        return "Diagnosing repair"
     if phase == "archive":
         eval_status = _eval_publish(bundle)
         if eval_status.get("published") is True:
             return "Eval archive published"
-        if eval_status.get("attempted"):
+        if eval_status.get("attempted") and not _is_record_not_ready(eval_status):
             return "Eval archive attempted"
-        return "Final artifacts pending" if status in {"pending", "running"} else "Final artifacts written"
+        return "Archive pending" if status in {"pending", "running"} else "Final artifacts written"
     if phase == "done":
         label, _ = _status_label(_verdict(bundle))
         return label
@@ -372,10 +439,20 @@ def _phase_status(bundle: dict[str, Any], phase: str, verdict: str) -> str:
     if phase == "recovery":
         if not steps.intersection(PHASE_STEPS["recovery"]):
             return "skipped" if terminal else "pending"
+        if any(_lower(e.get("step")) == "repair_validation" and _lower(e.get("status")) in FAILED_STATUSES for e in _events(bundle)):
+            return "failed"
         if any(_lower(e.get("step")) == "repair_validation" and _lower(e.get("status")) in SUCCESS_STATUSES for e in _events(bundle)):
             return "complete"
+        latest_recovery = _latest_recovery_event(bundle) or {}
+        latest_status = _lower(latest_recovery.get("status"))
+        if latest_status in FAILED_STATUSES:
+            return "failed"
+        return "warning" if terminal else "running"
 
     if phase == "live_validation":
+        blocker = _final_blocker_label(bundle)
+        if blocker and _final_visual_status(_verdict(bundle)) == "error":
+            return "Live validation incomplete"
         signals = _signals(bundle)
         smoke = _smoke(bundle)
         if signals.get("generation_smoke_passed") is True or _lower(smoke.get("status")) == "success":
@@ -389,10 +466,12 @@ def _phase_status(bundle: dict[str, Any], phase: str, verdict: str) -> str:
         if eval_status.get("published") is True:
             return "complete"
         if eval_status.get("attempted"):
+            if not terminal and _is_record_not_ready(eval_status):
+                return "pending"
             return "warning"
         if terminal:
             return "warning" if bundle.get("eval_record") else "pending"
-        return _phase_status_from_events(bundle, phase)
+        return "pending"
 
     if phase == "agent" and _pi_model_warning(bundle):
         return "warning"
@@ -429,7 +508,7 @@ def build_run_timeline_model(bundle: dict[str, Any]) -> dict[str, Any]:
     terminal = visual_status in {"success", "warn", "error", "stopped"}
     percent = 100 if terminal else int(progress.get("progress") or 0)
 
-    warnings = [w for w in (_pi_model_warning(bundle), _hardware_warning(bundle), _logs_warning(bundle), _archive_warning(bundle)) if w]
+    warnings = [w for w in (_pi_model_warning(bundle), _hardware_warning(bundle), _logs_warning(bundle), _archive_warning(bundle, terminal=terminal)) if w]
     phases: list[dict[str, Any]] = []
     for phase in PHASE_ORDER:
         status = _phase_status(bundle, phase, verdict)

src/version.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-ASF_APP_VERSION = "v189.5"
-ASF_RELEASE_NAME = "Agentic Space Factory v189.5"
+ASF_APP_VERSION = "v189.7"
+ASF_RELEASE_NAME = "Agentic Space Factory v189.7"
 
 
 def resolve_app_version(value: str | None = None) -> str:

src/worker_payload.py

@@ -52,14 +52,14 @@ INTERNAL_WORKSPACE_ARTIFACT_NAMES = {
 
 
 def internal_workspace_upload_ignore_patterns() -> list[str]:
-    patterns = [".git/*", "node_modules/*", "__pycache__/*", "*.pyc"]
+    patterns = [".git/*", ".cache/*", "**/.cache/*", "node_modules/*", "__pycache__/*", "*.pyc"]
     for name in sorted(INTERNAL_WORKSPACE_ARTIFACT_NAMES):
         patterns.extend([name, f"**/{name}"])
     return patterns
 
 
 def internal_workspace_copy_ignore(_dir: str, names: list[str]) -> set[str]:
-    ignored = {".git", "node_modules", "__pycache__"}
+    ignored = {".git", ".cache", "node_modules", "__pycache__"}
     ignored.update(name for name in names if name in INTERNAL_WORKSPACE_ARTIFACT_NAMES or name.endswith(".pyc"))
     return ignored
 
@@ -67,7 +67,7 @@ def internal_workspace_copy_ignore(_dir: str, names: list[str]) -> set[str]:
 def is_publishable_workspace_file(path: Path) -> bool:
     if path.name in INTERNAL_WORKSPACE_ARTIFACT_NAMES:
         return False
-    if any(part in {".git", "node_modules", "__pycache__"} for part in path.parts):
+    if any(part in {".git", ".cache", "node_modules", "__pycache__"} for part in path.parts):
         return False
     if path.suffix == ".pyc":
         return False
@@ -92,6 +92,43 @@ def append_event(path: Path, step: str, status: str, message: str, data: dict |
     print(line, flush=True)
 
 
+SAFE_SPACE_PYTHON_VERSIONS = {"3.10", "3.11", "3.12"}
+DEFAULT_SPACE_PYTHON_VERSION = "3.10"
+
+
+def normalize_space_python_version(value) -> tuple[str, bool, str]:
+    raw = str(value or "").strip().strip("\"'")
+    lowered = raw.lower().replace("python", "").strip()
+    match = re.search(r"(3)\.(\d+)(?:\.\d+)?", lowered)
+    if match:
+        normalized = f"{match.group(1)}.{match.group(2)}"
+    else:
+        normalized = lowered
+    if normalized in SAFE_SPACE_PYTHON_VERSIONS:
+        return normalized, normalized != raw, "allowed"
+    return DEFAULT_SPACE_PYTHON_VERSION, True, f"unsupported_or_ambiguous:{raw or 'missing'}"
+
+
+def requirements_has_package(lines: list[str], package: str) -> bool:
+    wanted = package.lower().replace("_", "-")
+    for line in lines:
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
+            continue
+        name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
+        if name == wanted:
+            return True
+    return False
+
+
+def workspace_app_imports_torch(workspace: Path) -> bool:
+    app_path = workspace / "app.py"
+    if not app_path.exists():
+        return False
+    text = app_path.read_text(encoding="utf-8", errors="ignore")
+    return bool(re.search(r"(?m)^\s*(import\s+torch\b|from\s+torch\b)", text))
+
+
 def _artifact_entry(run_dir: Path, rel_path: str, *, kind: str = "file") -> dict:
     path = run_dir / rel_path
     if kind == "folder":
@@ -2332,12 +2369,16 @@ def sanitize_readme_metadata(workspace: Path, events_path: Path):
     else:
         changed = True
 
+    raw_python_version = metadata.get("python_version") or DEFAULT_SPACE_PYTHON_VERSION
+    python_version, python_version_changed, python_version_reason = normalize_space_python_version(raw_python_version)
     required = {
         "title": metadata.get("title") or "Generated Model Space",
         "sdk": "gradio",
         "app_file": "app.py",
-        "python_version": metadata.get("python_version") or "3.10",
+        "python_version": python_version,
     }
+    if python_version_changed:
+        changed = True
     suggested = metadata.get("suggested_hardware")
     if suggested:
         required["suggested_hardware"] = suggested
@@ -2351,14 +2392,26 @@ def sanitize_readme_metadata(workspace: Path, events_path: Path):
     for key in ordered_keys:
         if key in required and required[key]:
             value = str(required[key]).strip()
-            if key in {"title", "short_description"} and not (value.startswith('"') or value.startswith("'")):
+            if key in {"title", "short_description", "python_version"} and not (value.startswith('"') or value.startswith("'")):
                 value = json.dumps(value, ensure_ascii=False)
             lines.append(f"{key}: {value}")
     normalized_body = body.lstrip("\n") or "# Generated Model Space\n"
     new_text = "---\n" + "\n".join(lines) + "\n---\n\n" + normalized_body
     if new_text != text:
         readme_path.write_text(new_text, encoding="utf-8")
-        append_event(events_path, "metadata_sanitize", "success", "Ensured README Space metadata", {"metadata_keys": [k for k in ordered_keys if k in required]})
+        append_event(
+            events_path,
+            "metadata_sanitize",
+            "success",
+            "Ensured README Space metadata",
+            {
+                "metadata_keys": [k for k in ordered_keys if k in required],
+                "python_version_before": str(raw_python_version).strip().strip("\"'"),
+                "python_version_after": python_version,
+                "python_version_normalized": bool(python_version_changed),
+                "python_version_reason": python_version_reason,
+            },
+        )
 
 def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
     """Normalize only broad known-dangerous base dependencies before upload.
@@ -2426,6 +2479,12 @@ def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
             stable_policy_lines.append(policy[canonical])
             changed = True
 
+    torch_added = False
+    if workspace_app_imports_torch(workspace) and not requirements_has_package(filtered + stable_policy_lines, "torch"):
+        stable_policy_lines.append("torch>=2.0.0")
+        torch_added = True
+        changed = True
+
     new_lines = prefix_lines + stable_policy_lines + filtered
     new = "\n".join(line for line in new_lines if line.strip()) + "\n"
     if new != raw:
@@ -2440,7 +2499,10 @@ def normalize_requirements_for_modern_hub(workspace: Path, events_path: Path):
             {
                 "huggingface_hub": policy["huggingface-hub"],
                 "transformers": policy["transformers"],
-                "reason": "Avoid uncontrolled Transformers 5.x while preserving model-specific dependency choices for Pi to repair from build logs.",
+                "reason": "Avoid uncontrolled Transformers 5.x while preserving model-specific dependency choices for Pi to repair from build logs; require torch when app.py imports torch.",
+                "torch_added": torch_added,
+                "torch_policy": "torch>=2.0.0",
+                "torch_reason": "app_imports_torch" if torch_added else "not_needed_or_already_present",
             },
         )
 
@@ -2538,7 +2600,7 @@ def workspace_file_inventory(workspace: Path, max_files: int = 80) -> list[str]:
     for path in sorted(workspace.rglob("*")):
         if not path.is_file():
             continue
-        if any(part in {".git", "node_modules", "__pycache__"} for part in path.parts):
+        if any(part in {".git", ".cache", "node_modules", "__pycache__"} for part in path.parts):
             continue
         try:
             files.append(str(path.relative_to(workspace)))
@@ -3740,6 +3802,43 @@ def append_event(path: Path, step: str, status: str, message: str, data: dict |
     print(line, flush=True)
 
 
+SAFE_SPACE_PYTHON_VERSIONS = {"3.10", "3.11", "3.12"}
+DEFAULT_SPACE_PYTHON_VERSION = "3.10"
+
+
+def normalize_space_python_version(value) -> tuple[str, bool, str]:
+    raw = str(value or "").strip().strip("\"'")
+    lowered = raw.lower().replace("python", "").strip()
+    match = re.search(r"(3)\.(\d+)(?:\.\d+)?", lowered)
+    if match:
+        normalized = f"{match.group(1)}.{match.group(2)}"
+    else:
+        normalized = lowered
+    if normalized in SAFE_SPACE_PYTHON_VERSIONS:
+        return normalized, normalized != raw, "allowed"
+    return DEFAULT_SPACE_PYTHON_VERSION, True, f"unsupported_or_ambiguous:{raw or 'missing'}"
+
+
+def requirements_has_package(lines: list[str], package: str) -> bool:
+    wanted = package.lower().replace("_", "-")
+    for line in lines:
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#") or stripped.startswith("-") or "://" in stripped:
+            continue
+        name = re.split(r"[<>=!~;\[]", stripped, 1)[0].strip().lower().replace("_", "-")
+        if name == wanted:
+            return True
+    return False
+
+
+def workspace_app_imports_torch(workspace: Path) -> bool:
+    app_path = workspace / "app.py"
+    if not app_path.exists():
+        return False
+    text = app_path.read_text(encoding="utf-8", errors="ignore")
+    return bool(re.search(r"(?m)^\s*(import\s+torch\b|from\s+torch\b)", text))
+
+
 def redact_text(text: str | None) -> str:
     if not text:
         return ""