diff --git "a/src/worker_payload.py" "b/src/worker_payload.py" --- "a/src/worker_payload.py" +++ "b/src/worker_payload.py" @@ -35,8 +35,8 @@ DEFAULT_PREFERRED_SPACE_HARDWARE = "zero-a10g" DEFAULT_FALLBACK_SPACE_HARDWARE = "a10g-large" DEFAULT_MODEL_ID = "sshleifer/tiny-gpt2" MAX_PI_REPAIR_ATTEMPTS = 3 -APP_VERSION = "v198.26.28" -app_version = "v198.26.28" +APP_VERSION = "v198.26.31" +app_version = "v198.26.31" # Internal agent/recovery files may be needed inside the transient Pi # workspace, but they should not be published to the generated Space or shown @@ -60,9 +60,11 @@ INTERNAL_WORKSPACE_ARTIFACT_NAMES = { "MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md", + "APP_RUNTIME_CONTRACT.json", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "PI_TASK_PACKET.json", + "RECIPE_AWARE_REPAIR_PACKET.json", "PATCH_REFUSAL.json", "TECHNICAL_BLOCKERS.json", "pi_feasibility_brief.json", @@ -970,6 +972,7 @@ def write_artifact_manifest(run_dir: Path, *, events_path: Path | None = None, r _artifact_entry(run_dir, "repair/MEMORY_DIAGNOSIS.json"), _artifact_entry(run_dir, "repair/REPAIR_DECISION.json"), _artifact_entry(run_dir, "repair/REPAIR_TASK_PACKET.json"), + _artifact_entry(run_dir, "repair/RECIPE_AWARE_REPAIR_PACKET.json"), _artifact_entry(run_dir, "repair/PI_TASK_PACKET.json"), _artifact_entry(run_dir, "repair/FROZEN_REPAIR_DECISION.json"), _artifact_entry(run_dir, "repair/LOG_EVIDENCE_PACKET.json"), @@ -1958,6 +1961,10 @@ def should_pause_generated_space(final_status: str | None, target_space: str | N "runtime_failed", "repair_failed", "repair_patch_failed", + "repair_exhausted", + "no_patch_produced_by_pi", + "no_relevant_patch_produced_by_pi", + "pi_patch_rejected_by_guard", "validation_failed", "dependency_error", "manual_hardware_required", @@ -3815,12 +3822,12 @@ def _health_text_values(validation: dict | None) -> list[tuple[str, str]]: values: list[tuple[str, str]] = [] if not isinstance(validation, dict): return values - for key in ("result_repr", "text", "body", "message", "error", "load_error", "pipeline_error", "model_error"): + for key in ("result_repr", "text", "body", "message", "error", "last_error", "load_error", "pipeline_error", "model_error"): value = validation.get(key) if value is not None: values.append((key, str(value))) for source, payload in _semantic_health_payloads(validation): - for key in ("status", "health", "state", "model", "pipeline", "generation", "message", "error", "load_error", "pipeline_error", "model_error"): + for key in ("status", "health", "state", "model", "pipeline", "generation", "message", "error", "last_error", "load_error", "pipeline_error", "model_error"): value = payload.get(key) if isinstance(value, str): values.append((f"{source}.{key}", value)) @@ -3836,14 +3843,20 @@ def validation_health_semantics(validation: dict | None) -> dict: counted as model-ready health or a completed repair. """ details = { - "schema_version": "health_semantics.v198_26_19", + "schema_version": "health_semantics.v198_26_30", "endpoint_reachable": False, "semantic_checked": False, "semantic_passed": None, "negative_markers": [], "positive_markers": [], "load_error": "", + "last_error": "", "model_ready": None, + "pipeline_ready": None, + "model_family": "", + "loader_strategy": "", + "expected_output_type": "", + "failure_class": "", "transport_status": "", } if not isinstance(validation, dict): @@ -3881,30 +3894,48 @@ def validation_health_semantics(validation: dict | None) -> dict: for key in ("pipeline_ready", "pipeline_loaded", "model_ready", "model_loaded", "generation_ready"): if key in payload: details["semantic_checked"] = True + if key in {"pipeline_ready", "pipeline_loaded"}: + details["pipeline_ready"] = bool(payload.get(key)) if isinstance(payload.get(key), bool) else details.get("pipeline_ready") if payload.get(key) is False: details["negative_markers"].append(f"{source}.{key}=false") elif payload.get(key) is True: details["positive_markers"].append(f"{source}.{key}=true") - for key in ("error", "load_error", "pipeline_error", "model_error"): + for key in ("model_family", "loader_strategy", "expected_output_type"): + value = payload.get(key) + if value and not details.get(key): + details[key] = str(value)[:200] + for key in ("error", "last_error", "load_error", "pipeline_error", "model_error"): value = payload.get(key) if value: details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}=present") if not details["load_error"]: details["load_error"] = str(value)[:1000] + if key == "last_error" and not details["last_error"]: + details["last_error"] = str(value)[:1000] + health_error_text = "\n".join(v for _, v in _health_text_values(validation)) + try: + if is_sdxl_lora_text_encoder_mismatch_text(health_error_text): + details["failure_class"] = "sdxl_lora_text_encoder_mismatch" + except Exception: + pass for key, value in _health_text_values(validation): lowered = value.strip().lower().strip("'\"") if lowered in HEALTH_NEGATIVE_STATUS_MARKERS or any(marker in lowered for marker in MODEL_NOT_READY_TEXT_MARKERS): details["semantic_checked"] = True details["negative_markers"].append(f"{key}=model_not_ready") - if not details["load_error"] and key in {"error", "load_error", "pipeline_error", "model_error", "result_repr", "text", "message"}: + if not details["load_error"] and key in {"error", "last_error", "load_error", "pipeline_error", "model_error", "result_repr", "text", "message"}: details["load_error"] = value[:1000] if details["negative_markers"]: details["semantic_passed"] = False details["model_ready"] = False + if details.get("pipeline_ready") is None: + details["pipeline_ready"] = False elif details["semantic_checked"] and details["positive_markers"]: details["semantic_passed"] = True details["model_ready"] = True + if details.get("pipeline_ready") is None: + details["pipeline_ready"] = True return details @@ -4632,7 +4663,11 @@ def run_generation_smoke(target_space_id: str, token: str, run_dir: Path, events api_name = "/generate" test_kwargs = {} timeout_s = smoke_timeout_seconds(expected_output_type) - append_event(events_path, "generation_smoke", "started", "Calling live generation endpoint for ZeroGPU timing", {"api_name": api_name, "expected_output_type": expected_output_type, "smoke_timeout_seconds": timeout_s}) + recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if workspace and (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") + if not isinstance(recipe, dict): + recipe = {} + app_runtime_contract = load_app_runtime_contract(workspace, run_dir) if workspace else build_app_runtime_contract(recipe) + append_event(events_path, "generation_smoke", "started", "Calling live generation endpoint for family-aware smoke", {"api_name": api_name, "expected_output_type": expected_output_type, "model_family": recipe.get("model_family"), "loader_strategy": recipe.get("loader_strategy"), "smoke_test_strategy": recipe.get("smoke_test_strategy"), "smoke_timeout_seconds": timeout_s}) client = make_gradio_client(target_space_id, token, timeout_s=timeout_s) identity = write_gradio_client_identity(client, target_space_id, run_dir, events_path, phase="generation_smoke") if identity.get("mismatch"): @@ -4676,7 +4711,7 @@ def run_generation_smoke(target_space_id: str, token: str, run_dir: Path, events canonical_source = "" if contract_payload and contract_payload.get("canonical_smoke_example_present"): canonical_example, canonical_source = _canonical_smoke_dict_from_contracts(contract, demo_quality_contract) - smoke_payload = {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "parameters": smoke_parameters, "source": smoke_source, "contract_present": bool(contract), "demo_quality_contract_present": bool(demo_quality_contract), "canonical_smoke_example_present": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s} + smoke_payload = {"api_name": api_name, "test_args": test_args, "raw_test_args": raw_test_args, "test_kwargs": test_kwargs, "parameters": smoke_parameters, "source": smoke_source, "contract_present": bool(contract), "app_runtime_contract_present": bool(app_runtime_contract), "model_family": recipe.get("model_family") or app_runtime_contract.get("model_family"), "loader_strategy": recipe.get("loader_strategy") or app_runtime_contract.get("loader_strategy"), "smoke_test_strategy": recipe.get("smoke_test_strategy") or app_runtime_contract.get("smoke_test_strategy"), "demo_quality_contract_present": bool(demo_quality_contract), "canonical_smoke_example_present": bool(canonical_example), "canonical_smoke_example_source": canonical_source, "choice_corrections": initial_choice_changes, "file_input_conversions": file_input_conversions, "smoke_timeout_seconds": timeout_s} payload_resolution_guard = detect_required_text_payload_resolution_issue(test_args, smoke_parameters, canonical_example) smoke_payload["payload_resolution_guard"] = payload_resolution_guard if canonical_example: @@ -4926,12 +4961,192 @@ def smoke_failure_repair_escalation_decision(generation_smoke: dict | None, *, v return payload + +def load_recipe_repair_context(workspace: Path | None, run_dir: Path | None) -> dict: + """Load MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json for repair.""" + workspace = Path(workspace) if workspace else None + run_dir = Path(run_dir) if run_dir else None + + def load_named(name: str) -> tuple[dict, str]: + candidates = [] + if workspace: + candidates.append(workspace / name) + if run_dir: + candidates.append(run_dir / name) + candidates.append(run_dir / "analysis_inputs" / name) + for candidate in candidates: + try: + if candidate.exists(): + payload = load_json_if_exists(candidate) + if isinstance(payload, dict): + return payload, str(candidate) + except Exception: + continue + return {}, "" + + recipe, recipe_source = load_named("MODEL_RECIPE.json") + runtime_contract, runtime_source = load_named("APP_RUNTIME_CONTRACT.json") + if not runtime_contract and recipe: + try: + runtime_contract = build_app_runtime_contract(recipe) + runtime_source = "derived_from_MODEL_RECIPE.json" + except Exception: + runtime_contract = {} + return { + "schema_version": "recipe_repair_context.v198_26_31", + "model_recipe": recipe, + "model_recipe_source": recipe_source, + "app_runtime_contract": runtime_contract, + "app_runtime_contract_source": runtime_source, + "recipe_available": bool(recipe), + "runtime_contract_available": bool(runtime_contract), + "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", + "artifact_role": recipe.get("artifact_role") or "", + "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", + "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", + "primary_endpoint": recipe.get("primary_endpoint") or runtime_contract.get("primary_endpoint") or "/generate", + } + + +def recipe_known_failure_catalog(recipe: dict | None = None, runtime_contract: dict | None = None) -> list[dict]: + recipe = recipe if isinstance(recipe, dict) else {} + runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} + catalog = [] + for item in recipe.get("known_failure_signatures") or []: + if isinstance(item, dict): + catalog.append(dict(item)) + required = [ + {"failure_class": "diffusers_duplicate_token_argument", "markers": ["got multiple values for keyword argument 'token'", "multiple values for keyword argument \"token\"", "duplicate token"], "repair_focus": "Remove duplicate token/use_auth_token injection from from_pretrained kwargs; do not change the model id or replace inference."}, + {"failure_class": "hf_cache_permission_denied", "markers": ["permission denied", "/data", "HF_HOME", "huggingface cache", "read-only file system", "errno 13"], "repair_focus": "Move HF_HOME/HF_HUB_CACHE/TRANSFORMERS_CACHE/DIFFUSERS_CACHE to a writable Space path before importing/loading models."}, + {"failure_class": "missing_spaces_gpu_decorator", "markers": ["missing @spaces.GPU", "ZeroGPU", "CUDA has been initialized before", "GPU task function", "@spaces.GPU"], "repair_focus": "Import spaces before CUDA-touching imports and protect the heavy inference function with @spaces.GPU while preserving fixed-GPU compatibility."}, + {"failure_class": "validator_schema_choice_type_mismatch", "markers": ["not in the list of choices", "Invalid value", "Dropdown", "CheckboxGroup", "choice"], "repair_focus": "Respect the published Gradio schema and canonical smoke payload; do not patch the model for validator-owned payload coercion errors."}, + {"failure_class": "sdxl_lora_text_encoder_mismatch", "markers": ["CLIPTextModel", "text_model", "has no attribute 'text_model'"], "repair_focus": "Treat the repo as an SDXL LoRA adapter: load the base SDXL pipeline first, then load_lora_weights(adapter_id, weight_name=...), and avoid loading the adapter as a standalone pipeline."}, + {"failure_class": "flux_template_syntax_error", "markers": ["FluxPipeline", "jinja", "template", "syntax error", "unexpected", "chat_template"], "repair_focus": "Repair FLUX/prompt-template syntax without changing FluxPipeline loader semantics or replacing image generation with placeholder output."}, + {"failure_class": "model_not_loaded_after_boot", "markers": ["model not loaded", "pipeline not loaded", "model_ready", "pipeline_ready", "last_error", "not initialized", "NoneType"], "repair_focus": "Fix loader initialization and expose the real load failure through /health.last_error; /generate must fail cleanly when the model is not ready."}, + ] + seen = {str(item.get("failure_class") or "") for item in catalog} + for item in required: + if item["failure_class"] not in seen: + catalog.append(item) + seen.add(item["failure_class"]) + return catalog + + +def match_recipe_known_failure_signature(error_text: str = "", recipe: dict | None = None, runtime_contract: dict | None = None, fallback_class: str = "") -> dict: + text = str(error_text or "") + low = text.lower() + recipe = recipe if isinstance(recipe, dict) else {} + runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} + fallback_class = str(fallback_class or "") + for sig in recipe_known_failure_catalog(recipe, runtime_contract): + failure_class = str(sig.get("failure_class") or sig.get("category") or "") + markers = [str(m) for m in (sig.get("markers") or sig.get("error_markers") or sig.get("signatures") or []) if str(m)] + class_matches = bool(failure_class and fallback_class and failure_class == fallback_class) + marker_matches = [m for m in markers if m.lower() in low] + if failure_class == "sdxl_lora_text_encoder_mismatch" and is_sdxl_lora_text_encoder_mismatch_text(text): + marker_matches = marker_matches or ["CLIPTextModel.text_model"] + if class_matches or marker_matches: + return { + "schema_version": "recipe_known_failure_match.v198_26_31", + "matched": True, + "failure_class": failure_class or fallback_class, + "matched_markers": marker_matches[:10], + "repair_focus": sig.get("repair_focus") or sig.get("recommended_repair_focus") or sig.get("recommendation") or "Patch according to MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json.", + "source_signature": sig, + "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", + "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", + "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", + } + if fallback_class: + return {"schema_version": "recipe_known_failure_match.v198_26_31", "matched": False, "failure_class": fallback_class, "matched_markers": [], "repair_focus": "No catalog signature matched; still use MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json as authoritative repair context.", "source_signature": {}, "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or ""} + return {"schema_version": "recipe_known_failure_match.v198_26_31", "matched": False, "failure_class": "", "matched_markers": [], "repair_focus": ""} + + +def recipe_family_repair_directives(recipe: dict | None = None, runtime_contract: dict | None = None, signature_match: dict | None = None) -> list[str]: + recipe = recipe if isinstance(recipe, dict) else {} + runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} + signature_match = signature_match if isinstance(signature_match, dict) else {} + family = str(recipe.get("model_family") or runtime_contract.get("model_family") or "") + directives = [ + "Use MODEL_RECIPE.json as the authoritative model-family plan, not a generic app repair brief.", + "Preserve APP_RUNTIME_CONTRACT.json: /health must expose model_ready, pipeline_ready, model_family, loader_strategy, last_error, and expected_output_type.", + "After patching, /generate must either return the expected output type or fail cleanly with the same underlying loader error visible in /health.last_error.", + ] + if signature_match.get("repair_focus"): + directives.append("Known failure focus: " + str(signature_match.get("repair_focus"))) + if family == "diffusers_sdxl_lora_adapter": + directives.extend([ + "This repo is an adapter. Do not call DiffusionPipeline.from_pretrained(adapter_id) as if it were a full pipeline.", + "Load the SDXL base model from model_recipe.base_model_id, then call load_lora_weights(adapter_id, weight_name=adapter_weight_name when present).", + "Keep the LoRA trigger words/prompt guidance from the model card when available.", + ]) + elif family == "diffusers_flux_pipeline": + directives.extend([ + "Keep FluxPipeline.from_pretrained semantics and the recipe-selected model id.", + "Do not replace FLUX inference with a text-only or placeholder response; preserve image output.", + ]) + elif family == "gguf_llamacpp": + directives.append("Use llama.cpp/llama-cpp-python style GGUF loading and text generation; do not attempt Transformers AutoModel loading for GGUF weights.") + elif family == "onnx_runtime_model": + directives.append("Use onnxruntime InferenceSession and preserve input/output tensor preprocessing; do not convert to a different runtime blindly.") + elif family == "transformers_pipeline": + directives.append("Use the Transformers pipeline/AutoModel strategy indicated by the recipe and preserve the task-specific output contract.") + return directives + + +def write_recipe_aware_repair_packet(workspace: Path, run_dir: Path, failure_reason: str = "", classification: dict | None = None, decision: dict | None = None) -> dict: + ctx = load_recipe_repair_context(workspace, run_dir) + recipe = ctx.get("model_recipe") or {} + runtime_contract = ctx.get("app_runtime_contract") or {} + category = str((classification or {}).get("category") or (classification or {}).get("failure_class") or (decision or {}).get("failure_class") or "") + match = match_recipe_known_failure_signature(failure_reason, recipe, runtime_contract, fallback_class=category) + packet = { + "schema_version": "recipe_aware_repair_packet.v198_26_31", + "authority": "MODEL_RECIPE.json + APP_RUNTIME_CONTRACT.json", + **ctx, + "known_failure_match": match, + "family_repair_directives": recipe_family_repair_directives(recipe, runtime_contract, match), + "repair_must_not_finish_not_started": True, + "no_patch_policy": "If Pi cannot produce a publishable patch, ASF must either launch another more direct recipe-aware attempt while budget remains or finish with no_patch_produced_by_pi / repair_exhausted, never post_repair_validation=not_started.", + } + repair_dir = run_dir / "repair" + repair_dir.mkdir(parents=True, exist_ok=True) + write_json(repair_dir / "RECIPE_AWARE_REPAIR_PACKET.json", packet) + try: + write_json(workspace / "RECIPE_AWARE_REPAIR_PACKET.json", packet) + except Exception: + pass + return packet + + +def recipe_repair_direct_followup_reason(run_dir: Path, terminal_status: str, failure_class: str = "") -> str: + latest = _latest_repair_attempt_result(run_dir) + return ( + "Previous Pi repair did not produce an uploadable publishable patch.\n" + f"Terminal status: {terminal_status}\n" + f"Latest attempt result: {latest.get('post_repair_result') or ''}\n" + f"Failure class: {failure_class or latest.get('failure_class') or ''}\n" + "Relaunch with a more direct recipe-aware instruction: implement the known failure focus from RECIPE_AWARE_REPAIR_PACKET.json, modify only allowed publishable files, or write PATCH_REFUSAL.json with a concrete blocker." + ) + + +def should_relaunch_after_no_patch(run_dir: Path, terminal_status: str, remaining_patch_budget: int) -> dict: + terminal_status = str(terminal_status or "") + no_patch_statuses = {"no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard", "pi_repair_failed_before_upload"} + latest = _latest_repair_attempt_result(run_dir) + result = str(latest.get("post_repair_result") or "") + result_is_no_patch = result in {"repair_noop", "expected_file_not_modified", "diff_gate_failed", "sanity_failed"} + can_retry = remaining_patch_budget > 0 and (terminal_status in no_patch_statuses or result_is_no_patch) + return {"schema_version": "recipe_repair_retry_decision.v198_26_31", "relaunch": bool(can_retry), "terminal_status": terminal_status, "remaining_patch_budget": int(remaining_patch_budget or 0), "latest_attempt": latest, "reason": "retry_with_more_direct_recipe_aware_brief" if can_retry else "repair_exhausted_or_not_retryable"} + def write_smoke_repair_primary_error_packet(run_dir: Path, workspace: Path, generation_smoke: dict | None, *, failure_class: str = "") -> dict: smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = read_inference_contract(workspace) recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") if not isinstance(recipe, dict): recipe = {} + repair_context = load_recipe_repair_context(workspace, run_dir) + app_runtime_contract = repair_context.get("app_runtime_contract") or {} error_text = str(smoke.get("error") or smoke.get("result_info") or smoke.get("result_repr") or smoke)[:6000] payload = { "schema_version": "smoke_repair_primary_error.v198_26_27", @@ -4944,6 +5159,8 @@ def write_smoke_repair_primary_error_packet(run_dir: Path, workspace: Path, gene "primary_api_name": smoke.get("api_name") or contract.get("primary_api_name") or contract.get("primary_endpoint") or "/generate", "contract": contract, "model_recipe": recipe, + "app_runtime_contract": app_runtime_contract, + "known_failure_match": match_recipe_known_failure_signature(error_text, recipe, app_runtime_contract, fallback_class=failure_class or smoke.get("failure_class") or smoke.get("failure_type") or ""), "sdxl_lora_text_encoder_mismatch": is_sdxl_lora_text_encoder_mismatch_text(error_text), "recommended_repair_focus": "sdxl_lora_base_loader_or_text_encoder_compatibility" if is_sdxl_lora_text_encoder_mismatch_text(error_text) else "model_loading_or_runtime_patch", } @@ -5151,6 +5368,7 @@ def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_ append_event(events_path, "generation_smoke_repair", "warning", "Writable cache repair was indicated but no deterministic cache patch was applied; falling back to Pi repair", cache_patch) primary_error_packet = write_smoke_repair_primary_error_packet(run_dir, workspace, generation_smoke, failure_class=failure_class) + recipe_repair_packet = write_recipe_aware_repair_packet(workspace, run_dir, str(generation_smoke.get("error") or generation_smoke.get("result_info") or generation_smoke), {"category": failure_class, "failure_phase": "generation_smoke"}, {"action": "patch_code", "classification": {"category": failure_class}}) failure_reason = ( "Automatic generation smoke failed after the Space became reachable.\n" f"Authoritative smoke error: {primary_error_packet.get('error') or ''}\n" @@ -5158,6 +5376,8 @@ def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_ f"Failure owner: {generation_smoke.get('failure_owner') or ''}\n" f"Actionability: {generation_smoke.get('actionability') or ''}\n" f"Recommended action: {generation_smoke.get('recommended_action') or ''}\n" + f"Recipe-aware repair focus: {(primary_error_packet.get('known_failure_match') or {}).get('repair_focus') or ''}\n" + f"Model family: {recipe_repair_packet.get('model_family') or ''}; loader: {recipe_repair_packet.get('loader_strategy') or ''}\n" f"Error/result: {generation_smoke.get('error') or generation_smoke.get('result_info') or generation_smoke}\n" f"Latest Space log brief: {latest_log_brief[:3000]}\n" ) @@ -5174,7 +5394,12 @@ def attempt_generation_smoke_repair(api, workspace: Path, run_dir: Path, events_ "smoke_primary_error_packet": "repair/SMOKE_PRIMARY_ERROR.json", "sdxl_lora_text_encoder_mismatch": failure_class == "sdxl_lora_text_encoder_mismatch", "missing_executable": generation_smoke.get("missing_executable") or "", + "recipe_aware_repair_packet": "repair/RECIPE_AWARE_REPAIR_PACKET.json", + "known_failure_match": primary_error_packet.get("known_failure_match") or {}, + "model_family": recipe_repair_packet.get("model_family") or "", + "loader_strategy": recipe_repair_packet.get("loader_strategy") or "", }, + "family_repair_directives": recipe_repair_packet.get("family_repair_directives") or [], "constraints": [ "Preserve real inference and the existing Gradio API contract.", "Do not replace the model call with placeholders or diagnostics if the failure is repairable.", @@ -5457,7 +5682,7 @@ def normalize_runtime_expected_output_type(value: str) -> str: def archive_rejected_workspace(workspace: Path, run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None) -> dict: """Persist the publishable workspace that failed pre-upload integrity. - v198.26.28 keeps blocked workspaces auditable and makes pre-upload + v198.26.29 keeps blocked workspaces auditable and makes pre-upload failures repairable: the exact rejected payload is preserved before the worker fails closed or requests a targeted repair pass. """ @@ -5490,7 +5715,7 @@ def archive_rejected_workspace(workspace: Path, run_dir: Path, events_path: Path def write_pre_upload_repair_attempt(run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None, deterministic_repairs: list[dict] | None = None, pi_repair_required: bool = False) -> dict: """Record the repair trajectory for a pre-upload template failure. - This artifact closes the visibility gap from v198.26.28: a pre-upload guard + This artifact closes the visibility gap from v198.26.29: a pre-upload guard must not fail opaquely. It must say which deterministic repairs were tried and whether a targeted Pi repair would be required before a final failure. """ @@ -5569,7 +5794,7 @@ def _insert_decorator_before_function(app_text: str, func_name: str, decorator: def _patch_common_python_syntax_defects(app_text: str) -> tuple[str, list[str]]: """Patch narrow syntax defects Pi has produced in real runs. - This is intentionally conservative. v198.26.28 covers the observed + This is intentionally conservative. v198.26.29 covers the observed `torch_dtype=DTYPE,,` class and repeated commas in function calls without attempting a broad Python formatter. """ @@ -5845,7 +6070,7 @@ def patch_workspace_for_missing_spaces_gpu_decorator(workspace: Path, run_dir: P def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path: Path, *, hardware_intent: dict | None = None, expected_output_type: str = "", reason: str = "pre_upload", raise_on_failure: bool = True) -> dict: """Pre-upload integrity guard for publishable runtime templates. - v198.26.28 extends the v198.26.24 repair-first policy with compile checks + v198.26.29 extends the v198.26.24 repair-first policy with compile checks and endpoint repair. The worker should not fail-fast merely because Pi forgot `api_name='generate'` or emitted a narrow syntax typo; it should repair, re-check, then fail closed only when the app is still invalid. @@ -5917,6 +6142,9 @@ def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path {"type": "post_patch_compile_guard", "result": post_patch_compile_guard}, ]) + recipe_alignment = recipe_runtime_alignment_guard(workspace, run_dir, events_path, reason=reason) + app_contract_guard = app_runtime_contract_guard(workspace, run_dir, events_path, expected_output_type=expected_output_type, reason=reason) + fatal = [] repairable_unresolved = [] if not compile_guard.get("passed") and not post_patch_compile_guard.get("passed"): @@ -5929,6 +6157,9 @@ def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path repairable_unresolved.append("zero_gpu_full_inference_without_spaces_gpu_decorator") if full_inference and expected in {"image", "video", "audio"} and not has_generate: repairable_unresolved.append("full_inference_media_contract_missing_generate_endpoint") + for problem in app_contract_guard.get("repairable_unresolved_problems") or []: + if problem not in repairable_unresolved: + repairable_unresolved.append(problem) if expected in {"image", "video", "audio"} and diagnostic_output: fatal.append("media_expected_output_replaced_by_text_diagnostic") if full_inference and not app_path.exists(): @@ -5953,6 +6184,8 @@ def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path "diffusers_token_patch": token_patch, "generate_endpoint_patch": generate_endpoint_patch, "spaces_gpu_decorator_patch": spaces_gpu_patch, + "recipe_runtime_alignment": recipe_alignment, + "app_runtime_contract_guard": app_contract_guard, } write_json(run_dir / "runtime_template_integrity_guard.json", result) append_event(events_path, "runtime_template_integrity_guard", "success" if result["passed"] else "failed", "Checked publishable runtime template integrity before upload", result) @@ -5983,7 +6216,7 @@ def _pre_upload_repair_failure_reason(guard_result: dict | None, workspace: Path f"Fatal problems: {', '.join(guard.get('fatal_problems') or guard.get('problems') or [])}.\n" f"Repairable unresolved problems: {', '.join(guard.get('repairable_unresolved_problems') or [])}.\n" f"Expected output type: {guard.get('expected_output_type') or ''}. Primary API: {guard.get('primary_api_name') or ''}. ZeroGPU target: {guard.get('zero_target')}.\n" - "Patch the publishable app.py minimally so it compiles, exposes the declared /generate endpoint, preserves the media output contract, and keeps @spaces.GPU when ZeroGPU is targeted.\n\n" + "Patch the publishable app.py minimally so it compiles, exposes the declared /generate endpoint, preserves the media output contract, returns structured /health fields required by APP_RUNTIME_CONTRACT.json, and keeps @spaces.GPU when ZeroGPU is targeted.\n\n" "Current app.py excerpt:\n" + app_excerpt ) @@ -6003,7 +6236,7 @@ def ensure_runtime_template_uploadable_with_pi_repair( """Run pre-upload guard, then execute one Pi repair pass if unresolved defects remain. Guards must protect ASF from publishing broken apps, but they should not be - terminal guillotines for repairable template defects. v198.26.28 guarantees + terminal guillotines for repairable template defects. v198.26.29 guarantees that a `pi_repair_required` pre-upload state is followed by either a repaired workspace that passes the guard, or an explicit terminal repair outcome. """ @@ -6034,13 +6267,13 @@ def ensure_runtime_template_uploadable_with_pi_repair( "failure_class": primary_problem, "failure_phase": "pre_upload_runtime_template_integrity", "logs_quality": "workspace_static_analysis", - "recommendation": "Patch app.py so the generated Space is publishable: compile cleanly, expose the declared /generate endpoint, preserve the media output type, and include @spaces.GPU for ZeroGPU.", + "recommendation": "Patch app.py so the generated Space is publishable: compile cleanly, expose the declared /generate endpoint, return structured /health fields, preserve the media output type, and include @spaces.GPU for ZeroGPU.", }, "constraints": [ "Patch only publishable runtime files needed to pass pre-upload integrity.", "Preserve the real model inference path and expected output type.", "Do not replace the app with diagnostics or placeholders.", - "Keep /health and /generate API contracts coherent with INFERENCE_CONTRACT.json.", + "Keep /health and /generate API contracts coherent with APP_RUNTIME_CONTRACT.json and INFERENCE_CONTRACT.json.", ], } write_json(run_dir / "pre_upload_pi_repair_decision.json", decision) @@ -8153,20 +8386,437 @@ def write_pi_planning_review(workspace: Path, run_dir: Path, events_path: Path, + +def _recipe_file_names(repo_tree: dict | None, model_analysis: dict | None = None) -> list[str]: + repo_tree = repo_tree if isinstance(repo_tree, dict) else {} + model_analysis = model_analysis if isinstance(model_analysis, dict) else {} + names: list[str] = [] + for key in ("sample_files", "top_level_entries"): + values = repo_tree.get(key) + if isinstance(values, list): + names.extend(str(v) for v in values if str(v).strip()) + siblings = model_analysis.get("siblings") + if isinstance(siblings, list): + names.extend(str(v) for v in siblings if str(v).strip()) + seen: set[str] = set() + out: list[str] = [] + for name in names: + clean = str(name).strip().strip("/") + if clean and clean not in seen: + seen.add(clean) + out.append(clean) + return out + + +def _recipe_extract_base_model(card: str, model_analysis: dict | None = None) -> str: + text = card or "" + candidates = [] + # Model card frontmatter often contains `base_model: owner/name`. + for pattern in [ + r"(?im)^\s*base_model\s*:\s*['\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", + r"(?im)^\s*base[_ -]?model[_ -]?id\s*:\s*['\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", + r"(?i)base model(?:\s+is|\s*:)?\s+[`'\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", + r"(?i)based on\s+[`'\"]?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", + ]: + candidates.extend(re.findall(pattern, text)) + if isinstance(model_analysis, dict): + for key in ("base_model", "base_model_id"): + value = model_analysis.get(key) + if isinstance(value, str): + candidates.append(value) + for value in candidates: + clean = str(value or "").strip().strip("`'\".,;)") + if "/" in clean: + return clean + return "" + + +def _recipe_extract_trigger_words(card: str) -> list[str]: + text = card or "" + triggers: list[str] = [] + for pattern in [ + r"(?i)trigger\s+word[s]?\s*[:\-]?\s*[`'\"]?([^`'\"\n,;]+)", + r"(?i)trigger\s*[:\-]?\s*[`'\"]?([^`'\"\n,;]+)", + ]: + for match in re.findall(pattern, text): + for part in re.split(r"[,/]|\band\b", str(match)): + clean = part.strip().strip("`'\".:- ") + if clean and len(clean) <= 80 and clean not in triggers: + triggers.append(clean) + # Common LoRA trigger tokens are often shown in examples but not labelled. + for token in re.findall(r"\b(?:sdxl|flux|lora)[-_][A-Za-z0-9_.-]{3,}\b", text, flags=re.IGNORECASE): + if token not in triggers: + triggers.append(token) + return triggers[:8] + + +def _recipe_weight_files(files: list[str]) -> list[str]: + return [name for name in files if name.lower().endswith((".safetensors", ".ckpt", ".bin", ".pt", ".pth", ".gguf", ".onnx"))] + + +def _recipe_first_file(files: list[str], suffixes: tuple[str, ...]) -> str: + for name in files: + if name.lower().endswith(suffixes): + return name + return "" + + +def classify_model_recipe_family(model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> dict: + """Return the worker-owned model-family decision used to constrain Pi. + + This is metadata/readme/repo-tree based and intentionally conservative. It + does not decide terminal feasibility by itself; it gives Pi and later guards + an authoritative recipe to compare against. + """ + model_analysis = model_analysis or {} + analysis_inputs = analysis_inputs or {} + card = str(analysis_inputs.get("model_card.md") or "") + card_low = card.lower() + tags = [str(t).lower() for t in (model_analysis.get("tags") or []) if str(t).strip()] + pipeline_tag = str(model_analysis.get("pipeline_tag") or "").lower() + library_name = str(model_analysis.get("library_name") or "").lower() + repo_tree = analysis_inputs.get("model_repo_tree.json") if isinstance(analysis_inputs.get("model_repo_tree.json"), dict) else {} + components = [str(c).lower() for c in (repo_tree.get("detected_components") or [])] + files = _recipe_file_names(repo_tree, model_analysis) + lower_files = [f.lower() for f in files] + weight_files = _recipe_weight_files(files) + has_model_index = any(f.endswith("model_index.json") or f == "model_index.json" for f in lower_files) or "diffusers_model_index" in components + has_diffusers_components = bool({"transformer", "text_encoder", "tokenizer", "vae", "scheduler"} & set(components)) + has_lora = ( + "lora" in components + or "lora" in tags + or "adapter" in tags + or "peft" in tags + or "lora" in card_low + or any("lora" in f for f in lower_files) + ) + has_sdxl = any(x in card_low for x in ["sdxl", "stable diffusion xl", "stable-diffusion-xl"]) or any("sdxl" in t for t in tags) or "xl" in model_id.lower() + has_flux = "flux" in model_id.lower() or "fluxpipeline" in card_low or any("flux" in t for t in tags) + has_gguf = any(f.endswith(".gguf") for f in lower_files) or "gguf" in tags + has_onnx = any(f.endswith(".onnx") for f in lower_files) or "onnx" in tags + has_single_file_checkpoint = any(f.endswith((".safetensors", ".ckpt")) for f in lower_files) and not has_model_index and not has_diffusers_components + expected = "image" if pipeline_tag in {"text-to-image", "image-to-image"} or "text-to-image" in tags or "diffusers" in tags else "text" + if pipeline_tag in {"text-to-video", "image-to-video"}: + expected = "video" + elif pipeline_tag in {"text-to-audio", "text-to-speech", "audio-to-audio"}: + expected = "audio" + elif pipeline_tag in {"automatic-speech-recognition"}: + expected = "text" + + family = "generic_model" + artifact_role = "unknown" + loader_strategy = "inspect_model_card_then_select_loader" + base_model_id = _recipe_extract_base_model(card, model_analysis) + adapter_id = "" + weight_name = _recipe_first_file(weight_files, (".safetensors", ".bin", ".pt", ".pth")) + evidence: list[str] = [] + known_failure_signatures: list[dict] = [] + smoke_strategy = "generic_minimal_smoke" + required_code_markers: list[str] = [] + guard_expectations: list[str] = [] + + if has_gguf: + family = "gguf_llamacpp" + artifact_role = "quantized_runtime" + loader_strategy = "llama_cpp_or_compatible_gguf_runtime" + expected = "text" if expected == "text" else expected + smoke_strategy = "short_text_generation_smoke" + required_code_markers = ["gguf_runtime_or_llama_cpp_loader", "/generate"] + evidence.append("repo_contains_gguf_weight") + elif has_onnx: + family = "onnx_runtime_model" + artifact_role = "onnx_graph" + loader_strategy = "onnxruntime_session_or_task_specific_pipeline" + smoke_strategy = "task_specific_onnx_smoke" + required_code_markers = ["onnxruntime_or_transformers_onnx_loader", "/generate"] + evidence.append("repo_contains_onnx_weight") + elif (library_name == "diffusers" or "diffusers" in tags or pipeline_tag in {"text-to-image", "image-to-image", "text-to-video", "image-to-video"}) and has_lora and has_sdxl and not has_model_index: + family = "diffusers_sdxl_lora_adapter" + artifact_role = "adapter" + adapter_id = model_id + loader_strategy = "sdxl_base_from_pretrained_or_single_file_then_load_lora_weights" + expected = "image" + smoke_strategy = "sdxl_lora_trigger_word_low_steps_image_smoke" + required_code_markers = ["StableDiffusionXLPipeline", "load_lora_weights", "/generate", "@spaces.GPU_if_zero_gpu"] + guard_expectations = ["base_model_resolved", "adapter_loaded", "health_exposes_model_error"] + known_failure_signatures.append({"match": "CLIPTextModel object has no attribute text_model", "failure_class": "sdxl_lora_text_encoder_mismatch", "repair_hint": "base_or_loader_incompatible; try alternate SDXL base loader or dependency matrix"}) + evidence.extend(["lora_marker_present", "sdxl_marker_present", "no_diffusers_model_index"]) + elif (library_name == "diffusers" or "diffusers" in tags or pipeline_tag in {"text-to-image", "image-to-image", "text-to-video", "image-to-video"}) and has_flux: + family = "diffusers_flux_pipeline" + artifact_role = "full_model" + loader_strategy = "FluxPipeline.from_pretrained" + expected = "image" + smoke_strategy = "diffusers_image_low_steps_schema_aware_smoke" + required_code_markers = ["FluxPipeline", "/generate", "@spaces.GPU_if_zero_gpu"] + guard_expectations = ["health_exposes_model_error", "generate_endpoint_exposed"] + known_failure_signatures.append({"match": "FluxPipeline", "failure_class": "diffusers_flux_runtime_error", "repair_hint": "preserve FluxPipeline.from_pretrained and reduce smoke steps/resolution"}) + evidence.append("flux_marker_present") + elif library_name == "diffusers" or "diffusers" in tags or pipeline_tag in {"text-to-image", "image-to-image", "text-to-video", "image-to-video"}: + family = "diffusers_full_pipeline" if has_model_index or has_diffusers_components else ("diffusers_single_file_checkpoint" if has_single_file_checkpoint else "diffusers_pipeline_or_checkpoint") + artifact_role = "full_model" if has_model_index or has_diffusers_components else ("single_file_checkpoint" if has_single_file_checkpoint else "unknown_diffusers_artifact") + loader_strategy = "DiffusionPipeline_or_family_specific_from_pretrained" if family != "diffusers_single_file_checkpoint" else "family_pipeline.from_single_file" + expected = "image" if expected == "text" else expected + smoke_strategy = "diffusers_media_low_cost_smoke" + required_code_markers = ["diffusers_pipeline_loader", "/generate"] + if expected in {"image", "video"}: + required_code_markers.append("@spaces.GPU_if_zero_gpu") + evidence.append("diffusers_or_media_task_marker_present") + elif library_name == "transformers" or "transformers" in tags or pipeline_tag: + family = "transformers_pipeline" + artifact_role = "full_model_or_processor_bundle" + loader_strategy = "transformers_pipeline_or_auto_model_for_task" + smoke_strategy = "task_specific_transformers_smoke" + required_code_markers = ["transformers_loader", "/generate_or_predict"] + evidence.append("transformers_or_pipeline_tag_marker_present") + + hardware = { + "preferred": "zero-a10g" if expected in {"image", "video", "audio"} or family.startswith("diffusers") else (model_analysis.get("preferred_hardware") or "cpu-basic"), + "fallback": "a10g-large" if expected in {"image", "video", "audio"} or family.startswith("diffusers") else (model_analysis.get("fallback_hardware") or "cpu-basic"), + "try_zero_gpu_first": bool(expected in {"image", "video", "audio"} or family.startswith("diffusers")), + "allow_fixed_gpu_fallback": bool(expected in {"image", "video", "audio"} or family.startswith("diffusers")), + } + return { + "model_family": family, + "artifact_role": artifact_role, + "loader_strategy": loader_strategy, + "base_model_id": base_model_id, + "adapter_id": adapter_id, + "adapter_weight_name": weight_name if artifact_role == "adapter" else "", + "primary_weight_name": weight_name if artifact_role != "adapter" else "", + "trigger_words": _recipe_extract_trigger_words(card), + "expected_output_type": expected, + "primary_endpoint": "/generate" if expected in {"image", "video", "audio", "text"} else "/predict", + "health_endpoint": "/health", + "health_contract": { + "endpoint": "/health", + "required_fields": ["status", "model_ready", "pipeline_ready", "model_family", "loader_strategy", "last_error", "expected_output_type"], + "cheap": True, + "must_not_load_model": True, + "model_ready_false_exposes_last_error": True, + }, + "generation_contract": { + "endpoint": "/generate" if expected in {"image", "video", "audio", "text"} else "/predict", + "must_fail_cleanly_when_not_ready": True, + "must_surface_model_load_error": True, + "no_placeholder_success": True, + }, + "hardware_strategy": hardware, + "smoke_test_strategy": smoke_strategy, + "required_code_markers": required_code_markers, + "guard_expectations": guard_expectations, + "known_failure_signatures": known_failure_signatures, + "evidence": evidence, + "repo_weight_files_sample": weight_files[:20], + } + + + +APP_RUNTIME_HEALTH_REQUIRED_FIELDS = ["status", "model_ready", "pipeline_ready", "model_family", "loader_strategy", "last_error", "expected_output_type"] + + +def build_app_runtime_contract(recipe: dict | None) -> dict: + recipe = recipe if isinstance(recipe, dict) else {} + health_contract = recipe.get("health_contract") if isinstance(recipe.get("health_contract"), dict) else {} + generation_contract = recipe.get("generation_contract") if isinstance(recipe.get("generation_contract"), dict) else {} + required_fields = health_contract.get("required_fields") if isinstance(health_contract.get("required_fields"), list) else APP_RUNTIME_HEALTH_REQUIRED_FIELDS + return { + "schema_version": "app_runtime_contract.v198_26_30", + "authority": "worker_required_generated_app_contract", + "model_family": recipe.get("model_family") or "generic_model", + "artifact_role": recipe.get("artifact_role") or "unknown", + "loader_strategy": recipe.get("loader_strategy") or "inspect_model_card_then_select_loader", + "expected_output_type": recipe.get("expected_output_type") or "any", + "primary_endpoint": recipe.get("primary_endpoint") or generation_contract.get("endpoint") or "/generate", + "health_endpoint": recipe.get("health_endpoint") or health_contract.get("endpoint") or "/health", + "required_health_fields": [str(x) for x in required_fields if str(x).strip()], + "health_must_be_cheap": True, + "health_must_not_load_model": True, + "generate_must_fail_cleanly_when_not_ready": True, + "generate_must_surface_model_load_error": True, + "generate_must_not_return_placeholder_success": True, + "smoke_test_strategy": recipe.get("smoke_test_strategy") or "generic_minimal_smoke", + "known_failure_signatures": recipe.get("known_failure_signatures") or [], + "notes": [ + "/health must return a structured payload, not only a transport OK.", + "When the model or pipeline is unavailable, /health must set model_ready=false or pipeline_ready=false and include last_error.", + "/generate must expose concrete model-load/runtime errors instead of TypeError/None placeholders.", + ], + } + + +def load_app_runtime_contract(workspace: Path, run_dir: Path | None = None) -> dict: + for base in [workspace, run_dir]: + if not base: + continue + path = base / "APP_RUNTIME_CONTRACT.json" + if path.exists(): + payload = load_json_if_exists(path) + if isinstance(payload, dict) and payload: + return payload + recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else {} + if (not isinstance(recipe, dict) or not recipe) and run_dir and (run_dir / "MODEL_RECIPE.json").exists(): + recipe = load_json_if_exists(run_dir / "MODEL_RECIPE.json") + return build_app_runtime_contract(recipe if isinstance(recipe, dict) else {}) + + +def _app_text_has_health_endpoint(app_text: str) -> bool: + low = str(app_text or "").lower() + return bool('/health' in low or 'api_name="health"' in low or "api_name='health'" in low or re.search(r"def\s+health\s*\(", app_text or "")) + + +def _app_text_mentions_primary_endpoint(app_text: str, endpoint: str) -> bool: + endpoint = normalize_api_name(endpoint or "/generate") + if endpoint == "/generate": + return _has_generate_api_endpoint(app_text) + bare = endpoint.lstrip("/") + return bool(endpoint in app_text or f'api_name="{bare}"' in app_text or f"api_name='{bare}'" in app_text or f'api_name="{endpoint}"' in app_text or f"api_name='{endpoint}'" in app_text) + + +def app_runtime_contract_guard(workspace: Path, run_dir: Path, events_path: Path, *, expected_output_type: str = "", reason: str = "pre_upload") -> dict: + """Verify the v198.26.30 family-aware generated app contract before upload. + + This guard is intentionally static and cheap. It does not prove inference + success; it proves Pi produced an app that exposes the right health and + generation surfaces for the live validator and future recipe-aware repair. + """ + app_path = workspace / "app.py" + app_text = app_path.read_text(encoding="utf-8", errors="ignore") if app_path.exists() else "" + contract = read_inference_contract(workspace) + runtime_contract = load_app_runtime_contract(workspace, run_dir) + recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else load_json_if_exists(run_dir / "MODEL_RECIPE.json") + if not isinstance(recipe, dict): + recipe = {} + required_health_fields = runtime_contract.get("required_health_fields") if isinstance(runtime_contract.get("required_health_fields"), list) else APP_RUNTIME_HEALTH_REQUIRED_FIELDS + required_health_fields = [str(x) for x in required_health_fields if str(x).strip()] + missing_health_fields = [field for field in required_health_fields if field not in app_text] + expected = normalize_runtime_expected_output_type(expected_output_type or contract.get("expected_output_type") or runtime_contract.get("expected_output_type") or recipe.get("expected_output_type") or "") + primary = normalize_api_name(str(contract.get("primary_api_name") or contract.get("primary_endpoint") or runtime_contract.get("primary_endpoint") or recipe.get("primary_endpoint") or "/generate")) + full_inference = contract.get("full_inference_implemented") is True + blockers = load_json_if_exists(workspace / "TECHNICAL_BLOCKERS.json") if (workspace / "TECHNICAL_BLOCKERS.json").exists() else {} + try: + blockers_count = len(blockers.get("blockers") or []) if isinstance(blockers, dict) and isinstance(blockers.get("blockers"), list) else int(contract.get("blockers_count") or 0) + except Exception: + blockers_count = 0 + has_health = _app_text_has_health_endpoint(app_text) + has_primary = _app_text_mentions_primary_endpoint(app_text, primary) + not_ready_markers = ["last_error", "model_ready", "pipeline_ready", "not ready", "not_ready", "not loaded", "not_loaded", "load_error", "pipeline_error", "model_error"] + has_not_ready_error_path = any(marker in app_text.lower() for marker in not_ready_markers) + problems: list[str] = [] + repairable: list[str] = [] + if not has_health: + problems.append("app_runtime_contract_missing_health_endpoint") + repairable.append("app_runtime_contract_missing_health_endpoint") + if missing_health_fields: + problems.append("app_runtime_contract_missing_structured_health_fields") + repairable.append("app_runtime_contract_missing_structured_health_fields") + if full_inference and primary and primary != "/health" and not has_primary: + problems.append("app_runtime_contract_missing_primary_generation_endpoint") + repairable.append("app_runtime_contract_missing_primary_generation_endpoint") + if full_inference and not has_not_ready_error_path: + problems.append("app_runtime_contract_missing_not_ready_error_path") + repairable.append("app_runtime_contract_missing_not_ready_error_path") + if expected in {"image", "video", "audio"} and full_inference and primary == "/health": + problems.append("app_runtime_contract_primary_endpoint_is_health_for_media") + repairable.append("app_runtime_contract_primary_endpoint_is_health_for_media") + payload = { + "schema_version": "app_runtime_contract_guard.v198_26_30", + "reason": reason, + "passed": not problems, + "problems": problems, + "repairable_unresolved_problems": repairable, + "model_family": runtime_contract.get("model_family") or recipe.get("model_family") or contract.get("model_family") or "", + "artifact_role": runtime_contract.get("artifact_role") or recipe.get("artifact_role") or contract.get("artifact_role") or "", + "loader_strategy": runtime_contract.get("loader_strategy") or recipe.get("loader_strategy") or contract.get("loader_strategy") or "", + "expected_output_type": expected, + "primary_endpoint": primary, + "health_endpoint": runtime_contract.get("health_endpoint") or "/health", + "required_health_fields": required_health_fields, + "missing_health_fields": missing_health_fields, + "has_health_endpoint": has_health, + "has_primary_endpoint": has_primary, + "has_not_ready_error_path": has_not_ready_error_path, + "full_inference_implemented": full_inference, + "blockers_count": blockers_count, + "technical_blockers_present": bool(blockers_count), + "runtime_contract": runtime_contract, + } + write_json(run_dir / "app_runtime_contract_guard.json", payload) + append_event(events_path, "app_runtime_contract_guard", "success" if payload["passed"] else "failed", "Checked family-aware /health and /generate app contract", payload) + return payload + +def recipe_runtime_alignment_guard(workspace: Path, run_dir: Path, events_path: Path, *, reason: str = "pre_upload") -> dict: + """Check generated app shape against the worker-owned MODEL_RECIPE. + + v198.26.30 keeps this recipe alignment diagnostic non-fatal; + APP_RUNTIME_CONTRACT.json is enforced separately by the pre-upload guard. + """ + recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") if (workspace / "MODEL_RECIPE.json").exists() else {} + if not isinstance(recipe, dict) or not recipe: + recipe = load_json_if_exists(run_dir / "MODEL_RECIPE.json") if (run_dir / "MODEL_RECIPE.json").exists() else {} + app_text = (workspace / "app.py").read_text(encoding="utf-8", errors="ignore") if (workspace / "app.py").exists() else "" + family = str(recipe.get("model_family") or "") + expected = normalize_runtime_expected_output_type(str(recipe.get("expected_output_type") or "")) + mismatches: list[str] = [] + satisfied: list[str] = [] + if family == "diffusers_sdxl_lora_adapter": + for marker, label in [("load_lora_weights", "missing_lora_loader"), ("StableDiffusionXLPipeline", "missing_sdxl_pipeline"), ("api_name=\"generate\"", "missing_generate_endpoint")]: + alt = marker.replace('"', "'") + if marker in app_text or alt in app_text: + satisfied.append(label.replace("missing_", "has_")) + else: + mismatches.append(label) + if not recipe.get("base_model_id"): + mismatches.append("sdxl_lora_base_model_unresolved") + elif family == "diffusers_flux_pipeline": + if "FluxPipeline" in app_text: + satisfied.append("has_flux_pipeline") + else: + mismatches.append("missing_flux_pipeline") + if expected in {"image", "video", "audio"} and not _has_generate_api_endpoint(app_text): + mismatches.append("media_recipe_missing_generate_endpoint") + payload = { + "schema_version": "recipe_runtime_alignment_guard.v198_26_30", + "reason": reason, + "non_blocking": True, + "recipe_present": bool(recipe), + "model_family": family, + "expected_output_type": expected, + "mismatches": mismatches, + "satisfied": satisfied, + "passed": not mismatches, + } + write_json(run_dir / "recipe_runtime_alignment_guard.json", payload) + append_event(events_path, "recipe_runtime_alignment_guard", "success" if not mismatches else "warning", "Checked generated runtime against worker-owned model recipe", payload) + return payload + def build_model_recipe(model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> dict: model_analysis = model_analysis or {} analysis_inputs = analysis_inputs or {} card = str(analysis_inputs.get("model_card.md") or "") card_low = card.lower() repo_tree = analysis_inputs.get("model_repo_tree.json") if isinstance(analysis_inputs.get("model_repo_tree.json"), dict) else {} + family = classify_model_recipe_family(model_id, model_analysis, analysis_inputs) dep_hints = [] - for pkg in ["diffusers", "transformers", "sentencepiece", "protobuf", "accelerate", "safetensors", "peft", "imageio", "imageio-ffmpeg", "torchvision", "spaces", "gradio"]: + for pkg in ["diffusers", "transformers", "sentencepiece", "protobuf", "accelerate", "safetensors", "peft", "imageio", "imageio-ffmpeg", "torchvision", "spaces", "gradio", "onnxruntime", "llama-cpp-python"]: if pkg.lower() in card_low or pkg in json.dumps(repo_tree)[:20000].lower(): dep_hints.append(pkg) + # Family-derived dependencies are authoritative hints even when the card omits them. + if str(family.get("model_family")) == "diffusers_sdxl_lora_adapter": + for pkg in ["diffusers", "safetensors", "peft", "transformers", "accelerate", "spaces"]: + if pkg not in dep_hints: + dep_hints.append(pkg) + if str(family.get("model_family")) == "diffusers_flux_pipeline": + for pkg in ["diffusers", "transformers", "accelerate", "safetensors", "spaces"]: + if pkg not in dep_hints: + dep_hints.append(pkg) pipeline_hints = [] for name in re.findall(r"\b[A-Za-z0-9_]*(?:Pipeline|Model|Processor|Tokenizer)\b", card)[:40]: if name not in pipeline_hints: pipeline_hints.append(name) + if family.get("model_family") == "diffusers_sdxl_lora_adapter" and "StableDiffusionXLPipeline" not in pipeline_hints: + pipeline_hints.insert(0, "StableDiffusionXLPipeline") + if family.get("model_family") == "diffusers_flux_pipeline" and "FluxPipeline" not in pipeline_hints: + pipeline_hints.insert(0, "FluxPipeline") constraints = [] if "divisible by 32" in card_low or "divisible_by_32" in card_low: constraints.append("resolution_divisible_by_32") @@ -8177,8 +8827,9 @@ def build_model_recipe(model_id: str, model_analysis: dict | None = None, analys if "fp16" in card_low or "float16" in card_low: constraints.append("fp16_hint") return { - "schema_version": "model_recipe.v198_25", - "purpose": "Context guide for Pi initial build. This does not replace the full model card; it indexes critical facts to verify in analysis_inputs/model_card.md.", + "schema_version": "model_recipe.v198_26_30", + "authority": "worker_authoritative_runtime_planning", + "purpose": "Worker-owned model-family recipe. Pi must implement code consistent with this recipe or explicitly write TECHNICAL_BLOCKERS.json; live validation remains the final success authority.", "model_id": model_id, "pipeline_tag": model_analysis.get("pipeline_tag"), "library_name": model_analysis.get("library_name"), @@ -8187,11 +8838,29 @@ def build_model_recipe(model_id: str, model_analysis: dict | None = None, analys "recommended_pipeline_or_class_hints": pipeline_hints[:20], "dependency_hints": dep_hints, "input_constraints_hints": constraints, - "source_priority": ["analysis_inputs/model_card.md", "analysis_inputs/model_repo_tree.json", "analysis_inputs/source_policy.md", "analysis_inputs/hf_spaces_operational_gist.md"], - "warning": "Pi must verify these hints against the full model card and official examples before coding.", + "model_family": family.get("model_family"), + "artifact_role": family.get("artifact_role"), + "loader_strategy": family.get("loader_strategy"), + "base_model_id": family.get("base_model_id"), + "adapter_id": family.get("adapter_id"), + "adapter_weight_name": family.get("adapter_weight_name"), + "primary_weight_name": family.get("primary_weight_name"), + "trigger_words": family.get("trigger_words", []), + "expected_output_type": family.get("expected_output_type"), + "primary_endpoint": family.get("primary_endpoint"), + "health_endpoint": family.get("health_endpoint"), + "hardware_strategy": family.get("hardware_strategy"), + "smoke_test_strategy": family.get("smoke_test_strategy"), + "required_code_markers": family.get("required_code_markers", []), + "guard_expectations": family.get("guard_expectations", []), + "known_failure_signatures": family.get("known_failure_signatures", []), + "app_runtime_contract": build_app_runtime_contract(family), + "recipe_evidence": family.get("evidence", []), + "repo_weight_files_sample": family.get("repo_weight_files_sample", []), + "source_priority": ["MODEL_RECIPE.json", "analysis_inputs/model_card.md", "analysis_inputs/model_repo_tree.json", "analysis_inputs/source_policy.md", "analysis_inputs/hf_spaces_operational_gist.md"], + "warning": "Pi must implement a runtime matching this recipe. If the model card contradicts it, write the contradiction in pi_feasibility_brief.json and do not silently change model family or output type.", } - def build_context_index(analysis_inputs: dict | None = None) -> dict: analysis_inputs = analysis_inputs or {} files = [] @@ -8202,6 +8871,7 @@ def build_context_index(analysis_inputs: dict | None = None) -> dict: ("analysis_inputs/source_policy.md", "Source priority and snippet distrust policy."), ("analysis_inputs/hf_spaces_operational_gist.md", "HF Spaces operational guidance snapshot/fallback."), ("MODEL_RECIPE.json", "Compact worker-generated guide. Start here, then verify in full sources."), + ("APP_RUNTIME_CONTRACT.json", "Mandatory generated-app contract for structured /health, /generate error behavior, and family-aware smoke validation."), ("SOURCE_PRIORITY.md", "Short source ordering guide for Pi."), ]: files.append({"path": name, "description": description}) @@ -8211,11 +8881,27 @@ def build_context_index(analysis_inputs: dict | None = None) -> dict: def write_initial_pi_context_artifacts(workspace: Path, model_id: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None) -> list[str]: recipe = build_model_recipe(model_id, model_analysis, analysis_inputs) index = build_context_index(analysis_inputs) - source_priority = """# Source priority for Pi\n\nStart with `MODEL_RECIPE.json` and `CONTEXT_INDEX.json` to orient yourself, but do not treat them as replacements for the rich sources. Verify critical claims in this order:\n\n1. `analysis_inputs/model_card.md` official/model-card instructions and examples.\n2. `analysis_inputs/model_repo_tree.json` repo structure, configs, component names, weights.\n3. `analysis_inputs/source_policy.md` trust rules for snippets and generated examples.\n4. `analysis_inputs/hf_spaces_operational_gist.md` HF Spaces operational constraints.\n\nFor initial build, you are expected to use the rich context. For later repairs, ASF will provide narrower task packets.\n""" + app_runtime_contract = build_app_runtime_contract(recipe) + source_priority = """# Source priority for Pi + +Start with `MODEL_RECIPE.json`. It is the worker-owned runtime plan for model family, artifact role, loader strategy, expected output type, endpoint, smoke strategy, and hardware. Then apply `APP_RUNTIME_CONTRACT.json`, which is the mandatory generated-app contract for structured `/health`, `/generate` error behavior, and family-aware smoke validation. Treat both as authoritative unless the model card/repo tree directly contradicts them; if so, write the contradiction explicitly in planning artifacts. + +Verify critical facts in this order: + +1. `MODEL_RECIPE.json` worker-owned runtime plan and required code markers. +2. `APP_RUNTIME_CONTRACT.json` mandatory generated-app health/generation contract. +3. `analysis_inputs/model_card.md` official/model-card instructions and examples. +4. `analysis_inputs/model_repo_tree.json` repo structure, configs, component names, weights. +5. `analysis_inputs/source_policy.md` trust rules for snippets and generated examples. +6. `analysis_inputs/hf_spaces_operational_gist.md` HF Spaces operational constraints. + +For initial build, you are expected to use the rich context. For later repairs, ASF will provide narrower task packets. +""" write_json(workspace / "MODEL_RECIPE.json", recipe) + write_json(workspace / "APP_RUNTIME_CONTRACT.json", app_runtime_contract) write_json(workspace / "CONTEXT_INDEX.json", index) (workspace / "SOURCE_PRIORITY.md").write_text(source_priority, encoding="utf-8") - return ["MODEL_RECIPE.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md"] + return ["MODEL_RECIPE.json", "APP_RUNTIME_CONTRACT.json", "CONTEXT_INDEX.json", "SOURCE_PRIORITY.md"] def create_initial_workspace(workspace: Path, model_id: str, target_space_id: str, preferred_hardware: str, fallback_hardware: str, allow_fallback: bool, implementation_mode: str, model_analysis: dict | None = None, analysis_inputs: dict | None = None): workspace.mkdir(parents=True, exist_ok=True) @@ -8226,33 +8912,48 @@ def create_initial_workspace(workspace: Path, model_id: str, target_space_id: st siblings = model_analysis.get("siblings", [])[:60] analysis_inputs_result = write_analysis_inputs_dir(workspace, analysis_inputs) initial_context_artifacts = write_initial_pi_context_artifacts(workspace, model_id, model_analysis, analysis_inputs) + initial_recipe = load_json_if_exists(workspace / "MODEL_RECIPE.json") + if not isinstance(initial_recipe, dict): + initial_recipe = {} app_py = f"""import gradio as gr from huggingface_hub import model_info, list_repo_files MODEL_ID = {model_id!r} TARGET_SPACE_ID = {target_space_id!r} +MODEL_FAMILY = {str(initial_recipe.get("model_family") or "generic_model")!r} +LOADER_STRATEGY = {str(initial_recipe.get("loader_strategy") or "inspect_model_card_then_select_loader")!r} +EXPECTED_OUTPUT_TYPE = {str(initial_recipe.get("expected_output_type") or "any")!r} +LAST_ERROR = "Initial scaffold has not loaded the model yet. Pi must replace it with a model-specific runtime or write TECHNICAL_BLOCKERS.json." def health(): return {{ - "status": "booted", + "status": "unavailable", + "model_ready": False, + "pipeline_ready": False, + "model_family": MODEL_FAMILY, + "loader_strategy": LOADER_STRATEGY, + "last_error": LAST_ERROR, + "expected_output_type": EXPECTED_OUTPUT_TYPE, "model_id": MODEL_ID, "target_space_id": TARGET_SPACE_ID, "stage": "initial-scaffold", - "note": "Pi should replace this scaffold with a model-specific demo while preserving a cheap health endpoint.", + "note": "Pi should replace this scaffold with a model-specific demo while preserving this structured health contract.", }} -def placeholder(*args): - return "Initial scaffold. Pi should replace this with a model-specific inference path, or write TECHNICAL_BLOCKERS.json." +def generate(prompt="Hello from Agentic Space Factory"): + raise RuntimeError(LAST_ERROR) with gr.Blocks(title="Generated Model Space — Agentic Space Factory") as demo: gr.Markdown("# Generated Model Space — Agentic Space Factory") gr.Markdown(f"Private generated Space for `{{MODEL_ID}}`.") gr.JSON(label="Health", value=health(), every=None) gr.Button("Health check").click(fn=health, inputs=None, outputs=gr.JSON(), api_name="health") - gr.Textbox(label="Input", value="Hello from Agentic Space Factory").submit(fn=placeholder, inputs=None, outputs=gr.Textbox(), api_name="predict") - gr.Button("Run placeholder").click(fn=placeholder, inputs=None, outputs=gr.Textbox(), api_name="predict") + prompt = gr.Textbox(label="Input", value="Hello from Agentic Space Factory") + output = gr.Textbox(label="Output") + prompt.submit(fn=generate, inputs=prompt, outputs=output, api_name="generate") + gr.Button("Run").click(fn=generate, inputs=prompt, outputs=output, api_name="generate") if __name__ == "__main__": demo.launch() @@ -8309,7 +9010,9 @@ BUILD-TIME MODEL ANALYSIS INPUTS: - `analysis_inputs/hf_spaces_operational_gist.md`: run-local HF Spaces operational gist snapshot or fallback summary. - `analysis_inputs/hf_spaces_gist_source.json`: fetch status, URL fallback, and hash/source metadata for the gist guidance. -Before writing code, first read `MODEL_RECIPE.json`, `CONTEXT_INDEX.json`, and `SOURCE_PRIORITY.md` to orient yourself, then read `analysis_inputs/model_card.md`, `analysis_inputs/model_repo_tree.json`, `analysis_inputs/source_policy.md`, `analysis_inputs/hf_spaces_operational_gist.md`, and `analysis_inputs/hf_spaces_gist_source.json`. Treat the model card as the canonical source when it was resolved from README.md. Auto-generated Hugging Face “Use this model” snippets are hints only and must not override README.md/model-card instructions. The recipe/index guide you through the rich context; they do not replace it. +Before writing code, first read `MODEL_RECIPE.json`, `APP_RUNTIME_CONTRACT.json`, `CONTEXT_INDEX.json`, and `SOURCE_PRIORITY.md`, then read `analysis_inputs/model_card.md`, `analysis_inputs/model_repo_tree.json`, `analysis_inputs/source_policy.md`, `analysis_inputs/hf_spaces_operational_gist.md`, and `analysis_inputs/hf_spaces_gist_source.json`. + +`MODEL_RECIPE.json` is the worker-owned runtime plan for model family, artifact role, loader strategy, output type, endpoint, health contract, smoke test strategy, and hardware. `APP_RUNTIME_CONTRACT.json` is the mandatory generated-app contract for structured `/health`, `/generate` not-ready behavior, and family-aware validation. Treat it as authoritative unless you can cite a direct contradiction from the model card or repo tree. If you disagree with the recipe, write the contradiction explicitly in `pi_feasibility_brief.json`; do not silently change family, output type, or endpoint. Treat the model card as canonical for model facts when it was resolved from README.md. Auto-generated Hugging Face “Use this model” snippets are hints only and must not override README.md/model-card instructions. {hf_spaces_guidance_goal_section()} @@ -8348,6 +9051,9 @@ Implementation contract: - Try to implement the closest real inference path for the model card using evidence from `analysis_inputs/model_card.md`, model metadata, config files, and repo files. If a generic snippet conflicts with the README/model-card, the README/model-card wins. - You may choose an appropriate Gradio UI for the task: text, image, audio, video, multimodal, embeddings, classification, etc. - If the model is standard and feasible, implement a real generate/predict function and expose it as a Gradio endpoint. +- Your app must conform to `MODEL_RECIPE.json`: preserve `model_family`, `artifact_role`, `loader_strategy`, `expected_output_type`, `primary_endpoint`, and `smoke_test_strategy`. For adapters such as SDXL LoRA, implement the base-model + adapter relationship explicitly; do not load an adapter repo as if it were a standalone full model. +- Your app must conform to `APP_RUNTIME_CONTRACT.json`: expose `/health` with literal structured fields `status`, `model_ready`, `pipeline_ready`, `model_family`, `loader_strategy`, `last_error`, and `expected_output_type`. Health must be cheap and must not trigger model loading or GPU work. +- `/generate` must fail cleanly when the model/pipeline is not ready: raise or return a structured error containing the concrete load/runtime error, not a vague `TypeError`, `None`, or UI-only message. If the model load hits `CLIPTextModel.text_model` or another family-specific known error, surface that exact error in both `/health.last_error` and `/generate`. - If the model requires GPU, add ZeroGPU-compatible `@spaces.GPU(...)` only around the inference function and preserve fixed-GPU compatibility. Do not decorate health. Do not assume dedicated A100/H200 hardware is available automatically; if such hardware is needed, document it as a manual requirement. If a pipeline load fails, `/generate` should raise the concrete load error instead of a vague silent `None` state. - If the model requires special dependencies, include them only when needed and document risks. - Investigate compatibility fallbacks before declaring a blocker: PyTorch SDPA, xformers, HF Kernels/Kernel Hub where relevant, Transformers AttentionInterface, Diffusers attention processors, CPU/offload/lazy loading, smaller resolution/steps, safe smoke-test inputs. @@ -8361,10 +9067,10 @@ Deliverables: - MODEL_DEPENDENCIES.json should summarize whether the repo is standalone or dependent, including LoRA/PEFT/adapter/component/base-model dependencies when present. - DEMO_QUALITY_CONTRACT.json should summarize the model-card promise, the primary user flow, Gradio examples, canonical_smoke_example, whether real inference is implemented, and any promise fulfillment risks. - app.py must boot on Hugging Face Spaces. -- app.py must expose health/api_name="health". +- app.py must expose health/api_name="health" and return the structured `APP_RUNTIME_CONTRACT.json` fields. - If real generation is implemented, generate/predict must attempt a real model call, not only return a textual diagnostic. - If real generation is not implemented, write TECHNICAL_BLOCKERS.json with: full_inference_implemented=false, blockers[], evidence[], minimum_runtime, and suggested_next_step. -- Write INFERENCE_CONTRACT.json with: full_inference_implemented, inference_strategy, health_endpoint, primary_api_name, expected_output_type, validation_level, requires_gpu, recommended_target_space_hardware, estimated_vram, and blockers_count. +- Write INFERENCE_CONTRACT.json with: full_inference_implemented, inference_strategy, health_endpoint, primary_api_name, expected_output_type, validation_level, requires_gpu, recommended_target_space_hardware, estimated_vram, blockers_count, model_family, artifact_role, loader_strategy, app_runtime_contract_version, health_required_fields, generate_not_ready_behavior, and recipe_conformance_notes. - README.md must explain the runtime strategy, task, limitations, how to test, and at least one example input when the demo is not purely diagnostic. - Write PI_SOURCE_USAGE.json with: used_model_card, used_official_code_examples, used_repo_tree, used_hf_spaces_gist, critical_claims_verified[], and any skipped_sources[]. This is an audit artifact only; it must not block implementation. - Write a concise PI_SUMMARY.md with what you changed, whether full inference is implemented, and any `gist_rules_used` if they are not already captured in JSON. @@ -9317,7 +10023,7 @@ def normalize_repair_decision(decision: dict, classification: dict, budgets: dic return normalized -def build_repair_task_packet(classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None) -> dict: +def build_repair_task_packet(classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None, *, workspace: Path | None = None, run_dir: Path | None = None) -> dict: category = (classification or {}).get("category") or "unknown_runtime_error" signature = compute_failure_signature(failure_reason, build_log, runtime_log, classification=classification or {}) evidence = [] @@ -9331,6 +10037,11 @@ def build_repair_task_packet(classification: dict, failure_reason: str = "", bui allowed_files = ["app.py", "requirements.txt"] expected_fix = "Apply the smallest code/dependency patch that addresses the classified failure." patch_mode = "pi_targeted_patch" + recipe_packet = write_recipe_aware_repair_packet(workspace, run_dir, failure_reason, classification, decision) if workspace is not None and run_dir is not None else {"known_failure_match": match_recipe_known_failure_signature(failure_reason, {}, {}, fallback_class=category)} + known_failure_match = recipe_packet.get("known_failure_match") if isinstance(recipe_packet, dict) else {} + matched_class = str((known_failure_match or {}).get("failure_class") or category) + if matched_class and matched_class != category: + category = matched_class if category == "dependency_error": task_type = "dependency_repair" allowed_files = ["requirements.txt", "requirements_policy.json"] @@ -9360,8 +10071,28 @@ def build_repair_task_packet(classification: dict, failure_reason: str = "", bui allowed_files = ["app.py"] expected_fix = "Only apply concrete memory optimizations supported by MEMORY_DIAGNOSIS.json evidence." patch_mode = "pi_memory_patch_after_diagnosis" + elif category == "sdxl_lora_text_encoder_mismatch": + task_type = "recipe_aware_sdxl_lora_loader_repair" + allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] + expected_fix = "Repair the SDXL LoRA adapter loader according to MODEL_RECIPE.json: base SDXL pipeline first, then load_lora_weights(adapter_id, weight_name), with health.last_error preserving load failures." + elif category == "flux_template_syntax_error": + task_type = "recipe_aware_flux_runtime_repair" + allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] + expected_fix = "Patch FLUX runtime/template handling while preserving FluxPipeline image generation and the family-aware runtime contract." + elif category == "missing_spaces_gpu_decorator": + task_type = "zerogpu_decorator_repair" + allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json"] + expected_fix = "Add/restore the spaces import and @spaces.GPU protection around heavy inference without touching model semantics." + elif category == "model_not_loaded_after_boot": + task_type = "runtime_loader_contract_repair" + allowed_files = ["app.py", "requirements.txt", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] + expected_fix = "Fix model initialization so /health reports readiness accurately and /generate fails cleanly instead of raising vague NoneType/model-not-loaded errors." + elif category == "validator_schema_choice_type_mismatch": + task_type = "validator_payload_or_schema_alignment" + allowed_files = ["app.py", "INFERENCE_CONTRACT.json", "APP_RUNTIME_CONTRACT.json"] + expected_fix = "Align the published Gradio schema and canonical smoke payload without changing model semantics; do not patch if this is validator-owned." packet = { - "schema_version": "repair_task_packet.v198_25", + "schema_version": "repair_task_packet.v198_26_31", "task_type": task_type, "repair_mode": "deep_repair_reanalysis" if is_deep_repair_category(category) else "surgical_repair", "patch_mode": patch_mode, @@ -9390,6 +10121,16 @@ def build_repair_task_packet(classification: dict, failure_reason: str = "", bui "full_context_files_available_if_needed": ["REPAIR_BRIEF.md", "INCIDENT_BRIEF.md", "DEPENDENCY_ERROR_BRIEF.md"], "default_context_files": ["REPAIR_TASK_PACKET.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], "refusal_mode": "If the frozen decision appears wrong or the failure belongs to the Factory, write PATCH_REFUSAL.json and do not edit publishable files.", + "recipe_aware_repair_packet": "RECIPE_AWARE_REPAIR_PACKET.json" if workspace is not None and run_dir is not None else "", + "model_recipe_summary": { + "model_family": (recipe_packet.get("model_family") if isinstance(recipe_packet, dict) else "") or "", + "artifact_role": (recipe_packet.get("artifact_role") if isinstance(recipe_packet, dict) else "") or "", + "loader_strategy": (recipe_packet.get("loader_strategy") if isinstance(recipe_packet, dict) else "") or "", + "expected_output_type": (recipe_packet.get("expected_output_type") if isinstance(recipe_packet, dict) else "") or "", + "primary_endpoint": (recipe_packet.get("primary_endpoint") if isinstance(recipe_packet, dict) else "") or "/generate", + }, + "known_failure_match": known_failure_match or {}, + "family_repair_directives": (recipe_packet.get("family_repair_directives") if isinstance(recipe_packet, dict) else []) or [], } if decision: packet["diagnosis_decision"] = decision @@ -9397,7 +10138,7 @@ def build_repair_task_packet(classification: dict, failure_reason: str = "", bui def write_repair_task_packet(workspace: Path, run_dir: Path, classification: dict, failure_reason: str = "", build_log: str = "", runtime_log: str = "", decision: dict | None = None) -> dict: - packet = build_repair_task_packet(classification, failure_reason, build_log, runtime_log, decision) + packet = build_repair_task_packet(classification, failure_reason, build_log, runtime_log, decision, workspace=workspace, run_dir=run_dir) repair_dir = run_dir / "repair" repair_dir.mkdir(parents=True, exist_ok=True) write_json(repair_dir / "REPAIR_TASK_PACKET.json", packet) @@ -10734,7 +11475,7 @@ def repair_workspace_with_pi(workspace: Path, run_dir: Path, events_path: Path, goal = f"""You are Pi in STRUCTURED REPAIR MODE / STRUCTURED PATCH REPAIR MODE for Agentic Space Factory. -First read `REPAIR_TASK_PACKET.json`, `FROZEN_REPAIR_DECISION.json`, and `LOG_EVIDENCE_PACKET.json`. It is the authoritative compact task, with frozen decision and log evidence packets attached. Do not reinterpret the whole run. Only read `REPAIR_BRIEF.md`, `INCIDENT_BRIEF.md`, and `DEPENDENCY_ERROR_BRIEF.md` if the packet is insufficient or the packet explicitly requests deep_repair_reanalysis. +First read `REPAIR_TASK_PACKET.json`, `RECIPE_AWARE_REPAIR_PACKET.json`, `FROZEN_REPAIR_DECISION.json`, and `LOG_EVIDENCE_PACKET.json`. `MODEL_RECIPE.json` and `APP_RUNTIME_CONTRACT.json` are authoritative for model-family repair. Do not reinterpret the whole run. Only read `REPAIR_BRIEF.md`, `INCIDENT_BRIEF.md`, and `DEPENDENCY_ERROR_BRIEF.md` if the packet is insufficient or the packet explicitly requests deep_repair_reanalysis. {hf_spaces_guidance_repair_section()} @@ -10760,6 +11501,11 @@ Critical method: Failure category: {classification.get('category')} Recommended strategy: {classification.get('recommendation')} +Model family: {repair_task_packet.get('model_recipe_summary', {}).get('model_family') or ''} +Loader strategy: {repair_task_packet.get('model_recipe_summary', {}).get('loader_strategy') or ''} +Known failure focus: {(repair_task_packet.get('known_failure_match') or {}).get('repair_focus') or ''} +Family directives: +{chr(10).join('- ' + str(x) for x in (repair_task_packet.get('family_repair_directives') or []))} Hard constraints: - Do not rebuild from scratch unless you clearly justify it in REPAIR_PLAN.md. @@ -10777,8 +11523,8 @@ Required deliverables: REPAIR_PLAN.md, patched files, REPAIR_SUMMARY.md. (repair_dir / f"REPAIR_GOAL_attempt_{repair_attempt}.md").write_text(goal, encoding="utf-8") write_repair_history(run_dir, load_repair_attempts(run_dir)) append_event(events_path, "repair_plan", "started", "Running Pi minimal patch repair", {"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type")}) - estimate_pi_context_budget(run_dir, workspace, phase="repair_patch", task_type=repair_task_packet.get("task_type") or "runtime_patch", direct_prompt=goal, referenced_files=["REPAIR_TASK_PACKET.json", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], max_context_chars=32000, omitted_sections=["full_build_logs", "full_runtime_logs", "full_report", "full_pi_output"], events_path=events_path, artifact_dir=repair_dir) - write_agent_trace_record(run_dir, phase="repair_patch", event="command_started", status="started", message="Pi structured patch repair started", data={"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type"), "decision": decision or {}}, artifacts=["repair/REPAIR_TASK_PACKET.json", "repair/FROZEN_REPAIR_DECISION.json", "repair/LOG_EVIDENCE_PACKET.json", "repair/REPAIR_GOAL.md", "repair/REPAIR_DECISION.json", "repair/PI_PROMPT_BUDGET_repair_patch.json"]) + estimate_pi_context_budget(run_dir, workspace, phase="repair_patch", task_type=repair_task_packet.get("task_type") or "runtime_patch", direct_prompt=goal, referenced_files=["REPAIR_TASK_PACKET.json", "RECIPE_AWARE_REPAIR_PACKET.json", "MODEL_RECIPE.json", "APP_RUNTIME_CONTRACT.json", "FROZEN_REPAIR_DECISION.json", "LOG_EVIDENCE_PACKET.json", "actionable_error_excerpt.txt", "app.py", "requirements.txt"], max_context_chars=32000, omitted_sections=["full_build_logs", "full_runtime_logs", "full_report", "full_pi_output"], events_path=events_path, artifact_dir=repair_dir) + write_agent_trace_record(run_dir, phase="repair_patch", event="command_started", status="started", message="Pi structured patch repair started", data={"model": pi_model, "category": classification.get("category"), "task_type": repair_task_packet.get("task_type"), "decision": decision or {}}, artifacts=["repair/REPAIR_TASK_PACKET.json", "repair/RECIPE_AWARE_REPAIR_PACKET.json", "repair/FROZEN_REPAIR_DECISION.json", "repair/LOG_EVIDENCE_PACKET.json", "repair/REPAIR_GOAL.md", "repair/REPAIR_DECISION.json", "repair/PI_PROMPT_BUDGET_repair_patch.json"]) code, out = run_cmd(["pi", "-p", goal], cwd=workspace, timeout=1500) logs_dir.mkdir(parents=True, exist_ok=True) (logs_dir / "pi_repair_output.txt").write_text(out, encoding="utf-8") @@ -11102,10 +11848,17 @@ def recover_after_live_validation_failure(api, workspace: Path, run_dir: Path, e repair_attempt, _repair_payload = next_pi_repair_attempt(run_dir) repaired = repair_workspace_with_pi(workspace, run_dir, events_path, pi_model, target_space_id, model_id, current_error, implementation_mode, expected_output_type, decision=decision, repair_attempt=repair_attempt, repair_trigger=(decision.get("classification") or {}).get("category") or "live_validation_failure") if not repaired: - write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation="not_started", failure_type="repair_patch_failed", final_user_message="Structured patch repair failed before redeploy.") - write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="failed_after_repair") - append_event(events_path, "failure", "failed", "Structured patch repair failed before redeploy", {"decision": decision}) - raise RuntimeError("Structured patch repair failed before redeploy") + terminal = failed_pi_repair_terminal_status(run_dir) + retry_decision = should_relaunch_after_no_patch(run_dir, terminal["post_repair_validation"], budgets.get(action, 0)) + write_repair_outcome(run_dir, events_path, patch_applied=False, upload_success=False, post_repair_validation=terminal["post_repair_validation"], failure_type=terminal["failure_type"], final_user_message=terminal["final_user_message"], latest_repair_attempt=terminal.get("latest_repair_attempt"), recipe_repair_retry=retry_decision) + append_event(events_path, "repair_patch", "failed", terminal["final_user_message"], {"decision": decision, "retry_decision": retry_decision}) + if retry_decision.get("relaunch"): + current_error = current_error + "\n\n" + recipe_repair_direct_followup_reason(run_dir, terminal["post_repair_validation"], (decision.get("classification") or {}).get("category") or "") + append_event(events_path, "repair_patch", "started", "Relaunching Pi with a more direct recipe-aware repair brief because budget remains", retry_decision) + continue + write_blockage_artifact(workspace, run_dir, events_path, decision, current_error, status="repair_exhausted") + append_event(events_path, "failure", "failed", "Structured patch repair exhausted without an uploadable patch", {"decision": decision, "terminal_status": terminal["post_repair_validation"]}) + raise RuntimeError(terminal["final_user_message"]) write_repair_outcome(run_dir, events_path, patch_applied=True, post_repair_validation="not_started") append_event(events_path, "repair_upload", "started", "Uploading repaired workspace") write_auth_probe(run_dir, events_path, "before_repair_upload", token, minimum_required_seconds=minimum_auth_seconds_for_phase("before_repair_upload"), raise_on_unsafe=True) @@ -11762,7 +12515,8 @@ def main(): if "/health" not in app_text and "api_name=\"health\"" not in app_text and "api_name='health'" not in app_text: append_event(events_path, "pi_verification", "failed", "app.py does not appear to expose /health; injecting safe health endpoint is not implemented") fail(run_dir, events_path, "Pi output did not preserve a /health endpoint") - append_event(events_path, "pi_verification", "success", "Pi output preserved health validation endpoint") + early_app_contract_guard = app_runtime_contract_guard(workspace, run_dir, events_path, expected_output_type=expected_output_type, reason="post_pi_pre_integrity") + append_event(events_path, "pi_verification", "success" if early_app_contract_guard.get("passed") else "warning", "Pi output preserved health endpoint; structured app contract will be enforced before upload", {"app_runtime_contract_passed": early_app_contract_guard.get("passed"), "problems": early_app_contract_guard.get("problems")}) output_type_resolution = contract_effective_output_type(workspace, expected_output_type) effective_expected_output_type = output_type_resolution.get("effective_expected_output_type") or expected_output_type @@ -13254,7 +14008,7 @@ def _semantic_health_payloads(validation: dict | None) -> list[tuple[str, dict]] def validation_health_semantics(validation: dict | None) -> dict: - details = {"schema_version": "health_semantics.v198_26_19", "endpoint_reachable": False, "semantic_checked": False, "semantic_passed": None, "negative_markers": [], "positive_markers": [], "load_error": "", "model_ready": None, "transport_status": ""} + details = {"schema_version": "health_semantics.v198_26_30", "endpoint_reachable": False, "semantic_checked": False, "semantic_passed": None, "negative_markers": [], "positive_markers": [], "load_error": "", "last_error": "", "model_ready": None, "pipeline_ready": None, "model_family": "", "loader_strategy": "", "expected_output_type": "", "failure_class": "", "transport_status": ""} if not isinstance(validation, dict): return details method = str(validation.get("method") or "").lower() @@ -13278,17 +14032,31 @@ def validation_health_semantics(validation: dict | None) -> dict: for key in ("pipeline_ready", "pipeline_loaded", "model_ready", "model_loaded", "generation_ready"): if key in payload: details["semantic_checked"] = True + if key in {"pipeline_ready", "pipeline_loaded"}: + details["pipeline_ready"] = bool(payload.get(key)) if isinstance(payload.get(key), bool) else details.get("pipeline_ready") if payload.get(key) is False: details["negative_markers"].append(f"{source}.{key}=false") elif payload.get(key) is True: details["positive_markers"].append(f"{source}.{key}=true") - for key in ("error", "load_error", "pipeline_error", "model_error"): + for key in ("model_family", "loader_strategy", "expected_output_type"): + value = payload.get(key) + if value and not details.get(key): + details[key] = str(value)[:200] + for key in ("error", "last_error", "load_error", "pipeline_error", "model_error"): if payload.get(key): details["semantic_checked"] = True details["negative_markers"].append(f"{source}.{key}=present") if not details["load_error"]: details["load_error"] = str(payload.get(key))[:1000] - for key in ("result_repr", "text", "message", "error", "load_error", "pipeline_error", "model_error"): + if key == "last_error" and not details["last_error"]: + details["last_error"] = str(payload.get(key))[:1000] + health_error_text = "\n".join(str(payload.get(k) or "") for _, payload in _semantic_health_payloads(validation) for k in ("last_error", "error", "load_error", "pipeline_error", "model_error")) + try: + if is_sdxl_lora_text_encoder_mismatch_text(health_error_text): + details["failure_class"] = "sdxl_lora_text_encoder_mismatch" + except Exception: + pass + for key in ("result_repr", "text", "message", "error", "last_error", "load_error", "pipeline_error", "model_error"): value = validation.get(key) if value is None: continue @@ -13301,9 +14069,13 @@ def validation_health_semantics(validation: dict | None) -> dict: if details["negative_markers"]: details["semantic_passed"] = False details["model_ready"] = False + if details.get("pipeline_ready") is None: + details["pipeline_ready"] = False elif details["semantic_checked"] and details["positive_markers"]: details["semantic_passed"] = True details["model_ready"] = True + if details.get("pipeline_ready") is None: + details["pipeline_ready"] = True return details @@ -14245,6 +15017,184 @@ def smoke_failure_repair_escalation_decision(generation_smoke: dict | None, *, v return payload + +def load_recipe_repair_context(workspace: Path | None, run_dir: Path | None) -> dict: + """Load MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json for repair.""" + workspace = Path(workspace) if workspace else None + run_dir = Path(run_dir) if run_dir else None + + def load_named(name: str) -> tuple[dict, str]: + candidates = [] + if workspace: + candidates.append(workspace / name) + if run_dir: + candidates.append(run_dir / name) + candidates.append(run_dir / "analysis_inputs" / name) + for candidate in candidates: + try: + if candidate.exists(): + payload = load_json_if_exists(candidate) + if isinstance(payload, dict): + return payload, str(candidate) + except Exception: + continue + return {}, "" + + recipe, recipe_source = load_named("MODEL_RECIPE.json") + runtime_contract, runtime_source = load_named("APP_RUNTIME_CONTRACT.json") + if not runtime_contract and recipe: + try: + runtime_contract = build_app_runtime_contract(recipe) + runtime_source = "derived_from_MODEL_RECIPE.json" + except Exception: + runtime_contract = {} + return { + "schema_version": "recipe_repair_context.v198_26_31", + "model_recipe": recipe, + "model_recipe_source": recipe_source, + "app_runtime_contract": runtime_contract, + "app_runtime_contract_source": runtime_source, + "recipe_available": bool(recipe), + "runtime_contract_available": bool(runtime_contract), + "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", + "artifact_role": recipe.get("artifact_role") or "", + "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", + "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", + "primary_endpoint": recipe.get("primary_endpoint") or runtime_contract.get("primary_endpoint") or "/generate", + } + + +def recipe_known_failure_catalog(recipe: dict | None = None, runtime_contract: dict | None = None) -> list[dict]: + recipe = recipe if isinstance(recipe, dict) else {} + runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} + catalog = [] + for item in recipe.get("known_failure_signatures") or []: + if isinstance(item, dict): + catalog.append(dict(item)) + required = [ + {"failure_class": "diffusers_duplicate_token_argument", "markers": ["got multiple values for keyword argument 'token'", "multiple values for keyword argument \"token\"", "duplicate token"], "repair_focus": "Remove duplicate token/use_auth_token injection from from_pretrained kwargs; do not change the model id or replace inference."}, + {"failure_class": "hf_cache_permission_denied", "markers": ["permission denied", "/data", "HF_HOME", "huggingface cache", "read-only file system", "errno 13"], "repair_focus": "Move HF_HOME/HF_HUB_CACHE/TRANSFORMERS_CACHE/DIFFUSERS_CACHE to a writable Space path before importing/loading models."}, + {"failure_class": "missing_spaces_gpu_decorator", "markers": ["missing @spaces.GPU", "ZeroGPU", "CUDA has been initialized before", "GPU task function", "@spaces.GPU"], "repair_focus": "Import spaces before CUDA-touching imports and protect the heavy inference function with @spaces.GPU while preserving fixed-GPU compatibility."}, + {"failure_class": "validator_schema_choice_type_mismatch", "markers": ["not in the list of choices", "Invalid value", "Dropdown", "CheckboxGroup", "choice"], "repair_focus": "Respect the published Gradio schema and canonical smoke payload; do not patch the model for validator-owned payload coercion errors."}, + {"failure_class": "sdxl_lora_text_encoder_mismatch", "markers": ["CLIPTextModel", "text_model", "has no attribute 'text_model'"], "repair_focus": "Treat the repo as an SDXL LoRA adapter: load the base SDXL pipeline first, then load_lora_weights(adapter_id, weight_name=...), and avoid loading the adapter as a standalone pipeline."}, + {"failure_class": "flux_template_syntax_error", "markers": ["FluxPipeline", "jinja", "template", "syntax error", "unexpected", "chat_template"], "repair_focus": "Repair FLUX/prompt-template syntax without changing FluxPipeline loader semantics or replacing image generation with placeholder output."}, + {"failure_class": "model_not_loaded_after_boot", "markers": ["model not loaded", "pipeline not loaded", "model_ready", "pipeline_ready", "last_error", "not initialized", "NoneType"], "repair_focus": "Fix loader initialization and expose the real load failure through /health.last_error; /generate must fail cleanly when the model is not ready."}, + ] + seen = {str(item.get("failure_class") or "") for item in catalog} + for item in required: + if item["failure_class"] not in seen: + catalog.append(item) + seen.add(item["failure_class"]) + return catalog + + +def match_recipe_known_failure_signature(error_text: str = "", recipe: dict | None = None, runtime_contract: dict | None = None, fallback_class: str = "") -> dict: + text = str(error_text or "") + low = text.lower() + recipe = recipe if isinstance(recipe, dict) else {} + runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} + fallback_class = str(fallback_class or "") + for sig in recipe_known_failure_catalog(recipe, runtime_contract): + failure_class = str(sig.get("failure_class") or sig.get("category") or "") + markers = [str(m) for m in (sig.get("markers") or sig.get("error_markers") or sig.get("signatures") or []) if str(m)] + class_matches = bool(failure_class and fallback_class and failure_class == fallback_class) + marker_matches = [m for m in markers if m.lower() in low] + if failure_class == "sdxl_lora_text_encoder_mismatch" and is_sdxl_lora_text_encoder_mismatch_text(text): + marker_matches = marker_matches or ["CLIPTextModel.text_model"] + if class_matches or marker_matches: + return { + "schema_version": "recipe_known_failure_match.v198_26_31", + "matched": True, + "failure_class": failure_class or fallback_class, + "matched_markers": marker_matches[:10], + "repair_focus": sig.get("repair_focus") or sig.get("recommended_repair_focus") or sig.get("recommendation") or "Patch according to MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json.", + "source_signature": sig, + "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", + "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", + "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or "", + } + if fallback_class: + return {"schema_version": "recipe_known_failure_match.v198_26_31", "matched": False, "failure_class": fallback_class, "matched_markers": [], "repair_focus": "No catalog signature matched; still use MODEL_RECIPE.json and APP_RUNTIME_CONTRACT.json as authoritative repair context.", "source_signature": {}, "model_family": recipe.get("model_family") or runtime_contract.get("model_family") or "", "loader_strategy": recipe.get("loader_strategy") or runtime_contract.get("loader_strategy") or "", "expected_output_type": recipe.get("expected_output_type") or runtime_contract.get("expected_output_type") or ""} + return {"schema_version": "recipe_known_failure_match.v198_26_31", "matched": False, "failure_class": "", "matched_markers": [], "repair_focus": ""} + + +def recipe_family_repair_directives(recipe: dict | None = None, runtime_contract: dict | None = None, signature_match: dict | None = None) -> list[str]: + recipe = recipe if isinstance(recipe, dict) else {} + runtime_contract = runtime_contract if isinstance(runtime_contract, dict) else {} + signature_match = signature_match if isinstance(signature_match, dict) else {} + family = str(recipe.get("model_family") or runtime_contract.get("model_family") or "") + directives = [ + "Use MODEL_RECIPE.json as the authoritative model-family plan, not a generic app repair brief.", + "Preserve APP_RUNTIME_CONTRACT.json: /health must expose model_ready, pipeline_ready, model_family, loader_strategy, last_error, and expected_output_type.", + "After patching, /generate must either return the expected output type or fail cleanly with the same underlying loader error visible in /health.last_error.", + ] + if signature_match.get("repair_focus"): + directives.append("Known failure focus: " + str(signature_match.get("repair_focus"))) + if family == "diffusers_sdxl_lora_adapter": + directives.extend([ + "This repo is an adapter. Do not call DiffusionPipeline.from_pretrained(adapter_id) as if it were a full pipeline.", + "Load the SDXL base model from model_recipe.base_model_id, then call load_lora_weights(adapter_id, weight_name=adapter_weight_name when present).", + "Keep the LoRA trigger words/prompt guidance from the model card when available.", + ]) + elif family == "diffusers_flux_pipeline": + directives.extend([ + "Keep FluxPipeline.from_pretrained semantics and the recipe-selected model id.", + "Do not replace FLUX inference with a text-only or placeholder response; preserve image output.", + ]) + elif family == "gguf_llamacpp": + directives.append("Use llama.cpp/llama-cpp-python style GGUF loading and text generation; do not attempt Transformers AutoModel loading for GGUF weights.") + elif family == "onnx_runtime_model": + directives.append("Use onnxruntime InferenceSession and preserve input/output tensor preprocessing; do not convert to a different runtime blindly.") + elif family == "transformers_pipeline": + directives.append("Use the Transformers pipeline/AutoModel strategy indicated by the recipe and preserve the task-specific output contract.") + return directives + + +def write_recipe_aware_repair_packet(workspace: Path, run_dir: Path, failure_reason: str = "", classification: dict | None = None, decision: dict | None = None) -> dict: + ctx = load_recipe_repair_context(workspace, run_dir) + recipe = ctx.get("model_recipe") or {} + runtime_contract = ctx.get("app_runtime_contract") or {} + category = str((classification or {}).get("category") or (classification or {}).get("failure_class") or (decision or {}).get("failure_class") or "") + match = match_recipe_known_failure_signature(failure_reason, recipe, runtime_contract, fallback_class=category) + packet = { + "schema_version": "recipe_aware_repair_packet.v198_26_31", + "authority": "MODEL_RECIPE.json + APP_RUNTIME_CONTRACT.json", + **ctx, + "known_failure_match": match, + "family_repair_directives": recipe_family_repair_directives(recipe, runtime_contract, match), + "repair_must_not_finish_not_started": True, + "no_patch_policy": "If Pi cannot produce a publishable patch, ASF must either launch another more direct recipe-aware attempt while budget remains or finish with no_patch_produced_by_pi / repair_exhausted, never post_repair_validation=not_started.", + } + repair_dir = run_dir / "repair" + repair_dir.mkdir(parents=True, exist_ok=True) + write_json(repair_dir / "RECIPE_AWARE_REPAIR_PACKET.json", packet) + try: + write_json(workspace / "RECIPE_AWARE_REPAIR_PACKET.json", packet) + except Exception: + pass + return packet + + +def recipe_repair_direct_followup_reason(run_dir: Path, terminal_status: str, failure_class: str = "") -> str: + latest = _latest_repair_attempt_result(run_dir) + return ( + "Previous Pi repair did not produce an uploadable publishable patch.\n" + f"Terminal status: {terminal_status}\n" + f"Latest attempt result: {latest.get('post_repair_result') or ''}\n" + f"Failure class: {failure_class or latest.get('failure_class') or ''}\n" + "Relaunch with a more direct recipe-aware instruction: implement the known failure focus from RECIPE_AWARE_REPAIR_PACKET.json, modify only allowed publishable files, or write PATCH_REFUSAL.json with a concrete blocker." + ) + + +def should_relaunch_after_no_patch(run_dir: Path, terminal_status: str, remaining_patch_budget: int) -> dict: + terminal_status = str(terminal_status or "") + no_patch_statuses = {"no_patch_produced_by_pi", "no_relevant_patch_produced_by_pi", "pi_patch_rejected_by_guard", "pi_repair_failed_before_upload"} + latest = _latest_repair_attempt_result(run_dir) + result = str(latest.get("post_repair_result") or "") + result_is_no_patch = result in {"repair_noop", "expected_file_not_modified", "diff_gate_failed", "sanity_failed"} + can_retry = remaining_patch_budget > 0 and (terminal_status in no_patch_statuses or result_is_no_patch) + return {"schema_version": "recipe_repair_retry_decision.v198_26_31", "relaunch": bool(can_retry), "terminal_status": terminal_status, "remaining_patch_budget": int(remaining_patch_budget or 0), "latest_attempt": latest, "reason": "retry_with_more_direct_recipe_aware_brief" if can_retry else "repair_exhausted_or_not_retryable"} + def write_smoke_repair_primary_error_packet(run_dir: Path, workspace: Path, generation_smoke: dict | None, *, failure_class: str = "") -> dict: smoke = generation_smoke if isinstance(generation_smoke, dict) else {} contract = read_inference_contract(workspace) @@ -14770,7 +15720,7 @@ def normalize_runtime_expected_output_type(value: str) -> str: def archive_rejected_workspace(workspace: Path, run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None) -> dict: """Persist the publishable workspace that failed pre-upload integrity. - v198.26.28 keeps blocked workspaces auditable and makes pre-upload + v198.26.29 keeps blocked workspaces auditable and makes pre-upload failures repairable: the exact rejected payload is preserved before the worker fails closed or requests a targeted repair pass. """ @@ -14803,7 +15753,7 @@ def archive_rejected_workspace(workspace: Path, run_dir: Path, events_path: Path def write_pre_upload_repair_attempt(run_dir: Path, events_path: Path, *, reason: str, guard_result: dict | None = None, deterministic_repairs: list[dict] | None = None, pi_repair_required: bool = False) -> dict: """Record the repair trajectory for a pre-upload template failure. - This artifact closes the visibility gap from v198.26.28: a pre-upload guard + This artifact closes the visibility gap from v198.26.29: a pre-upload guard must not fail opaquely. It must say which deterministic repairs were tried and whether a targeted Pi repair would be required before a final failure. """ @@ -14882,7 +15832,7 @@ def _insert_decorator_before_function(app_text: str, func_name: str, decorator: def _patch_common_python_syntax_defects(app_text: str) -> tuple[str, list[str]]: """Patch narrow syntax defects Pi has produced in real runs. - This is intentionally conservative. v198.26.28 covers the observed + This is intentionally conservative. v198.26.29 covers the observed `torch_dtype=DTYPE,,` class and repeated commas in function calls without attempting a broad Python formatter. """ @@ -15158,7 +16108,7 @@ def patch_workspace_for_missing_spaces_gpu_decorator(workspace: Path, run_dir: P def runtime_template_integrity_guard(workspace: Path, run_dir: Path, events_path: Path, *, hardware_intent: dict | None = None, expected_output_type: str = "", reason: str = "pre_upload", raise_on_failure: bool = True) -> dict: """Pre-upload integrity guard for publishable runtime templates. - v198.26.28 extends the v198.26.24 repair-first policy with compile checks + v198.26.29 extends the v198.26.24 repair-first policy with compile checks and endpoint repair. The worker should not fail-fast merely because Pi forgot `api_name='generate'` or emitted a narrow syntax typo; it should repair, re-check, then fail closed only when the app is still invalid. @@ -15316,7 +16266,7 @@ def ensure_runtime_template_uploadable_with_pi_repair( """Run pre-upload guard, then execute one Pi repair pass if unresolved defects remain. Guards must protect ASF from publishing broken apps, but they should not be - terminal guillotines for repairable template defects. v198.26.28 guarantees + terminal guillotines for repairable template defects. v198.26.29 guarantees that a `pi_repair_required` pre-upload state is followed by either a repaired workspace that passes the guard, or an explicit terminal repair outcome. """ @@ -15720,7 +16670,7 @@ def smoke_generate(target_space_id: str, token: str, run_dir: Path, events_path: # write_json(run_dir / "tests" / "payload_source.json", payload_source_record) # write_json(run_dir / "tests" / "replay_source.json", replay_source) # write_json(run_dir / "tests" / "validation_preflight.json", validation_preflight) - app_version = "v198.26.28" + app_version = "v198.26.31" engine_version = "unified_gradio_validation_harness_v198_25_3" parent_build_run_id = os.environ.get("PARENT_BUILD_RUN_ID", "").strip() validation_mode = os.environ.get("VALIDATION_MODE") or os.environ.get("SPACE_TEST_POLICY_MODE") or "linked"