Switch to ZeroGPU POC pattern: duration=1 billing, subprocess retains CUDA context

- app.py: pre-spawn generate.py from main process, @spaces.GPU(duration=1) only
for CUDA init signal; stream incremental logs from JSON result file
- generate.py: add --signal-path polling, --result-path incremental JSON logging,
add distillation LoRA (ltx-2.3-22b-distilled-lora-384.safetensors at 0.5),
experimental LoRA at 1.0; guidance_scale default 5.0

app.py

@@ -1,38 +1,43 @@
 """
 Sulphur — Image to Video (HF Spaces).
-Clones Wan2GP and downloads models on first run.
-Generation is handled by generate.py called as a subprocess inside @spaces.GPU.
+
+POC-proven pattern: subprocess is pre-spawned from the main Gradio process,
+@spaces.GPU(duration=1) is used only to signal CUDA initialisation.
+The subprocess retains GPU access for the full generation duration.
+Billing cost: 1 second per generation regardless of inference time.
 """
 
+import json
+import multiprocessing
 import os
-import sys
-import subprocess
 import shutil
+import subprocess
+import sys
 import tempfile
 import threading
-import json
+import time
 from pathlib import Path
 
 import gradio as gr
 import spaces
 
-_HF_TOKEN    = os.environ.get("HF_TOKEN")
-_PERSISTENT  = Path("/data") if Path("/data").exists() else Path(tempfile.gettempdir())
-WAN2GP_ROOT  = _PERSISTENT / "Wan2GP"
-CKPTS_DIR    = WAN2GP_ROOT / "ckpts"
-LORAS_DIR    = WAN2GP_ROOT / "loras" / "ltx2"
+_HF_TOKEN     = os.environ.get("HF_TOKEN")
+_PERSISTENT   = Path("/data") if Path("/data").exists() else Path(tempfile.gettempdir())
+WAN2GP_ROOT   = _PERSISTENT / "Wan2GP"
+CKPTS_DIR     = WAN2GP_ROOT / "ckpts"
+LORAS_DIR     = WAN2GP_ROOT / "loras" / "ltx2"
 FINETUNES_DIR = WAN2GP_ROOT / "finetunes"
-GENERATE_PY  = Path(__file__).parent / "generate.py"
+GENERATE_PY   = Path(__file__).parent / "generate.py"
 
 SULPHUR_ASSETS = [
     ("SulphurAI/Sulphur-2-base", "sulphur_distil_bf16.safetensors", CKPTS_DIR),
 ]
 LTX_ASSETS = [
     ("SulphurAI/Sulphur-2-base", "experimental/sulphur_experimental_lora_v1.safetensors", LORAS_DIR),
-    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b-distilled-lora-384.safetensors",     LORAS_DIR),
-    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b_vae.safetensors",                    CKPTS_DIR),
-    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b_text_embedding_projection.safetensors", CKPTS_DIR),
-    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b_embeddings_connector.safetensors",   CKPTS_DIR),
+    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b-distilled-lora-384.safetensors",              LORAS_DIR),
+    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b_vae.safetensors",                             CKPTS_DIR),
+    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b_text_embedding_projection.safetensors",       CKPTS_DIR),
+    ("DeepBeepMeep/LTX-2", "ltx-2.3-22b_embeddings_connector.safetensors",            CKPTS_DIR),
 ]
 
 SULPHUR_FINETUNE = {
@@ -42,15 +47,14 @@ SULPHUR_FINETUNE = {
         "architecture": "ltx2_22B",
         "parent_model_type": "ltx2_22B",
         "description": "LTX-2.3 fine-tuned i2v. Distilled checkpoint.",
-        # Full distilled model — do NOT also preload the rank-768 LoRA (README: use one or the other)
         "URLs": [str(CKPTS_DIR / "sulphur_distil_bf16.safetensors")],
         "preload_URLs": [],
     },
     "num_inference_steps": 8,
     "video_length": 81,
     "resolution": "832x480",
-    "guidance_scale": 3.5,
-    "alt_guidance_scale": 3.5,
+    "guidance_scale": 5.0,
+    "alt_guidance_scale": 5.0,
 }
 
 _setup_lock = threading.Lock()
@@ -60,14 +64,13 @@ _setup_done = False
 def _download(repo_id, filename, dest_dir):
     from huggingface_hub import hf_hub_download
     dest_dir.mkdir(parents=True, exist_ok=True)
-    dest = dest_dir / Path(filename).name  # flat — strip any subfolder
+    dest = dest_dir / Path(filename).name
     if dest.exists():
         print(f"[download] cached: {dest.name}")
         return
     print(f"[download] {repo_id}/{filename}")
     hf_hub_download(repo_id=repo_id, filename=filename,
                     local_dir=str(dest_dir), token=_HF_TOKEN)
-    # hf_hub_download preserves subfolder structure; flatten to dest_dir root
     downloaded = dest_dir / filename
     if downloaded.exists() and not dest.exists():
         shutil.move(str(downloaded), str(dest))
@@ -92,10 +95,9 @@ def setup():
     for repo, fname, dest in SULPHUR_ASSETS + LTX_ASSETS:
         _download(repo, fname, dest)
 
-    # Gemma text encoder — must stay in its subfolder (Wan2GP looks there by name)
     _gemma_folder = "gemma-3-12b-it-qat-q4_0-unquantized"
-    _gemma_file = f"{_gemma_folder}_quanto_bf16_int8.safetensors"
-    gemma_dest = CKPTS_DIR / _gemma_folder / _gemma_file
+    _gemma_file   = f"{_gemma_folder}_quanto_bf16_int8.safetensors"
+    gemma_dest    = CKPTS_DIR / _gemma_folder / _gemma_file
     if not gemma_dest.exists():
         from huggingface_hub import hf_hub_download
         print("[download] Gemma text encoder...")
@@ -109,7 +111,9 @@ def setup():
         print("[download] cached: Gemma text encoder")
 
     FINETUNES_DIR.mkdir(parents=True, exist_ok=True)
-    (FINETUNES_DIR / "sulphur_2_base.json").write_text(json.dumps(SULPHUR_FINETUNE, indent=2))
+    (FINETUNES_DIR / "sulphur_2_base.json").write_text(
+        json.dumps(SULPHUR_FINETUNE, indent=2)
+    )
     print("[setup] Done.")
 
 
@@ -118,77 +122,102 @@ setup()
 RESOLUTIONS = ["832x480", "480x832", "640x640", "1024x576", "576x1024"]
 
 
-@spaces.GPU(duration=80)
+@spaces.GPU(duration=1)
+def _signal_cuda_init(signal_path):
+    """Acquire a 1-second GPU lease just long enough for the worker to init CUDA."""
+    Path(signal_path).write_text("go")
+    time.sleep(0.5)
+
+
 def generate_video(image, prompt, resolution, steps, guidance_scale, frames, seed):
     if image is None:
         raise gr.Error("Please upload an image.")
     if not prompt.strip():
         raise gr.Error("Please enter a prompt.")
 
-    out_file = Path(tempfile.mkdtemp()) / "output.mp4"
-    env = {**os.environ, "WAN2GP_ROOT": str(WAN2GP_ROOT)}
+    signal_path = tempfile.mktemp(suffix=".signal")
+    result_path = tempfile.mktemp(suffix=".json")
+    out_dir     = tempfile.mkdtemp()
+    out_file    = os.path.join(out_dir, "output.mp4")
 
     cmd = [
         sys.executable, str(GENERATE_PY),
-        "--image",  image,
-        "--prompt", prompt,
-        "--output", str(out_file),
-        "--model",  "sulphur-2",
-        "--seed",   str(int(seed)),
-        "--resolution", resolution,
-        "--steps",  str(int(steps)),
+        "--image",          image,
+        "--prompt",         prompt,
+        "--output",         out_file,
+        "--model",          "sulphur-2",
+        "--seed",           str(int(seed)),
+        "--resolution",     resolution,
+        "--steps",          str(int(steps)),
         "--guidance_scale", str(float(guidance_scale)),
-        "--frames", str(int(frames)),
+        "--frames",         str(int(frames)),
+        "--signal-path",    signal_path,
+        "--result-path",    result_path,
     ]
 
-    log_lines = []
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                            text=True, bufsize=0, env=env)
-
-    buf = ""
-    while True:
-        chunk = proc.stdout.read(256)
-        if not chunk:
-            break
-        buf += chunk
-        # Split on \r or \n — tqdm uses \r to overwrite progress lines
-        parts = buf.replace("\r", "\n").split("\n")
-        buf = parts[-1]
-        for part in parts[:-1]:
-            stripped = part.strip()
-            if not stripped:
-                continue
-            # Overwrite last line if it looks like a progress bar update
-            if log_lines and ("%" in stripped or "it/s" in stripped or "step" in stripped.lower()):
-                log_lines[-1] = stripped
-            else:
-                log_lines.append(stripped)
-            print(stripped)
-            yield None, "\n".join(log_lines[-30:])
+    env = {**os.environ, "WAN2GP_ROOT": str(WAN2GP_ROOT)}
+
+    # Spawn from main Gradio process (not inside ZeroGPU daemon — avoids daemon-child restriction)
+    proc = subprocess.Popen(cmd, env=env)
+
+    log_lines = ["Worker spawned, acquiring GPU lease..."]
+    yield None, "\n".join(log_lines)
+
+    # 1-second lease — signals worker to init CUDA, then expires
+    _signal_cuda_init(signal_path)
+
+    log_lines.append("GPU lease expired. Worker retains CUDA context and is generating...")
+    yield None, "\n".join(log_lines)
+
+    last_log_len = 0
+    deadline = time.monotonic() + 600  # 10 min hard timeout
+
+    while time.monotonic() < deadline:
+        time.sleep(2)
+
+        if os.path.exists(result_path):
+            try:
+                with open(result_path) as f:
+                    data = json.load(f)
+                new_entries = data.get("log", [])[last_log_len:]
+                if new_entries:
+                    log_lines.extend(new_entries)
+                    last_log_len += len(new_entries)
+                yield None, "\n".join(log_lines[-40:])
+                if data.get("done"):
+                    break
+            except Exception:
+                pass
+        else:
+            yield None, "\n".join(log_lines)
 
     proc.wait()
-    log = "\n".join(log_lines)
 
-    if proc.returncode != 0 or not out_file.exists():
-        yield None, log + "\n\n[ERROR] Generation failed."
-        return
+    for path in (signal_path, result_path):
+        try:
+            os.unlink(path)
+        except Exception:
+            pass
 
-    final = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
-    shutil.copy2(out_file, final.name)
-    yield final.name, log + "\n\n[DONE]"
+    if os.path.exists(out_file):
+        final = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
+        shutil.copy2(out_file, final.name)
+        yield final.name, "\n".join(log_lines) + "\n\n[DONE]"
+    else:
+        yield None, "\n".join(log_lines) + "\n\n[ERROR] No output file produced."
 
 
 with gr.Blocks(title="Sulphur — Image to Video") as demo:
-    gr.Markdown("# Sulphur — Image to Video\nUsing New Experimental Lora v1")
+    gr.Markdown("# Sulphur — Image to Video\nUsing Experimental LoRA v1 + Distillation LoRA")
     with gr.Row():
         with gr.Column(scale=1):
             image_in  = gr.Image(type="filepath", label="Input Image")
             prompt_in = gr.Textbox(label="Prompt", placeholder="Describe the motion…", lines=3)
             with gr.Accordion("Advanced", open=False):
                 resolution_dd = gr.Dropdown(RESOLUTIONS, value="832x480", label="Resolution")
-                steps_sl      = gr.Slider(1, 50, value=8,   step=1,   label="Steps")
+                steps_sl      = gr.Slider(1, 50, value=8, step=1, label="Steps")
                 guidance_sl   = gr.Slider(1.0, 10.0, value=5.0, step=0.5, label="Guidance Scale")
-                frames_sl     = gr.Slider(17, 257, value=81, step=8,  label="Frames")
+                frames_sl     = gr.Slider(17, 257, value=81, step=8, label="Frames")
                 seed_num      = gr.Number(value=-1, label="Seed (-1 = random)", precision=0)
             run_btn = gr.Button("Generate", variant="primary")
         with gr.Column(scale=1):

generate.py

@@ -1,11 +1,14 @@
 """
-HF Spaces version of generate.py — same logic, paths adapted for /tmp/Wan2GP.
-Called as a subprocess from app.py inside @spaces.GPU.
+HF Spaces version of generate.py — paths adapted for /data/Wan2GP.
+Called as a subprocess from app.py.
+Waits for --signal-path before touching CUDA, writes incremental JSON to --result-path.
 """
 
 import argparse
+import json
 import os
 import sys
+import time
 from pathlib import Path
 
 WAN2GP_ROOT = Path(os.environ.get("WAN2GP_ROOT", "/tmp/Wan2GP"))
@@ -23,9 +26,21 @@ DEFAULTS = {
     },
 }
 
+_log_entries = []
+_result_path = None
 
-def p(*args, **kwargs):
-    print(*args, **kwargs, flush=True)
+
+def p(msg):
+    ts = time.strftime("%H:%M:%S")
+    entry = f"[{ts}] {msg}"
+    _log_entries.append(entry)
+    print(entry, flush=True)
+    if _result_path:
+        try:
+            with open(_result_path, "w") as f:
+                json.dump({"log": _log_entries, "done": False}, f)
+        except Exception:
+            pass
 
 
 def parse_args():
@@ -39,18 +54,32 @@ def parse_args():
     ap.add_argument("--frames",         type=int,   default=None)
     ap.add_argument("--resolution",     default=None)
     ap.add_argument("--seed",           type=int,   default=-1)
+    ap.add_argument("--signal-path",    default=None, dest="signal_path")
+    ap.add_argument("--result-path",    default=None, dest="result_path")
     return ap.parse_args()
 
 
 def main():
+    global _result_path
+
     args = parse_args()
+    _result_path = args.result_path
+
+    if args.signal_path:
+        p("Waiting for GPU lease signal...")
+        while not Path(args.signal_path).exists():
+            time.sleep(0.05)
+        p("Signal received — GPU lease active, initialising CUDA...")
 
     model_type = MODEL_SHORTHANDS.get(args.model, args.model)
     defaults   = DEFAULTS.get(model_type, DEFAULTS["sulphur_2_base"])
 
     image_path = str(Path(args.image.strip()).resolve())
     if not Path(image_path).exists():
-        print(f"Fatal: image not found: {image_path}", flush=True)
+        p(f"Fatal: image not found: {image_path}")
+        if _result_path:
+            with open(_result_path, "w") as f:
+                json.dump({"log": _log_entries, "done": True, "error": "image not found"}, f)
         sys.exit(1)
 
     resolution = args.resolution or defaults["resolution"]
@@ -83,9 +112,10 @@ def main():
         "image_prompt_type":    "S",
         "input_video_strength": 1.0,
         "activated_loras": [
+            "ltx-2.3-22b-distilled-lora-384.safetensors",
             "sulphur_experimental_lora_v1.safetensors",
         ],
-        "loras_multipliers": ["0.5"],
+        "loras_multipliers": ["0.5", "1.0"],
     }
 
     p(f"Model:      {model_type}")
@@ -102,7 +132,7 @@ def main():
     output_dir = Path(args.output).parent
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    p("Starting session...")
+    p("Starting WanGPSession...")
     session = WanGPSession(root=WAN2GP_ROOT, output_dir=output_dir, console_output=True)
 
     p("Running generation...")
@@ -115,7 +145,6 @@ def main():
         if src and Path(src).exists():
             output_file = src
 
-    # Fallback: scan the output dir for any video file Wan2GP may have written
     if output_file is None:
         candidates = sorted(output_dir.glob("**/*.mp4"), key=lambda f: f.stat().st_mtime, reverse=True)
         if candidates:
@@ -130,10 +159,17 @@ def main():
         p(f"No output found in {output_dir}")
         if result.errors:
             p(f"Errors: {result.errors}")
+        if _result_path:
+            with open(_result_path, "w") as f:
+                json.dump({"log": _log_entries, "done": True, "error": "no output"}, f)
         sys.exit(1)
 
     session.close()
 
+    if _result_path:
+        with open(_result_path, "w") as f:
+            json.dump({"log": _log_entries, "done": True}, f)
+
 
 if __name__ == "__main__":
     main()