Add prompt assist: auto-describe reference sheet + suggest shot (Qwen2.5-VL via Inference Providers) (#6)

- Add prompt assist: auto-describe reference sheet + suggest shot (Qwen2.5-VL via Inference Providers) (86919aa4981b55932a18070fc75eea130e9c1591)

app.py

@@ -21,6 +21,8 @@ subprocess.run([sys.executable, "-m", "pip", "install", "--force-reinstall", "--
 sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
 sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
 
+import base64
+import io
 import logging
 import random
 import tempfile
@@ -35,7 +37,7 @@ torch._dynamo.config.disable = True
 
 import spaces
 import gradio as gr
-from huggingface_hub import hf_hub_download, snapshot_download
+from huggingface_hub import hf_hub_download, snapshot_download, InferenceClient
 
 # Import LTX modules in the proven order — importing ltx_core.quantization/loader FIRST hits a
 # circular import (fp8_cast <-> loader.fuse_loras). Importing the model modules first forces the
@@ -325,6 +327,94 @@ def generate(sheet_image, sheet, action, seed, randomize, progress=gr.Progress(t
     return out_path, seed
 
 
+# ============================ PROMPT ASSIST (Inference Providers) ============================
+# Vision-LLM helpers that auto-write the two prompt fields. These are plain network calls
+# (no GPU), so they live OUTSIDE @spaces.GPU and never touch the LTX pipeline / VRAM.
+PROMPT_VLM = os.environ.get("PROMPT_VLM", "Qwen/Qwen2.5-VL-72B-Instruct")
+_vlm = InferenceClient(model=PROMPT_VLM, token=HF_TOKEN)
+
+_DESCRIBE_PROMPT = (
+    "You are writing the 'reference sheet description' for the LTX-2.3 Ingredients video model. "
+    "List every distinct subject in the image in this dense style.\n"
+    "FORMAT:\n"
+    "- One running description, subjects separated by '; '.\n"
+    "- Each CHARACTER: rich visual detail — build, skin/fur, hair, eyes, distinctive features, full clothing — then how "
+    "they are shown, e.g. 'shown in a face close-up and a full-body turnaround'.\n"
+    "- Each PROP: a detailed visual description.\n"
+    "- The LOCATION last, ONLY if a real environment is shown, described with its lighting.\n"
+    "- Describe each DISTINCT subject EXACTLY ONCE; merge multiple views of the same subject, never repeat a character. "
+    "IGNORE plain, neutral or studio backgrounds behind cut-outs (those are not a location).\n"
+    "- No preamble; never mention 'reference sheet', grid, panels, photo, or image.\n"
+    "EXAMPLE STYLE (a different image): \"a young Asian woman with a warm skin tone, dark hair parted down the middle in "
+    "two long braids, an olive-green t-shirt and khaki cargo pants, shown in a face close-up and a full-body turnaround; "
+    "a large heavy-duty blue hiking backpack with an external silver frame; a sweeping mountain landscape where a dirt "
+    "path winds toward snow-capped peaks under a bright blue sky\".\n"
+    "Now write the description for THIS image."
+)
+
+_ACTION_PROMPT = (
+    "You write the 'action / shot' prompt for the LTX-2.3 Ingredients video model, which animates the subjects from the "
+    "reference sheet into a ~5 second talking clip WITH audio. Match this STYLE and RULES.\n"
+    "STYLE: \"cinematic [genre] scene, a [shot type] of [subjects re-described briefly]. [staged action]; X says in a "
+    "[voice]: '...'. [reaction / second beat]; Y replies in a [voice]: '...'. [camera move]; [film look / lighting]. "
+    "clear immersive audio: [voices + ambient sounds], no background music\".\n"
+    "RULES:\n"
+    "- ONE continuous shot, no cuts. If there are 2+ characters, a SINGLE held two-shot with ALL of them visible together "
+    "in one shared location and consistent lighting; never a split screen or side-by-side panels.\n"
+    "- Briefly RE-DESCRIBE each subject's key appearance so identity holds (as in the style line).\n"
+    "- Each character gets at least one line of dialogue as a COMPLETE self-contained sentence, attributed with a described voice.\n"
+    "- Natural conversational distance for non-romantic pairings; no leaning in / intimacy.\n"
+    "- End with 'clear immersive audio:' listing the voices and ambient sounds; 'no background music' unless music truly fits.\n"
+    "- Output ONLY the action paragraph, present tense, lowercase cinematic style, ~90-140 words."
+)
+
+
+def _img_data_uri(img, max_side=1024):
+    im = img.convert("RGB")
+    im.thumbnail((max_side, max_side))
+    buf = io.BytesIO()
+    im.save(buf, format="JPEG", quality=90)
+    return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
+
+
+def describe_sheet(sheet_image):
+    """Auto-fill the reference-sheet description from the composed sheet image."""
+    if sheet_image is None:
+        raise gr.Error("Add or build a reference sheet first.")
+    try:
+        out = _vlm.chat_completion(
+            messages=[{"role": "user", "content": [
+                {"type": "text", "text": _DESCRIBE_PROMPT},
+                {"type": "image_url", "image_url": {"url": _img_data_uri(sheet_image)}}]}],
+            max_tokens=320, temperature=0.3,
+        )
+        return out.choices[0].message.content.strip()
+    except Exception as e:
+        raise gr.Error(f"Auto-describe failed ({type(e).__name__}). Check the HF_TOKEN secret / model access.")
+
+
+def suggest_action(sheet_image, sheet_desc, action_idea):
+    """Write the action/shot prompt. Expands the idea in the action box; invents one if empty."""
+    if sheet_image is None:
+        raise gr.Error("Add or build a reference sheet first.")
+    idea = (action_idea or "").strip()
+    subjects = (sheet_desc or "").strip() or "(infer the subjects from the image)"
+    user = f"Subjects: {subjects}\n\nIdea: {idea if idea else '(none — invent a fitting short scene)'}"
+    try:
+        out = _vlm.chat_completion(
+            messages=[
+                {"role": "system", "content": _ACTION_PROMPT},
+                {"role": "user", "content": [
+                    {"type": "text", "text": user},
+                    {"type": "image_url", "image_url": {"url": _img_data_uri(sheet_image)}}]}],
+            max_tokens=420, temperature=0.7,
+        )
+        return out.choices[0].message.content.strip()
+    except Exception as e:
+        raise gr.Error(f"Suggest shot failed ({type(e).__name__}). Check the HF_TOKEN secret / model access.")
+# =============================================================================================
+
+
 with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
     gr.Markdown(
         "# ⚡ LTX-2.3 Ingredients — Fast (Distilled)\n"
@@ -345,8 +435,10 @@ with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
                     gr.Markdown("*Tiles your images into one sheet and loads it into the **Reference sheet** tab.*")
             sheet = gr.Textbox(label="Reference sheet description", lines=3,
                                placeholder="a young woman with red hair in a green jacket (face close-up + turnaround); a brass pocket watch; a cobblestone alley at night")
+            describe_btn = gr.Button("✨ Auto-describe sheet", size="sm")
             action = gr.Textbox(label="Generated video — action / shot, plus any speech & sounds", lines=3,
                                 placeholder="the woman walks down the alley, checks the pocket watch and whispers 'almost time'; footsteps on cobblestone, distant city hum")
+            suggest_btn = gr.Button("✨ Suggest shot — type a one-line idea above, or leave empty to invent", size="sm")
             with gr.Accordion("Settings", open=False):
                 randomize = gr.Checkbox(True, label="Randomize seed")
                 seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed")
@@ -354,7 +446,13 @@ with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
         with gr.Column():
             video_out = gr.Video(label="Generated video")
 
-    build_btn.click(compose_sheet, inputs=gallery, outputs=sheet_image)
+    # Build the sheet, then auto-describe it into the description box.
+    build_btn.click(compose_sheet, inputs=gallery, outputs=sheet_image).then(
+        describe_sheet, inputs=sheet_image, outputs=sheet)
+    # Auto-describe when a sheet image is uploaded directly, or via the button.
+    sheet_image.upload(describe_sheet, inputs=sheet_image, outputs=sheet)
+    describe_btn.click(describe_sheet, inputs=sheet_image, outputs=sheet)
+    suggest_btn.click(suggest_action, inputs=[sheet_image, sheet, action], outputs=action)
     run.click(generate, inputs=[sheet_image, sheet, action, seed, randomize], outputs=[video_out, seed])
 
     gr.Examples(