Spaces:
Running on Zero
Running on Zero
Add joyfox/MiniMax-H3-Turbo as a fourth LoRA set
Browse filesComfyUI-layout 4-step turbo LoRA covering attention, MLP, AdaLN and
both output heads; per-key alpha folded into lora_B at load. Suggested
4 steps; H3_JOYFOX=off skips loading it.
- README.md +7 -1
- app.py +3 -3
- h3_lora.py +32 -2
README.md
CHANGED
|
@@ -60,6 +60,11 @@ A third, non-turbo set is [`fal/MiniMax-H3-Realism-People-LoRA`](https://hugging
|
|
| 60 |
reference-tree layout as larry under a `diffusion_model.` prefix, so it folds through the same key mapping.
|
| 61 |
`H3_REALISM=off` skips loading it.
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
## AoTI-compiled blocks
|
| 64 |
|
| 65 |
With `H3_AOTI=1` the 50 repeated transformer blocks run from a compiled package,
|
|
@@ -187,7 +192,8 @@ one-time `PIPE.to("cuda")` is inside the first row's 339 s and does not reappear
|
|
| 187 |
| `H3_LORA_STRENGTH` | `1.0` | Scales the larry LoRA delta (sharpness/artifact trade-off). |
|
| 188 |
| `H3_LIGHTX` | `on` | Set to `off` to skip loading the lightx2v LoRA set. |
|
| 189 |
| `H3_REALISM` | `on` | Set to `off` to skip loading the fal realism-people LoRA set. |
|
| 190 |
-
| `
|
|
|
|
| 191 |
|
| 192 |
## Whose GPU quota pays
|
| 193 |
|
|
|
|
| 60 |
reference-tree layout as larry under a `diffusion_model.` prefix, so it folds through the same key mapping.
|
| 61 |
`H3_REALISM=off` skips loading it.
|
| 62 |
|
| 63 |
+
A fourth set is [`joyfox/MiniMax-H3-Turbo`](https://huggingface.co/joyfox/MiniMax-H3-Turbo) (`joyfox`, 4 steps):
|
| 64 |
+
another ComfyUI-layout turbo LoRA, additionally covering the MLPs, the AdaLN projections, and both output heads
|
| 65 |
+
(`video_out` -> `proj_out`, `audio_out` -> `audio_proj_out`), with a per-key `alpha` folded into `lora_B` at load.
|
| 66 |
+
`H3_JOYFOX=off` skips loading it.
|
| 67 |
+
|
| 68 |
## AoTI-compiled blocks
|
| 69 |
|
| 70 |
With `H3_AOTI=1` the 50 repeated transformer blocks run from a compiled package,
|
|
|
|
| 192 |
| `H3_LORA_STRENGTH` | `1.0` | Scales the larry LoRA delta (sharpness/artifact trade-off). |
|
| 193 |
| `H3_LIGHTX` | `on` | Set to `off` to skip loading the lightx2v LoRA set. |
|
| 194 |
| `H3_REALISM` | `on` | Set to `off` to skip loading the fal realism-people LoRA set. |
|
| 195 |
+
| `H3_JOYFOX` | `on` | Set to `off` to skip loading the joyfox 4-step turbo LoRA set. |
|
| 196 |
+
| `H3_LORA_DEFAULT` | `larry` | Which loaded LoRA set starts folded (`larry` / `lightx` / `realism` / `joyfox`). |
|
| 197 |
|
| 198 |
## Whose GPU quota pays
|
| 199 |
|
app.py
CHANGED
|
@@ -304,8 +304,8 @@ def _fit_keyframe(image_path, current_canvas):
|
|
| 304 |
|
| 305 |
|
| 306 |
def _resolve_lora(lora, use_lora) -> str:
|
| 307 |
-
"""`lora` (`larry` / `lightx` / `realism` / `off`) wins; the legacy
|
| 308 |
-
if isinstance(lora, str) and lora in ("larry", "lightx", "realism", "off"):
|
| 309 |
return lora
|
| 310 |
return "larry" if use_lora else "off"
|
| 311 |
|
|
@@ -423,7 +423,7 @@ def studio_config():
|
|
| 423 |
# The LoRA dropdown: value -> {label, suggested steps}.
|
| 424 |
"loras": {
|
| 425 |
**{
|
| 426 |
-
name: {"label": spec["label"], "steps": {"larry": 6, "lightx": 4, "realism": 28}.get(name, 6)}
|
| 427 |
for name, spec in sets.items()
|
| 428 |
},
|
| 429 |
"off": {"label": "off (base model)", "steps": 28},
|
|
|
|
| 304 |
|
| 305 |
|
| 306 |
def _resolve_lora(lora, use_lora) -> str:
|
| 307 |
+
"""`lora` (`larry` / `lightx` / `realism` / `joyfox` / `off`) wins; the legacy bool maps onto `larry` / `off`."""
|
| 308 |
+
if isinstance(lora, str) and lora in ("larry", "lightx", "realism", "joyfox", "off"):
|
| 309 |
return lora
|
| 310 |
return "larry" if use_lora else "off"
|
| 311 |
|
|
|
|
| 423 |
# The LoRA dropdown: value -> {label, suggested steps}.
|
| 424 |
"loras": {
|
| 425 |
**{
|
| 426 |
+
name: {"label": spec["label"], "steps": {"larry": 6, "lightx": 4, "realism": 28, "joyfox": 4}.get(name, 6)}
|
| 427 |
for name, spec in sets.items()
|
| 428 |
},
|
| 429 |
"off": {"label": "off (base model)", "steps": 28},
|
h3_lora.py
CHANGED
|
@@ -26,9 +26,14 @@ The two supported LoRAs ship in different layouts:
|
|
| 26 |
`out_proj` on the 52 blocks), rank 16, `alpha == rank` (the card's scale 1.0), so it goes through the same
|
| 27 |
`_larry_targets` mapping. It wants the full step count, not 4–6.
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
`H3_LORA` selects the larry file (`off` skips loading it), `H3_LIGHTX=off` skips lightx, `H3_REALISM=off` skips
|
| 30 |
-
realism, `H3_LORA_DEFAULT` picks which set starts folded, and `H3_LORA_STRENGTH` is
|
| 31 |
-
sharpness/artifact dial.
|
| 32 |
"""
|
| 33 |
|
| 34 |
from __future__ import annotations
|
|
@@ -45,6 +50,8 @@ LIGHTX_FILE = os.environ.get("H3_LIGHTX_FILE", "minimax_h3_fl2v_turbo_4step_v0.1
|
|
| 45 |
LIGHTX_ALPHA = 8
|
| 46 |
REALISM_REPO = os.environ.get("H3_REALISM_REPO", "fal/MiniMax-H3-Realism-People-LoRA")
|
| 47 |
REALISM_FILE = os.environ.get("H3_REALISM_FILE", "h3-realism-people-t2v.safetensors")
|
|
|
|
|
|
|
| 48 |
# The card's sharpness/artifact dial for the larry LoRA: >1 against blurry ghosting/smear, <1 against grain.
|
| 49 |
LARRY_STRENGTH = float(os.environ.get("H3_LORA_STRENGTH", "1.0"))
|
| 50 |
DEFAULT_LORA = os.environ.get("H3_LORA_DEFAULT", "larry")
|
|
@@ -59,6 +66,7 @@ def _larry_targets(name: str, b: torch.Tensor, inner_dim: int) -> list[tuple[str
|
|
| 59 |
else:
|
| 60 |
target = name
|
| 61 |
target = target.replace("final_layer.adaln_proj.linear", "norm_out.linear")
|
|
|
|
| 62 |
|
| 63 |
if target.endswith(".attn.qkv_proj"):
|
| 64 |
prefix = target.removesuffix("qkv_proj")
|
|
@@ -131,6 +139,26 @@ def _load_realism(inner_dim: int) -> dict:
|
|
| 131 |
}
|
| 132 |
|
| 133 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
def _apply(entries, params, sign: float) -> None:
|
| 135 |
for key, a, b in entries:
|
| 136 |
param = params.get(key)
|
|
@@ -163,6 +191,8 @@ def apply_lora(transformer) -> str | None:
|
|
| 163 |
sets["lightx"] = _load_lightx()
|
| 164 |
if os.environ.get("H3_REALISM", "on").lower() not in ("", "off", "none"):
|
| 165 |
sets["realism"] = _load_realism(inner_dim)
|
|
|
|
|
|
|
| 166 |
if not sets:
|
| 167 |
return None
|
| 168 |
|
|
|
|
| 26 |
`out_proj` on the 52 blocks), rank 16, `alpha == rank` (the card's scale 1.0), so it goes through the same
|
| 27 |
`_larry_targets` mapping. It wants the full step count, not 4–6.
|
| 28 |
|
| 29 |
+
* `joyfox` (`joyfox/MiniMax-H3-Turbo`) is another 4-step turbo LoRA, ComfyUI-native like realism but covering more
|
| 30 |
+
of the tree: attention, MLP, the block and final AdaLN projections, and both output heads (`final_layer.video_out`
|
| 31 |
+
-> `proj_out`, `final_layer.audio_out` -> `audio_proj_out`). Ranks are mixed (32 attention/MLP, 8 modulation/heads)
|
| 32 |
+
with a scalar `alpha` per key, so each `lora_B` is prescaled by `alpha / rank` at load (here always 1.0).
|
| 33 |
+
|
| 34 |
`H3_LORA` selects the larry file (`off` skips loading it), `H3_LIGHTX=off` skips lightx, `H3_REALISM=off` skips
|
| 35 |
+
realism, `H3_JOYFOX=off` skips joyfox, `H3_LORA_DEFAULT` picks which set starts folded, and `H3_LORA_STRENGTH` is
|
| 36 |
+
the larry card's sharpness/artifact dial.
|
| 37 |
"""
|
| 38 |
|
| 39 |
from __future__ import annotations
|
|
|
|
| 50 |
LIGHTX_ALPHA = 8
|
| 51 |
REALISM_REPO = os.environ.get("H3_REALISM_REPO", "fal/MiniMax-H3-Realism-People-LoRA")
|
| 52 |
REALISM_FILE = os.environ.get("H3_REALISM_FILE", "h3-realism-people-t2v.safetensors")
|
| 53 |
+
JOYFOX_REPO = os.environ.get("H3_JOYFOX_REPO", "joyfox/MiniMax-H3-Turbo")
|
| 54 |
+
JOYFOX_FILE = os.environ.get("H3_JOYFOX_FILE", "minimax_h3_fl2va_4step_lora.safetensors")
|
| 55 |
# The card's sharpness/artifact dial for the larry LoRA: >1 against blurry ghosting/smear, <1 against grain.
|
| 56 |
LARRY_STRENGTH = float(os.environ.get("H3_LORA_STRENGTH", "1.0"))
|
| 57 |
DEFAULT_LORA = os.environ.get("H3_LORA_DEFAULT", "larry")
|
|
|
|
| 66 |
else:
|
| 67 |
target = name
|
| 68 |
target = target.replace("final_layer.adaln_proj.linear", "norm_out.linear")
|
| 69 |
+
target = target.replace("final_layer.video_out", "proj_out").replace("final_layer.audio_out", "audio_proj_out")
|
| 70 |
|
| 71 |
if target.endswith(".attn.qkv_proj"):
|
| 72 |
prefix = target.removesuffix("qkv_proj")
|
|
|
|
| 139 |
}
|
| 140 |
|
| 141 |
|
| 142 |
+
def _load_joyfox(inner_dim: int) -> dict:
|
| 143 |
+
from huggingface_hub import hf_hub_download
|
| 144 |
+
from safetensors.torch import load_file
|
| 145 |
+
|
| 146 |
+
lora = load_file(hf_hub_download(JOYFOX_REPO, JOYFOX_FILE))
|
| 147 |
+
bases = sorted({key.rsplit(".lora_", 1)[0].removeprefix("diffusion_model.") for key in lora if ".lora_" in key})
|
| 148 |
+
entries = []
|
| 149 |
+
for name in bases:
|
| 150 |
+
prefixed = f"diffusion_model.{name}"
|
| 151 |
+
a = lora[f"{prefixed}.lora_A.weight"]
|
| 152 |
+
# Per-key alpha (scalar tensor), unlike larry's fixed alpha == rank.
|
| 153 |
+
b = lora[f"{prefixed}.lora_B.weight"] * (lora[f"{prefixed}.alpha"].item() / a.shape[0])
|
| 154 |
+
entries.extend((key, a, b_part) for key, b_part in _larry_targets(name, b, inner_dim))
|
| 155 |
+
return {
|
| 156 |
+
"label": f"{JOYFOX_REPO}/{JOYFOX_FILE}",
|
| 157 |
+
"scale": 1.0, # alpha / rank already folded into lora_B
|
| 158 |
+
"entries": entries,
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
def _apply(entries, params, sign: float) -> None:
|
| 163 |
for key, a, b in entries:
|
| 164 |
param = params.get(key)
|
|
|
|
| 191 |
sets["lightx"] = _load_lightx()
|
| 192 |
if os.environ.get("H3_REALISM", "on").lower() not in ("", "off", "none"):
|
| 193 |
sets["realism"] = _load_realism(inner_dim)
|
| 194 |
+
if os.environ.get("H3_JOYFOX", "on").lower() not in ("", "off", "none"):
|
| 195 |
+
sets["joyfox"] = _load_joyfox(inner_dim)
|
| 196 |
if not sets:
|
| 197 |
return None
|
| 198 |
|