akhaliq HF Staff commited on
Commit
b683692
·
1 Parent(s): 2b67062

Add joyfox/MiniMax-H3-Turbo as a fourth LoRA set

Browse files

ComfyUI-layout 4-step turbo LoRA covering attention, MLP, AdaLN and
both output heads; per-key alpha folded into lora_B at load. Suggested
4 steps; H3_JOYFOX=off skips loading it.

Files changed (3) hide show
  1. README.md +7 -1
  2. app.py +3 -3
  3. h3_lora.py +32 -2
README.md CHANGED
@@ -60,6 +60,11 @@ A third, non-turbo set is [`fal/MiniMax-H3-Realism-People-LoRA`](https://hugging
60
  reference-tree layout as larry under a `diffusion_model.` prefix, so it folds through the same key mapping.
61
  `H3_REALISM=off` skips loading it.
62
 
 
 
 
 
 
63
  ## AoTI-compiled blocks
64
 
65
  With `H3_AOTI=1` the 50 repeated transformer blocks run from a compiled package,
@@ -187,7 +192,8 @@ one-time `PIPE.to("cuda")` is inside the first row's 339 s and does not reappear
187
  | `H3_LORA_STRENGTH` | `1.0` | Scales the larry LoRA delta (sharpness/artifact trade-off). |
188
  | `H3_LIGHTX` | `on` | Set to `off` to skip loading the lightx2v LoRA set. |
189
  | `H3_REALISM` | `on` | Set to `off` to skip loading the fal realism-people LoRA set. |
190
- | `H3_LORA_DEFAULT` | `larry` | Which loaded LoRA set starts folded (`larry` / `lightx` / `realism`). |
 
191
 
192
  ## Whose GPU quota pays
193
 
 
60
  reference-tree layout as larry under a `diffusion_model.` prefix, so it folds through the same key mapping.
61
  `H3_REALISM=off` skips loading it.
62
 
63
+ A fourth set is [`joyfox/MiniMax-H3-Turbo`](https://huggingface.co/joyfox/MiniMax-H3-Turbo) (`joyfox`, 4 steps):
64
+ another ComfyUI-layout turbo LoRA, additionally covering the MLPs, the AdaLN projections, and both output heads
65
+ (`video_out` -> `proj_out`, `audio_out` -> `audio_proj_out`), with a per-key `alpha` folded into `lora_B` at load.
66
+ `H3_JOYFOX=off` skips loading it.
67
+
68
  ## AoTI-compiled blocks
69
 
70
  With `H3_AOTI=1` the 50 repeated transformer blocks run from a compiled package,
 
192
  | `H3_LORA_STRENGTH` | `1.0` | Scales the larry LoRA delta (sharpness/artifact trade-off). |
193
  | `H3_LIGHTX` | `on` | Set to `off` to skip loading the lightx2v LoRA set. |
194
  | `H3_REALISM` | `on` | Set to `off` to skip loading the fal realism-people LoRA set. |
195
+ | `H3_JOYFOX` | `on` | Set to `off` to skip loading the joyfox 4-step turbo LoRA set. |
196
+ | `H3_LORA_DEFAULT` | `larry` | Which loaded LoRA set starts folded (`larry` / `lightx` / `realism` / `joyfox`). |
197
 
198
  ## Whose GPU quota pays
199
 
app.py CHANGED
@@ -304,8 +304,8 @@ def _fit_keyframe(image_path, current_canvas):
304
 
305
 
306
  def _resolve_lora(lora, use_lora) -> str:
307
- """`lora` (`larry` / `lightx` / `realism` / `off`) wins; the legacy `use_lora` bool maps onto `larry` / `off`."""
308
- if isinstance(lora, str) and lora in ("larry", "lightx", "realism", "off"):
309
  return lora
310
  return "larry" if use_lora else "off"
311
 
@@ -423,7 +423,7 @@ def studio_config():
423
  # The LoRA dropdown: value -> {label, suggested steps}.
424
  "loras": {
425
  **{
426
- name: {"label": spec["label"], "steps": {"larry": 6, "lightx": 4, "realism": 28}.get(name, 6)}
427
  for name, spec in sets.items()
428
  },
429
  "off": {"label": "off (base model)", "steps": 28},
 
304
 
305
 
306
  def _resolve_lora(lora, use_lora) -> str:
307
+ """`lora` (`larry` / `lightx` / `realism` / `joyfox` / `off`) wins; the legacy bool maps onto `larry` / `off`."""
308
+ if isinstance(lora, str) and lora in ("larry", "lightx", "realism", "joyfox", "off"):
309
  return lora
310
  return "larry" if use_lora else "off"
311
 
 
423
  # The LoRA dropdown: value -> {label, suggested steps}.
424
  "loras": {
425
  **{
426
+ name: {"label": spec["label"], "steps": {"larry": 6, "lightx": 4, "realism": 28, "joyfox": 4}.get(name, 6)}
427
  for name, spec in sets.items()
428
  },
429
  "off": {"label": "off (base model)", "steps": 28},
h3_lora.py CHANGED
@@ -26,9 +26,14 @@ The two supported LoRAs ship in different layouts:
26
  `out_proj` on the 52 blocks), rank 16, `alpha == rank` (the card's scale 1.0), so it goes through the same
27
  `_larry_targets` mapping. It wants the full step count, not 4–6.
28
 
 
 
 
 
 
29
  `H3_LORA` selects the larry file (`off` skips loading it), `H3_LIGHTX=off` skips lightx, `H3_REALISM=off` skips
30
- realism, `H3_LORA_DEFAULT` picks which set starts folded, and `H3_LORA_STRENGTH` is the larry card's
31
- sharpness/artifact dial.
32
  """
33
 
34
  from __future__ import annotations
@@ -45,6 +50,8 @@ LIGHTX_FILE = os.environ.get("H3_LIGHTX_FILE", "minimax_h3_fl2v_turbo_4step_v0.1
45
  LIGHTX_ALPHA = 8
46
  REALISM_REPO = os.environ.get("H3_REALISM_REPO", "fal/MiniMax-H3-Realism-People-LoRA")
47
  REALISM_FILE = os.environ.get("H3_REALISM_FILE", "h3-realism-people-t2v.safetensors")
 
 
48
  # The card's sharpness/artifact dial for the larry LoRA: >1 against blurry ghosting/smear, <1 against grain.
49
  LARRY_STRENGTH = float(os.environ.get("H3_LORA_STRENGTH", "1.0"))
50
  DEFAULT_LORA = os.environ.get("H3_LORA_DEFAULT", "larry")
@@ -59,6 +66,7 @@ def _larry_targets(name: str, b: torch.Tensor, inner_dim: int) -> list[tuple[str
59
  else:
60
  target = name
61
  target = target.replace("final_layer.adaln_proj.linear", "norm_out.linear")
 
62
 
63
  if target.endswith(".attn.qkv_proj"):
64
  prefix = target.removesuffix("qkv_proj")
@@ -131,6 +139,26 @@ def _load_realism(inner_dim: int) -> dict:
131
  }
132
 
133
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
  def _apply(entries, params, sign: float) -> None:
135
  for key, a, b in entries:
136
  param = params.get(key)
@@ -163,6 +191,8 @@ def apply_lora(transformer) -> str | None:
163
  sets["lightx"] = _load_lightx()
164
  if os.environ.get("H3_REALISM", "on").lower() not in ("", "off", "none"):
165
  sets["realism"] = _load_realism(inner_dim)
 
 
166
  if not sets:
167
  return None
168
 
 
26
  `out_proj` on the 52 blocks), rank 16, `alpha == rank` (the card's scale 1.0), so it goes through the same
27
  `_larry_targets` mapping. It wants the full step count, not 4–6.
28
 
29
+ * `joyfox` (`joyfox/MiniMax-H3-Turbo`) is another 4-step turbo LoRA, ComfyUI-native like realism but covering more
30
+ of the tree: attention, MLP, the block and final AdaLN projections, and both output heads (`final_layer.video_out`
31
+ -> `proj_out`, `final_layer.audio_out` -> `audio_proj_out`). Ranks are mixed (32 attention/MLP, 8 modulation/heads)
32
+ with a scalar `alpha` per key, so each `lora_B` is prescaled by `alpha / rank` at load (here always 1.0).
33
+
34
  `H3_LORA` selects the larry file (`off` skips loading it), `H3_LIGHTX=off` skips lightx, `H3_REALISM=off` skips
35
+ realism, `H3_JOYFOX=off` skips joyfox, `H3_LORA_DEFAULT` picks which set starts folded, and `H3_LORA_STRENGTH` is
36
+ the larry card's sharpness/artifact dial.
37
  """
38
 
39
  from __future__ import annotations
 
50
  LIGHTX_ALPHA = 8
51
  REALISM_REPO = os.environ.get("H3_REALISM_REPO", "fal/MiniMax-H3-Realism-People-LoRA")
52
  REALISM_FILE = os.environ.get("H3_REALISM_FILE", "h3-realism-people-t2v.safetensors")
53
+ JOYFOX_REPO = os.environ.get("H3_JOYFOX_REPO", "joyfox/MiniMax-H3-Turbo")
54
+ JOYFOX_FILE = os.environ.get("H3_JOYFOX_FILE", "minimax_h3_fl2va_4step_lora.safetensors")
55
  # The card's sharpness/artifact dial for the larry LoRA: >1 against blurry ghosting/smear, <1 against grain.
56
  LARRY_STRENGTH = float(os.environ.get("H3_LORA_STRENGTH", "1.0"))
57
  DEFAULT_LORA = os.environ.get("H3_LORA_DEFAULT", "larry")
 
66
  else:
67
  target = name
68
  target = target.replace("final_layer.adaln_proj.linear", "norm_out.linear")
69
+ target = target.replace("final_layer.video_out", "proj_out").replace("final_layer.audio_out", "audio_proj_out")
70
 
71
  if target.endswith(".attn.qkv_proj"):
72
  prefix = target.removesuffix("qkv_proj")
 
139
  }
140
 
141
 
142
+ def _load_joyfox(inner_dim: int) -> dict:
143
+ from huggingface_hub import hf_hub_download
144
+ from safetensors.torch import load_file
145
+
146
+ lora = load_file(hf_hub_download(JOYFOX_REPO, JOYFOX_FILE))
147
+ bases = sorted({key.rsplit(".lora_", 1)[0].removeprefix("diffusion_model.") for key in lora if ".lora_" in key})
148
+ entries = []
149
+ for name in bases:
150
+ prefixed = f"diffusion_model.{name}"
151
+ a = lora[f"{prefixed}.lora_A.weight"]
152
+ # Per-key alpha (scalar tensor), unlike larry's fixed alpha == rank.
153
+ b = lora[f"{prefixed}.lora_B.weight"] * (lora[f"{prefixed}.alpha"].item() / a.shape[0])
154
+ entries.extend((key, a, b_part) for key, b_part in _larry_targets(name, b, inner_dim))
155
+ return {
156
+ "label": f"{JOYFOX_REPO}/{JOYFOX_FILE}",
157
+ "scale": 1.0, # alpha / rank already folded into lora_B
158
+ "entries": entries,
159
+ }
160
+
161
+
162
  def _apply(entries, params, sign: float) -> None:
163
  for key, a, b in entries:
164
  param = params.get(key)
 
191
  sets["lightx"] = _load_lightx()
192
  if os.environ.get("H3_REALISM", "on").lower() not in ("", "off", "none"):
193
  sets["realism"] = _load_realism(inner_dim)
194
+ if os.environ.get("H3_JOYFOX", "on").lower() not in ("", "off", "none"):
195
+ sets["joyfox"] = _load_joyfox(inner_dim)
196
  if not sets:
197
  return None
198