Commit ·
9f7f795
1
Parent(s): 6caadef
fixed normalization error, update README with GitHub link
Browse files- README.md +10 -2
- modeling_emuru.py +4 -5
README.md
CHANGED
|
@@ -26,7 +26,8 @@ library_name: t5
|
|
| 26 |
|
| 27 |
# Emuru
|
| 28 |
|
| 29 |
-
**Emuru** is a conditional generative model that integrates a T5-based decoder with a Variational Autoencoder (VAE) for image generation conditioned on text and style images. It allows users to combine textual prompts (e.g., style text, generation text) and style images to create new, synthesized images.
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
## Model description
|
|
@@ -121,6 +122,13 @@ for idx, pil_img in enumerate(output_images):
|
|
| 121 |
If you use Emuru in your research or wish to refer to it, please cite:
|
| 122 |
|
| 123 |
```
|
| 124 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
```
|
| 126 |
|
|
|
|
| 26 |
|
| 27 |
# Emuru
|
| 28 |
|
| 29 |
+
**Emuru** is a conditional generative model that integrates a T5-based decoder with a Variational Autoencoder (VAE) for image generation conditioned on text and style images. It allows users to combine textual prompts (e.g., style text, generation text) and style images to create new, synthesized images.
|
| 30 |
+
Training code is released on [GitHub](https://github.com/aimagelab/Emuru)
|
| 31 |
|
| 32 |
|
| 33 |
## Model description
|
|
|
|
| 122 |
If you use Emuru in your research or wish to refer to it, please cite:
|
| 123 |
|
| 124 |
```
|
| 125 |
+
@InProceedings{Pippi_2025_CVPR,
|
| 126 |
+
author = {Pippi, Vittorio and Quattrini, Fabio and Cascianelli, Silvia and Tonioni, Alessio and Cucchiara, Rita},
|
| 127 |
+
title = {Zero-Shot Styled Text Image Generation, but Make It Autoregressive},
|
| 128 |
+
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
|
| 129 |
+
month = {June},
|
| 130 |
+
year = {2025},
|
| 131 |
+
pages = {7910-7919}
|
| 132 |
+
}
|
| 133 |
```
|
| 134 |
|
modeling_emuru.py
CHANGED
|
@@ -54,15 +54,15 @@ class Emuru(PreTrainedModel):
|
|
| 54 |
self.padding_token = nn.Parameter(torch.empty((1, vae_latent_size)), requires_grad=False)
|
| 55 |
self.padding_token_threshold = nn.Parameter(torch.empty(1), requires_grad=False)
|
| 56 |
|
| 57 |
-
self.vae = AutoencoderKL.from_pretrained(config.vae_name_or_path)
|
| 58 |
-
self.set_training(self.vae, False)
|
| 59 |
-
|
| 60 |
self.query_rearrange = Rearrange('b c h (w q) -> b w (q c h)', q=config.slices_per_query)
|
| 61 |
self.z_rearrange = Rearrange('b w (q c h) -> b c h (w q)', c=config.vae_channels, q=config.slices_per_query)
|
| 62 |
|
| 63 |
self.mse_criterion = nn.MSELoss()
|
| 64 |
self.init_weights()
|
| 65 |
|
|
|
|
|
|
|
|
|
|
| 66 |
def set_training(self, model: nn.Module, training: bool) -> None:
|
| 67 |
"""
|
| 68 |
Set the training mode for a given model and freeze/unfreeze parameters accordingly.
|
|
@@ -168,7 +168,6 @@ class Emuru(PreTrainedModel):
|
|
| 168 |
texts = [style_text + ' ' + gen_text for style_text, gen_text in zip(style_texts, gen_texts)]
|
| 169 |
|
| 170 |
imgs, _, img_ends = self._generate(texts=texts, imgs=style_imgs, lengths=lengths, **kwargs)
|
| 171 |
-
imgs = (imgs + 1) / 2
|
| 172 |
|
| 173 |
out_imgs = []
|
| 174 |
for i, end in enumerate(img_ends):
|
|
@@ -265,7 +264,7 @@ class Emuru(PreTrainedModel):
|
|
| 265 |
elif stopping_criteria == 'none':
|
| 266 |
pass
|
| 267 |
|
| 268 |
-
imgs = torch.clamp(self.vae.decode(self.z_rearrange(canvas_sequence)).sample,
|
| 269 |
return imgs, canvas_sequence, seq_stops * 8
|
| 270 |
|
| 271 |
def _img_encode(
|
|
|
|
| 54 |
self.padding_token = nn.Parameter(torch.empty((1, vae_latent_size)), requires_grad=False)
|
| 55 |
self.padding_token_threshold = nn.Parameter(torch.empty(1), requires_grad=False)
|
| 56 |
|
|
|
|
|
|
|
|
|
|
| 57 |
self.query_rearrange = Rearrange('b c h (w q) -> b w (q c h)', q=config.slices_per_query)
|
| 58 |
self.z_rearrange = Rearrange('b w (q c h) -> b c h (w q)', c=config.vae_channels, q=config.slices_per_query)
|
| 59 |
|
| 60 |
self.mse_criterion = nn.MSELoss()
|
| 61 |
self.init_weights()
|
| 62 |
|
| 63 |
+
self.vae = AutoencoderKL.from_pretrained(config.vae_name_or_path)
|
| 64 |
+
self.set_training(self.vae, False)
|
| 65 |
+
|
| 66 |
def set_training(self, model: nn.Module, training: bool) -> None:
|
| 67 |
"""
|
| 68 |
Set the training mode for a given model and freeze/unfreeze parameters accordingly.
|
|
|
|
| 168 |
texts = [style_text + ' ' + gen_text for style_text, gen_text in zip(style_texts, gen_texts)]
|
| 169 |
|
| 170 |
imgs, _, img_ends = self._generate(texts=texts, imgs=style_imgs, lengths=lengths, **kwargs)
|
|
|
|
| 171 |
|
| 172 |
out_imgs = []
|
| 173 |
for i, end in enumerate(img_ends):
|
|
|
|
| 264 |
elif stopping_criteria == 'none':
|
| 265 |
pass
|
| 266 |
|
| 267 |
+
imgs = torch.clamp(self.vae.decode(self.z_rearrange(canvas_sequence)).sample, 0, 1)
|
| 268 |
return imgs, canvas_sequence, seq_stops * 8
|
| 269 |
|
| 270 |
def _img_encode(
|