Spaces:
Runtime error
Runtime error
File size: 10,942 Bytes
5055c26 bd83a96 5055c26 bd83a96 5055c26 bd83a96 5055c26 3f02c7c 5055c26 3f02c7c 5055c26 3f02c7c 5055c26 3f02c7c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 |
from huggingface_hub import from_pretrained_keras
from keras_cv import models
import gradio as gr
from tensorflow import keras
from diffusers import StableDiffusionPipeline
keras.mixed_precision.set_global_policy("mixed_float16")
# prepare model
resolution = 512
# sd_dreambooth_model = models.StableDiffusion(
# img_width=resolution, img_height=resolution
# )
# db_diffusion_model = from_pretrained_keras("keras-dreambooth/dreambooth_diffusion_model")
# sd_dreambooth_model._diffusion_model = db_diffusion_model
# checkpoint of the converted Stable Diffusion from KerasCV
model_ckpt = "nielsgl/dreambooth-keras-pug-ace-sd2.1"
pipeline = StableDiffusionPipeline.from_pretrained(model_ckpt)
pipeline = pipeline.to("cuda")
unique_id = "puggieace"
class_label = "dog"
prompt = f"A photo of {unique_id} {class_label} on the beach"
image = pipeline(prompt, num_inference_steps=50).images[0]
# generate images
def infer(prompt, negative_prompt, guidance_scale=10, num_inference_steps=50):
neg = negative_prompt if negative_prompt else None
imgs = []
while len(imgs) != 4:
next_prompt = pipeline(prompt, negative_prompt=neg, guidance_scale=guidance_scale, num_inference_steps=num_inference_steps, num_images_per_prompt=5)
for img, is_neg in zip(next_prompt.images, next_prompt.nsfw_content_detected):
if not is_neg:
imgs.append(img)
if len(imgs) == 4:
break
return imgs
output = gr.Gallery(label="Outputs").style(grid=(1,2))
# customize interface
title = "KerasCV Stable Diffusion Demo on images of Ace."
description = "This is a dreambooth model fine-tuned on images of my pug named Ace. To try it, input the concept with `puggieace dog`."
examples=[
["Portrait photo of puggieace dog on a beachtowel wearing sunglasses on the beach, sunset in background, golden hour", "", 12, 50],
["A photo of a cute puggieace dog getting a haircut in a barbershop, ultra realistic, intricate details, highly detailed, photorealistic, octane render, 8 k, unreal engine. art by artgerm and greg rutkowski and charlie bowater and magali villeneuve and alphonse mucha", "", 12, 75],
["Portrait of puggieace dog as a Roman Emperor, city in background, ultra realistic, intricate details, eerie, highly detailed, photorealistic, octane render, 8 k, unreal engine. art by artgerm and greg rutkowski and charlie bowater and magali villeneuve and alphonse mucha", "", 15, 75],
["Photo of cute puggieace dog as an astronaut, space and planet in background, ultra realistic, concept art, intricate details, highly detailed, photorealistic, octane render, 8 k, unreal engine. trending on artstation", "", 15, 75],
["Photo of cute puggieace dog as super hero, futuristic city in background, cinematic light, high dynamic range, insane intricate details, stunning cinema effects, aesthetic, masterpiece, trending on artstation, cartoon art", "", 12, 75],
]
base_14 = "https://huggingface.co./nielsgl/dreambooth-pug-ace-sd1.4/resolve/main/"
base_21 = "https://huggingface.co./nielsgl/dreambooth-keras-pug-ace-sd2.1/resolve/main/"
model_card_1 = f"""
# KerasCV Stable Diffusion in Diffusers 🧨🤗
DreamBooth model for the `puggieace` concept trained by nielsgl on the [nielsgl/dreambooth-ace](https://huggingface.co./datasets/nielsgl/dreambooth-ace) dataset.
It can be used by modifying the `instance_prompt`: **a photo of puggieace**.
The examples are from 2 different Keras CV models (`StableDiffusion` and `StableDiffusionV2`, corresponding to Stable Diffusion V1.4 and V2.1, respectively) trained on the same dataset (`nielsgl/dreambooth-ace`).
## Description
The Stable Diffusion V2 pipeline contained in the corresponding repository (`nielsgl/dreambooth-keras-pug-ace-sd2.1`) was created using a modified version of [this Space](https://huggingface.co./spaces/sayakpaul/convert-kerascv-sd-diffusers) for StableDiffusionV2 from KerasCV. The purpose is to convert the KerasCV Stable Diffusion weights in a way that is compatible with [Diffusers](https://github.com/huggingface/diffusers). This allows users to fine-tune using KerasCV and use the fine-tuned weights in Diffusers taking advantage of its nifty features (like [schedulers](https://huggingface.co./docs/diffusers/main/en/using-diffusers/schedulers), [fast attention](https://huggingface.co./docs/diffusers/optimization/fp16), etc.).
This model was created as part of the Keras DreamBooth Sprint 🔥. Visit the [organisation page](https://huggingface.co./keras-dreambooth) for instructions on how to take part!
## Demo
"""
model_card_2 = f"""
## Examples
### Stable Diffusion V1.4
> Portrait of puggieace dog as a Roman Emperor, city in background
> Photo of puggieace dog wearing sunglasses on the beach, sunset in background, golden hour
> Photo of cute puggieace dog as an astronaut, planet and spaceship in background
### Stable Diffusion V2.1
> Portrait painting of a cute puggieace dog as a samurai
> Photo of cute puggieace dog as an astronaut, space and planet in background
> A photo of a cute puggieace dog getting a haircut in a barbershop
> Portrait photo of puggieace dog in New York
> Portrait of puggieace dog as a Roman Emperor, city in background
## Usage with Stable Diffusion V1.4
```python
from huggingface_hub import from_pretrained_keras
import keras_cv
import matplotlib.pyplot as plt
model = keras_cv.models.StableDiffusion(img_width=512, img_height=512, jit_compile=True)
model._diffusion_model = from_pretrained_keras("nielsgl/dreambooth-pug-ace")
model._text_encoder = from_pretrained_keras("nielsgl/dreambooth-pug-ace-text-encoder")
images = model.text_to_image("a photo of puggieace dog on the beach", batch_size=3)
plt.imshow(image[0])
```
## Usage with Stable Diffusion V2.1
```python
from diffusers import StableDiffusionPipeline
pipeline = StableDiffusionPipeline.from_pretrained('nielsgl/dreambooth-keras-pug-ace-sd2.1')
image = pipeline().images[0]
image
```
### Training hyperparameters
The following hyperparameters were used during training for Stable Diffusion v1.4:
| Hyperparameters | Value |
| :-- | :-- |
| name | RMSprop |
| weight_decay | None |
| clipnorm | None |
| global_clipnorm | None |
| clipvalue | None |
| use_ema | False |
| ema_momentum | 0.99 |
| ema_overwrite_frequency | 100 |
| jit_compile | True |
| is_legacy_optimizer | False |
| learning_rate | 0.0010000000474974513 |
| rho | 0.9 |
| momentum | 0.0 |
| epsilon | 1e-07 |
| centered | False |
| training_precision | float32 |
"""
with gr.Blocks() as demo:
with gr.Row():
gr.Markdown(model_card_1)
with gr.Row():
with gr.Column():
prompt_pos = gr.Textbox(label="Positive Prompt", value="a photo of puggieace dog getting a haircut")
prompt_neg = gr.Textbox(label="Negative Prompt", value="bad anatomy, blurry")
# gr.Slider(label='Number of gen image', minimum=1, maximum=4, value=2, step=1),
prompt_gs = gr.Number(label='Guidance scale', value=12)
prompt_steps = gr.Slider(label="Inference Steps",value=50)
prompt_btn = gr.Button("Generate")
with gr.Column():
output = gr.Gallery(label="Outputs").style(grid=(1,2))
prompt_btn.click(infer, inputs=[prompt_pos, prompt_neg, prompt_gs, prompt_steps], outputs=[output])
with gr.Row():
gr.Examples(examples, inputs=[prompt_pos, prompt_neg, prompt_gs, prompt_steps], outputs=output, fn=infer, cache_examples=True)
# gr.Interface(infer, inputs=[gr.Textbox(label="Positive Prompt", value="a photo of puggieace dog getting a haircut"),
# gr.Textbox(label="Negative Prompt", value="bad anatomy, blurry"),
# # gr.Slider(label='Number of gen image', minimum=1, maximum=4, value=2, step=1),
# gr.Number(label='Guidance scale', value=12),
# gr.Slider(label="Inference Steps",value=50),
# ], outputs=[output], title=title, description=description, examples=examples).queue()
with gr.Row():
with gr.Column():
gr.Markdown(model_card_2)
with gr.Column():
gr.Markdown(" ")
demo.queue().launch()
# with gr.Blocks() as card_interface:
# gr.Markdown(model_card)
# demo_interface = gr.Interface(infer, inputs=[gr.Textbox(label="Positive Prompt", value="a photo of puggieace dog getting a haircut"),
# gr.Textbox(label="Negative Prompt", value="bad anatomy, blurry"),
# # gr.Slider(label='Number of gen image', minimum=1, maximum=4, value=2, step=1),
# gr.Number(label='Guidance scale', value=12),
# gr.Slider(label="Inference Steps",value=50),
# ], outputs=[output], title=title, description=description, examples=examples).queue()
# gr.TabbedInterface([card_interface, demo_interface], ["Model Card", "Demo 🤗"]).launch() |