|
The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`. |
|
Running 1 job |
|
0it [00:00, ?it/s]
0it [00:00, ?it/s] |
|
/usr/local/lib/python3.10/dist-packages/controlnet_aux/mediapipe_face/mediapipe_face_common.py:7: UserWarning: The module 'mediapipe' is not installed. The package will have limited functionality. Please install it using the command: pip install 'mediapipe' |
|
warnings.warn( |
|
/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_5m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_5m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected. |
|
return register_model(fn_wrapper) |
|
/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_11m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_11m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected. |
|
return register_model(fn_wrapper) |
|
/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected. |
|
return register_model(fn_wrapper) |
|
/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_384 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_384. This is because the name being registered conflicts with an existing name. Please check if this is not expected. |
|
return register_model(fn_wrapper) |
|
/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_512 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_512. This is because the name being registered conflicts with an existing name. Please check if this is not expected. |
|
return register_model(fn_wrapper) |
|
You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers |
|
{ |
|
"type": "sd_trainer", |
|
"training_folder": "output", |
|
"device": "cuda:0", |
|
"network": { |
|
"type": "lora", |
|
"linear": 16, |
|
"linear_alpha": 16 |
|
}, |
|
"save": { |
|
"dtype": "float16", |
|
"save_every": 250, |
|
"max_step_saves_to_keep": 4, |
|
"push_to_hub": false |
|
}, |
|
"datasets": [ |
|
{ |
|
"folder_path": "/workspace/ai-toolkit/images", |
|
"caption_ext": "txt", |
|
"caption_dropout_rate": 0.05, |
|
"shuffle_tokens": false, |
|
"cache_latents_to_disk": true, |
|
"resolution": [ |
|
512, |
|
768, |
|
1024 |
|
] |
|
} |
|
], |
|
"train": { |
|
"batch_size": 1, |
|
"steps": 2000, |
|
"gradient_accumulation_steps": 1, |
|
"train_unet": true, |
|
"train_text_encoder": false, |
|
"gradient_checkpointing": true, |
|
"noise_scheduler": "flowmatch", |
|
"optimizer": "adamw8bit", |
|
"lr": 0.0001, |
|
"ema_config": { |
|
"use_ema": true, |
|
"ema_decay": 0.99 |
|
}, |
|
"dtype": "bf16" |
|
}, |
|
"model": { |
|
"name_or_path": "black-forest-labs/FLUX.1-dev", |
|
"is_flux": true, |
|
"quantize": true |
|
}, |
|
"sample": { |
|
"sampler": "flowmatch", |
|
"sample_every": 250, |
|
"width": 1024, |
|
"height": 1024, |
|
"prompts": [ |
|
"woman with red hair, playing chess at the park, bomb going off in the background", |
|
"a woman holding a coffee cup, in a beanie, sitting at a cafe", |
|
"a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini", |
|
"a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background", |
|
"a bear building a log cabin in the snow covered mountains", |
|
"woman playing the guitar, on stage, singing a song, laser lights, punk rocker", |
|
"hipster man with a beard, building a chair, in a wood shop", |
|
"photo of a man, white background, medium shot, modeling clothing, studio lighting, white backdrop", |
|
"a man holding a sign that says, 'this is a sign'", |
|
"a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle" |
|
], |
|
"neg": "", |
|
"seed": 42, |
|
"walk_seed": true, |
|
"guidance_scale": 4, |
|
"sample_steps": 20 |
|
} |
|
} |
|
Using EMA |
|
|
|
############################################# |
|
# Running job: my_first_flux_lora_v1 |
|
############################################# |
|
|
|
|
|
Running 1 process |
|
Loading Flux model |
|
Loading transformer |
|
Quantizing transformer |
|
Loading vae |
|
Loading t5 |
|
Downloading shards: 0%| | 0/2 [00:00<?, ?it/s]
Downloading shards: 50%|βββββ | 1/2 [00:04<00:04, 4.38s/it]
Downloading shards: 100%|ββββββββββ| 2/2 [00:08<00:00, 4.22s/it]
Downloading shards: 100%|ββββββββββ| 2/2 [00:08<00:00, 4.25s/it] |
|
Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards: 50%|βββββ | 1/2 [00:00<00:00, 8.42it/s]
Loading checkpoint shards: 100%|ββββββββββ| 2/2 [00:00<00:00, 9.07it/s]
Loading checkpoint shards: 100%|ββββββββββ| 2/2 [00:00<00:00, 8.96it/s] |
|
Quantizing T5 |
|
Loading clip |
|
making pipe |
|
preparing |
|
create LoRA network. base dim (rank): 16, alpha: 16 |
|
neuron dropout: p=None, rank dropout: p=None, module dropout: p=None |
|
create LoRA for Text Encoder: 0 modules. |
|
create LoRA for U-Net: 494 modules. |
|
enable LoRA for U-Net |
|
Error running job: [Errno 2] No such file or directory: '/workspace/ai-toolkit/images' |
|
|
|
======================================== |
|
Result: |
|
- 0 completed jobs |
|
- 1 failure |
|
======================================== |
|
Traceback (most recent call last): |
|
File "/workspace/ai-toolkit/run.py", line 90, in <module> |
|
main() |
|
File "/workspace/ai-toolkit/run.py", line 86, in main |
|
raise e |
|
File "/workspace/ai-toolkit/run.py", line 78, in main |
|
job.run() |
|
File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run |
|
process.run() |
|
File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1567, in run |
|
self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) |
|
File "/workspace/ai-toolkit/toolkit/data_loader.py", line 571, in get_dataloader_from_datasets |
|
dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) |
|
File "/workspace/ai-toolkit/toolkit/data_loader.py", line 411, in __init__ |
|
with open(self.dataset_path, 'r') as f: |
|
FileNotFoundError: [Errno 2] No such file or directory: '/workspace/ai-toolkit/images' |
|
|