consolidate controlnets

A1111_controlnet_extension_weights/anime_control_dreamshaper.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26b896ee1b5ae1321f055bbd712cfa2b7eb90d7096c80b64973ee33119a89204
+size 722596338

A1111_controlnet_extension_weights/anime_control_neverending.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c37cbd4bd653ea420e848a344fe5ffaacfb60d986b97521759c9de3d3a48bb6e
+size 722596338

A1111_controlnet_extension_weights/anime_control_protogen.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2618357f330a5d4faa0e034011a637374c587031444469ea85b0676a7ec52bb7
+size 722596338

A1111_controlnet_extension_weights/anime_control_realdosmix.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:407ced814fb5701f8e3c6d28dcaff173055202299262974d13626ccabc938071
+size 722596338

README.md

@@ -0,0 +1,37 @@
+---
+license: openrail
+---
+## Source
+
+The controlnets `canny_control` and `lineart_anime_control` were converted to `diffusers` format directly from the v1.1 originals in https://huggingface.co/lllyasviel
+
+## [Try Style Controlnet with A1111 WebUI](https://github.com/1lint/style_controlnet)
+
+Use anime styling controlnet with A1111 Stable Diffusion WebUI by downloading the weights from the A1111_webui_weights folder inside this repository. These weights can be used directly with the existing [A1111 Webui Controlnet Extension](https://github.com/Mikubill/sd-webui-controlnet), see this reddit post for [instructions](https://www.reddit.com/r/StableDiffusion/comments/119o71b/a1111_controlnet_extension_explained_like_youre_5/) on using the controlnet extension. 
+
+For each anime controlnet there is a standard variant, and a no-hint variant. 
+
+### Pass a black square as the controlnet conditioning image if you only want to add anime style guidance to image generation, or pass an anime image with canny preprocessing if you want to add both anime style and canny guidance to the image. See `assets` folder for example hints. 
+_________________________________________________
+
+### Generated using `anime_control_dreamshaper.safetensors` controlnet with canny hint
+![](./assets/hint_grid.png)
+_________________________________________________
+### Generated using `anime_control_dreamshaper.safetensors` controlnet with black square (numpy array of zeros) as hint
+![](./assets/zerohint_grid.png)
+_________________________________________________
+### Generated using `anime_styler_dreamshaper.safetensors` controlnet with no controlnet conditioning hint
+![](./assets/nohint_grid.png)
+_________________________________________________
+
+### Grid from left to right: Controlnet weight 0.0 (base model output), Controlnet weight 0.5, Controlnet weight 1.0, Controlnet hint (white means no controlnet hint passed)
+
+Generation settings for examples: Prompt: "1girl, blue eyes", Seed: 2048, all other settings are A1111 Webui defaults
+
+Base model used for examples: [Dreamshaper](https://civitai.com/models/4384/dreamshaper)
+_________________________________________________
+
+## Details
+
+These controlnets were initialized from a distinct UNet (`andite/anything-v4.5`), and predominantly trained without any controlnet conditioning image on a synthetically generated anime image dataset from the base model. (see `lint/anybooru` for a subset example of the training set). Then the main controlnet weights were frozen, the input hint block weights added back in and trained on the same dataset using canny image processing to generate the controlnet conditioning image. 
+

anime_dream/config.json

@@ -0,0 +1,52 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "_name_or_path": "models/neverendingDreamNED_bakedVae_animestyler/checkpoint-24000",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "controlnet_conditioning_channels": 0,
+  "controlnet_conditioning_embedding_type": "null",
+  "conv_in_kernel": 3,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

anime_dream/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47ab156c099f97e6f97952acc86a08dd22ccd72809d68406533afc7c95cdb4d7
+size 722672331

anime_dreamshaper/config.json

@@ -0,0 +1,45 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "_name_or_path": "/home/user/style_controlnet/models/dreamshaper_331BakedVae_animestyler/checkpoint-320000",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "controlnet_conditioning_channels": 0,
+  "controlnet_conditioning_embedding_type": "null",
+  "conv_in_kernel": 3,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

anime_dreamshaper/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0fdd80f915fbe7f61a31cb51b0d65653dd36ae4bf2dcc669bd6c4fe6ee99eba
+size 720423957

anime_neverending/config.json

@@ -0,0 +1,44 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "controlnet_conditioning_channels": 3,
+  "controlnet_conditioning_embedding_type": "null",
+  "conv_in_kernel": 3,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

anime_neverending/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5de72b8f81788c7d8ebbd514c15f3ec73df8246ab86275b71dcedcd90c3f1035
+size 1440809480

anime_protogen/config.json

@@ -0,0 +1,52 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "_name_or_path": "models/protogenX58RebuiltScifi_10_animestyler/checkpoint-57544",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "controlnet_conditioning_channels": 0,
+  "controlnet_conditioning_embedding_type": "null",
+  "conv_in_kernel": 3,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

anime_protogen/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ded318fedc1fc6aab56ffe1f3574257541b013ffcd2b9d62918481998002c298
+size 722673357

anime_realdosmix/config.json

@@ -0,0 +1,45 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "_name_or_path": "models/realdosmix__animestyler/checkpoint-70000",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "controlnet_conditioning_channels": 0,
+  "controlnet_conditioning_embedding_type": "null",
+  "conv_in_kernel": 3,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

anime_realdosmix/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ad6185126aa57a7ca86366c04eab912a040ea845326e769a8c84541f82d8d8a
+size 1440809480

anime_vinteprotomix/config.json

@@ -0,0 +1,45 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.14.0",
+  "_name_or_path": "lint/simpathizer",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "controlnet_conditioning_channels": 3,
+  "controlnet_conditioning_embedding_type": "null",
+  "conv_in_kernel": 3,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

anime_vinteprotomix/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e00b52dc99375b4d6f3ac592584569ffe7eee90c55a85aa292ed98116473b37
+size 720423962

lineart_anime_control/config.json

@@ -0,0 +1,41 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "class_embed_type": null,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

lineart_anime_control/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cbb004daa1a79a954eb2d268826417fbbc0327c25f1e7580244912204fef135
+size 722696633