Release 0.1.0 beta

Release 0.1.0 beta of "AnimaEngine" AI with based-model LoRa and tencent/HunyuanVideo using trojblue dataset

.gitattributes

@@ -1,35 +1,37 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ipynb filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,71 @@
 ---
 license: mit
+datasets:
+- trojblue/test-HunyuanVideo-anime-stills
+base_model:
+- tencent/HunyuanVideo
+pipeline_tag: text-to-video
+tags:
+- hunyuan-video
+- text-to-video
+- lora
+- diffusers
+- template:diffusion-lora
+instance_prompt: anime
+widget:
+- text: 'anime girl 1girl, alcohol carton, blush, braid, bridge, crosswalk, dress, green dress, holding carton, long hair, long sleeves, multiple girls, night, open mouth, outdoors, pedestrian bridge, purple eyes, red hair, single braid, solo focus, spaghetti strap'
+  output:
+    url: samples/ComfyUI_00017_.webp
+- text: '<prompt in metadata>'
+  output:
+    url: samples/ComfyUI_00024_.webp
+- text: '<prompt in metadata>'
+  output:
+    url: samples/ComfyUI_00068_.webp
+- text: 'anime scene of a vibrant carnival with colorful rides, games, and food stalls, and a clown handing balloons to a group of laughing children.'
+  output:
+    url: samples/ComfyUI_00071_.webp
 ---
+
+# **AnimaEngine**
+
+<Gallery />
+
+
+[v0.1 - testing version]
+
+
+
+An anime-style lora trained on anime screencaps and illustrations, aimed to create vibrant, bright and colorful anime style motions. It's good at generating single person motions (and girls better than boys).
+
+
+
+## Usage
+
+To use the lora (and to use HunyuanVideo in general) in ComfyUI, it's recommended to install the [VideoHelperSuite](https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite), and update to torch 2.5.1+cu124:
+
+```
+pip install -U torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu124
+```
+
+
+
+A sample workflow for using the lora can be found in the Huggingface repo:
+
+- [v0.1/ComfyUI_00024_.webp · trojblue/HunyuanVideo-lora-AnimeShot at main](https://huggingface.co/trojblue/HunyuanVideo-lora-AnimeShot/blob/main/v0.1/ComfyUI_00024_.webp)
+
+
+
+## Prompting
+
+Use prompts in the format of `anime <subject> <description>` to get the best results, use resolution `544x960` (and usually horizontal works a little bit better than vertical). for example:
+
+
+
+- anime girl with pink twin tails and green eyes, wearing a school uniform, holding a stack of books in a bustling library filled with sunlight streaming through tall windows.
+- anime boy with silver hair and blue eyes, wearing a casual hoodie, sitting on a park bench, feeding pigeons with a gentle smile.
+- anime girl 1girl, alcohol carton, blush, braid, bridge, crosswalk, dress, green dress, holding carton, long hair, long sleeves, multiple girls, night, open mouth, outdoors, pedestrian bridge, purple eyes, red hair, single braid, solo focus, spaghetti strap
+
+## Limitations
+
+It's trained as a test model so sometimes when body movements are large it gets disconnected. Also some concepts are less anime-like compared to others. I do plan to update the model later with more training time and dataset.

v0.1/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "linear1",
+    "linear2",
+    "img_mlp.fc1",
+    "img_attn_proj",
+    "txt_attn_proj",
+    "txt_mlp.fc2",
+    "img_mod.linear",
+    "img_attn_qkv",
+    "img_mlp.fc2",
+    "modulation.linear",
+    "txt_mlp.fc1",
+    "txt_mod.linear",
+    "txt_attn_qkv"
+  ],
+  "task_type": null,
+  "use_dora": false,
+  "use_rslora": false
+}

v0.1/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55eaaaea0420811a2bcd3225a616a6694c696a433843ca71421926309a32faff
+size 322519480