Upload folder using huggingface_hub

README.md

@@ -0,0 +1,28 @@
+---
+license: mit
+license_name: deepseek
+license_link: LICENSE
+pipeline_tag: text-generation
+library_name: transformers
+base_model:
+  - deepseek-ai/Janus-Pro-1B
+tags:
+  - chat
+---
+
+This model is derived from https://huggingface.co/deepseek-ai/Janus-Pro-1B and the main modifications are as follows
+
+- bin files are updated to safetensors
+- Add chat_template
+
+`4bit` mainly refers to quantifying the LLM part to 4 bits.
+
+## Quick Start
+
+In Macos (Apple silicon), use [mlx](https://github.com/ml-explore/mlx) framework https://github.com/wnma3mz/tLLM
+
+```bash
+tllm.server --model_path $MODEL_PATH --hostname localhost --is_local --client_size 1
+```
+
+`$MODEL_PATH` like `wnma3mz/Janus-Pro-1B-4bit`

config.json

@@ -0,0 +1,87 @@
+{
+  "aligner_config": {
+    "cls": "MlpProjector",
+    "model_type": "aligner",
+    "params": {
+      "depth": 2,
+      "input_dim": 1024,
+      "n_embed": 2048,
+      "projector_type": "mlp_gelu"
+    }
+  },
+  "architectures": [
+    "JanusProConditionalGeneration"
+  ],
+  "gen_aligner_config": {
+    "cls": "MlpProjector",
+    "model_type": "gen_aligner",
+    "params": {
+      "depth": 2,
+      "input_dim": 8,
+      "n_embed": 2048,
+      "projector_type": "mlp_gelu"
+    }
+  },
+  "gen_head_config": {
+    "cls": "vision_head",
+    "model_type": "gen_head",
+    "params": {
+      "image_token_embed": 2048,
+      "image_token_size": 16384,
+      "n_embed": 2048
+    }
+  },
+  "gen_vision_config": {
+    "cls": "VQ-16",
+    "model_type": "gen_vision",
+    "params": {
+      "image_token_size": 16384,
+      "n_embed": 8
+    }
+  },
+  "num_hidden_layers": 24,
+  "language_config": {
+    "hidden_size": 2048,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 16384,
+    "model_type": "llama",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 16,
+    "torch_dtype": "bfloat16",
+    "rms_norm_eps": 1e-6,
+    "tie_word_embeddings": false,
+    "vocab_size": 102400
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.33.1",
+  "quantization": {
+    "group_size": 64,
+    "bits": 4
+  },
+  "quantization_config": {
+      "group_size": 64,
+      "bits": 4
+  },  
+  "vision_config": {
+    "cls": "CLIPVisionTower",
+    "model_type": "vision",
+    "use_head": true,
+    "ignore_head": true,
+    "image_size": 384,
+    "patch_size": 16,
+    "hidden_size": 1024,
+    "num_hidden_layers": 24,
+    "num_attention_heads": 16,
+    "intermediate_size": 4096,
+    "mlp_ratio": 4,
+    "global_pool": "map",
+    "use_checkpoint": false,    
+    "params": {
+      "image_size": 384,
+      "model_name": "siglip_large_patch16_384",
+      "select_feature": "same",
+      "select_layer": -1
+    }
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1306d068356ec26c1ef49f3205c9b4ac61e461ba4154eb509ebae4abc13928b2
+size 1803190693

preprocessor_config.json

@@ -0,0 +1,23 @@
+{
+  "background_color": [
+    127,
+    127,
+    127
+  ],
+  "do_normalize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "VLMImageProcessor",
+  "image_size": 384,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "min_size": 14,
+  "processor_class": "VLChatProcessor",
+  "rescale_factor": 0.00392156862745098
+}

processor_config.json

@@ -0,0 +1,9 @@
+{
+  "add_special_token": false,
+  "ignore_id": -100,
+  "image_tag": "<image_placeholder>",
+  "mask_prompt": true,
+  "num_image_tokens": 576,
+  "processor_class": "VLChatProcessor",
+  "sft_format": "deepseek"
+}

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "additional_special_tokens": [
+    "<image_placeholder>",
+    "<patch_placeholder>",
+    "<|ref|>",
+    "<|/ref|>",
+    "<|det|>",
+    "<|/det|>",
+    "<|grounding|>",
+    "<|User|>",
+    "<|Assistant|>"
+  ],
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "pad_token": "<｜▁pad▁｜>"
+}

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "model_max_length": 16384,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{% break %}{%- endif %}{%- endfor %}{{ns.system_prompt + '\n\n'}}{%- for message in messages %}{%- if message['role'] == 'user' %}{{'<|User|>: ' + message['content'] + '\n\n'}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<|Assistant|>: ' + content + '<｜end of sentence｜>'}}{%- endif %}{%- endfor -%}{% if add_generation_prompt %}{{'<|Assistant|>:'}}{% endif %}",
+  "use_default_system_prompt": true
+}