Feat KoAlpaca Multimodel Gradio

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+.DS_Store
+*.ipynb

app.py

@@ -1,6 +1,11 @@
 import gradio as gr
 from gradio.themes.utils import colors
 from t5 import T5
+from koalpaca import KoAlpaca
+
+LOCAL_TEST = True
+MODELS = []
+cur_index = 0
 
 def prepare_theme():
     theme = gr.themes.Default(primary_hue=colors.gray, 
@@ -37,22 +42,31 @@ def prepare_theme():
     return theme
 
 def chat(message, chat_history):
-    global model
-    response = model.chat(message)
+    response = MODELS[cur_index].generate(message)
     chat_history.append((message, response))
     return "", chat_history
 
+def change_model_index(idx):
+    global cur_index
+    cur_index = idx
+    print(cur_index)
+    return
+
 if __name__=='__main__':
     theme = prepare_theme()
-    model = T5()
+    
+    MODELS.append(T5())
+    if not LOCAL_TEST:
+        MODELS.append(KoAlpaca())
 
     with gr.Blocks(theme=theme) as demo:
         with gr.Row():
+            rd = gr.Radio(['T5','KoAlpaca'], value='T5', type='index', label='Model Selection', show_label=True, interactive=True)
             with gr.Column(scale=5): # 챗봇 부분 
                 chatbot = gr.Chatbot(label="T5", bubble_full_width=False)
                 with gr.Row():
                     txt = gr.Textbox(show_label=False, placeholder='Send a message...', container=False)
 
         txt.submit(chat, [txt, chatbot], [txt, chatbot])
-
+        rd.select(change_model_index, [rd])
     demo.launch(debug=True, share=True)

koalpaca.py

@@ -0,0 +1,33 @@
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, GenerationConfig
+from peft import PeftModel, PeftConfig
+
+class KoAlpaca:
+    def __init__(self):
+        peft_model_id = "4n3mone/Komuchat-koalpaca-polyglot-12.8B"
+        config = PeftConfig.from_pretrained(peft_model_id)
+        self.bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16
+        )
+        self.model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, quantization_config=self.bnb_config, device_map={"":0})
+        self.model = PeftModel.from_pretrained(self.model, peft_model_id)
+        self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
+        self.gen_config = GenerationConfig.from_pretrained('./models/koalpaca', 'gen_config.json')
+        self.INPUT_FORMAT = "### 질문: <INPUT>\n\n### 답변:"
+        self.model.eval()
+        
+    def generate(self, inputs):
+        inputs = self.INPUT_FORMAT.replace('<INPUT>', inputs)
+        output_ids = self.model.generate(
+            **self.tokenizer(
+                inputs, 
+                return_tensors='pt', 
+                return_token_type_ids=False
+            ).to('cuda'), 
+            generation_config=self.gen_config
+        )
+        outputs = self.tokenizer.decode(output_ids[0]).split("### 답변: ")[-1]
+        return outputs

model/pko_t5_fm_patience10/config.json

@@ -1,29 +0,0 @@
-{
-  "_name_or_path": "paust/pko-t5-large",
-  "architectures": [
-    "T5ForConditionalGeneration"
-  ],
-  "d_ff": 2816,
-  "d_kv": 64,
-  "d_model": 1024,
-  "decoder_start_token_id": 0,
-  "dropout_rate": 0.1,
-  "eos_token_id": 1,
-  "feed_forward_proj": "gated-gelu",
-  "initializer_factor": 1.0,
-  "is_encoder_decoder": true,
-  "layer_norm_epsilon": 1e-06,
-  "model_type": "t5",
-  "num_decoder_layers": 24,
-  "num_heads": 16,
-  "num_layers": 24,
-  "output_past": true,
-  "pad_token_id": 0,
-  "relative_attention_max_distance": 128,
-  "relative_attention_num_buckets": 32,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float32",
-  "transformers_version": "4.16.2",
-  "use_cache": true,
-  "vocab_size": 50364
-}

model/pko_t5_fm_patience10/pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ff427d0fac31330c4f73047080eee6fb4280c44e64ec586d8c897c45dd88997b
-size 3282255749

model/pko_t5_fm_patience10/runs/Apr08_08-56-15_main1/1680944176.4623845/events.out.tfevents.1680944176.main1.37029.1

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5d711899223dcba6a3f996fd53d46a45c177f4deea6f9a41c7d3a8e587ed4c29
-size 5024

model/pko_t5_fm_patience10/runs/Apr08_08-56-15_main1/events.out.tfevents.1680944176.main1.37029.0

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:673b3ea74f0648feca11e100846b45ff35c80ff61a724dbc65ef5637d41599a7
-size 9060

model/pko_t5_fm_patience10/special_tokens_map.json

@@ -1 +0,0 @@
-{"eos_token": "</s>", "unk_token": "<pad>", "pad_token": "<pad>", "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"]}

model/pko_t5_fm_patience10/tokenizer_config.json

@@ -1 +0,0 @@
-{"eos_token": "</s>", "unk_token": "<pad>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"], "special_tokens_map_file": "./models/pko-t5/special_tokens_map.json", "name_or_path": "paust/pko-t5-large", "tokenizer_class": "T5Tokenizer"}

model/pko_t5_fm_patience10/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b1c51a0cfd21bf9256ed7d47851a2a4cba51264bf102db415ebea5927746bba4
-size 3195

model/pko_t5_instiz_patience10/config.json

@@ -1,29 +0,0 @@
-{
-  "_name_or_path": "paust/pko-t5-large",
-  "architectures": [
-    "T5ForConditionalGeneration"
-  ],
-  "d_ff": 2816,
-  "d_kv": 64,
-  "d_model": 1024,
-  "decoder_start_token_id": 0,
-  "dropout_rate": 0.1,
-  "eos_token_id": 1,
-  "feed_forward_proj": "gated-gelu",
-  "initializer_factor": 1.0,
-  "is_encoder_decoder": true,
-  "layer_norm_epsilon": 1e-06,
-  "model_type": "t5",
-  "num_decoder_layers": 24,
-  "num_heads": 16,
-  "num_layers": 24,
-  "output_past": true,
-  "pad_token_id": 0,
-  "relative_attention_max_distance": 128,
-  "relative_attention_num_buckets": 32,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float32",
-  "transformers_version": "4.16.2",
-  "use_cache": true,
-  "vocab_size": 50364
-}

models/koalpaca/gen_config.json

@@ -0,0 +1,11 @@
+{
+    "_from_model_config": true,
+    "bos_token_id": 0,
+    "do_sample": true,
+    "early_stopping": true,
+    "eos_token_id": 2,
+    "max_new_tokens": 50,
+    "no_repeat_ngram_size": 3,
+    "num_beams": 2,
+    "transformers_version": "4.28.0"
+}

models/pko_t5_COMU_patience10/gen_config.json

@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "do_sample": true,
+  "eos_token_id": 1,
+  "max_new_tokens": 64,
+  "min_length": 10,
+  "no_repeat_ngram_size": 2,
+  "num_beams": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.28.0"
+}

requirements.txt

@@ -1,5 +1,6 @@
-transformers==4.16.2
+transformers==4.28.0
 gradio==4.1.1
 numpy==1.26.1
 pandas==2.1.2
-torch==2.0.1
+torch==2.0.1
+git+https://github.com/huggingface/peft.git

t5.py

@@ -1,32 +1,30 @@
-from transformers import T5TokenizerFast, T5ForConditionalGeneration
+from transformers import T5TokenizerFast, T5ForConditionalGeneration, GenerationConfig
 
 class T5:
     def __init__(self,
-            model_dir:str='./model/pko_t5_COMU_patience10',
+            model_dir:str='./models/pko_t5_COMU_patience10',
             max_input_length:int=64,
-            max_target_length:int=64,
-            prefix:str='qa question: '
+            max_target_length:int=64
             ):
         self.model = T5ForConditionalGeneration.from_pretrained(model_dir)
         self.tokenizer = T5TokenizerFast.from_pretrained(model_dir)
+        self.gen_config = GenerationConfig.from_pretrained(model_dir, 'gen_config.json')
+        
         self.max_input_length = max_input_length
         self.max_target_length = max_target_length
-        self.prefix = prefix
+        self.INPUT_FORMAT = 'qa question: <INPUT>'
         
         # add tokens
         self.tokenizer.add_tokens(["#화자#", "#청자#", "#(남자)청자#", "#(남자)화자#", "#(여자)청자#", "(여자)화자"])
         self.model.resize_token_embeddings(len(self.tokenizer))
         self.model.config.max_length = max_target_length
         self.tokenizer.model_max_length = max_target_length
-        
-    def chat(self, inputs):
-        inputs = [self.prefix + inputs]
+    
+    def generate(self, inputs):
+        inputs = self.INPUT_FORMAT.replace("<INPUT>", inputs)
         input_ids = self.tokenizer(inputs, max_length=self.max_input_length, truncation=True, return_tensors="pt")
-        output_tensor = self.model.generate(**input_ids, num_beams=2, do_sample=True, min_length=10, max_length=self.max_target_length, no_repeat_ngram_size=2) #repetition_penalty=2.5
+        output_tensor = self.model.generate(**input_ids, generation_config=self.gen_config)
         output_ids = self.tokenizer.batch_decode(output_tensor, skip_special_tokens=True, clean_up_tokenization_spaces=True)
         outputs = str(output_ids)
         outputs = outputs.replace('[', '').replace(']', '').replace("'", '').replace("'", '')
-        return outputs
-        
-        
-        
+        return outputs