Update README.md

README.md

@@ -1,63 +1,37 @@
-
-어느정도 이제는 말을 조금 알아 듣는다.
-
-### 8bit 학습 기준
-
-{
-  "_name_or_path": "saltlux/Ko-Llama3-Luxia-8B",
-  "architectures": [
-    "LlamaForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "bos_token_id": 128000,
-  "eos_token_id": 128001,
-  "hidden_act": "silu",
-  "hidden_size": 4096,
-  "initializer_range": 0.02,
-  "intermediate_size": 14336,
-  "max_position_embeddings": 8192,
-  "model_type": "llama",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 8,
-  "pretraining_tp": 1,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 500000.0,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float16",
-  "transformers_version": "4.39.1",
-  "use_cache": false,
-  "vocab_size": 145792
-}
-
-batch_size = 16
-num_epochs = 1
-micro_batch = 1
-gradient_accumulation_steps = batch_size
-
-### 훈련 방법에 대한 하이퍼 파라미터
-cutoff_len = model.config.hidden_size
-lr_scheduler = 'cosine'
-warmup_ratio = 0.06 # warmup_steps = 100
-learning_rate = 2e-4
-optimizer = 'adamw_torch'
-weight_decay = 0.01
-max_grad_norm = 0.8 # 모델이 너무 과적합되는거같거나 이상한 방향으로 학습이 되는거같으면 값을 줄여보자.
-
-### LoRA config
-lora_r = 16
-lora_alpha = 16 
-lora_dropout = 0.05
-lora_target_modules = ["gate_proj", "down_proj", "up_proj"]
-
-### Tokenizer에서 나오는 input값 설정 옵션
-train_on_inputs = False
-add_eos_token = True
-
-val_data = None
-
-# Others
-resume_from_checkpoint = False # !! 만약 모델을 이어서 훈련하고 싶다면, './custom_LLM/checkpoint-[xxx]'와 같이 파일 경로를 입력해야 합니다!
-output_dir = './custom_LLM'
+---
+datasets:
+- kyujinpy/KOpen-platypus
+language:
+- ko
+- en
+pipeline_tag: text-generation
+---
+
+### Model Card for Model ID
+base_model : [Ko-Llama3-Luxia-8B](https://huggingface.co/saltlux/Ko-Llama3-Luxia-8B)
+
+### Basic usage
+```python
+# pip install accelerate
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+tokenizer = AutoTokenizer.from_pretrained("MDDDDR/Ko-Luxia-8B-it-v0.3")
+model = AutoModelForCausalLM.from_pretrained(
+    "MDDDDR/Ko-Luxia-8B-it-v0.3",
+    device_map="auto",
+    torch_dtype=torch.bfloat16
+)
+
+input_text = "Write me a poem about Machine Learning."
+input_ids = tokenizer(input_text, return_tensors="pt").to("cuda")
+
+outputs = model.generate(**input_ids)
+print(tokenizer.decode(outputs[0]))
+```
+
+### Training dataset
+dataset : [kyujinpy/KOpen-platypus](https://huggingface.co/datasets/kyujinpy/KOpen-platypus)
+
+### Hardware
+RTX 3090 Ti 24GB x 1