Model save

README.md

@@ -0,0 +1,66 @@
+---
+library_name: transformers
+base_model: kykim0/Llama-2-7b-ultrachat200k-2e
+tags:
+- trl
+- sft
+- generated_from_trainer
+model-index:
+- name: Llama-2-7b-ultrachat-syn
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Llama-2-7b-ultrachat-syn
+
+This model is a fine-tuned version of [kykim0/Llama-2-7b-ultrachat200k-2e](https://huggingface.co/kykim0/Llama-2-7b-ultrachat200k-2e) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0863
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 1
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 2
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 0.0994        | 0.9984 | 156  | 0.0907          |
+| 0.0916        | 1.9968 | 312  | 0.0863          |
+
+
+### Framework versions
+
+- Transformers 4.44.2
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.9968,
+    "total_flos": 101696396132352.0,
+    "train_loss": 0.11124454142573552,
+    "train_runtime": 7872.3532,
+    "train_samples": 20000,
+    "train_samples_per_second": 5.081,
+    "train_steps_per_second": 0.04
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.2"
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:979dcd3bbae6176eed808bcb15495288597c811720fae1122cd4f4e5093a0fbb
+oid sha256:be99ac7eade85d96adf96ef4fa4c75697bc21f1ebee313c71607536035dfd4bf
 size 4938985352

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b73b886df008716c9ee3e617f40a7e2a4dcc7b0143bbf85ce252706afd0a87b3
+oid sha256:724d489befa9d61b02af7f0c63b89b9b254957f82a726006630b0cb48285ce61
 size 4947390880

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2bd044aa1348aa0db12a2c081d362c16a675ef52764cc7b5198dfc02556cfee
+oid sha256:9f08ebb0ac1104c9c0d0f7de849185c96727a2339a375961c5ae044b20f2e2a5
 size 3590488816

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.9968,
+    "total_flos": 101696396132352.0,
+    "train_loss": 0.11124454142573552,
+    "train_runtime": 7872.3532,
+    "train_samples": 20000,
+    "train_samples_per_second": 5.081,
+    "train_steps_per_second": 0.04
+}

trainer_state.json

@@ -0,0 +1,275 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9968,
+  "eval_steps": 500,
+  "global_step": 312,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.064,
+      "grad_norm": 2.304359943215424,
+      "learning_rate": 6.25e-06,
+      "loss": 0.5192,
+      "step": 10
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 3.367745922034929,
+      "learning_rate": 1.25e-05,
+      "loss": 0.1579,
+      "step": 20
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 0.3120002563540976,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 0.1235,
+      "step": 30
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 0.34393102439669215,
+      "learning_rate": 1.9959742939952393e-05,
+      "loss": 0.1141,
+      "step": 40
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.22846734971092744,
+      "learning_rate": 1.9796753984232357e-05,
+      "loss": 0.1154,
+      "step": 50
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 0.22187942259321225,
+      "learning_rate": 1.9510565162951538e-05,
+      "loss": 0.1013,
+      "step": 60
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 0.1937516417577725,
+      "learning_rate": 1.9104775466588162e-05,
+      "loss": 0.0975,
+      "step": 70
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 0.18776075554849062,
+      "learning_rate": 1.8584487936018663e-05,
+      "loss": 0.1003,
+      "step": 80
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 0.16374083531357114,
+      "learning_rate": 1.795624548881781e-05,
+      "loss": 0.1011,
+      "step": 90
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.3083074608208316,
+      "learning_rate": 1.7227948638273918e-05,
+      "loss": 0.1019,
+      "step": 100
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 0.14996630876365036,
+      "learning_rate": 1.6408756139850243e-05,
+      "loss": 0.096,
+      "step": 110
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 0.10713257524125,
+      "learning_rate": 1.5508969814521026e-05,
+      "loss": 0.0918,
+      "step": 120
+    },
+    {
+      "epoch": 0.832,
+      "grad_norm": 0.16430611401390374,
+      "learning_rate": 1.4539904997395468e-05,
+      "loss": 0.0938,
+      "step": 130
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 0.13018547178323414,
+      "learning_rate": 1.3513748240813429e-05,
+      "loss": 0.0974,
+      "step": 140
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.249248518955414,
+      "learning_rate": 1.2443404061378941e-05,
+      "loss": 0.0994,
+      "step": 150
+    },
+    {
+      "epoch": 0.9984,
+      "eval_loss": 0.09072314202785492,
+      "eval_runtime": 40.5957,
+      "eval_samples_per_second": 24.633,
+      "eval_steps_per_second": 1.552,
+      "step": 156
+    },
+    {
+      "epoch": 1.024,
+      "grad_norm": 0.13728098111933706,
+      "learning_rate": 1.1342332658176556e-05,
+      "loss": 0.088,
+      "step": 160
+    },
+    {
+      "epoch": 1.088,
+      "grad_norm": 0.16275246072291852,
+      "learning_rate": 1.0224380642958052e-05,
+      "loss": 0.0947,
+      "step": 170
+    },
+    {
+      "epoch": 1.152,
+      "grad_norm": 0.15001498024621265,
+      "learning_rate": 9.103606910965666e-06,
+      "loss": 0.0939,
+      "step": 180
+    },
+    {
+      "epoch": 1.216,
+      "grad_norm": 0.13963224848380856,
+      "learning_rate": 7.994105842167274e-06,
+      "loss": 0.0876,
+      "step": 190
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.137367259845856,
+      "learning_rate": 6.909830056250527e-06,
+      "loss": 0.0869,
+      "step": 200
+    },
+    {
+      "epoch": 1.3439999999999999,
+      "grad_norm": 0.11133790927990313,
+      "learning_rate": 5.864414950334796e-06,
+      "loss": 0.0907,
+      "step": 210
+    },
+    {
+      "epoch": 1.408,
+      "grad_norm": 0.09323151899431767,
+      "learning_rate": 4.87100722594094e-06,
+      "loss": 0.0953,
+      "step": 220
+    },
+    {
+      "epoch": 1.472,
+      "grad_norm": 0.15783290675271067,
+      "learning_rate": 3.942099561591802e-06,
+      "loss": 0.0911,
+      "step": 230
+    },
+    {
+      "epoch": 1.536,
+      "grad_norm": 0.13377672988354583,
+      "learning_rate": 3.089373510131354e-06,
+      "loss": 0.0867,
+      "step": 240
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.1192199124767211,
+      "learning_rate": 2.323552596419889e-06,
+      "loss": 0.0871,
+      "step": 250
+    },
+    {
+      "epoch": 1.6640000000000001,
+      "grad_norm": 0.11020679845316991,
+      "learning_rate": 1.6542674627869738e-06,
+      "loss": 0.0896,
+      "step": 260
+    },
+    {
+      "epoch": 1.728,
+      "grad_norm": 0.09198353508041567,
+      "learning_rate": 1.0899347581163222e-06,
+      "loss": 0.0928,
+      "step": 270
+    },
+    {
+      "epoch": 1.792,
+      "grad_norm": 0.11654532424334699,
+      "learning_rate": 6.37651293602628e-07,
+      "loss": 0.0917,
+      "step": 280
+    },
+    {
+      "epoch": 1.8559999999999999,
+      "grad_norm": 0.10356962879053153,
+      "learning_rate": 3.0310479623313125e-07,
+      "loss": 0.0847,
+      "step": 290
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.0958617515134905,
+      "learning_rate": 9.0502382320653e-08,
+      "loss": 0.0868,
+      "step": 300
+    },
+    {
+      "epoch": 1.984,
+      "grad_norm": 0.08860455805163223,
+      "learning_rate": 2.5176505749346937e-09,
+      "loss": 0.0916,
+      "step": 310
+    },
+    {
+      "epoch": 1.9968,
+      "eval_loss": 0.08628984540700912,
+      "eval_runtime": 40.3884,
+      "eval_samples_per_second": 24.76,
+      "eval_steps_per_second": 1.56,
+      "step": 312
+    },
+    {
+      "epoch": 1.9968,
+      "step": 312,
+      "total_flos": 101696396132352.0,
+      "train_loss": 0.11124454142573552,
+      "train_runtime": 7872.3532,
+      "train_samples_per_second": 5.081,
+      "train_steps_per_second": 0.04
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 101696396132352.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}