init adapters

README.md

@@ -0,0 +1,73 @@
+---
+base_model: meta-llama/Llama-2-13b-hf
+tags:
+- generated_from_trainer
+datasets:
+- yhavinga/mc4_nl_cleaned
+model-index:
+- name: tiny-3e-4lr+1152tbs+1ep+0.1wd
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# tiny-3e-4lr+1152tbs+1ep+0.1wd
+
+This model is a fine-tuned version of [meta-llama/Llama-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf) on the yhavinga/mc4_nl_cleaned micro dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.7676
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 12
+- eval_batch_size: 12
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 16
+- gradient_accumulation_steps: 6
+- total_train_batch_size: 1152
+- total_eval_batch_size: 192
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.03
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.8784        | 0.09  | 90   | 1.8820          |
+| 1.8344        | 0.19  | 180  | 1.8542          |
+| 1.8351        | 0.28  | 270  | 1.8355          |
+| 1.8206        | 0.37  | 360  | 1.8212          |
+| 1.8021        | 0.47  | 450  | 1.8088          |
+| 1.8102        | 0.56  | 540  | 1.7982          |
+| 1.7991        | 0.65  | 630  | 1.7890          |
+| 1.7788        | 0.74  | 720  | 1.7811          |
+| 1.7915        | 0.84  | 810  | 1.7742          |
+| 1.7715        | 0.93  | 900  | 1.7676          |
+
+
+### Framework versions
+
+- Transformers 4.31.0.dev0
+- Pytorch 2.0.1+cu117
+- Datasets 2.13.1
+- Tokenizers 0.13.3

adapter_config.json

@@ -0,0 +1,21 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1cb1f00c70442197b147fecd625a413fa98a8c06757cd262823421f54012d2b
+size 104913037

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 1.7676039934158325,
+    "eval_runtime": 2546.7948,
+    "eval_samples": 58498,
+    "eval_samples_per_second": 22.969,
+    "eval_steps_per_second": 0.12,
+    "perplexity": 5.856803596460139,
+    "train_loss": 0.28749041867971176,
+    "train_runtime": 27407.5144,
+    "train_samples": 1114177,
+    "train_samples_per_second": 40.652,
+    "train_steps_per_second": 0.035
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 1.7676039934158325,
+    "eval_runtime": 2546.7948,
+    "eval_samples": 58498,
+    "eval_samples_per_second": 22.969,
+    "eval_steps_per_second": 0.12,
+    "perplexity": 5.856803596460139
+}

info.txt

@@ -0,0 +1,2 @@
+CMD: -n 4 -g 4 -t 80 -o llama2-13b-ft-mc4_nl_cleaned_tiny/tiny-3e-4lr+1152tbs+1ep+0.1wd -p llama2-13b-ft-mc4_nl_cleaned -e --preprocessed_dataset /dodrio/scratch/projects/2023_005/llm-finetuning/preprocessed_datasets/mc4_nl_cleaned--tiny-Llama-2-70b-hf-4096 --learning_rate 3e-4 --model_name_or_path meta-llama/Llama-2-13b-hf --per_device_train_batch_size 12 --per_device_eval_batch_size 12 --gradient_accumulation_steps 6 --eval_accumulation_steps 24 --save_total_limit 3 --eval_steps 90 --save_steps 90 --logging_first_step --weight_decay 0.1 --lr_scheduler_type cosine --early_stopping_patience 4 --warmup_ratio 0.03 --deepspeed ds_config_zero2.json --use_flash_attention -r evocpo0z
+

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.28749041867971176,
+    "train_runtime": 27407.5144,
+    "train_samples": 1114177,
+    "train_samples_per_second": 40.652,
+    "train_steps_per_second": 0.035
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4eaea2ba1845c2e33afb34a72565b01d41e7f85d17d0213020d84aeb1ac48c
+size 5691