Model save

README.md

@@ -0,0 +1,69 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-UFB-0
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-UFB-0
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.1492
+- Rewards/chosen: -1.5452
+- Rewards/rejected: -7.2115
+- Rewards/accuracies: 0.8359
+- Rewards/margins: 5.6663
+- Logps/rejected: -171.0846
+- Logps/chosen: -143.6666
+- Logits/rejected: -2.3237
+- Logits/chosen: -2.3692
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.36.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.369189977645874,
+    "eval_logits/rejected": -2.323690414428711,
+    "eval_logps/chosen": -143.66659545898438,
+    "eval_logps/rejected": -171.08460998535156,
+    "eval_loss": 0.14922186732292175,
+    "eval_rewards/accuracies": 0.8359375,
+    "eval_rewards/chosen": -1.5451841354370117,
+    "eval_rewards/margins": 5.666274070739746,
+    "eval_rewards/rejected": -7.2114577293396,
+    "eval_runtime": 155.3805,
+    "eval_samples": 506,
+    "eval_samples_per_second": 3.257,
+    "eval_steps_per_second": 0.103,
+    "train_loss": 0.2053286122078282,
+    "train_runtime": 6354.2323,
+    "train_samples": 14176,
+    "train_samples_per_second": 2.231,
+    "train_steps_per_second": 0.07
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.369189977645874,
+    "eval_logits/rejected": -2.323690414428711,
+    "eval_logps/chosen": -143.66659545898438,
+    "eval_logps/rejected": -171.08460998535156,
+    "eval_loss": 0.14922186732292175,
+    "eval_rewards/accuracies": 0.8359375,
+    "eval_rewards/chosen": -1.5451841354370117,
+    "eval_rewards/margins": 5.666274070739746,
+    "eval_rewards/rejected": -7.2114577293396,
+    "eval_runtime": 155.3805,
+    "eval_samples": 506,
+    "eval_samples_per_second": 3.257,
+    "eval_steps_per_second": 0.103
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf3b184609aecb2affdc013bbcd2c6e38a2f598c1fdee8a4208c9fec64a08017
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49abe7a9af3b868d215888761f2f689e4627177869db29a002579766882b920b
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8833c25cb140d8fc7b846c48d039faa25825a185d1ce3da91127a0d7fd509f9
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Feb04_16-28-17_node01/events.out.tfevents.1707035407.node01.3855476.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a7207a2faa5fd931a9eb178d0e073cf4e322debf1dcb964f347d0b01f444df7
+size 53568

runs/Feb04_16-28-17_node01/events.out.tfevents.1707048530.node01.3855476.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7eaae45021b57a201ae50d0e5b2c10c6be94ad66686f2cac7f67788f5292b9
+size 828

runs/Feb05_16-47-09_node01/events.out.tfevents.1707122857.node01.4036661.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ada0357fc5ef3c6ad2f89f417a6f8e18b5e698ae78ce218aed98b5d1839faf5
+size 102598

runs/Feb05_16-47-09_node01/events.out.tfevents.1707131668.node01.4036661.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17d49293b9d1238f346a8a93d133d92ec96387625e40fa705551a08196f50774
+size 828

runs/Feb07_09-21-48_node01/events.out.tfevents.1707269266.node01.133687.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:849d7ed8b31e72ea486502fe8941239597ab54576694d05dab5de679a27ab45d
+size 33174

runs/Feb07_09-21-48_node01/events.out.tfevents.1707275776.node01.133687.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30fa91daf3a171cf360b10ad4bd0495f46d59e825bdac1f8e538f8ac33b17c87
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.2053286122078282,
+    "train_runtime": 6354.2323,
+    "train_samples": 14176,
+    "train_samples_per_second": 2.231,
+    "train_steps_per_second": 0.07
+}

trainer_state.json

@@ -0,0 +1,660 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 443,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.111111111111111e-08,
+      "logits/chosen": -2.737483501434326,
+      "logits/rejected": -2.581545829772949,
+      "logps/chosen": -157.5369873046875,
+      "logps/rejected": -100.35362243652344,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.111111111111111e-07,
+      "logits/chosen": -2.61592960357666,
+      "logits/rejected": -2.572232484817505,
+      "logps/chosen": -124.513671875,
+      "logps/rejected": -91.37435913085938,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.04052966088056564,
+      "rewards/margins": 0.01970052346587181,
+      "rewards/rejected": 0.020829135552048683,
+      "step": 10
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.222222222222222e-07,
+      "logits/chosen": -2.565521001815796,
+      "logits/rejected": -2.4634904861450195,
+      "logps/chosen": -114.85684967041016,
+      "logps/rejected": -80.09593963623047,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.4715304970741272,
+      "rewards/margins": 0.5839148759841919,
+      "rewards/rejected": -0.11238440126180649,
+      "step": 20
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.333333333333333e-07,
+      "logits/chosen": -2.5056052207946777,
+      "logits/rejected": -2.4653897285461426,
+      "logps/chosen": -123.69754791259766,
+      "logps/rejected": -103.3398208618164,
+      "loss": 0.4006,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.01100767869502306,
+      "rewards/margins": 1.2376883029937744,
+      "rewards/rejected": -1.2266806364059448,
+      "step": 30
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.444444444444444e-07,
+      "logits/chosen": -2.4814658164978027,
+      "logits/rejected": -2.4201178550720215,
+      "logps/chosen": -136.6073760986328,
+      "logps/rejected": -117.58763122558594,
+      "loss": 0.3056,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.353680282831192,
+      "rewards/margins": 1.6959145069122314,
+      "rewards/rejected": -2.0495948791503906,
+      "step": 40
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.937185929648241e-07,
+      "logits/chosen": -2.458440065383911,
+      "logits/rejected": -2.3304443359375,
+      "logps/chosen": -124.83992004394531,
+      "logps/rejected": -106.70866394042969,
+      "loss": 0.2679,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.31784066557884216,
+      "rewards/margins": 2.447019100189209,
+      "rewards/rejected": -2.764859914779663,
+      "step": 50
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.811557788944723e-07,
+      "logits/chosen": -2.396515369415283,
+      "logits/rejected": -2.293870687484741,
+      "logps/chosen": -135.10882568359375,
+      "logps/rejected": -122.99256896972656,
+      "loss": 0.2447,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10166273266077042,
+      "rewards/margins": 3.037165641784668,
+      "rewards/rejected": -3.1388285160064697,
+      "step": 60
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.685929648241206e-07,
+      "logits/chosen": -2.355457305908203,
+      "logits/rejected": -2.2316598892211914,
+      "logps/chosen": -122.27568054199219,
+      "logps/rejected": -113.7423095703125,
+      "loss": 0.2718,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5933733582496643,
+      "rewards/margins": 2.744062662124634,
+      "rewards/rejected": -3.3374359607696533,
+      "step": 70
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.5603015075376885e-07,
+      "logits/chosen": -2.361450672149658,
+      "logits/rejected": -2.286520481109619,
+      "logps/chosen": -127.52728271484375,
+      "logps/rejected": -126.6650161743164,
+      "loss": 0.2313,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.7555137872695923,
+      "rewards/margins": 3.271893262863159,
+      "rewards/rejected": -4.027407169342041,
+      "step": 80
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.4346733668341706e-07,
+      "logits/chosen": -2.3498244285583496,
+      "logits/rejected": -2.257256507873535,
+      "logps/chosen": -122.7756118774414,
+      "logps/rejected": -121.1026382446289,
+      "loss": 0.2307,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7029477953910828,
+      "rewards/margins": 3.352973222732544,
+      "rewards/rejected": -4.0559210777282715,
+      "step": 90
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.309045226130653e-07,
+      "logits/chosen": -2.4514331817626953,
+      "logits/rejected": -2.378006935119629,
+      "logps/chosen": -129.557861328125,
+      "logps/rejected": -123.76164245605469,
+      "loss": 0.2078,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.20816242694854736,
+      "rewards/margins": 4.0717453956604,
+      "rewards/rejected": -3.8635826110839844,
+      "step": 100
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.1834170854271357e-07,
+      "logits/chosen": -2.358485698699951,
+      "logits/rejected": -2.279703378677368,
+      "logps/chosen": -133.0589141845703,
+      "logps/rejected": -129.48451232910156,
+      "loss": 0.2371,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.7045987248420715,
+      "rewards/margins": 3.171238422393799,
+      "rewards/rejected": -3.8758368492126465,
+      "step": 110
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.057788944723618e-07,
+      "logits/chosen": -2.3714640140533447,
+      "logits/rejected": -2.3009307384490967,
+      "logps/chosen": -129.02052307128906,
+      "logps/rejected": -130.68905639648438,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.2331945896148682,
+      "rewards/margins": 3.5529701709747314,
+      "rewards/rejected": -4.7861647605896,
+      "step": 120
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9321608040201003e-07,
+      "logits/chosen": -2.385488510131836,
+      "logits/rejected": -2.304129123687744,
+      "logps/chosen": -128.46060180664062,
+      "logps/rejected": -135.4064483642578,
+      "loss": 0.1812,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6662181615829468,
+      "rewards/margins": 4.080785274505615,
+      "rewards/rejected": -4.74700403213501,
+      "step": 130
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.806532663316583e-07,
+      "logits/chosen": -2.331386089324951,
+      "logits/rejected": -2.2633156776428223,
+      "logps/chosen": -138.77073669433594,
+      "logps/rejected": -150.08639526367188,
+      "loss": 0.1728,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.8451410531997681,
+      "rewards/margins": 4.410196781158447,
+      "rewards/rejected": -5.255337715148926,
+      "step": 140
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.6809045226130655e-07,
+      "logits/chosen": -2.261727809906006,
+      "logits/rejected": -2.253206729888916,
+      "logps/chosen": -128.15460205078125,
+      "logps/rejected": -137.04916381835938,
+      "loss": 0.2448,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.2643935680389404,
+      "rewards/margins": 4.03481912612915,
+      "rewards/rejected": -5.299212455749512,
+      "step": 150
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5552763819095475e-07,
+      "logits/chosen": -2.3198587894439697,
+      "logits/rejected": -2.2583937644958496,
+      "logps/chosen": -140.9436798095703,
+      "logps/rejected": -147.12094116210938,
+      "loss": 0.1615,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.0014054775238037,
+      "rewards/margins": 4.486364364624023,
+      "rewards/rejected": -5.487771034240723,
+      "step": 160
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.42964824120603e-07,
+      "logits/chosen": -2.291534185409546,
+      "logits/rejected": -2.208200454711914,
+      "logps/chosen": -141.3668670654297,
+      "logps/rejected": -155.08602905273438,
+      "loss": 0.216,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.5920617580413818,
+      "rewards/margins": 4.606629371643066,
+      "rewards/rejected": -6.198691368103027,
+      "step": 170
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.3040201005025127e-07,
+      "logits/chosen": -2.387359142303467,
+      "logits/rejected": -2.28928279876709,
+      "logps/chosen": -130.8974609375,
+      "logps/rejected": -142.79708862304688,
+      "loss": 0.2023,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.409898519515991,
+      "rewards/margins": 3.318312168121338,
+      "rewards/rejected": -5.72821044921875,
+      "step": 180
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.178391959798995e-07,
+      "logits/chosen": -2.378145456314087,
+      "logits/rejected": -2.3165206909179688,
+      "logps/chosen": -131.51109313964844,
+      "logps/rejected": -142.97760009765625,
+      "loss": 0.1793,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.6101114749908447,
+      "rewards/margins": 3.964582920074463,
+      "rewards/rejected": -5.574694633483887,
+      "step": 190
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.0527638190954773e-07,
+      "logits/chosen": -2.330974817276001,
+      "logits/rejected": -2.276876449584961,
+      "logps/chosen": -137.87515258789062,
+      "logps/rejected": -152.93994140625,
+      "loss": 0.1638,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.3574166297912598,
+      "rewards/margins": 4.367352485656738,
+      "rewards/rejected": -5.724769115447998,
+      "step": 200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.92713567839196e-07,
+      "logits/chosen": -2.4077656269073486,
+      "logits/rejected": -2.3121707439422607,
+      "logps/chosen": -144.79739379882812,
+      "logps/rejected": -147.3674774169922,
+      "loss": 0.1603,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5326565504074097,
+      "rewards/margins": 3.852278470993042,
+      "rewards/rejected": -5.384934902191162,
+      "step": 210
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.8015075376884424e-07,
+      "logits/chosen": -2.2944283485412598,
+      "logits/rejected": -2.2064151763916016,
+      "logps/chosen": -130.70140075683594,
+      "logps/rejected": -147.8914794921875,
+      "loss": 0.1641,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.207008957862854,
+      "rewards/margins": 4.792324542999268,
+      "rewards/rejected": -5.999332427978516,
+      "step": 220
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6758793969849245e-07,
+      "logits/chosen": -2.348940849304199,
+      "logits/rejected": -2.164310932159424,
+      "logps/chosen": -128.39413452148438,
+      "logps/rejected": -140.557373046875,
+      "loss": 0.166,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.3905706405639648,
+      "rewards/margins": 4.686975479125977,
+      "rewards/rejected": -6.077546119689941,
+      "step": 230
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.550251256281407e-07,
+      "logits/chosen": -2.395198106765747,
+      "logits/rejected": -2.2796173095703125,
+      "logps/chosen": -149.087646484375,
+      "logps/rejected": -157.66110229492188,
+      "loss": 0.1402,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.3839337825775146,
+      "rewards/margins": 4.870881080627441,
+      "rewards/rejected": -6.254815578460693,
+      "step": 240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.424623115577889e-07,
+      "logits/chosen": -2.4120841026306152,
+      "logits/rejected": -2.2922260761260986,
+      "logps/chosen": -133.91209411621094,
+      "logps/rejected": -144.4380340576172,
+      "loss": 0.1872,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.2101858854293823,
+      "rewards/margins": 4.869531154632568,
+      "rewards/rejected": -6.079718112945557,
+      "step": 250
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2989949748743717e-07,
+      "logits/chosen": -2.331249237060547,
+      "logits/rejected": -2.2102646827697754,
+      "logps/chosen": -124.6241683959961,
+      "logps/rejected": -138.6643524169922,
+      "loss": 0.1476,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8653093576431274,
+      "rewards/margins": 4.82875394821167,
+      "rewards/rejected": -5.694063186645508,
+      "step": 260
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1733668341708543e-07,
+      "logits/chosen": -2.2793853282928467,
+      "logits/rejected": -2.1484787464141846,
+      "logps/chosen": -141.07778930664062,
+      "logps/rejected": -158.54676818847656,
+      "loss": 0.1925,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.8885013461112976,
+      "rewards/margins": 5.88812780380249,
+      "rewards/rejected": -6.776629447937012,
+      "step": 270
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0477386934673366e-07,
+      "logits/chosen": -2.3435816764831543,
+      "logits/rejected": -2.222494125366211,
+      "logps/chosen": -132.23934936523438,
+      "logps/rejected": -152.41224670410156,
+      "loss": 0.1719,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.8898094892501831,
+      "rewards/margins": 5.38167667388916,
+      "rewards/rejected": -6.271485805511475,
+      "step": 280
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.922110552763819e-07,
+      "logits/chosen": -2.337050437927246,
+      "logits/rejected": -2.2481775283813477,
+      "logps/chosen": -143.0239715576172,
+      "logps/rejected": -167.92628479003906,
+      "loss": 0.1408,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.425594449043274,
+      "rewards/margins": 5.652303695678711,
+      "rewards/rejected": -7.0778985023498535,
+      "step": 290
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7964824120603015e-07,
+      "logits/chosen": -2.3036749362945557,
+      "logits/rejected": -2.1755778789520264,
+      "logps/chosen": -143.76683044433594,
+      "logps/rejected": -168.64463806152344,
+      "loss": 0.1393,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.336169719696045,
+      "rewards/margins": 5.389135837554932,
+      "rewards/rejected": -6.725306034088135,
+      "step": 300
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.6708542713567838e-07,
+      "logits/chosen": -2.2387304306030273,
+      "logits/rejected": -2.1477370262145996,
+      "logps/chosen": -134.2172393798828,
+      "logps/rejected": -152.63461303710938,
+      "loss": 0.1579,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.4482789039611816,
+      "rewards/margins": 4.973227500915527,
+      "rewards/rejected": -6.421506404876709,
+      "step": 310
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5452261306532663e-07,
+      "logits/chosen": -2.2377147674560547,
+      "logits/rejected": -2.1322357654571533,
+      "logps/chosen": -135.234130859375,
+      "logps/rejected": -161.66934204101562,
+      "loss": 0.1088,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.7265955209732056,
+      "rewards/margins": 5.747531890869141,
+      "rewards/rejected": -7.474127769470215,
+      "step": 320
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4195979899497487e-07,
+      "logits/chosen": -2.236829996109009,
+      "logits/rejected": -2.154937982559204,
+      "logps/chosen": -141.66830444335938,
+      "logps/rejected": -166.32823181152344,
+      "loss": 0.1488,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.782355546951294,
+      "rewards/margins": 5.470345497131348,
+      "rewards/rejected": -7.2527008056640625,
+      "step": 330
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.2939698492462312e-07,
+      "logits/chosen": -2.3523616790771484,
+      "logits/rejected": -2.24237322807312,
+      "logps/chosen": -134.5939178466797,
+      "logps/rejected": -169.53317260742188,
+      "loss": 0.1579,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.731003999710083,
+      "rewards/margins": 6.211657524108887,
+      "rewards/rejected": -7.942662239074707,
+      "step": 340
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1683417085427135e-07,
+      "logits/chosen": -2.2935945987701416,
+      "logits/rejected": -2.2166569232940674,
+      "logps/chosen": -137.88473510742188,
+      "logps/rejected": -160.505126953125,
+      "loss": 0.1508,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.9919565916061401,
+      "rewards/margins": 5.5159502029418945,
+      "rewards/rejected": -6.507906913757324,
+      "step": 350
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0427135678391958e-07,
+      "logits/chosen": -2.320068120956421,
+      "logits/rejected": -2.2384986877441406,
+      "logps/chosen": -134.75381469726562,
+      "logps/rejected": -160.6112518310547,
+      "loss": 0.1696,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.4626049995422363,
+      "rewards/margins": 5.313817024230957,
+      "rewards/rejected": -7.77642297744751,
+      "step": 360
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 9.170854271356783e-08,
+      "logits/chosen": -2.2218480110168457,
+      "logits/rejected": -2.1257214546203613,
+      "logps/chosen": -147.00196838378906,
+      "logps/rejected": -171.19464111328125,
+      "loss": 0.1453,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.5424976348876953,
+      "rewards/margins": 5.8535027503967285,
+      "rewards/rejected": -7.395999908447266,
+      "step": 370
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.914572864321607e-08,
+      "logits/chosen": -2.27662992477417,
+      "logits/rejected": -2.157721519470215,
+      "logps/chosen": -131.88534545898438,
+      "logps/rejected": -158.86459350585938,
+      "loss": 0.1955,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.0121521949768066,
+      "rewards/margins": 4.82122802734375,
+      "rewards/rejected": -6.833380222320557,
+      "step": 380
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.658291457286432e-08,
+      "logits/chosen": -2.4072272777557373,
+      "logits/rejected": -2.285789966583252,
+      "logps/chosen": -140.86204528808594,
+      "logps/rejected": -169.72288513183594,
+      "loss": 0.1312,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.6351500749588013,
+      "rewards/margins": 6.939029693603516,
+      "rewards/rejected": -7.574179649353027,
+      "step": 390
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.4020100502512555e-08,
+      "logits/chosen": -2.2212846279144287,
+      "logits/rejected": -2.141895294189453,
+      "logps/chosen": -126.89100646972656,
+      "logps/rejected": -161.2249755859375,
+      "loss": 0.2109,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.795007050037384,
+      "rewards/margins": 6.2724409103393555,
+      "rewards/rejected": -7.067447662353516,
+      "step": 400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.14572864321608e-08,
+      "logits/chosen": -2.370591878890991,
+      "logits/rejected": -2.2174134254455566,
+      "logps/chosen": -146.6661834716797,
+      "logps/rejected": -165.058837890625,
+      "loss": 0.1407,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.8686304092407227,
+      "rewards/margins": 6.0188398361206055,
+      "rewards/rejected": -6.8874711990356445,
+      "step": 410
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.8894472361809044e-08,
+      "logits/chosen": -2.3981685638427734,
+      "logits/rejected": -2.2463793754577637,
+      "logps/chosen": -147.17486572265625,
+      "logps/rejected": -157.16622924804688,
+      "loss": 0.1245,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.544703483581543,
+      "rewards/margins": 5.296902179718018,
+      "rewards/rejected": -6.841605186462402,
+      "step": 420
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6331658291457288e-08,
+      "logits/chosen": -2.3454318046569824,
+      "logits/rejected": -2.2091023921966553,
+      "logps/chosen": -153.2548370361328,
+      "logps/rejected": -168.34854125976562,
+      "loss": 0.1576,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.147238850593567,
+      "rewards/margins": 5.976444721221924,
+      "rewards/rejected": -7.123683929443359,
+      "step": 430
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.768844221105527e-09,
+      "logits/chosen": -2.2823286056518555,
+      "logits/rejected": -2.1865620613098145,
+      "logps/chosen": -123.73600769042969,
+      "logps/rejected": -146.88194274902344,
+      "loss": 0.1626,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.5477970838546753,
+      "rewards/margins": 5.064129829406738,
+      "rewards/rejected": -6.611927032470703,
+      "step": 440
+    },
+    {
+      "epoch": 1.0,
+      "step": 443,
+      "total_flos": 0.0,
+      "train_loss": 0.2053286122078282,
+      "train_runtime": 6354.2323,
+      "train_samples_per_second": 2.231,
+      "train_steps_per_second": 0.07
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 443,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64dd99ea4e3db5e3c5f9c8f8d9dd2e5460a809b4afb9888ceee912c4d2152f63
+size 5371