Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,79 @@
+---
+library_name: transformers
+license: gemma
+base_model: google/gemma-7b
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: gemma-7b-borpo-low-quality
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-7b-borpo-low-quality
+
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5380
+- Rewards/chosen: -0.0547
+- Rewards/rejected: -0.0625
+- Rewards/accuracies: 0.5468
+- Rewards/margins: 0.0079
+- Logps/rejected: -1.2508
+- Logps/chosen: -1.0933
+- Logits/rejected: 267.2346
+- Logits/chosen: 296.6808
+- Nll Loss: 1.4703
+- Log Odds Ratio: -0.7039
+- Log Odds Chosen: 0.2721
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 1.436         | 0.9955 | 167  | 1.4639          | -0.0502        | -0.0571          | 0.5540             | 0.0068          | -1.1413        | -1.0048      | 294.2689        | 322.9157      | 1.4152   | -0.6882        | 0.2192          |
+| 1.0918        | 1.9970 | 335  | 1.4233          | -0.0501        | -0.0574          | 0.4964             | 0.0073          | -1.1475        | -1.0012      | 284.8744        | 313.3100      | 1.3661   | -0.7028        | 0.2209          |
+| 0.576         | 2.9866 | 501  | 1.5380          | -0.0547        | -0.0625          | 0.5468             | 0.0079          | -1.2508        | -1.0933      | 267.2346        | 296.6808      | 1.4703   | -0.7039        | 0.2721          |
+
+
+### Framework versions
+
+- Transformers 4.44.2
+- Pytorch 2.4.0+cu121
+- Datasets 3.0.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9865871833084947,
+    "total_flos": 0.0,
+    "train_loss": 1.4583633707431025,
+    "train_runtime": 13840.413,
+    "train_samples": 5364,
+    "train_samples_per_second": 1.163,
+    "train_steps_per_second": 0.036
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "google/gemma-7b",
+  "architectures": [
+    "GemmaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "head_dim": 256,
+  "hidden_act": "gelu",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 16,
+  "pad_token_id": 107,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": false,
+  "vocab_size": 256000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "pad_token_id": 107,
+  "transformers_version": "4.44.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:939232d17e3967c3caafed0eba01dd5ff3ffc46d415abd2cb256c1118aade1d4
+size 4995496656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4abb51cfb7e74d6ab27e2b66b730506921d56b53413ab1dabdb30d4acbdd963
+size 4982953168

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d2c95e59e11f7826294c7603e84850e1664e4c5074896a0e5dbcdf7e5c55366
+size 4982953200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a76fec30f621da3f8585db962d2d84ad4cc84496bd827e491478c191a5af87ea
+size 2113988336

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 17075361792
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Sep20_21-28-51_65ecb96dba42/events.out.tfevents.1726867828.65ecb96dba42.202062.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f8b8961d4d9997efa5b656c0170916f1782815f0c92d2e2b23808ee6d577efe
+size 5408

runs/Sep20_21-31-58_65ecb96dba42/events.out.tfevents.1726868017.65ecb96dba42.204459.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a175c7523f6f2458ff2dbe1cb0bf1b3398334a8500108018703bb67150a064fc
+size 5408

runs/Sep20_21-39-49_65ecb96dba42/events.out.tfevents.1726868472.65ecb96dba42.209667.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee308f27eccbb4f164bd1776aae8b50b7548601bb20f46734b749a2920abc97f
+size 5408

runs/Sep20_21-43-04_65ecb96dba42/events.out.tfevents.1726868682.65ecb96dba42.212134.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df2112b4a40d4e5523ea2183d882680333ee69e26ee27415d7da60d18437e393
+size 5408

runs/Sep20_22-20-48_65ecb96dba42/events.out.tfevents.1726870945.65ecb96dba42.233626.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3c7244b71d0d742156b9d76f941d5137d78a67bf61a0dab88448efac5232a22
+size 7098

runs/Sep20_22-31-39_65ecb96dba42/events.out.tfevents.1726871582.65ecb96dba42.240178.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58559ee73d657b893b9004715e2dfeb78f55f940bd13d025f82563d4c5c97b3b
+size 94114

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:322a5f52ab5cab196761ab397a022d6fa3a2e1418585e532bb6efb2fedd2ae94
+size 17477501

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": null,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9865871833084947,
+    "total_flos": 0.0,
+    "train_loss": 1.4583633707431025,
+    "train_runtime": 13840.413,
+    "train_samples": 5364,
+    "train_samples_per_second": 1.163,
+    "train_steps_per_second": 0.036
+}

trainer_state.json

@@ -0,0 +1,1899 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9865871833084947,
+  "eval_steps": 500,
+  "global_step": 501,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.029806259314456036,
+      "grad_norm": 1766.112548828125,
+      "learning_rate": 2.5000000000000004e-07,
+      "log_odds_chosen": -0.22085151076316833,
+      "log_odds_ratio": -1.0071768760681152,
+      "logits/chosen": 204.30807495117188,
+      "logits/rejected": 202.9915771484375,
+      "logps/chosen": -14.827531814575195,
+      "logps/rejected": -14.60667896270752,
+      "loss": 14.9615,
+      "nll_loss": 14.545796394348145,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.7413766384124756,
+      "rewards/margins": -0.011042552068829536,
+      "rewards/rejected": -0.7303339838981628,
+      "step": 5
+    },
+    {
+      "epoch": 0.05961251862891207,
+      "grad_norm": 1192.7847900390625,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.2549552619457245,
+      "log_odds_ratio": -0.769985020160675,
+      "logits/chosen": 219.4311981201172,
+      "logits/rejected": 223.46994018554688,
+      "logps/chosen": -12.237886428833008,
+      "logps/rejected": -12.492194175720215,
+      "loss": 12.6124,
+      "nll_loss": 12.336746215820312,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.6118943095207214,
+      "rewards/margins": 0.012715431861579418,
+      "rewards/rejected": -0.6246097683906555,
+      "step": 10
+    },
+    {
+      "epoch": 0.08941877794336811,
+      "grad_norm": 725.3335571289062,
+      "learning_rate": 7.5e-07,
+      "log_odds_chosen": 0.0513768307864666,
+      "log_odds_ratio": -0.7725777626037598,
+      "logits/chosen": 282.11224365234375,
+      "logits/rejected": 261.1120910644531,
+      "logps/chosen": -7.968921661376953,
+      "logps/rejected": -8.020242691040039,
+      "loss": 8.2811,
+      "nll_loss": 7.956255912780762,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.39844608306884766,
+      "rewards/margins": 0.0025660558603703976,
+      "rewards/rejected": -0.401012122631073,
+      "step": 15
+    },
+    {
+      "epoch": 0.11922503725782414,
+      "grad_norm": 216.32925415039062,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": -0.1569918394088745,
+      "log_odds_ratio": -0.9539459347724915,
+      "logits/chosen": 280.9331359863281,
+      "logits/rejected": 275.239013671875,
+      "logps/chosen": -5.379445552825928,
+      "logps/rejected": -5.223770618438721,
+      "loss": 5.4483,
+      "nll_loss": 5.457978248596191,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.268972247838974,
+      "rewards/margins": -0.007783788256347179,
+      "rewards/rejected": -0.26118847727775574,
+      "step": 20
+    },
+    {
+      "epoch": 0.14903129657228018,
+      "grad_norm": 152.3154296875,
+      "learning_rate": 1.25e-06,
+      "log_odds_chosen": -0.03973072022199631,
+      "log_odds_ratio": -0.8836725354194641,
+      "logits/chosen": 300.5466003417969,
+      "logits/rejected": 309.90667724609375,
+      "logps/chosen": -3.2718262672424316,
+      "logps/rejected": -3.2141456604003906,
+      "loss": 3.4993,
+      "nll_loss": 3.386289596557617,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.16359131038188934,
+      "rewards/margins": -0.0028840256854891777,
+      "rewards/rejected": -0.1607072949409485,
+      "step": 25
+    },
+    {
+      "epoch": 0.17883755588673622,
+      "grad_norm": 89.40418243408203,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": -0.05680186673998833,
+      "log_odds_ratio": -0.8815790414810181,
+      "logits/chosen": 350.63232421875,
+      "logits/rejected": 379.04736328125,
+      "logps/chosen": -2.606602668762207,
+      "logps/rejected": -2.523834466934204,
+      "loss": 2.5544,
+      "nll_loss": 2.63269305229187,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1303301304578781,
+      "rewards/margins": -0.0041384161449968815,
+      "rewards/rejected": -0.12619172036647797,
+      "step": 30
+    },
+    {
+      "epoch": 0.20864381520119224,
+      "grad_norm": 49.261566162109375,
+      "learning_rate": 1.75e-06,
+      "log_odds_chosen": 0.16310584545135498,
+      "log_odds_ratio": -0.7383562922477722,
+      "logits/chosen": 385.07769775390625,
+      "logits/rejected": 373.39593505859375,
+      "logps/chosen": -1.8015142679214478,
+      "logps/rejected": -1.905853033065796,
+      "loss": 2.1302,
+      "nll_loss": 2.0047783851623535,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.09007571637630463,
+      "rewards/margins": 0.005216942168772221,
+      "rewards/rejected": -0.09529266506433487,
+      "step": 35
+    },
+    {
+      "epoch": 0.23845007451564829,
+      "grad_norm": 35.88943099975586,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.03436558321118355,
+      "log_odds_ratio": -0.7623429298400879,
+      "logits/chosen": 378.38153076171875,
+      "logits/rejected": 376.9698791503906,
+      "logps/chosen": -1.6465238332748413,
+      "logps/rejected": -1.670940637588501,
+      "loss": 1.9446,
+      "nll_loss": 2.038804769515991,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.08232619613409042,
+      "rewards/margins": 0.0012208283878862858,
+      "rewards/rejected": -0.08354702591896057,
+      "step": 40
+    },
+    {
+      "epoch": 0.26825633383010433,
+      "grad_norm": 37.345130920410156,
+      "learning_rate": 2.25e-06,
+      "log_odds_chosen": 0.058513958007097244,
+      "log_odds_ratio": -0.7263653874397278,
+      "logits/chosen": 390.2134704589844,
+      "logits/rejected": 399.8643493652344,
+      "logps/chosen": -1.5079405307769775,
+      "logps/rejected": -1.5511177778244019,
+      "loss": 1.8369,
+      "nll_loss": 1.7173048257827759,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.07539702951908112,
+      "rewards/margins": 0.002158870454877615,
+      "rewards/rejected": -0.07755588740110397,
+      "step": 45
+    },
+    {
+      "epoch": 0.29806259314456035,
+      "grad_norm": 50.422019958496094,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.2006622552871704,
+      "log_odds_ratio": -0.6733421087265015,
+      "logits/chosen": 399.89837646484375,
+      "logits/rejected": 420.90509033203125,
+      "logps/chosen": -1.418306827545166,
+      "logps/rejected": -1.5650049448013306,
+      "loss": 1.8378,
+      "nll_loss": 1.8607208728790283,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0709153264760971,
+      "rewards/margins": 0.007334905676543713,
+      "rewards/rejected": -0.07825024425983429,
+      "step": 50
+    },
+    {
+      "epoch": 0.32786885245901637,
+      "grad_norm": 47.060543060302734,
+      "learning_rate": 2.7500000000000004e-06,
+      "log_odds_chosen": 0.04189341515302658,
+      "log_odds_ratio": -0.7426876425743103,
+      "logits/chosen": 387.83795166015625,
+      "logits/rejected": 379.873291015625,
+      "logps/chosen": -1.4415570497512817,
+      "logps/rejected": -1.463037133216858,
+      "loss": 1.8704,
+      "nll_loss": 1.915564775466919,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.07207784801721573,
+      "rewards/margins": 0.0010740077123045921,
+      "rewards/rejected": -0.07315186411142349,
+      "step": 55
+    },
+    {
+      "epoch": 0.35767511177347244,
+      "grad_norm": 34.15119934082031,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.14166930317878723,
+      "log_odds_ratio": -0.6689570546150208,
+      "logits/chosen": 395.5494079589844,
+      "logits/rejected": 387.44561767578125,
+      "logps/chosen": -1.3572882413864136,
+      "logps/rejected": -1.4584226608276367,
+      "loss": 1.705,
+      "nll_loss": 1.769514799118042,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06786441057920456,
+      "rewards/margins": 0.005056709982454777,
+      "rewards/rejected": -0.07292111963033676,
+      "step": 60
+    },
+    {
+      "epoch": 0.38748137108792846,
+      "grad_norm": 59.435298919677734,
+      "learning_rate": 3.2500000000000002e-06,
+      "log_odds_chosen": 0.09232059866189957,
+      "log_odds_ratio": -0.7047046422958374,
+      "logits/chosen": 390.05853271484375,
+      "logits/rejected": 390.36346435546875,
+      "logps/chosen": -1.2730042934417725,
+      "logps/rejected": -1.3218134641647339,
+      "loss": 1.6272,
+      "nll_loss": 1.5361205339431763,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0636502131819725,
+      "rewards/margins": 0.0024404595606029034,
+      "rewards/rejected": -0.06609068065881729,
+      "step": 65
+    },
+    {
+      "epoch": 0.4172876304023845,
+      "grad_norm": 32.72905349731445,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.03600925952196121,
+      "log_odds_ratio": -0.7415024042129517,
+      "logits/chosen": 381.87835693359375,
+      "logits/rejected": 389.978271484375,
+      "logps/chosen": -1.496351718902588,
+      "logps/rejected": -1.5099536180496216,
+      "loss": 1.7008,
+      "nll_loss": 1.8172271251678467,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07481758296489716,
+      "rewards/margins": 0.0006800902774557471,
+      "rewards/rejected": -0.07549767941236496,
+      "step": 70
+    },
+    {
+      "epoch": 0.44709388971684055,
+      "grad_norm": 31.950422286987305,
+      "learning_rate": 3.7500000000000005e-06,
+      "log_odds_chosen": 0.12743942439556122,
+      "log_odds_ratio": -0.6888688206672668,
+      "logits/chosen": 396.3226318359375,
+      "logits/rejected": 383.38067626953125,
+      "logps/chosen": -1.2637438774108887,
+      "logps/rejected": -1.34519362449646,
+      "loss": 1.6454,
+      "nll_loss": 1.5880029201507568,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06318718940019608,
+      "rewards/margins": 0.0040724934078752995,
+      "rewards/rejected": -0.06725968420505524,
+      "step": 75
+    },
+    {
+      "epoch": 0.47690014903129657,
+      "grad_norm": 29.037189483642578,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.18037892878055573,
+      "log_odds_ratio": -0.6654509902000427,
+      "logits/chosen": 381.05133056640625,
+      "logits/rejected": 403.25115966796875,
+      "logps/chosen": -1.220245599746704,
+      "logps/rejected": -1.3405919075012207,
+      "loss": 1.6425,
+      "nll_loss": 1.6346063613891602,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.061012279242277145,
+      "rewards/margins": 0.006017312873154879,
+      "rewards/rejected": -0.06702959537506104,
+      "step": 80
+    },
+    {
+      "epoch": 0.5067064083457526,
+      "grad_norm": 48.85629653930664,
+      "learning_rate": 4.25e-06,
+      "log_odds_chosen": 0.13455167412757874,
+      "log_odds_ratio": -0.7552928924560547,
+      "logits/chosen": 406.13067626953125,
+      "logits/rejected": 390.22216796875,
+      "logps/chosen": -1.419901967048645,
+      "logps/rejected": -1.4787737131118774,
+      "loss": 1.6961,
+      "nll_loss": 1.7476110458374023,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07099510729312897,
+      "rewards/margins": 0.002943576779216528,
+      "rewards/rejected": -0.07393868267536163,
+      "step": 85
+    },
+    {
+      "epoch": 0.5365126676602087,
+      "grad_norm": 383.9244079589844,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.579018771648407,
+      "log_odds_ratio": -0.5751021504402161,
+      "logits/chosen": 401.8145446777344,
+      "logits/rejected": 417.1114807128906,
+      "logps/chosen": -1.2194725275039673,
+      "logps/rejected": -1.6738338470458984,
+      "loss": 1.4692,
+      "nll_loss": 1.5031993389129639,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0609736330807209,
+      "rewards/margins": 0.02271805889904499,
+      "rewards/rejected": -0.08369167894124985,
+      "step": 90
+    },
+    {
+      "epoch": 0.5663189269746647,
+      "grad_norm": 83.88288116455078,
+      "learning_rate": 4.75e-06,
+      "log_odds_chosen": 0.20128187537193298,
+      "log_odds_ratio": -0.6592803597450256,
+      "logits/chosen": 368.6065368652344,
+      "logits/rejected": 382.3212890625,
+      "logps/chosen": -1.1970250606536865,
+      "logps/rejected": -1.2992660999298096,
+      "loss": 1.6441,
+      "nll_loss": 1.5431461334228516,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.059851258993148804,
+      "rewards/margins": 0.00511204544454813,
+      "rewards/rejected": -0.06496329605579376,
+      "step": 95
+    },
+    {
+      "epoch": 0.5961251862891207,
+      "grad_norm": 54.11213684082031,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 0.13380606472492218,
+      "log_odds_ratio": -0.6779340505599976,
+      "logits/chosen": 404.63409423828125,
+      "logits/rejected": 441.92938232421875,
+      "logps/chosen": -1.2189857959747314,
+      "logps/rejected": -1.3092880249023438,
+      "loss": 1.596,
+      "nll_loss": 1.5926085710525513,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06094928830862045,
+      "rewards/margins": 0.004515114240348339,
+      "rewards/rejected": -0.06546439975500107,
+      "step": 100
+    },
+    {
+      "epoch": 0.6259314456035767,
+      "grad_norm": 24.880090713500977,
+      "learning_rate": 4.8795003647426654e-06,
+      "log_odds_chosen": 0.2874816656112671,
+      "log_odds_ratio": -0.6406186819076538,
+      "logits/chosen": 386.64764404296875,
+      "logits/rejected": 400.5997009277344,
+      "logps/chosen": -1.0471909046173096,
+      "logps/rejected": -1.2103850841522217,
+      "loss": 1.5244,
+      "nll_loss": 1.5330688953399658,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.05235954001545906,
+      "rewards/margins": 0.008159706369042397,
+      "rewards/rejected": -0.060519248247146606,
+      "step": 105
+    },
+    {
+      "epoch": 0.6557377049180327,
+      "grad_norm": 65.00325775146484,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 0.17585398256778717,
+      "log_odds_ratio": -0.6596522331237793,
+      "logits/chosen": 370.3504333496094,
+      "logits/rejected": 369.62982177734375,
+      "logps/chosen": -1.0811831951141357,
+      "logps/rejected": -1.2061924934387207,
+      "loss": 1.5148,
+      "nll_loss": 1.530368685722351,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.054059166461229324,
+      "rewards/margins": 0.006250458303838968,
+      "rewards/rejected": -0.06030962988734245,
+      "step": 110
+    },
+    {
+      "epoch": 0.6855439642324889,
+      "grad_norm": 44.05381393432617,
+      "learning_rate": 4.662524041201569e-06,
+      "log_odds_chosen": 0.3691500723361969,
+      "log_odds_ratio": -0.5788921117782593,
+      "logits/chosen": 388.53631591796875,
+      "logits/rejected": 396.33770751953125,
+      "logps/chosen": -0.9749990701675415,
+      "logps/rejected": -1.168736219406128,
+      "loss": 1.5042,
+      "nll_loss": 1.417229413986206,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.048749953508377075,
+      "rewards/margins": 0.009686857461929321,
+      "rewards/rejected": -0.05843681842088699,
+      "step": 115
+    },
+    {
+      "epoch": 0.7153502235469449,
+      "grad_norm": 38.68400955200195,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 0.3533535599708557,
+      "log_odds_ratio": -0.6087926626205444,
+      "logits/chosen": 379.69366455078125,
+      "logits/rejected": 379.962158203125,
+      "logps/chosen": -1.0634129047393799,
+      "logps/rejected": -1.2383795976638794,
+      "loss": 1.5529,
+      "nll_loss": 1.5442397594451904,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.053170643746852875,
+      "rewards/margins": 0.008748333901166916,
+      "rewards/rejected": -0.06191897392272949,
+      "step": 120
+    },
+    {
+      "epoch": 0.7451564828614009,
+      "grad_norm": 21.89837646484375,
+      "learning_rate": 4.47213595499958e-06,
+      "log_odds_chosen": 0.17026309669017792,
+      "log_odds_ratio": -0.6800710558891296,
+      "logits/chosen": 375.1714782714844,
+      "logits/rejected": 387.94122314453125,
+      "logps/chosen": -1.0947885513305664,
+      "logps/rejected": -1.1978260278701782,
+      "loss": 1.4603,
+      "nll_loss": 1.4590686559677124,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05473943427205086,
+      "rewards/margins": 0.005151876248419285,
+      "rewards/rejected": -0.059891313314437866,
+      "step": 125
+    },
+    {
+      "epoch": 0.7749627421758569,
+      "grad_norm": 22.374597549438477,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 0.1504744589328766,
+      "log_odds_ratio": -0.6961590051651001,
+      "logits/chosen": 401.68170166015625,
+      "logits/rejected": 390.8814392089844,
+      "logps/chosen": -1.1183803081512451,
+      "logps/rejected": -1.1522414684295654,
+      "loss": 1.5124,
+      "nll_loss": 1.4771168231964111,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.055919013917446136,
+      "rewards/margins": 0.0016930631827563047,
+      "rewards/rejected": -0.05761207267642021,
+      "step": 130
+    },
+    {
+      "epoch": 0.8047690014903129,
+      "grad_norm": 54.11912155151367,
+      "learning_rate": 4.303314829119352e-06,
+      "log_odds_chosen": 0.09426576644182205,
+      "log_odds_ratio": -0.7113849520683289,
+      "logits/chosen": 408.5809326171875,
+      "logits/rejected": 412.0887756347656,
+      "logps/chosen": -1.1269690990447998,
+      "logps/rejected": -1.195809245109558,
+      "loss": 1.519,
+      "nll_loss": 1.582168698310852,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.056348465383052826,
+      "rewards/margins": 0.0034419982694089413,
+      "rewards/rejected": -0.059790462255477905,
+      "step": 135
+    },
+    {
+      "epoch": 0.834575260804769,
+      "grad_norm": 23.130447387695312,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": -0.09378698468208313,
+      "log_odds_ratio": -0.8063527345657349,
+      "logits/chosen": 395.3038635253906,
+      "logits/rejected": 402.5538330078125,
+      "logps/chosen": -1.1263329982757568,
+      "logps/rejected": -1.0577051639556885,
+      "loss": 1.5115,
+      "nll_loss": 1.6241823434829712,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.056316643953323364,
+      "rewards/margins": -0.0034313846845179796,
+      "rewards/rejected": -0.052885256707668304,
+      "step": 140
+    },
+    {
+      "epoch": 0.8643815201192251,
+      "grad_norm": 46.85090255737305,
+      "learning_rate": 4.1522739926869985e-06,
+      "log_odds_chosen": -0.024486612528562546,
+      "log_odds_ratio": -0.735202431678772,
+      "logits/chosen": 391.74871826171875,
+      "logits/rejected": 396.13165283203125,
+      "logps/chosen": -1.1859979629516602,
+      "logps/rejected": -1.1640434265136719,
+      "loss": 1.5112,
+      "nll_loss": 1.5025219917297363,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.05929989740252495,
+      "rewards/margins": -0.0010977205820381641,
+      "rewards/rejected": -0.05820217728614807,
+      "step": 145
+    },
+    {
+      "epoch": 0.8941877794336811,
+      "grad_norm": 32.988243103027344,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 0.3807345926761627,
+      "log_odds_ratio": -0.559662401676178,
+      "logits/chosen": 399.6306457519531,
+      "logits/rejected": 417.28515625,
+      "logps/chosen": -1.0708329677581787,
+      "logps/rejected": -1.319719672203064,
+      "loss": 1.4701,
+      "nll_loss": 1.3841121196746826,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.053541649132966995,
+      "rewards/margins": 0.012444335967302322,
+      "rewards/rejected": -0.06598598510026932,
+      "step": 150
+    },
+    {
+      "epoch": 0.9239940387481371,
+      "grad_norm": 19.93175506591797,
+      "learning_rate": 4.016096644512495e-06,
+      "log_odds_chosen": 0.13164076209068298,
+      "log_odds_ratio": -0.6793235540390015,
+      "logits/chosen": 381.662353515625,
+      "logits/rejected": 396.54168701171875,
+      "logps/chosen": -1.1232904195785522,
+      "logps/rejected": -1.203460931777954,
+      "loss": 1.4306,
+      "nll_loss": 1.322166919708252,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05616452172398567,
+      "rewards/margins": 0.004008529242128134,
+      "rewards/rejected": -0.06017305329442024,
+      "step": 155
+    },
+    {
+      "epoch": 0.9538002980625931,
+      "grad_norm": 48.30046844482422,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 0.010590496473014355,
+      "log_odds_ratio": -0.794399619102478,
+      "logits/chosen": 387.1076354980469,
+      "logits/rejected": 432.69384765625,
+      "logps/chosen": -1.0055660009384155,
+      "logps/rejected": -1.0430043935775757,
+      "loss": 1.3903,
+      "nll_loss": 1.3440879583358765,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.050278306007385254,
+      "rewards/margins": 0.0018719173967838287,
+      "rewards/rejected": -0.052150219678878784,
+      "step": 160
+    },
+    {
+      "epoch": 0.9836065573770492,
+      "grad_norm": 17.599102020263672,
+      "learning_rate": 3.892494720807615e-06,
+      "log_odds_chosen": 0.021682387217879295,
+      "log_odds_ratio": -0.7344300746917725,
+      "logits/chosen": 397.306884765625,
+      "logits/rejected": 409.34222412109375,
+      "logps/chosen": -1.1095333099365234,
+      "logps/rejected": -1.1364104747772217,
+      "loss": 1.436,
+      "nll_loss": 1.4027729034423828,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.05547667294740677,
+      "rewards/margins": 0.0013438586611300707,
+      "rewards/rejected": -0.05682053044438362,
+      "step": 165
+    },
+    {
+      "epoch": 0.9955290611028316,
+      "eval_log_odds_chosen": 0.21922710537910461,
+      "eval_log_odds_ratio": -0.6881988644599915,
+      "eval_logits/chosen": 322.9157409667969,
+      "eval_logits/rejected": 294.26885986328125,
+      "eval_logps/chosen": -1.0047651529312134,
+      "eval_logps/rejected": -1.141276478767395,
+      "eval_loss": 1.4639270305633545,
+      "eval_nll_loss": 1.4151870012283325,
+      "eval_rewards/accuracies": 0.5539568066596985,
+      "eval_rewards/chosen": -0.05023825913667679,
+      "eval_rewards/margins": 0.006825567223131657,
+      "eval_rewards/rejected": -0.05706382542848587,
+      "eval_runtime": 112.2165,
+      "eval_samples_per_second": 4.928,
+      "eval_steps_per_second": 1.239,
+      "step": 167
+    },
+    {
+      "epoch": 1.0134128166915053,
+      "grad_norm": 19.598791122436523,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 0.45991507172584534,
+      "log_odds_ratio": -0.5599726438522339,
+      "logits/chosen": 381.36883544921875,
+      "logits/rejected": 404.8132629394531,
+      "logps/chosen": -0.8878811001777649,
+      "logps/rejected": -1.1503413915634155,
+      "loss": 1.2939,
+      "nll_loss": 1.1531199216842651,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.044394053518772125,
+      "rewards/margins": 0.013123014941811562,
+      "rewards/rejected": -0.057517070323228836,
+      "step": 170
+    },
+    {
+      "epoch": 1.0432190760059612,
+      "grad_norm": 20.02121353149414,
+      "learning_rate": 3.7796447300922724e-06,
+      "log_odds_chosen": 0.8954402804374695,
+      "log_odds_ratio": -0.4191034436225891,
+      "logits/chosen": 360.7755126953125,
+      "logits/rejected": 400.88580322265625,
+      "logps/chosen": -0.6612526774406433,
+      "logps/rejected": -1.1563608646392822,
+      "loss": 1.0774,
+      "nll_loss": 1.1424416303634644,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.033062636852264404,
+      "rewards/margins": 0.024755412712693214,
+      "rewards/rejected": -0.05781804770231247,
+      "step": 175
+    },
+    {
+      "epoch": 1.0730253353204173,
+      "grad_norm": 31.31179428100586,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 0.9305903315544128,
+      "log_odds_ratio": -0.4297749102115631,
+      "logits/chosen": 358.54632568359375,
+      "logits/rejected": 333.3828125,
+      "logps/chosen": -0.7619189023971558,
+      "logps/rejected": -1.288059949874878,
+      "loss": 1.1343,
+      "nll_loss": 1.1943809986114502,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03809594735503197,
+      "rewards/margins": 0.02630705200135708,
+      "rewards/rejected": -0.0644029974937439,
+      "step": 180
+    },
+    {
+      "epoch": 1.1028315946348732,
+      "grad_norm": 18.475496292114258,
+      "learning_rate": 3.6760731104690393e-06,
+      "log_odds_chosen": 0.9789319038391113,
+      "log_odds_ratio": -0.39594078063964844,
+      "logits/chosen": 388.92523193359375,
+      "logits/rejected": 379.08624267578125,
+      "logps/chosen": -0.6655952334403992,
+      "logps/rejected": -1.1871401071548462,
+      "loss": 1.0198,
+      "nll_loss": 0.9870488047599792,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.03327976167201996,
+      "rewards/margins": 0.026077240705490112,
+      "rewards/rejected": -0.05935700982809067,
+      "step": 185
+    },
+    {
+      "epoch": 1.1326378539493294,
+      "grad_norm": 15.556231498718262,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 0.5961565971374512,
+      "log_odds_ratio": -0.5247331857681274,
+      "logits/chosen": 364.07965087890625,
+      "logits/rejected": 406.9730529785156,
+      "logps/chosen": -0.7741492986679077,
+      "logps/rejected": -1.1193947792053223,
+      "loss": 1.0997,
+      "nll_loss": 1.0372028350830078,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03870746120810509,
+      "rewards/margins": 0.017262274399399757,
+      "rewards/rejected": -0.055969737470149994,
+      "step": 190
+    },
+    {
+      "epoch": 1.1624441132637853,
+      "grad_norm": 20.581634521484375,
+      "learning_rate": 3.5805743701971648e-06,
+      "log_odds_chosen": 0.9057777523994446,
+      "log_odds_ratio": -0.40288910269737244,
+      "logits/chosen": 387.034912109375,
+      "logits/rejected": 400.3788146972656,
+      "logps/chosen": -0.8112454414367676,
+      "logps/rejected": -1.2839386463165283,
+      "loss": 1.143,
+      "nll_loss": 1.1492406129837036,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.04056227579712868,
+      "rewards/margins": 0.023634660989046097,
+      "rewards/rejected": -0.06419692933559418,
+      "step": 195
+    },
+    {
+      "epoch": 1.1922503725782414,
+      "grad_norm": 25.256078720092773,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 0.968602180480957,
+      "log_odds_ratio": -0.41492360830307007,
+      "logits/chosen": 408.2672424316406,
+      "logits/rejected": 392.72406005859375,
+      "logps/chosen": -0.7001940011978149,
+      "logps/rejected": -1.2566159963607788,
+      "loss": 1.1031,
+      "nll_loss": 0.9941505193710327,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03500969707965851,
+      "rewards/margins": 0.02782110869884491,
+      "rewards/rejected": -0.06283080577850342,
+      "step": 200
+    },
+    {
+      "epoch": 1.2220566318926975,
+      "grad_norm": 20.430418014526367,
+      "learning_rate": 3.4921514788478916e-06,
+      "log_odds_chosen": 0.9490750432014465,
+      "log_odds_ratio": -0.43051719665527344,
+      "logits/chosen": 366.13177490234375,
+      "logits/rejected": 359.82196044921875,
+      "logps/chosen": -0.7287150025367737,
+      "logps/rejected": -1.228252649307251,
+      "loss": 1.1093,
+      "nll_loss": 1.1782151460647583,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.036435749381780624,
+      "rewards/margins": 0.024976884946227074,
+      "rewards/rejected": -0.061412639915943146,
+      "step": 205
+    },
+    {
+      "epoch": 1.2518628912071534,
+      "grad_norm": 29.456756591796875,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 1.1124473810195923,
+      "log_odds_ratio": -0.36473917961120605,
+      "logits/chosen": 371.9077453613281,
+      "logits/rejected": 400.37908935546875,
+      "logps/chosen": -0.6593618392944336,
+      "logps/rejected": -1.292751431465149,
+      "loss": 1.0725,
+      "nll_loss": 1.041669487953186,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.03296809643507004,
+      "rewards/margins": 0.03166947886347771,
+      "rewards/rejected": -0.06463757157325745,
+      "step": 210
+    },
+    {
+      "epoch": 1.2816691505216096,
+      "grad_norm": 51.35958480834961,
+      "learning_rate": 3.409971697352368e-06,
+      "log_odds_chosen": 1.1422733068466187,
+      "log_odds_ratio": -0.3576507568359375,
+      "logits/chosen": 396.532470703125,
+      "logits/rejected": 379.38525390625,
+      "logps/chosen": -0.7450262904167175,
+      "logps/rejected": -1.4131982326507568,
+      "loss": 1.0757,
+      "nll_loss": 1.0690205097198486,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.0372513122856617,
+      "rewards/margins": 0.033408597111701965,
+      "rewards/rejected": -0.07065991312265396,
+      "step": 215
+    },
+    {
+      "epoch": 1.3114754098360657,
+      "grad_norm": 16.197195053100586,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 0.7627310752868652,
+      "log_odds_ratio": -0.46217769384384155,
+      "logits/chosen": 383.55572509765625,
+      "logits/rejected": 377.1234436035156,
+      "logps/chosen": -0.7894420623779297,
+      "logps/rejected": -1.196644902229309,
+      "loss": 1.0384,
+      "nll_loss": 1.0212104320526123,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.03947211056947708,
+      "rewards/margins": 0.02036014385521412,
+      "rewards/rejected": -0.05983225256204605,
+      "step": 220
+    },
+    {
+      "epoch": 1.3412816691505216,
+      "grad_norm": 26.38709831237793,
+      "learning_rate": 3.3333333333333333e-06,
+      "log_odds_chosen": 0.5317455530166626,
+      "log_odds_ratio": -0.5077921152114868,
+      "logits/chosen": 383.05657958984375,
+      "logits/rejected": 375.5099792480469,
+      "logps/chosen": -0.8590075373649597,
+      "logps/rejected": -1.1633431911468506,
+      "loss": 1.0481,
+      "nll_loss": 1.092240333557129,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.04295038431882858,
+      "rewards/margins": 0.01521677989512682,
+      "rewards/rejected": -0.05816715955734253,
+      "step": 225
+    },
+    {
+      "epoch": 1.3710879284649775,
+      "grad_norm": 22.215499877929688,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 1.152860403060913,
+      "log_odds_ratio": -0.33905166387557983,
+      "logits/chosen": 360.29229736328125,
+      "logits/rejected": 377.34967041015625,
+      "logps/chosen": -0.6110543012619019,
+      "logps/rejected": -1.235459804534912,
+      "loss": 1.0305,
+      "nll_loss": 0.8882306814193726,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.03055271878838539,
+      "rewards/margins": 0.031220275908708572,
+      "rewards/rejected": -0.061772990971803665,
+      "step": 230
+    },
+    {
+      "epoch": 1.4008941877794336,
+      "grad_norm": 15.053303718566895,
+      "learning_rate": 3.2616403652672114e-06,
+      "log_odds_chosen": 1.173457384109497,
+      "log_odds_ratio": -0.39741092920303345,
+      "logits/chosen": 382.1279602050781,
+      "logits/rejected": 394.14453125,
+      "logps/chosen": -0.6574608087539673,
+      "logps/rejected": -1.3275513648986816,
+      "loss": 1.041,
+      "nll_loss": 0.9579538106918335,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.032873041927814484,
+      "rewards/margins": 0.033504534512758255,
+      "rewards/rejected": -0.06637756526470184,
+      "step": 235
+    },
+    {
+      "epoch": 1.4307004470938898,
+      "grad_norm": 17.994874954223633,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 0.7307401895523071,
+      "log_odds_ratio": -0.4488092362880707,
+      "logits/chosen": 405.40802001953125,
+      "logits/rejected": 409.69647216796875,
+      "logps/chosen": -0.7471104264259338,
+      "logps/rejected": -1.1478822231292725,
+      "loss": 1.062,
+      "nll_loss": 1.034090518951416,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03735552355647087,
+      "rewards/margins": 0.02003859169781208,
+      "rewards/rejected": -0.057394109666347504,
+      "step": 240
+    },
+    {
+      "epoch": 1.4605067064083457,
+      "grad_norm": 14.603346824645996,
+      "learning_rate": 3.1943828249997e-06,
+      "log_odds_chosen": 0.9838510751724243,
+      "log_odds_ratio": -0.39035505056381226,
+      "logits/chosen": 394.9382629394531,
+      "logits/rejected": 383.0425109863281,
+      "logps/chosen": -0.604079008102417,
+      "logps/rejected": -1.1444944143295288,
+      "loss": 1.0828,
+      "nll_loss": 1.0692133903503418,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.03020395338535309,
+      "rewards/margins": 0.027020767331123352,
+      "rewards/rejected": -0.05722472816705704,
+      "step": 245
+    },
+    {
+      "epoch": 1.4903129657228018,
+      "grad_norm": 24.562320709228516,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 1.0623723268508911,
+      "log_odds_ratio": -0.41066059470176697,
+      "logits/chosen": 374.6037902832031,
+      "logits/rejected": 381.32269287109375,
+      "logps/chosen": -0.6770502924919128,
+      "logps/rejected": -1.2543606758117676,
+      "loss": 1.0094,
+      "nll_loss": 0.9133090972900391,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.03385251387953758,
+      "rewards/margins": 0.028865519911050797,
+      "rewards/rejected": -0.06271803379058838,
+      "step": 250
+    },
+    {
+      "epoch": 1.520119225037258,
+      "grad_norm": 16.239404678344727,
+      "learning_rate": 3.131121455425748e-06,
+      "log_odds_chosen": 1.1072403192520142,
+      "log_odds_ratio": -0.34550726413726807,
+      "logits/chosen": 394.74688720703125,
+      "logits/rejected": 396.5551452636719,
+      "logps/chosen": -0.6057699918746948,
+      "logps/rejected": -1.1659581661224365,
+      "loss": 1.0541,
+      "nll_loss": 0.9463102221488953,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.03028850257396698,
+      "rewards/margins": 0.028009409084916115,
+      "rewards/rejected": -0.058297913521528244,
+      "step": 255
+    },
+    {
+      "epoch": 1.5499254843517138,
+      "grad_norm": 29.02210235595703,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 0.9046269655227661,
+      "log_odds_ratio": -0.42992085218429565,
+      "logits/chosen": 375.5458984375,
+      "logits/rejected": 415.7400817871094,
+      "logps/chosen": -0.7795925140380859,
+      "logps/rejected": -1.3336765766143799,
+      "loss": 1.021,
+      "nll_loss": 1.037724256515503,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.038979630917310715,
+      "rewards/margins": 0.02770419791340828,
+      "rewards/rejected": -0.066683828830719,
+      "step": 260
+    },
+    {
+      "epoch": 1.5797317436661698,
+      "grad_norm": 16.736574172973633,
+      "learning_rate": 3.0714755841697565e-06,
+      "log_odds_chosen": 1.028351902961731,
+      "log_odds_ratio": -0.4398733079433441,
+      "logits/chosen": 384.7108154296875,
+      "logits/rejected": 406.8509521484375,
+      "logps/chosen": -0.717097282409668,
+      "logps/rejected": -1.318499207496643,
+      "loss": 1.104,
+      "nll_loss": 1.0128785371780396,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03585486486554146,
+      "rewards/margins": 0.030070099979639053,
+      "rewards/rejected": -0.06592496484518051,
+      "step": 265
+    },
+    {
+      "epoch": 1.6095380029806259,
+      "grad_norm": 17.67503547668457,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 0.7785763740539551,
+      "log_odds_ratio": -0.4546143412590027,
+      "logits/chosen": 369.1441955566406,
+      "logits/rejected": 379.18487548828125,
+      "logps/chosen": -0.7971286177635193,
+      "logps/rejected": -1.235899567604065,
+      "loss": 1.0973,
+      "nll_loss": 1.1573312282562256,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.0398564375936985,
+      "rewards/margins": 0.021938541904091835,
+      "rewards/rejected": -0.06179497763514519,
+      "step": 270
+    },
+    {
+      "epoch": 1.639344262295082,
+      "grad_norm": 15.161911964416504,
+      "learning_rate": 3.0151134457776365e-06,
+      "log_odds_chosen": 0.9045177698135376,
+      "log_odds_ratio": -0.4321528375148773,
+      "logits/chosen": 360.2083435058594,
+      "logits/rejected": 351.1839904785156,
+      "logps/chosen": -0.6309309005737305,
+      "logps/rejected": -1.0601168870925903,
+      "loss": 1.0614,
+      "nll_loss": 0.976634681224823,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.031546544283628464,
+      "rewards/margins": 0.021459298208355904,
+      "rewards/rejected": -0.05300584435462952,
+      "step": 275
+    },
+    {
+      "epoch": 1.669150521609538,
+      "grad_norm": 14.985346794128418,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 0.8889838457107544,
+      "log_odds_ratio": -0.5245683193206787,
+      "logits/chosen": 404.428955078125,
+      "logits/rejected": 391.63189697265625,
+      "logps/chosen": -0.6854910850524902,
+      "logps/rejected": -1.1978435516357422,
+      "loss": 1.0314,
+      "nll_loss": 1.0269334316253662,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03427455574274063,
+      "rewards/margins": 0.02561761811375618,
+      "rewards/rejected": -0.05989217013120651,
+      "step": 280
+    },
+    {
+      "epoch": 1.698956780923994,
+      "grad_norm": 13.728422164916992,
+      "learning_rate": 2.961744388795462e-06,
+      "log_odds_chosen": 0.9078812599182129,
+      "log_odds_ratio": -0.4240701198577881,
+      "logits/chosen": 369.9886779785156,
+      "logits/rejected": 374.25213623046875,
+      "logps/chosen": -0.640867292881012,
+      "logps/rejected": -1.1350326538085938,
+      "loss": 1.0129,
+      "nll_loss": 0.9724606275558472,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03204336389899254,
+      "rewards/margins": 0.024708271026611328,
+      "rewards/rejected": -0.05675163120031357,
+      "step": 285
+    },
+    {
+      "epoch": 1.7287630402384502,
+      "grad_norm": 13.05510425567627,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 0.9024085998535156,
+      "log_odds_ratio": -0.41897106170654297,
+      "logits/chosen": 383.33331298828125,
+      "logits/rejected": 421.84710693359375,
+      "logps/chosen": -0.7976008057594299,
+      "logps/rejected": -1.2910783290863037,
+      "loss": 1.0749,
+      "nll_loss": 1.0102927684783936,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.039880041033029556,
+      "rewards/margins": 0.024673879146575928,
+      "rewards/rejected": -0.06455391645431519,
+      "step": 290
+    },
+    {
+      "epoch": 1.758569299552906,
+      "grad_norm": 22.535524368286133,
+      "learning_rate": 2.9111125486979104e-06,
+      "log_odds_chosen": 0.82624751329422,
+      "log_odds_ratio": -0.44848528504371643,
+      "logits/chosen": 358.3087463378906,
+      "logits/rejected": 401.21539306640625,
+      "logps/chosen": -0.7310397028923035,
+      "logps/rejected": -1.1755989789962769,
+      "loss": 1.0777,
+      "nll_loss": 1.026176929473877,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03655198961496353,
+      "rewards/margins": 0.022227967157959938,
+      "rewards/rejected": -0.05877995491027832,
+      "step": 295
+    },
+    {
+      "epoch": 1.788375558867362,
+      "grad_norm": 20.802898406982422,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 1.3168494701385498,
+      "log_odds_ratio": -0.3224945366382599,
+      "logits/chosen": 398.9639587402344,
+      "logits/rejected": 377.6863098144531,
+      "logps/chosen": -0.6201862096786499,
+      "logps/rejected": -1.2738819122314453,
+      "loss": 0.9939,
+      "nll_loss": 0.9130023121833801,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.031009310856461525,
+      "rewards/margins": 0.03268478438258171,
+      "rewards/rejected": -0.06369409710168839,
+      "step": 300
+    },
+    {
+      "epoch": 1.8181818181818183,
+      "grad_norm": 17.038707733154297,
+      "learning_rate": 2.862991671569341e-06,
+      "log_odds_chosen": 0.5965818762779236,
+      "log_odds_ratio": -0.5011720657348633,
+      "logits/chosen": 391.8568115234375,
+      "logits/rejected": 401.20648193359375,
+      "logps/chosen": -0.8930565714836121,
+      "logps/rejected": -1.2144049406051636,
+      "loss": 1.0267,
+      "nll_loss": 1.134427785873413,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.044652827084064484,
+      "rewards/margins": 0.016067421063780785,
+      "rewards/rejected": -0.060720253735780716,
+      "step": 305
+    },
+    {
+      "epoch": 1.8479880774962743,
+      "grad_norm": 16.012582778930664,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 0.9494487047195435,
+      "log_odds_ratio": -0.5154255032539368,
+      "logits/chosen": 374.0430908203125,
+      "logits/rejected": 383.86016845703125,
+      "logps/chosen": -0.7265419960021973,
+      "logps/rejected": -1.3359992504119873,
+      "loss": 1.0959,
+      "nll_loss": 1.0941182374954224,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03632710129022598,
+      "rewards/margins": 0.03047286346554756,
+      "rewards/rejected": -0.06679996103048325,
+      "step": 310
+    },
+    {
+      "epoch": 1.8777943368107302,
+      "grad_norm": 13.797101974487305,
+      "learning_rate": 2.817180849095055e-06,
+      "log_odds_chosen": 0.4637278616428375,
+      "log_odds_ratio": -0.5917452573776245,
+      "logits/chosen": 352.93426513671875,
+      "logits/rejected": 371.59765625,
+      "logps/chosen": -0.9915712475776672,
+      "logps/rejected": -1.311402440071106,
+      "loss": 1.0986,
+      "nll_loss": 1.2321771383285522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04957855865359306,
+      "rewards/margins": 0.015991564840078354,
+      "rewards/rejected": -0.06557012349367142,
+      "step": 315
+    },
+    {
+      "epoch": 1.9076005961251863,
+      "grad_norm": 15.759064674377441,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 1.112966537475586,
+      "log_odds_ratio": -0.37106412649154663,
+      "logits/chosen": 374.7521667480469,
+      "logits/rejected": 395.431396484375,
+      "logps/chosen": -0.642137885093689,
+      "logps/rejected": -1.249108910560608,
+      "loss": 0.9927,
+      "nll_loss": 0.9422394037246704,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.03210689499974251,
+      "rewards/margins": 0.03034854494035244,
+      "rewards/rejected": -0.0624554380774498,
+      "step": 320
+    },
+    {
+      "epoch": 1.9374068554396424,
+      "grad_norm": 18.297237396240234,
+      "learning_rate": 2.773500981126146e-06,
+      "log_odds_chosen": 1.2229207754135132,
+      "log_odds_ratio": -0.34499865770339966,
+      "logits/chosen": 373.3489685058594,
+      "logits/rejected": 404.5666809082031,
+      "logps/chosen": -0.6735208630561829,
+      "logps/rejected": -1.3717957735061646,
+      "loss": 0.983,
+      "nll_loss": 0.9137915372848511,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.03367604315280914,
+      "rewards/margins": 0.034913744777441025,
+      "rewards/rejected": -0.06858979165554047,
+      "step": 325
+    },
+    {
+      "epoch": 1.9672131147540983,
+      "grad_norm": 21.227933883666992,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 0.8807786107063293,
+      "log_odds_ratio": -0.40755215287208557,
+      "logits/chosen": 366.36541748046875,
+      "logits/rejected": 375.8720703125,
+      "logps/chosen": -0.6931061148643494,
+      "logps/rejected": -1.1950337886810303,
+      "loss": 1.0248,
+      "nll_loss": 0.8495981097221375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03465530276298523,
+      "rewards/margins": 0.025096386671066284,
+      "rewards/rejected": -0.059751689434051514,
+      "step": 330
+    },
+    {
+      "epoch": 1.9970193740685542,
+      "grad_norm": 15.724347114562988,
+      "learning_rate": 2.7317918235407652e-06,
+      "log_odds_chosen": 0.43006667494773865,
+      "log_odds_ratio": -0.608497142791748,
+      "logits/chosen": 391.9339904785156,
+      "logits/rejected": 385.08636474609375,
+      "logps/chosen": -0.966523289680481,
+      "logps/rejected": -1.2040079832077026,
+      "loss": 1.0918,
+      "nll_loss": 1.2487541437149048,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04832616448402405,
+      "rewards/margins": 0.011874236166477203,
+      "rewards/rejected": -0.06020040065050125,
+      "step": 335
+    },
+    {
+      "epoch": 1.9970193740685542,
+      "eval_log_odds_chosen": 0.2209286242723465,
+      "eval_log_odds_ratio": -0.7027979493141174,
+      "eval_logits/chosen": 313.30999755859375,
+      "eval_logits/rejected": 284.8743896484375,
+      "eval_logps/chosen": -1.0012338161468506,
+      "eval_logps/rejected": -1.1474785804748535,
+      "eval_loss": 1.423282504081726,
+      "eval_nll_loss": 1.36614990234375,
+      "eval_rewards/accuracies": 0.49640288949012756,
+      "eval_rewards/chosen": -0.05006168410181999,
+      "eval_rewards/margins": 0.007312240544706583,
+      "eval_rewards/rejected": -0.057373929768800735,
+      "eval_runtime": 112.3167,
+      "eval_samples_per_second": 4.924,
+      "eval_steps_per_second": 1.238,
+      "step": 335
+    },
+    {
+      "epoch": 2.0268256333830106,
+      "grad_norm": 22.363670349121094,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 1.9970035552978516,
+      "log_odds_ratio": -0.19410225749015808,
+      "logits/chosen": 394.3930358886719,
+      "logits/rejected": 370.94024658203125,
+      "logps/chosen": -0.36809104681015015,
+      "logps/rejected": -1.342511534690857,
+      "loss": 0.6352,
+      "nll_loss": 0.6902519464492798,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.018404554575681686,
+      "rewards/margins": 0.04872102662920952,
+      "rewards/rejected": -0.0671255812048912,
+      "step": 340
+    },
+    {
+      "epoch": 2.0566318926974665,
+      "grad_norm": 17.443252563476562,
+      "learning_rate": 2.691909510290828e-06,
+      "log_odds_chosen": 2.4873645305633545,
+      "log_odds_ratio": -0.12294415384531021,
+      "logits/chosen": 353.0312194824219,
+      "logits/rejected": 357.0645751953125,
+      "logps/chosen": -0.3527178168296814,
+      "logps/rejected": -1.623201608657837,
+      "loss": 0.5679,
+      "nll_loss": 0.5656304359436035,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.01763588935136795,
+      "rewards/margins": 0.06352418661117554,
+      "rewards/rejected": -0.08116006851196289,
+      "step": 345
+    },
+    {
+      "epoch": 2.0864381520119224,
+      "grad_norm": 13.353521347045898,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 2.534590005874634,
+      "log_odds_ratio": -0.11770644038915634,
+      "logits/chosen": 355.3052673339844,
+      "logits/rejected": 389.3919372558594,
+      "logps/chosen": -0.3792329430580139,
+      "logps/rejected": -1.8535314798355103,
+      "loss": 0.5665,
+      "nll_loss": 0.5773240327835083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.018961649388074875,
+      "rewards/margins": 0.07371493428945541,
+      "rewards/rejected": -0.09267657995223999,
+      "step": 350
+    },
+    {
+      "epoch": 2.1162444113263787,
+      "grad_norm": 15.57699203491211,
+      "learning_rate": 2.6537244621713765e-06,
+      "log_odds_chosen": 2.286215305328369,
+      "log_odds_ratio": -0.15262895822525024,
+      "logits/chosen": 352.2881164550781,
+      "logits/rejected": 369.99462890625,
+      "logps/chosen": -0.3723372220993042,
+      "logps/rejected": -1.5929332971572876,
+      "loss": 0.5759,
+      "nll_loss": 0.6265019178390503,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.01861685886979103,
+      "rewards/margins": 0.06102980300784111,
+      "rewards/rejected": -0.07964666187763214,
+      "step": 355
+    },
+    {
+      "epoch": 2.1460506706408347,
+      "grad_norm": 11.725736618041992,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 2.5719923973083496,
+      "log_odds_ratio": -0.12209601700305939,
+      "logits/chosen": 359.28118896484375,
+      "logits/rejected": 400.27215576171875,
+      "logps/chosen": -0.3560808300971985,
+      "logps/rejected": -1.588376760482788,
+      "loss": 0.5392,
+      "nll_loss": 0.539588451385498,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017804041504859924,
+      "rewards/margins": 0.06161479279398918,
+      "rewards/rejected": -0.0794188380241394,
+      "step": 360
+    },
+    {
+      "epoch": 2.1758569299552906,
+      "grad_norm": 11.369336128234863,
+      "learning_rate": 2.6171196129510684e-06,
+      "log_odds_chosen": 2.051750421524048,
+      "log_odds_ratio": -0.1606641709804535,
+      "logits/chosen": 347.43212890625,
+      "logits/rejected": 334.7518310546875,
+      "logps/chosen": -0.3488374650478363,
+      "logps/rejected": -1.3597064018249512,
+      "loss": 0.5618,
+      "nll_loss": 0.5548380613327026,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.017441872507333755,
+      "rewards/margins": 0.05054344981908798,
+      "rewards/rejected": -0.06798531860113144,
+      "step": 365
+    },
+    {
+      "epoch": 2.2056631892697465,
+      "grad_norm": 14.05441665649414,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 2.0613701343536377,
+      "log_odds_ratio": -0.20328454673290253,
+      "logits/chosen": 318.46661376953125,
+      "logits/rejected": 339.12286376953125,
+      "logps/chosen": -0.36191436648368835,
+      "logps/rejected": -1.3976794481277466,
+      "loss": 0.5944,
+      "nll_loss": 0.6158479452133179,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.018095718696713448,
+      "rewards/margins": 0.051788248121738434,
+      "rewards/rejected": -0.06988397240638733,
+      "step": 370
+    },
+    {
+      "epoch": 2.235469448584203,
+      "grad_norm": 13.069171905517578,
+      "learning_rate": 2.5819888974716113e-06,
+      "log_odds_chosen": 1.9356635808944702,
+      "log_odds_ratio": -0.19234418869018555,
+      "logits/chosen": 366.2290344238281,
+      "logits/rejected": 386.2459411621094,
+      "logps/chosen": -0.4155746400356293,
+      "logps/rejected": -1.442433476448059,
+      "loss": 0.5766,
+      "nll_loss": 0.5814634561538696,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.020778732374310493,
+      "rewards/margins": 0.051342952996492386,
+      "rewards/rejected": -0.07212167233228683,
+      "step": 375
+    },
+    {
+      "epoch": 2.2652757078986587,
+      "grad_norm": 13.310582160949707,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 2.3414645195007324,
+      "log_odds_ratio": -0.1297520399093628,
+      "logits/chosen": 361.81097412109375,
+      "logits/rejected": 347.11090087890625,
+      "logps/chosen": -0.2917303442955017,
+      "logps/rejected": -1.3597028255462646,
+      "loss": 0.5492,
+      "nll_loss": 0.5354436039924622,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.014586517587304115,
+      "rewards/margins": 0.05339862033724785,
+      "rewards/rejected": -0.06798513978719711,
+      "step": 380
+    },
+    {
+      "epoch": 2.2950819672131146,
+      "grad_norm": 11.30221939086914,
+      "learning_rate": 2.5482359571881276e-06,
+      "log_odds_chosen": 2.427121162414551,
+      "log_odds_ratio": -0.14391574263572693,
+      "logits/chosen": 351.34454345703125,
+      "logits/rejected": 348.7616271972656,
+      "logps/chosen": -0.30433347821235657,
+      "logps/rejected": -1.4669969081878662,
+      "loss": 0.5393,
+      "nll_loss": 0.5202994346618652,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.015216675586998463,
+      "rewards/margins": 0.058133166283369064,
+      "rewards/rejected": -0.07334984838962555,
+      "step": 385
+    },
+    {
+      "epoch": 2.3248882265275705,
+      "grad_norm": 13.66861629486084,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 2.2526602745056152,
+      "log_odds_ratio": -0.12812598049640656,
+      "logits/chosen": 367.953857421875,
+      "logits/rejected": 390.6175537109375,
+      "logps/chosen": -0.3860087990760803,
+      "logps/rejected": -1.594990611076355,
+      "loss": 0.5879,
+      "nll_loss": 0.6172261834144592,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.019300442188978195,
+      "rewards/margins": 0.060449086129665375,
+      "rewards/rejected": -0.07974953204393387,
+      "step": 390
+    },
+    {
+      "epoch": 2.354694485842027,
+      "grad_norm": 9.35374641418457,
+      "learning_rate": 2.515773027133138e-06,
+      "log_odds_chosen": 2.5175037384033203,
+      "log_odds_ratio": -0.13300301134586334,
+      "logits/chosen": 368.17181396484375,
+      "logits/rejected": 361.74591064453125,
+      "logps/chosen": -0.266899436712265,
+      "logps/rejected": -1.3636187314987183,
+      "loss": 0.5233,
+      "nll_loss": 0.47403964400291443,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.01334497332572937,
+      "rewards/margins": 0.0548359639942646,
+      "rewards/rejected": -0.06818093359470367,
+      "step": 395
+    },
+    {
+      "epoch": 2.384500745156483,
+      "grad_norm": 17.207456588745117,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 2.6303577423095703,
+      "log_odds_ratio": -0.12364117056131363,
+      "logits/chosen": 359.56134033203125,
+      "logits/rejected": 384.45733642578125,
+      "logps/chosen": -0.3128165900707245,
+      "logps/rejected": -1.7372194528579712,
+      "loss": 0.5523,
+      "nll_loss": 0.4810587465763092,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.015640828758478165,
+      "rewards/margins": 0.07122014462947845,
+      "rewards/rejected": -0.08686096966266632,
+      "step": 400
+    },
+    {
+      "epoch": 2.4143070044709387,
+      "grad_norm": 13.166074752807617,
+      "learning_rate": 2.484519974999767e-06,
+      "log_odds_chosen": 2.396026849746704,
+      "log_odds_ratio": -0.1698196679353714,
+      "logits/chosen": 415.81658935546875,
+      "logits/rejected": 383.0643310546875,
+      "logps/chosen": -0.3519137501716614,
+      "logps/rejected": -1.5836241245269775,
+      "loss": 0.5829,
+      "nll_loss": 0.5322110652923584,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0175956878811121,
+      "rewards/margins": 0.061585523188114166,
+      "rewards/rejected": -0.07918120920658112,
+      "step": 405
+    },
+    {
+      "epoch": 2.444113263785395,
+      "grad_norm": 12.805488586425781,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 2.3219261169433594,
+      "log_odds_ratio": -0.1753077507019043,
+      "logits/chosen": 361.06781005859375,
+      "logits/rejected": 375.9926452636719,
+      "logps/chosen": -0.39748460054397583,
+      "logps/rejected": -1.5706228017807007,
+      "loss": 0.5614,
+      "nll_loss": 0.6001583337783813,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.019874228164553642,
+      "rewards/margins": 0.05865691229701042,
+      "rewards/rejected": -0.07853114604949951,
+      "step": 410
+    },
+    {
+      "epoch": 2.473919523099851,
+      "grad_norm": 10.840022087097168,
+      "learning_rate": 2.4544034683690802e-06,
+      "log_odds_chosen": 2.408691883087158,
+      "log_odds_ratio": -0.15550675988197327,
+      "logits/chosen": 363.71136474609375,
+      "logits/rejected": 392.62689208984375,
+      "logps/chosen": -0.33126166462898254,
+      "logps/rejected": -1.5768396854400635,
+      "loss": 0.565,
+      "nll_loss": 0.5147169828414917,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.016563083976507187,
+      "rewards/margins": 0.062278907746076584,
+      "rewards/rejected": -0.07884199172258377,
+      "step": 415
+    },
+    {
+      "epoch": 2.503725782414307,
+      "grad_norm": 14.33762264251709,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 2.0540168285369873,
+      "log_odds_ratio": -0.1838049441576004,
+      "logits/chosen": 360.244873046875,
+      "logits/rejected": 338.4053649902344,
+      "logps/chosen": -0.382302850484848,
+      "logps/rejected": -1.4437496662139893,
+      "loss": 0.5427,
+      "nll_loss": 0.6479641199111938,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.01911514438688755,
+      "rewards/margins": 0.05307234078645706,
+      "rewards/rejected": -0.07218748331069946,
+      "step": 420
+    },
+    {
+      "epoch": 2.533532041728763,
+      "grad_norm": 11.913477897644043,
+      "learning_rate": 2.4253562503633297e-06,
+      "log_odds_chosen": 2.7403793334960938,
+      "log_odds_ratio": -0.09055573493242264,
+      "logits/chosen": 360.6409912109375,
+      "logits/rejected": 357.6117858886719,
+      "logps/chosen": -0.3066481351852417,
+      "logps/rejected": -1.7102562189102173,
+      "loss": 0.533,
+      "nll_loss": 0.5301586389541626,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015332406386733055,
+      "rewards/margins": 0.07018040120601654,
+      "rewards/rejected": -0.08551281690597534,
+      "step": 425
+    },
+    {
+      "epoch": 2.563338301043219,
+      "grad_norm": 12.434874534606934,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 2.650773525238037,
+      "log_odds_ratio": -0.11397852003574371,
+      "logits/chosen": 359.2686767578125,
+      "logits/rejected": 372.0355224609375,
+      "logps/chosen": -0.29058974981307983,
+      "logps/rejected": -1.6085436344146729,
+      "loss": 0.5369,
+      "nll_loss": 0.48550575971603394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.014529486186802387,
+      "rewards/margins": 0.06589768826961517,
+      "rewards/rejected": -0.08042718470096588,
+      "step": 430
+    },
+    {
+      "epoch": 2.593144560357675,
+      "grad_norm": 15.130928993225098,
+      "learning_rate": 2.3973165074269213e-06,
+      "log_odds_chosen": 2.4082794189453125,
+      "log_odds_ratio": -0.14481699466705322,
+      "logits/chosen": 368.16680908203125,
+      "logits/rejected": 336.6341247558594,
+      "logps/chosen": -0.37292951345443726,
+      "logps/rejected": -1.5544594526290894,
+      "loss": 0.5655,
+      "nll_loss": 0.5464103817939758,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.018646476790308952,
+      "rewards/margins": 0.059076495468616486,
+      "rewards/rejected": -0.07772296667098999,
+      "step": 435
+    },
+    {
+      "epoch": 2.6229508196721314,
+      "grad_norm": 13.672365188598633,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 2.712573289871216,
+      "log_odds_ratio": -0.11169672012329102,
+      "logits/chosen": 354.31842041015625,
+      "logits/rejected": 366.07745361328125,
+      "logps/chosen": -0.2754111886024475,
+      "logps/rejected": -1.5493534803390503,
+      "loss": 0.5762,
+      "nll_loss": 0.5381069779396057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.013770559802651405,
+      "rewards/margins": 0.06369712203741074,
+      "rewards/rejected": -0.07746767997741699,
+      "step": 440
+    },
+    {
+      "epoch": 2.6527570789865873,
+      "grad_norm": 12.229787826538086,
+      "learning_rate": 2.3702273156998867e-06,
+      "log_odds_chosen": 2.407763957977295,
+      "log_odds_ratio": -0.13709630072116852,
+      "logits/chosen": 339.0660705566406,
+      "logits/rejected": 373.5878601074219,
+      "logps/chosen": -0.3847058415412903,
+      "logps/rejected": -1.715640664100647,
+      "loss": 0.5811,
+      "nll_loss": 0.6087844967842102,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.019235290586948395,
+      "rewards/margins": 0.06654674559831619,
+      "rewards/rejected": -0.08578203618526459,
+      "step": 445
+    },
+    {
+      "epoch": 2.682563338301043,
+      "grad_norm": 11.220720291137695,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 2.3630270957946777,
+      "log_odds_ratio": -0.12073234468698502,
+      "logits/chosen": 364.763427734375,
+      "logits/rejected": 365.3312072753906,
+      "logps/chosen": -0.3996626138687134,
+      "logps/rejected": -1.661350965499878,
+      "loss": 0.589,
+      "nll_loss": 0.5471979379653931,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01998312957584858,
+      "rewards/margins": 0.06308441609144211,
+      "rewards/rejected": -0.08306754380464554,
+      "step": 450
+    },
+    {
+      "epoch": 2.712369597615499,
+      "grad_norm": 14.05644702911377,
+      "learning_rate": 2.3440361546924774e-06,
+      "log_odds_chosen": 2.527470588684082,
+      "log_odds_ratio": -0.128330796957016,
+      "logits/chosen": 393.28240966796875,
+      "logits/rejected": 373.51959228515625,
+      "logps/chosen": -0.3907659351825714,
+      "logps/rejected": -1.6387542486190796,
+      "loss": 0.5997,
+      "nll_loss": 0.5538510084152222,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.01953829452395439,
+      "rewards/margins": 0.06239941716194153,
+      "rewards/rejected": -0.08193771541118622,
+      "step": 455
+    },
+    {
+      "epoch": 2.742175856929955,
+      "grad_norm": 14.722325325012207,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 2.5439341068267822,
+      "log_odds_ratio": -0.1302451193332672,
+      "logits/chosen": 381.04022216796875,
+      "logits/rejected": 400.01812744140625,
+      "logps/chosen": -0.36085230112075806,
+      "logps/rejected": -1.748968482017517,
+      "loss": 0.5786,
+      "nll_loss": 0.6514776945114136,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.018042614683508873,
+      "rewards/margins": 0.06940580904483795,
+      "rewards/rejected": -0.08744843304157257,
+      "step": 460
+    },
+    {
+      "epoch": 2.7719821162444114,
+      "grad_norm": 12.233352661132812,
+      "learning_rate": 2.3186944788008413e-06,
+      "log_odds_chosen": 2.5529284477233887,
+      "log_odds_ratio": -0.1319517195224762,
+      "logits/chosen": 376.39300537109375,
+      "logits/rejected": 381.1004333496094,
+      "logps/chosen": -0.2920375168323517,
+      "logps/rejected": -1.539701223373413,
+      "loss": 0.581,
+      "nll_loss": 0.5538360476493835,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01460187416523695,
+      "rewards/margins": 0.06238318607211113,
+      "rewards/rejected": -0.07698506116867065,
+      "step": 465
+    },
+    {
+      "epoch": 2.8017883755588673,
+      "grad_norm": 12.4392728805542,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 2.2491307258605957,
+      "log_odds_ratio": -0.18098264932632446,
+      "logits/chosen": 377.3009948730469,
+      "logits/rejected": 349.0381164550781,
+      "logps/chosen": -0.3590669333934784,
+      "logps/rejected": -1.5034451484680176,
+      "loss": 0.5715,
+      "nll_loss": 0.5520719289779663,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.01795334741473198,
+      "rewards/margins": 0.05721891671419144,
+      "rewards/rejected": -0.07517226040363312,
+      "step": 470
+    },
+    {
+      "epoch": 2.8315946348733236,
+      "grad_norm": 13.7266263961792,
+      "learning_rate": 2.2941573387056174e-06,
+      "log_odds_chosen": 2.518183946609497,
+      "log_odds_ratio": -0.1277451366186142,
+      "logits/chosen": 350.8870544433594,
+      "logits/rejected": 374.2940979003906,
+      "logps/chosen": -0.36442482471466064,
+      "logps/rejected": -1.6510089635849,
+      "loss": 0.5518,
+      "nll_loss": 0.5228261947631836,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.018221240490674973,
+      "rewards/margins": 0.06432920694351196,
+      "rewards/rejected": -0.08255044370889664,
+      "step": 475
+    },
+    {
+      "epoch": 2.8614008941877795,
+      "grad_norm": 10.832075119018555,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 2.2293262481689453,
+      "log_odds_ratio": -0.12778417766094208,
+      "logits/chosen": 361.9057922363281,
+      "logits/rejected": 402.4067077636719,
+      "logps/chosen": -0.34709399938583374,
+      "logps/rejected": -1.4698781967163086,
+      "loss": 0.5087,
+      "nll_loss": 0.5175037384033203,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017354700714349747,
+      "rewards/margins": 0.05613920837640762,
+      "rewards/rejected": -0.07349390536546707,
+      "step": 480
+    },
+    {
+      "epoch": 2.8912071535022354,
+      "grad_norm": 11.188936233520508,
+      "learning_rate": 2.270383045932499e-06,
+      "log_odds_chosen": 2.5824456214904785,
+      "log_odds_ratio": -0.12692959606647491,
+      "logits/chosen": 357.77374267578125,
+      "logits/rejected": 380.9786376953125,
+      "logps/chosen": -0.3558903634548187,
+      "logps/rejected": -1.7740790843963623,
+      "loss": 0.5513,
+      "nll_loss": 0.5267875790596008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017794519662857056,
+      "rewards/margins": 0.07090945541858673,
+      "rewards/rejected": -0.08870397508144379,
+      "step": 485
+    },
+    {
+      "epoch": 2.9210134128166914,
+      "grad_norm": 12.93087100982666,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 2.304809331893921,
+      "log_odds_ratio": -0.18847742676734924,
+      "logits/chosen": 370.79241943359375,
+      "logits/rejected": 337.0198059082031,
+      "logps/chosen": -0.46047478914260864,
+      "logps/rejected": -1.6731350421905518,
+      "loss": 0.6251,
+      "nll_loss": 0.5758861303329468,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.023023737594485283,
+      "rewards/margins": 0.060633014887571335,
+      "rewards/rejected": -0.08365674316883087,
+      "step": 490
+    },
+    {
+      "epoch": 2.9508196721311473,
+      "grad_norm": 12.990720748901367,
+      "learning_rate": 2.2473328748774737e-06,
+      "log_odds_chosen": 2.1052613258361816,
+      "log_odds_ratio": -0.17354901134967804,
+      "logits/chosen": 362.78375244140625,
+      "logits/rejected": 391.7388610839844,
+      "logps/chosen": -0.4515438973903656,
+      "logps/rejected": -1.5366592407226562,
+      "loss": 0.5494,
+      "nll_loss": 0.5879518985748291,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.0225771926343441,
+      "rewards/margins": 0.05425576493144035,
+      "rewards/rejected": -0.07683296501636505,
+      "step": 495
+    },
+    {
+      "epoch": 2.9806259314456036,
+      "grad_norm": 11.811108589172363,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 2.437211513519287,
+      "log_odds_ratio": -0.13766226172447205,
+      "logits/chosen": 372.21160888671875,
+      "logits/rejected": 370.1877136230469,
+      "logps/chosen": -0.32970958948135376,
+      "logps/rejected": -1.5284183025360107,
+      "loss": 0.576,
+      "nll_loss": 0.47357138991355896,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01648547872900963,
+      "rewards/margins": 0.05993543937802315,
+      "rewards/rejected": -0.07642091810703278,
+      "step": 500
+    },
+    {
+      "epoch": 2.9865871833084947,
+      "eval_log_odds_chosen": 0.27211081981658936,
+      "eval_log_odds_ratio": -0.7039158344268799,
+      "eval_logits/chosen": 296.6807556152344,
+      "eval_logits/rejected": 267.234619140625,
+      "eval_logps/chosen": -1.0933129787445068,
+      "eval_logps/rejected": -1.2508149147033691,
+      "eval_loss": 1.5379581451416016,
+      "eval_nll_loss": 1.4702765941619873,
+      "eval_rewards/accuracies": 0.5467625856399536,
+      "eval_rewards/chosen": -0.054665643721818924,
+      "eval_rewards/margins": 0.007875093258917332,
+      "eval_rewards/rejected": -0.06254073977470398,
+      "eval_runtime": 112.2732,
+      "eval_samples_per_second": 4.925,
+      "eval_steps_per_second": 1.238,
+      "step": 501
+    },
+    {
+      "epoch": 2.9865871833084947,
+      "step": 501,
+      "total_flos": 0.0,
+      "train_loss": 1.4583633707431025,
+      "train_runtime": 13840.413,
+      "train_samples_per_second": 1.163,
+      "train_steps_per_second": 0.036
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 501,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef0399d23b46cfbc5e3cf2cd08aa71405ef6d9278762be54afa0ba8d0d2376a9
+size 6776